From c1193f7d9ca0332a828e437191c37d0c55bb56f4 Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Tue, 6 Feb 2024 10:41:27 -0500 Subject: [PATCH] fix: format py files with black --- SlideRule/S01_SlideRule_compare.py | 582 ++-- SlideRule/S02_SlideRule_trials.py | 263 +- analyis_publish/PB04_angle_priors.py | 2 +- analysis/B01_filter_regrid.py | 390 ++- analysis/B01_filter_regrid_ALT07_test.py | 168 +- analysis/B01_filter_regrid_curve_lin.py | 513 +-- analysis/B01_filter_regrid_lat_lon.py | 373 +- .../B01_filter_regrid_segment_test_regridd.py | 800 +++-- analysis/B01_filter_regrid_single.py | 393 ++- analysis/B02_make_spectra.py | 648 ++-- analysis/B02_make_spectra_FFT.py | 135 +- analysis/B02_make_spectra_LS.py | 496 +-- analysis/B02_make_spectra_LS_new.py | 368 +- analysis/B02_make_spectra_LS_new_new.py | 404 ++- analysis/B02_make_spectra_gFT_tester.py | 765 ++-- analysis/B02_make_spectra_get_phase.py | 638 ++-- analysis/B03_coherence_direction.py | 336 +- analysis/SA01_download_tracks.py | 99 +- analysis/SB02_make_spectra_LS.py | 632 ++-- analysis/SB02_make_spectra_LS_adjust_amp.py | 875 +++-- ...ake_spectra_LS_adjust_amp_working_exmpl.py | 445 ++- analysis/SB02_make_spectra_linear_inverse.py | 331 +- ...B02_make_spectra_linear_inverse_bestfit.py | 279 +- analysis/SB03_posterior_phase_fitting.py | 124 +- analysis/SB03_swell_phase_fitting.py | 495 +-- analysis/SB03_swell_phase_fitting_fake.py | 631 ++-- .../SB03_swell_phase_fitting_iter_freq.py | 560 +-- ...B03_swell_phase_fitting_iter_with_prior.py | 846 +++-- ...03_swell_phase_fitting_posterior_direct.py | 267 +- analysis/SB04_2d_wavefield_emulator.py | 298 +- analysis/X01_track_visualization.py | 180 +- analysis/_B02_make_spectra.py | 499 +-- analysis/load_ICEsat2.py | 211 +- analysis/load_ICEsat2_AL07.py | 273 +- analysis_db/A01b_ALT10_variance_tester.py | 389 ++- analysis_db/A01c_mergefiles.py | 151 +- analysis_db/A02a_sentinal1_prior.py | 312 +- analysis_db/A02b_WW3_hindcast_prior.py | 566 +-- analysis_db/B01_SL_load_batch.py | 156 +- analysis_db/B01_filter_regrid_segments.py | 756 ++-- .../B01_filter_regrid_segments_tester2.py | 760 ++-- analysis_db/S01_derive_tracklist_from_file.py | 92 +- .../SB01_SL_kml_orbits_to_geopandas.py | 19 +- analysis_fake_data/A01_1D_spectra_and_ice.py | 112 +- analysis_fake_data/A01_1d_fake_surface_obs.py | 103 +- .../S01_1D_fake_data_variance_conservation.py | 385 +- analysis_fake_data/wave_realizations.py | 37 +- config/2021_IceSAT2_startup_local.py | 359 +- config/generate_colors.py | 82 +- .../ICEsat2_SI_tools/__init__.py | 2 +- .../ICEsat2_SI_tools/angle_optimizer.py | 249 +- .../ICEsat2_SI_tools/convert_GPS_time.py | 181 +- .../ICEsat2_SI_tools/convert_julian.py | 110 +- .../ICEsat2_SI_tools/filter_regrid.py | 396 ++- .../ICEsat2_SI_tools/lanczos.py | 77 +- .../ICEsat2_SI_tools/read_ground_tracks.py | 112 +- .../ICEsat2_SI_tools/spectral_estimates.py | 1152 +++--- .../ICEsat2_SI_tools/spicke_remover.py | 116 +- .../ICEsat2_SI_tools/wave_tools.py | 28 +- src/icesat2_tracks/__init__.py | 2 +- .../analysis/SA01b_download_tester.py | 129 +- .../A01b_ALT07_SHNH_variance_tester.py | 610 ++-- .../analysis_db/B01_SL_load_single_file.py | 1 + .../analysis_db/B03_plot_spectra_ov.py | 1 + .../analysis_db/B05_define_angle.py | 13 +- .../analysis_db/B06_correct_separate_var.py | 10 +- src/icesat2_tracks/analysis_db/__init__.py | 2 +- .../analysis_fake_data/A01_1D_spectra.py | 70 +- .../S02_spectra_estimates_test.py | 45 +- .../S03_spectra_estimates_lomb_scargle.py | 281 +- .../X03_MCMC_surface_smapling.py | 194 +- src/icesat2_tracks/config/IceSAT2_startup.py | 110 +- src/icesat2_tracks/config/__init__.py | 2 +- src/icesat2_tracks/config/generate_colors.py | 80 +- .../nsidc_icesat2_associated2.py | 431 ++- .../local_modules/JONSWAP_gamma.py | 718 ++-- src/icesat2_tracks/local_modules/__init__.py | 2 +- .../local_modules/m_colormanager_ph3.py | 794 +++-- .../local_modules/m_general_ph3.py | 3083 +++++++++-------- .../local_modules/m_spectrum_ph3.py | 1273 +++---- .../local_modules/m_tools_ph3.py | 356 +- 81 files changed, 16968 insertions(+), 12260 deletions(-) diff --git a/SlideRule/S01_SlideRule_compare.py b/SlideRule/S01_SlideRule_compare.py index 29d6e8aa..d3cd4ddd 100644 --- a/SlideRule/S01_SlideRule_compare.py +++ b/SlideRule/S01_SlideRule_compare.py @@ -24,6 +24,7 @@ import datetime import numpy as np import shapely + sliderule.__version__ import matplotlib @@ -37,11 +38,12 @@ import copy import spicke_remover import generalized_FT as gFT -xr.set_options(display_style='text') -#matplotlib.use('agg') -%matplotlib inline -%matplotlib widget +xr.set_options(display_style="text") + +# matplotlib.use('agg') +# %matplotlib inline +# %matplotlib widget # %% define functions @@ -53,62 +55,73 @@ def select_beam_section(data, spot, lims): lims tuple of min max latitude limits """ if type(data) is dict: - mask =(data[spot].lats > lims[0]) & (data[spot].lats < lims[1]) - data2= data[spot][mask] + mask = (data[spot].lats > lims[0]) & (data[spot].lats < lims[1]) + data2 = data[spot][mask] # if abs(data2.lats[0]) > abs(data2.lats[-1]): # data2.sort_values('distance', inplace=True, ascending=False) else: - ii= (data.spot==spot) - mask = (data[ii].geometry.y > lims[0]) & (data[ii].geometry.y < lims[1]) - data2= data[ii][mask] + ii = data.spot == spot + mask = (data[ii].geometry.y > lims[0]) & (data[ii].geometry.y < lims[1]) + data2 = data[ii][mask] if abs(data2.geometry.y[0]) > abs(data2.geometry.y[-1]): - data2.sort_values('distance', inplace=True, ascending=False) + data2.sort_values("distance", inplace=True, ascending=False) return data2 -plot_path = mconfig['paths']['plot'] +'sliderule_tests/' +plot_path = mconfig["paths"]["plot"] + "sliderule_tests/" MT.mkdirs_r(plot_path) -#import s3fs +# import s3fs # %% -ID_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#ID_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#ID_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#ID_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False -#ID_name, batch_key, test_flag = '20190208152826_06440210_004_01', 'SH_batch01', False -#ID_name, batch_key, test_flag = '20190213133330_07190212_004_01', 'SH_batch02', False -#ID_name, batch_key, test_flag = '20190207002436_06190212_004_01', 'SH_batch02', False -#ID_name, batch_key, test_flag = '20190206022433_06050212_004_01', 'SH_batch02', False - -#ID_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False -#ID_name, batch_key, test_flag = '20190502021224_05160312_004_01', 'SH_batch02', False - -#ID_name, batch_key, test_flag = 'SH_20190208_06440212', 'SH_publish', True -#ID_name, batch_key, test_flag = 'SH_20190219_08070210', 'SH_publish', True -ID_name, batch_key, test_flag = 'SH_20190502_05160312', 'SH_publish', True - -#ID_name, batch_key, test_flag = 'NH_20190311_11200203', 'NH_batch06', True -#ID_name, batch_key, test_flag = 'NH_20210312_11961005', 'NH_batch07', True +ID_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# ID_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# ID_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# ID_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False +# ID_name, batch_key, test_flag = '20190208152826_06440210_004_01', 'SH_batch01', False +# ID_name, batch_key, test_flag = '20190213133330_07190212_004_01', 'SH_batch02', False +# ID_name, batch_key, test_flag = '20190207002436_06190212_004_01', 'SH_batch02', False +# ID_name, batch_key, test_flag = '20190206022433_06050212_004_01', 'SH_batch02', False + +# ID_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +# ID_name, batch_key, test_flag = '20190502021224_05160312_004_01', 'SH_batch02', False + +# ID_name, batch_key, test_flag = 'SH_20190208_06440212', 'SH_publish', True +# ID_name, batch_key, test_flag = 'SH_20190219_08070210', 'SH_publish', True +ID_name, batch_key, test_flag = "SH_20190502_05160312", "SH_publish", True + +# ID_name, batch_key, test_flag = 'NH_20190311_11200203', 'NH_batch06', True +# ID_name, batch_key, test_flag = 'NH_20210312_11961005', 'NH_batch07', True ##### use init_data to load experiment metadata -ID, _, _, _ = io.init_data(ID_name, batch_key, True, mconfig['paths']['work'], ) +ID, _, _, _ = io.init_data( + ID_name, + batch_key, + True, + mconfig["paths"]["work"], +) -#track_name = 'ATL03_20190502021224_05160312_005_01' -track_name = ID['tracks']['ATL03'][0] +'.h5' -#track_name = 'ATL03_20190502021224_05160312_005_01.h5' +# track_name = 'ATL03_20190502021224_05160312_005_01' +track_name = ID["tracks"]["ATL03"][0] + ".h5" +# track_name = 'ATL03_20190502021224_05160312_005_01.h5' -#print(ID_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') +# print(ID_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") -all_beams = mconfig['beams']['all_beams'] +all_beams = mconfig["beams"]["all_beams"] -load_path_work = mconfig['paths']['work'] +'/'+ batch_key +'/' -B2_hdf5 = h5py.File(load_path_work +'B01_regrid'+'/'+ID_name + '_B01_regridded.h5', 'r') -B3_hdf5 = h5py.File(load_path_work +'B01_regrid'+'/'+ID_name + '_B01_binned.h5', 'r') +load_path_work = mconfig["paths"]["work"] + "/" + batch_key + "/" +B2_hdf5 = h5py.File( + load_path_work + "B01_regrid" + "/" + ID_name + "_B01_regridded.h5", "r" +) +B3_hdf5 = h5py.File( + load_path_work + "B01_regrid" + "/" + ID_name + "_B01_binned.h5", "r" +) B2, B3 = dict(), dict() for b in all_beams: @@ -118,45 +131,54 @@ def select_beam_section(data, spot, lims): B2_hdf5.close(), B2_hdf5.close() - # %% # Configure Session # -#icesat2.init("icesat2sliderule.org", True) -icesat2.init("slideruleearth.io", True) #doesn't work -asset = 'nsidc-s3' +# icesat2.init("icesat2sliderule.org", True) +icesat2.init("slideruleearth.io", True) # doesn't work +asset = "nsidc-s3" # %% # latR=[-67.5, -64.5] # lonR=[ 59.6, 67.7] # print('org ', latR, lonR) -latR = [np.round(ID['pars']['start']['latitude'], 1), np.round(ID['pars']['end']['latitude'], 1) ] -lonR = [np.round(ID['pars']['start']['longitude'], 1), np.round(ID['pars']['end']['longitude'], 1) ] +latR = [ + np.round(ID["pars"]["start"]["latitude"], 1), + np.round(ID["pars"]["end"]["latitude"], 1), +] +lonR = [ + np.round(ID["pars"]["start"]["longitude"], 1), + np.round(ID["pars"]["end"]["longitude"], 1), +] latR.sort() lonR.sort() -latR = [latR[0]-0.0, latR[1]+0.5] -lonR = [lonR[0]-1.0, lonR[1]+6.5] +latR = [latR[0] - 0.0, latR[1] + 0.5] +lonR = [lonR[0] - 1.0, lonR[1] + 6.5] -poly=[{'lat':latR[ii], 'lon':lonR[jj]} for ii, jj in zip([1, 1, 0, 0, 1], [1, 0, 0, 1, 1])] -print('new ', latR, lonR) +poly = [ + {"lat": latR[ii], "lon": lonR[jj]} + for ii, jj in zip([1, 1, 0, 0, 1], [1, 0, 0, 1, 1]) +] +print("new ", latR, lonR) # %% ## Generate ATL06-type segments using the ATL03-native photon classification # Use the ocean classification for photons with a confidence parmeter to 2 or higher (low confidence or better) -params={'srt': 1, # Ocean classification - 'len': 20, # 10-meter segments - 'ats':3, # require that each segment contain photons separated by at least 5 m - 'res':10, # return one photon every 5 m - 'track': 0, # return all ground tracks - 'pass_invalid': True, - 'cnf': 2, # require classification confidence of 2 or more - #'iterations':10, # iterate the fit - 't0': '2019-05-02T02:12:24', # time range (not needed in this case) - 't1': '2019-05-02T03:00:00', - 'poly': poly, # polygon within which to select photons, +params = { + "srt": 1, # Ocean classification + "len": 20, # 10-meter segments + "ats": 3, # require that each segment contain photons separated by at least 5 m + "res": 10, # return one photon every 5 m + "track": 0, # return all ground tracks + "pass_invalid": True, + "cnf": 2, # require classification confidence of 2 or more + #'iterations':10, # iterate the fit + "t0": "2019-05-02T02:12:24", # time range (not needed in this case) + "t1": "2019-05-02T03:00:00", + "poly": poly, # polygon within which to select photons, } ## Generate ATL06-type segments using the YAPC photon classification @@ -165,194 +187,239 @@ def select_beam_section(data, spot, lims): # Adjusting the YAPC 'score' parameter to something higher than 100 will return fewer photons that are more tightly clustered. -#icesat2.init("slideruleearth.io", True, organization="sliderule") -#icesat2.init("slideruleearth.io", True)#, organization="sliderule") - -params_yapc={'srt': 1, - 'len': 20, - 'ats':3, - 'res':10, - 'track': 0, - 'pass_invalid': True, - 'cnf': -2, - 'iterations':10, - 't0': '2019-05-02T02:12:24', - 't1': '2019-05-02T03:00:00', +# icesat2.init("slideruleearth.io", True, organization="sliderule") +# icesat2.init("slideruleearth.io", True)#, organization="sliderule") + +params_yapc = { + "srt": 1, + "len": 20, + "ats": 3, + "res": 10, + "track": 0, + "pass_invalid": True, + "cnf": -2, + "iterations": 10, + "t0": "2019-05-02T02:12:24", + "t1": "2019-05-02T03:00:00", # "yapc": dict(knn=0, win_h=6, win_x=11, min_ph=4, score=100), # use the YAPC photon classifier; these are the recommended parameters, but the results might be more specific with a smaller win_h value, or a higher score cutoff - "yapc": dict(knn=0, win_h=3, win_x=11, min_ph=4, score=50), # use the YAPC photon classifier; these are the recommended parameters, but the results might be more specific with a smaller win_h value, or a higher score cutoff - 'poly':poly} + "yapc": dict( + knn=0, win_h=3, win_x=11, min_ph=4, score=50 + ), # use the YAPC photon classifier; these are the recommended parameters, but the results might be more specific with a smaller win_h value, or a higher score cutoff + "poly": poly, +} -ATL06_native =dict() +ATL06_native = dict() ATL06_yapc = dict() for llen in [10, 20, 30]: - params['len'] = llen - params_yapc['len'] = llen + params["len"] = llen + params_yapc["len"] = llen # Run the parallel version of the ATL06 algorithm, for a specified granule: - ATL06_native[llen] = icesat2.atl06p(params, asset="nsidc-s3", resources=[track_name]) - ATL06_yapc[llen] = icesat2.atl06p(params_yapc, asset="nsidc-s3", resources=[track_name])#, callbacks = {"atl03rec": atl03rec_cb}) + ATL06_native[llen] = icesat2.atl06p( + params, asset="nsidc-s3", resources=[track_name] + ) + ATL06_yapc[llen] = icesat2.atl06p( + params_yapc, asset="nsidc-s3", resources=[track_name] + ) # , callbacks = {"atl03rec": atl03rec_cb}) # %% -#for b in all_beams: +# for b in all_beams: + def init_compare_figure(B2i, B3i, lims): - F = M.figure_axis_xy(6,4, view_scale=0.7, container=True) + F = M.figure_axis_xy(6, 4, view_scale=0.7, container=True) grid = F.fig.add_gridspec(2, 4, wspace=0.1, hspace=0.8) # Add the subplots axx = dict() - axx[1] = F.fig.add_subplot(grid[0, 0:3])#, xticklabels=[]) + axx[1] = F.fig.add_subplot(grid[0, 0:3]) # , xticklabels=[]) axx[2] = F.fig.add_subplot(grid[1, 0:3], sharex=axx[1], yticklabels=[]) - axx[3] = F.fig.add_subplot(grid[0, -1], sharey=axx[1])# yticklabels=[]) - axx[4] = F.fig.add_subplot(grid[1, -1], sharey=axx[2], yticklabels=[]) #yticklabels=[]) - + axx[3] = F.fig.add_subplot(grid[0, -1], sharey=axx[1]) # yticklabels=[]) + axx[4] = F.fig.add_subplot( + grid[1, -1], sharey=axx[2], yticklabels=[] + ) # yticklabels=[]) # init basic data plt.sca(axx[1]) - plt.plot( B2i.lats, B2i.heights, '.', color = 'gray', markersize=0.8, alpha = 0.6) - plt.plot( B3i.lats, B3i.heights, '-k', lw=0.8, label='B03 H&H 2023', zorder=12) - #plt.plot( B3i.lats, B3i.heights, '.k', markersize = 1.2) - + plt.plot(B2i.lats, B2i.heights, ".", color="gray", markersize=0.8, alpha=0.6) + plt.plot(B3i.lats, B3i.heights, "-k", lw=0.8, label="B03 H&H 2023", zorder=12) + # plt.plot( B3i.lats, B3i.heights, '.k', markersize = 1.2) plt.sca(axx[2]) B3i_gradient = -np.gradient(B3i.heights_c_weighted_mean) # plt.plot( B2i.lats, B2i.heights_c, '.k', markersize=0.5) - plt.plot( B3i.lats, B3i_gradient, '-k', lw=0.8, label='B03 H&H 2023', zorder=12) + plt.plot(B3i.lats, B3i_gradient, "-k", lw=0.8, label="B03 H&H 2023", zorder=12) + axx[1].set_title("Height Timeseries", loc="left") + # ax1.set_xlabel('Latitude') + axx[1].set_ylabel("Height") - axx[1].set_title('Height Timeseries', loc='left') - #ax1.set_xlabel('Latitude') - axx[1].set_ylabel('Height') + axx[2].set_title("Gradient", loc="left") + axx[2].set_xlabel("Latitude") + axx[2].set_ylabel("Gradient") - axx[2].set_title('Gradient', loc='left') - axx[2].set_xlabel('Latitude') - axx[2].set_ylabel('Gradient') + axx[3].set_title("Scatter", loc="left") + axx[3].set_xlabel("H&H 2023 Height") + # axx[3].set_ylabel('Height') - axx[3].set_title('Scatter', loc='left') - axx[3].set_xlabel('H&H 2023 Height') - #axx[3].set_ylabel('Height') - - axx[4].set_title('Scatter', loc='left') - #axx[4].set_xlabel('Gradient') - #axx[4].set_ylabel('Gradient') + axx[4].set_title("Scatter", loc="left") + # axx[4].set_xlabel('Gradient') + # axx[4].set_ylabel('Gradient') return F, axx, B3i_gradient + # %% -def add_data(axx, data, delta_y, dx, color, label, alpha=1.0, scatter=True): +def add_data(axx, data, delta_y, dx, color, label, alpha=1.0, scatter=True): data_x, h_mean = data.geometry.y, data.h_mean - #data_dx = data.dh_fit_dx * dx - data_dx = - np.gradient(h_mean) #* dx + # data_dx = data.dh_fit_dx * dx + data_dx = -np.gradient(h_mean) # * dx - axx[1].plot( data_x, h_mean+ delta_y, color, lw=1.2, label=label, alpha=alpha) - #axx[1].plot( data_x, h_mean+ delta_y, '.k', markersize=1.2) + axx[1].plot(data_x, h_mean + delta_y, color, lw=1.2, label=label, alpha=alpha) + # axx[1].plot( data_x, h_mean+ delta_y, '.k', markersize=1.2) - axx[2].plot( data_x, data_dx, color, lw=1.2, label=label, alpha=alpha) + axx[2].plot(data_x, data_dx, color, lw=1.2, label=label, alpha=alpha) if scatter: if B3i.heights.size > h_mean.size: - axx[3].scatter( B3i.heights[0:-1], h_mean, 1, c=color, label=label) - axx[4].scatter( B3i_gradient[0:-1], data_dx, 1, c=color , label=label) + axx[3].scatter(B3i.heights[0:-1], h_mean, 1, c=color, label=label) + axx[4].scatter(B3i_gradient[0:-1], data_dx, 1, c=color, label=label) else: - axx[3].scatter( B3i.heights, h_mean, 1, c=color, label=label) - axx[4].scatter( B3i_gradient, data_dx, 1, c=color , label=label) + axx[3].scatter(B3i.heights, h_mean, 1, c=color, label=label) + axx[4].scatter(B3i_gradient, data_dx, 1, c=color, label=label) - axx[3].plot( h_mean, h_mean, '-k', lw=0.5) - axx[4].plot( B3i_gradient[0:-1], B3i_gradient[0:-1] , '-k', lw=0.5) + axx[3].plot(h_mean, h_mean, "-k", lw=0.5) + axx[4].plot(B3i_gradient[0:-1], B3i_gradient[0:-1], "-k", lw=0.5) -b, spot = 'gt1r', 2 +b, spot = "gt1r", 2 B3i = B3[b][3:110] # b, spot = 'gt3r', 6 # B3i = B3[b][2:102] -lims = [B3i.lats.min(), B3i.lats.max() ] +lims = [B3i.lats.min(), B3i.lats.max()] # select data given limits -B2i = select_beam_section(B2, b, lims) +B2i = select_beam_section(B2, b, lims) -F, axx,B3i_gradient = init_compare_figure(B2i, B3i, lims) +F, axx, B3i_gradient = init_compare_figure(B2i, B3i, lims) ## add more data B3_atl06_native = select_beam_section(ATL06_native[20], spot, lims) -B3_atl06_yapc = select_beam_section(ATL06_yapc[20], spot, lims) +B3_atl06_yapc = select_beam_section(ATL06_yapc[20], spot, lims) delta_y = 0.0 -add_data(axx, B3_atl06_native, delta_y,10, 'orange', 'ATL06 +'+str(delta_y), alpha=0.8, scatter=False) -#axx[2].plot(B3_atl06_native.geometry.y, -np.gradient(B3_atl06_native.h_mean)+0.02, 'darkgreen' ) - -add_data(axx, B3_atl06_yapc, delta_y, 10, 'g', 'ATL06 YAPC +'+str(delta_y), alpha=0.8, scatter=False) -#axx[2].plot(B3_atl06_yapc.geometry.y, -np.gradient(B3_atl06_yapc.h_mean) - 0.02 , 'lightgreen' ) +add_data( + axx, + B3_atl06_native, + delta_y, + 10, + "orange", + "ATL06 +" + str(delta_y), + alpha=0.8, + scatter=False, +) +# axx[2].plot(B3_atl06_native.geometry.y, -np.gradient(B3_atl06_native.h_mean)+0.02, 'darkgreen' ) + +add_data( + axx, + B3_atl06_yapc, + delta_y, + 10, + "g", + "ATL06 YAPC +" + str(delta_y), + alpha=0.8, + scatter=False, +) +# axx[2].plot(B3_atl06_yapc.geometry.y, -np.gradient(B3_atl06_yapc.h_mean) - 0.02 , 'lightgreen' ) axx[1].legend(ncol=3) -F.fig.suptitle('ATL06 native vs. YAPC') +F.fig.suptitle("ATL06 native vs. YAPC") plt.show() -F.save_light(name ='SlideRule_ATL06_compare_weak', path=plot_path) +F.save_light(name="SlideRule_ATL06_compare_weak", path=plot_path) # %% Test windowing ATL06_native -#b, spot = 'gt3r', 6 -b, spot = 'gt1r', 2 +# b, spot = 'gt3r', 6 +b, spot = "gt1r", 2 B3i = B3[b][3:110] -lims = [B3i.lats.min(), B3i.lats.max() ] +lims = [B3i.lats.min(), B3i.lats.max()] # select data given limits -B2i = select_beam_section(B2, b, lims) +B2i = select_beam_section(B2, b, lims) F, axx, B3i_gradient = init_compare_figure(B2i, B3i, lims) ## add data -delta_y=0.0 -for ll,ccol in zip([10, 20, 30], ['green', 'darkorange', 'red']): +delta_y = 0.0 +for ll, ccol in zip([10, 20, 30], ["green", "darkorange", "red"]): B3_atl06_native = select_beam_section(ATL06_native[ll], spot, lims) delta_y += 0.1 - add_data(axx, B3_atl06_native, delta_y,10, ccol, 'len='+ str(ll) +'| +' +str(np.round(delta_y, 2)), alpha=0.8, scatter=False) - #axx[2].plot(B3_atl06_native.geometry.y, -np.gradient(B3_atl06_native.h_mean)+0.02, 'darkgreen' ) + add_data( + axx, + B3_atl06_native, + delta_y, + 10, + ccol, + "len=" + str(ll) + "| +" + str(np.round(delta_y, 2)), + alpha=0.8, + scatter=False, + ) + # axx[2].plot(B3_atl06_native.geometry.y, -np.gradient(B3_atl06_native.h_mean)+0.02, 'darkgreen' ) axx[1].legend(ncol=3) -axx[1].set_ylim(B3i.heights.min(), B3i.heights.max()+0.2) -axx[2].set_ylim( np.nanmin(B3i_gradient), np.nanmax(B3i_gradient)) -F.fig.suptitle('ATL06 native') +axx[1].set_ylim(B3i.heights.min(), B3i.heights.max() + 0.2) +axx[2].set_ylim(np.nanmin(B3i_gradient), np.nanmax(B3i_gradient)) +F.fig.suptitle("ATL06 native") -F.save_light(name ='SlideRule_ATL06_native_window_weak', path=plot_path) +F.save_light(name="SlideRule_ATL06_native_window_weak", path=plot_path) plt.show() # %% Test windowing ATL06_yapc -b, spot = 'gt1r', 2 +b, spot = "gt1r", 2 B3i = B3[b][3:102] -lims = [B3i.lats.min(), B3i.lats.max() ] +lims = [B3i.lats.min(), B3i.lats.max()] # select data given limits -B2i = select_beam_section(B2, b, lims) +B2i = select_beam_section(B2, b, lims) F, axx, B3i_gradient = init_compare_figure(B2i, B3i, lims) ## add data -delta_y=0.0 -for ll,ccol in zip([10, 20, 30], ['green', 'darkorange', 'red']): +delta_y = 0.0 +for ll, ccol in zip([10, 20, 30], ["green", "darkorange", "red"]): B3_atl06_yapc = select_beam_section(ATL06_yapc[ll], spot, lims) delta_y += 0.1 - add_data(axx, B3_atl06_yapc, delta_y,10, ccol, 'len='+str(ll)+'| +' +str(np.round(delta_y, 2)), alpha=0.8, scatter=False) - #axx[2].plot(B3_atl06_native.geometry.y, -np.gradient(B3_atl06_native.h_mean)+0.02, 'darkgreen' ) + add_data( + axx, + B3_atl06_yapc, + delta_y, + 10, + ccol, + "len=" + str(ll) + "| +" + str(np.round(delta_y, 2)), + alpha=0.8, + scatter=False, + ) + # axx[2].plot(B3_atl06_native.geometry.y, -np.gradient(B3_atl06_native.h_mean)+0.02, 'darkgreen' ) axx[1].legend(ncol=3) -axx[1].set_ylim(B3i.heights.min(), B3i.heights.max()+0.2) -axx[2].set_ylim( np.nanmin(B3i_gradient), np.nanmax(B3i_gradient)) -F.fig.suptitle('ATL06 YAPC') +axx[1].set_ylim(B3i.heights.min(), B3i.heights.max() + 0.2) +axx[2].set_ylim(np.nanmin(B3i_gradient), np.nanmax(B3i_gradient)) +F.fig.suptitle("ATL06 YAPC") plt.show() -#F.save_light(name ='SlideRule_ATL06_yapc_window_weak', path=plot_path) +# F.save_light(name ='SlideRule_ATL06_yapc_window_weak', path=plot_path) # %% YAPC classifications: # --K, -k @@ -373,18 +440,22 @@ def add_data(axx, data, delta_y, dx, color, label, alpha=1.0, scatter=True): # Use 0 for dynamic window height -params_yapc={'srt': 1, - 'len': 20, - 'ats':5, - 'res':10, - 'track': 0, - 'pass_invalid': True, - 'cnf': -2, - 'iterations':10, - 't0': '2019-05-02T02:12:24', - 't1': '2019-05-02T03:00:00', - "yapc": dict(knn=0, win_h=6, win_x=20, min_ph=4, score=100), # use the YAPC photon classifier; these are the recommended parameters, but the results might be more specific with a smaller win_h value, or a higher score cutoff - 'poly':poly} +params_yapc = { + "srt": 1, + "len": 20, + "ats": 5, + "res": 10, + "track": 0, + "pass_invalid": True, + "cnf": -2, + "iterations": 10, + "t0": "2019-05-02T02:12:24", + "t1": "2019-05-02T03:00:00", + "yapc": dict( + knn=0, win_h=6, win_x=20, min_ph=4, score=100 + ), # use the YAPC photon classifier; these are the recommended parameters, but the results might be more specific with a smaller win_h value, or a higher score cutoff + "poly": poly, +} win_x_list = [10, 20, 30] @@ -393,45 +464,58 @@ def add_data(axx, data, delta_y, dx, color, label, alpha=1.0, scatter=True): ATL06_yapc_winx = dict() wwin_h = 10 for wwin_x in [10, 20, 30]: - params_yapc['yapc'] = dict(knn=0,win_x=wwin_x, win_h=wwin_h, min_ph=4, score=100) - # Run the parallel version of the ATL06 algorithm, for a specified granule: - #case_name = str(wwin_x) +'_' +str(wwin_h) - ATL06_yapc_winx[wwin_x] = icesat2.atl06p(params_yapc, asset="nsidc-s3", resources=[track_name]) + params_yapc["yapc"] = dict(knn=0, win_x=wwin_x, win_h=wwin_h, min_ph=4, score=100) + # Run the parallel version of the ATL06 algorithm, for a specified granule: + # case_name = str(wwin_x) +'_' +str(wwin_h) + ATL06_yapc_winx[wwin_x] = icesat2.atl06p( + params_yapc, asset="nsidc-s3", resources=[track_name] + ) ATL06_yapc_winh = dict() wwin_x = 20 -for wwin_h in [0, 10, 20]: - params_yapc['yapc'] = dict(knn=0,win_x=wwin_x, win_h=wwin_h, min_ph=4, score=100) - # Run the parallel version of the ATL06 algorithm, for a specified granule: - #case_name = str(wwin_x) +'_' +str(wwin_h) - ATL06_yapc_winh[wwin_h] = icesat2.atl06p(params_yapc, asset="nsidc-s3", resources=[track_name]) +for wwin_h in [0, 10, 20]: + params_yapc["yapc"] = dict(knn=0, win_x=wwin_x, win_h=wwin_h, min_ph=4, score=100) + # Run the parallel version of the ATL06 algorithm, for a specified granule: + # case_name = str(wwin_x) +'_' +str(wwin_h) + ATL06_yapc_winh[wwin_h] = icesat2.atl06p( + params_yapc, asset="nsidc-s3", resources=[track_name] + ) # %% -b, spot = 'gt1r', 2 +b, spot = "gt1r", 2 B3i = B3[b][3:102] -lims = [B3i.lats.min(), B3i.lats.max() ] +lims = [B3i.lats.min(), B3i.lats.max()] # select data given limits -B2i = select_beam_section(B2, b, lims) +B2i = select_beam_section(B2, b, lims) F, axx, B3i_gradient = init_compare_figure(B2i, B3i, lims) ## add data -delta_y=0.0 -for ll,ccol in zip( list(ATL06_yapc_winx.keys()) , ['green', 'darkorange', 'red']): +delta_y = 0.0 +for ll, ccol in zip(list(ATL06_yapc_winx.keys()), ["green", "darkorange", "red"]): B3_atl06_yapc = select_beam_section(ATL06_yapc_winx[ll], spot, lims) delta_y += 0.0 - add_data(axx, B3_atl06_yapc, delta_y,10, ccol, 'win_x='+str(ll)+'| +' +str(np.round(delta_y, 2)), alpha=0.8, scatter=False) - #axx[2].plot(B3_atl06_native.geometry.y, -np.gradient(B3_atl06_native.h_mean)+0.02, 'darkgreen' ) + add_data( + axx, + B3_atl06_yapc, + delta_y, + 10, + ccol, + "win_x=" + str(ll) + "| +" + str(np.round(delta_y, 2)), + alpha=0.8, + scatter=False, + ) + # axx[2].plot(B3_atl06_native.geometry.y, -np.gradient(B3_atl06_native.h_mean)+0.02, 'darkgreen' ) axx[1].legend(ncol=2) -axx[1].set_ylim(B3i.heights.min(), B3i.heights.max()+0.2) -axx[2].set_ylim( np.nanmin(B3i_gradient), np.nanmax(B3i_gradient)) -F.fig.suptitle('ATL06 YAPC win_x comapare') +axx[1].set_ylim(B3i.heights.min(), B3i.heights.max() + 0.2) +axx[2].set_ylim(np.nanmin(B3i_gradient), np.nanmax(B3i_gradient)) +F.fig.suptitle("ATL06 YAPC win_x comapare") plt.show() @@ -439,103 +523,129 @@ def add_data(axx, data, delta_y, dx, color, label, alpha=1.0, scatter=True): F, axx, B3i_gradient = init_compare_figure(B2i, B3i, lims) ## add data -delta_y=0.0 -for ll,ccol in zip( list(ATL06_yapc_winh.keys()) , ['green', 'darkorange', 'red']): +delta_y = 0.0 +for ll, ccol in zip(list(ATL06_yapc_winh.keys()), ["green", "darkorange", "red"]): B3_atl06_yapc = select_beam_section(ATL06_yapc_winh[ll], spot, lims) delta_y += 0.0 - add_data(axx, B3_atl06_yapc, delta_y,10, ccol, 'win_h='+str(ll)+'| +' +str(np.round(delta_y, 2)), alpha=0.8, scatter=False) - #axx[2].plot(B3_atl06_native.geometry.y, -np.gradient(B3_atl06_native.h_mean)+0.02, 'darkgreen' ) + add_data( + axx, + B3_atl06_yapc, + delta_y, + 10, + ccol, + "win_h=" + str(ll) + "| +" + str(np.round(delta_y, 2)), + alpha=0.8, + scatter=False, + ) + # axx[2].plot(B3_atl06_native.geometry.y, -np.gradient(B3_atl06_native.h_mean)+0.02, 'darkgreen' ) axx[1].legend(ncol=2) -axx[1].set_ylim(B3i.heights.min(), B3i.heights.max()+0.2) -axx[2].set_ylim( np.nanmin(B3i_gradient), np.nanmax(B3i_gradient)) -F.fig.suptitle('ATL06 YAPC win_h comapare') +axx[1].set_ylim(B3i.heights.min(), B3i.heights.max() + 0.2) +axx[2].set_ylim(np.nanmin(B3i_gradient), np.nanmax(B3i_gradient)) +F.fig.suptitle("ATL06 YAPC win_h comapare") plt.show() # %% plot YAPC classifications for case above just for checking # --- look up again what classifies as icesat2.SRT_OCEAN and icesat2.SRT_SEA_ICE ? -#The SRT categorization is non-exclusive. Ocean photons can be also Sea ice photons. +# The SRT categorization is non-exclusive. Ocean photons can be also Sea ice photons. parms = { # processing parameters - 'release' :'005', + "release": "005", "srt": icesat2.SRT_OCEAN, - 'time_start':'2019-05-02T02:12:24', - 'time_end':'2019-05-02T03:00:00', + "time_start": "2019-05-02T02:12:24", + "time_end": "2019-05-02T03:00:00", "len": 20, - 'track':1, # select gt1l + "track": 1, # select gt1l # classification and checks # still return photon segments that fail checks - "pass_invalid": True, + "pass_invalid": True, # all photons - "cnf": -2, - "yapc": dict(knn=0, win_h=10, win_x=11, min_ph=4, score=0), - 'atl03_ph_fields' : ['dist_ph_along','dist_ph_across'], - 'poly':poly + "cnf": -2, + "yapc": dict(knn=0, win_h=10, win_x=11, min_ph=4, score=0), + "atl03_ph_fields": ["dist_ph_along", "dist_ph_across"], + "poly": poly, } -gdf = icesat2.atl03s(parms, asset=asset, resource=track_name)#granules_list) +gdf = icesat2.atl03s(parms, asset=asset, resource=track_name) # granules_list) # %% parms = { # processing parameters - 'release' :'005', + "release": "005", "srt": icesat2.SRT_SEA_ICE, - 'time_start':'2019-05-02T02:12:24', - 'time_end':'2019-05-02T03:00:00', + "time_start": "2019-05-02T02:12:24", + "time_end": "2019-05-02T03:00:00", "len": 20, - 'track':1, # select gt1l + "track": 1, # select gt1l # classification and checks # still return photon segments that fail checks - "pass_invalid": True, + "pass_invalid": True, # all photons - "cnf": -2, - "yapc": dict(knn=0, win_h=10, win_x=11, min_ph=4, score=0), - 'atl03_ph_fields' : ['dist_ph_along','dist_ph_across'], - 'poly':poly + "cnf": -2, + "yapc": dict(knn=0, win_h=10, win_x=11, min_ph=4, score=0), + "atl03_ph_fields": ["dist_ph_along", "dist_ph_across"], + "poly": poly, } -gdf_seaice = icesat2.atl03s(parms, asset=asset, resource=track_name)#granules_list) +gdf_seaice = icesat2.atl03s(parms, asset=asset, resource=track_name) # granules_list) # %% -b, spot = 'gt1r', 2 -#B3i = B3[b][600:702] +b, spot = "gt1r", 2 +# B3i = B3[b][600:702] B3i = B3[b][0:102] -lims = [B3i.lats.min(), B3i.lats.max() ] +lims = [B3i.lats.min(), B3i.lats.max()] # select data given limits -B2i = select_beam_section(B2, b, lims) +B2i = select_beam_section(B2, b, lims) -#F, axx, B3i_gradient = init_compare_figure(B2i, B3i, lims) +# F, axx, B3i_gradient = init_compare_figure(B2i, B3i, lims) -gdfi = select_beam_section(gdf, spot, lims) -gdf_seaicei = select_beam_section(gdf_seaice, spot, lims) +gdfi = select_beam_section(gdf, spot, lims) +gdf_seaicei = select_beam_section(gdf_seaice, spot, lims) # Make 1 panel figure with B3i.heights and gdfi.height F = M.figure_axis_xy(7.5, 3) -ax =F.ax - -ax.plot( B2i.lats, B2i.heights, '.', color = 'gray', markersize=0.8, alpha = 0.6) -ax.plot( B3i.lats, B3i.heights, '-k', lw=0.8, label='B03 H&H 2023', zorder=12) - -score_curoff = 10 #230 -mask = gdfi['yapc_score'] > score_curoff -#ax.plot( gdfi[~mask].geometry.y , gdfi[~mask].height, '.', color='red', markersize=1.5, alpha = 0.6, label='YAPC score < '+str(score_curoff)) - - -axx[1].plot( gdfi['segment_dist'] + gdfi['distance'], gdfi.height, '.') -hm = ax.scatter( gdfi[mask].geometry.y , gdfi[mask].height, c=gdfi[mask]['yapc_score'], s=2, cmap='viridis', vmin = 0, vmax = 256, zorder= 10) - -ax.plot( gdf_seaicei.geometry.y , gdf_seaicei.height, '+', color = 'red', markersize=4, alpha = 1, zorder= 3) - - -plt.ylim( list(gdfi[mask].height.quantile([0.02, 0.98])) ) -#plt.ylim( list(B3i.heights.quantile([0.01, 0.99])) ) +ax = F.ax + +ax.plot(B2i.lats, B2i.heights, ".", color="gray", markersize=0.8, alpha=0.6) +ax.plot(B3i.lats, B3i.heights, "-k", lw=0.8, label="B03 H&H 2023", zorder=12) + +score_curoff = 10 # 230 +mask = gdfi["yapc_score"] > score_curoff +# ax.plot( gdfi[~mask].geometry.y , gdfi[~mask].height, '.', color='red', markersize=1.5, alpha = 0.6, label='YAPC score < '+str(score_curoff)) + + +axx[1].plot(gdfi["segment_dist"] + gdfi["distance"], gdfi.height, ".") +hm = ax.scatter( + gdfi[mask].geometry.y, + gdfi[mask].height, + c=gdfi[mask]["yapc_score"], + s=2, + cmap="viridis", + vmin=0, + vmax=256, + zorder=10, +) + +ax.plot( + gdf_seaicei.geometry.y, + gdf_seaicei.height, + "+", + color="red", + markersize=4, + alpha=1, + zorder=3, +) + + +plt.ylim(list(gdfi[mask].height.quantile([0.02, 0.98]))) +# plt.ylim( list(B3i.heights.quantile([0.01, 0.99])) ) plt.legend() plt.colorbar(hm) diff --git a/SlideRule/S02_SlideRule_trials.py b/SlideRule/S02_SlideRule_trials.py index 93c6e3a7..c8acf125 100644 --- a/SlideRule/S02_SlideRule_trials.py +++ b/SlideRule/S02_SlideRule_trials.py @@ -24,10 +24,12 @@ import datetime import numpy as np import shapely + sliderule.__version__ import matplotlib -#matplotlib.use('agg') + +# matplotlib.use('agg') # %matplotlib inline # %matplotlib widget @@ -35,33 +37,37 @@ # %% # Configure Session # -#icesat2.init("icesat2sliderule.org", True) -icesat2.init("slideruleearth.io", True) #doesn't work -asset = 'nsidc-s3' +# icesat2.init("icesat2sliderule.org", True) +icesat2.init("slideruleearth.io", True) # doesn't work +asset = "nsidc-s3" # %% -latR=[-67.5, -64.5] -lonR=[59.6, 67.7] +latR = [-67.5, -64.5] +lonR = [59.6, 67.7] -poly=[{'lat':latR[ii], 'lon':lonR[jj]} for ii, jj in zip([1, 1, 0, 0, 1], [1, 0, 0, 1, 1])] +poly = [ + {"lat": latR[ii], "lon": lonR[jj]} + for ii, jj in zip([1, 1, 0, 0, 1], [1, 0, 0, 1, 1]) +] -track_name = 'ATL03_20190502021224_05160312_005_01.h5' +track_name = "ATL03_20190502021224_05160312_005_01.h5" ## Generate ATL06-type segments using the ATL03-native photon classification # Use the ocean classification for photons with a confidence parmeter to 2 or higher (low confidence or better) -params={'srt': 1, # Ocean classification - 'len': 10, # 10-meter segments - 'ats':5, # require that each segment contain photons separated by at least 5 m - 'res':5, # return one photon every 5 m - 'track': 0, # return all ground tracks - 'pass_invalid': True, - 'cnf': 2, # require classification confidence of 2 or more - 'iterations':10, # iterate the fit - 't0': '2019-05-02T02:12:24', # time range (not needed in this case) - 't1': '2019-05-02T03:00:00', - 'poly': poly, # polygon within which to select photons, - 'atl03_ph_fields' : ['dist_ph_along','dist_ph_across'], +params = { + "srt": 1, # Ocean classification + "len": 10, # 10-meter segments + "ats": 5, # require that each segment contain photons separated by at least 5 m + "res": 5, # return one photon every 5 m + "track": 0, # return all ground tracks + "pass_invalid": True, + "cnf": 2, # require classification confidence of 2 or more + "iterations": 10, # iterate the fit + "t0": "2019-05-02T02:12:24", # time range (not needed in this case) + "t1": "2019-05-02T03:00:00", + "poly": poly, # polygon within which to select photons, + "atl03_ph_fields": ["dist_ph_along", "dist_ph_across"], } # Run the parallel version of the ATL06 algorithm, for a specified granule: @@ -77,37 +83,53 @@ # Adjusting the YAPC 'score' parameter to something higher than 100 will return fewer photons that are more tightly clustered. -#icesat2.init("slideruleearth.io", True, organization="sliderule") -icesat2.init("slideruleearth.io", True)#, organization="sliderule") - -params={'srt': 1, - 'len': 10, - 'ats':5, - 'res':5, - 'track': 0, - 'pass_invalid': True, - 'cnf': -2, - 'iterations':10, - 't0': '2019-05-02T02:12:24', - 't1': '2019-05-02T03:00:00', - "yapc": dict(knn=0, win_h=6, win_x=11, min_ph=4, score=100), # use the YAPC photon classifier; these are the recommended parameters, but the results might be more specific with a smaller win_h value, or a higher score cutoff -'atl03_ph_fields' : ['dist_ph_along','dist_ph_across'], - 'poly':poly} +# icesat2.init("slideruleearth.io", True, organization="sliderule") +icesat2.init("slideruleearth.io", True) # , organization="sliderule") + +params = { + "srt": 1, + "len": 10, + "ats": 5, + "res": 5, + "track": 0, + "pass_invalid": True, + "cnf": -2, + "iterations": 10, + "t0": "2019-05-02T02:12:24", + "t1": "2019-05-02T03:00:00", + "yapc": dict( + knn=0, win_h=6, win_x=11, min_ph=4, score=100 + ), # use the YAPC photon classifier; these are the recommended parameters, but the results might be more specific with a smaller win_h value, or a higher score cutoff + "atl03_ph_fields": ["dist_ph_along", "dist_ph_across"], + "poly": poly, +} -gdf_yapc = icesat2.atl06p(params, asset="nsidc-s3", resources=[track_name])#, callbacks = {"atl03rec": atl03rec_cb}) +gdf_yapc = icesat2.atl06p( + params, asset="nsidc-s3", resources=[track_name] +) # , callbacks = {"atl03rec": atl03rec_cb}) # %% Plot the results by spot: -hf, hax=plt.subplots(2,3, sharex=True, sharey=True) -hax=hax.T.ravel() -npoints= -1 -for ha, spot in zip(hax.ravel(), np.arange(1,7)): - - ii=(gdf_yapc.spot==spot) & (gdf_yapc.w_surface_window_final < 5) - ha.plot( gdf_yapc.geometry.y[ii][0:npoints], gdf_yapc.h_mean[ii][0:npoints],'.', markersize=2) - - ii=(gdf_sea_ice.spot==spot) - ha.plot( gdf_sea_ice.geometry.y[ii][0:npoints], gdf_sea_ice.h_mean[ii][0:npoints],'.', markersize=2) +hf, hax = plt.subplots(2, 3, sharex=True, sharey=True) +hax = hax.T.ravel() +npoints = -1 +for ha, spot in zip(hax.ravel(), np.arange(1, 7)): + + ii = (gdf_yapc.spot == spot) & (gdf_yapc.w_surface_window_final < 5) + ha.plot( + gdf_yapc.geometry.y[ii][0:npoints], + gdf_yapc.h_mean[ii][0:npoints], + ".", + markersize=2, + ) + + ii = gdf_sea_ice.spot == spot + ha.plot( + gdf_sea_ice.geometry.y[ii][0:npoints], + gdf_sea_ice.h_mean[ii][0:npoints], + ".", + markersize=2, + ) ha.set_title(spot) plt.ylim(22.5, 30) @@ -117,75 +139,72 @@ # Downloading a whole granule worth of ATL03 photons takes a lot of time and bandwidth, but it can be instructive to look at a small portion of one granule. Here's how to look at the YAPC classification for one track for part of the granule - parms = { # processing parameters - 'release' :'005', + "release": "005", "srt": icesat2.SRT_OCEAN, - 'time_start':'2019-05-02T02:12:24', - 'time_end':'2019-05-02T03:00:00', + "time_start": "2019-05-02T02:12:24", + "time_end": "2019-05-02T03:00:00", "len": 20, - 'track':2, # select gt1l + "track": 2, # select gt1l # classification and checks # still return photon segments that fail checks - "pass_invalid": True, + "pass_invalid": True, # all photons - "cnf": -2, - "yapc": dict(knn=0, win_h=6, win_x=11, min_ph=4, score=0), - 'atl03_ph_fields' : ['dist_ph_along','dist_ph_across'], - 'poly':poly_sub + "cnf": -2, + "yapc": dict(knn=0, win_h=6, win_x=11, min_ph=4, score=0), + "atl03_ph_fields": ["dist_ph_along", "dist_ph_across"], + "poly": poly_sub, } -gdf = icesat2.atl03s(parms, asset=asset, resource=track_name)#granules_list) +gdf = icesat2.atl03s(parms, asset=asset, resource=track_name) # granules_list) gdf.size # %% -F = M.figure_axis_xy(4,3) +F = M.figure_axis_xy(4, 3) -for ispot in range(1,5): - ii=(gdf.spot==ispot) +for ispot in range(1, 5): + ii = gdf.spot == ispot gdf2 = gdf[ii][1:200:2] - plt.plot( gdf2['segment_dist'] + gdf2['distance'], gdf2.dist_ph_across, '.') - #plt.plot( gdf2['distance'], gdf2.dist_ph_across, 'k.') - plt.plot( gdf2['segment_dist'] +gdf2.dist_ph_along, gdf2.dist_ph_across+10, '.') - + plt.plot(gdf2["segment_dist"] + gdf2["distance"], gdf2.dist_ph_across, ".") + # plt.plot( gdf2['distance'], gdf2.dist_ph_across, 'k.') + plt.plot(gdf2["segment_dist"] + gdf2.dist_ph_along, gdf2.dist_ph_across + 10, ".") - - #plt.plot( gdf_yapc2['distance'], gdf_yapc2['distance']*0, '.') + # plt.plot( gdf_yapc2['distance'], gdf_yapc2['distance']*0, '.') plt.show() -#gdf_yapc.segment_id +# gdf_yapc.segment_id # %% -F = M.figure_axis_xy(4,3) - -for ispot in range(1,5): +F = M.figure_axis_xy(4, 3) +for ispot in range(1, 5): - ii=(gdf_yapc.spot==ispot) + ii = gdf_yapc.spot == ispot gdf_yapc2 = gdf_yapc[ii][1:200:2] - #plt.plot( gdf_yapc2['distance'], gdf2.dist_ph_across, '.') - plt.plot( gdf_yapc2['distance'], gdf_yapc2['segment_id'], '.') - #plt.plot( gdf2.geometry.x, gdf2.geometry.y, 'k.') + # plt.plot( gdf_yapc2['distance'], gdf2.dist_ph_across, '.') + plt.plot(gdf_yapc2["distance"], gdf_yapc2["segment_id"], ".") + # plt.plot( gdf2.geometry.x, gdf2.geometry.y, 'k.') plt.show() - # %% -#plt.hist(gdf.yapc_score , bins= 30) -ii=np.argsort(gdf.yapc_score) +# plt.hist(gdf.yapc_score , bins= 30) +ii = np.argsort(gdf.yapc_score) -plt.figure(); plt.scatter(gdf.geometry.y[ii], gdf.height[ii], 1, c=gdf.yapc_score[ii]) +plt.figure() +plt.scatter(gdf.geometry.y[ii], gdf.height[ii], 1, c=gdf.yapc_score[ii]) plt.colorbar() # %% -gdf_sel = gdf[gdf.yapc_score>50] +gdf_sel = gdf[gdf.yapc_score > 50] -ii=np.argsort(gdf_sel.yapc_score) -plt.figure(); plt.scatter(gdf_sel.geometry.y[ii], gdf_sel.height[ii], 1, c=gdf_sel.yapc_score[ii]) +ii = np.argsort(gdf_sel.yapc_score) +plt.figure() +plt.scatter(gdf_sel.geometry.y[ii], gdf_sel.height[ii], 1, c=gdf_sel.yapc_score[ii]) plt.colorbar() plt.ylim(1100, 1400) @@ -194,30 +213,29 @@ # gdf_yapc.geometry # gdf_sea_ice.geometry # %% plot sections -F = M.figure_axis_xy(6, 2, view_scale = 0.7) - -#hf, hax=plt.subplots(2,3, sharex=True, sharey=True) -hax=[F.ax] -npoints= -1#3000 -for ha, spot in zip(hax, np.arange(1,2)): - - ii=(gdf_yapc.spot==spot) & (gdf_yapc.w_surface_window_final < 5) - data= gdf_yapc.h_mean[ii][0:npoints] - ha.plot( gdf_yapc.geometry.y[ii][0:npoints],data,'.', markersize=2) - - ii=(gdf_sea_ice.spot==spot) - data= gdf_sea_ice.h_mean[ii][0:npoints] - ha.plot( gdf_sea_ice.geometry.y[ii][0:npoints], data,'.', markersize=2) - ha.set_title(spot) +F = M.figure_axis_xy(6, 2, view_scale=0.7) +# hf, hax=plt.subplots(2,3, sharex=True, sharey=True) +hax = [F.ax] +npoints = -1 # 3000 +for ha, spot in zip(hax, np.arange(1, 2)): + ii = (gdf_yapc.spot == spot) & (gdf_yapc.w_surface_window_final < 5) + data = gdf_yapc.h_mean[ii][0:npoints] + ha.plot(gdf_yapc.geometry.y[ii][0:npoints], data, ".", markersize=2) -ii=np.argsort(gdf.yapc_score) + ii = gdf_sea_ice.spot == spot + data = gdf_sea_ice.h_mean[ii][0:npoints] + ha.plot(gdf_sea_ice.geometry.y[ii][0:npoints], data, ".", markersize=2) + ha.set_title(spot) -#ha.scatter(gdf.geometry.y[ii], gdf.height[ii], 1, c=gdf.yapc_score[ii]) -#plt.colorbar() -#plt.xlim(-65.1, -64.8) -#plt.ylim(data.quantile([0.01, 0.99]) ) + +ii = np.argsort(gdf.yapc_score) + +# ha.scatter(gdf.geometry.y[ii], gdf.height[ii], 1, c=gdf.yapc_score[ii]) +# plt.colorbar() +# plt.xlim(-65.1, -64.8) +# plt.ylim(data.quantile([0.01, 0.99]) ) # %% @@ -231,10 +249,11 @@ gdf_sea_ice.geometry.y # %% -ii=(gdf_sea_ice.spot==spot) -gdf_sea_ice[ii]['distance'].plot() -ii=(gdf_sea_ice.spot==3) -gdf_sea_ice[ii]['distance'].plot() +ii = gdf_sea_ice.spot == spot +gdf_sea_ice[ii]["distance"].plot() +ii = gdf_sea_ice.spot == 3 +gdf_sea_ice[ii]["distance"].plot() + # %% def pole_ward_table(T): @@ -244,9 +263,12 @@ def pole_ward_table(T): """ if T is None: return None - time = T['time']['delta_time'] - lat = T['ref']['latitude'] - print('1st lat =' + str(abs(lat.iloc[time.argmin()])) , ';last lat =' + str(abs(lat.iloc[time.argmax()])) ) + time = T["time"]["delta_time"] + lat = T["ref"]["latitude"] + print( + "1st lat =" + str(abs(lat.iloc[time.argmin()])), + ";last lat =" + str(abs(lat.iloc[time.argmax()])), + ) return abs(lat.iloc[time.argmax()]) > abs(lat.iloc[time.argmin()]) @@ -260,34 +282,33 @@ def pole_ward_table(T): gdf_yapc.sample(1000).geometry.plot() gdf2 = gdf_yapc.to_crs(3857) -#gdf3 = gdf_sea_ice.to_crs(4979) +# gdf3 = gdf_sea_ice.to_crs(4979) gdf2.geometry.get_coordinates()[0:10] gdf2.geometry.rotate gdf2.geometry.translate gdf2.crs # gdf3.crs -gdf2[gdf2.spot == 1][0:30].geometry.plot(marker = '.', markersize=0.4) +gdf2[gdf2.spot == 1][0:30].geometry.plot(marker=".", markersize=0.4) gdf2.sample(1000).geometry.plot() +x0, y0 = gdf2s.geometry.x.min(), gdf2s.geometry.y.min() -x0,y0 = gdf2s.geometry.x.min(), gdf2s.geometry.y.min() - -dist = np.sqrt( (gdf2s.geometry.x- x0)**2 + (gdf2s.geometry.y- y0)**2 ) -dist.sort_values()[0:1000].diff().plot(marker= '.') -#dist.diff()[0:100].plot(marker= '.') +dist = np.sqrt((gdf2s.geometry.x - x0) ** 2 + (gdf2s.geometry.y - y0) ** 2) +dist.sort_values()[0:1000].diff().plot(marker=".") +# dist.diff()[0:100].plot(marker= '.') # %% -gdf2 = gdf_sea_ice#.to_crs(3857) -gdf2s = gdf2[gdf2.spot == 1]#.sample(1000) +gdf2 = gdf_sea_ice # .to_crs(3857) +gdf2s = gdf2[gdf2.spot == 1] # .sample(1000) plt.figure() -#gdf2s.sort_values('distance')['distance'].diff().hist(bins=np.arange(2.5, 20, 1)) -gdf2s['distance'].diff().hist(bins=np.arange(2.5, 20, 1)) +# gdf2s.sort_values('distance')['distance'].diff().hist(bins=np.arange(2.5, 20, 1)) +gdf2s["distance"].diff().hist(bins=np.arange(2.5, 20, 1)) -#plt.show() +# plt.show() # %% diff --git a/analyis_publish/PB04_angle_priors.py b/analyis_publish/PB04_angle_priors.py index ce31051e..a4e3a76a 100644 --- a/analyis_publish/PB04_angle_priors.py +++ b/analyis_publish/PB04_angle_priors.py @@ -24,7 +24,7 @@ import concurrent.futures as futures #from numba import jit -%matplotlib widget +#%matplotlib widget #from ICEsat2_SI_tools import angle_optimizer import ICEsat2_SI_tools.wave_tools as waves import concurrent.futures as futures diff --git a/analysis/B01_filter_regrid.py b/analysis/B01_filter_regrid.py index 72d6fcf3..d2e82e88 100644 --- a/analysis/B01_filter_regrid.py +++ b/analysis/B01_filter_regrid.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,15 +9,16 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -36,75 +38,91 @@ from random import sample import imp -#import s3fs +# import s3fs # Python reader based on Pandas. Other reader examples available in readers.py -track_name= 'ATL03_20190515060436_07170312_002_02' -load_path = base_path + 'data/data1/' -load_file = load_path + 'processed_'+track_name+'.h5' +track_name = "ATL03_20190515060436_07170312_002_02" +load_path = base_path + "data/data1/" +load_file = load_path + "processed_" + track_name + ".h5" -track_name= 'ATL03_20191215230028_12220512_004_01' -track_name= 'ATL03_20210414065545_03121112_004_01' -track_name= 'ATL03_20210414174223_03191110_004_01' +track_name = "ATL03_20191215230028_12220512_004_01" +track_name = "ATL03_20210414065545_03121112_004_01" +track_name = "ATL03_20210414174223_03191110_004_01" -load_path = base_path + 'data/data4/' -load_file = load_path + 'processed_'+track_name+'.h5' +load_path = base_path + "data/data4/" +load_file = load_path + "processed_" + track_name + ".h5" # % # test which beams exist: -all_beams = ['gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r'] -low_beams = ['gt1l', 'gt2l', 'gt3l'] -high_beams = ['gt1r', 'gt2r', 'gt3r'] +all_beams = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] +low_beams = ["gt1l", "gt2l", "gt3l"] +high_beams = ["gt1r", "gt2r", "gt3r"] -f = h5py.File(load_file, 'r') -beams = [b if b in f.keys() else None for b in all_beams] +f = h5py.File(load_file, "r") +beams = [b if b in f.keys() else None for b in all_beams] -def correct_heights(T03, T03c, coord = 'delta_time'): - T03['heights_c']= T03['heights'] - np.interp( T03[coord],T03c[coord], T03c['dem_h'] ) + +def correct_heights(T03, T03c, coord="delta_time"): + T03["heights_c"] = T03["heights"] - np.interp( + T03[coord], T03c[coord], T03c["dem_h"] + ) return T03 + # Load fata and apply height corrections # This needs version 2 of the ALT 03 dataset -hist= 'Beam stats' -B= dict() +hist = "Beam stats" +B = dict() for k in beams: - T = io.getATL03_beam(load_file, beam= k) + T = io.getATL03_beam(load_file, beam=k) ho = k - ho = MT.add_line_var(ho, 'size', str(T.shape[0])) - ho = MT.add_line_var(ho, 'by confidence levels:' + str(np.arange(0, 5)), [ (T['signal_confidence'] == i).sum() for i in np.arange(0, 5) ]) + ho = MT.add_line_var(ho, "size", str(T.shape[0])) + ho = MT.add_line_var( + ho, + "by confidence levels:" + str(np.arange(0, 5)), + [(T["signal_confidence"] == i).sum() for i in np.arange(0, 5)], + ) # filter: - Tsel = T[(T['signal_confidence']>2) & (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] + Tsel = T[ + (T["signal_confidence"] > 2) & (T["heights"] < 100) & (T["heights"] > -100) + ] # & (T['delta_time']>5) & (T['delta_time']<24) ] if len(Tsel) == 0: - ho = MT.add_line_var(ho, 'no photons found', '') - Tsel= T[(T['signal_confidence'] ==-1 ) & (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] + ho = MT.add_line_var(ho, "no photons found", "") + Tsel = T[ + (T["signal_confidence"] == -1) + & (T["heights"] < 100) + & (T["heights"] > -100) + ] # & (T['delta_time']>5) & (T['delta_time']<24) ] + Tsel_c = io.getATL03_height_correction(load_file) + Tsel_c = Tsel_c[Tsel_c["dem_h"] < 1e5] # cute out weird references + B[k] = correct_heights(Tsel, Tsel_c) - Tsel_c = io.getATL03_height_correction(load_file) - Tsel_c = Tsel_c[Tsel_c['dem_h'] < 1e5] # cute out weird references - B[k] = correct_heights(Tsel, Tsel_c) - - ho = MT.add_line_var(ho, 'selected size', str(Tsel.shape[0])) - ho = MT.add_line_var(ho, 'final size ', str(Tsel_c.shape[0])) + ho = MT.add_line_var(ho, "selected size", str(Tsel.shape[0])) + ho = MT.add_line_var(ho, "final size ", str(Tsel_c.shape[0])) print(ho) - hist = MT.write_log(hist, ho) + hist = MT.write_log(hist, ho) # %% + def track_type(T): """ Returns if track acending or desending T is a pandas table """ - #T = B[k] - #T = B[beams_list[0]] - return (T['lats'].iloc[T['delta_time'].argmax()] - T['lats'].iloc[T['delta_time'].argmin()] ) < 0 - + # T = B[k] + # T = B[beams_list[0]] + return ( + T["lats"].iloc[T["delta_time"].argmax()] + - T["lats"].iloc[T["delta_time"].argmin()] + ) < 0 def lat_min_max(B, beams_list): @@ -117,28 +135,29 @@ def lat_min_max(B, beams_list): returns: min_lat, max_lat, accent min and max latitudes of the beams, (True/False) True if the track is accending """ - #B, beams_list = B , high_beams - accent = track_type( B[beams_list[0]] ) + # B, beams_list = B , high_beams + accent = track_type(B[beams_list[0]]) - if B[beams_list[0]]['lats'].iloc[0] < 0: - hemis = 'SH' + if B[beams_list[0]]["lats"].iloc[0] < 0: + hemis = "SH" else: - hemis = 'NH' + hemis = "NH" - track_lat_mins, track_lat_maxs= list(), list() + track_lat_mins, track_lat_maxs = list(), list() for k in beams_list: - track_lat_mins.append( B[k]['lats'].min() ) - track_lat_maxs.append( B[k]['lats'].max() ) + track_lat_mins.append(B[k]["lats"].min()) + track_lat_maxs.append(B[k]["lats"].max()) - if hemis == 'SH': - return max(track_lat_maxs) , min(track_lat_mins), accent + if hemis == "SH": + return max(track_lat_maxs), min(track_lat_mins), accent else: return min(track_lat_mins), max(track_lat_maxs), accent # %% -def derive_axis(TT, lat_lims = None): + +def derive_axis(TT, lat_lims=None): """ returns TT distance along track 'dist' in meters input: @@ -147,43 +166,43 @@ def derive_axis(TT, lat_lims = None): returns: TT with x,y,dist and order by dist """ - #TT, lat_lims = B[key], lat_lims_high + # TT, lat_lims = B[key], lat_lims_high # derive distances in meters - r_e= 6.3710E+6 - dy= r_e*2*np.pi/360.0 - #deglon_in_m= np.cos(T2['lats']*np.pi/180.0)*dy + r_e = 6.3710e6 + dy = r_e * 2 * np.pi / 360.0 + # deglon_in_m= np.cos(T2['lats']*np.pi/180.0)*dy # either use position of the 1st photon or use defined start latitude if lat_lims is None: - TT['y']=(TT['lats'].max() - TT['lats']) *dy + TT["y"] = (TT["lats"].max() - TT["lats"]) * dy else: - TT['y']=(lat_lims[0] - TT['lats']) *dy - - #TT['y'] = (TT['lats']) *dy + TT["y"] = (lat_lims[0] - TT["lats"]) * dy + # TT['y'] = (TT['lats']) *dy - if (lat_lims[2] == True): + if lat_lims[2] == True: # accending track - lon_min = TT['lons'].max() + lon_min = TT["lons"].max() else: # decending track - lon_min = TT['lons'].min() + lon_min = TT["lons"].min() - #print(lon_min) - TT['x'] = (TT['lons'] - lon_min) * np.cos( TT['lats']*np.pi/180.0 ) * dy - #TT['x'] = (TT['lons'] ) * np.cos( TT['lats']*np.pi/180.0 ) * dy - TT['dist'] = np.sqrt(TT['x']**2 + TT['y']**2) + # print(lon_min) + TT["x"] = (TT["lons"] - lon_min) * np.cos(TT["lats"] * np.pi / 180.0) * dy + # TT['x'] = (TT['lons'] ) * np.cos( TT['lats']*np.pi/180.0 ) * dy + TT["dist"] = np.sqrt(TT["x"] ** 2 + TT["y"] ** 2) # set 1st dist to 0, not used if global limits are used if lat_lims is None: - TT['dist']= TT['dist']- TT['dist'].min() + TT["dist"] = TT["dist"] - TT["dist"].min() else: - TT['dist']= TT['dist']#- lat_lims[0] + TT["dist"] = TT["dist"] # - lat_lims[0] - TT=TT.sort_values(by='dist') + TT = TT.sort_values(by="dist") return TT -def reduce_to_height_distance(TT, key, dx=1, lat_lims = None): + +def reduce_to_height_distance(TT, key, dx=1, lat_lims=None): """ interpolates key (photos heights) to regular grid using 'dist' in pandas table TT. dx is the interpolation interval @@ -194,17 +213,18 @@ def reduce_to_height_distance(TT, key, dx=1, lat_lims = None): x1, y1 position, height """ from scipy.interpolate import interp1d + if type(dx) is np.ndarray: x1 = dx else: - x1 = np.arange(0,TT['dist'].max(), dx) - y1 = np.interp(x1, TT['dist'], TT[key] ) + x1 = np.arange(0, TT["dist"].max(), dx) + y1 = np.interp(x1, TT["dist"], TT[key]) return x1, y1 + # this is not need anymore def poly_correct(x, y, poly_order=7, plot_flag=False): - """ subtracts a fitted polynom to y inputs: @@ -214,12 +234,22 @@ def poly_correct(x, y, poly_order=7, plot_flag=False): returns y' y - polynom fit """ - z = np.polyfit(x , y , poly_order) + z = np.polyfit(x, y, poly_order) p = np.poly1d(z) if plot_flag: - plt.plot(x,y, '.', markersize=0.2,) - plt.plot(x, p(x), '-', markersize=0.2,) - #return z + plt.plot( + x, + y, + ".", + markersize=0.2, + ) + plt.plot( + x, + p(x), + "-", + markersize=0.2, + ) + # return z return y - p(x) @@ -230,51 +260,52 @@ def poly_correct(x, y, poly_order=7, plot_flag=False): ##### 1.) derive common axis for beams # %% 1st all stong beams -B2=dict() -colors = iter(['red','blue','orange','green','black','yellow']) -dist_list =np.array([np.nan, np.nan]) +B2 = dict() +colors = iter(["red", "blue", "orange", "green", "black", "yellow"]) +dist_list = np.array([np.nan, np.nan]) for key in high_beams: - T2 = derive_axis(B[key], lat_lims_high) + T2 = derive_axis(B[key], lat_lims_high) # the ends of the tracks are about 600m appart.. i don't know why .. # this should have no influence on the distrance along the track - #plt.plot( list(T2['x'][-200:]), list(T2['y'][-200:]), c = next(colors)) - #plt.plot( list(T2['x'][15000:15005]), list(T2['y'][15000:15005]), '.', c = next(colors)) - #plt.plot( list(T2['lons'][:2]), list(T2['lats'][:2]), c = next(colors)) - #plt.axis('equal') + # plt.plot( list(T2['x'][-200:]), list(T2['y'][-200:]), c = next(colors)) + # plt.plot( list(T2['x'][15000:15005]), list(T2['y'][15000:15005]), '.', c = next(colors)) + # plt.plot( list(T2['lons'][:2]), list(T2['lats'][:2]), c = next(colors)) + # plt.axis('equal') # depreciated - #T2['heights_adjusted'] = poly_correct(T2['dist'], T2['heights']) # 'dist' is length from start position + # T2['heights_adjusted'] = poly_correct(T2['dist'], T2['heights']) # 'dist' is length from start position # rolling 100 photon window. May have to be replaced by a window of fixed length. # if rollin window is replaced by fixed lengrh the following step is obsolute. - #T2['heights_smth'] = T2['heights_adjusted'].rolling(100).median() - B2[key] = T2 - dist_list = np.vstack([ dist_list, [T2['dist'].min(), T2['dist'].max()] ]) + # T2['heights_smth'] = T2['heights_adjusted'].rolling(100).median() + B2[key] = T2 + dist_list = np.vstack([dist_list, [T2["dist"].min(), T2["dist"].max()]]) ##### 2.) regridding and averaging # %% define functions -def get_mode(y, bins = np.arange(-5,5, 0.1)): + +def get_mode(y, bins=np.arange(-5, 5, 0.1)): "returns modes of histogram of y defined by bins" - hist, xbin = np.histogram(y, bins = bins ) + hist, xbin = np.histogram(y, bins=bins) return xbin[hist.argmax()] + def weighted_mean(x_rel, y): "returns the gaussian weighted mean for stencil" def weight_fnk(x): "returns gaussian weight given the distance to the center x" - return np.exp(- (x/.5)**2 ) + return np.exp(-((x / 0.5) ** 2)) w = weight_fnk(x_rel) - return (w*y).sum()/w.sum() + return (w * y).sum() / w.sum() # this function is applied to beam: -def get_stencil_stats(T2, stencil_iter, key , Nphoton_min = 5): - +def get_stencil_stats(T2, stencil_iter, key, Nphoton_min=5): """ T2 pd.DAtaframe with beam data needs at least 'dist' and key stencil_iter iterable that constains the stancil boundaries and center [left boundary, center, right boundary] @@ -293,46 +324,46 @@ def get_stencil_stats(T2, stencil_iter, key , Nphoton_min = 5): """ - x_data = T2['dist'] + x_data = T2["dist"] y_data = T2[key] + # apply this funcion to each stancil def calc_stencil_stats(istencil): "returns stats per stencil" - i_mask=(x_data >= istencil[0]) & (x_data < istencil[2]) + i_mask = (x_data >= istencil[0]) & (x_data < istencil[2]) Nphoton = i_mask.sum() if Nphoton < Nphoton_min: Tmedian = T2[i_mask].median() - Tmedian[key+ '_weighted_mean'] = np.nan - Tmedian[key+ '_mode'] = np.nan - Tmedian['N_photos'] = i_mask.sum() - Tmedian[key+ '_std'] = np.nan + Tmedian[key + "_weighted_mean"] = np.nan + Tmedian[key + "_mode"] = np.nan + Tmedian["N_photos"] = i_mask.sum() + Tmedian[key + "_std"] = np.nan return istencil[1], Tmedian - Tmedian = T2[i_mask].median() - x_rel = (x_data[i_mask] - istencil[1])/(L/2) - y = y_data[i_mask] + x_rel = (x_data[i_mask] - istencil[1]) / (L / 2) + y = y_data[i_mask] - Tmedian[key+ '_weighted_mean'] = weighted_mean(x_rel, y) - Tmedian[key+ '_mode'] = get_mode(y) - Tmedian['N_photos'] = i_mask.sum() - Tmedian[key+ '_std'] = y.std() + Tmedian[key + "_weighted_mean"] = weighted_mean(x_rel, y) + Tmedian[key + "_mode"] = get_mode(y) + Tmedian["N_photos"] = i_mask.sum() + Tmedian[key + "_std"] = y.std() return istencil[1], Tmedian # apply func to all stancils - D_filt = dict(map(calc_stencil_stats,stencil_iter)) + D_filt = dict(map(calc_stencil_stats, stencil_iter)) - DF_filt = pd.DataFrame.from_dict(D_filt, orient='index') - DF_filt = DF_filt.rename(columns={key: key+'_median', 'dist': 'median_dist'}) - DF_filt['dist'] = DF_filt.index + DF_filt = pd.DataFrame.from_dict(D_filt, orient="index") + DF_filt = DF_filt.rename(columns={key: key + "_median", "dist": "median_dist"}) + DF_filt["dist"] = DF_filt.index DF_filt = DF_filt.reset_index() return DF_filt @@ -340,103 +371,140 @@ def calc_stencil_stats(istencil): # %% old version # define common dist_grid: -#dx= 5 # 2 * resolution in meters, datapoint +-dx are used to take the mean -#dist_grid = np.arange( np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0), dx ) +# dx= 5 # 2 * resolution in meters, datapoint +-dx are used to take the mean +# dist_grid = np.arange( np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0), dx ) + # derive bin means def bin_means(T2, dist_grid): - dF_mean = pd.DataFrame(index =T2.columns) - ilim = int(len(dist_grid)) - N_i = list() + dF_mean = pd.DataFrame(index=T2.columns) + ilim = int(len(dist_grid)) + N_i = list() - for i in np.arange(1,ilim-1, 1): - if i % 5000 ==0: + for i in np.arange(1, ilim - 1, 1): + if i % 5000 == 0: print(i) - i_mask=(T2['dist'] >= dist_grid[i-1]) & (T2['dist'] < dist_grid[i+1]) - #if ( (T2['dist'] >= dist_grid[i-1]) & (T2['dist'] < dist_grid[i+1]) ).sum() > 0: + i_mask = (T2["dist"] >= dist_grid[i - 1]) & (T2["dist"] < dist_grid[i + 1]) + # if ( (T2['dist'] >= dist_grid[i-1]) & (T2['dist'] < dist_grid[i+1]) ).sum() > 0: dF_mean[i] = T2[i_mask].mean() - #dF_median[i] = T2[i_mask].median() + # dF_median[i] = T2[i_mask].median() N_i.append(i_mask.sum()) - dF_mean = dF_mean.T - dF_mean['N_photos'] = N_i - dF_mean['dist'] = dist_grid[np.arange(1,ilim-1, 1)] + dF_mean = dF_mean.T + dF_mean["N_photos"] = N_i + dF_mean["dist"] = dist_grid[np.arange(1, ilim - 1, 1)] return dF_mean + # %% estimating wave length for given period T = 5 -g= 9.81 -lam = g *T**2 / (2 * np.pi) +g = 9.81 +lam = g * T**2 / (2 * np.pi) print(lam) # define parameters: -L = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 -Nphoton_min = 5 # mininum need photons per stancil to return results +L = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 +Nphoton_min = 5 # mininum need photons per stancil to return results -G=dict() +G = dict() B3 = dict() -for key,Ti in B2.items(): +for key, Ti in B2.items(): print(key) - stencil_iter = create_chunk_boundaries_unit_lengths( L, [ np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0) ], iter_flag=True ) + stencil_iter = create_chunk_boundaries_unit_lengths( + L, + [np.nanmin(dist_list[:, 0], 0), np.nanmax(dist_list[:, 1], 0)], + iter_flag=True, + ) - Ti2 = get_stencil_stats(Ti, stencil_iter, 'heights_c', Nphoton_min=Nphoton_min) - Ti2['heights_c_median'][ np.isnan(Ti2['heights_c_std']) ]= np.nan # replace median calculation with nans + Ti2 = get_stencil_stats(Ti, stencil_iter, "heights_c", Nphoton_min=Nphoton_min) + Ti2["heights_c_median"][ + np.isnan(Ti2["heights_c_std"]) + ] = np.nan # replace median calculation with nans - B3[key] =Ti2 # store in dict + B3[key] = Ti2 # store in dict # % save relevant data as xarray # G[key] =xr.DataArray(Ti2['heights_c_median'], coords={'dist': Ti2['dist']}, dims='dist', name=key) - print(key, 'done') + print(key, "done") # %% saving data # Gnew = xr.merge(G.values()) # Gnew.to_netcdf(load_path+'/'+track_name +'_filtered_photon_heights.nc') -io.save_pandas_table(B2, track_name + '_B01_corrected' , load_path) # all photos but heights adjusted and with distance coordinate -io.save_pandas_table(B3, track_name + '_B01_binned' , load_path) # regridding heights +io.save_pandas_table( + B2, track_name + "_B01_corrected", load_path +) # all photos but heights adjusted and with distance coordinate +io.save_pandas_table(B3, track_name + "_B01_binned", load_path) # regridding heights # %% plotting just for checking plot_flag = True if plot_flag: import m_tools_ph3 as MT - plot_path = base_path+'plots/B01_regridding/'+track_name+'/' - MT.mkdirs_r(plot_path) + plot_path = base_path + "plots/B01_regridding/" + track_name + "/" + MT.mkdirs_r(plot_path) Ti2 = B3[key] - T2 = B2[key] + T2 = B2[key] dl = 4000 - latlims = (Ti2['dist'].iloc[0] , Ti2['dist'].iloc[-1] ) - np.arange(latlims[0],latlims[1],dl )[0:80] - for ll in sample( list(np.arange(latlims[0],latlims[1],dl )[0:80]) ,10): + latlims = (Ti2["dist"].iloc[0], Ti2["dist"].iloc[-1]) + np.arange(latlims[0], latlims[1], dl)[0:80] + for ll in sample(list(np.arange(latlims[0], latlims[1], dl)[0:80]), 10): F = M.figure_axis_xy(7, 3, view_scale=0.8) - plt.plot( T2['dist'], T2['heights_c'], 'k.', markersize= 0.5, alpha =0.8 ) - #plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') - - plt.plot(Ti2['dist'], Ti2['heights_c_weighted_mean'] +1, '.-', color='darkblue', linewidth=0.5, markersize=2,alpha=0.9, label='x-gauss weighted mean +1') - plt.plot(Ti2['dist'], Ti2['heights_c_median'], 'r.-', linewidth=0.5, markersize=2,alpha=0.9, label='median') - - plt.plot(Ti2['dist'], Ti2['heights_c_mode']-1, 'g.-', linewidth=0.5, markersize=2,alpha=0.9, label='mode - 1') - - plt.plot(Ti2['dist'], Ti2['heights_c_std'] - 1.8, 'k-', linewidth=0.5,alpha=1) - plt.fill_between( Ti2['dist'], Ti2['heights_c_std'] -1.8 , y2=-1.8, color='gray',alpha=1) + plt.plot(T2["dist"], T2["heights_c"], "k.", markersize=0.5, alpha=0.8) + # plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') + + plt.plot( + Ti2["dist"], + Ti2["heights_c_weighted_mean"] + 1, + ".-", + color="darkblue", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="x-gauss weighted mean +1", + ) + plt.plot( + Ti2["dist"], + Ti2["heights_c_median"], + "r.-", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="median", + ) + + plt.plot( + Ti2["dist"], + Ti2["heights_c_mode"] - 1, + "g.-", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="mode - 1", + ) + + plt.plot(Ti2["dist"], Ti2["heights_c_std"] - 1.8, "k-", linewidth=0.5, alpha=1) + plt.fill_between( + Ti2["dist"], Ti2["heights_c_std"] - 1.8, y2=-1.8, color="gray", alpha=1 + ) # plt.plot( ALT03['delta_time'], ALT03['heights_c'], 'k.', markersize= 0.3, alpha =0.2 ) # plt.plot(ALT07['time']['delta_time'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') plt.legend(loc=1) - plt.xlim(ll, ll+dl) + plt.xlim(ll, ll + dl) plt.ylim(-2, 3) - plt.xlabel('Meters from the Sea Ice Edge') - plt.ylabel('Height Anomalie (meters)') - F.ax.axhline(y =-1.8, color='black', linewidth=0.5) - F.save_light(path= plot_path, name='ALT03_filt_compare'+ str(ll)) + plt.xlabel("Meters from the Sea Ice Edge") + plt.ylabel("Height Anomalie (meters)") + F.ax.axhline(y=-1.8, color="black", linewidth=0.5) + F.save_light(path=plot_path, name="ALT03_filt_compare" + str(ll)) # %% plot diff --git a/analysis/B01_filter_regrid_ALT07_test.py b/analysis/B01_filter_regrid_ALT07_test.py index 04d122ca..ba17494b 100644 --- a/analysis/B01_filter_regrid_ALT07_test.py +++ b/analysis/B01_filter_regrid_ALT07_test.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,15 +9,16 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -33,47 +35,48 @@ import ICEsat2_SI_tools.io as io import imp -#import s3fs + +# import s3fs # %% # Python reader based on Pandas. Other reader examples available in readers.py -#track_name= 'ATL07-01_20190909191519_11260401_004_01' -track_name = 'processed_ATL07-02_20190515045104_07170301_002_01' -load_path = base_path + 'data/data1/' -load_file = load_path +track_name+'.h5' +# track_name= 'ATL07-01_20190909191519_11260401_004_01' +track_name = "processed_ATL07-02_20190515045104_07170301_002_01" +load_path = base_path + "data/data1/" +load_file = load_path + track_name + ".h5" # % # test which beams exist: -all_beams = ['gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r'] -low_beams = ['gt1l', 'gt2l', 'gt3l'] -high_beams = ['gt1r', 'gt2r', 'gt3r'] +all_beams = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] +low_beams = ["gt1l", "gt2l", "gt3l"] +high_beams = ["gt1r", "gt2r", "gt3r"] -f = h5py.File(load_file, 'r') -beams = [b if b in f.keys() else None for b in all_beams] +f = h5py.File(load_file, "r") +beams = [b if b in f.keys() else None for b in all_beams] imp.reload(io) -ATL03 = h5py.File(load_file, 'r') +ATL03 = h5py.File(load_file, "r") -fileT=load_file -beam= high_beams[0] +fileT = load_file +beam = high_beams[0] imp.reload(io) ALT07 = io.getATL07_beam(fileT) # %% -#processed_ATL03_20190515060436_07170312_002_02 -track_name = 'processed_ATL03_20190515060436_07170312_002_02' -load_path = base_path + 'data/data1/' -load_file = load_path +track_name+'.h5' -ATL03 = h5py.File(load_file, 'r') +# processed_ATL03_20190515060436_07170312_002_02 +track_name = "processed_ATL03_20190515060436_07170312_002_02" +load_path = base_path + "data/data1/" +load_file = load_path + track_name + ".h5" +ATL03 = h5py.File(load_file, "r") ALT03 = io.getATL03_beam(load_file) -#ALT03 = ALT03[(ALT03['signal_confidence']>1) & (ALT03['heights']<100)] -ALT03 = ALT03[(ALT03['heights']<100) & (ALT03['heights'] > -100)] +# ALT03 = ALT03[(ALT03['signal_confidence']>1) & (ALT03['heights']<100)] +ALT03 = ALT03[(ALT03["heights"] < 100) & (ALT03["heights"] > -100)] # %% # latlims = (ALT07['ref']['latitude'].iloc[0] , ALT07['ref']['latitude'].iloc[-1] ) @@ -96,39 +99,43 @@ # %% D = ALT07 -plt.plot( D['ref']['seg_dist_x'], D['ref']['latitude'] ) -D['heights']['height_segment_ssh_flag'].plot() +plt.plot(D["ref"]["seg_dist_x"], D["ref"]["latitude"]) +D["heights"]["height_segment_ssh_flag"].plot() # %% -dist= D['ref']['seg_dist_x']- D['ref']['seg_dist_x'][0] -segl= D['heights']['height_segment_length_seg']/10 +dist = D["ref"]["seg_dist_x"] - D["ref"]["seg_dist_x"][0] +segl = D["heights"]["height_segment_length_seg"] / 10 lowest_possible_T = 2 * 2 * np.pi * np.sqrt(segl.median() / 9.81) + + # %% get surface height corretions def get_time_for_track(delta_time, atlas_epoch): "returns pandas dataframe" import pandas as pd import convert_GPS_time as cGPS + # Conversion of delta_time to a calendar date temp = cGPS.convert_GPS_time(atlas_epoch[0] + delta_time, OFFSET=0.0) - year = temp['year'][:].astype('int') - month = temp['month'][:].astype('int') - day = temp['day'][:].astype('int') - hour = temp['hour'][:].astype('int') - minute = temp['minute'][:].astype('int') - second = temp['second'][:].astype('int') - - return pd.DataFrame({'year':year, 'month':month, 'day':day, 'hour':hour, 'second':second}) + year = temp["year"][:].astype("int") + month = temp["month"][:].astype("int") + day = temp["day"][:].astype("int") + hour = temp["hour"][:].astype("int") + minute = temp["minute"][:].astype("int") + second = temp["second"][:].astype("int") + return pd.DataFrame( + {"year": year, "month": month, "day": day, "hour": hour, "second": second} + ) ALT07_cor = getATL07_height_corrections(fileT) ALT07_cor.shape podppd_flag.size -ALT07_cor['corrections'].keys() +ALT07_cor["corrections"].keys() # follow this correction: # Prior to surface finding, do the following to the photon heights: # • Remove data when padpodflag = 1. # not applied @@ -145,63 +152,76 @@ def get_time_for_track(delta_time, atlas_epoch): # h = h_ph - hMSS - hocean_tide - hlpe_tide - hIB # (sign convention is consistent with that used in ATL03) -h_correction =ALT07_cor['corrections']['height_segment_mss'] + ALT07_cor['corrections']['height_segment_ocean'] + ALT07_cor['corrections']['height_segment_lpe'] + ALT07_cor['corrections']['height_segment_dac'] +h_correction = ( + ALT07_cor["corrections"]["height_segment_mss"] + + ALT07_cor["corrections"]["height_segment_ocean"] + + ALT07_cor["corrections"]["height_segment_lpe"] + + ALT07_cor["corrections"]["height_segment_dac"] +) # 'height_segment_dac' can be replaced by 'height_segment_ib', not sure .. -ALT07_cor['h_correction'] = h_correction +ALT07_cor["h_correction"] = h_correction # %% -track_name = 'processed_ATL07-02_20190515045104_07170301_002_01' -load_path = base_path + 'data/data1/' -load_file = load_path +track_name+'.h5' +track_name = "processed_ATL07-02_20190515045104_07170301_002_01" +load_path = base_path + "data/data1/" +load_file = load_path + track_name + ".h5" imp.reload(io) ALT07 = io.getATL07_beam(load_file) -track_name = 'processed_ATL03_20190515060436_07170312_002_02' -load_path = base_path + 'data/data1/' -load_file = load_path +track_name+'.h5' -#ATL03 = h5py.File(load_file, 'r') +track_name = "processed_ATL03_20190515060436_07170312_002_02" +load_path = base_path + "data/data1/" +load_file = load_path + track_name + ".h5" +# ATL03 = h5py.File(load_file, 'r') ALT03 = io.getATL03_beam(load_file) # %% ALT07 -ALT03 = ALT03[ (ALT03['signal_confidence']>1) & (ALT03['heights']<100) & (ALT03['heights'] > -100)] -ALT03['heights'].plot() +ALT03 = ALT03[ + (ALT03["signal_confidence"] > 1) + & (ALT03["heights"] < 100) + & (ALT03["heights"] > -100) +] +ALT03["heights"].plot() -ALT03c = io.getATL03_height_correction(load_file) -ALT03c['dem_h'].plot() +ALT03c = io.getATL03_height_correction(load_file) +ALT03c["dem_h"].plot() # cut data -ALT03c = ALT03c[ALT03c['dem_h'] < 1e5] # cute out weird references -ALT03c['dem_h'].plot() +ALT03c = ALT03c[ALT03c["dem_h"] < 1e5] # cute out weird references +ALT03c["dem_h"].plot() # subtract h_ref -mask= (ALT03c['delta_time']>= ALT03['delta_time'].min() ) & (ALT03c['delta_time'] <= ALT03['delta_time'].max()) +mask = (ALT03c["delta_time"] >= ALT03["delta_time"].min()) & ( + ALT03c["delta_time"] <= ALT03["delta_time"].max() +) ALT03c = ALT03c[mask] -ALT03['heights_c']= ALT03['heights'] - np.interp( ALT03['delta_time'],ALT03c['delta_time'], ALT03c['dem_h'] ) +ALT03["heights_c"] = ALT03["heights"] - np.interp( + ALT03["delta_time"], ALT03c["delta_time"], ALT03c["dem_h"] +) # %% -plt.plot( ALT03c['delta_time'], ALT03c['dem_h'], zorder= 12) -plt.plot( ALT03['delta_time'], ALT03['heights'], 'k.', alpha =0.2 ) +plt.plot(ALT03c["delta_time"], ALT03c["dem_h"], zorder=12) +plt.plot(ALT03["delta_time"], ALT03["heights"], "k.", alpha=0.2) # %% test smooting -np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0) +np.nanmin(dist_list[:, 0], 0), np.nanmax(dist_list[:, 1], 0) # %% -latlims = (ALT07['ref']['latitude'].iloc[0] , ALT07['ref']['latitude'].iloc[-1] ) -#latlims = (ALT07['time']['delta_time'].iloc[0] , ALT07['time']['delta_time'].iloc[-1] ) -#latlims[1] -latlims[0] +latlims = (ALT07["ref"]["latitude"].iloc[0], ALT07["ref"]["latitude"].iloc[-1]) +# latlims = (ALT07['time']['delta_time'].iloc[0] , ALT07['time']['delta_time'].iloc[-1] ) +# latlims[1] -latlims[0] dl = 1 dl = 0.07 -for ll in np.arange(latlims[0],latlims[1],dl ): +for ll in np.arange(latlims[0], latlims[1], dl): F = M.figure_axis_xy(7, 2, view_scale=0.8) # ALT03i = ALT03[ALT03['signal_confidence']==4 ] @@ -210,13 +230,23 @@ def get_time_for_track(delta_time, atlas_epoch): # ALT03i = ALT03[(ALT03['signal_confidence']>1) & (ALT03['signal_confidence']<4)] # plt.plot(ALT03i['lats'] ,ALT03i['heights'] , 'g.', markersize=2, alpha = 1, label ='ALT03 sig.level 2-3') - plt.plot( ALT03['lats'], ALT03['heights_c'], 'k.', markersize= 0.3, alpha =0.2 ) - plt.plot(ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') + plt.plot(ALT03["lats"], ALT03["heights_c"], "k.", markersize=0.3, alpha=0.2) + plt.plot( + ALT07["ref"]["latitude"], + ALT07["heights"]["height_segment_height"], + "r.", + markersize=0.8, + alpha=1, + label="ALT07 seg. heights", + ) # plt.plot( ALT03['delta_time'], ALT03['heights_c'], 'k.', markersize= 0.3, alpha =0.2 ) # plt.plot(ALT07['time']['delta_time'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') - plt.xlim(ll, ll+dl) + plt.xlim(ll, ll + dl) plt.ylim(-2, 2) - #plt.legend() - F.save_light(path= base_path+'plots/tests/', name='ALT03_ALT07_comparison_corrected'+ str(ll)) + # plt.legend() + F.save_light( + path=base_path + "plots/tests/", + name="ALT03_ALT07_comparison_corrected" + str(ll), + ) diff --git a/analysis/B01_filter_regrid_curve_lin.py b/analysis/B01_filter_regrid_curve_lin.py index f4a08400..e442d3d7 100644 --- a/analysis/B01_filter_regrid_curve_lin.py +++ b/analysis/B01_filter_regrid_curve_lin.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,7 +9,7 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) import datetime @@ -17,7 +18,10 @@ import imp import ICEsat2_SI_tools.convert_GPS_time as cGPS import ICEsat2_SI_tools.io as io -from spectral_estimates import create_chunk_boundaries_unit_lengths, create_chunk_boundaries +from spectral_estimates import ( + create_chunk_boundaries_unit_lengths, + create_chunk_boundaries, +) import spectral_estimates as spec import m_tools_ph3 as MT import filter_regrid as regrid @@ -25,180 +29,203 @@ import concurrent.futures as futures - -#import s3fs -#processed_ATL03_20190605061807_10380310_004_01.h5 - -#imp.reload(io) -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207234532_06340210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190215184558_07530210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190208104534_06410210_004_01', 'SH_batch02', False - - - - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -load_path = mconfig['paths']['scratch'] +'/'+ batch_key +'/' -load_file = load_path + 'processed_'+ATlevel+'_'+track_name+'.h5' - -save_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' - -plot_path = mconfig['paths']['plot']+ '/'+hemis+'/'+batch_key+'/'+track_name +'/B_ov/' -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +# import s3fs +# processed_ATL03_20190605061807_10380310_004_01.h5 + +# imp.reload(io) +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207234532_06340210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190215184558_07530210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190208104534_06410210_004_01', 'SH_batch02', False + + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +load_path = mconfig["paths"]["scratch"] + "/" + batch_key + "/" +load_file = load_path + "processed_" + ATlevel + "_" + track_name + ".h5" + +save_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/B_ov/" +) +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" MT.mkdirs_r(save_path) # set pars # define parameters: -Lmeter = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 -Nphoton_min = 5 # mininum need photons per stancil to return results +Lmeter = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 +Nphoton_min = 5 # mininum need photons per stancil to return results -Lmeter_large= 100e3 # stancil width for testing photon density. stancils do not overlab. -minium_photon_density = 0.02 # minimum photon density per meter in Lmeter_large chunk to be counted as real signal +Lmeter_large = ( + 100e3 # stancil width for testing photon density. stancils do not overlab. +) +minium_photon_density = 0.02 # minimum photon density per meter in Lmeter_large chunk to be counted as real signal -plot_flag = True -Nworkers = 3 # number of threads for parallel processing -Nworkers_process =3 # number of threads for parallel processing +plot_flag = True +Nworkers = 3 # number of threads for parallel processing +Nworkers_process = 3 # number of threads for parallel processing # %% # test which beams exist: -all_beams = mconfig['beams']['all_beams'] +all_beams = mconfig["beams"]["all_beams"] # high_beams = mconfig['beams']['high_beams'] # low_beams = mconfig['beams']['low_beams'] -f = h5py.File(load_file, 'r') -beams = [b if b in f.keys() else None for b in all_beams] -accent = regrid.track_type_beam(f) +f = h5py.File(load_file, "r") +beams = [b if b in f.keys() else None for b in all_beams] +accent = regrid.track_type_beam(f) -print('accent is ' , accent) +print("accent is ", accent) # Load fata and apply height corrections # This needs version 2 of the ALT 03 dataset -#ATL03 = h5py.File(load_file, 'r') +# ATL03 = h5py.File(load_file, 'r') -#ATL03[k+'/heights/ph_id_channel'][0:100] +# ATL03[k+'/heights/ph_id_channel'][0:100] # %% -hist = 'Beam stats' -B = dict() -B1save = dict() -SEG = dict() +hist = "Beam stats" +B = dict() +B1save = dict() +SEG = dict() for k in beams: - #k = beams[0] + # k = beams[0] imp.reload(io) print(k) - T, seg = io.getATL03_beam(load_file, beam= k) - + T, seg = io.getATL03_beam(load_file, beam=k) - - print('loaded') - T = T[T['mask_seaice']] # only take sea ice points, no ocean points + print("loaded") + T = T[T["mask_seaice"]] # only take sea ice points, no ocean points ho = k - ho = MT.add_line_var(ho, 'size', str(T.shape[0])) - ho = MT.add_line_var(ho, 'by confidence levels:' + str(np.arange(0, 5)), [ (T['signal_confidence'] == i).sum() for i in np.arange(0, 5) ]) + ho = MT.add_line_var(ho, "size", str(T.shape[0])) + ho = MT.add_line_var( + ho, + "by confidence levels:" + str(np.arange(0, 5)), + [(T["signal_confidence"] == i).sum() for i in np.arange(0, 5)], + ) # filter: - Tsel = T[ (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] + Tsel = T[ + (T["heights"] < 100) & (T["heights"] > -100) + ] # & (T['delta_time']>5) & (T['delta_time']<24) ] # if len(Tsel) == 0: # ho = MT.add_line_var(ho, 'no photons found', '') # #Tsel= T[(T['signal_confidence'] ==-1 ) & (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] - Tsel_c = io.getATL03_height_correction(load_file) - Tsel_c = Tsel_c[Tsel_c['dem_h'] < 1e5] # cute out weird references - Tsel2 = regrid.correct_heights(Tsel, Tsel_c).reset_index(drop=True)# correct height - print('height corrected') + Tsel_c = io.getATL03_height_correction(load_file) + Tsel_c = Tsel_c[Tsel_c["dem_h"] < 1e5] # cute out weird references + Tsel2 = regrid.correct_heights(Tsel, Tsel_c).reset_index( + drop=True + ) # correct height + print("height corrected") ### cut data at the rear that has too much variance # cut last segments of data until variance is similar rear_mask = np.array(Tsel2.index) > -1 shape_old = Tsel2.shape - N_seg= 20 + N_seg = 20 cut_flag = True while cut_flag: - dd= Tsel2['heights_c'][rear_mask] + dd = Tsel2["heights_c"][rear_mask] nsize = dd.size - stencil_iter = create_chunk_boundaries( int(nsize/N_seg), nsize,ov =0, iter_flag=True ) + stencil_iter = create_chunk_boundaries( + int(nsize / N_seg), nsize, ov=0, iter_flag=True + ) def get_var(sti): - return dd[sti[0]: sti[1]].var() + return dd[sti[0] : sti[1]].var() var_list = np.array(list(map(get_var, stencil_iter))) - if var_list[0:3].mean()*10 < var_list[-1]: - #print('cut last '+ str(100/N_seg) +'% of data') - rear_mask[int(nsize* (N_seg-1) / N_seg):] = False + if var_list[0:3].mean() * 10 < var_list[-1]: + # print('cut last '+ str(100/N_seg) +'% of data') + rear_mask[int(nsize * (N_seg - 1) / N_seg) :] = False else: - cut_flag = False - - - Tsel2['process_mask'] = rear_mask - B1save[k] = Tsel2 - B[k] = Tsel2[rear_mask].drop(columns='process_mask') - SEG[k] = seg - - ho = MT.add_line_var(ho, 'cutted ', str( np.round( 100 *(rear_mask.size -rear_mask.sum() ) / rear_mask.size, 0)) + '% in the back of the track' ) - ho = MT.add_line_var(ho, 'selected size', str(Tsel.shape[0])) - ho = MT.add_line_var(ho, 'final size ', str(Tsel_c.shape[0])) + cut_flag = False + + Tsel2["process_mask"] = rear_mask + B1save[k] = Tsel2 + B[k] = Tsel2[rear_mask].drop(columns="process_mask") + SEG[k] = seg + + ho = MT.add_line_var( + ho, + "cutted ", + str(np.round(100 * (rear_mask.size - rear_mask.sum()) / rear_mask.size, 0)) + + "% in the back of the track", + ) + ho = MT.add_line_var(ho, "selected size", str(Tsel.shape[0])) + ho = MT.add_line_var(ho, "final size ", str(Tsel_c.shape[0])) print(ho) - hist = MT.write_log(hist, ho) + hist = MT.write_log(hist, ho) -print('done with 1st loop') +print("done with 1st loop") # %% define x- coodindate # find most equator ward segment length -total_segment_dist_x_min= list() -for k,I in SEG.items(): - total_segment_dist_x_min.append( I['segment_dist_x'].min() ) +total_segment_dist_x_min = list() +for k, I in SEG.items(): + total_segment_dist_x_min.append(I["segment_dist_x"].min()) total_segment_dist_x_min = min(total_segment_dist_x_min) # T = B['gt1r'] # seg = SEG['gt1r'] -def make_x_coorindate(T, seg): +def make_x_coorindate(T, seg): """ Returns the "true" along track coordindate but finding the correpsonding segment length also adds the segment_ID to the main table T """ - time_ticks = np.insert(np.diff(T['along_track_distance']), 0,0 ) - T['seg_ID_local'] = np.digitize(T['delta_time'], T[ time_ticks < - 10 ]['delta_time']) - Tg= T.groupby(by= 'seg_ID_local', axis=0, group_keys=seg['delta_time'] ) + time_ticks = np.insert(np.diff(T["along_track_distance"]), 0, 0) + T["seg_ID_local"] = np.digitize(T["delta_time"], T[time_ticks < -10]["delta_time"]) + Tg = T.groupby(by="seg_ID_local", axis=0, group_keys=seg["delta_time"]) def add_seg_length(Ti, seg): try: - seg_data = seg[seg['delta_time'] < Ti['delta_time'].iloc[0]].iloc[-1] - seg_l = seg_data['segment_dist_x'] - total_segment_dist_x_min - seg_ID = seg_data['segment_id'] + seg_data = seg[seg["delta_time"] < Ti["delta_time"].iloc[0]].iloc[-1] + seg_l = seg_data["segment_dist_x"] - total_segment_dist_x_min + seg_ID = seg_data["segment_id"] except: seg_l = np.nan - seg_ID = np.nan + seg_ID = np.nan - Ti['x'] = seg_l.astype('float64') + Ti['along_track_distance'].astype('float64') - Ti['seg_ID'] = seg_ID + Ti["x"] = seg_l.astype("float64") + Ti["along_track_distance"].astype("float64") + Ti["seg_ID"] = seg_ID return Ti return Tg.apply(add_seg_length, seg) + for k in B.keys(): print(k) B[k] = make_x_coorindate(B[k], SEG[k]) -dist_list = np.array([np.nan, np.nan]) +dist_list = np.array([np.nan, np.nan]) for k in B.keys(): - dist_list = np.vstack([ dist_list, [B[k]['x'].iloc[0] , B[k]['x'].iloc[-1] ] ]) + dist_list = np.vstack([dist_list, [B[k]["x"].iloc[0], B[k]["x"].iloc[-1]]]) del A @@ -272,27 +299,41 @@ def add_seg_length(Ti, seg): # %% test ave photon density -track_dist_bounds = [ np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0) ] -length_meter = track_dist_bounds[1] - track_dist_bounds[0] -#length_meter = (abs(lat_lims[1]) - abs(lat_lims[0])) * 110e3 -p_densities_r = list() -p_densities_l = list() +track_dist_bounds = [np.nanmin(dist_list[:, 0], 0), np.nanmax(dist_list[:, 1], 0)] +length_meter = track_dist_bounds[1] - track_dist_bounds[0] +# length_meter = (abs(lat_lims[1]) - abs(lat_lims[0])) * 110e3 +p_densities_r = list() +p_densities_l = list() for k, I in B.items(): - if 'r' in k: - p_densities_r.append( I.shape[0] /length_meter) + if "r" in k: + p_densities_r.append(I.shape[0] / length_meter) else: - p_densities_l.append( I.shape[0] /length_meter) - -if (np.array(p_densities_l).mean() < 0.5) & (np.array(p_densities_r).mean() < 0.5): # in photons per meter - print('photon density too low, this track is classified as bad track and pushed to bad list') - MT.json_save(track_name, bad_track_path, {'densities': [ np.array(p_densities_r).mean(), np.array(p_densities_l).mean()] , 'date': str(datetime.date.today()) }) - print('exit.') + p_densities_l.append(I.shape[0] / length_meter) + +if (np.array(p_densities_l).mean() < 0.5) & ( + np.array(p_densities_r).mean() < 0.5 +): # in photons per meter + print( + "photon density too low, this track is classified as bad track and pushed to bad list" + ) + MT.json_save( + track_name, + bad_track_path, + { + "densities": [ + np.array(p_densities_r).mean(), + np.array(p_densities_l).mean(), + ], + "date": str(datetime.date.today()), + }, + ) + print("exit.") exit() # %% save corrected data and delete from cash -#io.save_pandas_table(B1save, track_name + '_B01_corrected' , save_path) # all photos but heights adjusted and with distance coordinate +# io.save_pandas_table(B1save, track_name + '_B01_corrected' , save_path) # all photos but heights adjusted and with distance coordinate del B1save # for testing @@ -310,179 +351,245 @@ def add_seg_length(Ti, seg): # %% ##### 1.) derive common axis for beams and filter out low density area at the beginning -print('filter out low density area at the beginning') +print("filter out low density area at the beginning") + def derive_axis_and_boundaries(key): - #key = 'gt3r' + # key = 'gt3r' print(key) - #T2 # = regrid.derive_axis(B[key], lat_lims).sort_values('dist') - T2 = B[key]#['x'] + # T2 # = regrid.derive_axis(B[key], lat_lims).sort_values('dist') + T2 = B[key] # ['x'] + + x0 = get_better_lower_boundary(Lmeter_large, np.array(T2["x"])) - x0 = get_better_lower_boundary(Lmeter_large, np.array(T2['x'])) + print( + "cut off ", + 100 * (1 - T2[T2["x"] > x0].shape[0] / T2.shape[0]), + "% off all data points at the beginning", + ) + T2 = T2[T2["x"] > x0] # cut off low density start - print( 'cut off ' , 100 * (1 - T2[T2['x'] > x0].shape[0]/T2.shape[0]) , '% off all data points at the beginning' ) - T2 = T2[T2['x'] >x0] # cut off low density start + return key, T2, [T2["x"].min(), T2["x"].max()] - return key, T2, [T2['x'].min(), T2['x'].max()] def get_better_lower_boundary(Lmeter_large, dd): # T2 = regrid.derive_axis(B[key], lat_lims) - #dd = np.array(T2['x']) - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= True) + # dd = np.array(T2['x']) + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=True + ) def get_density(sti): - return sti[0], sum( (sti[0] <= dd) & (dd < sti[-1]) ) / Lmeter_large + return sti[0], sum((sti[0] <= dd) & (dd < sti[-1])) / Lmeter_large - with futures.ThreadPoolExecutor(max_workers= Nworkers_process) as executor_sub: - #var_list = np.array(list(map(get_density, stencil_iter))) + with futures.ThreadPoolExecutor(max_workers=Nworkers_process) as executor_sub: + # var_list = np.array(list(map(get_density, stencil_iter))) var_list = np.array(list(executor_sub.map(get_density, stencil_iter))) var_list = np.array(var_list) - #var_list[:,0] = np.random.rand(10) - #sort var_list - var_list = var_list[var_list[:,0].argsort(), :] - #print(var_list) - if sum(var_list[:,1] > minium_photon_density) > 1: - first_stencil = next((i for i, j in enumerate(var_list[:,1] > minium_photon_density) if j), None) #- 1 - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) + # var_list[:,0] = np.random.rand(10) + # sort var_list + var_list = var_list[var_list[:, 0].argsort(), :] + # print(var_list) + if sum(var_list[:, 1] > minium_photon_density) > 1: + first_stencil = next( + (i for i, j in enumerate(var_list[:, 1] > minium_photon_density) if j), None + ) # - 1 + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=False + ) return stencil_iter[0, first_stencil] else: - #first_stencil = next((i for i, j in enumerate(var_list[:,1] > 0) if j), None)# - 1 - print('no sufficient photon density found. return short stencil') + # first_stencil = next((i for i, j in enumerate(var_list[:,1] > 0) if j), None)# - 1 + print("no sufficient photon density found. return short stencil") # first_stencil= len(var_list[:,1]) -1 # stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) # return stencil_iter[0, first_stencil] # #print(first_stencil) - return var_list[-1,0]#[-1,0] + return var_list[-1, 0] # [-1,0] with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: - A = list( executor.map(derive_axis_and_boundaries, all_beams) ) + A = list(executor.map(derive_axis_and_boundaries, all_beams)) # %% -B2 = dict() -dist_list = np.array([np.nan, np.nan]) +B2 = dict() +dist_list = np.array([np.nan, np.nan]) for I in A: - k = I[0] - B2[k] = I[1] - #B2[k]['dist'] = B2[k]['x'] - dist_list = np.vstack([dist_list,I[2] ]) + k = I[0] + B2[k] = I[1] + # B2[k]['dist'] = B2[k]['x'] + dist_list = np.vstack([dist_list, I[2]]) del A -track_dist_bounds = [ np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0) ] +track_dist_bounds = [np.nanmin(dist_list[:, 0], 0), np.nanmax(dist_list[:, 1], 0)] # for testing: -#ts_s = np.copy(track_dist_bounds) +# ts_s = np.copy(track_dist_bounds) # %% -F= M.figure_axis_xy(5, 3, view_scale= 0.5) -for k,I in B2.items(): - plt.plot( I['x'] , I['across_track_distance'] , '.' , markersize = 0.3) - #plt.xlim(3e6, 3.25e6) - -F.ax.axvline(track_dist_bounds[0], color='k') -F.ax.axvline(track_dist_bounds[1], color='k') -plt.title('A01 filter & regrid | ' + track_name) -F.save_light(path= plot_path, name='ALT03_track_all') +F = M.figure_axis_xy(5, 3, view_scale=0.5) +for k, I in B2.items(): + plt.plot(I["x"], I["across_track_distance"], ".", markersize=0.3) + # plt.xlim(3e6, 3.25e6) + +F.ax.axvline(track_dist_bounds[0], color="k") +F.ax.axvline(track_dist_bounds[1], color="k") +plt.title("A01 filter & regrid | " + track_name) +F.save_light(path=plot_path, name="ALT03_track_all") # %% # for testing -#track_dist_bounds[1] = track_dist_bounds[0] + (track_dist_bounds[1] - track_dist_bounds[0])/20 -#track_dist_bounds = ts_s[0] ,ts_s[0] + (ts_s[1] - ts_s[0]) /6 +# track_dist_bounds[1] = track_dist_bounds[0] + (track_dist_bounds[1] - track_dist_bounds[0])/20 +# track_dist_bounds = ts_s[0] ,ts_s[0] + (ts_s[1] - ts_s[0]) /6 # %% -#imp.reload(regrid) +# imp.reload(regrid) ##### 2.) regridding and averaging -print('regrid') +print("regrid") + + def regridding_wrapper(I): key, Ti = I - print(key, Ti.shape,2* Ti.shape[0]/Lmeter) - stencil_iter = create_chunk_boundaries_unit_lengths( Lmeter, track_dist_bounds, iter_flag=True ) - with futures.ThreadPoolExecutor(max_workers= Nworkers) as executor_sub: - Bi = regrid.get_stencil_stats( Ti, stencil_iter, 'heights_c', 'x' , stancil_width= Lmeter/2, Nphoton_min=Nphoton_min, map_func = executor_sub.map) - #B3[key] = - print(key, 'done') + print(key, Ti.shape, 2 * Ti.shape[0] / Lmeter) + stencil_iter = create_chunk_boundaries_unit_lengths( + Lmeter, track_dist_bounds, iter_flag=True + ) + with futures.ThreadPoolExecutor(max_workers=Nworkers) as executor_sub: + Bi = regrid.get_stencil_stats( + Ti, + stencil_iter, + "heights_c", + "x", + stancil_width=Lmeter / 2, + Nphoton_min=Nphoton_min, + map_func=executor_sub.map, + ) + # B3[key] = + print(key, "done") return key, Bi with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: - B3 = dict( executor.map(regridding_wrapper, B2.items() ) ) + B3 = dict(executor.map(regridding_wrapper, B2.items())) # %% ---------------------------- define start and end position and same in Json file -I = B3['gt2l'].copy() +I = B3["gt2l"].copy() D_info = dict() -for k,I in B3.items(): +for k, I in B3.items(): # reset x coordinate - I['median_dist'] = I['median_x'] - track_dist_bounds[0] #- Lmeter/2 - I['dist'] = I['x'] - track_dist_bounds[0] #- Lmeter/2 - #I['index'] = I['x'] + I["median_dist"] = I["median_x"] - track_dist_bounds[0] # - Lmeter/2 + I["dist"] = I["x"] - track_dist_bounds[0] # - Lmeter/2 + # I['index'] = I['x'] # rename y cooridinate - I = I.rename(columns={'across_track_distance': 'y'}) + I = I.rename(columns={"across_track_distance": "y"}) # find starting and end position - Di_s = dict(I[I['seg_ID'] == I['seg_ID'].iloc[0] ].mean()[['lons', 'lats', 'seg_ID', 'delta_time']]) - Di_s['across_track_distance_0'] =track_dist_bounds[0] - - Di_e = dict(I[I['seg_ID'] == I['seg_ID'].iloc[-1] ].mean()[['lons', 'lats', 'seg_ID', 'delta_time']]) - Di_e['across_track_distance_0'] =track_dist_bounds[0] - - D_info[k] = {'start':Di_s, 'end':Di_e } + Di_s = dict( + I[I["seg_ID"] == I["seg_ID"].iloc[0]].mean()[ + ["lons", "lats", "seg_ID", "delta_time"] + ] + ) + Di_s["across_track_distance_0"] = track_dist_bounds[0] + + Di_e = dict( + I[I["seg_ID"] == I["seg_ID"].iloc[-1]].mean()[ + ["lons", "lats", "seg_ID", "delta_time"] + ] + ) + Di_e["across_track_distance_0"] = track_dist_bounds[0] + + D_info[k] = {"start": Di_s, "end": Di_e} # rerder indexes - column_names = ['index', 'x', 'y', 'median_x', 'lons', 'lats' ,'heights_c_weighted_mean', 'heights_c_median', 'heights_c_std', 'N_photos', ] - vars_ad = set(list(I[I['seg_ID'] == I['seg_ID'].iloc[0] ].mean().index)) - set(column_names) - I = I.reindex(columns=column_names + list(vars_ad)) + column_names = [ + "index", + "x", + "y", + "median_x", + "lons", + "lats", + "heights_c_weighted_mean", + "heights_c_median", + "heights_c_std", + "N_photos", + ] + vars_ad = set(list(I[I["seg_ID"] == I["seg_ID"].iloc[0]].mean().index)) - set( + column_names + ) + I = I.reindex(columns=column_names + list(vars_ad)) B3[k] = I # save Json -MT.json_save(track_name + '_B01_stats',save_path, D_info, verbose= True ) +MT.json_save(track_name + "_B01_stats", save_path, D_info, verbose=True) # %% saving data -io.save_pandas_table(B2, track_name + '_B01_regridded' , save_path) # all photos but heights adjusted and with distance coordinate -io.save_pandas_table(B3, track_name + '_B01_binned' , save_path) # regridding heights +io.save_pandas_table( + B2, track_name + "_B01_regridded", save_path +) # all photos but heights adjusted and with distance coordinate +io.save_pandas_table(B3, track_name + "_B01_binned", save_path) # regridding heights # %% plotting just for checking -key = 'gt2r' +key = "gt2r" if plot_flag: MT.mkdirs_r(plot_path) Ti2 = B3[key] - T2 = B2[key] + T2 = B2[key] dl = 4000 - latlims = (Ti2['x'].iloc[0] , Ti2['x'].iloc[-1] ) - #chunk_list = np.arange(latlims[0],latlims[1], dl ) - #chunk_list = sample( list(np.arange(latlims[0],latlims[1],dl )[0:80]) ,10) - chunk_list = np.arange(latlims[0],latlims[1], dl )[::10] - chunk_list = np.append( chunk_list, latlims[1]- dl-1) + latlims = (Ti2["x"].iloc[0], Ti2["x"].iloc[-1]) + # chunk_list = np.arange(latlims[0],latlims[1], dl ) + # chunk_list = sample( list(np.arange(latlims[0],latlims[1],dl )[0:80]) ,10) + chunk_list = np.arange(latlims[0], latlims[1], dl)[::10] + chunk_list = np.append(chunk_list, latlims[1] - dl - 1) for ll in chunk_list: F = M.figure_axis_xy(7, 3, view_scale=0.8) - plt.plot( T2['x'], T2['heights_c'], 'k.', markersize= 0.5, alpha =0.8 ) - #plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') - - plt.plot(Ti2['x'], Ti2['heights_c_weighted_mean'] +1, '.-', color='darkblue', linewidth=0.5, markersize=2,alpha=0.9, label='x-gauss weighted mean +1') - plt.plot(Ti2['x'], Ti2['heights_c_median'], 'r.-', linewidth=0.5, markersize=2,alpha=0.9, label='median') - - #plt.plot(Ti2['x'], Ti2['heights_c_mode']-1, 'g.-', linewidth=0.5, markersize=2,alpha=0.9, label='mode - 1') - - plt.plot(Ti2['x'], Ti2['heights_c_std'] - 1.8, 'k-', linewidth=0.5,alpha=1) - plt.fill_between( Ti2['x'], Ti2['heights_c_std'] -1.8 , y2=-1.8, color='gray',alpha=1) + plt.plot(T2["x"], T2["heights_c"], "k.", markersize=0.5, alpha=0.8) + # plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') + + plt.plot( + Ti2["x"], + Ti2["heights_c_weighted_mean"] + 1, + ".-", + color="darkblue", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="x-gauss weighted mean +1", + ) + plt.plot( + Ti2["x"], + Ti2["heights_c_median"], + "r.-", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="median", + ) + + # plt.plot(Ti2['x'], Ti2['heights_c_mode']-1, 'g.-', linewidth=0.5, markersize=2,alpha=0.9, label='mode - 1') + + plt.plot(Ti2["x"], Ti2["heights_c_std"] - 1.8, "k-", linewidth=0.5, alpha=1) + plt.fill_between( + Ti2["x"], Ti2["heights_c_std"] - 1.8, y2=-1.8, color="gray", alpha=1 + ) # plt.plot( ALT03['delta_time'], ALT03['heights_c'], 'k.', markersize= 0.3, alpha =0.2 ) # plt.plot(ALT07['time']['delta_time'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') plt.legend(loc=1) - plt.xlim(ll, ll+dl) + plt.xlim(ll, ll + dl) plt.ylim(-6, 6) - plt.xlabel('Meters from the Sea Ice Edge') - plt.ylabel('Height Anomalie (meters)') - F.ax.axhline(y =-1.8, color='black', linewidth=0.5) - #F.save_light(path= plot_path, name='ALT03_filt_compare'+ str(ll)) + plt.xlabel("Meters from the Sea Ice Edge") + plt.ylabel("Height Anomalie (meters)") + F.ax.axhline(y=-1.8, color="black", linewidth=0.5) + # F.save_light(path= plot_path, name='ALT03_filt_compare'+ str(ll)) diff --git a/analysis/B01_filter_regrid_lat_lon.py b/analysis/B01_filter_regrid_lat_lon.py index 704bb7ec..3e272e72 100644 --- a/analysis/B01_filter_regrid_lat_lon.py +++ b/analysis/B01_filter_regrid_lat_lon.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,7 +9,7 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) import datetime @@ -17,7 +18,10 @@ import imp import ICEsat2_SI_tools.convert_GPS_time as cGPS import ICEsat2_SI_tools.io as io -from spectral_estimates import create_chunk_boundaries_unit_lengths, create_chunk_boundaries +from spectral_estimates import ( + create_chunk_boundaries_unit_lengths, + create_chunk_boundaries, +) import spectral_estimates as spec import m_tools_ph3 as MT import filter_regrid as regrid @@ -25,282 +29,365 @@ import concurrent.futures as futures +# import s3fs +# processed_ATL03_20190605061807_10380310_004_01.h5 -#import s3fs -#processed_ATL03_20190605061807_10380310_004_01.h5 - -#imp.reload(io) -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207234532_06340210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190215184558_07530210_004_01', 'SH_batch02', False +# imp.reload(io) +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207234532_06340210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190215184558_07530210_004_01', 'SH_batch02', False +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' +load_path = mconfig["paths"]["scratch"] + "/" + batch_key + "/" +load_file = load_path + "processed_" + ATlevel + "_" + track_name + ".h5" -load_path = mconfig['paths']['scratch'] +'/'+ batch_key +'/' -load_file = load_path + 'processed_'+ATlevel+'_'+track_name+'.h5' +save_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" -save_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' - -plot_path = mconfig['paths']['plot']+ '/'+hemis+'/'+batch_key+'/'+track_name +'/B_ov/' -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/B_ov/" +) +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" MT.mkdirs_r(save_path) # set pars # define parameters: -Lmeter = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 -Nphoton_min = 5 # mininum need photons per stancil to return results +Lmeter = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 +Nphoton_min = 5 # mininum need photons per stancil to return results -Lmeter_large= 100e3 # stancil width for testing photon density. stancils do not overlab. -minium_photon_density = 0.05 # minimum photon density per meter in Lmeter_large chunk to be counted as real signal +Lmeter_large = ( + 100e3 # stancil width for testing photon density. stancils do not overlab. +) +minium_photon_density = 0.05 # minimum photon density per meter in Lmeter_large chunk to be counted as real signal -plot_flag = True -Nworkers = 3 # number of threads for parallel processing -Nworkers_process =3 # number of threads for parallel processing +plot_flag = True +Nworkers = 3 # number of threads for parallel processing +Nworkers_process = 3 # number of threads for parallel processing # %% # test which beams exist: -all_beams = mconfig['beams']['all_beams'] +all_beams = mconfig["beams"]["all_beams"] # high_beams = mconfig['beams']['high_beams'] # low_beams = mconfig['beams']['low_beams'] -f = h5py.File(load_file, 'r') -beams = [b if b in f.keys() else None for b in all_beams] -accent = regrid.track_type_beam(f) +f = h5py.File(load_file, "r") +beams = [b if b in f.keys() else None for b in all_beams] +accent = regrid.track_type_beam(f) -print('accent is ' , accent) +print("accent is ", accent) # Load fata and apply height corrections # This needs version 2 of the ALT 03 dataset -hist = 'Beam stats' -B = dict() -B1save = dict() +hist = "Beam stats" +B = dict() +B1save = dict() for k in beams: - #k = beams[0] - #imp.reload(io) + # k = beams[0] + # imp.reload(io) print(k) - T = io.getATL03_beam(load_file, beam= k) + T = io.getATL03_beam(load_file, beam=k) - print('loaded') - T = T[T['mask_seaice']] # only take sea ice points, no ocean points + print("loaded") + T = T[T["mask_seaice"]] # only take sea ice points, no ocean points ho = k - ho = MT.add_line_var(ho, 'size', str(T.shape[0])) - ho = MT.add_line_var(ho, 'by confidence levels:' + str(np.arange(0, 5)), [ (T['signal_confidence'] == i).sum() for i in np.arange(0, 5) ]) + ho = MT.add_line_var(ho, "size", str(T.shape[0])) + ho = MT.add_line_var( + ho, + "by confidence levels:" + str(np.arange(0, 5)), + [(T["signal_confidence"] == i).sum() for i in np.arange(0, 5)], + ) # filter: - Tsel = T[ (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] + Tsel = T[ + (T["heights"] < 100) & (T["heights"] > -100) + ] # & (T['delta_time']>5) & (T['delta_time']<24) ] # if len(Tsel) == 0: # ho = MT.add_line_var(ho, 'no photons found', '') # #Tsel= T[(T['signal_confidence'] ==-1 ) & (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] - Tsel_c = io.getATL03_height_correction(load_file) - Tsel_c = Tsel_c[Tsel_c['dem_h'] < 1e5] # cute out weird references - Tsel2 = regrid.correct_heights(Tsel, Tsel_c).reset_index(drop=True)# correct height - print('corrected') + Tsel_c = io.getATL03_height_correction(load_file) + Tsel_c = Tsel_c[Tsel_c["dem_h"] < 1e5] # cute out weird references + Tsel2 = regrid.correct_heights(Tsel, Tsel_c).reset_index( + drop=True + ) # correct height + print("corrected") ### cut data at the rear that has too much variance # cut last segments of data until variance is similar rear_mask = np.array(Tsel2.index) > -1 shape_old = Tsel2.shape - N_seg= 20 + N_seg = 20 cut_flag = True while cut_flag: - dd= Tsel2['heights_c'][rear_mask] + dd = Tsel2["heights_c"][rear_mask] nsize = dd.size - stencil_iter = create_chunk_boundaries( int(nsize/N_seg), nsize,ov =0, iter_flag=True ) + stencil_iter = create_chunk_boundaries( + int(nsize / N_seg), nsize, ov=0, iter_flag=True + ) def get_var(sti): - return dd[sti[0]: sti[1]].var() + return dd[sti[0] : sti[1]].var() var_list = np.array(list(map(get_var, stencil_iter))) - if var_list[0:3].mean()*10 < var_list[-1]: - #print('cut last '+ str(100/N_seg) +'% of data') - rear_mask[int(nsize* (N_seg-1) / N_seg):] = False + if var_list[0:3].mean() * 10 < var_list[-1]: + # print('cut last '+ str(100/N_seg) +'% of data') + rear_mask[int(nsize * (N_seg - 1) / N_seg) :] = False else: - cut_flag = False - - - Tsel2['process_mask'] = rear_mask - B1save[k] = Tsel2 - B[k] = Tsel2[rear_mask].drop(columns='process_mask') - - ho = MT.add_line_var(ho, 'cutted ', str( np.round( 100 *(rear_mask.size -rear_mask.sum() ) / rear_mask.size, 0)) + '% in the back of the track' ) - ho = MT.add_line_var(ho, 'selected size', str(Tsel.shape[0])) - ho = MT.add_line_var(ho, 'final size ', str(Tsel_c.shape[0])) + cut_flag = False + + Tsel2["process_mask"] = rear_mask + B1save[k] = Tsel2 + B[k] = Tsel2[rear_mask].drop(columns="process_mask") + + ho = MT.add_line_var( + ho, + "cutted ", + str(np.round(100 * (rear_mask.size - rear_mask.sum()) / rear_mask.size, 0)) + + "% in the back of the track", + ) + ho = MT.add_line_var(ho, "selected size", str(Tsel.shape[0])) + ho = MT.add_line_var(ho, "final size ", str(Tsel_c.shape[0])) print(ho) - hist = MT.write_log(hist, ho) + hist = MT.write_log(hist, ho) -print('done with 1st loop') +print("done with 1st loop") # %% # define latitude limits -lat_lims = regrid.lat_min_max(B, all_beams, accent = accent) +lat_lims = regrid.lat_min_max(B, all_beams, accent=accent) # %% test ave photon density -length_meter = (abs(lat_lims[1]) - abs(lat_lims[0])) * 110e3 +length_meter = (abs(lat_lims[1]) - abs(lat_lims[0])) * 110e3 p_densities_r = list() p_densities_l = list() for k, I in B.items(): - if 'r' in k: - p_densities_r.append( I.shape[0] /length_meter) + if "r" in k: + p_densities_r.append(I.shape[0] / length_meter) else: - p_densities_l.append( I.shape[0] /length_meter) - -if (np.array(p_densities_l).mean() < 0.5) & (np.array(p_densities_r).mean() < 0.5): # in photons per meter - print('photon density too low, this track is classified as bad track and pushed to bad list') - MT.json_save(track_name, bad_track_path, {'densities': [ np.array(p_densities_r).mean(), np.array(p_densities_l).mean()] , 'date': str(datetime.date.today()) }) - print('exit.') + p_densities_l.append(I.shape[0] / length_meter) + +if (np.array(p_densities_l).mean() < 0.5) & ( + np.array(p_densities_r).mean() < 0.5 +): # in photons per meter + print( + "photon density too low, this track is classified as bad track and pushed to bad list" + ) + MT.json_save( + track_name, + bad_track_path, + { + "densities": [ + np.array(p_densities_r).mean(), + np.array(p_densities_l).mean(), + ], + "date": str(datetime.date.today()), + }, + ) + print("exit.") exit() # %% save corrected data and delete from cash -#io.save_pandas_table(B1save, track_name + '_B01_corrected' , save_path) # all photos but heights adjusted and with distance coordinate +# io.save_pandas_table(B1save, track_name + '_B01_corrected' , save_path) # all photos but heights adjusted and with distance coordinate del B1save - - # %% ##### 1.) derive common axis for beams and filter out low density area at the beginning -print('derive common axis for beams and filter out low density area at the beginning') +print("derive common axis for beams and filter out low density area at the beginning") def derive_axis_and_boundaries(key): - #key = 'gt3r' + # key = 'gt3r' print(key) - T2 = regrid.derive_axis(B[key], lat_lims).sort_values('dist') + T2 = regrid.derive_axis(B[key], lat_lims).sort_values("dist") T2.shape - x0 = get_better_lower_boundary(Lmeter_large, np.array(T2['dist'])) - print( 'cut off ' , 100 * (1 - T2[T2['dist'] > x0].shape[0]/T2.shape[0]) , '% off all data points at the beginning' ) - T2 = T2[T2['dist'] >x0] # cut off low density start - #B2[key] = T2 - return key, T2, [T2['dist'].min(), T2['dist'].max()] + x0 = get_better_lower_boundary(Lmeter_large, np.array(T2["dist"])) + print( + "cut off ", + 100 * (1 - T2[T2["dist"] > x0].shape[0] / T2.shape[0]), + "% off all data points at the beginning", + ) + T2 = T2[T2["dist"] > x0] # cut off low density start + # B2[key] = T2 + return key, T2, [T2["dist"].min(), T2["dist"].max()] + def get_better_lower_boundary(Lmeter_large, dd): # T2 = regrid.derive_axis(B[key], lat_lims) # dd = np.array(T2['dist']) - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= True) + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=True + ) def get_density(sti): - return sti[0], sum( (sti[0] <= dd) & (dd < sti[-1]) ) / Lmeter_large + return sti[0], sum((sti[0] <= dd) & (dd < sti[-1])) / Lmeter_large - with futures.ThreadPoolExecutor(max_workers= Nworkers_process) as executor_sub: - #var_list = np.array(list(map(get_density, stencil_iter))) + with futures.ThreadPoolExecutor(max_workers=Nworkers_process) as executor_sub: + # var_list = np.array(list(map(get_density, stencil_iter))) var_list = np.array(list(executor_sub.map(get_density, stencil_iter))) var_list = np.array(var_list) - #var_list[:,0] = np.random.rand(10) - #sort var_list - var_list = var_list[var_list[:,0].argsort(), :] - #print(var_list) - if sum(var_list[:,1] > minium_photon_density) > 1: - first_stencil = next((i for i, j in enumerate(var_list[:,1] > minium_photon_density) if j), None) - 1 - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) + # var_list[:,0] = np.random.rand(10) + # sort var_list + var_list = var_list[var_list[:, 0].argsort(), :] + # print(var_list) + if sum(var_list[:, 1] > minium_photon_density) > 1: + first_stencil = ( + next( + (i for i, j in enumerate(var_list[:, 1] > minium_photon_density) if j), + None, + ) + - 1 + ) + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=False + ) return stencil_iter[0, first_stencil] else: - #first_stencil = next((i for i, j in enumerate(var_list[:,1] > 0) if j), None)# - 1 - print('no sufficient photon density found. return short stencil') + # first_stencil = next((i for i, j in enumerate(var_list[:,1] > 0) if j), None)# - 1 + print("no sufficient photon density found. return short stencil") # first_stencil= len(var_list[:,1]) -1 # stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) # return stencil_iter[0, first_stencil] # #print(first_stencil) - return var_list[-1,0]#[-1,0] - + return var_list[-1, 0] # [-1,0] # for bb in all_beams: # A = derive_axis_and_boundaries(bb) with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: - A = list( executor.map(derive_axis_and_boundaries, all_beams) ) + A = list(executor.map(derive_axis_and_boundaries, all_beams)) # %% -B2 = dict() -dist_list = np.array([np.nan, np.nan]) +B2 = dict() +dist_list = np.array([np.nan, np.nan]) for I in A: - k = I[0] - B2[k] = I[1] - dist_list = np.vstack([dist_list,I[2] ]) + k = I[0] + B2[k] = I[1] + dist_list = np.vstack([dist_list, I[2]]) del A -track_dist_bounds = [ np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0) ] +track_dist_bounds = [np.nanmin(dist_list[:, 0], 0), np.nanmax(dist_list[:, 1], 0)] # for testing: -#track_dist_bounds[1] = track_dist_bounds[0] + (track_dist_bounds[1] - track_dist_bounds[0])/20 +# track_dist_bounds[1] = track_dist_bounds[0] + (track_dist_bounds[1] - track_dist_bounds[0])/20 # %% ##### 2.) regridding and averaging -print('regrid') +print("regrid") + + def regridding_wrapper(I): key, Ti = I - print(key, Ti.shape, 2* Ti.shape[0]/Lmeter) - stencil_iter = create_chunk_boundaries_unit_lengths( Lmeter, track_dist_bounds, iter_flag=True ) - with futures.ThreadPoolExecutor(max_workers= Nworkers) as executor_sub: - Bi = regrid.get_stencil_stats( Ti, stencil_iter, 'heights_c', stancil_width= Lmeter/2, Nphoton_min=Nphoton_min, map_func = executor_sub.map) - #B3[key] = - print(key, 'done') + print(key, Ti.shape, 2 * Ti.shape[0] / Lmeter) + stencil_iter = create_chunk_boundaries_unit_lengths( + Lmeter, track_dist_bounds, iter_flag=True + ) + with futures.ThreadPoolExecutor(max_workers=Nworkers) as executor_sub: + Bi = regrid.get_stencil_stats( + Ti, + stencil_iter, + "heights_c", + stancil_width=Lmeter / 2, + Nphoton_min=Nphoton_min, + map_func=executor_sub.map, + ) + # B3[key] = + print(key, "done") return key, Bi with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: - B3 = dict( executor.map(regridding_wrapper, B2.items() ) ) + B3 = dict(executor.map(regridding_wrapper, B2.items())) # %% saving data -io.save_pandas_table(B2, track_name + '_B01_regridded' , save_path) # all photos but heights adjusted and with distance coordinate -io.save_pandas_table(B3, track_name + '_B01_binned' , save_path) # regridding heights +io.save_pandas_table( + B2, track_name + "_B01_regridded", save_path +) # all photos but heights adjusted and with distance coordinate +io.save_pandas_table(B3, track_name + "_B01_binned", save_path) # regridding heights # %% plotting just for checking -key = 'gt2r' +key = "gt2r" if plot_flag: MT.mkdirs_r(plot_path) Ti2 = B3[key] - T2 = B2[key] + T2 = B2[key] dl = 4000 - latlims = (Ti2['dist'].iloc[0] , Ti2['dist'].iloc[-1] ) - #chunk_list = np.arange(latlims[0],latlims[1], dl ) - #chunk_list = sample( list(np.arange(latlims[0],latlims[1],dl )[0:80]) ,10) - chunk_list = np.arange(latlims[0],latlims[1], dl )[::10] - chunk_list = np.append( chunk_list, latlims[1]- dl-1) + latlims = (Ti2["dist"].iloc[0], Ti2["dist"].iloc[-1]) + # chunk_list = np.arange(latlims[0],latlims[1], dl ) + # chunk_list = sample( list(np.arange(latlims[0],latlims[1],dl )[0:80]) ,10) + chunk_list = np.arange(latlims[0], latlims[1], dl)[::10] + chunk_list = np.append(chunk_list, latlims[1] - dl - 1) for ll in chunk_list: F = M.figure_axis_xy(7, 3, view_scale=0.8) - plt.plot( T2['dist'], T2['heights_c'], 'k.', markersize= 0.5, alpha =0.8 ) - #plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') - - plt.plot(Ti2['dist'], Ti2['heights_c_weighted_mean'] +1, '.-', color='darkblue', linewidth=0.5, markersize=2,alpha=0.9, label='x-gauss weighted mean +1') - plt.plot(Ti2['dist'], Ti2['heights_c_median'], 'r.-', linewidth=0.5, markersize=2,alpha=0.9, label='median') - - #plt.plot(Ti2['dist'], Ti2['heights_c_mode']-1, 'g.-', linewidth=0.5, markersize=2,alpha=0.9, label='mode - 1') - - plt.plot(Ti2['dist'], Ti2['heights_c_std'] - 1.8, 'k-', linewidth=0.5,alpha=1) - plt.fill_between( Ti2['dist'], Ti2['heights_c_std'] -1.8 , y2=-1.8, color='gray',alpha=1) + plt.plot(T2["dist"], T2["heights_c"], "k.", markersize=0.5, alpha=0.8) + # plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') + + plt.plot( + Ti2["dist"], + Ti2["heights_c_weighted_mean"] + 1, + ".-", + color="darkblue", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="x-gauss weighted mean +1", + ) + plt.plot( + Ti2["dist"], + Ti2["heights_c_median"], + "r.-", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="median", + ) + + # plt.plot(Ti2['dist'], Ti2['heights_c_mode']-1, 'g.-', linewidth=0.5, markersize=2,alpha=0.9, label='mode - 1') + + plt.plot(Ti2["dist"], Ti2["heights_c_std"] - 1.8, "k-", linewidth=0.5, alpha=1) + plt.fill_between( + Ti2["dist"], Ti2["heights_c_std"] - 1.8, y2=-1.8, color="gray", alpha=1 + ) # plt.plot( ALT03['delta_time'], ALT03['heights_c'], 'k.', markersize= 0.3, alpha =0.2 ) # plt.plot(ALT07['time']['delta_time'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') plt.legend(loc=1) - plt.xlim(ll, ll+dl) + plt.xlim(ll, ll + dl) plt.ylim(-6, 6) - plt.xlabel('Meters from the Sea Ice Edge') - plt.ylabel('Height Anomalie (meters)') - F.ax.axhline(y =-1.8, color='black', linewidth=0.5) - F.save_light(path= plot_path, name='ALT03_filt_compare'+ str(ll)) + plt.xlabel("Meters from the Sea Ice Edge") + plt.ylabel("Height Anomalie (meters)") + F.ax.axhline(y=-1.8, color="black", linewidth=0.5) + F.save_light(path=plot_path, name="ALT03_filt_compare" + str(ll)) diff --git a/analysis/B01_filter_regrid_segment_test_regridd.py b/analysis/B01_filter_regrid_segment_test_regridd.py index dc917dce..4e5e5a98 100644 --- a/analysis/B01_filter_regrid_segment_test_regridd.py +++ b/analysis/B01_filter_regrid_segment_test_regridd.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,7 +9,7 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) import datetime @@ -17,7 +18,10 @@ import imp import ICEsat2_SI_tools.convert_GPS_time as cGPS import ICEsat2_SI_tools.io as io -from spectral_estimates import create_chunk_boundaries_unit_lengths, create_chunk_boundaries +from spectral_estimates import ( + create_chunk_boundaries_unit_lengths, + create_chunk_boundaries, +) import spectral_estimates as spec import m_tools_ph3 as MT import filter_regrid as regrid @@ -25,153 +29,176 @@ import concurrent.futures as futures +# import s3fs +# processed_ATL03_20190605061807_10380310_004_01.h5 -#import s3fs -#processed_ATL03_20190605061807_10380310_004_01.h5 - -#imp.reload(io) -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207234532_06340210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190215184558_07530210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False - -track_name, batch_key, test_flag = '20190215184558_07530210_004_01', 'SH_batch02', False +# imp.reload(io) +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207234532_06340210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190215184558_07530210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +track_name, batch_key, test_flag = "20190215184558_07530210_004_01", "SH_batch02", False # equatorward track -#track_name, batch_key, test_flag = '20190208154150_06440212_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190208154150_06440212_004_01', 'SH_batch02', False # poleward track -#track_name, batch_key, test_flag = '20190209150245_06590210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190209150245_06590210_004_01', 'SH_batch02', False # -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -load_path = mconfig['paths']['scratch'] +'/'+ batch_key +'/' -load_file = load_path + 'processed_'+ATlevel+'_'+track_name+'.h5' - -save_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' - -plot_path = mconfig['paths']['plot']+ '/'+hemis+'/'+batch_key+'/'+track_name +'/B01/' -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +load_path = mconfig["paths"]["scratch"] + "/" + batch_key + "/" +load_file = load_path + "processed_" + ATlevel + "_" + track_name + ".h5" + +save_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/B01/" +) +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" MT.mkdirs_r(save_path) # set pars # define parameters: -Lmeter = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 -Nphoton_min = 5 # mininum need photons per stancil to return results +Lmeter = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 +Nphoton_min = 5 # mininum need photons per stancil to return results -Lmeter_large= 100e3 # stancil width for testing photon density. stancils do not overlab. -minium_photon_density = 0.02 # minimum photon density per meter in Lmeter_large chunk to be counted as real signal +Lmeter_large = ( + 100e3 # stancil width for testing photon density. stancils do not overlab. +) +minium_photon_density = 0.02 # minimum photon density per meter in Lmeter_large chunk to be counted as real signal -plot_flag = True -Nworkers = 4 # number of threads for parallel processing +plot_flag = True +Nworkers = 4 # number of threads for parallel processing Nworkers_process = 3 # number of threads for parallel processing # %% # test which beams exist: -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] # low_beams = mconfig['beams']['low_beams'] -f = h5py.File(load_file, 'r') -beams = [b if b in f.keys() else None for b in all_beams] +f = h5py.File(load_file, "r") +beams = [b if b in f.keys() else None for b in all_beams] imp.reload(regrid) -track_poleward = regrid.track_pole_ward_file(f) -print('poleward track is ' , track_poleward) +track_poleward = regrid.track_pole_ward_file(f) +print("poleward track is ", track_poleward) # Load fata and apply height corrections # This needs version 2 of the ALT 03 dataset # ATL03 = h5py.File(load_file, 'r') -#ATL03[k+'/heights/ph_id_channel'][0:100] -#accent = regrid.track_type( B[beams_list[0]] ) +# ATL03[k+'/heights/ph_id_channel'][0:100] +# accent = regrid.track_type( B[beams_list[0]] ) # %% -hist = 'Beam stats' -B = dict() -B1save = dict() -SEG = dict() +hist = "Beam stats" +B = dict() +B1save = dict() +SEG = dict() for k in beams: - #k = beams[0] + # k = beams[0] imp.reload(io) print(k) - T, seg = io.getATL03_beam(load_file, beam= k) - + T, seg = io.getATL03_beam(load_file, beam=k) - - print('loaded') - T = T[T['mask_seaice']] # only take sea ice points, no ocean points + print("loaded") + T = T[T["mask_seaice"]] # only take sea ice points, no ocean points ho = k - ho = MT.add_line_var(ho, 'size', str(T.shape[0])) - ho = MT.add_line_var(ho, 'by confidence levels:' + str(np.arange(0, 5)), [ (T['signal_confidence'] == i).sum() for i in np.arange(0, 5) ]) + ho = MT.add_line_var(ho, "size", str(T.shape[0])) + ho = MT.add_line_var( + ho, + "by confidence levels:" + str(np.arange(0, 5)), + [(T["signal_confidence"] == i).sum() for i in np.arange(0, 5)], + ) # filter: - Tsel = T[ (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] + Tsel = T[ + (T["heights"] < 100) & (T["heights"] > -100) + ] # & (T['delta_time']>5) & (T['delta_time']<24) ] # if len(Tsel) == 0: # ho = MT.add_line_var(ho, 'no photons found', '') # #Tsel= T[(T['signal_confidence'] ==-1 ) & (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] - Tsel_c = io.getATL03_height_correction(load_file) - Tsel_c = Tsel_c[Tsel_c['dem_h'] < 1e5] # cute out weird references - Tsel2 = regrid.correct_heights(Tsel, Tsel_c).reset_index(drop=True)# correct height - print('height corrected') + Tsel_c = io.getATL03_height_correction(load_file) + Tsel_c = Tsel_c[Tsel_c["dem_h"] < 1e5] # cute out weird references + Tsel2 = regrid.correct_heights(Tsel, Tsel_c).reset_index( + drop=True + ) # correct height + print("height corrected") ### cut data at the rear that has too much variance # cut last segments of data until variance is similar - rear_mask = np.array(Tsel2.index) > -1 # True + rear_mask = np.array(Tsel2.index) > -1 # True shape_old = Tsel2.shape - N_seg= 20 + N_seg = 20 cut_flag = True while cut_flag: - dd= Tsel2['heights_c'][rear_mask] + dd = Tsel2["heights_c"][rear_mask] nsize = dd.size - stencil_iter = create_chunk_boundaries( int(nsize/N_seg), nsize,ov =0, iter_flag=True ) + stencil_iter = create_chunk_boundaries( + int(nsize / N_seg), nsize, ov=0, iter_flag=True + ) def get_var(sti): - return dd[sti[0]: sti[1]].var() + return dd[sti[0] : sti[1]].var() var_list = np.array(list(map(get_var, stencil_iter))) - #print(var_list) + # print(var_list) if track_poleward: - if var_list[0:3].mean()*10 < var_list[-1]: - #print('cut last '+ str(100/N_seg) +'% of data') - rear_mask[int(nsize* (N_seg-1) / N_seg):] = False + if var_list[0:3].mean() * 10 < var_list[-1]: + # print('cut last '+ str(100/N_seg) +'% of data') + rear_mask[int(nsize * (N_seg - 1) / N_seg) :] = False else: - cut_flag = False + cut_flag = False else: - if var_list[-3:].mean()*10 < var_list[0]: - #print('cut last '+ str(100/N_seg) +'% of data') - #int(nsize* (N_seg-1) / N_seg) - rear_mask[: int(nsize* 1 / N_seg)] = False + if var_list[-3:].mean() * 10 < var_list[0]: + # print('cut last '+ str(100/N_seg) +'% of data') + # int(nsize* (N_seg-1) / N_seg) + rear_mask[: int(nsize * 1 / N_seg)] = False else: - cut_flag = False - - - Tsel2['process_mask'] = rear_mask - B1save[k] = Tsel2 - B[k] = Tsel2[rear_mask].drop(columns='process_mask') - SEG[k] = seg - - ho = MT.add_line_var(ho, 'cutted ', str( np.round( 100 *(rear_mask.size -rear_mask.sum() ) / rear_mask.size, 0)) + '% in the back of the track' ) - ho = MT.add_line_var(ho, 'selected size', str(Tsel.shape[0])) - ho = MT.add_line_var(ho, 'final size ', str(Tsel_c.shape[0])) + cut_flag = False + + Tsel2["process_mask"] = rear_mask + B1save[k] = Tsel2 + B[k] = Tsel2[rear_mask].drop(columns="process_mask") + SEG[k] = seg + + ho = MT.add_line_var( + ho, + "cutted ", + str(np.round(100 * (rear_mask.size - rear_mask.sum()) / rear_mask.size, 0)) + + "% in the back of the track", + ) + ho = MT.add_line_var(ho, "selected size", str(Tsel.shape[0])) + ho = MT.add_line_var(ho, "final size ", str(Tsel_c.shape[0])) print(ho) - hist = MT.write_log(hist, ho) + hist = MT.write_log(hist, ho) -print('done with 1st loop') +print("done with 1st loop") # %% define x- coodindate @@ -179,13 +206,12 @@ def get_var(sti): # this is on the equatorward side for poleward track, or # on the poleward side for equatorward tracks. -total_segment_dist_x_min= list() -for k,I in SEG.items(): - total_segment_dist_x_min.append( I['segment_dist_x'].min() ) +total_segment_dist_x_min = list() +for k, I in SEG.items(): + total_segment_dist_x_min.append(I["segment_dist_x"].min()) total_segment_dist_x_min = min(total_segment_dist_x_min) - # # # plt.plot( np.diff( T2.index), '.', markersize = 0.3 ) @@ -196,144 +222,152 @@ def get_var(sti): # plt.plot( np.diff( T3.index), '.', markersize = 0.3 ) # # plt.plot(T['delta_time'] , T.index , '.', markersize = 0.3 ) -T = B['gt1r'] -seg = SEG['gt1r'] +T = B["gt1r"] +seg = SEG["gt1r"] T -#T = T.sort_values('lats').reset_index(drop=True) -#T.sort_values('lats') +# T = T.sort_values('lats').reset_index(drop=True) +# T.sort_values('lats') # find positions where segmetn length is reset -time_ticks = np.insert(np.diff(T['along_track_distance']), 0,0 ) +time_ticks = np.insert(np.diff(T["along_track_distance"]), 0, 0) -#plt.plot(T['along_track_distance'][0:500], '.') +# plt.plot(T['along_track_distance'][0:500], '.') Ti = T[-500:] -plt.plot(Ti['along_track_distance'], '.') -plt.plot(np.diff(Ti['along_track_distance'])) +plt.plot(Ti["along_track_distance"], ".") +plt.plot(np.diff(Ti["along_track_distance"])) T # %% -#Ti = T[10200:10250] +# Ti = T[10200:10250] Ti = T[20000:20500] -#Ti = T[20000:20550] +# Ti = T[20000:20550] -#plt.plot(Ti['delta_time'], Ti['along_track_distance'], '.') -#plt.plot(Ti['delta_time'], np.insert(np.diff(Ti['along_track_distance']), 0,0 ) ) -delta_onehalf = seg['delta_time'].diff()/2 +# plt.plot(Ti['delta_time'], Ti['along_track_distance'], '.') +# plt.plot(Ti['delta_time'], np.insert(np.diff(Ti['along_track_distance']), 0,0 ) ) +delta_onehalf = seg["delta_time"].diff() / 2 delta_onehalf.iloc[0] = delta_onehalf.iloc[1] -seg['delta_half_time']= seg['delta_time'] - delta_onehalf #+ 5e-5 -#seg['delta_half_time']= seg['delta_time'] - seg['delta_time'].diff().mean()/2# - 1e-5 -#seg['segment_length'] -segi =seg[ (Ti['delta_time'].min() - 2 * delta_onehalf <= seg['delta_time'] ) & ( Ti['delta_time'].max() + 3 * delta_onehalf >= seg['delta_time'] ) ] -#segi.shape - -#Ti['ph_id_count'] -#segi['ph_index_beg'] + segi['reference_photon_index'] -1 +seg["delta_half_time"] = seg["delta_time"] - delta_onehalf # + 5e-5 +# seg['delta_half_time']= seg['delta_time'] - seg['delta_time'].diff().mean()/2# - 1e-5 +# seg['segment_length'] +segi = seg[ + (Ti["delta_time"].min() - 2 * delta_onehalf <= seg["delta_time"]) + & (Ti["delta_time"].max() + 3 * delta_onehalf >= seg["delta_time"]) +] +# segi.shape + +# Ti['ph_id_count'] +# segi['ph_index_beg'] + segi['reference_photon_index'] -1 # %% # % -plt.plot(Ti['delta_time'], Ti['delta_time'] *0, '.') +plt.plot(Ti["delta_time"], Ti["delta_time"] * 0, ".") -plt.plot( segi['delta_time'], segi['delta_time'] *0 , '.r' ) -plt.plot(Ti['delta_time'], Ti['along_track_distance'], '.') +plt.plot(segi["delta_time"], segi["delta_time"] * 0, ".r") +plt.plot(Ti["delta_time"], Ti["along_track_distance"], ".") # cutting using delta time from segments -for i in segi['delta_time']: - plt.gca().axvline(i, linewidth = 0.5, color= 'black') -plt.plot(Ti['delta_time'], np.digitize(Ti['delta_time'], segi['delta_time'] ) , '.k') +for i in segi["delta_time"]: + plt.gca().axvline(i, linewidth=0.5, color="black") +plt.plot(Ti["delta_time"], np.digitize(Ti["delta_time"], segi["delta_time"]), ".k") # cutting usind delta time from segments shifted -for i in segi['delta_half_time']: - plt.gca().axvline(i, linewidth = 0.5, color= 'blue') +for i in segi["delta_half_time"]: + plt.gca().axvline(i, linewidth=0.5, color="blue") -plt.xlabel('delta_time') -#np.digitize(T['delta_time'], T[ time_ticks < - 10 ]['delta_time']) +plt.xlabel("delta_time") +# np.digitize(T['delta_time'], T[ time_ticks < - 10 ]['delta_time']) # %% -plt.plot(Ti['delta_time'], Ti['delta_time'] *0, '.') +plt.plot(Ti["delta_time"], Ti["delta_time"] * 0, ".") -plt.plot( segi['delta_time'], segi['delta_time'] *0 , '.r' ) -plt.plot(Ti['delta_time'], Ti['along_track_distance'], '.') +plt.plot(segi["delta_time"], segi["delta_time"] * 0, ".r") +plt.plot(Ti["delta_time"], Ti["along_track_distance"], ".") # cutting usind delta time from segments shifted -for i in segi['delta_half_time']: - plt.gca().axvline(i, linewidth = 0.5, color= 'blue') +for i in segi["delta_half_time"]: + plt.gca().axvline(i, linewidth=0.5, color="blue") -segi['delta_half_time'].max() < Ti['delta_time'].max() +segi["delta_half_time"].max() < Ti["delta_time"].max() -Ti2 = Ti[ (Ti['delta_time'] > segi['delta_half_time'].iloc[0]) & (Ti['delta_time'] < segi['delta_half_time'].iloc[-1])] -bin_labels = np.digitize( Ti2['delta_time'], segi['delta_half_time'], right = True ) -plt.plot(Ti['delta_time'], bin_labels +5 , '.b') +Ti2 = Ti[ + (Ti["delta_time"] > segi["delta_half_time"].iloc[0]) + & (Ti["delta_time"] < segi["delta_half_time"].iloc[-1]) +] +bin_labels = np.digitize(Ti2["delta_time"], segi["delta_half_time"], right=True) +plt.plot(Ti["delta_time"], bin_labels + 5, ".b") -Ti2['delta_time0'] = Ti2['delta_time'] - Ti2['delta_time'].min() -Ti2['lats0'] = Ti2['lats'] - Ti2['lats'].max() +Ti2["delta_time0"] = Ti2["delta_time"] - Ti2["delta_time"].min() +Ti2["lats0"] = Ti2["lats"] - Ti2["lats"].max() for i in np.arange(bin_labels.max()): - print( Ti2[[ 'along_track_distance', 'delta_time0', 'lats0']][bin_labels ==i] ) + print(Ti2[["along_track_distance", "delta_time0", "lats0"]][bin_labels == i]) -#list(Ti2['delta_time']) -#bin_labels.min() -#bin_labels.max() +# list(Ti2['delta_time']) +# bin_labels.min() +# bin_labels.max() repeats.shape -SS = segi['segment_dist_x'] -repeats = np.bincount(bin_labels, minlength =SS.shape[0]) +SS = segi["segment_dist_x"] +repeats = np.bincount(bin_labels, minlength=SS.shape[0]) # check if repeats sumup if repeats.sum() != Ti2.shape[0]: - print('repeats do not sum up') + print("repeats do not sum up") SS = SS.repeat(repeats) SS.index = Ti2.index -Ti2['x'] = SS + Ti2['along_track_distance'] -#list(SS.repeat(repeats).reset_index(drop=True) +Ti2["x"] = SS + Ti2["along_track_distance"] +# list(SS.repeat(repeats).reset_index(drop=True) -#segi[segi['ph_index_beg'] != 0]['seg_ID_local'] = list(set(bin_labels)) +# segi[segi['ph_index_beg'] != 0]['seg_ID_local'] = list(set(bin_labels)) -#Tg= Ti.groupby(by= 'seg_ID_local', axis=0, group_keys=seg['delta_time'] ) +# Tg= Ti.groupby(by= 'seg_ID_local', axis=0, group_keys=seg['delta_time'] ) -#Tii, segii = Ti.loc[Tg.groups[1]], segi +# Tii, segii = Ti.loc[Tg.groups[1]], segi # %% - # %% -#T2 = Tg.apply(add_seg_length, segi) +# T2 = Tg.apply(add_seg_length, segi) -#list(T2['x'] - T2['x'].iloc[0] ) -plt.plot(Ti2['delta_time'], Ti2['x'] , 'b.') -plt.xlabel('delta_time') -#np.digitize(T['delta_time'], T[ time_ticks < - 10 ]['delta_time']) -#x_interp = np.interp(Ti2['delta_time'], segi['delta_time'], segi['segment_dist_x'] ) +# list(T2['x'] - T2['x'].iloc[0] ) +plt.plot(Ti2["delta_time"], Ti2["x"], "b.") +plt.xlabel("delta_time") +# np.digitize(T['delta_time'], T[ time_ticks < - 10 ]['delta_time']) +# x_interp = np.interp(Ti2['delta_time'], segi['delta_time'], segi['segment_dist_x'] ) -plt.plot( segi['delta_half_time'], segi['segment_dist_x'] + segi['segment_length'] , 'g-' ) +plt.plot(segi["delta_half_time"], segi["segment_dist_x"] + segi["segment_length"], "g-") # plt.plot( Ti2['delta_time'], x_interp - x_interp[0], '.' ) def find_anomalie_photons(Ti2, segi): - x_interp = np.interp(Ti2['delta_time'], segi['delta_half_time'], segi['segment_dist_x'] + segi['segment_length']) - - diff_x = x_interp - Ti2['x'] - diff_x = abs(diff_x-diff_x.mean()) - return diff_x > 3 *diff_x.std(), x_interp + x_interp = np.interp( + Ti2["delta_time"], + segi["delta_half_time"], + segi["segment_dist_x"] + segi["segment_length"], + ) -fail_mask, x_interp = find_anomalie_photons(Ti2, segi) + diff_x = x_interp - Ti2["x"] + diff_x = abs(diff_x - diff_x.mean()) + return diff_x > 3 * diff_x.std(), x_interp -#Ti3= Ti2[~fail_mask] -Ti2['x'][fail_mask] = x_interp[fail_mask] +fail_mask, x_interp = find_anomalie_photons(Ti2, segi) -plt.plot(Ti2['delta_time'], Ti2['x'] , 'r.') +# Ti3= Ti2[~fail_mask] -plt.plot(Ti2['delta_time'][fail_mask], x_interp[fail_mask] , 'g.', markersize= 20) -#plt.plot(Ti2['delta_time'], x_interp , 'g.') +Ti2["x"][fail_mask] = x_interp[fail_mask] +plt.plot(Ti2["delta_time"], Ti2["x"], "r.") +plt.plot(Ti2["delta_time"][fail_mask], x_interp[fail_mask], "g.", markersize=20) +# plt.plot(Ti2['delta_time'], x_interp , 'g.') # %% @@ -373,36 +407,38 @@ def find_anomalie_photons(Ti2, segi): def make_x_coorindate(k): - """ Returns the "true" along track coordindate but finding the correpsonding segment length also adds the segment_ID to the main table T """ - print(k, ' make coodindate') - T, seg= B[k], SEG[k] + print(k, " make coodindate") + T, seg = B[k], SEG[k] # make sure data is strictly ordered by delta_time - T = T.sort_values('delta_time').reset_index(drop=True) + T = T.sort_values("delta_time").reset_index(drop=True) # find positions where segmetn length is reset # shifts segment length postions in time - delta_onehalf = seg['delta_time'].diff()/2 + delta_onehalf = seg["delta_time"].diff() / 2 delta_onehalf.iloc[0] = delta_onehalf.iloc[1] - seg['delta_half_time']= seg['delta_time'] - delta_onehalf - 1e-5 + seg["delta_half_time"] = seg["delta_time"] - delta_onehalf - 1e-5 # cur phontos that are not in segmentns - T2 = T[ (T['delta_time'] > seg['delta_half_time'].iloc[0]) & (T['delta_time'] < seg['delta_half_time'].iloc[-1])] - bin_labels = np.digitize( T2['delta_time'], seg['delta_half_time'], right = True ) + T2 = T[ + (T["delta_time"] > seg["delta_half_time"].iloc[0]) + & (T["delta_time"] < seg["delta_half_time"].iloc[-1]) + ] + bin_labels = np.digitize(T2["delta_time"], seg["delta_half_time"], right=True) # select relevant data - SS = seg['segment_dist_x'] - SS_sid = seg['segment_id'] + SS = seg["segment_dist_x"] + SS_sid = seg["segment_id"] - repeats = np.bincount(bin_labels, minlength =SS.shape[0]) + repeats = np.bincount(bin_labels, minlength=SS.shape[0]) # check if repeats sumup if repeats.sum() != T2.shape[0]: - print('repeats do not sum up') + print("repeats do not sum up") # repeat segment dat accoridng to photons SS = SS.repeat(repeats) @@ -410,49 +446,51 @@ def make_x_coorindate(k): SS_sid = SS_sid.repeat(repeats) SS_sid.index = T2.index # define new coordinate - T2['x'] = SS + T2['along_track_distance'] - T2['segment_id'] = SS_sid + T2["x"] = SS + T2["along_track_distance"] + T2["segment_id"] = SS_sid # find bad photons def find_anomalie_photons(Ti2, segi): - x_interp = np.interp(Ti2['delta_time'], segi['delta_time'], segi['segment_dist_x'] ) + x_interp = np.interp( + Ti2["delta_time"], segi["delta_time"], segi["segment_dist_x"] + ) - diff_x = x_interp - Ti2['x'] - diff_x = abs(diff_x-diff_x.mean()) - return diff_x > 3 *diff_x.std() , x_interp + diff_x = x_interp - Ti2["x"] + diff_x = abs(diff_x - diff_x.mean()) + return diff_x > 3 * diff_x.std(), x_interp fail_mask, x_interp = find_anomalie_photons(T2, seg) - print('weird photon fraction:' , sum(fail_mask)/ fail_mask.size) + print("weird photon fraction:", sum(fail_mask) / fail_mask.size) # aply fail mask - #T2= T2[~fail_mask] - T2['x'][fail_mask] = x_interp[fail_mask] + # T2= T2[~fail_mask] + T2["x"][fail_mask] = x_interp[fail_mask] return k, T2 -#make_x_coorindate(k) +# make_x_coorindate(k) with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: - A = list( executor.map(make_x_coorindate, all_beams) ) + A = list(executor.map(make_x_coorindate, all_beams)) # %% -B= dict() -for I in A: # collect returns from from mapping - k = I[0] - B[ k ] = I[1][::-1] - - if ~track_poleward: # invert x- coordinate if there is an equatorward track - B[k] = B[k].reset_index(drop=True) - B[k]['x_true'] = B[k]['x'] - B[k]['x'] = abs(B[k]['x'] - B[k]['x'].iloc[0]) +B = dict() +for I in A: # collect returns from from mapping + k = I[0] + B[k] = I[1][::-1] + + if ~track_poleward: # invert x- coordinate if there is an equatorward track + B[k] = B[k].reset_index(drop=True) + B[k]["x_true"] = B[k]["x"] + B[k]["x"] = abs(B[k]["x"] - B[k]["x"].iloc[0]) else: - B[k]['x_true'] = B[k]['x'] + B[k]["x_true"] = B[k]["x"] -dist_list = np.array([np.nan, np.nan]) +dist_list = np.array([np.nan, np.nan]) for k in B.keys(): - dist_list = np.vstack([ dist_list, [ B[k]['x'].iloc[0] , B[k]['x'].iloc[-1] ] ]) + dist_list = np.vstack([dist_list, [B[k]["x"].iloc[0], B[k]["x"].iloc[-1]]]) del A @@ -461,27 +499,41 @@ def find_anomalie_photons(Ti2, segi): # %% test ave photon density abnd quit if necessary -track_dist_bounds = [ np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0) ] -length_meter = abs(track_dist_bounds[1] - track_dist_bounds[0]) -#length_meter = (abs(lat_lims[1]) - abs(lat_lims[0])) * 110e3 -p_densities_r = list() -p_densities_l = list() +track_dist_bounds = [np.nanmin(dist_list[:, 0], 0), np.nanmax(dist_list[:, 1], 0)] +length_meter = abs(track_dist_bounds[1] - track_dist_bounds[0]) +# length_meter = (abs(lat_lims[1]) - abs(lat_lims[0])) * 110e3 +p_densities_r = list() +p_densities_l = list() for k, I in B.items(): - if 'r' in k: - p_densities_r.append( I.shape[0] /length_meter) + if "r" in k: + p_densities_r.append(I.shape[0] / length_meter) else: - p_densities_l.append( I.shape[0] /length_meter) - -if (np.array(p_densities_l).mean() < 0.5) & (np.array(p_densities_r).mean() < 0.5): # in photons per meter - print('photon density too low, this track is classified as bad track and pushed to bad list') - MT.json_save(track_name, bad_track_path, {'densities': [ np.array(p_densities_r).mean(), np.array(p_densities_l).mean()] , 'date': str(datetime.date.today()) }) - print('exit.') + p_densities_l.append(I.shape[0] / length_meter) + +if (np.array(p_densities_l).mean() < 0.5) & ( + np.array(p_densities_r).mean() < 0.5 +): # in photons per meter + print( + "photon density too low, this track is classified as bad track and pushed to bad list" + ) + MT.json_save( + track_name, + bad_track_path, + { + "densities": [ + np.array(p_densities_r).mean(), + np.array(p_densities_l).mean(), + ], + "date": str(datetime.date.today()), + }, + ) + print("exit.") exit() # %% save corrected data and delete from cash -#io.save_pandas_table(B1save, track_name + '_B01_corrected' , save_path) # all photos but heights adjusted and with distance coordinate +# io.save_pandas_table(B1save, track_name + '_B01_corrected' , save_path) # all photos but heights adjusted and with distance coordinate del B1save # for testing @@ -498,230 +550,312 @@ def find_anomalie_photons(Ti2, segi): # %% -F = M.figure_axis_xy(4, 3, view_scale = 0.7) +F = M.figure_axis_xy(4, 3, view_scale=0.7) -for k,I in B.items(): - lmask = (I['lats'] > -68.1) & (I['lats'] < -67.7) - plt.plot(I['x'][lmask] - I['x'].iloc[0] , I['lats'][lmask] ,'.' , markersize = 0.2) +for k, I in B.items(): + lmask = (I["lats"] > -68.1) & (I["lats"] < -67.7) + plt.plot(I["x"][lmask] - I["x"].iloc[0], I["lats"][lmask], ".", markersize=0.2) -#plt.xlim(, -67.8) +# plt.xlim(, -67.8) -plt.xlabel('x') -plt.ylabel('lat') -#F.save_light(path= plot_path, name='B01_ALT03_'+track_name+'_tracks_check_lat_x') +plt.xlabel("x") +plt.ylabel("lat") +# F.save_light(path= plot_path, name='B01_ALT03_'+track_name+'_tracks_check_lat_x') # %% -F = M.figure_axis_xy(4, 3, view_scale = 0.7) -for k,I in B.items(): - plt.plot( I['delta_time'] , I['lats'], '.' , markersize = 0.3) +F = M.figure_axis_xy(4, 3, view_scale=0.7) +for k, I in B.items(): + plt.plot(I["delta_time"], I["lats"], ".", markersize=0.3) -plt.xlabel('delta time') -plt.ylabel('lat') -F.save_light(path= plot_path, name='B01_ALT03_'+track_name+'_tracks_check_time_lat') +plt.xlabel("delta time") +plt.ylabel("lat") +F.save_light(path=plot_path, name="B01_ALT03_" + track_name + "_tracks_check_time_lat") # %% -F = M.figure_axis_xy(4, 3, view_scale = 0.7) +F = M.figure_axis_xy(4, 3, view_scale=0.7) -for k,I in B.items(): - plt.plot( I['delta_time'] , I['x'], '.' , markersize = 0.3) +for k, I in B.items(): + plt.plot(I["delta_time"], I["x"], ".", markersize=0.3) -plt.xlabel('delta time') -plt.ylabel('x') +plt.xlabel("delta time") +plt.ylabel("x") -F.save_light(path= plot_path, name='B01_ALT03_'+track_name+'_tracks_check_time_x') +F.save_light(path=plot_path, name="B01_ALT03_" + track_name + "_tracks_check_time_x") # %% ##### 1.) derive common axis for beams and filter out low density area at the beginning -print('filter out low density area at the beginning') +print("filter out low density area at the beginning") + def derive_axis_and_boundaries(key): - #key = 'gt3r' + # key = 'gt3r' print(key) - #T2 # = regrid.derive_axis(B[key], lat_lims).sort_values('dist') - T2 = B[key]#['x'] + # T2 # = regrid.derive_axis(B[key], lat_lims).sort_values('dist') + T2 = B[key] # ['x'] + + x0 = get_better_lower_boundary(Lmeter_large, np.array(T2["x"])) - x0 = get_better_lower_boundary(Lmeter_large, np.array(T2['x'])) + print( + "cut off ", + 100 * (1 - T2[T2["x"] > x0].shape[0] / T2.shape[0]), + "% off all data points at the beginning", + ) + T2 = T2[T2["x"] > x0] # cut off low density start - print( 'cut off ' , 100 * (1 - T2[T2['x'] > x0].shape[0]/T2.shape[0]) , '% off all data points at the beginning' ) - T2 = T2[T2['x'] >x0] # cut off low density start + return key, T2, [T2["x"].min(), T2["x"].max()] - return key, T2, [T2['x'].min(), T2['x'].max()] def get_better_lower_boundary(Lmeter_large, dd): # T2 = regrid.derive_axis(B[key], lat_lims) - #dd = np.array(T2['x']) - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= True) + # dd = np.array(T2['x']) + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=True + ) def get_density(sti): - return sti[0], sum( (sti[0] <= dd) & (dd < sti[-1]) ) / Lmeter_large + return sti[0], sum((sti[0] <= dd) & (dd < sti[-1])) / Lmeter_large - with futures.ThreadPoolExecutor(max_workers= Nworkers_process) as executor_sub: - #var_list = np.array(list(map(get_density, stencil_iter))) + with futures.ThreadPoolExecutor(max_workers=Nworkers_process) as executor_sub: + # var_list = np.array(list(map(get_density, stencil_iter))) var_list = np.array(list(executor_sub.map(get_density, stencil_iter))) var_list = np.array(var_list) - #var_list[:,0] = np.random.rand(10) - #sort var_list - var_list = var_list[var_list[:,0].argsort(), :] - #print(var_list) - if sum(var_list[:,1] > minium_photon_density) > 1: - first_stencil = next((i for i, j in enumerate(var_list[:,1] > minium_photon_density) if j), None) #- 1 - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) + # var_list[:,0] = np.random.rand(10) + # sort var_list + var_list = var_list[var_list[:, 0].argsort(), :] + # print(var_list) + if sum(var_list[:, 1] > minium_photon_density) > 1: + first_stencil = next( + (i for i, j in enumerate(var_list[:, 1] > minium_photon_density) if j), None + ) # - 1 + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=False + ) return stencil_iter[0, first_stencil] else: - #first_stencil = next((i for i, j in enumerate(var_list[:,1] > 0) if j), None)# - 1 - print('no sufficient photon density found. return short stencil') + # first_stencil = next((i for i, j in enumerate(var_list[:,1] > 0) if j), None)# - 1 + print("no sufficient photon density found. return short stencil") # first_stencil= len(var_list[:,1]) -1 # stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) # return stencil_iter[0, first_stencil] # #print(first_stencil) - return var_list[-1,0]#[-1,0] + return var_list[-1, 0] # [-1,0] with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: - A = list( executor.map(derive_axis_and_boundaries, all_beams) ) + A = list(executor.map(derive_axis_and_boundaries, all_beams)) # %% -B2 = dict() -dist_list = np.array([np.nan, np.nan]) +B2 = dict() +dist_list = np.array([np.nan, np.nan]) for I in A: - k = I[0] - B2[k] = I[1] - #B2[k]['dist'] = B2[k]['x'] - dist_list = np.vstack([dist_list,I[2] ]) + k = I[0] + B2[k] = I[1] + # B2[k]['dist'] = B2[k]['x'] + dist_list = np.vstack([dist_list, I[2]]) del A -track_dist_bounds = [ np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0) ] +track_dist_bounds = [np.nanmin(dist_list[:, 0], 0), np.nanmax(dist_list[:, 1], 0)] # for testing: -#ts_s = np.copy(track_dist_bounds) +# ts_s = np.copy(track_dist_bounds) # %% -xscale= 1e3 -F= M.figure_axis_xy(5, 3, view_scale= 0.6) -for k,I in B2.items(): - plt.plot( I['x']/xscale , I['across_track_distance']/xscale , '.' , markersize = 0.3) - #plt.xlim(3e6, 3.25e6) +xscale = 1e3 +F = M.figure_axis_xy(5, 3, view_scale=0.6) +for k, I in B2.items(): + plt.plot(I["x"] / xscale, I["across_track_distance"] / xscale, ".", markersize=0.3) + # plt.xlim(3e6, 3.25e6) for k in high_beams: Ii = B2[k].iloc[0] - plt.text(Ii.x/xscale+ 5, Ii.across_track_distance/xscale , str(Ii[[ 'lats', 'lons'] ]).split('Name')[0] ) + plt.text( + Ii.x / xscale + 5, + Ii.across_track_distance / xscale, + str(Ii[["lats", "lons"]]).split("Name")[0], + ) Ii = B2[k].iloc[-1] - plt.text(Ii.x/xscale+ 5, Ii.across_track_distance/xscale , str(Ii[[ 'lats', 'lons'] ]).split('Name')[0], ha ='right' ) - -F.ax.axvline(track_dist_bounds[0]/xscale, color='gray', zorder= 2) -F.ax.axvline(track_dist_bounds[1]/xscale, color='gray', zorder= 2) -F.ax.axhline(0, color='gray', zorder= 2) - -plt.title('B01 filter and regrid | ' + track_name +'\npoleward '+str(track_poleward)+' \n \n', loc='left') -plt.xlabel('along track distance (km)') -plt.ylabel('across track distance (km)') - -F.save_light(path= plot_path +'../', name='B01_ALT03_'+track_name+'_tracks_all') + plt.text( + Ii.x / xscale + 5, + Ii.across_track_distance / xscale, + str(Ii[["lats", "lons"]]).split("Name")[0], + ha="right", + ) + +F.ax.axvline(track_dist_bounds[0] / xscale, color="gray", zorder=2) +F.ax.axvline(track_dist_bounds[1] / xscale, color="gray", zorder=2) +F.ax.axhline(0, color="gray", zorder=2) + +plt.title( + "B01 filter and regrid | " + + track_name + + "\npoleward " + + str(track_poleward) + + " \n \n", + loc="left", +) +plt.xlabel("along track distance (km)") +plt.ylabel("across track distance (km)") + +F.save_light(path=plot_path + "../", name="B01_ALT03_" + track_name + "_tracks_all") # %% # for testing -#track_dist_bounds[1] = track_dist_bounds[0] + (track_dist_bounds[1] - track_dist_bounds[0])/20 -#track_dist_bounds = ts_s[0] ,ts_s[0] + (ts_s[1] - ts_s[0]) /6 +# track_dist_bounds[1] = track_dist_bounds[0] + (track_dist_bounds[1] - track_dist_bounds[0])/20 +# track_dist_bounds = ts_s[0] ,ts_s[0] + (ts_s[1] - ts_s[0]) /6 # %% -#imp.reload(regrid) +# imp.reload(regrid) ##### 2.) regridding and averaging -print('regrid') +print("regrid") + + def regridding_wrapper(I): key, Ti = I - print(key, Ti.shape,2* Ti.shape[0]/Lmeter) - stencil_iter = create_chunk_boundaries_unit_lengths( Lmeter, track_dist_bounds, iter_flag=True ) - with futures.ThreadPoolExecutor(max_workers= Nworkers) as executor_sub: - Bi = regrid.get_stencil_stats( Ti, stencil_iter, 'heights_c', 'x' , stancil_width= Lmeter/2, Nphoton_min=Nphoton_min, map_func = executor_sub.map) - #B3[key] = - print(key, 'done') + print(key, Ti.shape, 2 * Ti.shape[0] / Lmeter) + stencil_iter = create_chunk_boundaries_unit_lengths( + Lmeter, track_dist_bounds, iter_flag=True + ) + with futures.ThreadPoolExecutor(max_workers=Nworkers) as executor_sub: + Bi = regrid.get_stencil_stats( + Ti, + stencil_iter, + "heights_c", + "x", + stancil_width=Lmeter / 2, + Nphoton_min=Nphoton_min, + map_func=executor_sub.map, + ) + # B3[key] = + print(key, "done") return key, Bi with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: - B3 = dict( executor.map(regridding_wrapper, B2.items() ) ) + B3 = dict(executor.map(regridding_wrapper, B2.items())) # %% ---define start and end position and same in Json file -#I = B3['gt2l'].copy() +# I = B3['gt2l'].copy() D_info = dict() -for k,I in B3.items(): +for k, I in B3.items(): # reset x coordinate - I['median_dist'] = I['median_x'] - track_dist_bounds[0] #- Lmeter/2 - I['dist'] = I['x'] - track_dist_bounds[0] #- Lmeter/2 - #I['index'] = I['x'] + I["median_dist"] = I["median_x"] - track_dist_bounds[0] # - Lmeter/2 + I["dist"] = I["x"] - track_dist_bounds[0] # - Lmeter/2 + # I['index'] = I['x'] # rename y coordinate - I = I.rename(columns={'across_track_distance': 'y'}) + I = I.rename(columns={"across_track_distance": "y"}) # find starting and end position - Di_s = dict(I[I['seg_ID'] == I['seg_ID'].iloc[0] ].mean()[['lons', 'lats', 'seg_ID', 'delta_time']]) - Di_s['across_track_distance_0'] =track_dist_bounds[0] - - Di_e = dict(I[I['seg_ID'] == I['seg_ID'].iloc[-1] ].mean()[['lons', 'lats', 'seg_ID', 'delta_time']]) - Di_e['across_track_distance_0'] =track_dist_bounds[0] - - D_info[k] = {'start':Di_s, 'end':Di_e , 'poleward': str(track_poleward) } + Di_s = dict( + I[I["seg_ID"] == I["seg_ID"].iloc[0]].mean()[ + ["lons", "lats", "seg_ID", "delta_time"] + ] + ) + Di_s["across_track_distance_0"] = track_dist_bounds[0] + + Di_e = dict( + I[I["seg_ID"] == I["seg_ID"].iloc[-1]].mean()[ + ["lons", "lats", "seg_ID", "delta_time"] + ] + ) + Di_e["across_track_distance_0"] = track_dist_bounds[0] + + D_info[k] = {"start": Di_s, "end": Di_e, "poleward": str(track_poleward)} # reorder indexes - column_names = ['index', 'x', 'y', 'median_x', 'lons', 'lats' ,'heights_c_weighted_mean', 'heights_c_median', 'heights_c_std', 'N_photos', ] - vars_ad = set(list(I[I['seg_ID'] == I['seg_ID'].iloc[0] ].mean().index)) - set(column_names) - I = I.reindex(columns=column_names + list(vars_ad)) + column_names = [ + "index", + "x", + "y", + "median_x", + "lons", + "lats", + "heights_c_weighted_mean", + "heights_c_median", + "heights_c_std", + "N_photos", + ] + vars_ad = set(list(I[I["seg_ID"] == I["seg_ID"].iloc[0]].mean().index)) - set( + column_names + ) + I = I.reindex(columns=column_names + list(vars_ad)) B3[k] = I # save Json -MT.json_save(track_name + '_B01_stats',save_path, D_info, verbose= True ) +MT.json_save(track_name + "_B01_stats", save_path, D_info, verbose=True) # %% saving data -io.save_pandas_table(B2, track_name + '_B01_regridded' , save_path) # all photos but heights adjusted and with distance coordinate -io.save_pandas_table(B3, track_name + '_B01_binned' , save_path) # regridding heights +io.save_pandas_table( + B2, track_name + "_B01_regridded", save_path +) # all photos but heights adjusted and with distance coordinate +io.save_pandas_table(B3, track_name + "_B01_binned", save_path) # regridding heights # %% plotting just for checking -key = 'gt2r' +key = "gt2r" if plot_flag: MT.mkdirs_r(plot_path) Ti2 = B3[key] - T2 = B2[key] + T2 = B2[key] dl = 4000 - latlims = (Ti2['x'].iloc[0] , Ti2['x'].iloc[-1] ) - #chunk_list = np.arange(latlims[0],latlims[1], dl ) - #chunk_list = sample( list(np.arange(latlims[0],latlims[1],dl )[0:80]) ,10) - chunk_list = np.arange(latlims[0],latlims[1], dl )[::10] - chunk_list = np.append( chunk_list, latlims[1]- dl-1) + latlims = (Ti2["x"].iloc[0], Ti2["x"].iloc[-1]) + # chunk_list = np.arange(latlims[0],latlims[1], dl ) + # chunk_list = sample( list(np.arange(latlims[0],latlims[1],dl )[0:80]) ,10) + chunk_list = np.arange(latlims[0], latlims[1], dl)[::10] + chunk_list = np.append(chunk_list, latlims[1] - dl - 1) for ll in chunk_list: F = M.figure_axis_xy(7, 3, view_scale=0.8) - plt.plot( T2['x'], T2['heights_c'], 'k.', markersize= 0.5, alpha =0.8 ) - #plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') - - plt.plot(Ti2['x'], Ti2['heights_c_weighted_mean'] +1, '.-', color='darkblue', linewidth=0.5, markersize=2,alpha=0.9, label='x-gauss weighted mean +1') - plt.plot(Ti2['x'], Ti2['heights_c_median'], 'r.-', linewidth=0.5, markersize=2,alpha=0.9, label='median') - - #plt.plot(Ti2['x'], Ti2['heights_c_mode']-1, 'g.-', linewidth=0.5, markersize=2,alpha=0.9, label='mode - 1') - - plt.plot(Ti2['x'], Ti2['heights_c_std'] - 1.8, 'k-', linewidth=0.5,alpha=1) - plt.fill_between( Ti2['x'], Ti2['heights_c_std'] -1.8 , y2=-1.8, color='gray',alpha=1) + plt.plot(T2["x"], T2["heights_c"], "k.", markersize=0.5, alpha=0.8) + # plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') + + plt.plot( + Ti2["x"], + Ti2["heights_c_weighted_mean"] + 1, + ".-", + color="darkblue", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="x-gauss weighted mean +1", + ) + plt.plot( + Ti2["x"], + Ti2["heights_c_median"], + "r.-", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="median", + ) + + # plt.plot(Ti2['x'], Ti2['heights_c_mode']-1, 'g.-', linewidth=0.5, markersize=2,alpha=0.9, label='mode - 1') + + plt.plot(Ti2["x"], Ti2["heights_c_std"] - 1.8, "k-", linewidth=0.5, alpha=1) + plt.fill_between( + Ti2["x"], Ti2["heights_c_std"] - 1.8, y2=-1.8, color="gray", alpha=1 + ) # plt.plot( ALT03['delta_time'], ALT03['heights_c'], 'k.', markersize= 0.3, alpha =0.2 ) # plt.plot(ALT07['time']['delta_time'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') plt.legend(loc=1) - plt.xlim(ll, ll+dl) + plt.xlim(ll, ll + dl) plt.ylim(-4, 4) - plt.xlabel('Meters from the Sea Ice Edge') - plt.ylabel('Height Anomalie (meters)') - F.ax.axhline(y =-1.8, color='black', linewidth=0.5) - F.save_light(path= plot_path, name='ALT03_filt_compare'+ str(ll)) + plt.xlabel("Meters from the Sea Ice Edge") + plt.ylabel("Height Anomalie (meters)") + F.ax.axhline(y=-1.8, color="black", linewidth=0.5) + F.save_light(path=plot_path, name="ALT03_filt_compare" + str(ll)) diff --git a/analysis/B01_filter_regrid_single.py b/analysis/B01_filter_regrid_single.py index b2002300..2f3017bf 100644 --- a/analysis/B01_filter_regrid_single.py +++ b/analysis/B01_filter_regrid_single.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,15 +9,15 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -#import m_general as M -#import m_tools as MT +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -36,79 +37,97 @@ from random import sample import imp -#import s3fs +# import s3fs # Python reader based on Pandas. Other reader examples available in readers.py -track_name= 'ATL03_20190515060436_07170312_002_02' -load_path = base_path + 'data/data1/' -load_file = load_path + 'processed_'+track_name+'.h5' - -track_name= 'ATL03_20191215230028_12220512_004_01' -track_name= 'ATL03_20210414065545_03121112_004_01' -load_path = base_path + 'data/data4/' -load_file = load_path + 'processed_'+track_name+'.h5' +track_name = "ATL03_20190515060436_07170312_002_02" +load_path = base_path + "data/data1/" +load_file = load_path + "processed_" + track_name + ".h5" + +track_name = "ATL03_20191215230028_12220512_004_01" +track_name = "ATL03_20210414065545_03121112_004_01" +load_path = base_path + "data/data4/" +load_file = load_path + "processed_" + track_name + ".h5" # %% -track_name= 'ATL03_20200520190502_08440701_003_01' -load_path = base_path + 'modules/read-ICESat-2/scripts/2020.05.20/' -load_file = load_path +track_name+'.h5' +track_name = "ATL03_20200520190502_08440701_003_01" +load_path = base_path + "modules/read-ICESat-2/scripts/2020.05.20/" +load_file = load_path + track_name + ".h5" # %% # test which beams exist: -all_beams = ['gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r'] -low_beams = ['gt1l', 'gt2l', 'gt3l'] -high_beams = ['gt1r', 'gt2r', 'gt3r'] +all_beams = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] +low_beams = ["gt1l", "gt2l", "gt3l"] +high_beams = ["gt1r", "gt2r", "gt3r"] + +f = h5py.File(load_file, "r") +beams = [b if b in f.keys() else None for b in all_beams] -f = h5py.File(load_file, 'r') -beams = [b if b in f.keys() else None for b in all_beams] -def correct_heights(T03, T03c, coord = 'delta_time'): - T03['heights_c']= T03['heights'] - np.interp( T03[coord],T03c[coord], T03c['dem_h'] ) +def correct_heights(T03, T03c, coord="delta_time"): + T03["heights_c"] = T03["heights"] - np.interp( + T03[coord], T03c[coord], T03c["dem_h"] + ) return T03 + # Load fata and apply height corrections # This needs version 2 of the ALT 03 dataset -hist= 'Beam stats' -B= dict() +hist = "Beam stats" +B = dict() for k in beams: - T = io.getATL03_beam(load_file, beam= k) + T = io.getATL03_beam(load_file, beam=k) ho = k - ho = MT.add_line_var(ho, 'size', str(T.shape[0])) - ho = MT.add_line_var(ho, 'by confidence levels:' + str(np.arange(0, 5)), [ (T['signal_confidence'] == i).sum() for i in np.arange(0, 5) ]) + ho = MT.add_line_var(ho, "size", str(T.shape[0])) + ho = MT.add_line_var( + ho, + "by confidence levels:" + str(np.arange(0, 5)), + [(T["signal_confidence"] == i).sum() for i in np.arange(0, 5)], + ) # filter: - Tsel = T[(T['signal_confidence']>2) & (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] + Tsel = T[ + (T["signal_confidence"] > 2) & (T["heights"] < 100) & (T["heights"] > -100) + ] # & (T['delta_time']>5) & (T['delta_time']<24) ] if len(Tsel) == 0: - ho = MT.add_line_var(ho, 'no photons found', '') - Tsel= T[(T['signal_confidence'] ==-1 ) & (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] + ho = MT.add_line_var(ho, "no photons found", "") + Tsel = T[ + (T["signal_confidence"] == -1) + & (T["heights"] < 100) + & (T["heights"] > -100) + ] # & (T['delta_time']>5) & (T['delta_time']<24) ] + Tsel_c = io.getATL03_height_correction(load_file) + Tsel_c = Tsel_c[Tsel_c["dem_h"] < 1e5] # cute out weird references + B[k] = correct_heights(Tsel, Tsel_c) - Tsel_c = io.getATL03_height_correction(load_file) - Tsel_c = Tsel_c[Tsel_c['dem_h'] < 1e5] # cute out weird references - B[k] = correct_heights(Tsel, Tsel_c) - - ho = MT.add_line_var(ho, 'selected size', str(Tsel.shape[0])) - ho = MT.add_line_var(ho, 'final size ', str(Tsel_c.shape[0])) + ho = MT.add_line_var(ho, "selected size", str(Tsel.shape[0])) + ho = MT.add_line_var(ho, "final size ", str(Tsel_c.shape[0])) print(ho) - hist = MT.write_log(hist, ho) + hist = MT.write_log(hist, ho) # %% -plt.plot( B[k].lats , B[k].heights_c, '.') +plt.plot(B[k].lats, B[k].heights_c, ".") # %% + def track_type(T): """ Returns if track acending or desending T is a pandas table """ - #T = B[k] - #T = B[beams_list[0]] - return (T['lats'].iloc[T['delta_time'].argmax()] - T['lats'].iloc[T['delta_time'].argmin()] ) < 0 + # T = B[k] + # T = B[beams_list[0]] + return ( + T["lats"].iloc[T["delta_time"].argmax()] + - T["lats"].iloc[T["delta_time"].argmin()] + ) < 0 + def lat_min_max(B, beams_list): """ @@ -120,28 +139,29 @@ def lat_min_max(B, beams_list): returns: min_lat, max_lat, accent min and max latitudes of the beams, (True/False) True if the track is accending """ - #B, beams_list = B , high_beams - accent = track_type( B[beams_list[0]] ) + # B, beams_list = B , high_beams + accent = track_type(B[beams_list[0]]) - if B[beams_list[0]]['lats'].iloc[0] < 0: - hemis = 'SH' + if B[beams_list[0]]["lats"].iloc[0] < 0: + hemis = "SH" else: - hemis = 'NH' + hemis = "NH" - track_lat_mins, track_lat_maxs= list(), list() + track_lat_mins, track_lat_maxs = list(), list() for k in beams_list: - track_lat_mins.append( B[k]['lats'].min() ) - track_lat_maxs.append( B[k]['lats'].max() ) + track_lat_mins.append(B[k]["lats"].min()) + track_lat_maxs.append(B[k]["lats"].max()) - if hemis == 'SH': - return max(track_lat_maxs) , min(track_lat_mins), accent + if hemis == "SH": + return max(track_lat_maxs), min(track_lat_mins), accent else: return min(track_lat_mins), max(track_lat_maxs), accent # %% -def derive_axis(TT, lat_lims = None): + +def derive_axis(TT, lat_lims=None): """ returns TT distance along track 'dist' in meters input: @@ -150,43 +170,43 @@ def derive_axis(TT, lat_lims = None): returns: TT with x,y,dist and order by dist """ - #TT, lat_lims = B[key], lat_lims_high + # TT, lat_lims = B[key], lat_lims_high # derive distances in meters - r_e= 6.3710E+6 - dy= r_e*2*np.pi/360.0 - #deglon_in_m= np.cos(T2['lats']*np.pi/180.0)*dy + r_e = 6.3710e6 + dy = r_e * 2 * np.pi / 360.0 + # deglon_in_m= np.cos(T2['lats']*np.pi/180.0)*dy # either use position of the 1st photon or use defined start latitude if lat_lims is None: - TT['y']=(TT['lats'].max() - TT['lats']) *dy + TT["y"] = (TT["lats"].max() - TT["lats"]) * dy else: - TT['y']=(lat_lims[0] - TT['lats']) *dy - - #TT['y'] = (TT['lats']) *dy + TT["y"] = (lat_lims[0] - TT["lats"]) * dy + # TT['y'] = (TT['lats']) *dy - if (lat_lims[2] == True): + if lat_lims[2] == True: # accending track - lon_min = TT['lons'].max() + lon_min = TT["lons"].max() else: # decending track - lon_min = TT['lons'].min() + lon_min = TT["lons"].min() - #print(lon_min) - TT['x'] = (TT['lons'] - lon_min) * np.cos( TT['lats']*np.pi/180.0 ) * dy - #TT['x'] = (TT['lons'] ) * np.cos( TT['lats']*np.pi/180.0 ) * dy - TT['dist'] = np.sqrt(TT['x']**2 + TT['y']**2) + # print(lon_min) + TT["x"] = (TT["lons"] - lon_min) * np.cos(TT["lats"] * np.pi / 180.0) * dy + # TT['x'] = (TT['lons'] ) * np.cos( TT['lats']*np.pi/180.0 ) * dy + TT["dist"] = np.sqrt(TT["x"] ** 2 + TT["y"] ** 2) # set 1st dist to 0, not used if global limits are used if lat_lims is None: - TT['dist']= TT['dist']- TT['dist'].min() + TT["dist"] = TT["dist"] - TT["dist"].min() else: - TT['dist']= TT['dist']#- lat_lims[0] + TT["dist"] = TT["dist"] # - lat_lims[0] - TT=TT.sort_values(by='dist') + TT = TT.sort_values(by="dist") return TT -def reduce_to_height_distance(TT, key, dx=1, lat_lims = None): + +def reduce_to_height_distance(TT, key, dx=1, lat_lims=None): """ interpolates key (photos heights) to regular grid using 'dist' in pandas table TT. dx is the interpolation interval @@ -197,17 +217,18 @@ def reduce_to_height_distance(TT, key, dx=1, lat_lims = None): x1, y1 position, height """ from scipy.interpolate import interp1d + if type(dx) is np.ndarray: x1 = dx else: - x1 = np.arange(0,TT['dist'].max(), dx) - y1 = np.interp(x1, TT['dist'], TT[key] ) + x1 = np.arange(0, TT["dist"].max(), dx) + y1 = np.interp(x1, TT["dist"], TT[key]) return x1, y1 + # this is not need anymore def poly_correct(x, y, poly_order=7, plot_flag=False): - """ subtracts a fitted polynom to y inputs: @@ -217,12 +238,22 @@ def poly_correct(x, y, poly_order=7, plot_flag=False): returns y' y - polynom fit """ - z = np.polyfit(x , y , poly_order) + z = np.polyfit(x, y, poly_order) p = np.poly1d(z) if plot_flag: - plt.plot(x,y, '.', markersize=0.2,) - plt.plot(x, p(x), '-', markersize=0.2,) - #return z + plt.plot( + x, + y, + ".", + markersize=0.2, + ) + plt.plot( + x, + p(x), + "-", + markersize=0.2, + ) + # return z return y - p(x) @@ -233,51 +264,52 @@ def poly_correct(x, y, poly_order=7, plot_flag=False): ##### 1.) derive common axis for beams # %% 1st all stong beams -B2=dict() -colors = iter(['red','blue','orange','green','black','yellow']) -dist_list =np.array([np.nan, np.nan]) +B2 = dict() +colors = iter(["red", "blue", "orange", "green", "black", "yellow"]) +dist_list = np.array([np.nan, np.nan]) for key in high_beams: - T2 = derive_axis(B[key], lat_lims_high) + T2 = derive_axis(B[key], lat_lims_high) # the ends of the tracks are about 600m appart.. i don't know why .. # this should have no influence on the distrance along the track - #plt.plot( list(T2['x'][-200:]), list(T2['y'][-200:]), c = next(colors)) - #plt.plot( list(T2['x'][15000:15005]), list(T2['y'][15000:15005]), '.', c = next(colors)) - #plt.plot( list(T2['lons'][:2]), list(T2['lats'][:2]), c = next(colors)) - #plt.axis('equal') + # plt.plot( list(T2['x'][-200:]), list(T2['y'][-200:]), c = next(colors)) + # plt.plot( list(T2['x'][15000:15005]), list(T2['y'][15000:15005]), '.', c = next(colors)) + # plt.plot( list(T2['lons'][:2]), list(T2['lats'][:2]), c = next(colors)) + # plt.axis('equal') # depreciated - #T2['heights_adjusted'] = poly_correct(T2['dist'], T2['heights']) # 'dist' is length from start position + # T2['heights_adjusted'] = poly_correct(T2['dist'], T2['heights']) # 'dist' is length from start position # rolling 100 photon window. May have to be replaced by a window of fixed length. # if rollin window is replaced by fixed lengrh the following step is obsolute. - #T2['heights_smth'] = T2['heights_adjusted'].rolling(100).median() - B2[key] = T2 - dist_list = np.vstack([ dist_list, [T2['dist'].min(), T2['dist'].max()] ]) + # T2['heights_smth'] = T2['heights_adjusted'].rolling(100).median() + B2[key] = T2 + dist_list = np.vstack([dist_list, [T2["dist"].min(), T2["dist"].max()]]) ##### 2.) regridding and averaging # %% define functions -def get_mode(y, bins = np.arange(-5,5, 0.1)): + +def get_mode(y, bins=np.arange(-5, 5, 0.1)): "returns modes of histogram of y defined by bins" - hist, xbin = np.histogram(y, bins = bins ) + hist, xbin = np.histogram(y, bins=bins) return xbin[hist.argmax()] + def weighted_mean(x_rel, y): "returns the gaussian weighted mean for stencil" def weight_fnk(x): "returns gaussian weight given the distance to the center x" - return np.exp(- (x/.5)**2 ) + return np.exp(-((x / 0.5) ** 2)) w = weight_fnk(x_rel) - return (w*y).sum()/w.sum() + return (w * y).sum() / w.sum() # this function is applied to beam: -def get_stencil_stats(T2, stencil_iter, key , Nphoton_min = 5): - +def get_stencil_stats(T2, stencil_iter, key, Nphoton_min=5): """ T2 pd.DAtaframe with beam data needs at least 'dist' and key stencil_iter iterable that constains the stancil boundaries and center [left boundary, center, right boundary] @@ -296,46 +328,46 @@ def get_stencil_stats(T2, stencil_iter, key , Nphoton_min = 5): """ - x_data = T2['dist'] + x_data = T2["dist"] y_data = T2[key] + # apply this funcion to each stancil def calc_stencil_stats(istencil): "returns stats per stencil" - i_mask=(x_data >= istencil[0]) & (x_data < istencil[2]) + i_mask = (x_data >= istencil[0]) & (x_data < istencil[2]) Nphoton = i_mask.sum() if Nphoton < Nphoton_min: Tmedian = T2[i_mask].median() - Tmedian[key+ '_weighted_mean'] = np.nan - #Tmedian[key+ '_mode'] = np.nan - Tmedian['N_photos'] = i_mask.sum() - Tmedian[key+ '_std'] = np.nan + Tmedian[key + "_weighted_mean"] = np.nan + # Tmedian[key+ '_mode'] = np.nan + Tmedian["N_photos"] = i_mask.sum() + Tmedian[key + "_std"] = np.nan return istencil[1], Tmedian - Tmedian = T2[i_mask].median() - x_rel = (x_data[i_mask] - istencil[1])/(L/2) - y = y_data[i_mask] + x_rel = (x_data[i_mask] - istencil[1]) / (L / 2) + y = y_data[i_mask] - Tmedian[key+ '_weighted_mean'] = weighted_mean(x_rel, y) - #Tmedian[key+ '_mode'] = get_mode(y) - Tmedian['N_photos'] = i_mask.sum() - Tmedian[key+ '_std'] = y.std() + Tmedian[key + "_weighted_mean"] = weighted_mean(x_rel, y) + # Tmedian[key+ '_mode'] = get_mode(y) + Tmedian["N_photos"] = i_mask.sum() + Tmedian[key + "_std"] = y.std() return istencil[1], Tmedian # apply func to all stancils - D_filt = dict(map(calc_stencil_stats,stencil_iter)) + D_filt = dict(map(calc_stencil_stats, stencil_iter)) - DF_filt = pd.DataFrame.from_dict(D_filt, orient='index') - DF_filt = DF_filt.rename(columns={key: key+'_median', 'dist': 'median_dist'}) - DF_filt['dist'] = DF_filt.index + DF_filt = pd.DataFrame.from_dict(D_filt, orient="index") + DF_filt = DF_filt.rename(columns={key: key + "_median", "dist": "median_dist"}) + DF_filt["dist"] = DF_filt.index DF_filt = DF_filt.reset_index() return DF_filt @@ -343,102 +375,139 @@ def calc_stencil_stats(istencil): # %% old version # define common dist_grid: -#dx= 5 # 2 * resolution in meters, datapoint +-dx are used to take the mean -#dist_grid = np.arange( np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0), dx ) +# dx= 5 # 2 * resolution in meters, datapoint +-dx are used to take the mean +# dist_grid = np.arange( np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0), dx ) + # derive bin means def bin_means(T2, dist_grid): - dF_mean = pd.DataFrame(index =T2.columns) - ilim = int(len(dist_grid)) - N_i = list() + dF_mean = pd.DataFrame(index=T2.columns) + ilim = int(len(dist_grid)) + N_i = list() - for i in np.arange(1,ilim-1, 1): - if i % 5000 ==0: + for i in np.arange(1, ilim - 1, 1): + if i % 5000 == 0: print(i) - i_mask=(T2['dist'] >= dist_grid[i-1]) & (T2['dist'] < dist_grid[i+1]) - #if ( (T2['dist'] >= dist_grid[i-1]) & (T2['dist'] < dist_grid[i+1]) ).sum() > 0: + i_mask = (T2["dist"] >= dist_grid[i - 1]) & (T2["dist"] < dist_grid[i + 1]) + # if ( (T2['dist'] >= dist_grid[i-1]) & (T2['dist'] < dist_grid[i+1]) ).sum() > 0: dF_mean[i] = T2[i_mask].mean() - #dF_median[i] = T2[i_mask].median() + # dF_median[i] = T2[i_mask].median() N_i.append(i_mask.sum()) - dF_mean = dF_mean.T - dF_mean['N_photos'] = N_i - dF_mean['dist'] = dist_grid[np.arange(1,ilim-1, 1)] + dF_mean = dF_mean.T + dF_mean["N_photos"] = N_i + dF_mean["dist"] = dist_grid[np.arange(1, ilim - 1, 1)] return dF_mean + # %% estimating wave length for given period T = 5 -g= 9.81 -lam = g *T**2 / (2 * np.pi) +g = 9.81 +lam = g * T**2 / (2 * np.pi) print(lam) # define parameters: -L = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 -Nphoton_min = 5 # mininum need photons per stancil to return results +L = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 +Nphoton_min = 5 # mininum need photons per stancil to return results -G=dict() +G = dict() B3 = dict() -for key,Ti in B2.items(): +for key, Ti in B2.items(): print(key) - stencil_iter = create_chunk_boundaries_unit_lengths( L, [ np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0) ], iter_flag=True ) + stencil_iter = create_chunk_boundaries_unit_lengths( + L, + [np.nanmin(dist_list[:, 0], 0), np.nanmax(dist_list[:, 1], 0)], + iter_flag=True, + ) - Ti2 = get_stencil_stats(Ti, stencil_iter, 'heights_c', Nphoton_min=Nphoton_min) - Ti2['heights_c_median'][ np.isnan(Ti2['heights_c_std']) ]= np.nan # replace median calculation with nans + Ti2 = get_stencil_stats(Ti, stencil_iter, "heights_c", Nphoton_min=Nphoton_min) + Ti2["heights_c_median"][ + np.isnan(Ti2["heights_c_std"]) + ] = np.nan # replace median calculation with nans - B3[key] =Ti2 # store in dict + B3[key] = Ti2 # store in dict # % save relevant data as xarray # G[key] =xr.DataArray(Ti2['heights_c_median'], coords={'dist': Ti2['dist']}, dims='dist', name=key) - print(key, 'done') + print(key, "done") # %% saving data # Gnew = xr.merge(G.values()) # Gnew.to_netcdf(load_path+'/'+track_name +'_filtered_photon_heights.nc') -io.save_pandas_table(B2, track_name + '_B01_corrected' , load_path) # all photos but heights adjusted and with distance coordinate -io.save_pandas_table(B3, track_name + '_B01_binned' , load_path) # regridding heights +io.save_pandas_table( + B2, track_name + "_B01_corrected", load_path +) # all photos but heights adjusted and with distance coordinate +io.save_pandas_table(B3, track_name + "_B01_binned", load_path) # regridding heights # %% plotting just for checking plot_flag = True if plot_flag: import m_tools_ph3 as MT - plot_path = base_path+'plots/B01_regridding/'+track_name+'/' - MT.mkdirs_r(plot_path) + plot_path = base_path + "plots/B01_regridding/" + track_name + "/" + MT.mkdirs_r(plot_path) Ti2 = B3[key] - T2 = B2[key] + T2 = B2[key] dl = 4000 - latlims = (Ti2['dist'].iloc[0] , Ti2['dist'].iloc[-1] ) - for ll in sample( list(np.arange(latlims[0],latlims[1],dl )[0:80]) ,10): + latlims = (Ti2["dist"].iloc[0], Ti2["dist"].iloc[-1]) + for ll in sample(list(np.arange(latlims[0], latlims[1], dl)[0:80]), 10): F = M.figure_axis_xy(7, 3, view_scale=0.8) - plt.plot( T2['dist'], T2['heights_c'], 'k.', markersize= 0.5, alpha =0.8 ) - #plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') - - plt.plot(Ti2['dist'], Ti2['heights_c_weighted_mean'] +1, '.-', color='darkblue', linewidth=0.5, markersize=2,alpha=0.9, label='x-gauss weighted mean +1') - plt.plot(Ti2['dist'], Ti2['heights_c_median'], 'r.-', linewidth=0.5, markersize=2,alpha=0.9, label='median') - - plt.plot(Ti2['dist'], Ti2['heights_c_mode']-1, 'g.-', linewidth=0.5, markersize=2,alpha=0.9, label='mode - 1') - - plt.plot(Ti2['dist'], Ti2['heights_c_std'] - 1.8, 'k-', linewidth=0.5,alpha=1) - plt.fill_between( Ti2['dist'], Ti2['heights_c_std'] -1.8 , y2=-1.8, color='gray',alpha=1) + plt.plot(T2["dist"], T2["heights_c"], "k.", markersize=0.5, alpha=0.8) + # plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') + + plt.plot( + Ti2["dist"], + Ti2["heights_c_weighted_mean"] + 1, + ".-", + color="darkblue", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="x-gauss weighted mean +1", + ) + plt.plot( + Ti2["dist"], + Ti2["heights_c_median"], + "r.-", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="median", + ) + + plt.plot( + Ti2["dist"], + Ti2["heights_c_mode"] - 1, + "g.-", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="mode - 1", + ) + + plt.plot(Ti2["dist"], Ti2["heights_c_std"] - 1.8, "k-", linewidth=0.5, alpha=1) + plt.fill_between( + Ti2["dist"], Ti2["heights_c_std"] - 1.8, y2=-1.8, color="gray", alpha=1 + ) # plt.plot( ALT03['delta_time'], ALT03['heights_c'], 'k.', markersize= 0.3, alpha =0.2 ) # plt.plot(ALT07['time']['delta_time'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') plt.legend(loc=1) - plt.xlim(ll, ll+dl) + plt.xlim(ll, ll + dl) plt.ylim(-2, 3) - plt.xlabel('Meters from the Sea Ice Edge') - plt.ylabel('Height Anomalie (meters)') - F.ax.axhline(y =-1.8, color='black', linewidth=0.5) - F.save_light(path= plot_path, name='ALT03_filt_compare'+ str(ll)) + plt.xlabel("Meters from the Sea Ice Edge") + plt.ylabel("Height Anomalie (meters)") + F.ax.axhline(y=-1.8, color="black", linewidth=0.5) + F.save_light(path=plot_path, name="ALT03_filt_compare" + str(ll)) # %% plot diff --git a/analysis/B02_make_spectra.py b/analysis/B02_make_spectra.py index 653b1258..9f82657d 100644 --- a/analysis/B02_make_spectra.py +++ b/analysis/B02_make_spectra.py @@ -1,16 +1,16 @@ - import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -22,112 +22,132 @@ import spicke_remover import datetime import concurrent.futures as futures -#import s3fs -# %% -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20200512030438_07110710_004_01', 'SH_batch02', False - - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' -load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' -load_file = load_path + 'processed_' + ATlevel + '_' + track_name + '.h5' - -save_path = mconfig['paths']['work'] + '/B02_spectra_'+hemis+'/' -save_name = 'B02_'+track_name - -plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/B_spectra/' +# import s3fs +# %% +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20200512030438_07110710_004_01', 'SH_batch02', False + + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +load_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" +load_file = load_path + "processed_" + ATlevel + "_" + track_name + ".h5" + +save_path = mconfig["paths"]["work"] + "/B02_spectra_" + hemis + "/" +save_name = "B02_" + track_name + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/B_spectra/" +) MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -#Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +# Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # -Nworkers= 2 +Nworkers = 2 # %% test amount of nans in the data -nan_fraction= list() +nan_fraction = list() for I in Gd.values(): - nan_fraction.append( np.sum(np.isnan(I['heights_c_std'])) / I['heights_c_std'].shape[0] ) + nan_fraction.append( + np.sum(np.isnan(I["heights_c_std"])) / I["heights_c_std"].shape[0] + ) if np.array(nan_fraction).mean() > 0.95: - print('nan fraction > 95%, pass this track, add to bad tracks') - MT.json_save(track_name, bad_track_path, {'nan_fraction': np.array(nan_fraction).mean(), 'date': str(datetime.date.today()) }) - print('exit.') + print("nan fraction > 95%, pass this track, add to bad tracks") + MT.json_save( + track_name, + bad_track_path, + { + "nan_fraction": np.array(nan_fraction).mean(), + "date": str(datetime.date.today()), + }, + ) + print("exit.") exit() # %% test LS with an even grid where missing values are set to 0 imp.reload(spec) print(Gd.keys()) -Gi =Gd[ list(Gd.keys())[0] ] # to select a test beam +Gi = Gd[list(Gd.keys())[0]] # to select a test beam # derive spectal limits # Longest deserved period: -T_max = 40 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 -x= np.array(Gi['dist']) +T_max = 40 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 +x = np.array(Gi["dist"]) dx = np.diff(x).mean() -min_datapoint = 1/k_0/dx +min_datapoint = 1 / k_0 / dx Lpoints = int(np.round(min_datapoint) * 20) -Lmeters =Lpoints * dx +Lmeters = Lpoints * dx -#plt.plot(np.diff(np.array(Gi['dist']))) -print('L number of gridpoint:', Lpoints) -print('L length in meters:', Lmeters) -print('approx number windows', 2* Gi['dist'].iloc[-1] /Lmeters-1 ) +# plt.plot(np.diff(np.array(Gi['dist']))) +print("L number of gridpoint:", Lpoints) +print("L length in meters:", Lmeters) +print("approx number windows", 2 * Gi["dist"].iloc[-1] / Lmeters - 1) -wavenumber_k = np.fft.rfftfreq( Lpoints, d=dx) +wavenumber_k = np.fft.rfftfreq(Lpoints, d=dx) dk = np.diff(wavenumber_k).mean() # %% -plot_data_model=False -G_LS= dict() -G_rar_fft= dict() +plot_data_model = False +G_LS = dict() +G_rar_fft = dict() Pars_optm = dict() -#imp.reload(spec) +# imp.reload(spec) for k in all_beams: # ------------------------------- use gridded data - hkey= 'heights_c_weighted_mean' - x = Gd[k]['dist'] - xlims = x.iloc[0], x.iloc[-1] - dd = np.copy(Gd[k][hkey]) + hkey = "heights_c_weighted_mean" + x = Gd[k]["dist"] + xlims = x.iloc[0], x.iloc[-1] + dd = np.copy(Gd[k][hkey]) - dd_error = np.copy(Gd[k]['heights_c_std']) + dd_error = np.copy(Gd[k]["heights_c_std"]) dd_error[np.isnan(dd_error)] = 100 - F = M.figure_axis_xy(6, 3) plt.subplot(2, 1, 1) - plt.plot(x, dd, 'gray', label='displacement (m) ') + plt.plot(x, dd, "gray", label="displacement (m) ") # compute slope spectra !! - dd = np.gradient(dd) - dd, _ = spicke_remover.spicke_remover(dd, spreed=10, verbose=False) - dd_nans = (np.isnan(dd) ) + (Gd[k]['N_photos'] <= 5) + dd = np.gradient(dd) + dd, _ = spicke_remover.spicke_remover(dd, spreed=10, verbose=False) + dd_nans = (np.isnan(dd)) + (Gd[k]["N_photos"] <= 5) - dd_filled = np.copy(dd) - dd_filled[dd_nans] = 0 + dd_filled = np.copy(dd) + dd_filled[dd_nans] = 0 # using gappy data - dd_no_nans = dd[~dd_nans] # windowing is applied here - x_no_nans = x[~dd_nans] - dd_error_no_nans = dd_error[~dd_nans] + dd_no_nans = dd[~dd_nans] # windowing is applied here + x_no_nans = x[~dd_nans] + dd_error_no_nans = dd_error[~dd_nans] dx = np.diff(x).mean() @@ -135,352 +155,468 @@ # dd= dd[~outlyers] # x= x[~outlyers] - plt.plot(x_no_nans, dd_no_nans, 'black', label='slope (m/m)') + plt.plot(x_no_nans, dd_no_nans, "black", label="slope (m/m)") plt.legend() imp.reload(spec) - print('LS') - - - S = spec.wavenumber_spectrogram_LS( np.array(x_no_nans), np.array(dd_no_nans), Lmeters, dx, dy = None, waven_method = wavenumber_k, ov=None, window=None) - #ThreadPoolExecutor - with futures.ThreadPoolExecutor(max_workers= Nworkers) as executor_sub: - G, PP = S.cal_spectrogram(xlims= xlims, weight_data=True, max_nfev = None , map_func=executor_sub.map) - - S.mean_spectral_error(mask= (G.N_per_stancil != 0).squeeze().data ) - S.parceval(add_attrs= True, weight_data=False) + print("LS") + + S = spec.wavenumber_spectrogram_LS( + np.array(x_no_nans), + np.array(dd_no_nans), + Lmeters, + dx, + dy=None, + waven_method=wavenumber_k, + ov=None, + window=None, + ) + # ThreadPoolExecutor + with futures.ThreadPoolExecutor(max_workers=Nworkers) as executor_sub: + G, PP = S.cal_spectrogram( + xlims=xlims, weight_data=True, max_nfev=None, map_func=executor_sub.map + ) + + S.mean_spectral_error(mask=(G.N_per_stancil != 0).squeeze().data) + S.parceval(add_attrs=True, weight_data=False) if plot_data_model: - for i in np.arange(60,120,2): - c1= 'blue' - c2= 'red' + for i in np.arange(60, 120, 2): + c1 = "blue" + c2 = "red" - xi_1=G.x[i] - xi_2=G.x[i+1] - #if k%2 ==0: + xi_1 = G.x[i] + xi_2 = G.x[i + 1] + # if k%2 ==0: F = M.figure_axis_xy(16, 2.5) - plt.plot(G.eta[0:-1] +xi_1, np.fft.irfft(G.sel(x=xi_1).Y_model_hat) ,'-', c=c1, linewidth=0.8, alpha=1, zorder=12) - plt.plot(G.eta[0:-1] +xi_2, np.fft.irfft(G.sel(x=xi_2).Y_model_hat) ,'-', c=c2, linewidth=0.8, alpha=1, zorder=12) - - plt.plot(x, dd, '-', c='k',linewidth=2, alpha =0.6) - #plt.plot(x, dd, '.', c='k',markersize=3, alpha =0.5) - - #plt.plot(x[~dd_nans], dd_error[~dd_nans], '.', c='green',linewidth=1, alpha =0.5) + plt.plot( + G.eta[0:-1] + xi_1, + np.fft.irfft(G.sel(x=xi_1).Y_model_hat), + "-", + c=c1, + linewidth=0.8, + alpha=1, + zorder=12, + ) + plt.plot( + G.eta[0:-1] + xi_2, + np.fft.irfft(G.sel(x=xi_2).Y_model_hat), + "-", + c=c2, + linewidth=0.8, + alpha=1, + zorder=12, + ) + + plt.plot(x, dd, "-", c="k", linewidth=2, alpha=0.6) + # plt.plot(x, dd, '.', c='k',markersize=3, alpha =0.5) + + # plt.plot(x[~dd_nans], dd_error[~dd_nans], '.', c='green',linewidth=1, alpha =0.5) F.ax.axvline(xi_1 + G.eta[0].data, linewidth=4, color=c1, alpha=0.5) - F.ax.axvline(xi_1 + G.eta[-1].data, linewidth=4, color=c1, alpha=0.5) - F.ax.axvline(xi_2 + G.eta[0].data, linewidth=4, color=c2, alpha=0.5) - F.ax.axvline(xi_2 + G.eta[-1].data, linewidth=4, color=c2, alpha=0.5) - - plt.text(xi_1 + G.eta[0].data, 0.5, ' N='+ str(G.sel(x=xi_1).N_per_stancil.data) ) - plt.text(xi_2 + G.eta[0].data, 0.5, ' N='+ str(G.sel(x=xi_2).N_per_stancil.data) ) - plt.xlim(xi_1 + G.eta[0].data*1.2, xi_2 + G.eta[-1].data*1.2 ) + F.ax.axvline(xi_1 + G.eta[-1].data, linewidth=4, color=c1, alpha=0.5) + F.ax.axvline(xi_2 + G.eta[0].data, linewidth=4, color=c2, alpha=0.5) + F.ax.axvline(xi_2 + G.eta[-1].data, linewidth=4, color=c2, alpha=0.5) + + plt.text( + xi_1 + G.eta[0].data, + 0.5, + " N=" + str(G.sel(x=xi_1).N_per_stancil.data), + ) + plt.text( + xi_2 + G.eta[0].data, + 0.5, + " N=" + str(G.sel(x=xi_2).N_per_stancil.data), + ) + plt.xlim(xi_1 + G.eta[0].data * 1.2, xi_2 + G.eta[-1].data * 1.2) plt.ylim(-1, 1) plt.show() - - # assign beam coordinate - G.coords['beam'] = str(k)#(('beam'), str(k)) - G = G.expand_dims(dim = 'beam', axis = 1) + G.coords["beam"] = str(k) # (('beam'), str(k)) + G = G.expand_dims(dim="beam", axis=1) # repack such that all coords are associated with beam - G.coords['N_per_stancil'] = (('x', 'beam' ), np.expand_dims(G['N_per_stancil'], 1)) + G.coords["N_per_stancil"] = (("x", "beam"), np.expand_dims(G["N_per_stancil"], 1)) # add more coodindates to the Dataset - x_coord_no_gaps = spec.linear_gap_fill( Gd[k], 'dist', 'x' ) - y_coord_no_gaps = spec.linear_gap_fill( Gd[k], 'dist', 'y' ) - mapped_coords = spec.sub_sample_coords(Gd[k]['dist'], x_coord_no_gaps, y_coord_no_gaps, S.stancil_iter , map_func = None ) - G = G.assign_coords(x_coord =('x',mapped_coords[:,1] ) ).assign_coords(y_coord =('x',mapped_coords[:,2] ) ) - - lons_no_gaps = spec.linear_gap_fill( Gd[k], 'dist', 'lons' ) - lats_no_gaps = spec.linear_gap_fill( Gd[k], 'dist', 'lats' ) - mapped_coords = spec.sub_sample_coords(Gd[k]['dist'], lons_no_gaps, lats_no_gaps, S.stancil_iter , map_func = None ) - G = G.assign_coords(lon =('x',mapped_coords[:,1] ) ).assign_coords(lat =('x',mapped_coords[:,2] ) ) - - for kkk in ['mean_El', 'mean_Eu']: - G.coords[kkk] = (('k', 'beam' ), np.expand_dims(S.G[kkk], 1)) - - for kkk in ['lon', 'lat', 'x_coord', 'y_coord']: - G.coords[kkk] = (('x', 'beam' ), np.expand_dims(G[kkk], 1)) + x_coord_no_gaps = spec.linear_gap_fill(Gd[k], "dist", "x") + y_coord_no_gaps = spec.linear_gap_fill(Gd[k], "dist", "y") + mapped_coords = spec.sub_sample_coords( + Gd[k]["dist"], x_coord_no_gaps, y_coord_no_gaps, S.stancil_iter, map_func=None + ) + G = G.assign_coords(x_coord=("x", mapped_coords[:, 1])).assign_coords( + y_coord=("x", mapped_coords[:, 2]) + ) + + lons_no_gaps = spec.linear_gap_fill(Gd[k], "dist", "lons") + lats_no_gaps = spec.linear_gap_fill(Gd[k], "dist", "lats") + mapped_coords = spec.sub_sample_coords( + Gd[k]["dist"], lons_no_gaps, lats_no_gaps, S.stancil_iter, map_func=None + ) + G = G.assign_coords(lon=("x", mapped_coords[:, 1])).assign_coords( + lat=("x", mapped_coords[:, 2]) + ) + + for kkk in ["mean_El", "mean_Eu"]: + G.coords[kkk] = (("k", "beam"), np.expand_dims(S.G[kkk], 1)) + + for kkk in ["lon", "lat", "x_coord", "y_coord"]: + G.coords[kkk] = (("x", "beam"), np.expand_dims(G[kkk], 1)) # calculate number data points def get_stancil_nans(stancil): x_mask = (stancil[0] < x) & (x <= stancil[-1]) - idata = Gd[k]['N_photos'][x_mask] + idata = Gd[k]["N_photos"][x_mask] return stancil[1], idata.sum() - photon_list = np.array(list(dict(map( get_stancil_nans, copy.copy(S.stancil_iter) )).values())) - G.coords['N_photons'] = (('x', 'beam' ), np.expand_dims(photon_list, 1)) + photon_list = np.array( + list(dict(map(get_stancil_nans, copy.copy(S.stancil_iter))).values()) + ) + G.coords["N_photons"] = (("x", "beam"), np.expand_dims(photon_list, 1)) # Save to dict - G_LS[k] = G + G_LS[k] = G Pars_optm[k] = PP # plot plt.subplot(2, 1, 2) GG = G.LS_spectal_power.squeeze() - plt.plot(GG.k, np.nanmean(GG,1), 'gray', label='mean LS power') - plt.plot(GG.k, np.nanmean(S.G, 1), 'k', label='mean LS power optm') - + plt.plot(GG.k, np.nanmean(GG, 1), "gray", label="mean LS power") + plt.plot(GG.k, np.nanmean(S.G, 1), "k", label="mean LS power optm") # standard FFT - print('FFT') - dd[dd_nans] = 0 + print("FFT") + dd[dd_nans] = 0 S = spec.wavenumber_spectrogram(x, dd, Lpoints) G = S.cal_spectrogram() - S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True) + S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.parceval(add_attrs=True) # assign beam coordinate - G.coords['beam'] = str(k)#(('beam'), str(k)) - G = G.expand_dims(dim = 'beam', axis = 2) - G.coords['mean_El'] = (('k', 'beam' ), np.expand_dims(G['mean_El'], 1)) - G.coords['mean_Eu'] = (('k', 'beam' ), np.expand_dims(G['mean_Eu'], 1)) - G.coords['x'] = G.coords['x'] * dx # adjust x-coodinate definition + G.coords["beam"] = str(k) # (('beam'), str(k)) + G = G.expand_dims(dim="beam", axis=2) + G.coords["mean_El"] = (("k", "beam"), np.expand_dims(G["mean_El"], 1)) + G.coords["mean_Eu"] = (("k", "beam"), np.expand_dims(G["mean_Eu"], 1)) + G.coords["x"] = G.coords["x"] * dx # adjust x-coodinate definition stancil_iter = spec.create_chunk_boundaries(int(Lpoints), dd_nans.size) + def get_stancil_nans(stancil): - idata = dd_nans[stancil[0]:stancil[-1]] + idata = dd_nans[stancil[0] : stancil[-1]] return stancil[1], idata.size - idata.sum() - N_list = np.array(list(dict(map( get_stancil_nans, stancil_iter )).values())) + N_list = np.array(list(dict(map(get_stancil_nans, stancil_iter)).values())) # repack such that all coords are associated with beam - G.coords['N_per_stancil'] = (('x', 'beam' ), np.expand_dims(N_list, 1)) + G.coords["N_per_stancil"] = (("x", "beam"), np.expand_dims(N_list, 1)) - #save to dict + # save to dict G_rar_fft[k] = G # for plotting - G_rar_fft_p =G.squeeze() - plt.plot(G.k, G_rar_fft_p[:, G_rar_fft_p['N_per_stancil'] > 10 ].mean('x'), 'darkblue', label='mean FFT') - #plt.plot(G.k, GG.mean('x'), 'lightblue', label='mean FFT') + G_rar_fft_p = G.squeeze() + plt.plot( + G.k, + G_rar_fft_p[:, G_rar_fft_p["N_per_stancil"] > 10].mean("x"), + "darkblue", + label="mean FFT", + ) + # plt.plot(G.k, GG.mean('x'), 'lightblue', label='mean FFT') plt.legend() plt.show() - F.save_light(path=plot_path, name = 'B02_control_'+k+'_' + track_name) - print('saved as '+'B02_control_'+k+'_' + track_name) + F.save_light(path=plot_path, name="B02_control_" + k + "_" + track_name) + print("saved as " + "B02_control_" + k + "_" + track_name) print(np.isinf(G).sum().data) # %% + def dict_weighted_mean(Gdict, weight_key): """ returns the weighted meean of a dict of xarray, data_arrays weight_key must be in the xr.DataArrays """ - #Gdict = G_LS + # Gdict = G_LS # weight_key='N_per_stancil' - akey = list( Gdict.keys() )[0] + akey = list(Gdict.keys())[0] GSUM = Gdict[akey].copy() - GSUM.data = np.zeros(GSUM.shape) + GSUM.data = np.zeros(GSUM.shape) N_per_stancil = GSUM.N_per_stancil * 0 - N_photons = np.zeros(GSUM.N_per_stancil.size) + N_photons = np.zeros(GSUM.N_per_stancil.size) - counter= 0 - for k,I in Gdict.items(): - I =I.squeeze() - #GSUM += I.where( ~np.isnan(I.squeeze()), 0) * I[weight_key] #.sel(x=GSUM.x) - nan_mask = np.isnan(I) - x_nan_mask = nan_mask.sum('k') != 0 - weights = I[weight_key].where( ~x_nan_mask, 0) - GSUM += I.where( ~nan_mask, 0) * weights - N_per_stancil += weights - if 'N_photons' in GSUM.coords: - N_photons += I['N_photons'] - counter+=1 + counter = 0 + for k, I in Gdict.items(): + I = I.squeeze() + # GSUM += I.where( ~np.isnan(I.squeeze()), 0) * I[weight_key] #.sel(x=GSUM.x) + nan_mask = np.isnan(I) + x_nan_mask = nan_mask.sum("k") != 0 + weights = I[weight_key].where(~x_nan_mask, 0) + GSUM += I.where(~nan_mask, 0) * weights + N_per_stancil += weights + if "N_photons" in GSUM.coords: + N_photons += I["N_photons"] + counter += 1 - GSUM = GSUM / N_per_stancil /counter + GSUM = GSUM / N_per_stancil / counter - if 'N_photons' in GSUM.coords: - GSUM.coords['N_photons'] = (('x', 'beam'), np.expand_dims(N_photons, 1) ) + if "N_photons" in GSUM.coords: + GSUM.coords["N_photons"] = (("x", "beam"), np.expand_dims(N_photons, 1)) - GSUM['beam'] = ['weighted_mean'] - GSUM.name='power_spec' + GSUM["beam"] = ["weighted_mean"] + GSUM.name = "power_spec" return GSUM + G_LS_sel = {} -for k,I in G_LS.items(): - G_LS_sel[k] = I['spectral_power_optm'] +for k, I in G_LS.items(): + G_LS_sel[k] = I["spectral_power_optm"] -G_LS_wmean = dict_weighted_mean(G_LS_sel, 'N_per_stancil') -G_fft_wmean = dict_weighted_mean(G_rar_fft, 'N_per_stancil') +G_LS_wmean = dict_weighted_mean(G_LS_sel, "N_per_stancil") +G_fft_wmean = dict_weighted_mean(G_rar_fft, "N_per_stancil") # %% plot -def plot_wavenumber_spectrogram(ax, Gi, clev, title= None, plot_photon_density=True ): +def plot_wavenumber_spectrogram(ax, Gi, clev, title=None, plot_photon_density=True): - if Gi.k[0] ==0: - Gi= Gi.sel(k=Gi.k[1:]) - x_lambda= 1/Gi.k - plt.pcolormesh(Gi.x/1e3, x_lambda , Gi, cmap=plt.cm.ocean_r , vmin = clev[0], vmax = clev[-1]) + if Gi.k[0] == 0: + Gi = Gi.sel(k=Gi.k[1:]) + x_lambda = 1 / Gi.k + plt.pcolormesh( + Gi.x / 1e3, x_lambda, Gi, cmap=plt.cm.ocean_r, vmin=clev[0], vmax=clev[-1] + ) - ax.set_yscale('log') + ax.set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') if plot_photon_density: - plt.plot(Gi.x/1e3, x_lambda[-1] + (Gi.N_per_stancil/Gi.N_per_stancil.max() ) * 10 , c='black', linewidth= 0.8, label='NAN-density' ) - plt.fill_between(Gi.x/1e3, x_lambda[-1] + (Gi.N_per_stancil/Gi.N_per_stancil.max() ) * 10, 0, color='gray', alpha = 0.3) - ax.axhline(30, color='black', linewidth=0.3) - - #plt.xlabel('Distance from the Ice Edge (km)') + plt.plot( + Gi.x / 1e3, + x_lambda[-1] + (Gi.N_per_stancil / Gi.N_per_stancil.max()) * 10, + c="black", + linewidth=0.8, + label="NAN-density", + ) + plt.fill_between( + Gi.x / 1e3, + x_lambda[-1] + (Gi.N_per_stancil / Gi.N_per_stancil.max()) * 10, + 0, + color="gray", + alpha=0.3, + ) + ax.axhline(30, color="black", linewidth=0.3) + + # plt.xlabel('Distance from the Ice Edge (km)') plt.ylim(x_lambda[-1], x_lambda[0]) - plt.title(title, loc='left') + plt.title(title, loc="left") -#Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() -#Gmean = G_LS_wmean.rolling(x=2, min_periods= 1, center=True).mean() + +# Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() +# Gmean = G_LS_wmean.rolling(x=2, min_periods= 1, center=True).mean() Gmean = G_LS_wmean.rolling(k=5, center=True).mean() -#Gmean = Gmean.where(~np.isnan(Gmean), 0) +# Gmean = Gmean.where(~np.isnan(Gmean), 0) -k_max_range = Gmean.k[Gmean.mean('x').argmax().data].data* 0.75, Gmean.k[Gmean.mean('x').argmax().data].data* 1, Gmean.k[Gmean.mean('x').argmax().data].data* 1.25 +k_max_range = ( + Gmean.k[Gmean.mean("x").argmax().data].data * 0.75, + Gmean.k[Gmean.mean("x").argmax().data].data * 1, + Gmean.k[Gmean.mean("x").argmax().data].data * 1.25, +) # %% font_for_print() -F = M.figure_axis_xy(6.5, 5.6, container= True, view_scale =1) +F = M.figure_axis_xy(6.5, 5.6, container=True, view_scale=1) -plt.suptitle('LS and FFT Slope Spectrograms\n' + track_name, y = 0.98) -gs = GridSpec(3,3, wspace=0.2, hspace=.5)#figure=fig, -#clev=np.arange(0, 6, 0.1)*3 +plt.suptitle("LS and FFT Slope Spectrograms\n" + track_name, y=0.98) +gs = GridSpec(3, 3, wspace=0.2, hspace=0.5) # figure=fig, +# clev=np.arange(0, 6, 0.1)*3 -#%matplotlib inline +# %matplotlib inline # -clev_log = M.clevels( [0, Gmean.quantile(0.95).data * 1.2], 31)* 1 -#clev_log = M.clevels( [Gmean.quantile(0.01).data, Gmean.quantile(0.99).data * 1], 31)* 1 -xlims= Gmean.x[0]/1e3, Gmean.x[-1]/1e3 +clev_log = M.clevels([0, Gmean.quantile(0.95).data * 1.2], 31) * 1 +# clev_log = M.clevels( [Gmean.quantile(0.01).data, Gmean.quantile(0.99).data * 1], 31)* 1 +xlims = Gmean.x[0] / 1e3, Gmean.x[-1] / 1e3 -for pos, k, pflag in zip([gs[0, 0],gs[0, 1],gs[0, 2] ], high_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[0, 0], gs[0, 1], gs[0, 2]], high_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G_LS_sel[k].squeeze()#.rolling(k=10, x=2, min_periods= 1, center=True).mean() - #Gplot.mean('x').plot() + Gplot = G_LS_sel[ + k + ].squeeze() # .rolling(k=10, x=2, min_periods= 1, center=True).mean() + # Gplot.mean('x').plot() # #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) # - #np.log(Gplot).plot() - plot_wavenumber_spectrogram(ax0, Gplot, clev_log, title =k, plot_photon_density=True ) + # np.log(Gplot).plot() + plot_wavenumber_spectrogram(ax0, Gplot, clev_log, title=k, plot_photon_density=True) plt.xlim(xlims) # if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -for pos, k, pflag in zip([gs[1, 0],gs[1, 1],gs[1, 2] ], low_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[1, 0], gs[1, 1], gs[1, 2]], low_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G_LS_sel[k].squeeze()#.rolling(k=10, x=2, min_periods= 1, center=True).mean() - #Gplot.mean('x').plot() + Gplot = G_LS_sel[ + k + ].squeeze() # .rolling(k=10, x=2, min_periods= 1, center=True).mean() + # Gplot.mean('x').plot() # #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) # - plot_wavenumber_spectrogram(ax0, Gplot, clev_log, title =k, plot_photon_density=True ) + plot_wavenumber_spectrogram(ax0, Gplot, clev_log, title=k, plot_photon_density=True) plt.xlim(xlims) # if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -pos, k, pflag = gs[2, 0], 'Power(weighted mean) \n10 $\log_{10}( (m/m)^2 m )$', True +pos, k, pflag = gs[2, 0], "Power(weighted mean) \n10 $\log_{10}( (m/m)^2 m )$", True ax0 = F.fig.add_subplot(pos) -Gplot = G_LS_wmean.squeeze().rolling(k=5, min_periods= 1, center=True).median().rolling(x=3, min_periods= 1, center=True).median() +Gplot = ( + G_LS_wmean.squeeze() + .rolling(k=5, min_periods=1, center=True) + .median() + .rolling(x=3, min_periods=1, center=True) + .median() +) dd = 10 * np.log10(Gplot) -dd= dd.where(~np.isinf(dd), np.nan ) +dd = dd.where(~np.isinf(dd), np.nan) -plot_wavenumber_spectrogram(ax0, dd, clev_log , title =k, plot_photon_density= True) +plot_wavenumber_spectrogram(ax0, dd, clev_log, title=k, plot_photon_density=True) plt.xlim(xlims) # plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) # ax0.axhline(30, color='black', linewidth=0.5) -ax0.axhline(1/k_max_range[0], color='red', linestyle= '--', linewidth= 0.5) -ax0.axhline(1/k_max_range[1], color='red', linestyle= '-', linewidth= 0.5) -ax0.axhline(1/k_max_range[2], color='red', linestyle= '--', linewidth= 0.5) +ax0.axhline(1 / k_max_range[0], color="red", linestyle="--", linewidth=0.5) +ax0.axhline(1 / k_max_range[1], color="red", linestyle="-", linewidth=0.5) +ax0.axhline(1 / k_max_range[2], color="red", linestyle="--", linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() pos = gs[2, 1] ax0 = F.fig.add_subplot(pos) -plt.title('Photons density ($m^{-1}$)', loc='left') - -for k,I in G_LS.items(): - plt.plot(Gplot.x/1e3, I.N_photons/Lmeters, label=k, linewidth=0.8) -plt.plot(Gplot.x/1e3, G_LS_wmean.N_photons/3/Lmeters , c='black', label='ave Photons' , linewidth=0.8) +plt.title("Photons density ($m^{-1}$)", loc="left") + +for k, I in G_LS.items(): + plt.plot(Gplot.x / 1e3, I.N_photons / Lmeters, label=k, linewidth=0.8) +plt.plot( + Gplot.x / 1e3, + G_LS_wmean.N_photons / 3 / Lmeters, + c="black", + label="ave Photons", + linewidth=0.8, +) plt.xlim(xlims) -plt.xlabel('Distance from the Ice Edge (km)') +plt.xlabel("Distance from the Ice Edge (km)") pos = gs[2, 2] ax0 = F.fig.add_subplot(pos) -ax0.set_yscale('log') - -plt.title('Peak Spectal Power', loc='left') - -for k,I in G_LS_sel.items(): - plt.scatter(I.x.data/1e3, I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k').data *1e3 , s=0.5, marker='.', color='red', alpha= 0.3) - -for k,I in G_rar_fft.items(): - I= I.squeeze() - I= I[:,I.N_per_stancil >= I.N_per_stancil.max().data*0.9] - plt.scatter(I.x/1e3, I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3 , s=0.5, marker='.', c='blue', alpha= 0.3) - - -Gplot= G_fft_wmean.squeeze() -Gplot = Gplot[:,Gplot.N_per_stancil >= Gplot.N_per_stancil.max().data*0.9] -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3, '.', markersize=1.5 , c='blue', label= 'FFT') - -Gplot= G_LS_wmean.squeeze() -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3 , '.' , markersize=1.5, c='red', label= 'LS') - -plt.ylabel('1e-3 $(m)^2~m$') +ax0.set_yscale("log") + +plt.title("Peak Spectal Power", loc="left") + +for k, I in G_LS_sel.items(): + plt.scatter( + I.x.data / 1e3, + I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k").data * 1e3, + s=0.5, + marker=".", + color="red", + alpha=0.3, + ) + +for k, I in G_rar_fft.items(): + I = I.squeeze() + I = I[:, I.N_per_stancil >= I.N_per_stancil.max().data * 0.9] + plt.scatter( + I.x / 1e3, + I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + s=0.5, + marker=".", + c="blue", + alpha=0.3, + ) + + +Gplot = G_fft_wmean.squeeze() +Gplot = Gplot[:, Gplot.N_per_stancil >= Gplot.N_per_stancil.max().data * 0.9] +plt.plot( + Gplot.x / 1e3, + Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + ".", + markersize=1.5, + c="blue", + label="FFT", +) + +Gplot = G_LS_wmean.squeeze() +plt.plot( + Gplot.x / 1e3, + Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + ".", + markersize=1.5, + c="red", + label="LS", +) + +plt.ylabel("1e-3 $(m)^2~m$") plt.legend() -plt.ylim(Gplot.min()*1.4, Gplot.max()*1.4 ) +plt.ylim(Gplot.min() * 1.4, Gplot.max() * 1.4) plt.xlim(xlims) -F.save_light(path=plot_path, name = 'B02_specs_' + track_name +'_L'+str(Lmeters)) - +F.save_light(path=plot_path, name="B02_specs_" + track_name + "_L" + str(Lmeters)) # %% save fitting parameters -MT.save_pandas_table(Pars_optm, save_name+'_params', save_path ) +MT.save_pandas_table(Pars_optm, save_name + "_params", save_path) # %% repack data def repack_attributes(DD): - #DD = G_LS + # DD = G_LS attr_dim_list = list(DD.keys()) for k in attr_dim_list: for ka in list(DD[k].attrs.keys()): I = DD[k] - I.coords[ka] = ( 'beam', np.expand_dims(I.attrs[ka], 0) ) + I.coords[ka] = ("beam", np.expand_dims(I.attrs[ka], 0)) return DD -G_LS[G_LS_wmean.beam.data[0]] =G_LS_wmean -G_rar_fft[G_fft_wmean.beam.data[0]] =G_fft_wmean + +G_LS[G_LS_wmean.beam.data[0]] = G_LS_wmean +G_rar_fft[G_fft_wmean.beam.data[0]] = G_fft_wmean G_LS = repack_attributes(G_LS) G_rar_fft = repack_attributes(G_rar_fft) # %% save results -G_LS_DS = xr.merge(G_LS.values()) -G_LS_DS['name'] = 'LS_power_spectra' -G_LS_DS['Y_model_hat_imag'] = G_LS_DS.Y_model_hat.imag -G_LS_DS['Y_model_hat_real'] = G_LS_DS.Y_model_hat.real -G_LS_DS = G_LS_DS.drop('Y_model_hat') -G_LS_DS.to_netcdf(save_path+save_name+'_LS.nc') - -G_fft_DS = xr.merge(G_rar_fft.values()) -G_fft_DS['name']= 'FFT_power_spectra' +G_LS_DS = xr.merge(G_LS.values()) +G_LS_DS["name"] = "LS_power_spectra" +G_LS_DS["Y_model_hat_imag"] = G_LS_DS.Y_model_hat.imag +G_LS_DS["Y_model_hat_real"] = G_LS_DS.Y_model_hat.real +G_LS_DS = G_LS_DS.drop("Y_model_hat") +G_LS_DS.to_netcdf(save_path + save_name + "_LS.nc") + +G_fft_DS = xr.merge(G_rar_fft.values()) +G_fft_DS["name"] = "FFT_power_spectra" try: - G_fft_DS = G_fft_DS.drop('Y_model_hat') + G_fft_DS = G_fft_DS.drop("Y_model_hat") except: pass -G_fft_DS.to_netcdf(save_path+save_name+'_FFT.nc') +G_fft_DS.to_netcdf(save_path + save_name + "_FFT.nc") diff --git a/analysis/B02_make_spectra_FFT.py b/analysis/B02_make_spectra_FFT.py index 15acd9d3..c1da7a75 100644 --- a/analysis/B02_make_spectra_FFT.py +++ b/analysis/B02_make_spectra_FFT.py @@ -1,23 +1,25 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -#sys.path.append(base_path +'modules/') -#sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +# base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' +# sys.path.append(base_path +'modules/') +# sys.path.append(base_path +'modules/ICEsat2_SI_tools/') import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -37,24 +39,23 @@ import imp - -#import s3fs +# import s3fs # %% # Python reader based on Pandas. Other reader examples available in readers.py -plot_path = mconfig['paths']['plot'] + '/tests/' +plot_path = mconfig["paths"]["plot"] + "/tests/" -track_name= 'ATL03_20190515060436_07170312_002_02' -load_path = base_path + 'data/data1/' +track_name = "ATL03_20190515060436_07170312_002_02" +load_path = base_path + "data/data1/" # test which beams exist: -all_beams = ['gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r'] -low_beams = ['gt1l', 'gt2l', 'gt3l'] -high_beams = ['gt1r', 'gt2r', 'gt3r'] +all_beams = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] +low_beams = ["gt1l", "gt2l", "gt3l"] +high_beams = ["gt1r", "gt2r", "gt3r"] # %% -Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') +Gall = xr.open_dataset(load_path + "/" + track_name + "_filtered_photon_heights.nc") # Gmask = Gall.dist *0 # for k in Gall.var(): @@ -78,100 +79,106 @@ # plt.plot( nan_mask.dist[ lfields[0] ==i ] ) # %% -L=600 # meters +L = 600 # meters imp.reload(spec) -G2= dict() +G2 = dict() for k in high_beams: - Gall[k+'_nans'] = np.isnan(Gall[k]) - Gall[k][Gall[k+'_nans']] = 0 + Gall[k + "_nans"] = np.isnan(Gall[k]) + Gall[k][Gall[k + "_nans"]] = 0 Gsel = Gall[k] - dd = (Gsel -Gsel.mean('dist')) + dd = Gsel - Gsel.mean("dist") dd = np.gradient(dd) - #np.isnan(dd).sum() + # np.isnan(dd).sum() S = spec.wavenumber_spetrogram(Gsel.dist, dd, L) G = S.cal_spectrogram() - S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True) + S.parceval(add_attrs=True) + stancil_iter = spec.create_chunk_boundaries(L, Gall[k + "_nans"].size) - stancil_iter = spec.create_chunk_boundaries(L, Gall[k+'_nans'].size) def get_stancil_nans(stancil): - idata = Gall[k+'_nans'][stancil[0]:stancil[-1]] - return stancil[1], idata.sum().data/idata.size + idata = Gall[k + "_nans"][stancil[0] : stancil[-1]] + return stancil[1], idata.sum().data / idata.size - nan_list = np.array(list(dict(map( get_stancil_nans, stancil_iter )).values())) + nan_list = np.array(list(dict(map(get_stancil_nans, stancil_iter)).values())) G2[k] = G - G2[k].coords['nan_density'] = (('x'), nan_list) + G2[k].coords["nan_density"] = (("x"), nan_list) -#plt.plot(G) +# plt.plot(G) # %% -#Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() -Gplot = G.rolling(k=10, x=2, min_periods= 1, center=True).mean() +# Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() +Gplot = G.rolling(k=10, x=2, min_periods=1, center=True).mean() import matplotlib.colors as colors from matplotlib.gridspec import GridSpec font_for_print() -F = M.figure_axis_xy(6.5, 5.6, container= True, view_scale =0.8) +F = M.figure_axis_xy(6.5, 5.6, container=True, view_scale=0.8) -plt.suptitle('FFT with gaps =0', y = 0.95) -gs = GridSpec(3,3, wspace=0.1, hspace=0.7)#figure=fig, -clev=np.arange(0, 6, 0.1) +plt.suptitle("FFT with gaps =0", y=0.95) +gs = GridSpec(3, 3, wspace=0.1, hspace=0.7) # figure=fig, +clev = np.arange(0, 6, 0.1) -for pos, k, pflag in zip([gs[0, 0],gs[0, 1],gs[0, 2] ], high_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[0, 0], gs[0, 1], gs[0, 2]], high_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G2[k]#.rolling(k=10, x=2, min_periods= 1, center=True).mean() + Gplot = G2[k] # .rolling(k=10, x=2, min_periods= 1, center=True).mean() - #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) + # plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) - plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap='PuBu', vmin = clev[0], vmax = clev[-1]) + plt.pcolormesh( + Gplot.x / 1e3, 1 / Gplot.k, Gplot, cmap="PuBu", vmin=clev[0], vmax=clev[-1] + ) - plt.gca().set_yscale('log') + plt.gca().set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') - plt.xlabel('Distance from the Ice Edge (km)') - plt.ylim(20,800) - plt.title(k, loc='left') + plt.xlabel("Distance from the Ice Edge (km)") + plt.ylim(20, 800) + plt.title(k, loc="left") - plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) - ax0.axhline(30, color='black', linewidth=0.5) + plt.plot(Gplot.x / 1e3, 10 * nan_list + 20, c="black", label="NAN-density") + ax0.axhline(30, color="black", linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -G3_sum = G2[k]*0 -G3_sum['nan_density'] =0 +G3_sum = G2[k] * 0 +G3_sum["nan_density"] = 0 for k in high_beams: G3_sum += G2[k] -G3_sum= G3_sum/3 -G3_sum['nan_density'] =G3_sum['nan_density']/3 +G3_sum = G3_sum / 3 +G3_sum["nan_density"] = G3_sum["nan_density"] / 3 -pos, k, pflag = gs[1, 0], 'mean', True +pos, k, pflag = gs[1, 0], "mean", True ax0 = F.fig.add_subplot(pos) -Gplot = G3_sum.rolling(k=10, x=2, min_periods= 1, center=True).mean() +Gplot = G3_sum.rolling(k=10, x=2, min_periods=1, center=True).mean() -#plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) +# plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) -plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap='PuBu', vmin = clev[0], vmax = clev[-1]) +plt.pcolormesh( + Gplot.x / 1e3, 1 / Gplot.k, Gplot, cmap="PuBu", vmin=clev[0], vmax=clev[-1] +) -plt.gca().set_yscale('log') +plt.gca().set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') -plt.xlabel('Distance from the Ice Edge (km)') -plt.ylim(20,800) -plt.title(k, loc='left') +plt.xlabel("Distance from the Ice Edge (km)") +plt.ylim(20, 800) +plt.title(k, loc="left") -plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) -ax0.axhline(30, color='black', linewidth=0.5) +plt.plot(Gplot.x / 1e3, 10 * nan_list + 20, c="black", label="NAN-density") +ax0.axhline(30, color="black", linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -F.save_light(path=plot_path, name='exmpl_spec_FFT_strong') +F.save_light(path=plot_path, name="exmpl_spec_FFT_strong") diff --git a/analysis/B02_make_spectra_LS.py b/analysis/B02_make_spectra_LS.py index 26848269..3343b3a4 100644 --- a/analysis/B02_make_spectra_LS.py +++ b/analysis/B02_make_spectra_LS.py @@ -1,23 +1,25 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -37,92 +39,104 @@ import imp - -#import s3fs +# import s3fs # %% # Python reader based on Pandas. Other reader examples available in readers.py -plot_path = mconfig['paths']['plot'] + '/tests/' +plot_path = mconfig["paths"]["plot"] + "/tests/" -track_name= 'ATL03_20190515060436_07170312_002_02' -load_path = base_path + 'data/data1/' +track_name = "ATL03_20190515060436_07170312_002_02" +load_path = base_path + "data/data1/" # test which beams exist: -all_beams = ['gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r'] -low_beams = ['gt1l', 'gt2l', 'gt3l'] -high_beams = ['gt1r', 'gt2r', 'gt3r'] +all_beams = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] +low_beams = ["gt1l", "gt2l", "gt3l"] +high_beams = ["gt1r", "gt2r", "gt3r"] # %% -#Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') +# Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') imp.reload(io) -Gfilt = io.load_pandas_table_dict(track_name + '_B01_corrected', load_path) -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) +Gfilt = io.load_pandas_table_dict(track_name + "_B01_corrected", load_path) +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # %% -Gi =Gd[high_beams[0]] -Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') -#np.isnan(Gi['dist']).sum() -Gi = Gi[(Gi['dist'] > 140000) & (Gi['dist'] < 160001)] +Gi = Gd[high_beams[0]] +Gi = Gi[~np.isnan(Gi["heights_c_weighted_mean"])].sort_values("dist") +# np.isnan(Gi['dist']).sum() +Gi = Gi[(Gi["dist"] > 140000) & (Gi["dist"] < 160001)] # %% -plt.plot( Gi['dist'], Gi['heights_c_weighted_mean'], '-') -#plt.xlim(140000, 145000) +plt.plot(Gi["dist"], Gi["heights_c_weighted_mean"], "-") +# plt.xlim(140000, 145000) plt.ylim(-1, 1) -#np.diff(np.array(Gi['dist'])) +# np.diff(np.array(Gi['dist'])) # %% test and compare to FFT from scipy.signal import detrend -y =detrend(np.array(Gi['heights_c_weighted_mean']) ) -f_fft2, df2 = spec.calc_freq_fft( np.array(Gi['dist']), y.size) -spec_fft2 = spec.calc_spectrum_fft(y, df2, np.array(Gi['dist']).size) + +y = detrend(np.array(Gi["heights_c_weighted_mean"])) +f_fft2, df2 = spec.calc_freq_fft(np.array(Gi["dist"]), y.size) +spec_fft2 = spec.calc_spectrum_fft(y, df2, np.array(Gi["dist"]).size) f_fft2_regular, spec_fft2_regular = f_fft2, spec_fft2 # %% simple spectra from astropy.timeseries import LombScargle -ls = LombScargle( np.array(Gi['dist']) , np.array(Gi['heights_c_weighted_mean']) ) -ls_auto_power = ls.power(f_fft2 , normalization='psd') -#ls_auto_f , ls_auto_power = ls.autopower(normalization='psd', samples_per_peak=0.1) +ls = LombScargle(np.array(Gi["dist"]), np.array(Gi["heights_c_weighted_mean"])) -plt.plot(f_fft2 , spec.LS_power_to_PSD(ls_auto_power, y.size, df2) ) +ls_auto_power = ls.power(f_fft2, normalization="psd") +# ls_auto_f , ls_auto_power = ls.autopower(normalization='psd', samples_per_peak=0.1) + +plt.plot(f_fft2, spec.LS_power_to_PSD(ls_auto_power, y.size, df2)) -plt.plot(f_fft2, spec_fft2 ) +plt.plot(f_fft2, spec_fft2) plt.xlim(0, 0.01) # %% tests # maximum to be resolved wavenumber -T = 40 #sec -k_0 = (2 * np.pi/ T)**2 / 9.81 +T = 40 # sec +k_0 = (2 * np.pi / T) ** 2 / 9.81 -1/k_0 -x= np.array(Gi.index) +1 / k_0 +x = np.array(Gi.index) dt = np.diff(x).mean() -#dt = np.diff(np.array(Gi['dist'])).mean() -min_datapoint = 1/k_0/dt +# dt = np.diff(np.array(Gi['dist'])).mean() +min_datapoint = 1 / k_0 / dt L = int(np.round(min_datapoint) * 10) -S_pwelch_fft = spec.wavenumber_spectrogram(x , np.array(Gi['heights_c_weighted_mean']), L) +S_pwelch_fft = spec.wavenumber_spectrogram( + x, np.array(Gi["heights_c_weighted_mean"]), L +) G_pwelch_fft = S_pwelch_fft.cal_spectrogram() -S_pwelch_ls = spec.wavenumber_spectrogram(x, np.array(Gi['heights_c_weighted_mean']), L) +S_pwelch_ls = spec.wavenumber_spectrogram(x, np.array(Gi["heights_c_weighted_mean"]), L) G_pwelch_fft = S_pwelch_fft.cal_spectrogram() # %% imp.reload(spec) -y =detrend(np.array(Gi['heights_c_weighted_mean']) ) +y = detrend(np.array(Gi["heights_c_weighted_mean"])) -S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( x, L, method='fftX2', minimum_frequency=None, maximum_frequency=None, samples_per_peak=0.5) +S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( + x, + L, + method="fftX2", + minimum_frequency=None, + maximum_frequency=None, + samples_per_peak=0.5, +) -S_pwelch_ls = spec.wavenumber_spectrogram_LS_even(x, y, L, waven_method = S_pwelch_k , dy=None , ov=None, window=None, kjumps=2) +S_pwelch_ls = spec.wavenumber_spectrogram_LS_even( + x, y, L, waven_method=S_pwelch_k, dy=None, ov=None, window=None, kjumps=2 +) G_pwelch_ls = S_pwelch_ls.cal_spectrogram() S_pwelch_ls.parceval() -plt.plot(G_pwelch_ls.k , G_pwelch_ls.mean('x') ) -plt.plot(G_pwelch_fft.k , G_pwelch_fft.mean('x') , 'k') +plt.plot(G_pwelch_ls.k, G_pwelch_ls.mean("x")) +plt.plot(G_pwelch_fft.k, G_pwelch_fft.mean("x"), "k") plt.xlim(0, 0.02) @@ -131,275 +145,311 @@ # %% test LS with an even grid where missing values are set to 0 imp.reload(spec) -Gi =Gd[high_beams[0]] -#Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') +Gi = Gd[high_beams[0]] +# Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') # derive spectal limits -T_max = 50 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 -x= np.array(Gi.index) +T_max = 50 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 +x = np.array(Gi.index) dx = np.diff(x).mean() -min_datapoint = 1/k_0/dx +min_datapoint = 1 / k_0 / dx L = int(np.round(min_datapoint) * 20) -plt.plot(np.diff(np.array(Gi['dist']))) +plt.plot(np.diff(np.array(Gi["dist"]))) print(L) -print(L*dx) -print(Gi['dist'].iloc[-1] /L ) - -S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( x, L, dx= dx, method='fftX2', minimum_frequency=None, maximum_frequency=None, samples_per_peak=0.01) -hkey= 'heights_c_weighted_mean' -G2= dict() +print(L * dx) +print(Gi["dist"].iloc[-1] / L) + +S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( + x, + L, + dx=dx, + method="fftX2", + minimum_frequency=None, + maximum_frequency=None, + samples_per_peak=0.01, +) +hkey = "heights_c_weighted_mean" +G2 = dict() for k in high_beams: - x= Gd[k].index - dd =Gd[k][hkey] + x = Gd[k].index + dd = Gd[k][hkey] dd_nans = np.isnan(dd) - dd= dd[~dd_nans] - x= x[~dd_nans] - dd = (dd -dd.mean()) - #dd = np.gradient(dd) - #np.isnan(dd).sum() - S = spec.wavenumber_spectrogram_LS_even( np.array(x), np.array(dd), L, waven_method = S_pwelch_k , dy=None , ov=None, window=None, kjumps=2) + dd = dd[~dd_nans] + x = x[~dd_nans] + dd = dd - dd.mean() + # dd = np.gradient(dd) + # np.isnan(dd).sum() + S = spec.wavenumber_spectrogram_LS_even( + np.array(x), + np.array(dd), + L, + waven_method=S_pwelch_k, + dy=None, + ov=None, + window=None, + kjumps=2, + ) G = S.cal_spectrogram() - S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True) + S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.parceval(add_attrs=True) - G['x'] = G.x * dx + G["x"] = G.x * dx stancil_iter = spec.create_chunk_boundaries(L, np.array(dd).size) + def get_stancil_nans(stancil): - idata = Gd[k]['N_photos'][stancil[0]:stancil[-1]] + idata = Gd[k]["N_photos"][stancil[0] : stancil[-1]] return stancil[1], idata.sum() - photon_list = np.array(list(dict(map( get_stancil_nans, stancil_iter )).values())) + photon_list = np.array(list(dict(map(get_stancil_nans, stancil_iter)).values())) G2[k] = G - G2[k].coords['N_photos'] = (('x'), photon_list) - + G2[k].coords["N_photos"] = (("x"), photon_list) # %% font_for_pres() -plt.plot(G.k, G.mean('x')) +plt.plot(G.k, G.mean("x")) # %% -#Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() -Gplot = G.rolling(k=10, x=2, min_periods= 1, center=True).mean() +# Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() +Gplot = G.rolling(k=10, x=2, min_periods=1, center=True).mean() import matplotlib.colors as colors from matplotlib.gridspec import GridSpec font_for_print() -F = M.figure_axis_xy(6.5, 5.6, container= True, view_scale =0.8) +F = M.figure_axis_xy(6.5, 5.6, container=True, view_scale=0.8) -plt.suptitle('LS on regular grid', y = 0.95) -gs = GridSpec(3,3, wspace=0.1, hspace=0.7)#figure=fig, -#clev=np.arange(0, 6, 0.1)*3 -clev = M.clevels(Gplot.data, 31)* 0.8 +plt.suptitle("LS on regular grid", y=0.95) +gs = GridSpec(3, 3, wspace=0.1, hspace=0.7) # figure=fig, +# clev=np.arange(0, 6, 0.1)*3 +clev = M.clevels(Gplot.data, 31) * 0.8 -for pos, k, pflag in zip([gs[0, 0],gs[0, 1],gs[0, 2] ], high_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[0, 0], gs[0, 1], gs[0, 2]], high_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G2[k]#.rolling(k=10, x=2, min_periods= 1, center=True).mean() + Gplot = G2[k] # .rolling(k=10, x=2, min_periods= 1, center=True).mean() - #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) + # plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) - plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap=plt.cm.ocean_r, vmin = clev[0], vmax = clev[-1]) + plt.pcolormesh( + Gplot.x / 1e3, + 1 / Gplot.k, + Gplot, + cmap=plt.cm.ocean_r, + vmin=clev[0], + vmax=clev[-1], + ) - plt.gca().set_yscale('log') + plt.gca().set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') - plt.xlabel('Distance from the Ice Edge (km)') - plt.ylim(20,L*dx/2) - plt.title(k, loc='left') + plt.xlabel("Distance from the Ice Edge (km)") + plt.ylim(20, L * dx / 2) + plt.title(k, loc="left") # plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) # ax0.axhline(30, color='black', linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -G3_sum = G2[k]*0 -G3_sum['N_photos'] = G3_sum['N_photos']*0 -G3_sum= G3_sum.sel(x=slice(G3_sum.x[0],G3_sum.x[-3]) ) +G3_sum = G2[k] * 0 +G3_sum["N_photos"] = G3_sum["N_photos"] * 0 +G3_sum = G3_sum.sel(x=slice(G3_sum.x[0], G3_sum.x[-3])) for k in high_beams: G3_sum += G2[k].sel(x=G3_sum.x) - G3_sum['N_photos'] += G2[k].sel(x=G3_sum.x)['N_photos'] + G3_sum["N_photos"] += G2[k].sel(x=G3_sum.x)["N_photos"] -G3_sum= G3_sum/3 -G3_sum['N_photos'] =G3_sum['N_photos']/3 +G3_sum = G3_sum / 3 +G3_sum["N_photos"] = G3_sum["N_photos"] / 3 -pos, k, pflag = gs[1, 0], 'mean', True +pos, k, pflag = gs[1, 0], "mean", True ax0 = F.fig.add_subplot(pos) -Gplot = G3_sum.rolling(k=5, x=2, min_periods= 1, center=True).mean() +Gplot = G3_sum.rolling(k=5, x=2, min_periods=1, center=True).mean() -#plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) +# plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) -plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap=plt.cm.ocean_r, vmin = clev[0], vmax = clev[-1]) +plt.pcolormesh( + Gplot.x / 1e3, 1 / Gplot.k, Gplot, cmap=plt.cm.ocean_r, vmin=clev[0], vmax=clev[-1] +) -plt.gca().set_yscale('log') +plt.gca().set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') -plt.xlabel('Distance from the Ice Edge (km)') -plt.ylim(20,L*dx/2) -plt.title(k, loc='left') +plt.xlabel("Distance from the Ice Edge (km)") +plt.ylim(20, L * dx / 2) +plt.title(k, loc="left") # plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) # ax0.axhline(30, color='black', linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() pos = gs[1, 1] ax0 = F.fig.add_subplot(pos) -plt.title('ave photon per chunk') -plt.plot(Gplot.x/1e3, Gplot.N_photos , c='black') +plt.title("ave photon per chunk") +plt.plot(Gplot.x / 1e3, Gplot.N_photos, c="black") pos = gs[1, 2] ax0 = F.fig.add_subplot(pos) -plt.title('spectal power decay') -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(1/500, 1/100)).integrate('k') , c='black') +plt.title("spectal power decay") +plt.plot(Gplot.x / 1e3, Gplot.sel(k=slice(1 / 500, 1 / 100)).integrate("k"), c="black") -F.save_light(path=plot_path, name='exmpl_spec_LS_strong_L'+str(L*dx)) +F.save_light(path=plot_path, name="exmpl_spec_LS_strong_L" + str(L * dx)) # %% rar photon data Gi = Gfilt[high_beams[0]] -Gi = Gi[(Gi['dist'] > 140000) & (Gi['dist'] < 160001)] +Gi = Gi[(Gi["dist"] > 140000) & (Gi["dist"] < 160001)] -plt.plot( Gi['dist'], Gi['heights_c_weighted_mean'], '.', alpha= 0.2) -#plt.xlim(140000, 145000) +plt.plot(Gi["dist"], Gi["heights_c_weighted_mean"], ".", alpha=0.2) +# plt.xlim(140000, 145000) plt.ylim(-1, 1) -#np.diff(np.array(Gi['dist'])) +# np.diff(np.array(Gi['dist'])) # %% from scipy.signal import detrend -y =detrend(np.array(Gi['heights']) ) -f_fft2, df2 = spec.calc_freq_fft( np.array(Gi['dist']), y.size) -f_fft2 = f_fft2[1:] -spec_fft2 = spec.calc_spectrum_fft(y, df2, np.array(Gi['dist']).size) +y = detrend(np.array(Gi["heights"])) +f_fft2, df2 = spec.calc_freq_fft(np.array(Gi["dist"]), y.size) +f_fft2 = f_fft2[1:] +spec_fft2 = spec.calc_spectrum_fft(y, df2, np.array(Gi["dist"]).size) # %% simple spectra from astropy.timeseries import LombScargle -ls = LombScargle( np.array(Gi['dist']) , np.array(Gi['heights']) ) -ls_power = ls.power(f_fft2_regular[1::1] , normalization='psd') -plt.plot(f_fft2_regular[1::1] , (spec.LS_power_to_PSD(ls_power, y.size, df2)) , 'r', zorder=12) +ls = LombScargle(np.array(Gi["dist"]), np.array(Gi["heights"])) + +ls_power = ls.power(f_fft2_regular[1::1], normalization="psd") +plt.plot( + f_fft2_regular[1::1], (spec.LS_power_to_PSD(ls_power, y.size, df2)), "r", zorder=12 +) # ls_auto_f , ls_auto_power = ls.autopower(normalization='psd', samples_per_peak=0.1) # plt.plot(ls_auto_f , spec.LS_power_to_PSD(ls_auto_power, y.size, df2) , 'r', zorder=12) -#plt.plot(f_fft2, spec_fft2 ) -plt.plot(f_fft2_regular, spec_fft2_regular, 'k-' ) +# plt.plot(f_fft2, spec_fft2 ) +plt.plot(f_fft2_regular, spec_fft2_regular, "k-") plt.xlim(0, 0.01) plt.ylim(0, 80) - - - - - - - - - - - # %% ueven version font_for_pres() Gi = Gfilt[high_beams[0]] -dist_diffs= Gi.sort_values('dist')['dist'].diff() +dist_diffs = Gi.sort_values("dist")["dist"].diff() dist_diffs.median() imp.reload(spec) # derive spectal limits -T_max = 30 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 +T_max = 30 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 -x = np.array(Gi.sort_values('dist')['dist']) +x = np.array(Gi.sort_values("dist")["dist"]) xdiff = np.diff(x) -xdiff = xdiff[xdiff< .1] +xdiff = xdiff[xdiff < 0.1] -#plt.hist( xdiff, bins= 40 ) -dx =np.mean(xdiff)*1e2 -#dx = 0.01#dist_diffs.median() #np.diff(np.array(Gi['dist'])).mean() -1/dx -min_datapoint = 1/k_0/dx -longest_wavelength= 1/k_0 # meters +# plt.hist( xdiff, bins= 40 ) +dx = np.mean(xdiff) * 1e2 +# dx = 0.01#dist_diffs.median() #np.diff(np.array(Gi['dist'])).mean() +1 / dx +min_datapoint = 1 / k_0 / dx +longest_wavelength = 1 / k_0 # meters Lmeters = int(np.round(longest_wavelength) * 20) print(Lmeters) -print(Gi['dist'].max()/ Lmeters ) +print(Gi["dist"].max() / Lmeters) # -shortest_wavenumber = 1/2 # 2 meter wave -desired_dx = 1/shortest_wavenumber +shortest_wavenumber = 1 / 2 # 2 meter wave +desired_dx = 1 / shortest_wavenumber Lpoints = Lmeters / dx print(Lpoints) -Gi.shape[0]/ Lpoints +Gi.shape[0] / Lpoints -S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( None, Lmeters, method='fixed_ratio', dx= dx) +S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS(None, Lmeters, method="fixed_ratio", dx=dx) -S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( np.array(Gi['dist']), Lpoints, method='LS_auto', minimum_frequency=f_fft2_regular[1], maximum_frequency=f_fft2_regular[-1], samples_per_peak=0.1) +S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( + np.array(Gi["dist"]), + Lpoints, + method="LS_auto", + minimum_frequency=f_fft2_regular[1], + maximum_frequency=f_fft2_regular[-1], + samples_per_peak=0.1, +) S_pwelch_k[-1] S_pwelch_k[0] S_pwelch_k.size -#f_regular, df2 = spec.calc_freq_fft_given_dx(desired_dx,Lmeters) -f_regular = np.linspace(k_0/100, 0.1, 300) +# f_regular, df2 = spec.calc_freq_fft_given_dx(desired_dx,Lmeters) +f_regular = np.linspace(k_0 / 100, 0.1, 300) -#f_regular = 1/np.linspace(1/shortest_wavenumber, 10/k_0, 1000)[::-1] +# f_regular = 1/np.linspace(1/shortest_wavenumber, 10/k_0, 1000)[::-1] dk = np.diff(f_regular).mean() f_regular[1:].size f_regular[-1] -G2= dict() -#k = high_beams[0] +G2 = dict() +# k = high_beams[0] for k in high_beams: Gi = Gfilt[k] # for testing amplitude - #Gi = Gi[(Gi['dist'] > 140000) & (Gi['dist'] < 160001)] - #Gi['nans'] = np.isnan(Gi['heights']) - #Gall[k][Gall[k+'_nans']] = 0 - #Gsel = Gi[~Gall[k+'_nans']] - dd = np.array( Gi['heights'] - Gi['heights'].mean() ) - #dd = np.gradient(dd) - #np.isnan(dd).sum() - S = spec.wavenumber_spectrogram_LS(np.array(Gi.dist), dd, Lmeters, waven_method = f_regular , dy=None , ov=None, window=None, kjumps=1) + # Gi = Gi[(Gi['dist'] > 140000) & (Gi['dist'] < 160001)] + # Gi['nans'] = np.isnan(Gi['heights']) + # Gall[k][Gall[k+'_nans']] = 0 + # Gsel = Gi[~Gall[k+'_nans']] + dd = np.array(Gi["heights"] - Gi["heights"].mean()) + # dd = np.gradient(dd) + # np.isnan(dd).sum() + S = spec.wavenumber_spectrogram_LS( + np.array(Gi.dist), + dd, + Lmeters, + waven_method=f_regular, + dy=None, + ov=None, + window=None, + kjumps=1, + ) G = S.cal_spectrogram() - S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True) + S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.parceval(add_attrs=True) G2[k] = G font_for_pres() -plt.plot(G.k, G, 'b', alpha = 0.2) -plt.plot(G.k, G.mean('x'), 'r-' ) +plt.plot(G.k, G, "b", alpha=0.2) +plt.plot(G.k, G.mean("x"), "r-") # for testing amplitude # !!!! vanriance is not conserved yet !!! -plt.plot(f_fft2_regular, spec_fft2_regular, 'k-' ) +plt.plot(f_fft2_regular, spec_fft2_regular, "k-") plt.xlim(0, 0.01) plt.ylim(0, 40) @@ -407,91 +457,103 @@ def get_stancil_nans(stancil): font_for_pres() -plt.plot(G.k, G.mean('x')) +plt.plot(G.k, G.mean("x")) # %% -#Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() -Gplot = G.rolling(k=10, x=2, min_periods= 1, center=True).mean() +# Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() +Gplot = G.rolling(k=10, x=2, min_periods=1, center=True).mean() # %% import matplotlib.colors as colors from matplotlib.gridspec import GridSpec font_for_print() -F = M.figure_axis_xy(6.5, 5.6, container= True, view_scale =0.8) +F = M.figure_axis_xy(6.5, 5.6, container=True, view_scale=0.8) -plt.suptitle('LS on uneven grid (dx= '+ str(int(np.diff(Gplot.x).mean())) +' m)', y = 0.95) -gs = GridSpec(3,3, wspace=0.1, hspace=0.7)#figure=fig, -#clev=np.arange(0, 6, 0.1)*3 -clev = M.clevels(Gplot.data, 31)* 0.9 +plt.suptitle( + "LS on uneven grid (dx= " + str(int(np.diff(Gplot.x).mean())) + " m)", y=0.95 +) +gs = GridSpec(3, 3, wspace=0.1, hspace=0.7) # figure=fig, +# clev=np.arange(0, 6, 0.1)*3 +clev = M.clevels(Gplot.data, 31) * 0.9 -for pos, k, pflag in zip([gs[0, 0],gs[0, 1],gs[0, 2] ], high_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[0, 0], gs[0, 1], gs[0, 2]], high_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G2[k]#.rolling(k=10, x=2, min_periods= 1, center=True).mean() - - #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) - ax0.set_yscale('log') - #plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap=plt.cm.ocean_r) - plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap=plt.cm.ocean_r, vmin = clev[0], vmax = clev[-1]) - - #plt.gca().set_yscale('log') + Gplot = G2[k] # .rolling(k=10, x=2, min_periods= 1, center=True).mean() + + # plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) + ax0.set_yscale("log") + # plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap=plt.cm.ocean_r) + plt.pcolormesh( + Gplot.x / 1e3, + 1 / Gplot.k, + Gplot, + cmap=plt.cm.ocean_r, + vmin=clev[0], + vmax=clev[-1], + ) + + # plt.gca().set_yscale('log') # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') - plt.xlabel('Distance from the Ice Edge (km)') - plt.ylim(20,Lmeters/2) - plt.title(k, loc='left') + plt.xlabel("Distance from the Ice Edge (km)") + plt.ylim(20, Lmeters / 2) + plt.title(k, loc="left") - - #ax0.axhline(30, color='black', linewidth=0.5) + # ax0.axhline(30, color='black', linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -k= high_beams[0] -G3_sum = G2[k]*0 -N_grid=G2[k]*0 -x_common =G2[k].x +k = high_beams[0] +G3_sum = G2[k] * 0 +N_grid = G2[k] * 0 +x_common = G2[k].x for k in high_beams: gii = G2[high_beams[0]] - N_grid +=~np.isnan(gii) - G3_sum += gii.where(~np.isnan(gii), 0).interp(x = x_common) + N_grid += ~np.isnan(gii) + G3_sum += gii.where(~np.isnan(gii), 0).interp(x=x_common) -G3_sum= G3_sum/N_grid +G3_sum = G3_sum / N_grid -pos, k, pflag = gs[1, 0], 'mean', True +pos, k, pflag = gs[1, 0], "mean", True ax0 = F.fig.add_subplot(pos) -Gplot = G3_sum.rolling(k=5, x=5, min_periods= 1, center=True).mean() +Gplot = G3_sum.rolling(k=5, x=5, min_periods=1, center=True).mean() -#plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) +# plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) -plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap=plt.cm.ocean_r, vmin = clev[0], vmax = clev[-1]) +plt.pcolormesh( + Gplot.x / 1e3, 1 / Gplot.k, Gplot, cmap=plt.cm.ocean_r, vmin=clev[0], vmax=clev[-1] +) -plt.gca().set_yscale('log') +plt.gca().set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') -plt.xlabel('Distance from the Ice Edge (km)') -plt.ylim(20,Lmeters/2) -plt.title(k, loc='left') +plt.xlabel("Distance from the Ice Edge (km)") +plt.ylim(20, Lmeters / 2) +plt.title(k, loc="left") -#plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) -ax0.axhline(30, color='black', linewidth=0.5) +# plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) +ax0.axhline(30, color="black", linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() pos = gs[1, 1] ax0 = F.fig.add_subplot(pos) -plt.title('mean # of datapoints / chunk (1e3)') -plt.plot(Gplot.x/1e3, Gplot.N_per_sample/1e3 , c='black') +plt.title("mean # of datapoints / chunk (1e3)") +plt.plot(Gplot.x / 1e3, Gplot.N_per_sample / 1e3, c="black") pos = gs[1, 2] ax0 = F.fig.add_subplot(pos) -plt.title('spectal power decay') -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(1/500, 1/100)).integrate('k') , c='black') +plt.title("spectal power decay") +plt.plot(Gplot.x / 1e3, Gplot.sel(k=slice(1 / 500, 1 / 100)).integrate("k"), c="black") -F.save_light(path=plot_path, name='exmpl_spec_LS_uneven_strong') +F.save_light(path=plot_path, name="exmpl_spec_LS_uneven_strong") diff --git a/analysis/B02_make_spectra_LS_new.py b/analysis/B02_make_spectra_LS_new.py index b70ad1d8..c5d31eca 100644 --- a/analysis/B02_make_spectra_LS_new.py +++ b/analysis/B02_make_spectra_LS_new.py @@ -1,23 +1,25 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -42,285 +44,351 @@ from matplotlib.gridspec import GridSpec -#import s3fs +# import s3fs # %% # Python reader based on Pandas. Other reader examples available in readers.py -plot_path = mconfig['paths']['plot'] + '/tests/' +plot_path = mconfig["paths"]["plot"] + "/tests/" -track_name= 'ATL03_20190515060436_07170312_002_02' -load_path = base_path + 'data/data1/' +track_name = "ATL03_20190515060436_07170312_002_02" +load_path = base_path + "data/data1/" -#track_name= 'ATL03_20191215230028_12220512_004_01' -track_name= 'ATL03_20210414174223_03191110_004_01' -load_path = base_path + 'data/data4/' +# track_name= 'ATL03_20191215230028_12220512_004_01' +track_name = "ATL03_20210414174223_03191110_004_01" +load_path = base_path + "data/data4/" # test which beams exist: -all_beams = ['gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r'] -low_beams = ['gt1l', 'gt2l', 'gt3l'] -high_beams = ['gt1r', 'gt2r', 'gt3r'] +all_beams = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] +low_beams = ["gt1l", "gt2l", "gt3l"] +high_beams = ["gt1r", "gt2r", "gt3r"] # %% -#Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') +# Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') imp.reload(io) -Gfilt = io.load_pandas_table_dict(track_name + '_B01_corrected', load_path) # rhis is the rar photon data -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # - - +Gfilt = io.load_pandas_table_dict( + track_name + "_B01_corrected", load_path +) # rhis is the rar photon data +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # # %% test LS with an even grid where missing values are set to 0 imp.reload(spec) -Gi =Gd[high_beams[0]] +Gi = Gd[high_beams[0]] -#Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') +# Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') # derive spectal limits # Longest deserved period: -T_max = 40 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 -x= np.array(Gi['dist']) +T_max = 40 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 +x = np.array(Gi["dist"]) dx = np.diff(x).mean() -min_datapoint = 1/k_0/dx +min_datapoint = 1 / k_0 / dx Lpoints = int(np.round(min_datapoint) * 10) -Lmeters =Lpoints * dx - -#plt.plot(np.diff(np.array(Gi['dist']))) -print('L number of gridpoint:', Lpoints) -print('L length in meters:', Lmeters) -print('approx number windows', 2* Gi['dist'].iloc[-1] /Lmeters-1 ) - - -S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( x, Lpoints, dx= dx, method='fft', minimum_frequency=None, maximum_frequency=None, samples_per_peak=0.005) +Lmeters = Lpoints * dx + +# plt.plot(np.diff(np.array(Gi['dist']))) +print("L number of gridpoint:", Lpoints) +print("L length in meters:", Lmeters) +print("approx number windows", 2 * Gi["dist"].iloc[-1] / Lmeters - 1) + + +S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( + x, + Lpoints, + dx=dx, + method="fft", + minimum_frequency=None, + maximum_frequency=None, + samples_per_peak=0.005, +) S_pwelch_k.shape -hkey= 'heights_c_weighted_mean' +hkey = "heights_c_weighted_mean" imp.reload(spicke_remover) font_for_pres() -G_LS= dict() -G_fft= dict() +G_LS = dict() +G_fft = dict() for k in high_beams: - #k = high_beams[1] - x= Gd[k]['dist'] + # k = high_beams[1] + x = Gd[k]["dist"] xlims = x.iloc[0], x.iloc[-1] - dd = np.copy(Gd[k][hkey]) - - + dd = np.copy(Gd[k][hkey]) M.figure_axis_xy(6, 3) plt.subplot(1, 2, 1) - plt.plot(x, dd, 'gray', label='displacement (m) ') + plt.plot(x, dd, "gray", label="displacement (m) ") # compute slope spectra !! dd = np.gradient(dd) dd, _ = spicke_remover.spicke_remover(dd, spreed=5, verbose=False) - dd_nans = (np.isnan(dd) ) + (Gd[k]['N_photos'] <= 5) + dd_nans = (np.isnan(dd)) + (Gd[k]["N_photos"] <= 5) - dd_no_nans= dd[~dd_nans] + dd_no_nans = dd[~dd_nans] x_no_nans = x[~dd_nans] - # outlyers = abs(dd) > dd.std() *5 # dd= dd[~outlyers] # x= x[~outlyers] - plt.plot(x_no_nans, dd_no_nans, 'black', label='slope (m/m)') + plt.plot(x_no_nans, dd_no_nans, "black", label="slope (m/m)") plt.legend() - print('LS') - S = spec.wavenumber_spectrogram_LS( np.array(x_no_nans), np.array(dd_no_nans), Lmeters, waven_method = S_pwelch_k[1:] , ov=None, window=None, kjumps=2) - G_ls_i = S.cal_spectrogram(xlims= xlims) - S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True) - - #G['x'] = G.x * dx + print("LS") + S = spec.wavenumber_spectrogram_LS( + np.array(x_no_nans), + np.array(dd_no_nans), + Lmeters, + waven_method=S_pwelch_k[1:], + ov=None, + window=None, + kjumps=2, + ) + G_ls_i = S.cal_spectrogram(xlims=xlims) + S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.parceval(add_attrs=True) + + # G['x'] = G.x * dx def get_stancil_nans(stancil): - x_mask= (stancil[0] < x) & (x <= stancil[-1]) - idata = Gd[k]['N_photos'][x_mask] + x_mask = (stancil[0] < x) & (x <= stancil[-1]) + idata = Gd[k]["N_photos"][x_mask] return stancil[1], idata.sum() - photon_list = np.array(list(dict(map( get_stancil_nans, copy.copy(S.stancil_iter) )).values())) + photon_list = np.array( + list(dict(map(get_stancil_nans, copy.copy(S.stancil_iter))).values()) + ) G_LS[k] = G_ls_i - G_LS[k].coords['N_photons'] = (('x'), photon_list) + G_LS[k].coords["N_photons"] = (("x"), photon_list) plt.subplot(1, 2, 2) - plt.plot(G_ls_i.k, G_ls_i.mean('x'), 'k', label='mean LS') + plt.plot(G_ls_i.k, G_ls_i.mean("x"), "k", label="mean LS") # standard FFT - print('FFT') - dd[dd_nans] = 0 + print("FFT") + dd[dd_nans] = 0 S = spec.wavenumber_spectrogram(x, dd, Lpoints) G = S.cal_spectrogram() - S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True) - + S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.parceval(add_attrs=True) stancil_iter = spec.create_chunk_boundaries(int(Lpoints), dd_nans.size) + def get_stancil_nans(stancil): - idata = dd_nans[stancil[0]:stancil[-1]] + idata = dd_nans[stancil[0] : stancil[-1]] return stancil[1], idata.size - idata.sum() - N_list = np.array(list(dict(map( get_stancil_nans, stancil_iter )).values())) + N_list = np.array(list(dict(map(get_stancil_nans, stancil_iter)).values())) G_fft[k] = G - G_fft[k].coords['N_per_stancil'] = (('x'), N_list) - G_fft[k].coords['x'] = G_fft[k].coords['x'] * dx + G_fft[k].coords["N_per_stancil"] = (("x"), N_list) + G_fft[k].coords["x"] = G_fft[k].coords["x"] * dx - plt.plot(G.k, G.mean('x'), 'darkblue', label='mean FFT') + plt.plot(G.k, G.mean("x"), "darkblue", label="mean FFT") plt.legend() plt.show() - - #print(np.isinf(G).sum().data) + # print(np.isinf(G).sum().data) # %% font_for_pres() -plt.plot(G.k, G.mean('x')) +plt.plot(G.k, G.mean("x")) + def dict_weighted_mean(Gdict, weight_key): """ returns the weighted meean of a dict of xarray, data_arrays weight_key must be in the xr.DataArrays """ - #Gdict = G_LS + # Gdict = G_LS # weight_key='N_per_stancil' - akey = list( Gdict.keys() )[0] + akey = list(Gdict.keys())[0] GSUM = Gdict[akey].copy() - GSUM.data = np.zeros(GSUM.shape) + GSUM.data = np.zeros(GSUM.shape) N_per_stancil = GSUM.N_per_stancil * 0 - N_photons = np.zeros(GSUM.N_per_stancil.size) + N_photons = np.zeros(GSUM.N_per_stancil.size) - counter= 0 - for k,I in Gdict.items(): - GSUM += I * I[weight_key] #.sel(x=GSUM.x) - N_per_stancil += I[weight_key] - if 'N_photons' in GSUM.coords: - N_photons += I['N_photons'] - counter+=1 + counter = 0 + for k, I in Gdict.items(): + GSUM += I * I[weight_key] # .sel(x=GSUM.x) + N_per_stancil += I[weight_key] + if "N_photons" in GSUM.coords: + N_photons += I["N_photons"] + counter += 1 - GSUM = GSUM / N_per_stancil /counter + GSUM = GSUM / N_per_stancil / counter - if 'N_photons' in GSUM.coords: - GSUM.coords['N_photons'] = (('x'), N_photons ) + if "N_photons" in GSUM.coords: + GSUM.coords["N_photons"] = (("x"), N_photons) return GSUM -G_LS_wmean = dict_weighted_mean(G_LS, 'N_per_stancil') -G_fft_wmean = dict_weighted_mean(G_fft, 'N_per_stancil') + +G_LS_wmean = dict_weighted_mean(G_LS, "N_per_stancil") +G_fft_wmean = dict_weighted_mean(G_fft, "N_per_stancil") # %% -def plot_wavenumber_spectrogram(ax, Gi, clev, title= None, plot_photon_density=True ): +def plot_wavenumber_spectrogram(ax, Gi, clev, title=None, plot_photon_density=True): - x_lambda= 1/Gi.k - plt.pcolormesh(Gi.x/1e3, x_lambda , Gi, cmap=plt.cm.ocean_r , vmin = clev[0], vmax = clev[-1]) + x_lambda = 1 / Gi.k + plt.pcolormesh( + Gi.x / 1e3, x_lambda, Gi, cmap=plt.cm.ocean_r, vmin=clev[0], vmax=clev[-1] + ) - ax.set_yscale('log') + ax.set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') if plot_photon_density: - plt.plot(Gi.x/1e3, x_lambda[-1] + (Gi.N_per_stancil/Gi.N_per_stancil.max() ) * 10 , c='black', linewidth= 0.8, label='NAN-density' ) - ax.axhline(30, color='black', linewidth=0.5) - - #plt.xlabel('Distance from the Ice Edge (km)') + plt.plot( + Gi.x / 1e3, + x_lambda[-1] + (Gi.N_per_stancil / Gi.N_per_stancil.max()) * 10, + c="black", + linewidth=0.8, + label="NAN-density", + ) + ax.axhline(30, color="black", linewidth=0.5) + + # plt.xlabel('Distance from the Ice Edge (km)') plt.ylim(x_lambda[-1], x_lambda[0]) - plt.title(title, loc='left') + plt.title(title, loc="left") -#Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() -Gmean = G_LS_wmean.rolling(k=10, x=2, min_periods= 1, center=True).mean() -Gmean = Gmean.where(~np.isnan(Gmean), 0) -k_max_range = Gmean.k[Gmean.mean('x').argmax().data].data* 0.75, Gmean.k[Gmean.mean('x').argmax().data].data* 1, Gmean.k[Gmean.mean('x').argmax().data].data* 1.25 +# Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() +Gmean = G_LS_wmean.rolling(k=10, x=2, min_periods=1, center=True).mean() +Gmean = Gmean.where(~np.isnan(Gmean), 0) +k_max_range = ( + Gmean.k[Gmean.mean("x").argmax().data].data * 0.75, + Gmean.k[Gmean.mean("x").argmax().data].data * 1, + Gmean.k[Gmean.mean("x").argmax().data].data * 1.25, +) font_for_print() -F = M.figure_axis_xy(6.5, 5.6, container= True, view_scale =1) +F = M.figure_axis_xy(6.5, 5.6, container=True, view_scale=1) -plt.suptitle('LS and FFT Slope Spectrograms\n' + track_name, y = 0.98) -gs = GridSpec(3,3, wspace=0.2, hspace=.4)#figure=fig, -#clev=np.arange(0, 6, 0.1)*3 -clev = M.clevels( Gmean.data, 31)* 1 -xlims= Gmean.x[0]/1e3, Gmean.x[-1]/1e3 +plt.suptitle("LS and FFT Slope Spectrograms\n" + track_name, y=0.98) +gs = GridSpec(3, 3, wspace=0.2, hspace=0.4) # figure=fig, +# clev=np.arange(0, 6, 0.1)*3 +clev = M.clevels(Gmean.data, 31) * 1 +xlims = Gmean.x[0] / 1e3, Gmean.x[-1] / 1e3 -for pos, k, pflag in zip([gs[0, 0],gs[0, 1],gs[0, 2] ], high_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[0, 0], gs[0, 1], gs[0, 2]], high_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G_LS[k]#.rolling(k=10, x=2, min_periods= 1, center=True).mean() - #Gplot.mean('x').plot() + Gplot = G_LS[k] # .rolling(k=10, x=2, min_periods= 1, center=True).mean() + # Gplot.mean('x').plot() # #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) # - plot_wavenumber_spectrogram(ax0, Gplot, clev, title =k, plot_photon_density=True ) + plot_wavenumber_spectrogram(ax0, Gplot, clev, title=k, plot_photon_density=True) plt.xlim(xlims) # if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -pos, k, pflag = gs[1, 0], 'weighted mean', True +pos, k, pflag = gs[1, 0], "weighted mean", True ax0 = F.fig.add_subplot(pos) -Gplot = G_LS_wmean.rolling(k=5, x=2, min_periods= 1, center=True).median() +Gplot = G_LS_wmean.rolling(k=5, x=2, min_periods=1, center=True).median() -plot_wavenumber_spectrogram(ax0, Gplot, clev, title =k, plot_photon_density= True) +plot_wavenumber_spectrogram(ax0, Gplot, clev, title=k, plot_photon_density=True) plt.xlim(xlims) # plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) # ax0.axhline(30, color='black', linewidth=0.5) -ax0.axhline(1/k_max_range[0], color='red', linestyle= '--', linewidth= 0.5) -ax0.axhline(1/k_max_range[1], color='red', linestyle= '-', linewidth= 0.5) -ax0.axhline(1/k_max_range[2], color='red', linestyle= '--', linewidth= 0.5) +ax0.axhline(1 / k_max_range[0], color="red", linestyle="--", linewidth=0.5) +ax0.axhline(1 / k_max_range[1], color="red", linestyle="-", linewidth=0.5) +ax0.axhline(1 / k_max_range[2], color="red", linestyle="--", linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() pos = gs[1, 1] ax0 = F.fig.add_subplot(pos) -plt.title('Photons density ($m^{-1}$)', loc='left') - -for k,I in G_LS.items(): - plt.plot(Gplot.x/1e3, I.N_photons/Lmeters, label=k, linewidth=0.8) -plt.plot(Gplot.x/1e3, G_LS_wmean.N_photons/3/Lmeters , c='black', label='ave Photons' , linewidth=0.8) +plt.title("Photons density ($m^{-1}$)", loc="left") + +for k, I in G_LS.items(): + plt.plot(Gplot.x / 1e3, I.N_photons / Lmeters, label=k, linewidth=0.8) +plt.plot( + Gplot.x / 1e3, + G_LS_wmean.N_photons / 3 / Lmeters, + c="black", + label="ave Photons", + linewidth=0.8, +) plt.xlim(xlims) -plt.xlabel('Distance from the Ice Edge (km)') +plt.xlabel("Distance from the Ice Edge (km)") pos = gs[1, 2] ax0 = F.fig.add_subplot(pos) -ax0.set_yscale('log') - -plt.title('Peak Spectal Power', loc='left') - -for k,I in G_LS.items(): - plt.scatter(I.x.data/1e3, I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k').data *1e3 , s=0.5, marker='.', color='red', alpha= 0.3) - -for k,I in G_fft.items(): - I= I[:,I.N_per_stancil >= I.N_per_stancil.max().data*0.9] - plt.scatter(I.x/1e3, I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3 , s=0.5, marker='.', c='blue', alpha= 0.3) - - -Gplot= G_fft_wmean[:,G_fft_wmean.N_per_stancil >= G_fft_wmean.N_per_stancil.max().data*0.9] -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3, '.', markersize=1.5 , c='blue', label= 'FFT') - -Gplot= G_LS_wmean -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3 , '.' , markersize=1.5, c='red', label= 'LS') - -plt.ylabel('1e-3 $(m)^2~m$') +ax0.set_yscale("log") + +plt.title("Peak Spectal Power", loc="left") + +for k, I in G_LS.items(): + plt.scatter( + I.x.data / 1e3, + I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k").data * 1e3, + s=0.5, + marker=".", + color="red", + alpha=0.3, + ) + +for k, I in G_fft.items(): + I = I[:, I.N_per_stancil >= I.N_per_stancil.max().data * 0.9] + plt.scatter( + I.x / 1e3, + I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + s=0.5, + marker=".", + c="blue", + alpha=0.3, + ) + + +Gplot = G_fft_wmean[ + :, G_fft_wmean.N_per_stancil >= G_fft_wmean.N_per_stancil.max().data * 0.9 +] +plt.plot( + Gplot.x / 1e3, + Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + ".", + markersize=1.5, + c="blue", + label="FFT", +) + +Gplot = G_LS_wmean +plt.plot( + Gplot.x / 1e3, + Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + ".", + markersize=1.5, + c="red", + label="LS", +) + +plt.ylabel("1e-3 $(m)^2~m$") plt.legend() -#plt.ylim(Gplot.min()*1.4, Gplot.max()*1.4 ) +# plt.ylim(Gplot.min()*1.4, Gplot.max()*1.4 ) plt.xlim(xlims) -F.save_light(path=plot_path, name=track_name +'_exmpl_spec_LS_strong_L'+str(Lmeters)) +F.save_light(path=plot_path, name=track_name + "_exmpl_spec_LS_strong_L" + str(Lmeters)) diff --git a/analysis/B02_make_spectra_LS_new_new.py b/analysis/B02_make_spectra_LS_new_new.py index bc32371b..7790dbf3 100644 --- a/analysis/B02_make_spectra_LS_new_new.py +++ b/analysis/B02_make_spectra_LS_new_new.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -7,18 +8,19 @@ """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -43,308 +45,382 @@ from matplotlib.gridspec import GridSpec -#import s3fs +# import s3fs # %% # Python reader based on Pandas. Other reader examples available in readers.py -plot_path = mconfig['paths']['plot'] + '/tests/' +plot_path = mconfig["paths"]["plot"] + "/tests/" -track_name= 'ATL03_20190515060436_07170312_002_02' -load_path = base_path + 'data/data1/' +track_name = "ATL03_20190515060436_07170312_002_02" +load_path = base_path + "data/data1/" -#track_name= 'ATL03_20191215230028_12220512_004_01' -track_name= 'ATL03_20210414174223_03191110_004_01' -load_path = base_path + 'data/data4/' +# track_name= 'ATL03_20191215230028_12220512_004_01' +track_name = "ATL03_20210414174223_03191110_004_01" +load_path = base_path + "data/data4/" # %% as in main analysis -#20190601220921_09870310_004_01 -track_name= '20190601220921_09870310_004_01' -ATlevel= 'ATL03' -hemis= 'SH' -load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' -load_file = load_path + 'processed_' + ATlevel + '_' + track_name + '.h5' - -bad_track_path =mconfig['paths']['work'] +'bad_tracks/' +# 20190601220921_09870310_004_01 +track_name = "20190601220921_09870310_004_01" +ATlevel = "ATL03" +hemis = "SH" +load_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" +load_file = load_path + "processed_" + ATlevel + "_" + track_name + ".h5" + +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" # %% # test which beams exist: -all_beams = ['gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r'] -low_beams = ['gt1l', 'gt2l', 'gt3l'] -high_beams = ['gt1r', 'gt2r', 'gt3r'] - +all_beams = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] +low_beams = ["gt1l", "gt2l", "gt3l"] +high_beams = ["gt1r", "gt2r", "gt3r"] # %% -#Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') +# Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') imp.reload(io) -Gfilt = io.load_pandas_table_dict(track_name + '_B01_corrected', load_path) # rhis is the rar photon data -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # - - +Gfilt = io.load_pandas_table_dict( + track_name + "_B01_corrected", load_path +) # rhis is the rar photon data +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # # %% test LS with an even grid where missing values are set to 0 imp.reload(spec) -nan_fraction= list() +nan_fraction = list() for I in Gd.values(): - nan_fraction.append( np.sum(np.isnan(I['heights_c_std'])) / I['heights_c_std'].shape[0] ) + nan_fraction.append( + np.sum(np.isnan(I["heights_c_std"])) / I["heights_c_std"].shape[0] + ) if np.array(nan_fraction).mean() > 0.95: - print('nan fraction > 95%, pass this track, add to bad tracks') - MT.json_save(track_name, bad_track_path, {'nan_fraction': np.array(nan_fraction).mean(), 'date': str(datetime.date.today()) }) + print("nan fraction > 95%, pass this track, add to bad tracks") + MT.json_save( + track_name, + bad_track_path, + { + "nan_fraction": np.array(nan_fraction).mean(), + "date": str(datetime.date.today()), + }, + ) exit() - -#Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') -Gi =Gd[high_beams[0]] +# Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') + +Gi = Gd[high_beams[0]] # derive spectal limits # Longest deserved period: -T_max = 40 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 -x= np.array(Gi['dist']) +T_max = 40 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 +x = np.array(Gi["dist"]) dx = np.diff(x).mean() -min_datapoint = 1/k_0/dx +min_datapoint = 1 / k_0 / dx Lpoints = int(np.round(min_datapoint) * 10) -Lmeters =Lpoints * dx - -#plt.plot(np.diff(np.array(Gi['dist']))) -print('L number of gridpoint:', Lpoints) -print('L length in meters:', Lmeters) -print('approx number windows', 2* Gi['dist'].iloc[-1] /Lmeters-1 ) - - -S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( x, Lpoints, dx= dx, method='fft', minimum_frequency=None, maximum_frequency=None, samples_per_peak=0.005) +Lmeters = Lpoints * dx + +# plt.plot(np.diff(np.array(Gi['dist']))) +print("L number of gridpoint:", Lpoints) +print("L length in meters:", Lmeters) +print("approx number windows", 2 * Gi["dist"].iloc[-1] / Lmeters - 1) + + +S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( + x, + Lpoints, + dx=dx, + method="fft", + minimum_frequency=None, + maximum_frequency=None, + samples_per_peak=0.005, +) S_pwelch_k.shape -hkey= 'heights_c_weighted_mean' +hkey = "heights_c_weighted_mean" imp.reload(spicke_remover) font_for_pres() -G_LS= dict() -G_fft= dict() +G_LS = dict() +G_fft = dict() for k in high_beams: - #k = high_beams[1] - x= Gd[k]['dist'] + # k = high_beams[1] + x = Gd[k]["dist"] xlims = x.iloc[0], x.iloc[-1] - dd = np.copy(Gd[k][hkey]) - - + dd = np.copy(Gd[k][hkey]) M.figure_axis_xy(6, 3) plt.subplot(1, 2, 1) - plt.plot(x, dd, 'gray', label='displacement (m) ') + plt.plot(x, dd, "gray", label="displacement (m) ") # compute slope spectra !! dd = np.gradient(dd) dd, _ = spicke_remover.spicke_remover(dd, spreed=5, verbose=False) - dd_nans = (np.isnan(dd) ) + (Gd[k]['N_photos'] <= 5) + dd_nans = (np.isnan(dd)) + (Gd[k]["N_photos"] <= 5) - dd_no_nans= dd[~dd_nans] + dd_no_nans = dd[~dd_nans] x_no_nans = x[~dd_nans] - # outlyers = abs(dd) > dd.std() *5 # dd= dd[~outlyers] # x= x[~outlyers] - plt.plot(x_no_nans, dd_no_nans, 'black', label='slope (m/m)') + plt.plot(x_no_nans, dd_no_nans, "black", label="slope (m/m)") plt.legend() - print('LS') - S = spec.wavenumber_spectrogram_LS( np.array(x_no_nans), np.array(dd_no_nans), Lmeters, waven_method = S_pwelch_k[1:] , ov=None, window=None, kjumps=2) - G_ls_i = S.cal_spectrogram(xlims= xlims) - S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True) - - #G['x'] = G.x * dx + print("LS") + S = spec.wavenumber_spectrogram_LS( + np.array(x_no_nans), + np.array(dd_no_nans), + Lmeters, + waven_method=S_pwelch_k[1:], + ov=None, + window=None, + kjumps=2, + ) + G_ls_i = S.cal_spectrogram(xlims=xlims) + S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.parceval(add_attrs=True) + + # G['x'] = G.x * dx def get_stancil_nans(stancil): - x_mask= (stancil[0] < x) & (x <= stancil[-1]) - idata = Gd[k]['N_photos'][x_mask] + x_mask = (stancil[0] < x) & (x <= stancil[-1]) + idata = Gd[k]["N_photos"][x_mask] return stancil[1], idata.sum() - photon_list = np.array(list(dict(map( get_stancil_nans, copy.copy(S.stancil_iter) )).values())) + photon_list = np.array( + list(dict(map(get_stancil_nans, copy.copy(S.stancil_iter))).values()) + ) G_LS[k] = G_ls_i - G_LS[k].coords['N_photons'] = (('x'), photon_list) + G_LS[k].coords["N_photons"] = (("x"), photon_list) plt.subplot(1, 2, 2) - plt.plot(G_ls_i.k, G_ls_i.mean('x'), 'k', label='mean LS') + plt.plot(G_ls_i.k, G_ls_i.mean("x"), "k", label="mean LS") # standard FFT - print('FFT') - dd[dd_nans] = 0 + print("FFT") + dd[dd_nans] = 0 S = spec.wavenumber_spectrogram(x, dd, Lpoints) G = S.cal_spectrogram() - S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True) - + S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.parceval(add_attrs=True) stancil_iter = spec.create_chunk_boundaries(int(Lpoints), dd_nans.size) + def get_stancil_nans(stancil): - idata = dd_nans[stancil[0]:stancil[-1]] + idata = dd_nans[stancil[0] : stancil[-1]] return stancil[1], idata.size - idata.sum() - N_list = np.array(list(dict(map( get_stancil_nans, stancil_iter )).values())) + N_list = np.array(list(dict(map(get_stancil_nans, stancil_iter)).values())) G_fft[k] = G - G_fft[k].coords['N_per_stancil'] = (('x'), N_list) - G_fft[k].coords['x'] = G_fft[k].coords['x'] * dx + G_fft[k].coords["N_per_stancil"] = (("x"), N_list) + G_fft[k].coords["x"] = G_fft[k].coords["x"] * dx - plt.plot(G.k, G.mean('x'), 'darkblue', label='mean FFT') + plt.plot(G.k, G.mean("x"), "darkblue", label="mean FFT") plt.legend() plt.show() - - #print(np.isinf(G).sum().data) + # print(np.isinf(G).sum().data) # %% font_for_pres() -plt.plot(G.k, G.mean('x')) +plt.plot(G.k, G.mean("x")) + def dict_weighted_mean(Gdict, weight_key): """ returns the weighted meean of a dict of xarray, data_arrays weight_key must be in the xr.DataArrays """ - #Gdict = G_LS + # Gdict = G_LS # weight_key='N_per_stancil' - akey = list( Gdict.keys() )[0] + akey = list(Gdict.keys())[0] GSUM = Gdict[akey].copy() - GSUM.data = np.zeros(GSUM.shape) + GSUM.data = np.zeros(GSUM.shape) N_per_stancil = GSUM.N_per_stancil * 0 - N_photons = np.zeros(GSUM.N_per_stancil.size) + N_photons = np.zeros(GSUM.N_per_stancil.size) - counter= 0 - for k,I in Gdict.items(): - GSUM += I * I[weight_key] #.sel(x=GSUM.x) - N_per_stancil += I[weight_key] - if 'N_photons' in GSUM.coords: - N_photons += I['N_photons'] - counter+=1 + counter = 0 + for k, I in Gdict.items(): + GSUM += I * I[weight_key] # .sel(x=GSUM.x) + N_per_stancil += I[weight_key] + if "N_photons" in GSUM.coords: + N_photons += I["N_photons"] + counter += 1 - GSUM = GSUM / N_per_stancil /counter + GSUM = GSUM / N_per_stancil / counter - if 'N_photons' in GSUM.coords: - GSUM.coords['N_photons'] = (('x'), N_photons ) + if "N_photons" in GSUM.coords: + GSUM.coords["N_photons"] = (("x"), N_photons) return GSUM -G_LS_wmean = dict_weighted_mean(G_LS, 'N_per_stancil') -G_fft_wmean = dict_weighted_mean(G_fft, 'N_per_stancil') + +G_LS_wmean = dict_weighted_mean(G_LS, "N_per_stancil") +G_fft_wmean = dict_weighted_mean(G_fft, "N_per_stancil") # %% -def plot_wavenumber_spectrogram(ax, Gi, clev, title= None, plot_photon_density=True ): +def plot_wavenumber_spectrogram(ax, Gi, clev, title=None, plot_photon_density=True): - x_lambda= 1/Gi.k - plt.pcolormesh(Gi.x/1e3, x_lambda , Gi, cmap=plt.cm.ocean_r , vmin = clev[0], vmax = clev[-1]) + x_lambda = 1 / Gi.k + plt.pcolormesh( + Gi.x / 1e3, x_lambda, Gi, cmap=plt.cm.ocean_r, vmin=clev[0], vmax=clev[-1] + ) - ax.set_yscale('log') + ax.set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') if plot_photon_density: - plt.plot(Gi.x/1e3, x_lambda[-1] + (Gi.N_per_stancil/Gi.N_per_stancil.max() ) * 10 , c='black', linewidth= 0.8, label='NAN-density' ) - ax.axhline(30, color='black', linewidth=0.5) - - #plt.xlabel('Distance from the Ice Edge (km)') + plt.plot( + Gi.x / 1e3, + x_lambda[-1] + (Gi.N_per_stancil / Gi.N_per_stancil.max()) * 10, + c="black", + linewidth=0.8, + label="NAN-density", + ) + ax.axhline(30, color="black", linewidth=0.5) + + # plt.xlabel('Distance from the Ice Edge (km)') plt.ylim(x_lambda[-1], x_lambda[0]) - plt.title(title, loc='left') + plt.title(title, loc="left") -#Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() -Gmean = G_LS_wmean.rolling(k=10, x=2, min_periods= 1, center=True).mean() -Gmean = Gmean.where(~np.isnan(Gmean), 0) -k_max_range = Gmean.k[Gmean.mean('x').argmax().data].data* 0.75, Gmean.k[Gmean.mean('x').argmax().data].data* 1, Gmean.k[Gmean.mean('x').argmax().data].data* 1.25 +# Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() +Gmean = G_LS_wmean.rolling(k=10, x=2, min_periods=1, center=True).mean() +Gmean = Gmean.where(~np.isnan(Gmean), 0) +k_max_range = ( + Gmean.k[Gmean.mean("x").argmax().data].data * 0.75, + Gmean.k[Gmean.mean("x").argmax().data].data * 1, + Gmean.k[Gmean.mean("x").argmax().data].data * 1.25, +) font_for_print() -F = M.figure_axis_xy(6.5, 5.6, container= True, view_scale =1) +F = M.figure_axis_xy(6.5, 5.6, container=True, view_scale=1) -plt.suptitle('LS and FFT Slope Spectrograms\n' + track_name, y = 0.98) -gs = GridSpec(3,3, wspace=0.2, hspace=.4)#figure=fig, -#clev=np.arange(0, 6, 0.1)*3 -clev = M.clevels( Gmean.data, 31)* 1 -xlims= Gmean.x[0]/1e3, Gmean.x[-1]/1e3 +plt.suptitle("LS and FFT Slope Spectrograms\n" + track_name, y=0.98) +gs = GridSpec(3, 3, wspace=0.2, hspace=0.4) # figure=fig, +# clev=np.arange(0, 6, 0.1)*3 +clev = M.clevels(Gmean.data, 31) * 1 +xlims = Gmean.x[0] / 1e3, Gmean.x[-1] / 1e3 -for pos, k, pflag in zip([gs[0, 0],gs[0, 1],gs[0, 2] ], high_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[0, 0], gs[0, 1], gs[0, 2]], high_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G_LS[k]#.rolling(k=10, x=2, min_periods= 1, center=True).mean() - #Gplot.mean('x').plot() + Gplot = G_LS[k] # .rolling(k=10, x=2, min_periods= 1, center=True).mean() + # Gplot.mean('x').plot() # #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) # - plot_wavenumber_spectrogram(ax0, Gplot, clev, title =k, plot_photon_density=True ) + plot_wavenumber_spectrogram(ax0, Gplot, clev, title=k, plot_photon_density=True) plt.xlim(xlims) # if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -pos, k, pflag = gs[1, 0], 'weighted mean', True +pos, k, pflag = gs[1, 0], "weighted mean", True ax0 = F.fig.add_subplot(pos) -Gplot = G_LS_wmean.rolling(k=5, x=2, min_periods= 1, center=True).median() +Gplot = G_LS_wmean.rolling(k=5, x=2, min_periods=1, center=True).median() -plot_wavenumber_spectrogram(ax0, Gplot, clev, title =k, plot_photon_density= True) +plot_wavenumber_spectrogram(ax0, Gplot, clev, title=k, plot_photon_density=True) plt.xlim(xlims) # plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) # ax0.axhline(30, color='black', linewidth=0.5) -ax0.axhline(1/k_max_range[0], color='red', linestyle= '--', linewidth= 0.5) -ax0.axhline(1/k_max_range[1], color='red', linestyle= '-', linewidth= 0.5) -ax0.axhline(1/k_max_range[2], color='red', linestyle= '--', linewidth= 0.5) +ax0.axhline(1 / k_max_range[0], color="red", linestyle="--", linewidth=0.5) +ax0.axhline(1 / k_max_range[1], color="red", linestyle="-", linewidth=0.5) +ax0.axhline(1 / k_max_range[2], color="red", linestyle="--", linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() pos = gs[1, 1] ax0 = F.fig.add_subplot(pos) -plt.title('Photons density ($m^{-1}$)', loc='left') - -for k,I in G_LS.items(): - plt.plot(Gplot.x/1e3, I.N_photons/Lmeters, label=k, linewidth=0.8) -plt.plot(Gplot.x/1e3, G_LS_wmean.N_photons/3/Lmeters , c='black', label='ave Photons' , linewidth=0.8) +plt.title("Photons density ($m^{-1}$)", loc="left") + +for k, I in G_LS.items(): + plt.plot(Gplot.x / 1e3, I.N_photons / Lmeters, label=k, linewidth=0.8) +plt.plot( + Gplot.x / 1e3, + G_LS_wmean.N_photons / 3 / Lmeters, + c="black", + label="ave Photons", + linewidth=0.8, +) plt.xlim(xlims) -plt.xlabel('Distance from the Ice Edge (km)') +plt.xlabel("Distance from the Ice Edge (km)") pos = gs[1, 2] ax0 = F.fig.add_subplot(pos) -ax0.set_yscale('log') - -plt.title('Peak Spectal Power', loc='left') - -for k,I in G_LS.items(): - plt.scatter(I.x.data/1e3, I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k').data *1e3 , s=0.5, marker='.', color='red', alpha= 0.3) - -for k,I in G_fft.items(): - I= I[:,I.N_per_stancil >= I.N_per_stancil.max().data*0.9] - plt.scatter(I.x/1e3, I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3 , s=0.5, marker='.', c='blue', alpha= 0.3) - - -Gplot= G_fft_wmean[:,G_fft_wmean.N_per_stancil >= G_fft_wmean.N_per_stancil.max().data*0.9] -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3, '.', markersize=1.5 , c='blue', label= 'FFT') - -Gplot= G_LS_wmean -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3 , '.' , markersize=1.5, c='red', label= 'LS') - -plt.ylabel('1e-3 $(m)^2~m$') +ax0.set_yscale("log") + +plt.title("Peak Spectal Power", loc="left") + +for k, I in G_LS.items(): + plt.scatter( + I.x.data / 1e3, + I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k").data * 1e3, + s=0.5, + marker=".", + color="red", + alpha=0.3, + ) + +for k, I in G_fft.items(): + I = I[:, I.N_per_stancil >= I.N_per_stancil.max().data * 0.9] + plt.scatter( + I.x / 1e3, + I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + s=0.5, + marker=".", + c="blue", + alpha=0.3, + ) + + +Gplot = G_fft_wmean[ + :, G_fft_wmean.N_per_stancil >= G_fft_wmean.N_per_stancil.max().data * 0.9 +] +plt.plot( + Gplot.x / 1e3, + Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + ".", + markersize=1.5, + c="blue", + label="FFT", +) + +Gplot = G_LS_wmean +plt.plot( + Gplot.x / 1e3, + Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + ".", + markersize=1.5, + c="red", + label="LS", +) + +plt.ylabel("1e-3 $(m)^2~m$") plt.legend() -#plt.ylim(Gplot.min()*1.4, Gplot.max()*1.4 ) +# plt.ylim(Gplot.min()*1.4, Gplot.max()*1.4 ) plt.xlim(xlims) -F.save_light(path=plot_path, name=track_name +'_exmpl_spec_LS_strong_L'+str(Lmeters)) +F.save_light(path=plot_path, name=track_name + "_exmpl_spec_LS_strong_L" + str(Lmeters)) diff --git a/analysis/B02_make_spectra_gFT_tester.py b/analysis/B02_make_spectra_gFT_tester.py index db592c0e..74b5cc68 100644 --- a/analysis/B02_make_spectra_gFT_tester.py +++ b/analysis/B02_make_spectra_gFT_tester.py @@ -1,15 +1,16 @@ # %% import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -25,190 +26,225 @@ # %% + def recreate_fft_weights(GG_xi, k): GG_xi.y_data[np.isnan(GG_xi.y_data)] = 0 y_gridded = GG_xi.y_data # take FFT to get peaj parameters - k_fft = np.fft.rfftfreq(GG_xi.eta.size, d=dx) * 2* np.pi - f_weight= np.sqrt(9.81 * k_fft) / (2 *np.pi) - data_fft = spec.Z_to_power(np.fft.rfft(y_gridded), np.diff(f_weight).mean(), GG_xi.eta.size) - + k_fft = np.fft.rfftfreq(GG_xi.eta.size, d=dx) * 2 * np.pi + f_weight = np.sqrt(9.81 * k_fft) / (2 * np.pi) + data_fft = spec.Z_to_power( + np.fft.rfft(y_gridded), np.diff(f_weight).mean(), GG_xi.eta.size + ) # create JONSWAP weight - Spec_fft = gFT.get_prior_spec(f_weight, data_fft ) - f= np.sqrt(9.81 * k) / (2 *np.pi) - weight = Spec_fft.create_weight(freq = f, plot_flag= False, max_nfev=None) + Spec_fft = gFT.get_prior_spec(f_weight, data_fft) + f = np.sqrt(9.81 * k) / (2 * np.pi) + weight = Spec_fft.create_weight(freq=f, plot_flag=False, max_nfev=None) return k_fft, data_fft, k, weight -#import s3fs -# %% -#track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190208152826_06440210_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190215184558_07530210_004_01', 'SH_batch01', False - - -ID_name, batch_key, ID_flag = 'SH_20190502_05160312', 'SH_publish', True -#ID_name, batch_key, ID_flag = 'SH_20190224_08800210', 'SH_publish', True -ID, _, hemis, batch = io.init_data(ID_name, batch_key, ID_flag, mconfig['paths']['work'], ) -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -load_path = mconfig['paths']['work'] + batch_key +'/B01_regrid/' -load_file = load_path + 'processed_' + ATlevel + '_' + ID_name + '.h5' - -save_path = mconfig['paths']['work'] + '/B02_spectra_'+hemis+'/' -save_name = 'B02_'+ID_name - -plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + ID_name + '/B_spectra/' +# import s3fs +# %% +# track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190208152826_06440210_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190215184558_07530210_004_01', 'SH_batch01', False + + +ID_name, batch_key, ID_flag = "SH_20190502_05160312", "SH_publish", True +# ID_name, batch_key, ID_flag = 'SH_20190224_08800210', 'SH_publish', True +ID, _, hemis, batch = io.init_data( + ID_name, + batch_key, + ID_flag, + mconfig["paths"]["work"], +) + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +load_path = mconfig["paths"]["work"] + batch_key + "/B01_regrid/" +load_file = load_path + "processed_" + ATlevel + "_" + ID_name + ".h5" + +save_path = mconfig["paths"]["work"] + "/B02_spectra_" + hemis + "/" +save_name = "B02_" + ID_name + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + ID_name + + "/B_spectra/" +) MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -#Gfilt = io.load_pandas_table_dict(ID_name + '_B01_regridded', load_path) # rhis is the rar photon data -Gd = io.load_pandas_table_dict(ID_name + '_B01_binned_' , load_path) # -Gd = h5py.File(load_path +'/'+ID_name + '_B01_binned.h5', 'r') +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +# Gfilt = io.load_pandas_table_dict(ID_name + '_B01_regridded', load_path) # rhis is the rar photon data +Gd = io.load_pandas_table_dict(ID_name + "_B01_binned_", load_path) # +Gd = h5py.File(load_path + "/" + ID_name + "_B01_binned.h5", "r") # %% test amount of nans in the data -nan_fraction= list() +nan_fraction = list() for I in Gd.values(): - nan_fraction.append( np.sum(np.isnan(I['heights_c_std'])) / I['heights_c_std'].shape[0] ) + nan_fraction.append( + np.sum(np.isnan(I["heights_c_std"])) / I["heights_c_std"].shape[0] + ) if np.array(nan_fraction).mean() > 0.95: - print('nan fraction > 95%, pass this track, add to bad tracks') - MT.json_save(ID_name, bad_track_path, {'nan_fraction': np.array(nan_fraction).mean(), 'date': str(datetime.date.today()) }) - print('exit.') + print("nan fraction > 95%, pass this track, add to bad tracks") + MT.json_save( + ID_name, + bad_track_path, + { + "nan_fraction": np.array(nan_fraction).mean(), + "date": str(datetime.date.today()), + }, + ) + print("exit.") exit() # %% test LS with an even grid where missing values are set to 0 imp.reload(spec) print(Gd.keys()) -Gi =Gd[ list(Gd.keys())[0] ] # to select a test beam +Gi = Gd[list(Gd.keys())[0]] # to select a test beam # derive spectal limits # Longest deserved period: -T_max = 40 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 -x = np.array(Gi['dist']) -dx = np.diff(x).mean() -min_datapoint = 2*np.pi/k_0/dx - -Lpoints = int(np.round(min_datapoint) * 10 ) -Lmeters = Lpoints * dx - -#plt.plot(np.diff(np.array(Gi['dist']))) -print('L number of gridpoint:', Lpoints) -print('L length in km:', Lmeters/1e3) -print('approx number windows', 2* Gi['dist'][:][-1] /Lmeters-1 ) - -T_min = 5 -lambda_min = 9.81 * T_min**2/ (2 *np.pi) -flim = 1/T_min - -oversample = 2 -dlambda = Lmeters * oversample -dk = 2 * np.pi/ dlambda -kk =np.arange(0, 1/lambda_min, 1/dlambda) * 2*np.pi -kk = kk[k_0<=kk] -#dk = np.diff(kk).mean() -print('2 M = ', kk.size *2 ) - -dk *150 +T_max = 40 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 +x = np.array(Gi["dist"]) +dx = np.diff(x).mean() +min_datapoint = 2 * np.pi / k_0 / dx + +Lpoints = int(np.round(min_datapoint) * 10) +Lmeters = Lpoints * dx + +# plt.plot(np.diff(np.array(Gi['dist']))) +print("L number of gridpoint:", Lpoints) +print("L length in km:", Lmeters / 1e3) +print("approx number windows", 2 * Gi["dist"][:][-1] / Lmeters - 1) + +T_min = 5 +lambda_min = 9.81 * T_min**2 / (2 * np.pi) +flim = 1 / T_min + +oversample = 2 +dlambda = Lmeters * oversample +dk = 2 * np.pi / dlambda +kk = np.arange(0, 1 / lambda_min, 1 / dlambda) * 2 * np.pi +kk = kk[k_0 <= kk] +# dk = np.diff(kk).mean() +print("2 M = ", kk.size * 2) + +dk * 150 # %% -G_gFT= dict() +G_gFT = dict() G_gFT_x = dict() -G_rar_fft= dict() +G_rar_fft = dict() Pars_optm = dict() imp.reload(spec) imp.reload(gFT) -k=all_beams[1] -#for k in all_beams: +k = all_beams[1] +# for k in all_beams: # ------------------------------- use gridded data -hkey= 'heights_c_weighted_mean' -x = Gd[k]['dist'][:] -xlims = x[0], x[-1] -dd = np.copy(Gd[k][hkey]) +hkey = "heights_c_weighted_mean" +x = Gd[k]["dist"][:] +xlims = x[0], x[-1] +dd = np.copy(Gd[k][hkey]) -dd_error = np.copy(Gd[k]['heights_c_std']) +dd_error = np.copy(Gd[k]["heights_c_std"]) dd_error[np.isnan(dd_error)] = 100 -#plt.hist(1/dd_weight, bins=40) +# plt.hist(1/dd_weight, bins=40) F = M.figure_axis_xy(6, 3) plt.subplot(2, 1, 1) -#plt.plot(x, dd, 'gray', label='displacement (m) ') +# plt.plot(x, dd, 'gray', label='displacement (m) ') # compute slope spectra !! -dd = np.gradient(dd) -dd, _ = spicke_remover.spicke_remover(dd, spreed=10, verbose=False) -dd_nans = (np.isnan(dd) ) + (Gd[k]['N_photos'][:] <= 5) +dd = np.gradient(dd) +dd, _ = spicke_remover.spicke_remover(dd, spreed=10, verbose=False) +dd_nans = (np.isnan(dd)) + (Gd[k]["N_photos"][:] <= 5) dd_filled = np.copy(dd) dd_filled[dd_nans] = 0 -#win = create_weighted_window(dd_filled) +# win = create_weighted_window(dd_filled) # using gappy data -Nsel= 10000 -dd_no_nans = dd[~dd_nans][0:Nsel] # windowing is applied here -x_no_nans = x[~dd_nans][0:Nsel] +Nsel = 10000 +dd_no_nans = dd[~dd_nans][0:Nsel] # windowing is applied here +x_no_nans = x[~dd_nans][0:Nsel] dd_error_no_nans = dd_error[~dd_nans][0:Nsel] -plt.plot(x_no_nans, dd_no_nans, '.', 'black', label='slope (m/m)') +plt.plot(x_no_nans, dd_no_nans, ".", "black", label="slope (m/m)") plt.legend() plt.show() - # %% -xlims = xlims[0], xlims[0] + (xlims[1] -xlims[0])/2 +xlims = xlims[0], xlims[0] + (xlims[1] - xlims[0]) / 2 -print('gFT') +print("gFT") font_for_print() -#S_pwelch_k2 = np.arange(S_pwelch_k[1], S_pwelch_k[-1], S_pwelch_dk*2 ) +# S_pwelch_k2 = np.arange(S_pwelch_k[1], S_pwelch_k[-1], S_pwelch_dk*2 ) imp.reload(gFT) -S = gFT.wavenumber_spectrogram_gFT( np.array(x_no_nans), np.array(dd_no_nans), Lmeters, dx, kk, data_error = dd_error_no_nans, ov=None) -GG, GG_x, Params = S.cal_spectrogram(xlims= xlims, max_nfev = None, plot_flag = True) +S = gFT.wavenumber_spectrogram_gFT( + np.array(x_no_nans), + np.array(dd_no_nans), + Lmeters, + dx, + kk, + data_error=dd_error_no_nans, + ov=None, +) +GG, GG_x, Params = S.cal_spectrogram(xlims=xlims, max_nfev=None, plot_flag=True) # %% -xsel= 0 +xsel = 0 GG_xi = GG_x.isel(x=xsel) GGi = GG.isel(x=xsel) fft_k, fft_data, k, weight = recreate_fft_weights(GG_xi, GGi.k) font_for_pres() -F= M.figure_axis_xy(6, 3, view_scale= 0.8) -GGi.gFT_PSD_data.plot(color ='red', lw= 0.8, label= 'GFT') -GGi.weight.plot(color = 'green', label= 'weight') +F = M.figure_axis_xy(6, 3, view_scale=0.8) +GGi.gFT_PSD_data.plot(color="red", lw=0.8, label="GFT") +GGi.weight.plot(color="green", label="weight") -plt.plot(fft_k, fft_data, 'gray', lw=0.5, label= 'FFT', zorder= 3) -plt.plot(k, weight, 'k--', label= 'fitten PM model') +plt.plot(fft_k, fft_data, "gray", lw=0.5, label="FFT", zorder=3) +plt.plot(k, weight, "k--", label="fitten PM model") -plt.legend(ncol =2) +plt.legend(ncol=2) plt.xlim(k[0], k[-1]) plt.ylim(0, np.quantile(fft_data, 0.98)) -plt.ylabel('PSD ( (m/m)$^2$/k)') -plt.xlabel('wavenumber (k)') +plt.ylabel("PSD ( (m/m)$^2$/k)") +plt.xlabel("wavenumber (k)") -plt.title('x='+ str(np.round(GG_xi.x.data/1e3,1))+ 'km, N=' + str(int(GGi.N_per_stancil)) ) +plt.title( + "x=" + str(np.round(GG_xi.x.data / 1e3, 1)) + "km, N=" + str(int(GGi.N_per_stancil)) +) # %% @@ -232,8 +268,8 @@ def recreate_fft_weights(GG_xi, k): # plt.plot(Params.T['alpha'], '.') # %% -def linear_gap_fill(F, key_lead, key_int): +def linear_gap_fill(F, key_lead, key_int): """ F pd.DataFrame key_lead key in F that determined the independent coordindate @@ -241,105 +277,135 @@ def linear_gap_fill(F, key_lead, key_int): """ y_g = np.array(F[key_int]) - nans, x2= np.isnan(y_g), lambda z: z.nonzero()[0] - y_g[nans]= np.interp(x2(nans), x2(~nans), y_g[~nans]) + nans, x2 = np.isnan(y_g), lambda z: z.nonzero()[0] + y_g[nans] = np.interp(x2(nans), x2(~nans), y_g[~nans]) return y_g - -plot_data_model=True +plot_data_model = True if plot_data_model: - for i in np.arange(30,60,2): - c1= 'blue' - c2= 'red' + for i in np.arange(30, 60, 2): + c1 = "blue" + c2 = "red" - GGi = GG.isel(x= i) + GGi = GG.isel(x=i) - xi_1=GG_x.x[i] - xi_2=GG_x.x[i+1] - #if k%2 ==0: + xi_1 = GG_x.x[i] + xi_2 = GG_x.x[i + 1] + # if k%2 ==0: F = M.figure_axis_xy(16, 2) - eta = GG_x.eta + eta = GG_x.eta # gFT model y_model = GG_x.y_model[:, i] - plt.plot(eta +xi_1, y_model ,'-', c=c1, linewidth=0.8, alpha=1, zorder=12) - y_model = GG_x.y_model[:, i+1] - plt.plot(eta +xi_2, y_model,'-', c=c2, linewidth=0.8, alpha=1, zorder=12) + plt.plot(eta + xi_1, y_model, "-", c=c1, linewidth=0.8, alpha=1, zorder=12) + y_model = GG_x.y_model[:, i + 1] + plt.plot(eta + xi_2, y_model, "-", c=c2, linewidth=0.8, alpha=1, zorder=12) # iterpolated model in gaps - FT = gFT.generalized_Fourier(eta +xi_1, None,GG.k ) + FT = gFT.generalized_Fourier(eta + xi_1, None, GG.k) _ = FT.get_H() - FT.b_hat=np.concatenate([ GGi.gFT_cos_coeff, GGi.gFT_sin_coeff ]) - plt.plot(eta +xi_1, FT.model() ,'-', c='orange', linewidth=0.8, alpha=1,zorder= 2) + FT.b_hat = np.concatenate([GGi.gFT_cos_coeff, GGi.gFT_sin_coeff]) + plt.plot( + eta + xi_1, FT.model(), "-", c="orange", linewidth=0.8, alpha=1, zorder=2 + ) - FT = gFT.generalized_Fourier(eta +xi_2, None,GG.k ) + FT = gFT.generalized_Fourier(eta + xi_2, None, GG.k) _ = FT.get_H() - FT.b_hat=np.concatenate([ GGi.gFT_cos_coeff, GGi.gFT_sin_coeff ]) - plt.plot(eta +xi_2, FT.model() ,'-', c='orange', linewidth=0.8, alpha=1,zorder= 2) - + FT.b_hat = np.concatenate([GGi.gFT_cos_coeff, GGi.gFT_sin_coeff]) + plt.plot( + eta + xi_2, FT.model(), "-", c="orange", linewidth=0.8, alpha=1, zorder=2 + ) # oringial data - #plt.plot(x, dd, '-', c='k',linewidth=3, alpha =0.6, zorder=11) - #plt.plot(x, dd, '.', c='k',markersize=3, alpha =0.5) + # plt.plot(x, dd, '-', c='k',linewidth=3, alpha =0.6, zorder=11) + # plt.plot(x, dd, '.', c='k',markersize=3, alpha =0.5) # oringal data from model_output y_data = GG_x.y_data[:, i] - plt.plot(eta +xi_1, y_data ,'-', c='k',linewidth=3, alpha =0.3, zorder=11) - y_data = GG_x.y_data[:, i+1] - plt.plot(eta +xi_2, y_data,'-', c='k',linewidth=3, alpha =0.3, zorder=11) - - - #plt.plot(x[~dd_nans], dd_error[~dd_nans], '.', c='green',linewidth=1, alpha =0.5) - - F.ax.axvline(xi_1 + eta[0].data , linewidth=4, color=c1, alpha=0.5) - F.ax.axvline(xi_1 + eta[-1].data, linewidth=4, color=c1, alpha=0.5) - F.ax.axvline(xi_2 + eta[0].data , linewidth=4, color=c2, alpha=0.5) - F.ax.axvline(xi_2 + eta[-1].data, linewidth=4, color=c2, alpha=0.5) - - ylims= -np.nanstd(dd)*2, np.nanstd(dd)*2 - plt.text(xi_1 + eta[0].data, ylims[-1], ' N='+ str(GG.sel(x=xi_1, method='nearest').N_per_stancil.data) + ' N/2M= '+ str(GG.sel(x=xi_1, method='nearest').N_per_stancil.data/2/kk.size) ) - plt.text(xi_2 + eta[0].data, ylims[-1], ' N='+ str(GG.sel(x=xi_2, method='nearest').N_per_stancil.data) + ' N/2M= '+ str(GG.sel(x=xi_2, method='nearest').N_per_stancil.data/2/kk.size) ) - plt.xlim(xi_1 + eta[0].data*1.2, xi_2 + eta[-1].data*1.2 ) - #plt.xlim(xi_1, xi_2 ) - + plt.plot(eta + xi_1, y_data, "-", c="k", linewidth=3, alpha=0.3, zorder=11) + y_data = GG_x.y_data[:, i + 1] + plt.plot(eta + xi_2, y_data, "-", c="k", linewidth=3, alpha=0.3, zorder=11) + + # plt.plot(x[~dd_nans], dd_error[~dd_nans], '.', c='green',linewidth=1, alpha =0.5) + + F.ax.axvline(xi_1 + eta[0].data, linewidth=4, color=c1, alpha=0.5) + F.ax.axvline(xi_1 + eta[-1].data, linewidth=4, color=c1, alpha=0.5) + F.ax.axvline(xi_2 + eta[0].data, linewidth=4, color=c2, alpha=0.5) + F.ax.axvline(xi_2 + eta[-1].data, linewidth=4, color=c2, alpha=0.5) + + ylims = -np.nanstd(dd) * 2, np.nanstd(dd) * 2 + plt.text( + xi_1 + eta[0].data, + ylims[-1], + " N=" + + str(GG.sel(x=xi_1, method="nearest").N_per_stancil.data) + + " N/2M= " + + str(GG.sel(x=xi_1, method="nearest").N_per_stancil.data / 2 / kk.size), + ) + plt.text( + xi_2 + eta[0].data, + ylims[-1], + " N=" + + str(GG.sel(x=xi_2, method="nearest").N_per_stancil.data) + + " N/2M= " + + str(GG.sel(x=xi_2, method="nearest").N_per_stancil.data / 2 / kk.size), + ) + plt.xlim(xi_1 + eta[0].data * 1.2, xi_2 + eta[-1].data * 1.2) + # plt.xlim(xi_1, xi_2 ) plt.ylim(ylims[0], ylims[-1]) plt.show() # % -#S.mean_spectral_error() # add x-mean spectal error estimate to xarray -S.parceval(add_attrs= True, weight_data=False) +# S.mean_spectral_error() # add x-mean spectal error estimate to xarray +S.parceval(add_attrs=True, weight_data=False) # assign beam coordinate -GG.coords['beam'], GG_x.coords['beam'] = str(k), str(k) -GG, GG_x = GG.expand_dims(dim = 'beam', axis = 1), GG_x.expand_dims(dim = 'beam', axis = 1) +GG.coords["beam"], GG_x.coords["beam"] = str(k), str(k) +GG, GG_x = GG.expand_dims(dim="beam", axis=1), GG_x.expand_dims(dim="beam", axis=1) # repack such that all coords are associated with beam -GG.coords['N_per_stancil'] = (('x', 'beam' ), np.expand_dims(GG['N_per_stancil'], 1)) +GG.coords["N_per_stancil"] = (("x", "beam"), np.expand_dims(GG["N_per_stancil"], 1)) # add more coodindates to the Dataset -x_coord_no_gaps = linear_gap_fill( Gd[k], 'dist', 'x' ) -y_coord_no_gaps = linear_gap_fill( Gd[k], 'dist', 'y' ) -mapped_coords = spec.sub_sample_coords(Gd[k]['dist'], x_coord_no_gaps, y_coord_no_gaps, S.stancil_iter , map_func = None ) - -GG.coords['x_coord'] = GG_x.coords['x_coord'] = (('x', 'beam' ), np.expand_dims(mapped_coords[:,1], 1) ) -GG.coords['y_coord'] = GG_x.coords['y_coord'] = (('x', 'beam' ), np.expand_dims(mapped_coords[:,2], 1) ) +x_coord_no_gaps = linear_gap_fill(Gd[k], "dist", "x") +y_coord_no_gaps = linear_gap_fill(Gd[k], "dist", "y") +mapped_coords = spec.sub_sample_coords( + Gd[k]["dist"], x_coord_no_gaps, y_coord_no_gaps, S.stancil_iter, map_func=None +) + +GG.coords["x_coord"] = GG_x.coords["x_coord"] = ( + ("x", "beam"), + np.expand_dims(mapped_coords[:, 1], 1), +) +GG.coords["y_coord"] = GG_x.coords["y_coord"] = ( + ("x", "beam"), + np.expand_dims(mapped_coords[:, 2], 1), +) # if data staarts with nans replace coords with nans again -if (GG.coords['N_per_stancil'] == 0).squeeze()[0].data: - nlabel = label( (GG.coords['N_per_stancil'] == 0).squeeze())[0] - nan_mask= nlabel ==nlabel[0] - GG.coords['x_coord'][nan_mask] =np.nan - GG.coords['y_coord'][nan_mask] =np.nan - -lons_no_gaps = linear_gap_fill( Gd[k], 'dist', 'lons' ) -lats_no_gaps = linear_gap_fill( Gd[k], 'dist', 'lats' ) -mapped_coords = spec.sub_sample_coords(Gd[k]['dist'], lons_no_gaps, lats_no_gaps, S.stancil_iter , map_func = None ) - -GG.coords['lon'] = GG_x.coords['lon'] = (('x', 'beam' ), np.expand_dims(mapped_coords[:,1], 1) ) -GG.coords['lat'] = GG_x.coords['lat'] = (('x', 'beam' ), np.expand_dims(mapped_coords[:,2], 1) ) +if (GG.coords["N_per_stancil"] == 0).squeeze()[0].data: + nlabel = label((GG.coords["N_per_stancil"] == 0).squeeze())[0] + nan_mask = nlabel == nlabel[0] + GG.coords["x_coord"][nan_mask] = np.nan + GG.coords["y_coord"][nan_mask] = np.nan + +lons_no_gaps = linear_gap_fill(Gd[k], "dist", "lons") +lats_no_gaps = linear_gap_fill(Gd[k], "dist", "lats") +mapped_coords = spec.sub_sample_coords( + Gd[k]["dist"], lons_no_gaps, lats_no_gaps, S.stancil_iter, map_func=None +) + +GG.coords["lon"] = GG_x.coords["lon"] = ( + ("x", "beam"), + np.expand_dims(mapped_coords[:, 1], 1), +) +GG.coords["lat"] = GG_x.coords["lat"] = ( + ("x", "beam"), + np.expand_dims(mapped_coords[:, 2], 1), +) # plt.plot(Gd[k]['x'], Gd[k]['y'] , '.') # plt.plot(GG.coords['x_coord'], GG.coords['y_coord'] , '.r') @@ -347,264 +413,351 @@ def linear_gap_fill(F, key_lead, key_int): # plt.plot(Gd[k]['lons'], Gd[k]['lats'] , '.') # plt.plot(GG.coords['lon'], GG.coords['lat'] , '.r') # spectral errors are cacualted within S and now repacked to main DataSet G -#G.coords['mean_El'] = (('k', 'beam' ), np.expand_dims(S.G['mean_El'], 1)) -#G.coords['mean_Eu'] = (('k', 'beam' ), np.expand_dims(S.G['mean_Eu'], 1)) +# G.coords['mean_El'] = (('k', 'beam' ), np.expand_dims(S.G['mean_El'], 1)) +# G.coords['mean_Eu'] = (('k', 'beam' ), np.expand_dims(S.G['mean_Eu'], 1)) + # calculate number data points def get_stancil_nans(stancil): x_mask = (stancil[0] < x) & (x <= stancil[-1]) - idata = Gd[k]['N_photos'][x_mask] + idata = Gd[k]["N_photos"][x_mask] return stancil[1], idata.sum() -photon_list = np.array(list(dict(map( get_stancil_nans, copy.copy(S.stancil_iter) )).values())) -GG.coords['N_photons'] = (('x', 'beam' ), np.expand_dims(photon_list, 1)) + +photon_list = np.array( + list(dict(map(get_stancil_nans, copy.copy(S.stancil_iter))).values()) +) +GG.coords["N_photons"] = (("x", "beam"), np.expand_dims(photon_list, 1)) # Save to dict -G_gFT[k] = GG -G_gFT_x[k] = GG_x +G_gFT[k] = GG +G_gFT_x[k] = GG_x Pars_optm[k] = Params # plot plt.subplot(2, 1, 2) G_gFT_power = GG.gFT_PSD_data.squeeze() -plt.plot(G_gFT_power.k, np.nanmean(G_gFT_power,1), 'gray', label='mean gFT power data ') +plt.plot( + G_gFT_power.k, np.nanmean(G_gFT_power, 1), "gray", label="mean gFT power data " +) G_gFT_power = GG.gFT_PSD_model.squeeze() -plt.plot(GG.k, np.nanmean(S.G, 1), 'k', label='mean LS power model') +plt.plot(GG.k, np.nanmean(S.G, 1), "k", label="mean LS power model") # standard FFT -print('FFT') -dd[dd_nans] = 0 +print("FFT") +dd[dd_nans] = 0 S = spec.wavenumber_spectrogram(x, dd, Lpoints) G = S.cal_spectrogram() -S.mean_spectral_error() # add x-mean spectal error estimate to xarray -S.parceval(add_attrs= True) +S.mean_spectral_error() # add x-mean spectal error estimate to xarray +S.parceval(add_attrs=True) # assign beam coordinate -G.coords['beam'] = str(k)#(('beam'), str(k)) -G = G.expand_dims(dim = 'beam', axis = 2) -G.coords['mean_El'] = (('k', 'beam' ), np.expand_dims(G['mean_El'], 1)) -G.coords['mean_Eu'] = (('k', 'beam' ), np.expand_dims(G['mean_Eu'], 1)) -G.coords['x'] = G.coords['x'] * dx # adjust x-coodinate definition +G.coords["beam"] = str(k) # (('beam'), str(k)) +G = G.expand_dims(dim="beam", axis=2) +G.coords["mean_El"] = (("k", "beam"), np.expand_dims(G["mean_El"], 1)) +G.coords["mean_Eu"] = (("k", "beam"), np.expand_dims(G["mean_Eu"], 1)) +G.coords["x"] = G.coords["x"] * dx # adjust x-coodinate definition stancil_iter = spec.create_chunk_boundaries(int(Lpoints), dd_nans.size) + + def get_stancil_nans(stancil): - idata = dd_nans[stancil[0]:stancil[-1]] + idata = dd_nans[stancil[0] : stancil[-1]] return stancil[1], idata.size - idata.sum() -N_list = np.array(list(dict(map( get_stancil_nans, stancil_iter )).values())) + +N_list = np.array(list(dict(map(get_stancil_nans, stancil_iter)).values())) # repack such that all coords are associated with beam -G.coords['N_per_stancil'] = (('x', 'beam' ), np.expand_dims(N_list, 1)) +G.coords["N_per_stancil"] = (("x", "beam"), np.expand_dims(N_list, 1)) -#save to dict +# save to dict G_rar_fft[k] = G # for plotting -G_rar_fft_p =G.squeeze() -plt.plot(G.k, G_rar_fft_p[:, G_rar_fft_p['N_per_stancil'] > 10 ].mean('x'), 'darkblue', label='mean FFT') -#plt.plot(G.k, GG.mean('x'), 'lightblue', label='mean FFT') +G_rar_fft_p = G.squeeze() +plt.plot( + G.k, + G_rar_fft_p[:, G_rar_fft_p["N_per_stancil"] > 10].mean("x"), + "darkblue", + label="mean FFT", +) +# plt.plot(G.k, GG.mean('x'), 'lightblue', label='mean FFT') plt.legend() plt.show() -F.save_light(path=plot_path, name = 'B02_control_'+k+'_' + track_name) -#print('saved as '+'B02_control_'+k+'_' + track_name) -#print(np.isinf(G).sum().data) - +F.save_light(path=plot_path, name="B02_control_" + k + "_" + track_name) +# print('saved as '+'B02_control_'+k+'_' + track_name) +# print(np.isinf(G).sum().data) # %% + def dict_weighted_mean(Gdict, weight_key): """ returns the weighted meean of a dict of xarray, data_arrays weight_key must be in the xr.DataArrays """ - #Gdict = G_LS + # Gdict = G_LS # weight_key='N_per_stancil' - akey = list( Gdict.keys() )[0] + akey = list(Gdict.keys())[0] GSUM = Gdict[akey].copy() - GSUM.data = np.zeros(GSUM.shape) + GSUM.data = np.zeros(GSUM.shape) N_per_stancil = GSUM.N_per_stancil * 0 - N_photons = np.zeros(GSUM.N_per_stancil.size) + N_photons = np.zeros(GSUM.N_per_stancil.size) - counter= 0 - for k,I in Gdict.items(): - I =I.squeeze() - GSUM += I.where( ~np.isnan(I), 0) * I[weight_key] #.sel(x=GSUM.x) - N_per_stancil += I[weight_key] - if 'N_photons' in GSUM.coords: - N_photons += I['N_photons'] - counter+=1 + counter = 0 + for k, I in Gdict.items(): + I = I.squeeze() + GSUM += I.where(~np.isnan(I), 0) * I[weight_key] # .sel(x=GSUM.x) + N_per_stancil += I[weight_key] + if "N_photons" in GSUM.coords: + N_photons += I["N_photons"] + counter += 1 - GSUM = GSUM / N_per_stancil /counter + GSUM = GSUM / N_per_stancil / counter - if 'N_photons' in GSUM.coords: - GSUM.coords['N_photons'] = (('x', 'beam'), np.expand_dims(N_photons, 1) ) + if "N_photons" in GSUM.coords: + GSUM.coords["N_photons"] = (("x", "beam"), np.expand_dims(N_photons, 1)) - GSUM['beam'] = ['weighted_mean'] - GSUM.name='power_spec' + GSUM["beam"] = ["weighted_mean"] + GSUM.name = "power_spec" return GSUM + G_gFT_sel = {} -for k,I in G_gFT.items(): - G_gFT_sel[k] = I['gFT_PSD_model'] +for k, I in G_gFT.items(): + G_gFT_sel[k] = I["gFT_PSD_model"] -G_gFT_wmean = dict_weighted_mean(G_gFT_sel, 'N_per_stancil') -G_fft_wmean = dict_weighted_mean(G_rar_fft, 'N_per_stancil') +G_gFT_wmean = dict_weighted_mean(G_gFT_sel, "N_per_stancil") +G_fft_wmean = dict_weighted_mean(G_rar_fft, "N_per_stancil") # %% plot -def plot_wavenumber_spectrogram(ax, Gi, clev, title= None, plot_photon_density=True ): +def plot_wavenumber_spectrogram(ax, Gi, clev, title=None, plot_photon_density=True): - if Gi.k[0] ==0: - Gi= Gi.sel(k=Gi.k[1:]) - x_lambda= 1/Gi.k - plt.pcolormesh(Gi.x/1e3, x_lambda , Gi, cmap=plt.cm.ocean_r , vmin = clev[0], vmax = clev[-1]) + if Gi.k[0] == 0: + Gi = Gi.sel(k=Gi.k[1:]) + x_lambda = 1 / Gi.k + plt.pcolormesh( + Gi.x / 1e3, x_lambda, Gi, cmap=plt.cm.ocean_r, vmin=clev[0], vmax=clev[-1] + ) - ax.set_yscale('log') + ax.set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') if plot_photon_density: - plt.plot(Gi.x/1e3, x_lambda[-1] + (Gi.N_per_stancil/Gi.N_per_stancil.max() ) * 10 , c='black', linewidth= 0.8, label='NAN-density' ) - plt.fill_between(Gi.x/1e3, x_lambda[-1] + (Gi.N_per_stancil/Gi.N_per_stancil.max() ) * 10, 0, color='gray', alpha = 0.3) - ax.axhline(30, color='black', linewidth=0.3) - - #plt.xlabel('Distance from the Ice Edge (km)') + plt.plot( + Gi.x / 1e3, + x_lambda[-1] + (Gi.N_per_stancil / Gi.N_per_stancil.max()) * 10, + c="black", + linewidth=0.8, + label="NAN-density", + ) + plt.fill_between( + Gi.x / 1e3, + x_lambda[-1] + (Gi.N_per_stancil / Gi.N_per_stancil.max()) * 10, + 0, + color="gray", + alpha=0.3, + ) + ax.axhline(30, color="black", linewidth=0.3) + + # plt.xlabel('Distance from the Ice Edge (km)') plt.ylim(x_lambda[-1], x_lambda[0]) - plt.title(title, loc='left') + plt.title(title, loc="left") + -#Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() -#Gmean = G_gFT_wmean.rolling(x=2, min_periods= 1, center=True).mean() +# Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() +# Gmean = G_gFT_wmean.rolling(x=2, min_periods= 1, center=True).mean() Gmean = G_gFT_wmean.rolling(k=5, center=True).mean() -#Gmean = Gmean.where(~np.isnan(Gmean), 0) -k_max_range = Gmean.k[Gmean.mean('x').argmax().data].data* 0.75, Gmean.k[Gmean.mean('x').argmax().data].data* 1, Gmean.k[Gmean.mean('x').argmax().data].data* 1.25 +# Gmean = Gmean.where(~np.isnan(Gmean), 0) +k_max_range = ( + Gmean.k[Gmean.mean("x").argmax().data].data * 0.75, + Gmean.k[Gmean.mean("x").argmax().data].data * 1, + Gmean.k[Gmean.mean("x").argmax().data].data * 1.25, +) # %% font_for_print() -F = M.figure_axis_xy(6.5, 5.6, container= True, view_scale =1) +F = M.figure_axis_xy(6.5, 5.6, container=True, view_scale=1) -plt.suptitle('LS and FFT Slope Spectrograms\n' + track_name, y = 0.98) -gs = GridSpec(3,3, wspace=0.2, hspace=.5)#figure=fig, -#clev=np.arange(0, 6, 0.1)*3 +plt.suptitle("LS and FFT Slope Spectrograms\n" + track_name, y=0.98) +gs = GridSpec(3, 3, wspace=0.2, hspace=0.5) # figure=fig, +# clev=np.arange(0, 6, 0.1)*3 -#%matplotlib inline +# %matplotlib inline -clev = M.clevels( [Gmean.quantile(0.8).data * 1e4, Gmean.quantile(0.99).data * 1e4], 31)/ 1e4 +clev = ( + M.clevels([Gmean.quantile(0.8).data * 1e4, Gmean.quantile(0.99).data * 1e4], 31) + / 1e4 +) -xlims= Gmean.x[0]/1e3, Gmean.x[-1]/1e3 +xlims = Gmean.x[0] / 1e3, Gmean.x[-1] / 1e3 -for pos, k, pflag in zip([gs[0, 0],gs[0, 1],gs[0, 2] ], high_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[0, 0], gs[0, 1], gs[0, 2]], high_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G_gFT_sel[k].squeeze()#.rolling(k=10, x=2, min_periods= 1, center=True).mean() - #Gplot.mean('x').plot() + Gplot = G_gFT_sel[ + k + ].squeeze() # .rolling(k=10, x=2, min_periods= 1, center=True).mean() + # Gplot.mean('x').plot() # #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) # - #np.log(Gplot).plot() - plot_wavenumber_spectrogram(ax0, Gplot, clev, title =k, plot_photon_density=True ) + # np.log(Gplot).plot() + plot_wavenumber_spectrogram(ax0, Gplot, clev, title=k, plot_photon_density=True) plt.xlim(xlims) # if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -for pos, k, pflag in zip([gs[1, 0],gs[1, 1],gs[1, 2] ], low_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[1, 0], gs[1, 1], gs[1, 2]], low_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G_gFT_sel[k].squeeze()#.rolling(k=10, x=2, min_periods= 1, center=True).mean() - #Gplot.mean('x').plot() + Gplot = G_gFT_sel[ + k + ].squeeze() # .rolling(k=10, x=2, min_periods= 1, center=True).mean() + # Gplot.mean('x').plot() # #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) # - plot_wavenumber_spectrogram(ax0, Gplot, clev, title =k, plot_photon_density=True ) + plot_wavenumber_spectrogram(ax0, Gplot, clev, title=k, plot_photon_density=True) plt.xlim(xlims) # if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -pos, k, pflag = gs[2, 0], 'Power(weighted mean) \n10 $\log_{10}( (m/m)^2 m )$', True +pos, k, pflag = gs[2, 0], "Power(weighted mean) \n10 $\log_{10}( (m/m)^2 m )$", True ax0 = F.fig.add_subplot(pos) -Gplot = G_gFT_wmean.squeeze().rolling(k=5, min_periods= 1, center=True).median().rolling(x=3, min_periods= 1, center=True).median() +Gplot = ( + G_gFT_wmean.squeeze() + .rolling(k=5, min_periods=1, center=True) + .median() + .rolling(x=3, min_periods=1, center=True) + .median() +) dd = 10 * np.log10(Gplot) -dd= dd.where(~np.isinf(dd), np.nan ) -clev_log = M.clevels( [dd.quantile(0.01).data, dd.quantile(0.98).data * 1.2], 31)* 1 -plot_wavenumber_spectrogram(ax0, dd, clev_log , title =k, plot_photon_density= True) +dd = dd.where(~np.isinf(dd), np.nan) +clev_log = M.clevels([dd.quantile(0.01).data, dd.quantile(0.98).data * 1.2], 31) * 1 +plot_wavenumber_spectrogram(ax0, dd, clev_log, title=k, plot_photon_density=True) plt.xlim(xlims) # plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) # ax0.axhline(30, color='black', linewidth=0.5) -ax0.axhline(1/k_max_range[0], color='red', linestyle= '--', linewidth= 0.5) -ax0.axhline(1/k_max_range[1], color='red', linestyle= '-', linewidth= 0.5) -ax0.axhline(1/k_max_range[2], color='red', linestyle= '--', linewidth= 0.5) +ax0.axhline(1 / k_max_range[0], color="red", linestyle="--", linewidth=0.5) +ax0.axhline(1 / k_max_range[1], color="red", linestyle="-", linewidth=0.5) +ax0.axhline(1 / k_max_range[2], color="red", linestyle="--", linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() pos = gs[2, 1] ax0 = F.fig.add_subplot(pos) -plt.title('Photons density ($m^{-1}$)', loc='left') - -for k,I in G_gFT.items(): - plt.plot(Gplot.x/1e3, I.N_photons/Lmeters, label=k, linewidth=0.8) -plt.plot(Gplot.x/1e3, G_gFT_wmean.N_photons/3/Lmeters , c='black', label='ave Photons' , linewidth=0.8) +plt.title("Photons density ($m^{-1}$)", loc="left") + +for k, I in G_gFT.items(): + plt.plot(Gplot.x / 1e3, I.N_photons / Lmeters, label=k, linewidth=0.8) +plt.plot( + Gplot.x / 1e3, + G_gFT_wmean.N_photons / 3 / Lmeters, + c="black", + label="ave Photons", + linewidth=0.8, +) plt.xlim(xlims) -plt.xlabel('Distance from the Ice Edge (km)') +plt.xlabel("Distance from the Ice Edge (km)") pos = gs[2, 2] ax0 = F.fig.add_subplot(pos) -ax0.set_yscale('log') - -plt.title('Peak Spectal Power', loc='left') - -for k,I in G_gFT_sel.items(): - plt.scatter(I.x.data/1e3, I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k').data *1e3 , s=0.5, marker='.', color='red', alpha= 0.3) - -for k,I in G_rar_fft.items(): - I= I.squeeze() - I= I[:,I.N_per_stancil >= I.N_per_stancil.max().data*0.9] - plt.scatter(I.x/1e3, I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3 , s=0.5, marker='.', c='blue', alpha= 0.3) - - -Gplot= G_fft_wmean.squeeze() -Gplot = Gplot[:,Gplot.N_per_stancil >= Gplot.N_per_stancil.max().data*0.9] -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3, '.', markersize=1.5 , c='blue', label= 'FFT') - -Gplot= G_gFT_wmean.squeeze() -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3 , '.' , markersize=1.5, c='red', label= 'gFT') - -plt.ylabel('1e-3 $(m)^2~m$') +ax0.set_yscale("log") + +plt.title("Peak Spectal Power", loc="left") + +for k, I in G_gFT_sel.items(): + plt.scatter( + I.x.data / 1e3, + I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k").data * 1e3, + s=0.5, + marker=".", + color="red", + alpha=0.3, + ) + +for k, I in G_rar_fft.items(): + I = I.squeeze() + I = I[:, I.N_per_stancil >= I.N_per_stancil.max().data * 0.9] + plt.scatter( + I.x / 1e3, + I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + s=0.5, + marker=".", + c="blue", + alpha=0.3, + ) + + +Gplot = G_fft_wmean.squeeze() +Gplot = Gplot[:, Gplot.N_per_stancil >= Gplot.N_per_stancil.max().data * 0.9] +plt.plot( + Gplot.x / 1e3, + Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + ".", + markersize=1.5, + c="blue", + label="FFT", +) + +Gplot = G_gFT_wmean.squeeze() +plt.plot( + Gplot.x / 1e3, + Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + ".", + markersize=1.5, + c="red", + label="gFT", +) + +plt.ylabel("1e-3 $(m)^2~m$") plt.legend() -plt.ylim(Gplot.min()*1.4, Gplot.max()*1.4 ) +plt.ylim(Gplot.min() * 1.4, Gplot.max() * 1.4) plt.xlim(xlims) -F.save_light(path=plot_path, name = 'B02_specs_' + track_name +'_L'+str(Lmeters)) +F.save_light(path=plot_path, name="B02_specs_" + track_name + "_L" + str(Lmeters)) # %% save fitting parameters -MT.save_pandas_table(Pars_optm, save_name+'_params', save_path ) +MT.save_pandas_table(Pars_optm, save_name + "_params", save_path) + # %% repack data def repack_attributes(DD): - #DD = G_LS + # DD = G_LS attr_dim_list = list(DD.keys()) for k in attr_dim_list: for ka in list(DD[k].attrs.keys()): I = DD[k] - I.coords[ka] = ( 'beam', np.expand_dims(I.attrs[ka], 0) ) + I.coords[ka] = ("beam", np.expand_dims(I.attrs[ka], 0)) return DD -G_gFT[G_gFT_wmean.beam.data[0]] =G_gFT_wmean -G_rar_fft[G_fft_wmean.beam.data[0]] =G_fft_wmean -G_gFT = repack_attributes(G_gFT) -G_gFT_x = repack_attributes(G_gFT_x) -G_rar_fft = repack_attributes(G_rar_fft) +G_gFT[G_gFT_wmean.beam.data[0]] = G_gFT_wmean +G_rar_fft[G_fft_wmean.beam.data[0]] = G_fft_wmean + +G_gFT = repack_attributes(G_gFT) +G_gFT_x = repack_attributes(G_gFT_x) +G_rar_fft = repack_attributes(G_rar_fft) # %% save results # G_gFT_DS = xr.merge(G_gFT.values()) diff --git a/analysis/B02_make_spectra_get_phase.py b/analysis/B02_make_spectra_get_phase.py index 45a21ffd..09fa3057 100644 --- a/analysis/B02_make_spectra_get_phase.py +++ b/analysis/B02_make_spectra_get_phase.py @@ -1,16 +1,16 @@ - import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -22,86 +22,108 @@ import spicke_remover import datetime -#import s3fs +# import s3fs # %% -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' -load_file = load_path + 'processed_' + ATlevel + '_' + track_name + '.h5' - -save_path = mconfig['paths']['work'] + '/B02_spectra_'+hemis+'/' -save_name = 'B02_'+track_name - -plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/B_spectra/' +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +track_name, batch_key, test_flag = "20190601094826_09790312_004_01", "SH_batch01", False +# track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +load_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" +load_file = load_path + "processed_" + ATlevel + "_" + track_name + ".h5" + +save_path = mconfig["paths"]["work"] + "/B02_spectra_" + hemis + "/" +save_name = "B02_" + track_name + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/B_spectra/" +) MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +Gfilt = io.load_pandas_table_dict( + track_name + "_B01_regridded", load_path +) # rhis is the rar photon data +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # # %% test amount of nans in the data -nan_fraction= list() +nan_fraction = list() for I in Gd.values(): - nan_fraction.append( np.sum(np.isnan(I['heights_c_std'])) / I['heights_c_std'].shape[0] ) + nan_fraction.append( + np.sum(np.isnan(I["heights_c_std"])) / I["heights_c_std"].shape[0] + ) if np.array(nan_fraction).mean() > 0.95: - print('nan fraction > 95%, pass this track, add to bad tracks') - MT.json_save(track_name, bad_track_path, {'nan_fraction': np.array(nan_fraction).mean(), 'date': str(datetime.date.today()) }) - print('exit.') + print("nan fraction > 95%, pass this track, add to bad tracks") + MT.json_save( + track_name, + bad_track_path, + { + "nan_fraction": np.array(nan_fraction).mean(), + "date": str(datetime.date.today()), + }, + ) + print("exit.") exit() # %% test LS with an even grid where missing values are set to 0 imp.reload(spec) print(Gd.keys()) -Gi =Gd[ list(Gd.keys())[0] ] # to select a test beam +Gi = Gd[list(Gd.keys())[0]] # to select a test beam # derive spectal limits # Longest deserved period: -T_max = 40 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 -x= np.array(Gi['dist']) +T_max = 40 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 +x = np.array(Gi["dist"]) dx = np.diff(x).mean() -min_datapoint = 1/k_0/dx +min_datapoint = 1 / k_0 / dx Lpoints = int(np.round(min_datapoint) * 20) -Lmeters =Lpoints * dx +Lmeters = Lpoints * dx -#plt.plot(np.diff(np.array(Gi['dist']))) -print('L number of gridpoint:', Lpoints) -print('L length in meters:', Lmeters) -print('approx number windows', 2* Gi['dist'].iloc[-1] /Lmeters-1 ) +# plt.plot(np.diff(np.array(Gi['dist']))) +print("L number of gridpoint:", Lpoints) +print("L length in meters:", Lmeters) +print("approx number windows", 2 * Gi["dist"].iloc[-1] / Lmeters - 1) # S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( x, Lpoints, dx= dx, method='fft', minimum_frequency=None, maximum_frequency=None, samples_per_peak=0.005) # S_pwelch_k.shape -S_pwelch_k = np.fft.rfftfreq( Lpoints, d=dx) +S_pwelch_k = np.fft.rfftfreq(Lpoints, d=dx) dk = np.diff(S_pwelch_k).mean() # %% -#plt.plot( Gfilt[k]['dist']-Gfilt[k]['dist'].min(), Gfilt[k]['heights_c'], '.' ) -#plt.xlim(200000, 201000) +# plt.plot( Gfilt[k]['dist']-Gfilt[k]['dist'].min(), Gfilt[k]['heights_c'], '.' ) +# plt.xlim(200000, 201000) for k in all_beams: - plt.plot(Gd[k]['lons'], Gd[k]['lats']) + plt.plot(Gd[k]["lons"], Gd[k]["lats"]) for k in all_beams: - plt.plot(Gd[k]['x'][0:1000], Gd[k]['y'][0:1000]) + plt.plot(Gd[k]["x"][0:1000], Gd[k]["y"][0:1000]) Gd[k].shape @@ -113,66 +135,69 @@ def linear_gap_fill(F, key_lead, key_int): y_g = np.array(F[key_int]) - nans, x2= np.isnan(y_g), lambda z: z.nonzero()[0] - y_g[nans]= np.interp(x2(nans), x2(~nans), y_g[~nans]) + nans, x2 = np.isnan(y_g), lambda z: z.nonzero()[0] + y_g[nans] = np.interp(x2(nans), x2(~nans), y_g[~nans]) return y_g -plt.plot( linear_gap_fill( Gi, 'dist', 'lons' ) , linear_gap_fill( Gi, 'dist', 'lats' ) ) +plt.plot(linear_gap_fill(Gi, "dist", "lons"), linear_gap_fill(Gi, "dist", "lats")) -lon_interp = linear_gap_fill( Gi, 'dist', 'lons' ) -lat_interp = linear_gap_fill( Gi, 'dist', 'lats' ) -plt.plot( np.gradient(lat_interp) ) -Gd[k]['lons'] = yy +lon_interp = linear_gap_fill(Gi, "dist", "lons") +lat_interp = linear_gap_fill(Gi, "dist", "lats") -plt.plot(Gd[k]['dist'], Gd[k]['lons']) +plt.plot(np.gradient(lat_interp)) +Gd[k]["lons"] = yy -nan_mask = np.isnan(Gi['lons']) +plt.plot(Gd[k]["dist"], Gd[k]["lons"]) + +nan_mask = np.isnan(Gi["lons"]) sum(nan_mask) xp = (~nan_mask).ravel().nonzero()[0] -yy =np.copy(Gi['lons']) -yy[nan_mask] = np.interp(Gi['dist'][nan_mask], Gi['dist'][~nan_mask],yy[~nan_mask] ).shape +yy = np.copy(Gi["lons"]) +yy[nan_mask] = np.interp( + Gi["dist"][nan_mask], Gi["dist"][~nan_mask], yy[~nan_mask] +).shape plt.plot(yy) plt.show() # %% -G_LS= dict() -G_rar_fft= dict() +G_LS = dict() +G_rar_fft = dict() imp.reload(spec) for k in all_beams: # ------------------------------- use gridded data - hkey= 'heights_c_weighted_mean' - x = Gd[k]['dist'] - xlims = x.iloc[0], x.iloc[-1] - dd = np.copy(Gd[k][hkey]) + hkey = "heights_c_weighted_mean" + x = Gd[k]["dist"] + xlims = x.iloc[0], x.iloc[-1] + dd = np.copy(Gd[k][hkey]) - dd_error = np.copy(Gd[k]['heights_c_std']) + dd_error = np.copy(Gd[k]["heights_c_std"]) dd_error[np.isnan(dd_error)] = 100 - #plt.hist(1/dd_weight, bins=40) + # plt.hist(1/dd_weight, bins=40) F = M.figure_axis_xy(6, 3) plt.subplot(2, 1, 1) - plt.plot(x, dd, 'gray', label='displacement (m) ') + plt.plot(x, dd, "gray", label="displacement (m) ") # compute slope spectra !! - dd = np.gradient(dd) - dd, _ = spicke_remover.spicke_remover(dd, spreed=10, verbose=False) - dd_nans = (np.isnan(dd) ) + (Gd[k]['N_photos'] <= 5) + dd = np.gradient(dd) + dd, _ = spicke_remover.spicke_remover(dd, spreed=10, verbose=False) + dd_nans = (np.isnan(dd)) + (Gd[k]["N_photos"] <= 5) dd_filled = np.copy(dd) dd_filled[dd_nans] = 0 - #win = create_weighted_window(dd_filled) + # win = create_weighted_window(dd_filled) # using gappy data - dd_no_nans = dd[~dd_nans] # windowing is applied here - x_no_nans = x[~dd_nans] + dd_no_nans = dd[~dd_nans] # windowing is applied here + x_no_nans = x[~dd_nans] dd_error_no_nans = dd_error[~dd_nans] # replace bad daa by zero @@ -189,7 +214,7 @@ def linear_gap_fill(F, key_lead, key_int): # dd= dd[~outlyers] # x= x[~outlyers] - plt.plot(x_no_nans, dd_no_nans, 'black', label='slope (m/m)') + plt.plot(x_no_nans, dd_no_nans, "black", label="slope (m/m)") plt.legend() # ---------------------------------- use ungridded data @@ -206,86 +231,128 @@ def linear_gap_fill(F, key_lead, key_int): # plt.legend() # for testing - #xlims = xlims[0], xlims[1]/2 + # xlims = xlims[0], xlims[1]/2 # %% - print('LS') + print("LS") imp.reload(spec) - #S_pwelch_k2 = np.arange(S_pwelch_k[1], S_pwelch_k[-1], S_pwelch_dk*2 ) + # S_pwelch_k2 = np.arange(S_pwelch_k[1], S_pwelch_k[-1], S_pwelch_dk*2 ) - kk = S_pwelch_k # includes 0 wavenumber - #kk= S_pwelch_k2 + kk = S_pwelch_k # includes 0 wavenumber + # kk= S_pwelch_k2 # plt.plot(1/kk) # kk = np.concatenate( [kk[kk <= 1/50] , kk[kk > 1/50][::1]] ) kk.shape - S = spec.wavenumber_spectrogram_LS( np.array(x_no_nans), np.array(dd_no_nans), Lmeters, dx, dy = None, waven_method = kk, ov=None, window=None) - G, PP = S.cal_spectrogram(xlims= xlims, weight_data=False, max_nfev = 200) + S = spec.wavenumber_spectrogram_LS( + np.array(x_no_nans), + np.array(dd_no_nans), + Lmeters, + dx, + dy=None, + waven_method=kk, + ov=None, + window=None, + ) + G, PP = S.cal_spectrogram(xlims=xlims, weight_data=False, max_nfev=200) # add more coodindates to the Dataset - x_coord_no_gaps = linear_gap_fill( Gd[k], 'dist', 'x' ) - y_coord_no_gaps = linear_gap_fill( Gd[k], 'dist', 'y' ) - mapped_coords = spec.sub_sample_coords(Gd[k]['dist'], x_coord_no_gaps, y_coord_no_gaps, S.stancil_iter , map_func = None ) - G = G.assign_coords(x_coord =('x',mapped_coords[:,1] ) ).assign_coords(y_coord =('x',mapped_coords[:,2] ) ) - - lons_no_gaps = linear_gap_fill( Gd[k], 'dist', 'lons' ) - lats_no_gaps = linear_gap_fill( Gd[k], 'dist', 'lats' ) - mapped_coords = spec.sub_sample_coords(Gd[k]['dist'], lons_no_gaps, lats_no_gaps, S.stancil_iter , map_func = None ) - G = G.assign_coords(lon =('x',mapped_coords[:,1] ) ).assign_coords(lat =('x',mapped_coords[:,2] ) ) - + x_coord_no_gaps = linear_gap_fill(Gd[k], "dist", "x") + y_coord_no_gaps = linear_gap_fill(Gd[k], "dist", "y") + mapped_coords = spec.sub_sample_coords( + Gd[k]["dist"], x_coord_no_gaps, y_coord_no_gaps, S.stancil_iter, map_func=None + ) + G = G.assign_coords(x_coord=("x", mapped_coords[:, 1])).assign_coords( + y_coord=("x", mapped_coords[:, 2]) + ) + + lons_no_gaps = linear_gap_fill(Gd[k], "dist", "lons") + lats_no_gaps = linear_gap_fill(Gd[k], "dist", "lats") + mapped_coords = spec.sub_sample_coords( + Gd[k]["dist"], lons_no_gaps, lats_no_gaps, S.stancil_iter, map_func=None + ) + G = G.assign_coords(lon=("x", mapped_coords[:, 1])).assign_coords( + lat=("x", mapped_coords[:, 2]) + ) # %% - plot_data_model=True + plot_data_model = True if plot_data_model: - for i in np.arange(60,120,2): - c1= 'blue' - c2= 'red' + for i in np.arange(60, 120, 2): + c1 = "blue" + c2 = "red" - xi_1=G.x[i] - xi_2=G.x[i+1] - #if k%2 ==0: + xi_1 = G.x[i] + xi_2 = G.x[i + 1] + # if k%2 ==0: F = M.figure_axis_xy(16, 2.5) - plt.plot(G.eta[0:-1] +xi_1, np.fft.irfft(G.sel(x=xi_1).Y_model_hat) ,'-', c=c1, linewidth=0.8, alpha=1, zorder=12) - plt.plot(G.eta[0:-1] +xi_2, np.fft.irfft(G.sel(x=xi_2).Y_model_hat) ,'-', c=c2, linewidth=0.8, alpha=1, zorder=12) - - plt.plot(x, dd, '-', c='k',linewidth=2, alpha =0.6) - #plt.plot(x, dd, '.', c='k',markersize=3, alpha =0.5) - - #plt.plot(x[~dd_nans], dd_error[~dd_nans], '.', c='green',linewidth=1, alpha =0.5) + plt.plot( + G.eta[0:-1] + xi_1, + np.fft.irfft(G.sel(x=xi_1).Y_model_hat), + "-", + c=c1, + linewidth=0.8, + alpha=1, + zorder=12, + ) + plt.plot( + G.eta[0:-1] + xi_2, + np.fft.irfft(G.sel(x=xi_2).Y_model_hat), + "-", + c=c2, + linewidth=0.8, + alpha=1, + zorder=12, + ) + + plt.plot(x, dd, "-", c="k", linewidth=2, alpha=0.6) + # plt.plot(x, dd, '.', c='k',markersize=3, alpha =0.5) + + # plt.plot(x[~dd_nans], dd_error[~dd_nans], '.', c='green',linewidth=1, alpha =0.5) F.ax.axvline(xi_1 + G.eta[0].data, linewidth=4, color=c1, alpha=0.5) - F.ax.axvline(xi_1 + G.eta[-1].data, linewidth=4, color=c1, alpha=0.5) - F.ax.axvline(xi_2 + G.eta[0].data, linewidth=4, color=c2, alpha=0.5) - F.ax.axvline(xi_2 + G.eta[-1].data, linewidth=4, color=c2, alpha=0.5) - - plt.text(xi_1 + G.eta[0].data, 0.5, ' N='+ str(G.sel(x=xi_1).N_per_stancil.data) ) - plt.text(xi_2 + G.eta[0].data, 0.5, ' N='+ str(G.sel(x=xi_2).N_per_stancil.data) ) - plt.xlim(xi_1 + G.eta[0].data*1.2, xi_2 + G.eta[-1].data*1.2 ) + F.ax.axvline(xi_1 + G.eta[-1].data, linewidth=4, color=c1, alpha=0.5) + F.ax.axvline(xi_2 + G.eta[0].data, linewidth=4, color=c2, alpha=0.5) + F.ax.axvline(xi_2 + G.eta[-1].data, linewidth=4, color=c2, alpha=0.5) + + plt.text( + xi_1 + G.eta[0].data, + 0.5, + " N=" + str(G.sel(x=xi_1).N_per_stancil.data), + ) + plt.text( + xi_2 + G.eta[0].data, + 0.5, + " N=" + str(G.sel(x=xi_2).N_per_stancil.data), + ) + plt.xlim(xi_1 + G.eta[0].data * 1.2, xi_2 + G.eta[-1].data * 1.2) plt.ylim(-1, 1) plt.show() # %% - S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True, weight_data=False) + S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.parceval(add_attrs=True, weight_data=False) # assign beam coordinate - G.coords['beam'] = str(k)#(('beam'), str(k)) - G = G.expand_dims(dim = 'beam', axis = 1) + G.coords["beam"] = str(k) # (('beam'), str(k)) + G = G.expand_dims(dim="beam", axis=1) # repack such that all coords are associated with beam - G.coords['N_per_stancil'] = (('x', 'beam' ), np.expand_dims(G['N_per_stancil'], 1)) + G.coords["N_per_stancil"] = (("x", "beam"), np.expand_dims(G["N_per_stancil"], 1)) # spectral errors are cacualted within S and now repacked to main DataSet G - G.coords['mean_El'] = (('k', 'beam' ), np.expand_dims(S.G['mean_El'], 1)) - G.coords['mean_Eu'] = (('k', 'beam' ), np.expand_dims(S.G['mean_Eu'], 1)) + G.coords["mean_El"] = (("k", "beam"), np.expand_dims(S.G["mean_El"], 1)) + G.coords["mean_Eu"] = (("k", "beam"), np.expand_dims(S.G["mean_Eu"], 1)) # calculate number data points def get_stancil_nans(stancil): x_mask = (stancil[0] < x) & (x <= stancil[-1]) - idata = Gd[k]['N_photos'][x_mask] + idata = Gd[k]["N_photos"][x_mask] return stancil[1], idata.sum() - photon_list = np.array(list(dict(map( get_stancil_nans, copy.copy(S.stancil_iter) )).values())) - G.coords['N_photons'] = (('x', 'beam' ), np.expand_dims(photon_list, 1)) + photon_list = np.array( + list(dict(map(get_stancil_nans, copy.copy(S.stancil_iter))).values()) + ) + G.coords["N_photons"] = (("x", "beam"), np.expand_dims(photon_list, 1)) # Save to dict G_LS[k] = G @@ -293,225 +360,299 @@ def get_stancil_nans(stancil): # plot plt.subplot(2, 1, 2) GG = G.LS_spectal_power.squeeze() - plt.plot(GG.k, np.nanmean(GG,1), 'gray', label='mean LS power') - plt.plot(GG.k, np.nanmean(S.G, 1), 'k', label='mean LS power optm') - + plt.plot(GG.k, np.nanmean(GG, 1), "gray", label="mean LS power") + plt.plot(GG.k, np.nanmean(S.G, 1), "k", label="mean LS power optm") # standard FFT - print('FFT') - dd[dd_nans] = 0 + print("FFT") + dd[dd_nans] = 0 S = spec.wavenumber_spectrogram(x, dd, Lpoints) G = S.cal_spectrogram() - S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True) + S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.parceval(add_attrs=True) # assign beam coordinate - G.coords['beam'] = str(k)#(('beam'), str(k)) - G = G.expand_dims(dim = 'beam', axis = 2) - G.coords['mean_El'] = (('k', 'beam' ), np.expand_dims(G['mean_El'], 1)) - G.coords['mean_Eu'] = (('k', 'beam' ), np.expand_dims(G['mean_Eu'], 1)) - G.coords['x'] = G.coords['x'] * dx # adjust x-coodinate definition + G.coords["beam"] = str(k) # (('beam'), str(k)) + G = G.expand_dims(dim="beam", axis=2) + G.coords["mean_El"] = (("k", "beam"), np.expand_dims(G["mean_El"], 1)) + G.coords["mean_Eu"] = (("k", "beam"), np.expand_dims(G["mean_Eu"], 1)) + G.coords["x"] = G.coords["x"] * dx # adjust x-coodinate definition stancil_iter = spec.create_chunk_boundaries(int(Lpoints), dd_nans.size) + def get_stancil_nans(stancil): - idata = dd_nans[stancil[0]:stancil[-1]] + idata = dd_nans[stancil[0] : stancil[-1]] return stancil[1], idata.size - idata.sum() - N_list = np.array(list(dict(map( get_stancil_nans, stancil_iter )).values())) + N_list = np.array(list(dict(map(get_stancil_nans, stancil_iter)).values())) # repack such that all coords are associated with beam - G.coords['N_per_stancil'] = (('x', 'beam' ), np.expand_dims(N_list, 1)) + G.coords["N_per_stancil"] = (("x", "beam"), np.expand_dims(N_list, 1)) - #save to dict + # save to dict G_rar_fft[k] = G # for plotting - G_rar_fft_p =G.squeeze() - plt.plot(G.k, G_rar_fft_p[:, G_rar_fft_p['N_per_stancil'] > 10 ].mean('x'), 'darkblue', label='mean FFT') - #plt.plot(G.k, GG.mean('x'), 'lightblue', label='mean FFT') + G_rar_fft_p = G.squeeze() + plt.plot( + G.k, + G_rar_fft_p[:, G_rar_fft_p["N_per_stancil"] > 10].mean("x"), + "darkblue", + label="mean FFT", + ) + # plt.plot(G.k, GG.mean('x'), 'lightblue', label='mean FFT') plt.legend() plt.show() - #F.save_light(path=plot_path, name = 'B02_control_'+k+'_' + track_name) - #print('saved as '+'B02_control_'+k+'_' + track_name) - #print(np.isinf(G).sum().data) - + # F.save_light(path=plot_path, name = 'B02_control_'+k+'_' + track_name) + # print('saved as '+'B02_control_'+k+'_' + track_name) + # print(np.isinf(G).sum().data) # %% + def dict_weighted_mean(Gdict, weight_key): """ returns the weighted meean of a dict of xarray, data_arrays weight_key must be in the xr.DataArrays """ - #Gdict = G_LS + # Gdict = G_LS # weight_key='N_per_stancil' - akey = list( Gdict.keys() )[0] + akey = list(Gdict.keys())[0] GSUM = Gdict[akey].copy() - GSUM.data = np.zeros(GSUM.shape) + GSUM.data = np.zeros(GSUM.shape) N_per_stancil = GSUM.N_per_stancil * 0 - N_photons = np.zeros(GSUM.N_per_stancil.size) + N_photons = np.zeros(GSUM.N_per_stancil.size) - counter= 0 - for k,I in Gdict.items(): - I =I.squeeze() - GSUM += I.where( ~np.isnan(I), 0) * I[weight_key] #.sel(x=GSUM.x) - N_per_stancil += I[weight_key] - if 'N_photons' in GSUM.coords: - N_photons += I['N_photons'] - counter+=1 + counter = 0 + for k, I in Gdict.items(): + I = I.squeeze() + GSUM += I.where(~np.isnan(I), 0) * I[weight_key] # .sel(x=GSUM.x) + N_per_stancil += I[weight_key] + if "N_photons" in GSUM.coords: + N_photons += I["N_photons"] + counter += 1 - GSUM = GSUM / N_per_stancil /counter + GSUM = GSUM / N_per_stancil / counter - if 'N_photons' in GSUM.coords: - GSUM.coords['N_photons'] = (('x', 'beam'), np.expand_dims(N_photons, 1) ) + if "N_photons" in GSUM.coords: + GSUM.coords["N_photons"] = (("x", "beam"), np.expand_dims(N_photons, 1)) - GSUM['beam'] = ['weighted_mean'] - GSUM.name='power_spec' + GSUM["beam"] = ["weighted_mean"] + GSUM.name = "power_spec" return GSUM + G_LS_sel = {} -for k,I in G_LS.items(): - G_LS_sel[k] = I['spectral_power_optm'] +for k, I in G_LS.items(): + G_LS_sel[k] = I["spectral_power_optm"] -G_LS_wmean = dict_weighted_mean(G_LS_sel, 'N_per_stancil') -G_fft_wmean = dict_weighted_mean(G_rar_fft, 'N_per_stancil') +G_LS_wmean = dict_weighted_mean(G_LS_sel, "N_per_stancil") +G_fft_wmean = dict_weighted_mean(G_rar_fft, "N_per_stancil") # %% plot -def plot_wavenumber_spectrogram(ax, Gi, clev, title= None, plot_photon_density=True ): +def plot_wavenumber_spectrogram(ax, Gi, clev, title=None, plot_photon_density=True): - if Gi.k[0] ==0: - Gi= Gi.sel(k=Gi.k[1:]) - x_lambda= 1/Gi.k - plt.pcolormesh(Gi.x/1e3, x_lambda , Gi, cmap=plt.cm.ocean_r , vmin = clev[0], vmax = clev[-1]) + if Gi.k[0] == 0: + Gi = Gi.sel(k=Gi.k[1:]) + x_lambda = 1 / Gi.k + plt.pcolormesh( + Gi.x / 1e3, x_lambda, Gi, cmap=plt.cm.ocean_r, vmin=clev[0], vmax=clev[-1] + ) - ax.set_yscale('log') + ax.set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') if plot_photon_density: - plt.plot(Gi.x/1e3, x_lambda[-1] + (Gi.N_per_stancil/Gi.N_per_stancil.max() ) * 10 , c='black', linewidth= 0.8, label='NAN-density' ) - plt.fill_between(Gi.x/1e3, x_lambda[-1] + (Gi.N_per_stancil/Gi.N_per_stancil.max() ) * 10, 0, color='gray', alpha = 0.3) - ax.axhline(30, color='black', linewidth=0.3) - - #plt.xlabel('Distance from the Ice Edge (km)') + plt.plot( + Gi.x / 1e3, + x_lambda[-1] + (Gi.N_per_stancil / Gi.N_per_stancil.max()) * 10, + c="black", + linewidth=0.8, + label="NAN-density", + ) + plt.fill_between( + Gi.x / 1e3, + x_lambda[-1] + (Gi.N_per_stancil / Gi.N_per_stancil.max()) * 10, + 0, + color="gray", + alpha=0.3, + ) + ax.axhline(30, color="black", linewidth=0.3) + + # plt.xlabel('Distance from the Ice Edge (km)') plt.ylim(x_lambda[-1], x_lambda[0]) - plt.title(title, loc='left') + plt.title(title, loc="left") + -#Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() -#Gmean = G_LS_wmean.rolling(x=2, min_periods= 1, center=True).mean() +# Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() +# Gmean = G_LS_wmean.rolling(x=2, min_periods= 1, center=True).mean() Gmean = G_LS_wmean.rolling(k=5, center=True).mean() -#Gmean = Gmean.where(~np.isnan(Gmean), 0) +# Gmean = Gmean.where(~np.isnan(Gmean), 0) -k_max_range = Gmean.k[Gmean.mean('x').argmax().data].data* 0.75, Gmean.k[Gmean.mean('x').argmax().data].data* 1, Gmean.k[Gmean.mean('x').argmax().data].data* 1.25 +k_max_range = ( + Gmean.k[Gmean.mean("x").argmax().data].data * 0.75, + Gmean.k[Gmean.mean("x").argmax().data].data * 1, + Gmean.k[Gmean.mean("x").argmax().data].data * 1.25, +) # %% font_for_print() -F = M.figure_axis_xy(6.5, 5.6, container= True, view_scale =1) +F = M.figure_axis_xy(6.5, 5.6, container=True, view_scale=1) -plt.suptitle('LS and FFT Slope Spectrograms\n' + track_name, y = 0.98) -gs = GridSpec(3,3, wspace=0.2, hspace=.5)#figure=fig, -#clev=np.arange(0, 6, 0.1)*3 +plt.suptitle("LS and FFT Slope Spectrograms\n" + track_name, y=0.98) +gs = GridSpec(3, 3, wspace=0.2, hspace=0.5) # figure=fig, +# clev=np.arange(0, 6, 0.1)*3 -#%matplotlib inline +# %matplotlib inline -clev = M.clevels( [Gmean.quantile(0.01).data, Gmean.quantile(0.99).data * 1.2], 31)* 1 -xlims= Gmean.x[0]/1e3, Gmean.x[-1]/1e3 +clev = M.clevels([Gmean.quantile(0.01).data, Gmean.quantile(0.99).data * 1.2], 31) * 1 +xlims = Gmean.x[0] / 1e3, Gmean.x[-1] / 1e3 -for pos, k, pflag in zip([gs[0, 0],gs[0, 1],gs[0, 2] ], high_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[0, 0], gs[0, 1], gs[0, 2]], high_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G_LS_sel[k].squeeze()#.rolling(k=10, x=2, min_periods= 1, center=True).mean() - #Gplot.mean('x').plot() + Gplot = G_LS_sel[ + k + ].squeeze() # .rolling(k=10, x=2, min_periods= 1, center=True).mean() + # Gplot.mean('x').plot() # #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) # - #np.log(Gplot).plot() - plot_wavenumber_spectrogram(ax0, Gplot, clev, title =k, plot_photon_density=True ) + # np.log(Gplot).plot() + plot_wavenumber_spectrogram(ax0, Gplot, clev, title=k, plot_photon_density=True) plt.xlim(xlims) # if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -for pos, k, pflag in zip([gs[1, 0],gs[1, 1],gs[1, 2] ], low_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[1, 0], gs[1, 1], gs[1, 2]], low_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G_LS_sel[k].squeeze()#.rolling(k=10, x=2, min_periods= 1, center=True).mean() - #Gplot.mean('x').plot() + Gplot = G_LS_sel[ + k + ].squeeze() # .rolling(k=10, x=2, min_periods= 1, center=True).mean() + # Gplot.mean('x').plot() # #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) # - plot_wavenumber_spectrogram(ax0, Gplot, clev, title =k, plot_photon_density=True ) + plot_wavenumber_spectrogram(ax0, Gplot, clev, title=k, plot_photon_density=True) plt.xlim(xlims) # if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -pos, k, pflag = gs[2, 0], 'Power(weighted mean) \n10 $\log_{10}( (m/m)^2 m )$', True +pos, k, pflag = gs[2, 0], "Power(weighted mean) \n10 $\log_{10}( (m/m)^2 m )$", True ax0 = F.fig.add_subplot(pos) -Gplot = G_LS_wmean.squeeze().rolling(k=5, min_periods= 1, center=True).median().rolling(x=3, min_periods= 1, center=True).median() +Gplot = ( + G_LS_wmean.squeeze() + .rolling(k=5, min_periods=1, center=True) + .median() + .rolling(x=3, min_periods=1, center=True) + .median() +) dd = 10 * np.log10(Gplot) -dd= dd.where(~np.isinf(dd), np.nan ) -clev_log = M.clevels( [dd.quantile(0.01).data, dd.quantile(0.98).data * 1.2], 31)* 1 -plot_wavenumber_spectrogram(ax0, dd, clev_log , title =k, plot_photon_density= True) +dd = dd.where(~np.isinf(dd), np.nan) +clev_log = M.clevels([dd.quantile(0.01).data, dd.quantile(0.98).data * 1.2], 31) * 1 +plot_wavenumber_spectrogram(ax0, dd, clev_log, title=k, plot_photon_density=True) plt.xlim(xlims) # plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) # ax0.axhline(30, color='black', linewidth=0.5) -ax0.axhline(1/k_max_range[0], color='red', linestyle= '--', linewidth= 0.5) -ax0.axhline(1/k_max_range[1], color='red', linestyle= '-', linewidth= 0.5) -ax0.axhline(1/k_max_range[2], color='red', linestyle= '--', linewidth= 0.5) +ax0.axhline(1 / k_max_range[0], color="red", linestyle="--", linewidth=0.5) +ax0.axhline(1 / k_max_range[1], color="red", linestyle="-", linewidth=0.5) +ax0.axhline(1 / k_max_range[2], color="red", linestyle="--", linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() pos = gs[2, 1] ax0 = F.fig.add_subplot(pos) -plt.title('Photons density ($m^{-1}$)', loc='left') - -for k,I in G_LS.items(): - plt.plot(Gplot.x/1e3, I.N_photons/Lmeters, label=k, linewidth=0.8) -plt.plot(Gplot.x/1e3, G_LS_wmean.N_photons/3/Lmeters , c='black', label='ave Photons' , linewidth=0.8) +plt.title("Photons density ($m^{-1}$)", loc="left") + +for k, I in G_LS.items(): + plt.plot(Gplot.x / 1e3, I.N_photons / Lmeters, label=k, linewidth=0.8) +plt.plot( + Gplot.x / 1e3, + G_LS_wmean.N_photons / 3 / Lmeters, + c="black", + label="ave Photons", + linewidth=0.8, +) plt.xlim(xlims) -plt.xlabel('Distance from the Ice Edge (km)') +plt.xlabel("Distance from the Ice Edge (km)") pos = gs[2, 2] ax0 = F.fig.add_subplot(pos) -ax0.set_yscale('log') - -plt.title('Peak Spectal Power', loc='left') - -for k,I in G_LS_sel.items(): - plt.scatter(I.x.data/1e3, I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k').data *1e3 , s=0.5, marker='.', color='red', alpha= 0.3) - -for k,I in G_rar_fft.items(): - I= I.squeeze() - I= I[:,I.N_per_stancil >= I.N_per_stancil.max().data*0.9] - plt.scatter(I.x/1e3, I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3 , s=0.5, marker='.', c='blue', alpha= 0.3) - - -Gplot= G_fft_wmean.squeeze() -Gplot = Gplot[:,Gplot.N_per_stancil >= Gplot.N_per_stancil.max().data*0.9] -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3, '.', markersize=1.5 , c='blue', label= 'FFT') - -Gplot= G_LS_wmean.squeeze() -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3 , '.' , markersize=1.5, c='red', label= 'LS') - -plt.ylabel('1e-3 $(m)^2~m$') +ax0.set_yscale("log") + +plt.title("Peak Spectal Power", loc="left") + +for k, I in G_LS_sel.items(): + plt.scatter( + I.x.data / 1e3, + I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k").data * 1e3, + s=0.5, + marker=".", + color="red", + alpha=0.3, + ) + +for k, I in G_rar_fft.items(): + I = I.squeeze() + I = I[:, I.N_per_stancil >= I.N_per_stancil.max().data * 0.9] + plt.scatter( + I.x / 1e3, + I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + s=0.5, + marker=".", + c="blue", + alpha=0.3, + ) + + +Gplot = G_fft_wmean.squeeze() +Gplot = Gplot[:, Gplot.N_per_stancil >= Gplot.N_per_stancil.max().data * 0.9] +plt.plot( + Gplot.x / 1e3, + Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + ".", + markersize=1.5, + c="blue", + label="FFT", +) + +Gplot = G_LS_wmean.squeeze() +plt.plot( + Gplot.x / 1e3, + Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + ".", + markersize=1.5, + c="red", + label="LS", +) + +plt.ylabel("1e-3 $(m)^2~m$") plt.legend() -plt.ylim(Gplot.min()*1.4, Gplot.max()*1.4 ) +plt.ylim(Gplot.min() * 1.4, Gplot.max() * 1.4) plt.xlim(xlims) -#F.save_light(path=plot_path, name = 'B02_specs_' + track_name +'_L'+str(Lmeters)) +# F.save_light(path=plot_path, name = 'B02_specs_' + track_name +'_L'+str(Lmeters)) + # %% repack data def repack_attributes(DD): @@ -520,20 +661,21 @@ def repack_attributes(DD): for k in attr_dim_list: for ka in list(DD[k].attrs.keys()): I = DD[k] - I.coords[ka] = ( 'beam', np.expand_dims(I.attrs[ka], 0) ) + I.coords[ka] = ("beam", np.expand_dims(I.attrs[ka], 0)) return DD -G_LS[G_LS_wmean.beam.data[0]] =G_LS_wmean -G_rar_fft[G_fft_wmean.beam.data[0]] =G_fft_wmean + +G_LS[G_LS_wmean.beam.data[0]] = G_LS_wmean +G_rar_fft[G_fft_wmean.beam.data[0]] = G_fft_wmean G_LS = repack_attributes(G_LS) G_rar_fft = repack_attributes(G_rar_fft) # %% save results -G_LS_DS = xr.merge(G_LS.values()) -G_LS_DS['name'] = 'LS_power_spectra' -#G_LS_DS.to_netcdf(save_path+save_name+'_LS.nc') +G_LS_DS = xr.merge(G_LS.values()) +G_LS_DS["name"] = "LS_power_spectra" +# G_LS_DS.to_netcdf(save_path+save_name+'_LS.nc') -G_fft_DS = xr.merge(G_rar_fft.values()) -G_fft_DS['name']= 'FFT_power_spectra' -#G_fft_DS.to_netcdf(save_path+save_name+'_FFT.nc') +G_fft_DS = xr.merge(G_rar_fft.values()) +G_fft_DS["name"] = "FFT_power_spectra" +# G_fft_DS.to_netcdf(save_path+save_name+'_FFT.nc') diff --git a/analysis/B03_coherence_direction.py b/analysis/B03_coherence_direction.py index 0c5e1f37..73aaf6aa 100644 --- a/analysis/B03_coherence_direction.py +++ b/analysis/B03_coherence_direction.py @@ -1,16 +1,16 @@ - import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -22,82 +22,92 @@ import spicke_remover import datetime import concurrent.futures as futures -#import s3fs -# %% -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False -track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False - - - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -save_path = mconfig['paths']['work'] + '/B03_spectra_'+hemis+'/' -save_name = 'B03_'+track_name - -plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/B_spectra/' +# import s3fs +# %% +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False +track_name, batch_key, test_flag = "20190219073735_08070210_004_01", "SH_batch02", False + + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +save_path = mconfig["paths"]["work"] + "/B03_spectra_" + hemis + "/" +save_name = "B03_" + track_name + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/B_spectra/" +) MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -#Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +# Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data -load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # -load_path = mconfig['paths']['work'] + '/B02_spectra_'+hemis+'/' -Gpars = io.load_pandas_table_dict('B02_'+ track_name + '_params' , load_path) # -Gspec = xr.open_dataset(load_path + 'B02_'+ track_name + '_LS.nc' ) # -Gspec['Y_model_hat'] = Gspec.Y_model_hat_real + Gspec.Y_model_hat_imag *1j -Gspec = Gspec.drop('Y_model_hat_real').drop('Y_model_hat_imag') +load_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # +load_path = mconfig["paths"]["work"] + "/B02_spectra_" + hemis + "/" +Gpars = io.load_pandas_table_dict("B02_" + track_name + "_params", load_path) # +Gspec = xr.open_dataset(load_path + "B02_" + track_name + "_LS.nc") # +Gspec["Y_model_hat"] = Gspec.Y_model_hat_real + Gspec.Y_model_hat_imag * 1j +Gspec = Gspec.drop("Y_model_hat_real").drop("Y_model_hat_imag") -dk = Gspec.k.diff('k').mean().data +dk = Gspec.k.diff("k").mean().data Lpoints = Gspec.Lpoints -Gspec = Gspec.sel(k = slice(0.000125, 0.025)).isel(x =slice(4, 30)) +Gspec = Gspec.sel(k=slice(0.000125, 0.025)).isel(x=slice(4, 30)) -Gspec.coords['f'] = (('k'), np.sqrt(Gspec.k.data * 9.81)/ 2/np.pi ) -Gspec.coords['T'] = 1/Gspec.coords['f'] -Gspec=Gspec.swap_dims({'k': 'f'}) +Gspec.coords["f"] = (("k"), np.sqrt(Gspec.k.data * 9.81) / 2 / np.pi) +Gspec.coords["T"] = 1 / Gspec.coords["f"] +Gspec = Gspec.swap_dims({"k": "f"}) -#Gspec.spectral_power_optm.sel(beam='weighted_mean').plot() +# Gspec.spectral_power_optm.sel(beam='weighted_mean').plot() # %% -#k_lim= 0.02 -A, B = Gspec.sel(beam= 'gt2r').Y_model_hat , Gspec.sel(beam= 'gt2l').Y_model_hat - -r_ave_kargs={'x':2, 'f':10, 'center':True, 'min_periods':2} -r_ave_kargs2={'f':10, 'center':True, 'min_periods':2} -#(abs(B) - abs(A)).plot() -S_aa = (A*A.conj()).real -S_bb = (B*B.conj()).real -#co_spec = (A.conj() *B) /S_aa/S_bb +# k_lim= 0.02 +A, B = Gspec.sel(beam="gt2r").Y_model_hat, Gspec.sel(beam="gt2l").Y_model_hat + +r_ave_kargs = {"x": 2, "f": 10, "center": True, "min_periods": 2} +r_ave_kargs2 = {"f": 10, "center": True, "min_periods": 2} +# (abs(B) - abs(A)).plot() +S_aa = (A * A.conj()).real +S_bb = (B * B.conj()).real +# co_spec = (A.conj() *B) /S_aa/S_bb # abs(co_spec).plot() -co_spec = (A.conj() *B) +co_spec = A.conj() * B np.log(abs(co_spec)).plot(levels=np.arange(-2, 3, 0.1)) -co_spec = (A.conj() *B).rolling(**r_ave_kargs).mean()# +co_spec = (A.conj() * B).rolling(**r_ave_kargs).mean() # np.log(abs(co_spec)).plot(levels=np.arange(-2, 3, 0.1)) (abs(co_spec)).plot(levels=np.exp(np.arange(-3, 2, 0.1))) -abs(co_spec).mean('x').plot() +abs(co_spec).mean("x").plot() -#(abs(co_spec)/(S_aa *S_bb).rolling(**r_ave_kargs).mean()).plot() +# (abs(co_spec)/(S_aa *S_bb).rolling(**r_ave_kargs).mean()).plot() # # (abs(A.conj() *B)/(S_aa *S_bb)).rolling(**r_ave_kargs).mean()[:,:].plot() from xhistogram.xarray import histogram -phase = np.arctan2( -co_spec.imag, co_spec.real ) +phase = np.arctan2(-co_spec.imag, co_spec.real) phase.plot() # %% @@ -107,65 +117,68 @@ f_pos = abs(co_spec).isel(x=0).rolling(f=10).mean().argmax().data - def adjust_phase_forward(pi, f_pos, f_size): - for fi in np.arange(f_pos+1, f_size): - #print(fi,pi[fi] ) - dpi1 = pi[fi] - pi[fi-1] - if abs(dpi1)/(2*np.pi) > (2 *np.pi): - dpi1= dpi1 - np.sign(dpi1) * np.floor(abs(dpi1)/(2*np.pi)) * 2 *np.pi + for fi in np.arange(f_pos + 1, f_size): + # print(fi,pi[fi] ) + dpi1 = pi[fi] - pi[fi - 1] + if abs(dpi1) / (2 * np.pi) > (2 * np.pi): + dpi1 = dpi1 - np.sign(dpi1) * np.floor(abs(dpi1) / (2 * np.pi)) * 2 * np.pi - dpi2 = - np.sign(dpi1) * (2* np.pi - np.sign(dpi1) * dpi1) + dpi2 = -np.sign(dpi1) * (2 * np.pi - np.sign(dpi1) * dpi1) dds = np.array([dpi1, dpi2]) - pi[fi] = pi[fi-1] + dds[np.argmin(abs(dds))] - #print(pi[fi] ) + pi[fi] = pi[fi - 1] + dds[np.argmin(abs(dds))] + # print(pi[fi] ) return pi + def adjust_phase_backward(pi, f_pos, f_size): - for fi in np.arange(f_pos-1, -1, -1): - #print(fi,pi[fi] ) - dpi1 = pi[fi] - pi[fi+1] - if abs(dpi1)/(2*np.pi) > (2 *np.pi): - dpi1= dpi1 - np.sign(dpi1) * np.floor(abs(dpi1)/(2*np.pi)) * 2 *np.pi + for fi in np.arange(f_pos - 1, -1, -1): + # print(fi,pi[fi] ) + dpi1 = pi[fi] - pi[fi + 1] + if abs(dpi1) / (2 * np.pi) > (2 * np.pi): + dpi1 = dpi1 - np.sign(dpi1) * np.floor(abs(dpi1) / (2 * np.pi)) * 2 * np.pi - dpi2 = - np.sign(dpi1) * (2* np.pi - np.sign(dpi1) * dpi1) + dpi2 = -np.sign(dpi1) * (2 * np.pi - np.sign(dpi1) * dpi1) dds = np.array([dpi1, dpi2]) - pi[fi] = pi[fi+1] + dds[np.argmin(abs(dds))] - #print(pi[fi] ) + pi[fi] = pi[fi + 1] + dds[np.argmin(abs(dds))] + # print(pi[fi] ) return pi + def adjust_phase_freq(pi, f_pos, f_size): pi = adjust_phase_forward(pi, f_pos, f_size) pi = adjust_phase_backward(pi, f_pos, f_size) return pi + + # %% pi = np.copy(pi_sel) -pi =adjust_phase_freq(pi, f_pos, f.size) -#pi =adjust_phase_freq(pi, f_pos, f.size) +pi = adjust_phase_freq(pi, f_pos, f.size) +# pi =adjust_phase_freq(pi, f_pos, f.size) -plt.plot(pi_sel/np.pi, 'k--') -plt.plot(pi/np.pi) +plt.plot(pi_sel / np.pi, "k--") +plt.plot(pi / np.pi) plt.grid() # %% -phase = np.arctan2( -co_spec.imag, co_spec.real ) -phase_data = np.zeros(phase.T.shape)*np.nan +phase = np.arctan2(-co_spec.imag, co_spec.real) +phase_data = np.zeros(phase.T.shape) * np.nan for xi in range(phase.x.size): - #print(xi.data) - pi = np.copy(phase.isel(x= xi)) - phase_data[xi,:] =adjust_phase_freq(pi, f_pos, f.size) + # print(xi.data) + pi = np.copy(phase.isel(x=xi)) + phase_data[xi, :] = adjust_phase_freq(pi, f_pos, f.size) phase_adjust = phase.copy() phase_adjust.data = phase_data.T / np.pi phase_adjust.plot() -phase_adjust.where(phase_adjust < np.pi, phase_adjust - 2* np.pi).plot() +phase_adjust.where(phase_adjust < np.pi, phase_adjust - 2 * np.pi).plot() # for fi in range(phase_adjust.f.size): # #print(xi.data) # pi = np.copy(phase_adjust.isel(f= fi)) @@ -181,119 +194,136 @@ def adjust_phase_freq(pi, f_pos, f_size): # # phase_adjust.plot() # %% -phase_adjust = phase_adjust.where(phase_adjust < 1 , phase_adjust -2) -phase_adjust = phase_adjust.where(phase_adjust > -1 , phase_adjust +2) +phase_adjust = phase_adjust.where(phase_adjust < 1, phase_adjust - 2) +phase_adjust = phase_adjust.where(phase_adjust > -1, phase_adjust + 2) phase_adjust.plot() # %% -phase_filtered= (phase_adjust.where(abs(co_spec)>0.2, np.nan)) +phase_filtered = phase_adjust.where(abs(co_spec) > 0.2, np.nan) phase_filtered.plot() -x_prime= (phase_filtered.x - phase_filtered.x[0] )/1e3 +x_prime = (phase_filtered.x - phase_filtered.x[0]) / 1e3 -y_prime = (phase_filtered.k/ 2/ np.pi) -plt.pcolormesh( x_prime , y_prime, phase_filtered.data) -plt.ylabel('1/lambda') +y_prime = phase_filtered.k / 2 / np.pi +plt.pcolormesh(x_prime, y_prime, phase_filtered.data) +plt.ylabel("1/lambda") # %% font_for_pres() -lala= 2* np.pi/phase_filtered.k -plt.pcolormesh( x_prime , 1/y_prime, phase_filtered * lala /2, vmin=-1000, vmax=1000, cmap=plt.cm.coolwarm) -plt.ylabel('wave length (meters)') -plt.colorbar(label='meters of phase shift') -plt.yscale('log') +lala = 2 * np.pi / phase_filtered.k +plt.pcolormesh( + x_prime, + 1 / y_prime, + phase_filtered * lala / 2, + vmin=-1000, + vmax=1000, + cmap=plt.cm.coolwarm, +) +plt.ylabel("wave length (meters)") +plt.colorbar(label="meters of phase shift") +plt.yscale("log") plt.ylim(0, 1e4) # %% -plt.plot(1/y_prime, (phase_filtered* lala /2), '-') +plt.plot(1 / y_prime, (phase_filtered * lala / 2), "-") # %% -histogram(phase_adjust, bins=[20], dim=['x'], weights=abs(co_spec)).rolling(**r_ave_kargs2).mean().T.plot() +histogram(phase_adjust, bins=[20], dim=["x"], weights=abs(co_spec)).rolling( + **r_ave_kargs2 +).mean().T.plot() # %% -phase_hist = histogram(phase_adjust, bins=[20], dim=['x'], weights=abs(co_spec)) +phase_hist = histogram(phase_adjust, bins=[20], dim=["x"], weights=abs(co_spec)) -phase_hist.rolling(**r_ave_kargs2).mean().T.plot(cmap=plt.cm.ocean_r, levels=np.arange(-1, 18, 0.5)) -plt.ylabel('radients (1/pi)') +phase_hist.rolling(**r_ave_kargs2).mean().T.plot( + cmap=plt.cm.ocean_r, levels=np.arange(-1, 18, 0.5) +) +plt.ylabel("radients (1/pi)") plt.grid() -#histogram(phase_adjust, bins=[20], dim=['x']).rolling(**r_ave_kargs2).mean().T.plot() +# histogram(phase_adjust, bins=[20], dim=['x']).rolling(**r_ave_kargs2).mean().T.plot() # %% -phase_hist = histogram(phase_adjust, bins=[100], dim=['f'], weights=abs(co_spec)) -phase_hist.T.plot(cmap=plt.cm.ocean_r, levels=np.arange(-1, 20, 0.5) ) -plt.ylabel('radients (1/pi)') +phase_hist = histogram(phase_adjust, bins=[100], dim=["f"], weights=abs(co_spec)) +phase_hist.T.plot(cmap=plt.cm.ocean_r, levels=np.arange(-1, 20, 0.5)) +plt.ylabel("radients (1/pi)") plt.grid() -#phase_hist.rolling(**r_ave_kargs2).mean().T.plot(cmap=plt.cm.ocean_r, levels=np.arange(-1, 20, 0.5)) +# phase_hist.rolling(**r_ave_kargs2).mean().T.plot(cmap=plt.cm.ocean_r, levels=np.arange(-1, 20, 0.5)) # %% -phase_adjust = phase_adjust.where(phase_adjust < 1 , phase_adjust -2) -phase_adjust = phase_adjust.where(phase_adjust > -1 , phase_adjust +2) -phase_hist = histogram(phase_adjust, bins=[60], dim=['f'], weights=abs(co_spec)) -phase_hist.T.plot(cmap=plt.cm.ocean_r, levels=np.arange(-1, 20, 0.5) ) -plt.ylabel('radients (1/pi)') +phase_adjust = phase_adjust.where(phase_adjust < 1, phase_adjust - 2) +phase_adjust = phase_adjust.where(phase_adjust > -1, phase_adjust + 2) +phase_hist = histogram(phase_adjust, bins=[60], dim=["f"], weights=abs(co_spec)) +phase_hist.T.plot(cmap=plt.cm.ocean_r, levels=np.arange(-1, 20, 0.5)) +plt.ylabel("radients (1/pi)") plt.grid() # %% -phase_hist.T.rolling(Y_model_hat_bin = 5, x=10, center= True, min_periods= 2).mean().plot(cmap=plt.cm.ocean_r, levels=np.arange(-1, 10, 0.5) ) -plt.ylabel('radients (1/pi)') +phase_hist.T.rolling(Y_model_hat_bin=5, x=10, center=True, min_periods=2).mean().plot( + cmap=plt.cm.ocean_r, levels=np.arange(-1, 10, 0.5) +) +plt.ylabel("radients (1/pi)") plt.grid() # %% -histogram(phase, bins=[60], dim=['f']).T.plot() +histogram(phase, bins=[60], dim=["f"]).T.plot() -plt.plot(phase, 'k', linewidth= 1, alpha =0.5) -#plt.plot(phase.k, phase, 'k', linewidth=0.8, alpha= 0.6) -phase.sel(x=slice(500000, 600000)).mean('x').rolling(k=20, center=True, min_periods=2).mean().plot()#.hist(bins= 40) -phase.sel(x=slice(500000, 600000)).mean('x').rolling(k=20, center=True, min_periods=2).std().plot()#.hist(bins= 40) +plt.plot(phase, "k", linewidth=1, alpha=0.5) +# plt.plot(phase.k, phase, 'k', linewidth=0.8, alpha= 0.6) +phase.sel(x=slice(500000, 600000)).mean("x").rolling( + k=20, center=True, min_periods=2 +).mean().plot() # .hist(bins= 40) +phase.sel(x=slice(500000, 600000)).mean("x").rolling( + k=20, center=True, min_periods=2 +).std().plot() # .hist(bins= 40) # %% -phase.mean('x').plot()#.hist(bins= 40) +phase.mean("x").plot() # .hist(bins= 40) -phase.where(phase > np.pi/2, phase + np.pi).plot() +phase.where(phase > np.pi / 2, phase + np.pi).plot() -C_ab= abs(co_spec)**2 #/ S_aa/ S_bb +C_ab = abs(co_spec) ** 2 # / S_aa/ S_bb np.log(C_ab).sel(k=slice(0, 0.03)).plot() -C_ab= abs(co_spec)**2 / S_aa/ S_bb -(C_ab).sel(k=slice(0, 0.03)).plot(levels = np.arange(0.5, 1+0.1, 0.05)) -#(C_ab.where(~np.isnan(C_ab), 0)).sel(k=slice(0, 0.03)).plot.hist(bins= 30) +C_ab = abs(co_spec) ** 2 / S_aa / S_bb +(C_ab).sel(k=slice(0, 0.03)).plot(levels=np.arange(0.5, 1 + 0.1, 0.05)) +# (C_ab.where(~np.isnan(C_ab), 0)).sel(k=slice(0, 0.03)).plot.hist(bins= 30) -x_nan_mask = (~np.isnan(C_ab)).sum('k') == 0 +x_nan_mask = (~np.isnan(C_ab)).sum("k") == 0 C_ab[:, ~x_nan_mask].min() -phase = np.arctan2( -co_spec.imag, co_spec.real ) +phase = np.arctan2(-co_spec.imag, co_spec.real) -plt.plot( phase ) -plt.plot(S_bb, 'k', linewidth= 0.5, alpha =0.5) +plt.plot(phase) +plt.plot(S_bb, "k", linewidth=0.5, alpha=0.5) np.nanmin(C_ab.imag) np.nanmax(C_ab.imag) Gspec.beam # %% -Ai = A.sel(x= 500000, method='nearest') -Bi = B.sel(x= 500000, method='nearest') +Ai = A.sel(x=500000, method="nearest") +Bi = B.sel(x=500000, method="nearest") -M.figure_axis_xy(10, 2.5, view_scale= 0.7) -plt.plot(np.fft.irfft(Ai) ) -plt.plot(np.fft.irfft(Bi) ) +M.figure_axis_xy(10, 2.5, view_scale=0.7) +plt.plot(np.fft.irfft(Ai)) +plt.plot(np.fft.irfft(Bi)) # %% -co_spec=Ai.imag *Bi.real -C_ab= abs(co_spec)**2 / ( (Ai * Ai.conj()).real * (Bi * Bi.conj()).real ) +co_spec = Ai.imag * Bi.real +C_ab = abs(co_spec) ** 2 / ((Ai * Ai.conj()).real * (Bi * Bi.conj()).real) plt.plot(C_ab.k, C_ab) -#plt.semilogx(C_ab.k, abs(Ai.imag *Bi.real)**2) +# plt.semilogx(C_ab.k, abs(Ai.imag *Bi.real)**2) -phase = np.arctan2( -co_spec.imag, co_spec.real ) -phase[phase <- np.pi/2] = phase[phase < - np.pi/2] + np.pi +phase = np.arctan2(-co_spec.imag, co_spec.real) +phase[phase < -np.pi / 2] = phase[phase < -np.pi / 2] + np.pi phase.plot.hist() plt.plot(phase.k, phase) # %% @@ -306,16 +336,26 @@ def get_coherency(d1, d2, Lint, dt=None): from scipy import signal if dt is None: - dt =np.diff((dd.time - dd.time[0]).data.astype('m8[D]')).mean().astype('int') + dt = np.diff((dd.time - dd.time[0]).data.astype("m8[D]")).mean().astype("int") else: - dt= dt - - coh = signal.coherence( d1, d2, fs=1/dt, window ='hann', nperseg=Lint, noverlap=None, detrend='linear') - - co_spec = signal.csd( d1, d2, fs=1/dt, window ='hann', nperseg=Lint,noverlap=None, detrend='linear') - phase = np.arctan2( -co_spec[1].imag, co_spec[1].real ) - - #Spec = m_spec.Spectrum(dd, dt=dt, verbose=False) - #co_spec = signal.csd( d1, d2, fs=1/dt, window ='hann', nperseg=Lint) - #np.arctan2(-) - return {'f' : coh[0], 'coh2' : coh[1], 'coh' : np.sqrt(abs(coh[1])), 'cospec' :co_spec , 'phase':phase} + dt = dt + + coh = signal.coherence( + d1, d2, fs=1 / dt, window="hann", nperseg=Lint, noverlap=None, detrend="linear" + ) + + co_spec = signal.csd( + d1, d2, fs=1 / dt, window="hann", nperseg=Lint, noverlap=None, detrend="linear" + ) + phase = np.arctan2(-co_spec[1].imag, co_spec[1].real) + + # Spec = m_spec.Spectrum(dd, dt=dt, verbose=False) + # co_spec = signal.csd( d1, d2, fs=1/dt, window ='hann', nperseg=Lint) + # np.arctan2(-) + return { + "f": coh[0], + "coh2": coh[1], + "coh": np.sqrt(abs(coh[1])), + "cospec": co_spec, + "phase": phase, + } diff --git a/analysis/SA01_download_tracks.py b/analysis/SA01_download_tracks.py index 9c3a1e50..01ac17af 100644 --- a/analysis/SA01_download_tracks.py +++ b/analysis/SA01_download_tracks.py @@ -1,46 +1,49 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This is a test file, playing with the Earth data login and icepyx. """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline -#from pprint import pprint +# %matplotlib inline +# from pprint import pprint import icepyx as ipx -import m_tools_ph3 as MT +import m_tools_ph3 as MT # %% -downlaod_path = mconfig['paths']['scratch'] +'/SH_batch02/' -path = mconfig['paths']['analysis']+'../track_lists/' +downlaod_path = mconfig["paths"]["scratch"] + "/SH_batch02/" +path = mconfig["paths"]["analysis"] + "../track_lists/" MT.mkdirs_r(downlaod_path) -flist = MT.json_load('Batch02_alex_tracks', path) +flist = MT.json_load("Batch02_alex_tracks", path) D = pd.DataFrame(flist) + def str2dt64(s): - return np.datetime64(s[0:4]+'-'+s[4:6]+'-'+s[6:8]) + return np.datetime64(s[0:4] + "-" + s[4:6] + "-" + s[6:8]) + -D['date'] = D[0].apply(lambda row: str2dt64(row[0:8]) ) +D["date"] = D[0].apply(lambda row: str2dt64(row[0:8])) -dmin, dmax = D['date'].min(), D['date'].max() +dmin, dmax = D["date"].min(), D["date"].max() dmin, dmax -D['RGT'] = D[1].apply(lambda row: row[0:4]) -D['cycle'] = D[1].apply(lambda row: row[4:6]) -D['segment'] = D[1].apply(lambda row: row[6:8]) -#D['segment'].hist() +D["RGT"] = D[1].apply(lambda row: row[0:4]) +D["cycle"] = D[1].apply(lambda row: row[4:6]) +D["segment"] = D[1].apply(lambda row: row[6:8]) +# D['segment'].hist() -D['id'] = D[0]+'_'+D[1] -#D['id_compare'] = D[0]+'_'+ -D['id_compare'] = D['RGT']+D['cycle'] +D["id"] = D[0] + "_" + D[1] +# D['id_compare'] = D[0]+'_'+ +D["id_compare"] = D["RGT"] + D["cycle"] Dsub = D[0:2] @@ -51,12 +54,17 @@ def str2dt64(s): # %% -date_range =[str(dmin).split(' ')[0],str(dmax).split(' ')[0]] -region_a = ipx.Query('ATL03',[30, -70, -30, -55],date_range, \ - start_time='00:00:00', end_time='23:59:59', \ - tracks = list(Dsub['RGT'])) - -region_a.earthdata_login('mhell','mhell@ucsd.edu') +date_range = [str(dmin).split(" ")[0], str(dmax).split(" ")[0]] +region_a = ipx.Query( + "ATL03", + [30, -70, -30, -55], + date_range, + start_time="00:00:00", + end_time="23:59:59", + tracks=list(Dsub["RGT"]), +) + +region_a.earthdata_login("mhell", "mhell@ucsd.edu") # @[49[4tK\-qBWB%5 # %% @@ -64,48 +72,57 @@ def str2dt64(s): region_a.avail_granules(ids=True) # %% -#region_a.visualize_spatial_extent() -#region_a.order_vars.remove(all=True) - -ATL03_var_list = ['dem_h', 'delta_time', 'lon_ph', 'lat_ph', 'h_ph', 'dist_ph_along', 'atlas_sdp_gps_epoch', 'signal_conf_ph'] -region_a.order_vars.append(var_list=ATL03_var_list)#, keyword_list=['orbit_info']) -region_a.order_vars.append( keyword_list=['orbit_info']) -#region_a.order_vars.wanted +# region_a.visualize_spatial_extent() +# region_a.order_vars.remove(all=True) + +ATL03_var_list = [ + "dem_h", + "delta_time", + "lon_ph", + "lat_ph", + "h_ph", + "dist_ph_along", + "atlas_sdp_gps_epoch", + "signal_conf_ph", +] +region_a.order_vars.append(var_list=ATL03_var_list) # , keyword_list=['orbit_info']) +region_a.order_vars.append(keyword_list=["orbit_info"]) +# region_a.order_vars.wanted region_a.subsetparams(Coverage=region_a.order_vars.wanted) region_a.tracks -#region_a.file_vars +# region_a.file_vars # %% help(region_a.granules) -#region_a.granules.__dict__ -#region_a.granules.avail[0].keys() +# region_a.granules.__dict__ +# region_a.granules.avail[0].keys() type(region_a.granules.avail) -gran_list = [i['producer_granule_id'] for i in region_a.granules.avail] +gran_list = [i["producer_granule_id"] for i in region_a.granules.avail] len(gran_list) -sub_set= list() -for id_wanted in Dsub['id_compare']: +sub_set = list() +for id_wanted in Dsub["id_compare"]: sub_set.append([i for i in gran_list if id_wanted in i][0]) gran_to_delete = list(set(gran_list) - set(sub_set)) # %% -wanted_granuals= list() +wanted_granuals = list() for i in region_a.granules.avail: - if any( [id_wanted in i['producer_granule_id'] for id_wanted in Dsub['id_compare']] ): + if any([id_wanted in i["producer_granule_id"] for id_wanted in Dsub["id_compare"]]): print(i) wanted_granuals.append(i) region_a.granules.avail = wanted_granuals -#help(region_a.granules) +# help(region_a.granules) len(region_a.granules.avail) region_a.avail_granules(ids=True) -region_a.granules.download(verbose= True, path=downlaod_path ) +region_a.granules.download(verbose=True, path=downlaod_path) # print('check how many granuals are available') diff --git a/analysis/SB02_make_spectra_LS.py b/analysis/SB02_make_spectra_LS.py index d82fff1e..6df11478 100644 --- a/analysis/SB02_make_spectra_LS.py +++ b/analysis/SB02_make_spectra_LS.py @@ -1,23 +1,25 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -37,59 +39,58 @@ import imp - -#import s3fs +# import s3fs # %% # Python reader based on Pandas. Other reader examples available in readers.py -plot_path = mconfig['paths']['plot'] + '/tests/' +plot_path = mconfig["paths"]["plot"] + "/tests/" -track_name= 'ATL03_20190515060436_07170312_002_02' -load_path = base_path + 'data/data1/' +track_name = "ATL03_20190515060436_07170312_002_02" +load_path = base_path + "data/data1/" # test which beams exist: -all_beams = ['gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r'] -low_beams = ['gt1l', 'gt2l', 'gt3l'] -high_beams = ['gt1r', 'gt2r', 'gt3r'] +all_beams = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] +low_beams = ["gt1l", "gt2l", "gt3l"] +high_beams = ["gt1r", "gt2r", "gt3r"] # %% -#Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') +# Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') imp.reload(io) -Gfilt = io.load_pandas_table_dict(track_name + '_B01_corrected', load_path) -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) +Gfilt = io.load_pandas_table_dict(track_name + "_B01_corrected", load_path) +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # %% -Gi =Gd[high_beams[0]] -Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') -#np.isnan(Gi['dist']).sum() -Gi = Gi[(Gi['dist'] > 140000) & (Gi['dist'] < 160001)] +Gi = Gd[high_beams[0]] +Gi = Gi[~np.isnan(Gi["heights_c_weighted_mean"])].sort_values("dist") +# np.isnan(Gi['dist']).sum() +Gi = Gi[(Gi["dist"] > 140000) & (Gi["dist"] < 160001)] # %% -plt.plot( Gi['dist'], Gi['heights_c_weighted_mean'], '-') -#plt.xlim(140000, 145000) +plt.plot(Gi["dist"], Gi["heights_c_weighted_mean"], "-") +# plt.xlim(140000, 145000) plt.ylim(-1, 1) -#np.diff(np.array(Gi['dist'])) +# np.diff(np.array(Gi['dist'])) # %% test and compare to FFT from scipy.signal import detrend -y =detrend(np.array(Gi['heights_c_weighted_mean']) ) -f_fft2, df2 = spec.calc_freq_fft( np.array(Gi['dist']), y.size) -spec_fft2 = spec.calc_spectrum_fft(y, df2, np.array(Gi['dist']).size) -f_fft2_regular, spec_fft2_regular = f_fft2, spec_fft2 +y = detrend(np.array(Gi["heights_c_weighted_mean"])) +f_fft2, df2 = spec.calc_freq_fft(np.array(Gi["dist"]), y.size) +spec_fft2 = spec.calc_spectrum_fft(y, df2, np.array(Gi["dist"]).size) +f_fft2_regular, spec_fft2_regular = f_fft2, spec_fft2 # %% # test getting phase -t_base =np.array(Gi['dist']) -t_base = t_base - t_base[0]#.mean() +t_base = np.array(Gi["dist"]) +t_base = t_base - t_base[0] # .mean() m1 = ls.offset() * np.ones(len(t_base)) m1_man = ls.offset() * np.ones(len(t_base)) -fi_divide= 1000 +fi_divide = 1000 thetas = np.zeros([2]) thetas_v2 = np.zeros([2]) @@ -99,51 +100,60 @@ fft_contstruct[0] = 0 + np.complex128(0) LS_power_construct = np.complex128(np.zeros(N_f)) -for fi, i in zip( f_fft2[1:], np.arange(1,N_f+1)): - #m1 += ls.model(t_base, fi) - #print(ls.model_parameters(fi)) +for fi, i in zip(f_fft2[1:], np.arange(1, N_f + 1)): + # m1 += ls.model(t_base, fi) + # print(ls.model_parameters(fi)) theta = ls.model_parameters(fi) - phi = t_base * 2 * np.pi *fi - THETA = 0#np.arctan2( np.sin(2* phi).sum(), np.cos(2* phi).sum() )/2 - m_i = theta[1]* np.cos(phi - THETA) + theta[0] * np.sin(phi- THETA) - m1_man += m_i + phi = t_base * 2 * np.pi * fi + THETA = 0 # np.arctan2( np.sin(2* phi).sum(), np.cos(2* phi).sum() )/2 + m_i = theta[1] * np.cos(phi - THETA) + theta[0] * np.sin(phi - THETA) + m1_man += m_i thetas = np.vstack([thetas, theta]) # follow Hocke anad Kämpfer - R = np.sum( theta[1] *np.cos(phi - THETA)*np.cos(phi - THETA) + theta[0] * np.sin(phi - THETA) *np.cos(phi - THETA) ) - I = np.sum( theta[1] *np.cos(phi - THETA)*np.sin(phi - THETA) + theta[0] * np.sin(phi - THETA) *np.sin(phi - THETA) ) - C = np.sum( np.cos(phi)*np.cos(phi) ) - S = np.sum( np.sin(phi)*np.sin(phi) ) + R = np.sum( + theta[1] * np.cos(phi - THETA) * np.cos(phi - THETA) + + theta[0] * np.sin(phi - THETA) * np.cos(phi - THETA) + ) + I = np.sum( + theta[1] * np.cos(phi - THETA) * np.sin(phi - THETA) + + theta[0] * np.sin(phi - THETA) * np.sin(phi - THETA) + ) + C = np.sum(np.cos(phi) * np.cos(phi)) + S = np.sum(np.sin(phi) * np.sin(phi)) m_ii_sigma = np.var(m_i) - P = ( (R**2 / C) + (I**2 /S) ) / (2 * m_ii_sigma) - A_ft = np.sqrt((R**2 / C + I**2 /S) * N_f/4 ) *2 - t_ave = (t_base[0] + t_base[-1])/2 - phi_ft =np.arctan2(I, R) + t_ave *fi + THETA + P = ((R**2 / C) + (I**2 / S)) / (2 * m_ii_sigma) + A_ft = np.sqrt((R**2 / C + I**2 / S) * N_f / 4) * 2 + t_ave = (t_base[0] + t_base[-1]) / 2 + phi_ft = np.arctan2(I, R) + t_ave * fi + THETA - fft_contstruct[i] = A_ft * (np.cos(phi_ft) + np.sin(phi_ft)* 1j) + fft_contstruct[i] = A_ft * (np.cos(phi_ft) + np.sin(phi_ft) * 1j) LS_power_construct[i] = P theta - np.sqrt(2/N_f) * I/ np.sqrt(S), np.sqrt(2/N_f) * R/ np.sqrt(C) + np.sqrt(2 / N_f) * I / np.sqrt(S), np.sqrt(2 / N_f) * R / np.sqrt(C) thetas = thetas[1:, :] # %% # %% simple spectra from astropy.timeseries import LombScargle -ls = LombScargle( np.array(Gi['dist']) , np.array(Gi['heights_c_weighted_mean']) , fit_mean=False) -ls_auto_power = ls.power(f_fft2[1::] , normalization='psd') -#ls_auto_f , ls_auto_power = ls.autopower(normalization='psd', samples_per_peak=0.1) +ls = LombScargle( + np.array(Gi["dist"]), np.array(Gi["heights_c_weighted_mean"]), fit_mean=False +) + +ls_auto_power = ls.power(f_fft2[1::], normalization="psd") +# ls_auto_f , ls_auto_power = ls.autopower(normalization='psd', samples_per_peak=0.1) -plt.plot(f_fft2[1::] , spec.LS_power_to_PSD(ls_auto_power, y.size, df2) , label= 'LS') +plt.plot(f_fft2[1::], spec.LS_power_to_PSD(ls_auto_power, y.size, df2), label="LS") -plt.plot(f_fft2[1:] , LS_power_construct[1:] /y.size , label= 'LS construct') +plt.plot(f_fft2[1:], LS_power_construct[1:] / y.size, label="LS construct") -plt.plot(f_fft2[:-1], spec_fft2 , label= 'standard FFT') +plt.plot(f_fft2[:-1], spec_fft2, label="standard FFT") plt.legend() plt.xlim(0, 0.01) @@ -153,11 +163,11 @@ # m2 += ls.model(t_base, fi) -plt.plot(t_base, m1_man, '-', label='LS model') +plt.plot(t_base, m1_man, "-", label="LS model") -#plt.plot(t_base, m2 - m2.mean()) +# plt.plot(t_base, m2 - m2.mean()) -plt.plot( t_base , np.array(Gi['heights_c_weighted_mean']) , c='black', label=' org. data') +plt.plot(t_base, np.array(Gi["heights_c_weighted_mean"]), c="black", label=" org. data") plt.xlim(t_base[100], t_base[700]) plt.legend() @@ -169,7 +179,7 @@ f_fft2.shape -N=y.size +N = y.size fft_contstruct.imag.std() phih_from_data.imag.std() @@ -183,65 +193,79 @@ abs(phih_from_data).sum() # compare FFT coefficients again -F= M.figure_axis_xy(10, 2.5, view_scale= 0.8) -plt.plot(f_fft2, abs(fft_contstruct) ) -plt.plot(f_fft2[0:-1], abs(phih_from_data) ) -#F.ax.set_xscale('log') +F = M.figure_axis_xy(10, 2.5, view_scale=0.8) +plt.plot(f_fft2, abs(fft_contstruct)) +plt.plot(f_fft2[0:-1], abs(phih_from_data)) +# F.ax.set_xscale('log') plt.xlim(0, 0.01) # %% -M.figure_axis_xy(10, 2.5, view_scale= 0.8) -plt.plot(f_fft2, (fft_contstruct.imag )) -plt.plot(f_fft2[0:-1], (phih_from_LS.imag ) ) +M.figure_axis_xy(10, 2.5, view_scale=0.8) +plt.plot(f_fft2, (fft_contstruct.imag)) +plt.plot(f_fft2[0:-1], (phih_from_LS.imag)) plt.xlim(0, 0.01) - # %% # spec_fft3 = 2.*(phih*phih.conj()).real / df2 /np.array(Gi['dist']).size**2 # # plt.semilogy(f_fft2[1:-1], spec_fft3[1:] , 'r') # plt.semilogy(f_fft2[1::] , spec.LS_power_to_PSD(ls_auto_power, y.size, df2), 'k' ) -#plt.xlim(0, 0.01) +# plt.xlim(0, 0.01) # %% # phih_from_data.shape # phih_flat = np.copy(phih) # phih_flat[150::] = 0 m1_man_inv = np.fft.irfft(phih_from_data) -plt.plot(t_base[1:], m1_man_inv , '-', label=' iFFT(FFT(LS model))') -#plt.plot(t_base, m1_man, '*') -plt.plot( t_base , np.array(Gi['heights_c_weighted_mean']) - np.array(Gi['heights_c_weighted_mean']).mean() , c='black', label=' org. data') - -fft_contstruct_inv = np.fft.irfft(1j*fft_contstruct) -plt.plot(t_base, fft_contstruct_inv[1:], '-', label=' iFFT( FFT constructed from LS)') +plt.plot(t_base[1:], m1_man_inv, "-", label=" iFFT(FFT(LS model))") +# plt.plot(t_base, m1_man, '*') +plt.plot( + t_base, + np.array(Gi["heights_c_weighted_mean"]) + - np.array(Gi["heights_c_weighted_mean"]).mean(), + c="black", + label=" org. data", +) + +fft_contstruct_inv = np.fft.irfft(1j * fft_contstruct) +plt.plot(t_base, fft_contstruct_inv[1:], "-", label=" iFFT( FFT constructed from LS)") plt.xlim(t_base[-500], t_base[-100]) plt.legend() # %% single frequency tests -fi = f_fft2[abs(phih_from_data ).argmax()] +fi = f_fft2[abs(phih_from_data).argmax()] t_base_short = t_base[0:100] - t_base[0] theta_i = ls.model_parameters(fi) -m_single = theta_i[0] * np.sin(t_base_short * 2 * np.pi *fi ) + theta_i[1]* np.cos(t_base_short * 2 * np.pi *fi) +m_single = theta_i[0] * np.sin(t_base_short * 2 * np.pi * fi) + theta_i[1] * np.cos( + t_base_short * 2 * np.pi * fi +) alpha = 3 -#m_single2 = alpha * theta_i[0] * np.sin(t_base_short * 2 * np.pi *fi ) + alpha * theta_i[1]* np.cos(t_base_short * 2 * np.pi *fi) -m_single3 = - np.sign(sum(theta_i)) * np.linalg.norm(theta_i) * np.sin(t_base_short * 2 * np.pi *fi + np.arctan2(theta_i[1],theta_i[0])) - -z_i_fft = phih_from_data[abs(phih_from_data ).argmax()] -m_single_fft = - np.sign(z_i_fft.real+z_i_fft.imag) * (z_i_fft).real * np.sin(t_base_short * 2 * np.pi *fi + np.arctan2(z_i_fft.imag,z_i_fft.real)) +# m_single2 = alpha * theta_i[0] * np.sin(t_base_short * 2 * np.pi *fi ) + alpha * theta_i[1]* np.cos(t_base_short * 2 * np.pi *fi) +m_single3 = ( + -np.sign(sum(theta_i)) + * np.linalg.norm(theta_i) + * np.sin(t_base_short * 2 * np.pi * fi + np.arctan2(theta_i[1], theta_i[0])) +) +z_i_fft = phih_from_data[abs(phih_from_data).argmax()] +m_single_fft = ( + -np.sign(z_i_fft.real + z_i_fft.imag) + * (z_i_fft).real + * np.sin(t_base_short * 2 * np.pi * fi + np.arctan2(z_i_fft.imag, z_i_fft.real)) +) plt.plot(t_base_short, m_single) -#plt.plot(t_base_short, m_single2) -plt.plot(t_base_short[:], m_single3[:], 'k.') -#plt.plot(t_base_short[:], m_single_fft[:], 'r') +# plt.plot(t_base_short, m_single2) +plt.plot(t_base_short[:], m_single3[:], "k.") +# plt.plot(t_base_short[:], m_single_fft[:], 'r') # %% @@ -259,48 +283,65 @@ def reconstruct_data_from_LS(LS, x_real_axis, freq): for fi in freq_seq: theta = LS.model_parameters(fi) - y_reconstruct += theta[0] + theta[1] * np.sin(x_real_axis * 2 * np.pi *fi ) + theta[2]* np.cos(x_real_axis * 2 * np.pi *fi) + y_reconstruct += ( + theta[0] + + theta[1] * np.sin(x_real_axis * 2 * np.pi * fi) + + theta[2] * np.cos(x_real_axis * 2 * np.pi * fi) + ) return y_reconstruct + def get_missfit_error(data, model): - return np.nanmean( (data-model)**2)/ np.nanstd(data) + return np.nanmean((data - model) ** 2) / np.nanstd(data) + -y_model = reconstruct_data_from_LS(ls, np.array(Gi['dist']), f_fft2 ) +y_model = reconstruct_data_from_LS(ls, np.array(Gi["dist"]), f_fft2) y_model_fft = np.fft.rfft(y_model) -y_model_r = get_missfit_error(np.array(Gi['heights_c_weighted_mean']) , y_model) +y_model_r = get_missfit_error(np.array(Gi["heights_c_weighted_mean"]), y_model) # %% tests # maximum to be resolved wavenumber -T = 40 #sec -k_0 = (2 * np.pi/ T)**2 / 9.81 +T = 40 # sec +k_0 = (2 * np.pi / T) ** 2 / 9.81 -1/k_0 -x= np.array(Gi['dist']) +1 / k_0 +x = np.array(Gi["dist"]) dt = np.diff(x).mean() -#dt = np.diff(np.array(Gi['dist'])).mean() -min_datapoint = 1/k_0/dt +# dt = np.diff(np.array(Gi['dist'])).mean() +min_datapoint = 1 / k_0 / dt L = int(np.round(min_datapoint) * 10) -S_pwelch_fft = spec.wavenumber_spectrogram(x , np.array(Gi['heights_c_weighted_mean']), L) +S_pwelch_fft = spec.wavenumber_spectrogram( + x, np.array(Gi["heights_c_weighted_mean"]), L +) G_pwelch_fft = S_pwelch_fft.cal_spectrogram() -S_pwelch_ls = spec.wavenumber_spectrogram(x, np.array(Gi['heights_c_weighted_mean']), L) +S_pwelch_ls = spec.wavenumber_spectrogram(x, np.array(Gi["heights_c_weighted_mean"]), L) G_pwelch_fft = S_pwelch_fft.cal_spectrogram() # %% imp.reload(spec) -y =detrend(np.array(Gi['heights_c_weighted_mean']) ) +y = detrend(np.array(Gi["heights_c_weighted_mean"])) -S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( x, L, method='fftX2', minimum_frequency=None, maximum_frequency=None, samples_per_peak=0.5) +S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( + x, + L, + method="fftX2", + minimum_frequency=None, + maximum_frequency=None, + samples_per_peak=0.5, +) -S_pwelch_ls = spec.wavenumber_spectrogram_LS_even(x, y, L, waven_method = S_pwelch_k , dy=None , ov=None, window=None, kjumps=2) +S_pwelch_ls = spec.wavenumber_spectrogram_LS_even( + x, y, L, waven_method=S_pwelch_k, dy=None, ov=None, window=None, kjumps=2 +) G_pwelch_ls = S_pwelch_ls.cal_spectrogram() S_pwelch_ls.parceval() -plt.plot(G_pwelch_ls.k , G_pwelch_ls.mean('x') ) -plt.plot(G_pwelch_fft.k , G_pwelch_fft.mean('x') , 'k') +plt.plot(G_pwelch_ls.k, G_pwelch_ls.mean("x")) +plt.plot(G_pwelch_fft.k, G_pwelch_fft.mean("x"), "k") plt.xlim(0, 0.02) @@ -309,274 +350,311 @@ def get_missfit_error(data, model): # %% test LS with an even grid where missing values are set to 0 imp.reload(spec) -Gi =Gd[high_beams[0]] -#Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') +Gi = Gd[high_beams[0]] +# Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') # derive spectal limits -T_max = 50 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 -x= np.array(Gi['dist']) +T_max = 50 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 +x = np.array(Gi["dist"]) dx = np.diff(x).mean() -min_datapoint = 1/k_0/dx +min_datapoint = 1 / k_0 / dx L = int(np.round(min_datapoint) * 20) -#plt.plot(np.diff(np.array(Gi['dist']))) +# plt.plot(np.diff(np.array(Gi['dist']))) print(L) -print(L*dx) -print(Gi['dist'].iloc[-1] /L ) - -S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( x, L, dx= dx, method='fftX2', minimum_frequency=None, maximum_frequency=None, samples_per_peak=0.01) -hkey= 'heights_c_weighted_mean' -G2= dict() +print(L * dx) +print(Gi["dist"].iloc[-1] / L) + +S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( + x, + L, + dx=dx, + method="fftX2", + minimum_frequency=None, + maximum_frequency=None, + samples_per_peak=0.01, +) +hkey = "heights_c_weighted_mean" +G2 = dict() for k in high_beams: - x= Gd[k]['dist'] - dd =Gd[k][hkey] + x = Gd[k]["dist"] + dd = Gd[k][hkey] dd_nans = np.isnan(dd) - dd= dd[~dd_nans] - x= x[~dd_nans] - dd = (dd -dd.mean()) - #dd = np.gradient(dd) - #np.isnan(dd).sum() - S = spec.wavenumber_spectrogram_LS_even( np.array(x), np.array(dd), L, waven_method = S_pwelch_k , dy=None , ov=None, window=None, kjumps=2) + dd = dd[~dd_nans] + x = x[~dd_nans] + dd = dd - dd.mean() + # dd = np.gradient(dd) + # np.isnan(dd).sum() + S = spec.wavenumber_spectrogram_LS_even( + np.array(x), + np.array(dd), + L, + waven_method=S_pwelch_k, + dy=None, + ov=None, + window=None, + kjumps=2, + ) G = S.cal_spectrogram() - S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True) + S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.parceval(add_attrs=True) - G['x'] = G.x * dx + G["x"] = G.x * dx stancil_iter = spec.create_chunk_boundaries(L, np.array(dd).size) + def get_stancil_nans(stancil): - idata = Gd[k]['N_photos'][stancil[0]:stancil[-1]] + idata = Gd[k]["N_photos"][stancil[0] : stancil[-1]] return stancil[1], idata.sum() - photon_list = np.array(list(dict(map( get_stancil_nans, stancil_iter )).values())) + photon_list = np.array(list(dict(map(get_stancil_nans, stancil_iter)).values())) G2[k] = G - G2[k].coords['N_photos'] = (('x'), photon_list) + G2[k].coords["N_photos"] = (("x"), photon_list) # %% font_for_pres() -plt.plot(G.k, G.mean('x')) +plt.plot(G.k, G.mean("x")) # %% -#Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() -Gplot = G.rolling(k=10, x=2, min_periods= 1, center=True).mean() +# Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() +Gplot = G.rolling(k=10, x=2, min_periods=1, center=True).mean() import matplotlib.colors as colors from matplotlib.gridspec import GridSpec font_for_print() -F = M.figure_axis_xy(6.5, 5.6, container= True, view_scale =0.8) +F = M.figure_axis_xy(6.5, 5.6, container=True, view_scale=0.8) -plt.suptitle('LS on regular grid', y = 0.95) -gs = GridSpec(3,3, wspace=0.1, hspace=0.7)#figure=fig, -#clev=np.arange(0, 6, 0.1)*3 -clev = M.clevels(Gplot.data, 31)* 0.8 +plt.suptitle("LS on regular grid", y=0.95) +gs = GridSpec(3, 3, wspace=0.1, hspace=0.7) # figure=fig, +# clev=np.arange(0, 6, 0.1)*3 +clev = M.clevels(Gplot.data, 31) * 0.8 -for pos, k, pflag in zip([gs[0, 0],gs[0, 1],gs[0, 2] ], high_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[0, 0], gs[0, 1], gs[0, 2]], high_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G2[k]#.rolling(k=10, x=2, min_periods= 1, center=True).mean() + Gplot = G2[k] # .rolling(k=10, x=2, min_periods= 1, center=True).mean() - #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) + # plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) - plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap=plt.cm.ocean_r, vmin = clev[0], vmax = clev[-1]) + plt.pcolormesh( + Gplot.x / 1e3, + 1 / Gplot.k, + Gplot, + cmap=plt.cm.ocean_r, + vmin=clev[0], + vmax=clev[-1], + ) - plt.gca().set_yscale('log') + plt.gca().set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') - plt.xlabel('Distance from the Ice Edge (km)') - plt.ylim(20,L*dx/2) - plt.title(k, loc='left') + plt.xlabel("Distance from the Ice Edge (km)") + plt.ylim(20, L * dx / 2) + plt.title(k, loc="left") # plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) # ax0.axhline(30, color='black', linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -G3_sum = G2[k]*0 -G3_sum['N_photos'] = G3_sum['N_photos']*0 -G3_sum= G3_sum.sel(x=slice(G3_sum.x[0],G3_sum.x[-3]) ) +G3_sum = G2[k] * 0 +G3_sum["N_photos"] = G3_sum["N_photos"] * 0 +G3_sum = G3_sum.sel(x=slice(G3_sum.x[0], G3_sum.x[-3])) for k in high_beams: G3_sum += G2[k].sel(x=G3_sum.x) - G3_sum['N_photos'] += G2[k].sel(x=G3_sum.x)['N_photos'] + G3_sum["N_photos"] += G2[k].sel(x=G3_sum.x)["N_photos"] -G3_sum= G3_sum/3 -G3_sum['N_photos'] =G3_sum['N_photos']/3 +G3_sum = G3_sum / 3 +G3_sum["N_photos"] = G3_sum["N_photos"] / 3 -pos, k, pflag = gs[1, 0], 'mean', True +pos, k, pflag = gs[1, 0], "mean", True ax0 = F.fig.add_subplot(pos) -Gplot = G3_sum.rolling(k=5, x=2, min_periods= 1, center=True).mean() +Gplot = G3_sum.rolling(k=5, x=2, min_periods=1, center=True).mean() -#plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) +# plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) -plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap=plt.cm.ocean_r, vmin = clev[0], vmax = clev[-1]) +plt.pcolormesh( + Gplot.x / 1e3, 1 / Gplot.k, Gplot, cmap=plt.cm.ocean_r, vmin=clev[0], vmax=clev[-1] +) -plt.gca().set_yscale('log') +plt.gca().set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') -plt.xlabel('Distance from the Ice Edge (km)') -plt.ylim(20,L*dx/2) -plt.title(k, loc='left') +plt.xlabel("Distance from the Ice Edge (km)") +plt.ylim(20, L * dx / 2) +plt.title(k, loc="left") # plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) # ax0.axhline(30, color='black', linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() pos = gs[1, 1] ax0 = F.fig.add_subplot(pos) -plt.title('ave photon per chunk') -plt.plot(Gplot.x/1e3, Gplot.N_photos , c='black') +plt.title("ave photon per chunk") +plt.plot(Gplot.x / 1e3, Gplot.N_photos, c="black") pos = gs[1, 2] ax0 = F.fig.add_subplot(pos) -plt.title('spectal power decay') -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(1/500, 1/100)).integrate('k') , c='black') +plt.title("spectal power decay") +plt.plot(Gplot.x / 1e3, Gplot.sel(k=slice(1 / 500, 1 / 100)).integrate("k"), c="black") -F.save_light(path=plot_path, name='exmpl_spec_LS_strong_L'+str(L*dx)) +F.save_light(path=plot_path, name="exmpl_spec_LS_strong_L" + str(L * dx)) # %% rar photon data Gi = Gfilt[high_beams[0]] -Gi = Gi[(Gi['dist'] > 140000) & (Gi['dist'] < 160001)] +Gi = Gi[(Gi["dist"] > 140000) & (Gi["dist"] < 160001)] -plt.plot( Gi['dist'], Gi['heights_c_weighted_mean'], '.', alpha= 0.2) -#plt.xlim(140000, 145000) +plt.plot(Gi["dist"], Gi["heights_c_weighted_mean"], ".", alpha=0.2) +# plt.xlim(140000, 145000) plt.ylim(-1, 1) -#np.diff(np.array(Gi['dist'])) +# np.diff(np.array(Gi['dist'])) # %% from scipy.signal import detrend -y =detrend(np.array(Gi['heights']) ) -f_fft2, df2 = spec.calc_freq_fft( np.array(Gi['dist']), y.size) -f_fft2 = f_fft2[1:] -spec_fft2 = spec.calc_spectrum_fft(y, df2, np.array(Gi['dist']).size) +y = detrend(np.array(Gi["heights"])) +f_fft2, df2 = spec.calc_freq_fft(np.array(Gi["dist"]), y.size) +f_fft2 = f_fft2[1:] +spec_fft2 = spec.calc_spectrum_fft(y, df2, np.array(Gi["dist"]).size) # %% simple spectra from astropy.timeseries import LombScargle -ls = LombScargle( np.array(Gi['dist']) , np.array(Gi['heights']) ) -ls_power = ls.power(f_fft2_regular[1::1] , normalization='psd') -plt.plot(f_fft2_regular[1::1] , (spec.LS_power_to_PSD(ls_power, y.size, df2)) , 'r', zorder=12) +ls = LombScargle(np.array(Gi["dist"]), np.array(Gi["heights"])) + +ls_power = ls.power(f_fft2_regular[1::1], normalization="psd") +plt.plot( + f_fft2_regular[1::1], (spec.LS_power_to_PSD(ls_power, y.size, df2)), "r", zorder=12 +) # ls_auto_f , ls_auto_power = ls.autopower(normalization='psd', samples_per_peak=0.1) # plt.plot(ls_auto_f , spec.LS_power_to_PSD(ls_auto_power, y.size, df2) , 'r', zorder=12) -#plt.plot(f_fft2, spec_fft2 ) -plt.plot(f_fft2_regular, spec_fft2_regular, 'k-' ) +# plt.plot(f_fft2, spec_fft2 ) +plt.plot(f_fft2_regular, spec_fft2_regular, "k-") plt.xlim(0, 0.01) plt.ylim(0, 80) - - - - - - - - - - - # %% ueven version font_for_pres() Gi = Gfilt[high_beams[0]] -dist_diffs= Gi.sort_values('dist')['dist'].diff() +dist_diffs = Gi.sort_values("dist")["dist"].diff() dist_diffs.median() imp.reload(spec) # derive spectal limits -T_max = 30 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 +T_max = 30 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 -x = np.array(Gi.sort_values('dist')['dist']) +x = np.array(Gi.sort_values("dist")["dist"]) xdiff = np.diff(x) -xdiff = xdiff[xdiff< .1] +xdiff = xdiff[xdiff < 0.1] -#plt.hist( xdiff, bins= 40 ) -dx =np.mean(xdiff)*1e2 -#dx = 0.01#dist_diffs.median() #np.diff(np.array(Gi['dist'])).mean() -1/dx -min_datapoint = 1/k_0/dx -longest_wavelength= 1/k_0 # meters +# plt.hist( xdiff, bins= 40 ) +dx = np.mean(xdiff) * 1e2 +# dx = 0.01#dist_diffs.median() #np.diff(np.array(Gi['dist'])).mean() +1 / dx +min_datapoint = 1 / k_0 / dx +longest_wavelength = 1 / k_0 # meters Lmeters = int(np.round(longest_wavelength) * 20) print(Lmeters) -print(Gi['dist'].max()/ Lmeters ) +print(Gi["dist"].max() / Lmeters) # -shortest_wavenumber = 1/2 # 2 meter wave -desired_dx = 1/shortest_wavenumber +shortest_wavenumber = 1 / 2 # 2 meter wave +desired_dx = 1 / shortest_wavenumber Lpoints = Lmeters / dx print(Lpoints) -Gi.shape[0]/ Lpoints +Gi.shape[0] / Lpoints -S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( None, Lmeters, method='fixed_ratio', dx= dx) +S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS(None, Lmeters, method="fixed_ratio", dx=dx) -S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( np.array(Gi['dist']), Lpoints, method='LS_auto', minimum_frequency=f_fft2_regular[1], maximum_frequency=f_fft2_regular[-1], samples_per_peak=0.1) +S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( + np.array(Gi["dist"]), + Lpoints, + method="LS_auto", + minimum_frequency=f_fft2_regular[1], + maximum_frequency=f_fft2_regular[-1], + samples_per_peak=0.1, +) S_pwelch_k[-1] S_pwelch_k[0] S_pwelch_k.size -#f_regular, df2 = spec.calc_freq_fft_given_dx(desired_dx,Lmeters) -f_regular = np.linspace(k_0/100, 0.1, 300) +# f_regular, df2 = spec.calc_freq_fft_given_dx(desired_dx,Lmeters) +f_regular = np.linspace(k_0 / 100, 0.1, 300) -#f_regular = 1/np.linspace(1/shortest_wavenumber, 10/k_0, 1000)[::-1] +# f_regular = 1/np.linspace(1/shortest_wavenumber, 10/k_0, 1000)[::-1] dk = np.diff(f_regular).mean() f_regular[1:].size f_regular[-1] -G2= dict() -#k = high_beams[0] +G2 = dict() +# k = high_beams[0] for k in high_beams: Gi = Gfilt[k] # for testing amplitude - #Gi = Gi[(Gi['dist'] > 140000) & (Gi['dist'] < 160001)] - #Gi['nans'] = np.isnan(Gi['heights']) - #Gall[k][Gall[k+'_nans']] = 0 - #Gsel = Gi[~Gall[k+'_nans']] - dd = np.array( Gi['heights'] - Gi['heights'].mean() ) - #dd = np.gradient(dd) - #np.isnan(dd).sum() - S = spec.wavenumber_spectrogram_LS(np.array(Gi.dist), dd, Lmeters, waven_method = f_regular , dy=None , ov=None, window=None, kjumps=1) + # Gi = Gi[(Gi['dist'] > 140000) & (Gi['dist'] < 160001)] + # Gi['nans'] = np.isnan(Gi['heights']) + # Gall[k][Gall[k+'_nans']] = 0 + # Gsel = Gi[~Gall[k+'_nans']] + dd = np.array(Gi["heights"] - Gi["heights"].mean()) + # dd = np.gradient(dd) + # np.isnan(dd).sum() + S = spec.wavenumber_spectrogram_LS( + np.array(Gi.dist), + dd, + Lmeters, + waven_method=f_regular, + dy=None, + ov=None, + window=None, + kjumps=1, + ) G = S.cal_spectrogram() - S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True) + S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.parceval(add_attrs=True) G2[k] = G font_for_pres() -plt.plot(G.k, G, 'b', alpha = 0.2) -plt.plot(G.k, G.mean('x'), 'r-' ) +plt.plot(G.k, G, "b", alpha=0.2) +plt.plot(G.k, G.mean("x"), "r-") # for testing amplitude # !!!! vanriance is not conserved yet !!! -plt.plot(f_fft2_regular, spec_fft2_regular, 'k-' ) +plt.plot(f_fft2_regular, spec_fft2_regular, "k-") plt.xlim(0, 0.01) plt.ylim(0, 40) @@ -584,91 +662,103 @@ def get_stancil_nans(stancil): font_for_pres() -plt.plot(G.k, G.mean('x')) +plt.plot(G.k, G.mean("x")) # %% -#Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() -Gplot = G.rolling(k=10, x=2, min_periods= 1, center=True).mean() +# Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() +Gplot = G.rolling(k=10, x=2, min_periods=1, center=True).mean() # %% import matplotlib.colors as colors from matplotlib.gridspec import GridSpec font_for_print() -F = M.figure_axis_xy(6.5, 5.6, container= True, view_scale =0.8) +F = M.figure_axis_xy(6.5, 5.6, container=True, view_scale=0.8) -plt.suptitle('LS on uneven grid (dx= '+ str(int(np.diff(Gplot.x).mean())) +' m)', y = 0.95) -gs = GridSpec(3,3, wspace=0.1, hspace=0.7)#figure=fig, -#clev=np.arange(0, 6, 0.1)*3 -clev = M.clevels(Gplot.data, 31)* 0.9 +plt.suptitle( + "LS on uneven grid (dx= " + str(int(np.diff(Gplot.x).mean())) + " m)", y=0.95 +) +gs = GridSpec(3, 3, wspace=0.1, hspace=0.7) # figure=fig, +# clev=np.arange(0, 6, 0.1)*3 +clev = M.clevels(Gplot.data, 31) * 0.9 -for pos, k, pflag in zip([gs[0, 0],gs[0, 1],gs[0, 2] ], high_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[0, 0], gs[0, 1], gs[0, 2]], high_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G2[k]#.rolling(k=10, x=2, min_periods= 1, center=True).mean() - - #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) - ax0.set_yscale('log') - #plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap=plt.cm.ocean_r) - plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap=plt.cm.ocean_r, vmin = clev[0], vmax = clev[-1]) - - #plt.gca().set_yscale('log') + Gplot = G2[k] # .rolling(k=10, x=2, min_periods= 1, center=True).mean() + + # plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) + ax0.set_yscale("log") + # plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap=plt.cm.ocean_r) + plt.pcolormesh( + Gplot.x / 1e3, + 1 / Gplot.k, + Gplot, + cmap=plt.cm.ocean_r, + vmin=clev[0], + vmax=clev[-1], + ) + + # plt.gca().set_yscale('log') # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') - plt.xlabel('Distance from the Ice Edge (km)') - plt.ylim(20,Lmeters/2) - plt.title(k, loc='left') + plt.xlabel("Distance from the Ice Edge (km)") + plt.ylim(20, Lmeters / 2) + plt.title(k, loc="left") - - #ax0.axhline(30, color='black', linewidth=0.5) + # ax0.axhline(30, color='black', linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -k= high_beams[0] -G3_sum = G2[k]*0 -N_grid=G2[k]*0 -x_common =G2[k].x +k = high_beams[0] +G3_sum = G2[k] * 0 +N_grid = G2[k] * 0 +x_common = G2[k].x for k in high_beams: gii = G2[high_beams[0]] - N_grid +=~np.isnan(gii) - G3_sum += gii.where(~np.isnan(gii), 0).interp(x = x_common) + N_grid += ~np.isnan(gii) + G3_sum += gii.where(~np.isnan(gii), 0).interp(x=x_common) -G3_sum= G3_sum/N_grid +G3_sum = G3_sum / N_grid -pos, k, pflag = gs[1, 0], 'mean', True +pos, k, pflag = gs[1, 0], "mean", True ax0 = F.fig.add_subplot(pos) -Gplot = G3_sum.rolling(k=5, x=5, min_periods= 1, center=True).mean() +Gplot = G3_sum.rolling(k=5, x=5, min_periods=1, center=True).mean() -#plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) +# plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) -plt.pcolormesh(Gplot.x/1e3, 1/Gplot.k , Gplot, cmap=plt.cm.ocean_r, vmin = clev[0], vmax = clev[-1]) +plt.pcolormesh( + Gplot.x / 1e3, 1 / Gplot.k, Gplot, cmap=plt.cm.ocean_r, vmin=clev[0], vmax=clev[-1] +) -plt.gca().set_yscale('log') +plt.gca().set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') -plt.xlabel('Distance from the Ice Edge (km)') -plt.ylim(20,Lmeters/2) -plt.title(k, loc='left') +plt.xlabel("Distance from the Ice Edge (km)") +plt.ylim(20, Lmeters / 2) +plt.title(k, loc="left") -#plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) -ax0.axhline(30, color='black', linewidth=0.5) +# plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) +ax0.axhline(30, color="black", linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() pos = gs[1, 1] ax0 = F.fig.add_subplot(pos) -plt.title('mean # of datapoints / chunk (1e3)') -plt.plot(Gplot.x/1e3, Gplot.N_per_sample/1e3 , c='black') +plt.title("mean # of datapoints / chunk (1e3)") +plt.plot(Gplot.x / 1e3, Gplot.N_per_sample / 1e3, c="black") pos = gs[1, 2] ax0 = F.fig.add_subplot(pos) -plt.title('spectal power decay') -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(1/500, 1/100)).integrate('k') , c='black') +plt.title("spectal power decay") +plt.plot(Gplot.x / 1e3, Gplot.sel(k=slice(1 / 500, 1 / 100)).integrate("k"), c="black") -F.save_light(path=plot_path, name='exmpl_spec_LS_uneven_strong') +F.save_light(path=plot_path, name="exmpl_spec_LS_uneven_strong") diff --git a/analysis/SB02_make_spectra_LS_adjust_amp.py b/analysis/SB02_make_spectra_LS_adjust_amp.py index 26035439..d558c95a 100644 --- a/analysis/SB02_make_spectra_LS_adjust_amp.py +++ b/analysis/SB02_make_spectra_LS_adjust_amp.py @@ -1,23 +1,25 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -36,55 +38,55 @@ import imp -#import s3fs +# import s3fs # %% # Python reader based on Pandas. Other reader examples available in readers.py -plot_path = mconfig['paths']['plot'] + '/tests/' +plot_path = mconfig["paths"]["plot"] + "/tests/" -track_name= 'ATL03_20190515060436_07170312_002_02' -load_path = base_path + 'data/data1/' +track_name = "ATL03_20190515060436_07170312_002_02" +load_path = base_path + "data/data1/" # test which beams exist: -all_beams = ['gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r'] -low_beams = ['gt1l', 'gt2l', 'gt3l'] -high_beams = ['gt1r', 'gt2r', 'gt3r'] +all_beams = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] +low_beams = ["gt1l", "gt2l", "gt3l"] +high_beams = ["gt1r", "gt2r", "gt3r"] # %% -#Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') +# Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') imp.reload(io) -Gfilt = io.load_pandas_table_dict(track_name + '_B01_corrected', load_path) -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) +Gfilt = io.load_pandas_table_dict(track_name + "_B01_corrected", load_path) +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # %% -Gi =Gd[high_beams[0]] -Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') -#np.isnan(Gi['dist']).sum() -Gi = Gi[(Gi['dist'] > 140000) & (Gi['dist'] < 170001)] +Gi = Gd[high_beams[0]] +Gi = Gi[~np.isnan(Gi["heights_c_weighted_mean"])].sort_values("dist") +# np.isnan(Gi['dist']).sum() +Gi = Gi[(Gi["dist"] > 140000) & (Gi["dist"] < 170001)] # %% +# plt.xlim(140000, 145000) +# plt.ylim(-1, 1) - -#plt.xlim(140000, 145000) -#plt.ylim(-1, 1) - -#np.diff(np.array(Gi['dist'])) +# np.diff(np.array(Gi['dist'])) # %% test and compare to FFT from scipy.signal import detrend -y =detrend(np.array(Gi['heights_c_weighted_mean']) ) -data_gap= np.copy(y) +y = detrend(np.array(Gi["heights_c_weighted_mean"])) + +data_gap = np.copy(y) data_gap[500:2000] = np.nan -nan_mask =np.isnan(data_gap) +nan_mask = np.isnan(data_gap) + +nan_mask.sum() / data_gap.size -nan_mask.sum()/data_gap.size +plt.plot(Gi["dist"], y, "-") +plt.plot(Gi["dist"], data_gap, "-") -plt.plot( Gi['dist'], y, '-') -plt.plot( Gi['dist'], data_gap, '-') # %% def create_weighted_window(data, window=None): @@ -94,43 +96,45 @@ def create_weighted_window(data, window=None): """ L = data.size if window is None: - #win=np.hanning(L) + # win=np.hanning(L) import scipy.signal.windows as WINDOWS - win = WINDOWS.tukey(L, alpha=0.1, sym=True) - else: - win=window + win = WINDOWS.tukey(L, alpha=0.1, sym=True) + else: + win = window - factor = np.sqrt( np.var(data) / np.var(( data* win) ) ) - #factor=np.sqrt( L/(win**2).sum()) - win *= factor + factor = np.sqrt(np.var(data) / np.var((data * win))) + # factor=np.sqrt( L/(win**2).sum()) + win *= factor return win def create_window_mask(nan_mask, ddata): from scipy.ndimage.measurements import label - nlabel = label(~nan_mask +0) + + nlabel = label(~nan_mask + 0) windows = np.zeros(nan_mask.size) - for i in np.arange(1, nlabel[1]+1): + for i in np.arange(1, nlabel[1] + 1): - sub_mask = nlabel[0] == i - win= create_weighted_window(ddata[sub_mask]) - #win = create_window(sum(sub_mask)) + sub_mask = nlabel[0] == i + win = create_weighted_window(ddata[sub_mask]) + # win = create_window(sum(sub_mask)) windows[sub_mask] = win - return windows#, window_inv + return windows # , window_inv + -windows = create_window_mask(nan_mask, data_gap ) +windows = create_window_mask(nan_mask, data_gap) np.nanvar(data_gap) -np.nanvar(( data_gap* windows)[~nan_mask] ) +np.nanvar((data_gap * windows)[~nan_mask]) -plt.plot(data_gap* windows) +plt.plot(data_gap * windows) # %% -f_fft2, df2 = spec.calc_freq_fft( np.array(Gi['dist']), y.size) +f_fft2, df2 = spec.calc_freq_fft(np.array(Gi["dist"]), y.size) spec_fft2 = spec.calc_spectrum_fft(y, df2, y.size) f_fft2_regular, spec_fft2_regular = f_fft2, spec_fft2 @@ -139,7 +143,7 @@ def create_window_mask(nan_mask, ddata): win = create_weighted_window(y) dd = np.copy(data_gap) -dd[nan_mask]= 0 +dd[nan_mask] = 0 spec_fft_gap_window1 = spec.calc_spectrum_fft(dd * window_single, df2, y.size) spec_fft_windowed = spec.calc_spectrum_fft(y * window_single, df2, y.size) @@ -148,46 +152,70 @@ def create_window_mask(nan_mask, ddata): # %% simple spectra from astropy.timeseries import LombScargle -ls = LombScargle( np.array(Gi['dist']) , y, fit_mean=False) -fft_sub = f_fft2[1:] -ls_auto_power = ls.power(fft_sub , normalization='psd', assume_regular_frequency= False) - -ls_windowed = LombScargle( np.array(Gi['dist']) , y *win, fit_mean=False) -ls_auto_power_windowed = ls_windowed.power(fft_sub , normalization='psd', assume_regular_frequency= False) - -ls_gap = LombScargle( np.array(Gi['dist'])[~nan_mask] , data_gap[~nan_mask] , fit_mean=False) -ls_gap_auto_power = ls_gap.power(fft_sub , normalization='psd', assume_regular_frequency= False) - -ls_gap_windowed = LombScargle( np.array(Gi['dist'])[~nan_mask], ( data_gap* windows)[~nan_mask] , fit_mean=False) -ls_auto_power_gap_windowed = ls_windowed.power(fft_sub , normalization='psd', assume_regular_frequency= False) - -ls_gap_window1 = LombScargle( np.array(Gi['dist'])[~nan_mask], (data_gap* window_single)[~nan_mask] , fit_mean=False) -ls_auto_power_gap_window1 = ls_windowed.power(fft_sub , normalization='psd', assume_regular_frequency= False) - - -#ls_auto_f , ls_auto_power = l s.autopower(normalization='psd', samples_per_peak=0.1) -F= M.figure_axis_xy(10, 4, view_scale= 0.8) -plt.subplot(2, 1,1) -#plt.plot(f_fft2[1::] , spec.LS_power_to_PSD(ls_auto_power_windowed, y.size, df2), '^' , label= 'LS single window, no gaps') -plt.plot(f_fft2[:-1], spec_fft_windowed, '*-' , label= 'single windowed FFT, no gaps') -plt.plot(f_fft2[:-1], spec_fft_gap_windowed, '-' , label= 'single windowed FFT, gaps') - -#plt.plot(f_fft2[:-1], spec_fft_gap_window1 , label= 'single windowed standard FFT, gaps') - -#plt.plot(fft_sub , spec.LS_power_to_PSD(ls_auto_power, y.size, df2) , label= 'LS no window', zorder=12) -plt.plot(f_fft2[1::] , spec.LS_power_to_PSD(ls_auto_power_gap_windowed, y.size, df2) , label= 'LS_gap_windows') -plt.plot(f_fft2[1::] , spec.LS_power_to_PSD(ls_auto_power_gap_window1, y.size, df2) , label= 'LS_gap_1 window') -#plt.plot(f_fft2[1::] , spec.LS_power_to_PSD(ls_gap_auto_power, y.size, df2) , label= 'LS_gap no windowed') - -#plt.plot(f_fft2[1::] , spec.LS_power_to_PSD(spec_fft_gap_windowed, y.size, df2) , label= 'FFFT_gap_windowed') - -#plt.plot(f_fft2[:-1], spec_fft2 , label= 'standard FFT') +ls = LombScargle(np.array(Gi["dist"]), y, fit_mean=False) +fft_sub = f_fft2[1:] +ls_auto_power = ls.power(fft_sub, normalization="psd", assume_regular_frequency=False) + +ls_windowed = LombScargle(np.array(Gi["dist"]), y * win, fit_mean=False) +ls_auto_power_windowed = ls_windowed.power( + fft_sub, normalization="psd", assume_regular_frequency=False +) + +ls_gap = LombScargle( + np.array(Gi["dist"])[~nan_mask], data_gap[~nan_mask], fit_mean=False +) +ls_gap_auto_power = ls_gap.power( + fft_sub, normalization="psd", assume_regular_frequency=False +) + +ls_gap_windowed = LombScargle( + np.array(Gi["dist"])[~nan_mask], (data_gap * windows)[~nan_mask], fit_mean=False +) +ls_auto_power_gap_windowed = ls_windowed.power( + fft_sub, normalization="psd", assume_regular_frequency=False +) + +ls_gap_window1 = LombScargle( + np.array(Gi["dist"])[~nan_mask], + (data_gap * window_single)[~nan_mask], + fit_mean=False, +) +ls_auto_power_gap_window1 = ls_windowed.power( + fft_sub, normalization="psd", assume_regular_frequency=False +) + + +# ls_auto_f , ls_auto_power = l s.autopower(normalization='psd', samples_per_peak=0.1) +F = M.figure_axis_xy(10, 4, view_scale=0.8) +plt.subplot(2, 1, 1) +# plt.plot(f_fft2[1::] , spec.LS_power_to_PSD(ls_auto_power_windowed, y.size, df2), '^' , label= 'LS single window, no gaps') +plt.plot(f_fft2[:-1], spec_fft_windowed, "*-", label="single windowed FFT, no gaps") +plt.plot(f_fft2[:-1], spec_fft_gap_windowed, "-", label="single windowed FFT, gaps") + +# plt.plot(f_fft2[:-1], spec_fft_gap_window1 , label= 'single windowed standard FFT, gaps') + +# plt.plot(fft_sub , spec.LS_power_to_PSD(ls_auto_power, y.size, df2) , label= 'LS no window', zorder=12) +plt.plot( + f_fft2[1::], + spec.LS_power_to_PSD(ls_auto_power_gap_windowed, y.size, df2), + label="LS_gap_windows", +) +plt.plot( + f_fft2[1::], + spec.LS_power_to_PSD(ls_auto_power_gap_window1, y.size, df2), + label="LS_gap_1 window", +) +# plt.plot(f_fft2[1::] , spec.LS_power_to_PSD(ls_gap_auto_power, y.size, df2) , label= 'LS_gap no windowed') + +# plt.plot(f_fft2[1::] , spec.LS_power_to_PSD(spec_fft_gap_windowed, y.size, df2) , label= 'FFFT_gap_windowed') + +# plt.plot(f_fft2[:-1], spec_fft2 , label= 'standard FFT') plt.legend() plt.xlim(0, 0.01) -plt.subplot(2, 1,2) +plt.subplot(2, 1, 2) # yn= 1 @@ -195,52 +223,59 @@ def create_window_mask(nan_mask, ddata): # f_fft_sub, df2 = spec.calc_freq_fft( np.array(Gi['dist'])[::yn], y_test.size) # plt.plot(f_fft_sub, np.angle(np.fft.rfft(y_test)) , label= 'standard FFT') -plt.plot(f_fft2[:-1],np.angle(np.fft.rfft(y * windows)) , label= 'windowed standard FFT') +plt.plot(f_fft2[:-1], np.angle(np.fft.rfft(y * windows)), label="windowed standard FFT") -plt.plot(f_fft2[:-1],np.angle(np.fft.rfft(y)) , label= 'standard FFT') +plt.plot(f_fft2[:-1], np.angle(np.fft.rfft(y)), label="standard FFT") plt.xlim(0, 0.01) - - # %% # test getting phase -t_base =np.array(Gi['dist']) -t_base = t_base# - t_base[0] +t_base = np.array(Gi["dist"]) +t_base = t_base # - t_base[0] -m1 = ls.offset() * np.ones(len(t_base)) +m1 = ls.offset() * np.ones(len(t_base)) m1_windowed = ls_windowed.offset() * np.ones(len(t_base)) -m1_gap = ls_gap.offset() * np.ones(len(t_base)) +m1_gap = ls_gap.offset() * np.ones(len(t_base)) m1_gap_window1 = ls_gap_window1.offset() * np.ones(len(t_base)) m1_gap_windowed = ls_gap_windowed.offset() * np.ones(len(t_base)) -fi_divide= 1000 +fi_divide = 1000 thetas = np.zeros([2]) thetas_v2 = np.zeros([2]) -for fi in fft_sub:#f_fft2[1:]: - #m1 += ls.model(t_base, fi) - #print(ls.model_parameters(fi)) +for fi in fft_sub: # f_fft2[1:]: + # m1 += ls.model(t_base, fi) + # print(ls.model_parameters(fi)) theta = ls.model_parameters(fi) - m1 += theta[0] * np.sin(t_base * 2 * np.pi *fi ) + theta[1]* np.cos(t_base * 2 * np.pi *fi) + m1 += theta[0] * np.sin(t_base * 2 * np.pi * fi) + theta[1] * np.cos( + t_base * 2 * np.pi * fi + ) theta = ls_windowed.model_parameters(fi) - m1_windowed += theta[0] * np.sin(t_base * 2 * np.pi *fi ) + theta[1]* np.cos(t_base * 2 * np.pi *fi) + m1_windowed += theta[0] * np.sin(t_base * 2 * np.pi * fi) + theta[1] * np.cos( + t_base * 2 * np.pi * fi + ) - #m1_man += theta[0] + theta[1] * np.sin(t_base * 2 * np.pi *fi ) + theta[2]* np.cos(t_base * 2 * np.pi *fi) + # m1_man += theta[0] + theta[1] * np.sin(t_base * 2 * np.pi *fi ) + theta[2]* np.cos(t_base * 2 * np.pi *fi) thetas = np.vstack([thetas, theta]) theta = ls_gap.model_parameters(fi) - m1_gap += theta[0] * np.sin(t_base * 2 * np.pi *fi ) + theta[1]* np.cos(t_base * 2 * np.pi *fi) + m1_gap += theta[0] * np.sin(t_base * 2 * np.pi * fi) + theta[1] * np.cos( + t_base * 2 * np.pi * fi + ) theta = ls_gap_window1.model_parameters(fi) - m1_gap_window1 += theta[0] * np.sin(t_base * 2 * np.pi *fi ) + theta[1]* np.cos(t_base * 2 * np.pi *fi) + m1_gap_window1 += theta[0] * np.sin(t_base * 2 * np.pi * fi) + theta[1] * np.cos( + t_base * 2 * np.pi * fi + ) theta = ls_gap_windowed.model_parameters(fi) - m1_gap_windowed += theta[0] * np.sin(t_base * 2 * np.pi *fi ) + theta[1]* np.cos(t_base * 2 * np.pi *fi) - + m1_gap_windowed += theta[0] * np.sin(t_base * 2 * np.pi * fi) + theta[1] * np.cos( + t_base * 2 * np.pi * fi + ) # ii= np.array([theta[0] * np.sqrt(2) *np.sin(np.arctan(theta[1]/ theta[0]) ) , theta[1]* np.sqrt(2) * np.cos(np.arctan(theta[1]/ theta[0])) ]) # thetas_v2 = np.vstack([ thetas_v2, ii]) @@ -248,7 +283,7 @@ def create_window_mask(nan_mask, ddata): thetas = thetas[1:, :] # thetas_v2 = thetas_v2[1:, :] -N=y.size +N = y.size # test # fft_contstruct = N* np.complex128(thetas[:,1] ) /2 @@ -261,30 +296,37 @@ def create_window_mask(nan_mask, ddata): # m2 += ls.model(t_base, fi) # %% -plt.plot( t_base , data_gap , linewidth=2, alpha=0.5 , c='black', label=' data_grap') -plt.plot(t_base, m1_gap, '-', label='LS model gap') +plt.plot(t_base, data_gap, linewidth=2, alpha=0.5, c="black", label=" data_grap") +plt.plot(t_base, m1_gap, "-", label="LS model gap") plt.xlim(t_base[100], t_base[1000]) plt.legend() # %% -#plt.plot( t_base , y , linewidth=2, alpha=0.5 , c='black', label=' org. data') -plt.plot( t_base , data_gap *window_single , linewidth=2, alpha=0.5 , c='black', label=' data_grap * win') +# plt.plot( t_base , y , linewidth=2, alpha=0.5 , c='black', label=' org. data') +plt.plot( + t_base, + data_gap * window_single, + linewidth=2, + alpha=0.5, + c="black", + label=" data_grap * win", +) -#plt.plot(t_base, m1) -#plt.plot(t_base, m1, '*', label='LS model', alpha=0.2) +# plt.plot(t_base, m1) +# plt.plot(t_base, m1, '*', label='LS model', alpha=0.2) -#plt.plot(t_base, m1_windowed, '-', label='LS model windowed', alpha =0.6) -plt.plot(t_base, m1_gap_window1, '-', label='LS model gap 1 window', alpha =0.6) +# plt.plot(t_base, m1_windowed, '-', label='LS model windowed', alpha =0.6) +plt.plot(t_base, m1_gap_window1, "-", label="LS model gap 1 window", alpha=0.6) plt.xlim(t_base[100], t_base[1000]) plt.legend() # %% -#plt.plot( t_base[1:-1] , (y -m1)[1:-1], linewidth=2, alpha=0.5 , c='black', label=' org. data') -#plt.plot( t_base[1:-1] , (y -m1_windowed)[1:-1], linewidth=2, alpha=0.5 , c='black', label=' org. data') -#plt.xlim(t_base[500], t_base[600]) +# plt.plot( t_base[1:-1] , (y -m1)[1:-1], linewidth=2, alpha=0.5 , c='black', label=' org. data') +# plt.plot( t_base[1:-1] , (y -m1_windowed)[1:-1], linewidth=2, alpha=0.5 , c='black', label=' org. data') +# plt.xlim(t_base[500], t_base[600]) # %% fft again @@ -296,22 +338,24 @@ def create_window_mask(nan_mask, ddata): # % parcevel test -print('with Hanning window') -print('data var:', y.var() ) -spec_data_fft3 = 2.*(phih_from_data*phih_from_data.conj()).real / df3 /y.size**2 -print('data fft sum:', df3 * spec_data_fft3.sum().data ) +print("with Hanning window") +print("data var:", y.var()) +spec_data_fft3 = 2.0 * (phih_from_data * phih_from_data.conj()).real / df3 / y.size**2 +print("data fft sum:", df3 * spec_data_fft3.sum().data) -print('LS model var:', m1.var() ) -spec_LS_model_fft3 = 2.*(phih_from_LS*phih_from_LS.conj()).real / df3 /m1.size**2 -print('LS model fft sum:', df3 * spec_LS_model_fft3.sum().data ) +print("LS model var:", m1.var()) +spec_LS_model_fft3 = 2.0 * (phih_from_LS * phih_from_LS.conj()).real / df3 / m1.size**2 +print("LS model fft sum:", df3 * spec_LS_model_fft3.sum().data) -print('Data gap var:', data_gap[~nan_mask].var() ) -print('LS model without gap var:', m1_gap[~nan_mask].var() ) +print("Data gap var:", data_gap[~nan_mask].var()) +print("LS model without gap var:", m1_gap[~nan_mask].var()) -print('LS model with gap var:', m1_gap.var() ) -spec_LS_model_gap_fft3 = 2.*(phih_from_LS_gap*phih_from_LS_gap.conj()).real / df3 /y.size**2 -print('Ls model with gap fft sum:', df3 * spec_LS_model_gap_fft3.sum().data ) +print("LS model with gap var:", m1_gap.var()) +spec_LS_model_gap_fft3 = ( + 2.0 * (phih_from_LS_gap * phih_from_LS_gap.conj()).real / df3 / y.size**2 +) +print("Ls model with gap fft sum:", df3 * spec_LS_model_gap_fft3.sum().data) # %% fft again @@ -322,131 +366,151 @@ def create_window_mask(nan_mask, ddata): df3 = np.gradient(f_fft3).mean() # % parcevel test -print('withOUT Hanning window') -print('data var:', y.var() ) -spec_data_fft3 = 2.*(phih_from_data*phih_from_data.conj()).real / df3 /y.size**2 -print('data fft sum:', df3 * spec_data_fft3.sum().data ) - -print('LS model var:', m1.var() ) -spec_LS_model_fft3 = 2.*(phih_from_LS*phih_from_LS.conj()).real / df3 /m1.size**2 -print('LS model fft sum:', df3 * spec_LS_model_fft3.sum().data ) +print("withOUT Hanning window") +print("data var:", y.var()) +spec_data_fft3 = 2.0 * (phih_from_data * phih_from_data.conj()).real / df3 / y.size**2 +print("data fft sum:", df3 * spec_data_fft3.sum().data) +print("LS model var:", m1.var()) +spec_LS_model_fft3 = 2.0 * (phih_from_LS * phih_from_LS.conj()).real / df3 / m1.size**2 +print("LS model fft sum:", df3 * spec_LS_model_fft3.sum().data) -print('Data gap var:', data_gap[~nan_mask].var() ) -print('LS model without gap var:', m1_gap[~nan_mask].var() ) -print('LS model with gap var:', m1_gap.var() ) -spec_LS_model_gap_fft3 = 2.*(phih_from_LS_gap*phih_from_LS_gap.conj()).real / df3 /y.size**2 -print('Ls model with gap fft sum:', df3 * spec_LS_model_gap_fft3.sum().data ) +print("Data gap var:", data_gap[~nan_mask].var()) +print("LS model without gap var:", m1_gap[~nan_mask].var()) +print("LS model with gap var:", m1_gap.var()) +spec_LS_model_gap_fft3 = ( + 2.0 * (phih_from_LS_gap * phih_from_LS_gap.conj()).real / df3 / y.size**2 +) +print("Ls model with gap fft sum:", df3 * spec_LS_model_gap_fft3.sum().data) # %% np.angle(phih_from_LS_gap) -F= M.figure_axis_xy(12, 4.1 * 2.5, view_scale= 0.5, container=True) -gs = GridSpec(4,1, wspace=0.1, hspace=0.7)#figure=fig, +F = M.figure_axis_xy(12, 4.1 * 2.5, view_scale=0.5, container=True) +gs = GridSpec(4, 1, wspace=0.1, hspace=0.7) # figure=fig, -pos0,pos1,pos2,pos3 = gs[0, 0],gs[1, 0],gs[2, 0],gs[3, 0] +pos0, pos1, pos2, pos3 = gs[0, 0], gs[1, 0], gs[2, 0], gs[3, 0] ax0 = F.fig.add_subplot(pos0) -#plt.subplot(4, 1, 1) -plt.plot(t_base, m1, '.', c='blue', label='LS model') -plt.plot(t_base, m1_gap, '-',c='red', label='LS model') -#plt.plot(t_base, m2 - m2.mean()) +# plt.subplot(4, 1, 1) +plt.plot(t_base, m1, ".", c="blue", label="LS model") +plt.plot(t_base, m1_gap, "-", c="red", label="LS model") +# plt.plot(t_base, m2 - m2.mean()) -plt.plot( t_base , y , linewidth=2, alpha=0.5 , c='black', label=' org. data') +plt.plot(t_base, y, linewidth=2, alpha=0.5, c="black", label=" org. data") plt.xlim(t_base[10], t_base[2500]) plt.ylim(-1, 1) plt.legend() -def gaus(x, x_0, amp, sigma_g ): - return amp* np.exp(-0.5 * ( (x-x_0)/sigma_g)**2) +def gaus(x, x_0, amp, sigma_g): + return amp * np.exp(-0.5 * ((x - x_0) / sigma_g) ** 2) -def gauss_weight(x, x_positions, amplitudes, sigma_g=5): +def gauss_weight(x, x_positions, amplitudes, sigma_g=5): """ postions and amplitudes must have the same length """ - y= np.ones(x.size) - for p,a in zip(x_positions, amplitudes): - y += gaus(x, p, a, sigma_g) + y = np.ones(x.size) + for p, a in zip(x_positions, amplitudes): + y += gaus(x, p, a, sigma_g) return y +weights = gauss_weight(f_fft3, [0.001, 0.005, 0.045], [0, -0.5, -0.9], sigma_g=0.002) - -weights = gauss_weight(f_fft3, [0.001, 0.005, 0.045], [0, -.5, -.9], sigma_g= 0.002) - -plt.plot(f_fft3, weights ) +plt.plot(f_fft3, weights) ax2 = F.fig.add_subplot(pos1) -#ax2 = plt.subplot(4, 1, 2) -plt.plot(f_fft3, np.abs(phih_from_LS), c='blue', linewidth = 0.5 ) -plt.plot(f_fft3, M.runningmean(np.abs(phih_from_LS) , 10, tailcopy=True), linewidth = 1.2, c='blue' ,label= 'LS full data') -plt.plot(f_fft3, np.abs(phih_from_LS_gap), c='red', linewidth = 0.5 , alpha= 0.6) -plt.plot(f_fft3, M.runningmean(np.abs(phih_from_LS_gap) , 10, tailcopy=True), linewidth = 1.2, c='red' ,label= 'LS gap data') - - -plt.plot(f_fft3, np.abs(phih_from_LS_gap) *weights , linewidth = 0.5, c='orange', alpha= 0.6) -plt.plot(f_fft3, M.runningmean(np.abs(phih_from_LS_gap) *weights, 10, tailcopy=True), linewidth = 1.2, c='orange' , label='LS gap weighted' ) +# ax2 = plt.subplot(4, 1, 2) +plt.plot(f_fft3, np.abs(phih_from_LS), c="blue", linewidth=0.5) +plt.plot( + f_fft3, + M.runningmean(np.abs(phih_from_LS), 10, tailcopy=True), + linewidth=1.2, + c="blue", + label="LS full data", +) +plt.plot(f_fft3, np.abs(phih_from_LS_gap), c="red", linewidth=0.5, alpha=0.6) +plt.plot( + f_fft3, + M.runningmean(np.abs(phih_from_LS_gap), 10, tailcopy=True), + linewidth=1.2, + c="red", + label="LS gap data", +) + + +plt.plot( + f_fft3, np.abs(phih_from_LS_gap) * weights, linewidth=0.5, c="orange", alpha=0.6 +) +plt.plot( + f_fft3, + M.runningmean(np.abs(phih_from_LS_gap) * weights, 10, tailcopy=True), + linewidth=1.2, + c="orange", + label="LS gap weighted", +) plt.legend() plt.xlim(f_fft3[1], f_fft3[-1]) -ax2.set_xscale('log') -ax2.set_yscale('log') +ax2.set_xscale("log") +ax2.set_yscale("log") ax3 = F.fig.add_subplot(pos2) -plt.plot(f_fft3, weights , linewidth = 0.5, c='k', label='LS gap weighted' ) - +plt.plot(f_fft3, weights, linewidth=0.5, c="k", label="LS gap weighted") plt.xlim(f_fft3[0], f_fft3[-1]) plt.xlim(f_fft3[1], f_fft3[-1]) -ax3.set_xscale('log') +ax3.set_xscale("log") -#ax3.set_yscale('log') +# ax3.set_yscale('log') ax4 = F.fig.add_subplot(pos3) -plt.plot(t_base, m1, '-', c='blue', linewidth = 0.5,label='LS model') -#plt.plot(t_base, m1_gap, '-',linewidth = 0.5, c='red', label='LS model', zorder=10) +plt.plot(t_base, m1, "-", c="blue", linewidth=0.5, label="LS model") +# plt.plot(t_base, m1_gap, '-',linewidth = 0.5, c='red', label='LS model', zorder=10) -data_model = np.fft.irfft(phih_from_LS_gap*weights) -plt.plot(t_base[1:], data_model , alpha= 0.8, c='orange', label='LS weighted model') -#plt.plot(t_base, m2 - m2.mean()) +data_model = np.fft.irfft(phih_from_LS_gap * weights) +plt.plot(t_base[1:], data_model, alpha=0.8, c="orange", label="LS weighted model") +# plt.plot(t_base, m2 - m2.mean()) -#plt.plot( t_base , y , linewidth=2, alpha=0.5 , c='black', label=' org. data') +# plt.plot( t_base , y , linewidth=2, alpha=0.5 , c='black', label=' org. data') plt.xlim(t_base[10], t_base[2500]) plt.ylim(-1, 1) plt.legend() -print(' cost', (abs(data_gap[~nan_mask][1:] - data_model[~nan_mask[1:]])**2).sum() ) +print(" cost", (abs(data_gap[~nan_mask][1:] - data_model[~nan_mask[1:]]) ** 2).sum()) # %% - -def objective_func(params, data_x, Z_results, weight_func, nan_mask = None, plot_flag=False): +def objective_func( + params, data_x, Z_results, weight_func, nan_mask=None, plot_flag=False +): # model_x= model_real_space # data= y - alpha =1e3 + alpha = 1e3 + def model_real_space(Z, weights): """ Both inputs must have the same length """ - return np.fft.irfft(Z*weights) + return np.fft.irfft(Z * weights) weights = weight_func(f_fft3, params) @@ -459,68 +523,103 @@ def model_real_space(Z, weights): if plot_flag: - F= M.figure_axis_xy(10, 4.1 * 2.5, view_scale= 0.5, container = True) + F = M.figure_axis_xy(10, 4.1 * 2.5, view_scale=0.5, container=True) - gs = GridSpec(5,1, wspace=0.1, hspace=0.2)#figure=fig, - pos0,pos1,pos2 = gs[0:3, 0],gs[3, 0],gs[4, 0]#,gs[3, 0] + gs = GridSpec(5, 1, wspace=0.1, hspace=0.2) # figure=fig, + pos0, pos1, pos2 = gs[0:3, 0], gs[3, 0], gs[4, 0] # ,gs[3, 0] ax1 = F.fig.add_subplot(pos0) - chunk_l= 400 - chunk_iter = spec.create_chunk_boundaries(chunk_l, data_x.size, ov=0, iter_flag = True) + chunk_l = 400 + chunk_iter = spec.create_chunk_boundaries( + chunk_l, data_x.size, ov=0, iter_flag=True + ) - ofsett0= 6 + ofsett0 = 6 ofsett = np.copy(ofsett0) for chi in chunk_iter: - v1= np.round(np.nanvar(dd), 4) - plt.plot(ofsett+ data_x[chi[0]:chi[-1]] , linewidth=3, alpha=0.5 , c='black', label=' org. data (var:'+str(v1)+')') - - v1= np.round(model_x(Z_results, weights*0 +1)[~nan_mask[1:]].var(), 4) - plt.plot(ofsett + model_x(Z_results, weights*0 +1)[chi[0]:chi[-1]] ,linewidth= 0.8, c='red', label='LS model init (var:'+str(v1)+')') - - v1= np.round(model.var(), 4) - plt.plot(ofsett + model_x(Z_results, weights)[chi[0]:chi[-1]],linewidth= 0.8, c='blue', label='LS model weighted (var:'+str(v1)+')') + v1 = np.round(np.nanvar(dd), 4) + plt.plot( + ofsett + data_x[chi[0] : chi[-1]], + linewidth=3, + alpha=0.5, + c="black", + label=" org. data (var:" + str(v1) + ")", + ) + + v1 = np.round(model_x(Z_results, weights * 0 + 1)[~nan_mask[1:]].var(), 4) + plt.plot( + ofsett + model_x(Z_results, weights * 0 + 1)[chi[0] : chi[-1]], + linewidth=0.8, + c="red", + label="LS model init (var:" + str(v1) + ")", + ) + + v1 = np.round(model.var(), 4) + plt.plot( + ofsett + model_x(Z_results, weights)[chi[0] : chi[-1]], + linewidth=0.8, + c="blue", + label="LS model weighted (var:" + str(v1) + ")", + ) if ofsett == ofsett0: plt.legend() ofsett -= 1 - plt.ylim(ofsett, ofsett0+1) - plt.xlim(0, chunk_l*2) - + plt.ylim(ofsett, ofsett0 + 1) + plt.xlim(0, chunk_l * 2) ax2 = F.fig.add_subplot(pos1) - #ax2 = plt.subplot(3, 1, 2) - plt.plot(weights ) - ax2.set_xscale('log') + # ax2 = plt.subplot(3, 1, 2) + plt.plot(weights) + ax2.set_xscale("log") ax3 = F.fig.add_subplot(pos2) - #ax3 = plt.subplot(3, 1, 3) + # ax3 = plt.subplot(3, 1, 3) # v2_fft= np.fft.rfft(data_x) # v2 = np.round( (2.*(v2_fft*v2_fft.conj()).real /data_x.size**2 ).sum(), 4) # plt.plot(abs(v2_fft) , linewidth=2, alpha=0.5 , c='black', label='org data (var: '+str(v2) +')') - v2 = np.round( (2.*(Z_results*Z_results.conj()).real /data_x.size**2 ).sum(), 4) - plt.plot(abs(Z_results), linewidth= 0.8, c='red', label='Z (var: '+str(v2) +')') - plt.plot(M.runningmean(abs(Z_results) , 20, tailcopy=True), linewidth= 1.5, c='red', zorder=12) - - Z2= Z_results* weights - v2 = np.round( (2.*(Z2*Z2.conj()).real /data_x.size**2 ).sum(), 4) - plt.plot(abs(Z2), linewidth= 0.8, c='blue', label='weighted Z(var: '+str(v2) +')') - plt.plot(M.runningmean(abs(Z2) , 20, tailcopy=True), linewidth= 1.5, c='blue', zorder=12) + v2 = np.round( + (2.0 * (Z_results * Z_results.conj()).real / data_x.size**2).sum(), 4 + ) + plt.plot( + abs(Z_results), linewidth=0.8, c="red", label="Z (var: " + str(v2) + ")" + ) + plt.plot( + M.runningmean(abs(Z_results), 20, tailcopy=True), + linewidth=1.5, + c="red", + zorder=12, + ) + + Z2 = Z_results * weights + v2 = np.round((2.0 * (Z2 * Z2.conj()).real / data_x.size**2).sum(), 4) + plt.plot( + abs(Z2), linewidth=0.8, c="blue", label="weighted Z(var: " + str(v2) + ")" + ) + plt.plot( + M.runningmean(abs(Z2), 20, tailcopy=True), + linewidth=1.5, + c="blue", + zorder=12, + ) plt.legend() - plt.ylim( np.percentile(abs(np.fft.rfft(data)), 1), abs(np.fft.rfft(data)).max()*1.3 ) + plt.ylim( + np.percentile(abs(np.fft.rfft(data)), 1), abs(np.fft.rfft(data)).max() * 1.3 + ) - ax3.set_xscale('log') - ax3.set_yscale('log') + ax3.set_xscale("log") + ax3.set_yscale("log") + fitting_cost = (abs(dd - model) / dd.std()) ** 2 + variance_cost = (abs(dd.var() - model.var()) / dd.std()) ** 2 - fitting_cost =( abs(dd - model) / dd.std() )**2 - variance_cost =( abs(dd.var() - model.var()) / dd.std() ) **2 + return fitting_cost.sum(), alpha * variance_cost - return fitting_cost.sum() ,alpha* variance_cost import lmfit @@ -528,30 +627,34 @@ def model_real_space(Z, weights): p_smothed = M.runningmean(np.abs(phih_from_LS_gap), 20, tailcopy=True) f_max = f_fft3[p_smothed[~np.isnan(p_smothed)].argmax()] -f_list = [f_max/ 2, f_max, f_max]# + 1 * (f_fft3[-1] - f_max)/2]#, f_max + 2 * (f_fft3[-1] - f_max)/3 ] +f_list = [ + f_max / 2, + f_max, + f_max, +] # + 1 * (f_fft3[-1] - f_max)/2]#, f_max + 2 * (f_fft3[-1] - f_max)/3 ] amp_list = [0 for i in f_list] -fi_low = 0 +fi_low = 0 params = lmfit.Parameters() for i in np.arange(len(f_list)): - f_name = 'freq'+str(i) - fi = f_list[i] - if i+1 < len(f_list): - fi_high = f_list[i+1] + f_name = "freq" + str(i) + fi = f_list[i] + if i + 1 < len(f_list): + fi_high = f_list[i + 1] else: fi_high = f_fft3[-1] params.add(f_name, fi, min=fi_low, max=fi_high) - fi_low = fi + fi_low = fi for i in np.arange(len(amp_list)): - a_name = 'amp'+str(i) + a_name = "amp" + str(i) params.add(a_name, 0.5, min=-1, max=2) -params.add('sigma_g', 0.001, min=0.001, max=0.1, vary= False) +params.add("sigma_g", 0.001, min=0.001, max=0.1, vary=False) -params['freq1'].vary = False -params['freq2'].vary = False +params["freq1"].vary = False +params["freq2"].vary = False params.pretty_print() @@ -561,113 +664,171 @@ def reorder_values(params): freq_l = list() amp_l = list() for i in list(params): - if 'freq' in i: + if "freq" in i: freq_l.append(params[i].value) - elif 'amp' in i: + elif "amp" in i: amp_l.append(params[i].value) return freq_l, amp_l if params is not None: freq_l, amp_l = reorder_values(params) - weights = gauss_weight(f_fft3, freq_l, amp_l, sigma_g= params['sigma_g']) + weights = gauss_weight(f_fft3, freq_l, amp_l, sigma_g=params["sigma_g"]) else: - weights = gauss_weight(f_fft3, [0.001, 0.005, 0.045], [0, -.5, -.9], sigma_g= params['sigma_g']) + weights = gauss_weight( + f_fft3, [0.001, 0.005, 0.045], [0, -0.5, -0.9], sigma_g=params["sigma_g"] + ) return weights -plt.plot(f_fft3, gauss_weight_function(f_fft3, params) ) +plt.plot(f_fft3, gauss_weight_function(f_fft3, params)) -objective_func(params, data_gap, phih_from_LS_gap, gauss_weight_function, nan_mask = nan_mask, plot_flag= True) + +objective_func( + params, + data_gap, + phih_from_LS_gap, + gauss_weight_function, + nan_mask=nan_mask, + plot_flag=True, +) # %% simplest model params = lmfit.Parameters() -params.add('amp', 0, min=-1, max=1) +params.add("amp", 0, min=-1, max=1) + def constant_weight(ff, amp): """ - zdgfsg + zdgfsg """ - y = (ff*0+1) + amp + y = (ff * 0 + 1) + amp return y - def constant_weight_function(ff, params): - return constant_weight(ff, params['amp'].value) + return constant_weight(ff, params["amp"].value) + -objective_func(params, data_gap, phih_from_LS_gap, constant_weight_function, nan_mask = nan_mask, plot_flag= True) +objective_func( + params, + data_gap, + phih_from_LS_gap, + constant_weight_function, + nan_mask=nan_mask, + plot_flag=True, +) # %% -cost_list=list() -amp_list= np.arange(-0.8, 1, 0.05) +cost_list = list() +amp_list = np.arange(-0.8, 1, 0.05) for aa in amp_list: - params['amp'].value = aa - cost= objective_func(params, data_gap, phih_from_LS_gap, constant_weight_function, nan_mask = nan_mask, plot_flag= False) + params["amp"].value = aa + cost = objective_func( + params, + data_gap, + phih_from_LS_gap, + constant_weight_function, + nan_mask=nan_mask, + plot_flag=False, + ) cost_list.append(cost) cost_list = np.array(cost_list) -plt.plot(amp_list, cost_list[:,0], label='fitting cost') -plt.plot(amp_list, cost_list[:,1], label='variance cost') -plt.plot(amp_list, cost_list[:,0]+ cost_list[:,1], label='total cost') +plt.plot(amp_list, cost_list[:, 0], label="fitting cost") +plt.plot(amp_list, cost_list[:, 1], label="variance cost") +plt.plot(amp_list, cost_list[:, 0] + cost_list[:, 1], label="total cost") plt.legend() # %% -#args=(y, model_real_space) -#kws={'nan_mask':nan_mask, 'sigma_g':0.002 } +# args=(y, model_real_space) +# kws={'nan_mask':nan_mask, 'sigma_g':0.002 } -fitter = lmfit.minimize(objective_func, params, args=(data_gap, phih_from_LS_gap, constant_weight_function), - kws={'nan_mask':nan_mask} , method='least_squares') +fitter = lmfit.minimize( + objective_func, + params, + args=(data_gap, phih_from_LS_gap, constant_weight_function), + kws={"nan_mask": nan_mask}, + method="least_squares", +) fitter.params.pretty_print() -objective_func(fitter.params, data_gap, phih_from_LS_gap, constant_weight_function, nan_mask = nan_mask, plot_flag= True) +objective_func( + fitter.params, + data_gap, + phih_from_LS_gap, + constant_weight_function, + nan_mask=nan_mask, + plot_flag=True, +) + # %% -def tanh_weight(x, x_positions, LF_amp, HF_amp, sigma_g): +def tanh_weight(x, x_positions, LF_amp, HF_amp, sigma_g): """ - zdgfsg + zdgfsg """ - HF_amp1 = (LF_amp-HF_amp) - decay = 0.5 - np.tanh( (x-x_positions)/sigma_g )/2 - y = decay * HF_amp1 + (1 - HF_amp1) - y = y- y[0] +LF_amp - #y = y * LF_amp + HF_amp1 = LF_amp - HF_amp + decay = 0.5 - np.tanh((x - x_positions) / sigma_g) / 2 + y = decay * HF_amp1 + (1 - HF_amp1) + y = y - y[0] + LF_amp + # y = y * LF_amp return y -#plt.plot( f_fft3, tanh_weight( f_fft3, 0.025, 1 , 1, 0.005) ) + +# plt.plot( f_fft3, tanh_weight( f_fft3, 0.025, 1 , 1, 0.005) ) + def tanh_weight_function(ff, params): - return tanh_weight(ff, params['x_pos'].value,params['LF_amp'].value,params['HF_amp'].value,params['sigma_g'].value, ) + return tanh_weight( + ff, + params["x_pos"].value, + params["LF_amp"].value, + params["HF_amp"].value, + params["sigma_g"].value, + ) + params = lmfit.Parameters() p_smothed = M.runningmean(np.abs(phih_from_LS_gap), 20, tailcopy=True) f_max = f_fft3[p_smothed[~np.isnan(p_smothed)].argmax()] -params.add('x_pos', f_max, min=f_max*0.75, max=f_max*1.25, vary=True) +params.add("x_pos", f_max, min=f_max * 0.75, max=f_max * 1.25, vary=True) -params.add('LF_amp', 1, min=.2, max=1.5) -params.add('HF_amp', 1, min=.2, max=1.5) -params.add('sigma_g', 0.002, min=0.001, max=0.05, vary= False) +params.add("LF_amp", 1, min=0.2, max=1.5) +params.add("HF_amp", 1, min=0.2, max=1.5) +params.add("sigma_g", 0.002, min=0.001, max=0.05, vary=False) -#tanh_weight_function(f_fft3, params) -objective_func(params, data_gap, phih_from_LS_gap, tanh_weight_function, nan_mask = nan_mask, plot_flag= True) +# tanh_weight_function(f_fft3, params) +objective_func( + params, + data_gap, + phih_from_LS_gap, + tanh_weight_function, + nan_mask=nan_mask, + plot_flag=True, +) # %% -def objective_func(params, data_x, Z_results, weight_func, nan_mask = None, plot_flag=False): + +def objective_func( + params, data_x, Z_results, weight_func, nan_mask=None, plot_flag=False +): # model_x= model_real_space # data= y - alpha =1e7 + alpha = 1e7 + def model_real_space(Z, weights): """ Both inputs must have the same length """ - return np.fft.irfft(Z*weights) + return np.fft.irfft(Z * weights) weights = weight_func(f_fft3, params) @@ -680,74 +841,107 @@ def model_real_space(Z, weights): if plot_flag: - F= M.figure_axis_xy(10, 4.1 * 2.5, view_scale= 0.5, container = True) + F = M.figure_axis_xy(10, 4.1 * 2.5, view_scale=0.5, container=True) - gs = GridSpec(5,1, wspace=0.1, hspace=0.4)#figure=fig, - pos0,pos1,pos2 = gs[0:3, 0],gs[3, 0],gs[4, 0]#,gs[3, 0] + gs = GridSpec(5, 1, wspace=0.1, hspace=0.4) # figure=fig, + pos0, pos1, pos2 = gs[0:3, 0], gs[3, 0], gs[4, 0] # ,gs[3, 0] ax1 = F.fig.add_subplot(pos0) - plt.title('Stacked Timeseries', loc='left') + plt.title("Stacked Timeseries", loc="left") - chunk_l= 400 - chunk_iter = spec.create_chunk_boundaries(chunk_l, data_x.size, ov=0, iter_flag = True) + chunk_l = 400 + chunk_iter = spec.create_chunk_boundaries( + chunk_l, data_x.size, ov=0, iter_flag=True + ) - ofsett0= 6 + ofsett0 = 6 ofsett = np.copy(ofsett0) for chi in chunk_iter: - v1= np.round(np.nanvar(dd), 4) - plt.plot(ofsett+ data_x[chi[0]:chi[-1]] , linewidth=3, alpha=0.5 , c='black', label=' org. data (var:'+str(v1)+')') - - v1= np.round(model_x(Z_results, weights*0 +1)[~nan_mask[1:]].var(), 4) - plt.plot(ofsett + model_x(Z_results, weights*0 +1)[chi[0]:chi[-1]] ,linewidth= 0.8, c='red', label='LS model init (var:'+str(v1)+')') - - v1= np.round(model.var(), 4) - plt.plot(ofsett + model_x(Z_results, weights)[chi[0]:chi[-1]],linewidth= 0.8, c='blue', label='LS model weighted (var:'+str(v1)+')') + v1 = np.round(np.nanvar(dd), 4) + plt.plot( + ofsett + data_x[chi[0] : chi[-1]], + linewidth=3, + alpha=0.5, + c="black", + label=" org. data (var:" + str(v1) + ")", + ) + + v1 = np.round(model_x(Z_results, weights * 0 + 1)[~nan_mask[1:]].var(), 4) + plt.plot( + ofsett + model_x(Z_results, weights * 0 + 1)[chi[0] : chi[-1]], + linewidth=0.8, + c="red", + label="LS model init (var:" + str(v1) + ")", + ) + + v1 = np.round(model.var(), 4) + plt.plot( + ofsett + model_x(Z_results, weights)[chi[0] : chi[-1]], + linewidth=0.8, + c="blue", + label="LS model weighted (var:" + str(v1) + ")", + ) if ofsett == ofsett0: plt.legend() ofsett -= 1 - plt.ylim(ofsett, ofsett0+1) - plt.xlim(0, chunk_l*2) - + plt.ylim(ofsett, ofsett0 + 1) + plt.xlim(0, chunk_l * 2) ax2 = F.fig.add_subplot(pos1) - #ax2 = plt.subplot(3, 1, 2) - plt.title('Amplitude Weight Function', loc='left') - plt.plot(weights , c='black') - ax2.set_xscale('log') + # ax2 = plt.subplot(3, 1, 2) + plt.title("Amplitude Weight Function", loc="left") + plt.plot(weights, c="black") + ax2.set_xscale("log") ax3 = F.fig.add_subplot(pos2) - plt.title('Initial and tuned |Z|', loc='left') + plt.title("Initial and tuned |Z|", loc="left") - #ax3 = plt.subplot(3, 1, 3) + # ax3 = plt.subplot(3, 1, 3) # v2_fft= np.fft.rfft(data_x) # v2 = np.round( (2.*(v2_fft*v2_fft.conj()).real /data_x.size**2 ).sum(), 4) # plt.plot(abs(v2_fft) , linewidth=2, alpha=0.5 , c='black', label='org data (var: '+str(v2) +')') - v2 = np.round( (4.*(Z_results*Z_results.conj()).real /data_x.size**2 ).sum(), 4) - plt.plot(abs(Z_results), linewidth= 0.8, c='red', label='Z (var: '+str(v2) +')') - plt.plot(M.runningmean(abs(Z_results) , 20, tailcopy=True), linewidth= 1.5, c='red', zorder=12) - - Z2= Z_results* weights - v2 = np.round( (4.*(Z2*Z2.conj()).real /data_x.size**2 ).sum(), 4) - plt.plot(abs(Z2), linewidth= 0.8, c='blue', label='weighted Z(var: '+str(v2) +')') - plt.plot(M.runningmean(abs(Z2) , 20, tailcopy=True), linewidth= 1.5, c='blue', zorder=12) + v2 = np.round( + (4.0 * (Z_results * Z_results.conj()).real / data_x.size**2).sum(), 4 + ) + plt.plot( + abs(Z_results), linewidth=0.8, c="red", label="Z (var: " + str(v2) + ")" + ) + plt.plot( + M.runningmean(abs(Z_results), 20, tailcopy=True), + linewidth=1.5, + c="red", + zorder=12, + ) + + Z2 = Z_results * weights + v2 = np.round((4.0 * (Z2 * Z2.conj()).real / data_x.size**2).sum(), 4) + plt.plot( + abs(Z2), linewidth=0.8, c="blue", label="weighted Z(var: " + str(v2) + ")" + ) + plt.plot( + M.runningmean(abs(Z2), 20, tailcopy=True), + linewidth=1.5, + c="blue", + zorder=12, + ) plt.legend() - plt.ylim( np.percentile(abs(np.fft.rfft(data)), 0.5), abs(np.fft.rfft(data)).max()*1.3 ) - plt.xlabel('wavenumber k') - ax3.set_xscale('log') - ax3.set_yscale('log') - - fitting_cost =( abs(dd - model) / dd.std() )**2 - variance_cost =( abs(dd.var() - model.var()) / dd.std() ) **2 - - return fitting_cost.sum() , alpha* variance_cost - + plt.ylim( + np.percentile(abs(np.fft.rfft(data)), 0.5), + abs(np.fft.rfft(data)).max() * 1.3, + ) + plt.xlabel("wavenumber k") + ax3.set_xscale("log") + ax3.set_yscale("log") + fitting_cost = (abs(dd - model) / dd.std()) ** 2 + variance_cost = (abs(dd.var() - model.var()) / dd.std()) ** 2 + return fitting_cost.sum(), alpha * variance_cost # D_data = data_gap *window_single @@ -755,13 +949,13 @@ def model_real_space(Z, weights): # Z_data = np.copy(data_gap *win) # Z_data[nan_mask]=0 # Z_data= np.fft.rfft(Z_data) -#Z_data.shape +# Z_data.shape D_data = data_gap * window_single -Z_data= np.fft.rfft(m1_gap_window1) +Z_data = np.fft.rfft(m1_gap_window1) -D_data = data_gap# * window_single -Z_data= np.fft.rfft(m1_gap) +D_data = data_gap # * window_single +Z_data = np.fft.rfft(m1_gap) # D_data = data_gap# * window_single # Z_data= np.fft.rfft(m1_gap_window1) @@ -773,21 +967,37 @@ def model_real_space(Z, weights): # Z_data.shape - - -fitter = lmfit.minimize(objective_func, params, args=(D_data, Z_data, tanh_weight_function), - kws={'nan_mask':nan_mask} , method='dual_annealing') +fitter = lmfit.minimize( + objective_func, + params, + args=(D_data, Z_data, tanh_weight_function), + kws={"nan_mask": nan_mask}, + method="dual_annealing", +) fitter.params.pretty_print() -objective_func(fitter.params, D_data, Z_data, tanh_weight_function, nan_mask = nan_mask, plot_flag= True) +objective_func( + fitter.params, + D_data, + Z_data, + tanh_weight_function, + nan_mask=nan_mask, + plot_flag=True, +) # v2_fft= np.fft.rfft(data_x) -#v2 = np.round( (2.*(v2_fft*v2_fft.conj()).real /data_x.size**2 ).sum(), 4) +# v2 = np.round( (2.*(v2_fft*v2_fft.conj()).real /data_x.size**2 ).sum(), 4) # plt.plot(abs(v2_fft) , linewidth=2, alpha=0.5 , c='black', label='org data (var: '+str(v2) +')') -#plt.plot(np.abs(phih_from_LS), c='gray', linewidth = 2, alpha=0.5 ) -plt.plot(M.runningmean(np.abs(phih_from_data) , 20, tailcopy=True), linewidth = 1.2, c='black' ,label= 'LS full data', zorder=12) +# plt.plot(np.abs(phih_from_LS), c='gray', linewidth = 2, alpha=0.5 ) +plt.plot( + M.runningmean(np.abs(phih_from_data), 20, tailcopy=True), + linewidth=1.2, + c="black", + label="LS full data", + zorder=12, +) plt.legend() data_gap @@ -804,14 +1014,13 @@ def model_real_space(Z, weights): phih_from_LS - ## %% phih_from_data.shape -plt.plot(y , linewidth=3, alpha=0.5 , c='black') +plt.plot(y, linewidth=3, alpha=0.5, c="black") -plt.plot(np.fft.irfft(phih_from_data), 'b') +plt.plot(np.fft.irfft(phih_from_data), "b") zz = np.copy(phih_from_data) -zz[300:]= 0 +zz[300:] = 0 plt.plot(np.fft.irfft(zz)) -plt.xlim(0,200) +plt.xlim(0, 200) diff --git a/analysis/SB02_make_spectra_LS_adjust_amp_working_exmpl.py b/analysis/SB02_make_spectra_LS_adjust_amp_working_exmpl.py index 76d4b648..338428e6 100644 --- a/analysis/SB02_make_spectra_LS_adjust_amp_working_exmpl.py +++ b/analysis/SB02_make_spectra_LS_adjust_amp_working_exmpl.py @@ -1,23 +1,25 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -37,57 +39,58 @@ import imp import lmfit -#import s3fs + +# import s3fs # %% # Python reader based on Pandas. Other reader examples available in readers.py -plot_path = mconfig['paths']['plot'] + '/tests/' +plot_path = mconfig["paths"]["plot"] + "/tests/" -track_name= 'ATL03_20190515060436_07170312_002_02' -load_path = base_path + 'data/data1/' +track_name = "ATL03_20190515060436_07170312_002_02" +load_path = base_path + "data/data1/" # test which beams exist: -all_beams = ['gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r'] -low_beams = ['gt1l', 'gt2l', 'gt3l'] -high_beams = ['gt1r', 'gt2r', 'gt3r'] +all_beams = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] +low_beams = ["gt1l", "gt2l", "gt3l"] +high_beams = ["gt1r", "gt2r", "gt3r"] # %% -#Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') +# Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') imp.reload(io) -Gfilt = io.load_pandas_table_dict(track_name + '_B01_corrected', load_path) -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) +Gfilt = io.load_pandas_table_dict(track_name + "_B01_corrected", load_path) +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # %% -Gi =Gd[high_beams[0]] -Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') -#np.isnan(Gi['dist']).sum() -Gi = Gi[(Gi['dist'] > 140000) & (Gi['dist'] < 170001)] +Gi = Gd[high_beams[0]] +Gi = Gi[~np.isnan(Gi["heights_c_weighted_mean"])].sort_values("dist") +# np.isnan(Gi['dist']).sum() +Gi = Gi[(Gi["dist"] > 140000) & (Gi["dist"] < 170001)] # %% +# plt.xlim(140000, 145000) +# plt.ylim(-1, 1) - -#plt.xlim(140000, 145000) -#plt.ylim(-1, 1) - -#np.diff(np.array(Gi['dist'])) +# np.diff(np.array(Gi['dist'])) # %% test and compare to FFT from scipy.signal import detrend -y =detrend(np.array(Gi['heights_c_weighted_mean']) ) -data_gap= np.copy(y) +y = detrend(np.array(Gi["heights_c_weighted_mean"])) + +data_gap = np.copy(y) data_gap[500:2000] = np.nan -nan_mask =np.isnan(data_gap) +nan_mask = np.isnan(data_gap) data_filled = np.copy(y) data_filled[nan_mask] = 0 -nan_mask.sum()/data_gap.size +nan_mask.sum() / data_gap.size + +plt.plot(Gi["dist"], y, "-") +plt.plot(Gi["dist"], data_gap, "-") -plt.plot( Gi['dist'], y, '-') -plt.plot( Gi['dist'], data_gap, '-') # %% def create_weighted_window(data, window=None): @@ -99,319 +102,393 @@ def create_weighted_window(data, window=None): L = data.size if window is None: - #win=np.hanning(L) - win = WINDOWS.tukey(L, alpha=0.1, sym=True) + # win=np.hanning(L) + win = WINDOWS.tukey(L, alpha=0.1, sym=True) else: - win=window - + win = window - factor = np.sqrt( np.var(data) / np.var(( data* win) ) ) - #factor=np.sqrt( L/(win**2).sum()) - win *= factor + factor = np.sqrt(np.var(data) / np.var((data * win))) + # factor=np.sqrt( L/(win**2).sum()) + win *= factor return win def create_window_mask(nan_mask, ddata): from scipy.ndimage.measurements import label - nlabel = label(~nan_mask +0) + + nlabel = label(~nan_mask + 0) windows = np.zeros(nan_mask.size) - for i in np.arange(1, nlabel[1]+1): + for i in np.arange(1, nlabel[1] + 1): - sub_mask = nlabel[0] == i - win= create_weighted_window(ddata[sub_mask]) - #win = create_window(sum(sub_mask)) + sub_mask = nlabel[0] == i + win = create_weighted_window(ddata[sub_mask]) + # win = create_window(sum(sub_mask)) windows[sub_mask] = win - return windows#, window_inv + return windows # , window_inv + -#windows = create_window_mask(nan_mask, data_gap ) +# windows = create_window_mask(nan_mask, data_gap ) win = create_weighted_window(data_filled) np.nanvar(data_gap) -np.nanvar(( data_gap* win)[~nan_mask] ) +np.nanvar((data_gap * win)[~nan_mask]) -plt.plot(data_gap* win) +plt.plot(data_gap * win) # %% -f_fft2, df2 = spec.calc_freq_fft( np.array(Gi['dist']), y.size) +f_fft2, df2 = spec.calc_freq_fft(np.array(Gi["dist"]), y.size) Y = spec.calc_spectrum_fft(y, df2, y.size) -Y_win= spec.calc_spectrum_fft(y * win, df2, y.size) +Y_win = spec.calc_spectrum_fft(y * win, df2, y.size) Y_gap_win = spec.calc_spectrum_fft(data_filled * win, df2, y.size) # %% simple spectra from astropy.timeseries import LombScargle -ls_gap_win = LombScargle( np.array(Gi['dist'])[~nan_mask], ( data_gap* win)[~nan_mask] , fit_mean=False) -ls_gap_win_power = ls_gap_win.power(f_fft2 , normalization='psd', assume_regular_frequency= False) -F= M.figure_axis_xy(10, 4, view_scale= 0.8) -plt.subplot(2, 1,1) -#plt.plot(f_fft2[1::] , spec.LS_power_to_PSD(ls_auto_power_windowed, y.size, df2), '^' , label= 'LS single window, no gaps') -plt.plot(f_fft2[:-1], Y_win, '-' , c='k' , label= 'single windowed FFT, no gaps') -plt.plot(f_fft2[:-1], Y_gap_win, '-' , label= 'single windowed FFT, gaps') +ls_gap_win = LombScargle( + np.array(Gi["dist"])[~nan_mask], (data_gap * win)[~nan_mask], fit_mean=False +) +ls_gap_win_power = ls_gap_win.power( + f_fft2, normalization="psd", assume_regular_frequency=False +) -plt.plot(f_fft2 , spec.LS_power_to_PSD(ls_gap_win_power, y.size, df2) , label= 'LS_gap_win') +F = M.figure_axis_xy(10, 4, view_scale=0.8) +plt.subplot(2, 1, 1) +# plt.plot(f_fft2[1::] , spec.LS_power_to_PSD(ls_auto_power_windowed, y.size, df2), '^' , label= 'LS single window, no gaps') +plt.plot(f_fft2[:-1], Y_win, "-", c="k", label="single windowed FFT, no gaps") +plt.plot(f_fft2[:-1], Y_gap_win, "-", label="single windowed FFT, gaps") + +plt.plot( + f_fft2, spec.LS_power_to_PSD(ls_gap_win_power, y.size, df2), label="LS_gap_win" +) plt.legend() plt.xlim(0, 0.01) -plt.subplot(2, 1,2) +plt.subplot(2, 1, 2) -plt.plot(f_fft2[:-1],np.angle(np.fft.rfft(y * win)), c='k' , label= 'windowed standard FFT') -plt.plot(f_fft2[:-1],np.angle(np.fft.rfft(y)), '--', c='k', label= 'standard FFT') +plt.plot( + f_fft2[:-1], np.angle(np.fft.rfft(y * win)), c="k", label="windowed standard FFT" +) +plt.plot(f_fft2[:-1], np.angle(np.fft.rfft(y)), "--", c="k", label="standard FFT") plt.legend() plt.xlim(0, 0.01) - - # %% reconstruct data # test getting phase -t_base =np.array(Gi['dist']) -t_base = t_base# - t_base[0] +t_base = np.array(Gi["dist"]) +t_base = t_base # - t_base[0] -y_model = ls_gap_win.offset() * np.ones(len(t_base)) +y_model = ls_gap_win.offset() * np.ones(len(t_base)) thetas = np.zeros([2]) -for fi in f_fft2[1:]:#f_fft2[1:]: +for fi in f_fft2[1:]: # f_fft2[1:]: theta = ls_gap_win.model_parameters(fi) thetas = np.vstack([thetas, theta]) - y_model += theta[1]* np.cos(t_base * 2 * np.pi *fi) + theta[0] * np.sin(t_base * 2 * np.pi *fi ) + y_model += theta[1] * np.cos(t_base * 2 * np.pi * fi) + theta[0] * np.sin( + t_base * 2 * np.pi * fi + ) -N=y.size +N = y.size # %% rewrite model as FFT Y_from_LS = np.fft.rfft(y_model) # %% test reconstructed model -plt.plot( t_base , data_gap *win , linewidth=2, alpha=0.9 , c='black', label=' data_grap') -plt.plot(t_base, y_model, '-', label='LS model gap') +plt.plot(t_base, data_gap * win, linewidth=2, alpha=0.9, c="black", label=" data_grap") +plt.plot(t_base, y_model, "-", label="LS model gap") -plt.plot(t_base[1:], np.fft.irfft(Y_from_LS), '-', label='inv FFT from fft(LS model)') +plt.plot(t_base[1:], np.fft.irfft(Y_from_LS), "-", label="inv FFT from fft(LS model)") plt.xlim(t_base[0], t_base[700]) plt.legend() - # %% fft again Y_from_data_win = np.fft.rfft(y * win) df3 = np.gradient(f_fft2).mean() # % parcevel test -print('data var:', y.var() ) -print('Data gap var:', data_gap[~nan_mask].var() ) +print("data var:", y.var()) +print("Data gap var:", data_gap[~nan_mask].var()) -print('with window') -print('data * win var:', (y*win).var() ) -spec_data_fft3 = 2.*(Y_from_data_win*Y_from_data_win.conj()).real / df3 /y.size**2 -print('data * win fft sum:', df3 * spec_data_fft3.sum().data ) +print("with window") +print("data * win var:", (y * win).var()) +spec_data_fft3 = 2.0 * (Y_from_data_win * Y_from_data_win.conj()).real / df3 / y.size**2 +print("data * win fft sum:", df3 * spec_data_fft3.sum().data) -print('LS model var:', y_model.var() ) -spec_LS_model_fft3 = 2.*(Y_from_LS*Y_from_LS.conj()).real / df3 /y.size**2 -print('LS model fft sum:', df3 * spec_LS_model_fft3.sum().data ) +print("LS model var:", y_model.var()) +spec_LS_model_fft3 = 2.0 * (Y_from_LS * Y_from_LS.conj()).real / df3 / y.size**2 +print("LS model fft sum:", df3 * spec_LS_model_fft3.sum().data) # %% - - # %% - # %% -def tanh_weight(x, x_cutoff , x_max_pos, LF_amp, HF_amp, Gauss_amp, sigma_g): +def tanh_weight(x, x_cutoff, x_max_pos, LF_amp, HF_amp, Gauss_amp, sigma_g): """ - zdgfsg + zdgfsg """ - HF_amp1 = (LF_amp-HF_amp) - decay = 0.5 - np.tanh( (x-x_cutoff)/sigma_g )/2 - y = decay * HF_amp1 + (1 - HF_amp1) - y = y- y[0] +LF_amp + HF_amp1 = LF_amp - HF_amp + decay = 0.5 - np.tanh((x - x_cutoff) / sigma_g) / 2 + y = decay * HF_amp1 + (1 - HF_amp1) + y = y - y[0] + LF_amp - def gaus(x, x_0, amp, sigma_g ): - return amp* np.exp(-0.5 * ( (x-x_0)/sigma_g)**2) + def gaus(x, x_0, amp, sigma_g): + return amp * np.exp(-0.5 * ((x - x_0) / sigma_g) ** 2) - y += gaus(x, x_max_pos, Gauss_amp, sigma_g ) + y += gaus(x, x_max_pos, Gauss_amp, sigma_g) - #y = y * LF_amp + # y = y * LF_amp return y -plt.plot( f_fft2, tanh_weight( f_fft2, 0.025 + 0.02 , 0.025 , 1 , 0.1, 0.5, 0.003) ) + +plt.plot(f_fft2, tanh_weight(f_fft2, 0.025 + 0.02, 0.025, 1, 0.1, 0.5, 0.003)) # %% # %% -def objective_func(params, data_x, Z_results, weight_func, freq, nan_mask = None, plot_flag=False): + +def objective_func( + params, data_x, Z_results, weight_func, freq, nan_mask=None, plot_flag=False +): # model_x= model_real_space # data= y - alpha =1e7 + alpha = 1e7 + def model_real_space(Z, weights): """ Both inputs must have the same length """ - return np.fft.irfft(Z*weights) + return np.fft.irfft(Z * weights) weights = weight_func(freq[1:], params) if nan_mask is not None: model = model_real_space(Z_results, weights)[~nan_mask[1:]] dd = data_x[~nan_mask][1:] - #print(sum(np.isnan(dd))) + # print(sum(np.isnan(dd))) else: model = model_real_space(Z_results, weights)[:] dd = data_x[1:] if plot_flag: - F= M.figure_axis_xy(10, 4.1 * 2.5, view_scale= 0.5, container = True) + F = M.figure_axis_xy(10, 4.1 * 2.5, view_scale=0.5, container=True) - gs = GridSpec(5,1, wspace=0.1, hspace=0.4)#figure=fig, - pos0,pos1,pos2 = gs[0:3, 0],gs[3, 0],gs[4, 0]#,gs[3, 0] + gs = GridSpec(5, 1, wspace=0.1, hspace=0.4) # figure=fig, + pos0, pos1, pos2 = gs[0:3, 0], gs[3, 0], gs[4, 0] # ,gs[3, 0] ax1 = F.fig.add_subplot(pos0) - plt.title('Stacked Timeseries', loc='left') + plt.title("Stacked Timeseries", loc="left") - chunk_l= 400 - chunk_iter = spec.create_chunk_boundaries(chunk_l, data_x.size, ov=0, iter_flag = True) + chunk_l = 400 + chunk_iter = spec.create_chunk_boundaries( + chunk_l, data_x.size, ov=0, iter_flag=True + ) - ofsett0= 6 + ofsett0 = 6 ofsett = np.copy(ofsett0) for chi in chunk_iter: - v1= np.round(np.nanvar(dd), 4) - plt.plot(ofsett+ data_x[chi[0]:chi[-1]] , linewidth=3, alpha=0.5 , c='black', label=' org. data (var:'+str(v1)+')') - - model_init = model_real_space(Z_results, weights*0 +1)[~nan_mask[1:]] - v1= np.round(model_init.var(), 4) - plt.plot(ofsett + model_real_space(Z_results, weights*0 +1)[chi[0]:chi[-1]] ,linewidth= 0.8, c='red', label='LS model init (var:'+str(v1)+')') - - v1= np.round(model.var(), 4) - plt.plot(ofsett + model_real_space(Z_results, weights)[chi[0]:chi[-1]],linewidth= 0.8, c='blue', label='LS model weighted (var:'+str(v1)+')') + v1 = np.round(np.nanvar(dd), 4) + plt.plot( + ofsett + data_x[chi[0] : chi[-1]], + linewidth=3, + alpha=0.5, + c="black", + label=" org. data (var:" + str(v1) + ")", + ) + + model_init = model_real_space(Z_results, weights * 0 + 1)[~nan_mask[1:]] + v1 = np.round(model_init.var(), 4) + plt.plot( + ofsett + model_real_space(Z_results, weights * 0 + 1)[chi[0] : chi[-1]], + linewidth=0.8, + c="red", + label="LS model init (var:" + str(v1) + ")", + ) + + v1 = np.round(model.var(), 4) + plt.plot( + ofsett + model_real_space(Z_results, weights)[chi[0] : chi[-1]], + linewidth=0.8, + c="blue", + label="LS model weighted (var:" + str(v1) + ")", + ) if ofsett == ofsett0: plt.legend() ofsett -= 1 - plt.ylim(ofsett, ofsett0+1) - plt.xlim(0, chunk_l*2) - + plt.ylim(ofsett, ofsett0 + 1) + plt.xlim(0, chunk_l * 2) ax2 = F.fig.add_subplot(pos1) - #ax2 = plt.subplot(3, 1, 2) - plt.title('Amplitude Weight Function', loc='left') - plt.plot(weights , c='black') - ax2.set_xscale('log') + # ax2 = plt.subplot(3, 1, 2) + plt.title("Amplitude Weight Function", loc="left") + plt.plot(weights, c="black") + ax2.set_xscale("log") ax3 = F.fig.add_subplot(pos2) - plt.title('Initial and tuned |Z|', loc='left') + plt.title("Initial and tuned |Z|", loc="left") - #ax3 = plt.subplot(3, 1, 3) + # ax3 = plt.subplot(3, 1, 3) # v2_fft= np.fft.rfft(data_x) # v2 = np.round( (2.*(v2_fft*v2_fft.conj()).real /data_x.size**2 ).sum(), 4) # plt.plot(abs(v2_fft) , linewidth=2, alpha=0.5 , c='black', label='org data (var: '+str(v2) +')') - v2 = np.round( (4.*(Z_results*Z_results.conj()).real /data_x.size**2 ).sum(), 4) - plt.plot(abs(Z_results), linewidth= 0.8, c='red', label='Z (var: '+str(v2) +')') - plt.plot(M.runningmean(abs(Z_results) , 20, tailcopy=True), linewidth= 1.5, c='red', zorder=12) - - Z2= Z_results* weights - v2 = np.round( (4.*(Z2*Z2.conj()).real /data_x.size**2 ).sum(), 4) - plt.plot(abs(Z2), linewidth= 0.8, c='blue', label='weighted Z(var: '+str(v2) +')') - plt.plot(M.runningmean(abs(Z2) , 20, tailcopy=True), linewidth= 1.5, c='blue', zorder=12) + v2 = np.round( + (4.0 * (Z_results * Z_results.conj()).real / data_x.size**2).sum(), 4 + ) + plt.plot( + abs(Z_results), linewidth=0.8, c="red", label="Z (var: " + str(v2) + ")" + ) + plt.plot( + M.runningmean(abs(Z_results), 20, tailcopy=True), + linewidth=1.5, + c="red", + zorder=12, + ) + + Z2 = Z_results * weights + v2 = np.round((4.0 * (Z2 * Z2.conj()).real / data_x.size**2).sum(), 4) + plt.plot( + abs(Z2), linewidth=0.8, c="blue", label="weighted Z(var: " + str(v2) + ")" + ) + plt.plot( + M.runningmean(abs(Z2), 20, tailcopy=True), + linewidth=1.5, + c="blue", + zorder=12, + ) plt.legend() - plt.ylim( np.percentile(abs(Z_results), 0.5), abs(Z_results).max()*1.3 ) - plt.xlabel('wavenumber k') - ax3.set_xscale('log') - ax3.set_yscale('log') + plt.ylim(np.percentile(abs(Z_results), 0.5), abs(Z_results).max() * 1.3) + plt.xlabel("wavenumber k") + ax3.set_xscale("log") + ax3.set_yscale("log") - fitting_cost =( abs(dd - model) / dd.std() )**2 - variance_cost =( abs(dd.var() - model.var()) / dd.std() ) **2 + fitting_cost = (abs(dd - model) / dd.std()) ** 2 + variance_cost = (abs(dd.var() - model.var()) / dd.std()) ** 2 - return fitting_cost.sum() , alpha* variance_cost + return fitting_cost.sum(), alpha * variance_cost def tanh_weight_function(ff, params): - return tanh_weight(ff, params['x_cutoff'].value, - params['x_max_pos'].value, - params['LF_amp'].value, - params['HF_amp'].value, - params['Gauss_amp'].value, - params['sigma_g'].value ) + return tanh_weight( + ff, + params["x_cutoff"].value, + params["x_max_pos"].value, + params["LF_amp"].value, + params["HF_amp"].value, + params["Gauss_amp"].value, + params["sigma_g"].value, + ) + params = lmfit.Parameters() p_smothed = M.runningmean(np.abs(Y_from_LS), 20, tailcopy=True) f_max = f_fft2[p_smothed[~np.isnan(p_smothed)].argmax()] -lambda_max = 9.81 * 5 **2/ (2* np.pi) -params.add('x_cutoff', 1/lambda_max , min=f_max*0.75, max=f_max*5, vary=False) -params.add('x_max_pos', f_max , min=f_max*0.75, max=f_max*5, vary=False) -params.add('LF_amp', 1 , min=0.5 , max=1.2, vary= True) -params.add('HF_amp', 0.5 , min=0 , max=1.5, vary= True) -params.add('sigma_g', 0.002 , min=0.001 , max=0.05, vary= False) -params.add('Gauss_amp', 0.5 , min=0.01 , max=2, vary= True) +lambda_max = 9.81 * 5**2 / (2 * np.pi) +params.add("x_cutoff", 1 / lambda_max, min=f_max * 0.75, max=f_max * 5, vary=False) +params.add("x_max_pos", f_max, min=f_max * 0.75, max=f_max * 5, vary=False) +params.add("LF_amp", 1, min=0.5, max=1.2, vary=True) +params.add("HF_amp", 0.5, min=0, max=1.5, vary=True) +params.add("sigma_g", 0.002, min=0.001, max=0.05, vary=False) +params.add("Gauss_amp", 0.5, min=0.01, max=2, vary=True) # % -dx = t_base[2]- t_base[1] -stancil=t_base[0], t_base[-1] -x_model = np.arange(stancil[0], stancil[-1] + dx, dx) +dx = t_base[2] - t_base[1] +stancil = t_base[0], t_base[-1] +x_model = np.arange(stancil[0], stancil[-1] + dx, dx) -x_pos = np.round( (t_base[~nan_mask])/ 10.0, 0).astype('int') -x_pos = x_pos- x_pos[0] -#x_pos[x_pos >= t_base.size] = t_base.size-1 +x_pos = np.round((t_base[~nan_mask]) / 10.0, 0).astype("int") +x_pos = x_pos - x_pos[0] +# x_pos[x_pos >= t_base.size] = t_base.size-1 -y_gridded= np.copy(x_model) * 0 -y_gridded[x_pos] = (data_gap*win)[~nan_mask] -y_gridded=y_gridded[:-1] +y_gridded = np.copy(x_model) * 0 +y_gridded[x_pos] = (data_gap * win)[~nan_mask] +y_gridded = y_gridded[:-1] data_gap.shape y_gridded.shape -#tanh_weight_function(f_fft3, params - - -objective_func(params, (data_gap*win), Y_from_LS, tanh_weight_function, f_fft2, nan_mask = nan_mask , plot_flag= True) +# tanh_weight_function(f_fft3, params + + +objective_func( + params, + (data_gap * win), + Y_from_LS, + tanh_weight_function, + f_fft2, + nan_mask=nan_mask, + plot_flag=True, +) # %% params.pretty_print() -D_data = (data_gap * win)#[~nan_mask] -Z_data= Y_from_LS# np.fft.rfft(m1_gap_windowed) +D_data = data_gap * win # [~nan_mask] +Z_data = Y_from_LS # np.fft.rfft(m1_gap_windowed) D_data.shape # Z_mask= Z_data_final <.5 # Z_2 = np.copy(Z_data_final) # Z_2[Z_mask] =0 # Z_data = Z_2 -1/f_fft2[450] +1 / f_fft2[450] f_fft2[450] -1/30 - +1 / 30 -fitter = lmfit.minimize(objective_func, params, args=(D_data, Z_data, tanh_weight_function, f_fft2), - kws={'nan_mask':nan_mask} , method='dual_annealing') +fitter = lmfit.minimize( + objective_func, + params, + args=(D_data, Z_data, tanh_weight_function, f_fft2), + kws={"nan_mask": nan_mask}, + method="dual_annealing", +) fitter.params.pretty_print() -objective_func(fitter.params, D_data, Z_data, tanh_weight_function, f_fft2, nan_mask = nan_mask, plot_flag= True) - -plt.plot(M.runningmean(np.abs(Y_from_data_win) , 20, tailcopy=True), linewidth = 1.2, c='black' ,label= 'LS full data', zorder=12) +objective_func( + fitter.params, + D_data, + Z_data, + tanh_weight_function, + f_fft2, + nan_mask=nan_mask, + plot_flag=True, +) + +plt.plot( + M.runningmean(np.abs(Y_from_data_win), 20, tailcopy=True), + linewidth=1.2, + c="black", + label="LS full data", + zorder=12, +) plt.legend() data_gap @@ -422,22 +499,22 @@ def tanh_weight_function(ff, params): Z_data_final = Z_data * tanh_weight_function(f_fft2[1:], fitter.params) -Z_mask= Z_data_final <3 +Z_mask = Z_data_final < 3 plt.plot(Z_data_final) Z_2 = np.copy(Z_data_final) -Z_2[Z_mask] =0 +Z_2[Z_mask] = 0 # %% -plt.plot( data_gap ) -plt.plot( np.fft.irfft(Z_2), 'k' ) -plt.xlim(0, 10`00) +plt.plot(data_gap) +plt.plot(np.fft.irfft(Z_2), "k") +plt.xlim(0, 1000) # old: plt.xlim(0, 10`00) # %% # %% imp.reload(spec) -P = spec.conserve_variance(Z_data, f_fft2, D_data, nan_mask = nan_mask ) +P = spec.conserve_variance(Z_data, f_fft2, D_data, nan_mask=nan_mask) P.set_parameters() P.params.pretty_print() P.test_ojective_func(P.tanh_weight_function, plot_flag=False) @@ -446,4 +523,4 @@ def tanh_weight_function(ff, params): P.plot_result() -plt.plot(np.fft.ifft(P.best_guess_Z() )) +plt.plot(np.fft.ifft(P.best_guess_Z())) diff --git a/analysis/SB02_make_spectra_linear_inverse.py b/analysis/SB02_make_spectra_linear_inverse.py index 8ca687e8..557e653b 100644 --- a/analysis/SB02_make_spectra_linear_inverse.py +++ b/analysis/SB02_make_spectra_linear_inverse.py @@ -1,23 +1,25 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -37,83 +39,83 @@ import imp import lmfit -#import s3fs + +# import s3fs # %% # Python reader based on Pandas. Other reader examples available in readers.py -plot_path = mconfig['paths']['plot'] + '/tests/' +plot_path = mconfig["paths"]["plot"] + "/tests/" -track_name= 'ATL03_20190515060436_07170312_002_02' -load_path = base_path + 'data/data1/' +track_name = "ATL03_20190515060436_07170312_002_02" +load_path = base_path + "data/data1/" # test which beams exist: -all_beams = ['gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r'] -low_beams = ['gt1l', 'gt2l', 'gt3l'] -high_beams = ['gt1r', 'gt2r', 'gt3r'] +all_beams = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] +low_beams = ["gt1l", "gt2l", "gt3l"] +high_beams = ["gt1r", "gt2r", "gt3r"] # %% -#Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') +# Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') imp.reload(io) -Gfilt = io.load_pandas_table_dict(track_name + '_B01_corrected', load_path) -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) +Gfilt = io.load_pandas_table_dict(track_name + "_B01_corrected", load_path) +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # %% -Gi =Gd[high_beams[0]] -Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') -#np.isnan(Gi['dist']).sum() -Gi = Gi[(Gi['dist'] >= 140000) & (Gi['dist'] <= 180001)] -Gi = Gi[(Gi['dist'] >= 140000) & (Gi['dist'] <= 170001)] +Gi = Gd[high_beams[0]] +Gi = Gi[~np.isnan(Gi["heights_c_weighted_mean"])].sort_values("dist") +# np.isnan(Gi['dist']).sum() +Gi = Gi[(Gi["dist"] >= 140000) & (Gi["dist"] <= 180001)] +Gi = Gi[(Gi["dist"] >= 140000) & (Gi["dist"] <= 170001)] # %% +# plt.xlim(140000, 145000) +# plt.ylim(-1, 1) - -#plt.xlim(140000, 145000) -#plt.ylim(-1, 1) - -#np.diff(np.array(Gi['dist'])) +# np.diff(np.array(Gi['dist'])) # %% test and compare to FFT from scipy.signal import detrend -y =detrend(np.array(Gi['heights_c_weighted_mean']) ) -y_gap= np.copy(y) +y = detrend(np.array(Gi["heights_c_weighted_mean"])) + +y_gap = np.copy(y) y_gap[200:2800] = np.nan -nan_mask =np.isnan(y_gap) +nan_mask = np.isnan(y_gap) data_filled = np.copy(y) data_filled[nan_mask] = 0 -nan_mask.sum()/y_gap.size +nan_mask.sum() / y_gap.size -plt.plot( Gi['dist'], y, '-') -plt.plot( Gi['dist'], y_gap, '-') +plt.plot(Gi["dist"], y, "-") +plt.plot(Gi["dist"], y_gap, "-") # %% -x= Gi['dist'] - Gi['dist'].iloc[0] +x = Gi["dist"] - Gi["dist"].iloc[0] dx = np.diff(x).mean() -plt.plot( x, y_gap) +plt.plot(x, y_gap) -k_full = np.copy(np.fft.rfftfreq(x.size, d=dx) * 2* np.pi) -#k = k_full[1:201] -klim= 301# 301 +k_full = np.copy(np.fft.rfftfreq(x.size, d=dx) * 2 * np.pi) +# k = k_full[1:201] +klim = 301 # 301 k = k_full[1:klim] MM = k.size NN = x.size -2*np.pi/k[-1] +2 * np.pi / k[-1] x_gap = x[~nan_mask] -#plt.plot( x_gap , np.cos(np.outer(x_gap, k[0:10])), '.' ) +# plt.plot( x_gap , np.cos(np.outer(x_gap, k[0:10])), '.' ) -G = np.vstack([ np.cos(np.outer(x_gap, k)).T , np.sin(np.outer(x_gap, k)).T ] ).T -G_full = np.vstack([ np.cos(np.outer(x, k)).T , np.sin(np.outer(x, k)).T ] ).T +G = np.vstack([np.cos(np.outer(x_gap, k)).T, np.sin(np.outer(x_gap, k)).T]).T +G_full = np.vstack([np.cos(np.outer(x, k)).T, np.sin(np.outer(x, k)).T]).T G.shape -b = np.zeros([2 *MM]) +b = np.zeros([2 * MM]) b[4] = 1 b[300] = 1 @@ -121,50 +123,53 @@ G.shape y_gap.shape -plt.plot( (G *b).sum(1) ) +plt.plot((G * b).sum(1)) # Model Penalty # regullization . 1/weight # choose simple decay along diagonal import ICEsat2_SI_tools.generalized_FT as gFT + imp.reload(gFT) -k_fft = np.fft.rfftfreq(x.size, d=dx) * 2* np.pi -f_weight= np.sqrt(9.81 * k_fft) / (2 *np.pi) -data_weight = spec.Z_to_power(np.fft.rfft(data_filled), np.diff(f_weight).mean(), x.size) +k_fft = np.fft.rfftfreq(x.size, d=dx) * 2 * np.pi +f_weight = np.sqrt(9.81 * k_fft) / (2 * np.pi) +data_weight = spec.Z_to_power( + np.fft.rfft(data_filled), np.diff(f_weight).mean(), x.size +) -S = gFT.get_prior_spec(f_weight, data_weight ) +S = gFT.get_prior_spec(f_weight, data_weight) pars = S.set_parameters() -f= np.sqrt(9.81 * k) / (2 *np.pi) -weight = S.create_weight(freq = f, plot_flag= False) -weight = weight + weight.max()* 0.1 # add pemnalty floor -#weight[weight< 0] =0.001 +f = np.sqrt(9.81 * k) / (2 * np.pi) +weight = S.create_weight(freq=f, plot_flag=False) +weight = weight + weight.max() * 0.1 # add pemnalty floor +# weight[weight< 0] =0.001 plt.plot(k, weight) -#weights = np.concatenate([ weight , weight ]) -#weights = np.concatenate([ weight , weight ]) +# weights = np.concatenate([ weight , weight ]) +# weights = np.concatenate([ weight , weight ]) -#plt.plot( np.arange(MM, 0, -1)/MM ) -#weights = np.concatenate([ np.arange(MM, 0, -1)/MM , np.arange(MM, 0, -1)/MM ]) -weights = np.concatenate([ np.arange(MM, 0, -1)/MM , np.arange(MM, 0, -1)/MM ])# *0+1 -weights= weights+ 0.1 +# plt.plot( np.arange(MM, 0, -1)/MM ) +# weights = np.concatenate([ np.arange(MM, 0, -1)/MM , np.arange(MM, 0, -1)/MM ]) +weights = np.concatenate([np.arange(MM, 0, -1) / MM, np.arange(MM, 0, -1) / MM]) # *0+1 +weights = weights + 0.1 plt.plot(weights) -#weights = np.concatenate([ np.exp(-4 *np.arange(0, MM)/MM), np.exp(-4 *np.arange(0, MM)/MM) ]) +# weights = np.concatenate([ np.exp(-4 *np.arange(0, MM)/MM), np.exp(-4 *np.arange(0, MM)/MM) ]) P = np.diag(weights) P.shape - # uncertainty of the data -uncertaintiy = np.array(Gi['heights_c_std'])[~nan_mask] -#uncertaintiy = np.array(Gi['heights_c_std']) -R = np.diag( uncertaintiy ) +uncertaintiy = np.array(Gi["heights_c_std"])[~nan_mask] +# uncertaintiy = np.array(Gi['heights_c_std']) +R = np.diag(uncertaintiy) R.shape from numpy import linalg + # R = dvar is either a scalar or a vector n_data long # estimate is: # phat = (G'*R^-1*G + P^-1)^-1 * G'*R^-1*y < ------------------------------- @@ -173,74 +178,70 @@ inv(R).shape G.T.shape -#inv( (G.T @ inv(R) @ G ) + inv(P) ).shape -b_hat = inv( (G.T @ inv(R) @ G ) + inv(P) ) @ G.T @ inv(R) @ y[~nan_mask] +# inv( (G.T @ inv(R) @ G ) + inv(P) ).shape +b_hat = inv((G.T @ inv(R) @ G) + inv(P)) @ G.T @ inv(R) @ y[~nan_mask] plt.plot(b_hat) # %% -M.figure_axis_xy(10,5, view_scale =0.6) +M.figure_axis_xy(10, 5, view_scale=0.6) -plt.subplot(2,1, 1) +plt.subplot(2, 1, 1) -#plt.plot( x_gap, y_gap[~nan_mask], '-k', linewidth= 2) -plt.plot( x, y, '-k', linewidth= 2) +# plt.plot( x_gap, y_gap[~nan_mask], '-k', linewidth= 2) +plt.plot(x, y, "-k", linewidth=2) -#plt.plot( x , (b_hat * G).sum(1), 'r-', markersize= 1 ) -plt.plot( x[~nan_mask] , (b_hat * G).sum(1), 'r-', markersize= 1 ) -plt.plot( x[nan_mask] , (b_hat * G_full).sum(1)[nan_mask], '-', color='orange', linewidth = 0.5 ) -#plt.xlim(1000,8000) +# plt.plot( x , (b_hat * G).sum(1), 'r-', markersize= 1 ) +plt.plot(x[~nan_mask], (b_hat * G).sum(1), "r-", markersize=1) +plt.plot( + x[nan_mask], (b_hat * G_full).sum(1)[nan_mask], "-", color="orange", linewidth=0.5 +) +# plt.xlim(1000,8000) plt.ylim(-1, 1) -plt.subplot(2,1, 2) +plt.subplot(2, 1, 2) -plt.plot( x[~nan_mask], (y[~nan_mask] - (b_hat * G).sum(1)) , '.k', markersize= 0.8 ) -#plt.plot( x, y_gap - (b_hat * G).sum(1) , '.k', markersize= 0.8 ) +plt.plot(x[~nan_mask], (y[~nan_mask] - (b_hat * G).sum(1)), ".k", markersize=0.8) +# plt.plot( x, y_gap - (b_hat * G).sum(1) , '.k', markersize= 0.8 ) # rr= y - (b_hat * G).sum(1) # y.var() - (b_hat * G).sum(1).var() - rr.var() - -rr= y[~nan_mask] - (b_hat * G).sum(1) +rr = y[~nan_mask] - (b_hat * G).sum(1) y[~nan_mask].var() - (b_hat * G).sum(1).var() - rr.var() # rr= y - (b_hat * G).sum(1) # y.var() - (b_hat * G).sum(1).var() - rr.var() -rr.var()/uncertaintiy.var() +rr.var() / uncertaintiy.var() -plt.xlim(1000,8000) +plt.xlim(1000, 8000) plt.ylim(-1, 1) - # %% - - # %% k.shape sum1 = (b_hat * G).sum(0) R = sum1[0:MM] I = sum1[MM:] -C =(G**2)[:, 0:MM].sum(0) -S =(G**2)[:, MM:].sum(0) -ampl_spec = np.sqrt( 16* MM * (R**2/ C + I**2/ S) ) -plt.plot( k, ampl_spec) +C = (G**2)[:, 0:MM].sum(0) +S = (G**2)[:, MM:].sum(0) +ampl_spec = np.sqrt(16 * MM * (R**2 / C + I**2 / S)) +plt.plot(k, ampl_spec) -power_spec = ( MM * (R**2/ C + I**2/ S) ) * 2* np.pi / y_gap[~nan_mask].std()/4 +power_spec = (MM * (R**2 / C + I**2 / S)) * 2 * np.pi / y_gap[~nan_mask].std() / 4 power_spec.sum() - - dk = np.diff(k).mean() -1/dk +1 / dk dx @@ -250,29 +251,29 @@ plt.plot(Z.real) plt.plot(Z2_real) -#plt.ylim(0,0.06) +# plt.ylim(0,0.06) # %% Z_fft = np.fft.rfft(y) -Z = b_hat[0:MM] - b_hat[MM:] *1j # this is per size(k) +Z = b_hat[0:MM] - b_hat[MM:] * 1j # this is per size(k) x.shape - #%% +# %% plt.plot(k, power_spec) ############################################ # %% -Z = b_hat[0:MM] - b_hat[MM:] *1j # this is per size(k) +Z = b_hat[0:MM] - b_hat[MM:] * 1j # this is per size(k) (b_hat * G).sum(1).var() (b_hat * G).sum(1).var() / y[~nan_mask].var() (b_hat * G_full).sum(1).var() -#y.var() - rr.var() +# y.var() - rr.var() k.shape k_full.size x.size @@ -282,22 +283,22 @@ k.shape # for non gappy data -#Z_power = spec.Z_to_power(Z * k_full.size, dk, x.size) -k.size/k_full.size -k_full.size/k.size +# Z_power = spec.Z_to_power(Z * k_full.size, dk, x.size) +k.size / k_full.size +k_full.size / k.size -x_gap.size/x.size +x_gap.size / x.size -datat_point_frac= x_gap.size/x.size +datat_point_frac = x_gap.size / x.size -1/datat_point_frac -#1/np.sqrt(datat_point_frac) +1 / datat_point_frac +# 1/np.sqrt(datat_point_frac) -#Z_power = spec.Z_to_power(Z * k_full.size* datat_point_frac , dk, x_gap.size) -#Z_power = spec.Z_to_power(Z * k_full.size , dk, x.size) #/datat_point_frac -#Z_power = spec.Z_to_power(Z * k_full.size , dk, x.size) #/datat_point_frac -Z_power = spec.Z_to_power(Z * (x.size/2+1) , dk, x.size) #/datat_point_frac -#Z_power = spec.Z_to_power(Z * k.size , dk, x_gap.size) #/datat_point_frac +# Z_power = spec.Z_to_power(Z * k_full.size* datat_point_frac , dk, x_gap.size) +# Z_power = spec.Z_to_power(Z * k_full.size , dk, x.size) #/datat_point_frac +# Z_power = spec.Z_to_power(Z * k_full.size , dk, x.size) #/datat_point_frac +Z_power = spec.Z_to_power(Z * (x.size / 2 + 1), dk, x.size) # /datat_point_frac +# Z_power = spec.Z_to_power(Z * k.size , dk, x_gap.size) #/datat_point_frac Z_power.shape Z_power.sum() * dk @@ -306,25 +307,29 @@ # Z_power2 = spec.Z_to_power(Z * (x.size/2 +1) , dk, x_gap.size) # Z_power2.sum() * dk * datat_point_frac -Z_power2 = (Z*Z.conj()).real * x.size / 2 / x_gap.size +Z_power2 = (Z * Z.conj()).real * x.size / 2 / x_gap.size Z_power2.sum() Z_power2.sum() / (y[~nan_mask].var() - rr.var()) -Z_power2 = (Z*Z.conj()).real * x.size / 2 / x_gap.size /dk -Z_power2.sum() *dk -Z_power2.sum() *dk / (y[~nan_mask].var() - rr.var()) -#np.sqrt(2) -#Z_power.sum() * dk / datat_point_frac +Z_power2 = (Z * Z.conj()).real * x.size / 2 / x_gap.size / dk +Z_power2.sum() * dk +Z_power2.sum() * dk / (y[~nan_mask].var() - rr.var()) +# np.sqrt(2) +# Z_power.sum() * dk / datat_point_frac -Z_power2 = ((Z * (x.size/2+1))*(Z * (x.size/2+1)).conj()).real * 2 / ( x.size * x_gap.size * dk) -Z_power2.sum() *dk -Z_power2.sum() *dk / (y[~nan_mask].var() - rr.var()) +Z_power2 = ( + ((Z * (x.size / 2 + 1)) * (Z * (x.size / 2 + 1)).conj()).real + * 2 + / (x.size * x_gap.size * dk) +) +Z_power2.sum() * dk +Z_power2.sum() * dk / (y[~nan_mask].var() - rr.var()) -1/dk/x.size -#plt.plot(k, 4*abs(Z* k_full.size)/dk/x_gap.size**2, 'b') -plt.plot(k, Z_power, 'k*') +1 / dk / x.size +# plt.plot(k, 4*abs(Z* k_full.size)/dk/x_gap.size**2, 'b') +plt.plot(k, Z_power, "k*") # # plt.plot(Z_power, abs(Z)) @@ -337,21 +342,21 @@ # tpi= 2*np.pi # -Z_b_hat_full = np.fft.rfft( (b_hat * G_full).sum(1) ) -Z_b_hat_full_power = spec.Z_to_power(Z_b_hat_full , dk, x.size) +Z_b_hat_full = np.fft.rfft((b_hat * G_full).sum(1)) +Z_b_hat_full_power = spec.Z_to_power(Z_b_hat_full, dk, x.size) Z_b_hat_full_power -plt.plot(k, Z_b_hat_full_power[1:klim], '-g') +plt.plot(k, Z_b_hat_full_power[1:klim], "-g") -Z_b_hat_full_power[1:klim].sum() *dk +Z_b_hat_full_power[1:klim].sum() * dk Z_fft = np.fft.rfft(data_filled) # #Z_fft = np.fft.rfft(y) Z_fft_power = spec.Z_to_power(Z_fft, dk, x.size) -plt.plot(k, Z_fft_power[1:klim], 'r') -#Z_fft_power[1:klim].sum() *dk/tpi -Z_fft_power[1:klim].sum() *dk -#Z_fft_power.sum() *dk +plt.plot(k, Z_fft_power[1:klim], "r") +# Z_fft_power[1:klim].sum() *dk/tpi +Z_fft_power[1:klim].sum() * dk +# Z_fft_power.sum() *dk ################################################ @@ -369,13 +374,13 @@ def complex_represenation(b_hat, M, N_x_full): this returns a power spectral density with the same variance as the data without gaps. """ - Z = b_hat[0:M] - b_hat[M:] *1j - Z = Z * (N_x_full/2+1) # this + Z = b_hat[0:M] - b_hat[M:] * 1j + Z = Z * (N_x_full / 2 + 1) # this return Z -def Z_to_power_gFT(Z, dk, N_x, N_x_full): - """ compute the 1d Power spectral density of a field Z +def Z_to_power_gFT(Z, dk, N_x, N_x_full): + """compute the 1d Power spectral density of a field Z inputs: Z complex fourier coefficients, output of .complex_represenation method dk delta wavenumber asssuming Z is on regular grid @@ -387,25 +392,23 @@ def Z_to_power_gFT(Z, dk, N_x, N_x_full): spec_complete spectal density representing the (moddeled) complete data ( [b_hat]^2 / dk) """ - spec = 2.*(Z*Z.conj()).real - neven = True if (N_x_full%2) else False + spec = 2.0 * (Z * Z.conj()).real + neven = True if (N_x_full % 2) else False # the zeroth frequency should be counted only once - spec[0] = spec[0]/2. + spec[0] = spec[0] / 2.0 if neven: - spec[-1] = spec[-1]/2. + spec[-1] = spec[-1] / 2.0 # spectral density respesenting the incomplete data ( [b_hat]^2 / dk) spec_incomplete = spec / dk / N_x / N_x_full # spectal density representing the (moddeled) complete data ( [b_hat]^2 / dk) spec_complete = spec / dk / N_x_full**2 - return spec_incomplete, spec_complete - -def power_from_model(b_hat, dk, M, N_x, N_x_full): - """ compute the 1d Power spectral density from the model coefficients in b_hat +def power_from_model(b_hat, dk, M, N_x, N_x_full): + """compute the 1d Power spectral density from the model coefficients in b_hat b_hat is the model coefficient matrix M size of the model vector, size of k @@ -417,13 +420,13 @@ def power_from_model(b_hat, dk, M, N_x, N_x_full): spectral density respesenting the incomplete data ( [b_hat]^2 / dk) """ - Z = b_hat[0:M] - b_hat[M:] *1j - spec = (Z*Z.conj()).real * N_x_full / 2 / N_x /dk + Z = b_hat[0:M] - b_hat[M:] * 1j + spec = (Z * Z.conj()).real * N_x_full / 2 / N_x / dk - neven = True if (N_x_full%2) else False - spec[0] = spec[0]/2. + neven = True if (N_x_full % 2) else False + spec[0] = spec[0] / 2.0 if neven: - spec[-1] = spec[-1]/2. + spec[-1] = spec[-1] / 2.0 # spectral density respesenting the incomplete data return spec @@ -434,36 +437,36 @@ def power_from_model(b_hat, dk, M, N_x, N_x_full): spec_c.sum() * dk (b_hat * G_full).sum(1).var() -spec_in.sum() *dk -power_from_model(b_hat, dk, MM, x_gap.size, x.size).sum() *dk +spec_in.sum() * dk +power_from_model(b_hat, dk, MM, x_gap.size, x.size).sum() * dk (b_hat * G).sum(1).var() # %% -plt.plot(k, abs(Z) ) -plt.plot( k, ampl_spec , 'k') -#np.median( abs(Z_fft) / (abs(Z)*MM) ) +plt.plot(k, abs(Z)) +plt.plot(k, ampl_spec, "k") +# np.median( abs(Z_fft) / (abs(Z)*MM) ) # %% compate phase -plt.plot(Z_fft.real[1:klim]/ k_full.size, Z.real, '.') -plt.axis('equal') +plt.plot(Z_fft.real[1:klim] / k_full.size, Z.real, ".") +plt.axis("equal") -(Z_fft.real[1:klim]/ k_full.size - (Z.real)).mean() -Z_fft.imag[1:klim].sum()/ k_full.size +(Z_fft.real[1:klim] / k_full.size - (Z.real)).mean() +Z_fft.imag[1:klim].sum() / k_full.size Z.imag.sum() -Z_fft.real[1:klim].sum()/ k_full.size +Z_fft.real[1:klim].sum() / k_full.size Z.real.sum() -( (Z_fft.imag[1:klim]/ k_full.size) - (Z.imag) ).mean() -#plt.plot( (Z_fft[1:klim].imag) - (Z.imag*k_full.size)) -plt.plot( (Z_fft[1:klim].imag) ) -plt.plot( (Z.imag*k_full.size)) +((Z_fft.imag[1:klim] / k_full.size) - (Z.imag)).mean() +# plt.plot( (Z_fft[1:klim].imag) - (Z.imag*k_full.size)) +plt.plot((Z_fft[1:klim].imag)) +plt.plot((Z.imag * k_full.size)) -plt.plot( (Z_fft[1:klim].real) ) -plt.plot( (Z.real*k_full.size)) +plt.plot((Z_fft[1:klim].real)) +plt.plot((Z.real * k_full.size)) -plt.plot( (Z_fft[1:klim].imag) , (Z.imag*k_full.size), '.') +plt.plot((Z_fft[1:klim].imag), (Z.imag * k_full.size), ".") # plt.plot( (Z_fft.real) - (Z.real*MM)) -#plt.plot( ( (Z_fft.imag) - (Z.imag*MM) ) - ( (Z_fft.real) - (Z.real*MM) ) ) +# plt.plot( ( (Z_fft.imag) - (Z.imag*MM) ) - ( (Z_fft.real) - (Z.real*MM) ) ) diff --git a/analysis/SB02_make_spectra_linear_inverse_bestfit.py b/analysis/SB02_make_spectra_linear_inverse_bestfit.py index 98e0eb3f..ef014767 100644 --- a/analysis/SB02_make_spectra_linear_inverse_bestfit.py +++ b/analysis/SB02_make_spectra_linear_inverse_bestfit.py @@ -1,24 +1,26 @@ # %% import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -#%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -40,105 +42,108 @@ import lmfit import generalized_FT as gFT -#import s3fs + +# import s3fs # %% # Python reader based on Pandas. Other reader examples available in readers.py -plot_path = mconfig['paths']['plot'] + '/tests/' +plot_path = mconfig["paths"]["plot"] + "/tests/" -track_name= 'ATL03_20190515060436_07170312_002_02' -load_path = base_path + 'data/data1/' +track_name = "ATL03_20190515060436_07170312_002_02" +load_path = base_path + "data/data1/" # test which beams exist: -all_beams = ['gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r'] -low_beams = ['gt1l', 'gt2l', 'gt3l'] -high_beams = ['gt1r', 'gt2r', 'gt3r'] +all_beams = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] +low_beams = ["gt1l", "gt2l", "gt3l"] +high_beams = ["gt1r", "gt2r", "gt3r"] # %% -#Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') +# Gall= xr.open_dataset(load_path+'/'+track_name +'_filtered_photon_heights.nc') imp.reload(io) -Gfilt = io.load_pandas_table_dict(track_name + '_B01_corrected', load_path) -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) +Gfilt = io.load_pandas_table_dict(track_name + "_B01_corrected", load_path) +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # %% -Gi =Gd[high_beams[0]] -Gi = Gi[~np.isnan(Gi['heights_c_weighted_mean'])].sort_values('dist') -#np.isnan(Gi['dist']).sum() -Gi = Gi[(Gi['dist'] >= 140000) & (Gi['dist'] <= 170001)][0:2500] +Gi = Gd[high_beams[0]] +Gi = Gi[~np.isnan(Gi["heights_c_weighted_mean"])].sort_values("dist") +# np.isnan(Gi['dist']).sum() +Gi = Gi[(Gi["dist"] >= 140000) & (Gi["dist"] <= 170001)][0:2500] # %% test and compare to FFT from scipy.signal import detrend -y =detrend(np.array(Gi['heights_c_weighted_mean']) ) + +y = detrend(np.array(Gi["heights_c_weighted_mean"])) y.shape -y_gap= np.copy(y) +y_gap = np.copy(y) y_gap[200:1000] = np.nan -nan_mask =np.isnan(y_gap) +nan_mask = np.isnan(y_gap) data_filled = np.copy(y) data_filled[nan_mask] = 0 -nan_mask.sum()/y_gap.size +nan_mask.sum() / y_gap.size -plt.plot( Gi['dist'], y, '-') -plt.plot( Gi['dist'], y_gap, '-') +plt.plot(Gi["dist"], y, "-") +plt.plot(Gi["dist"], y_gap, "-") # %% -x= Gi['dist'] - Gi['dist'].iloc[0] +x = Gi["dist"] - Gi["dist"].iloc[0] dx = np.median(np.diff(x)) -plt.plot( x, y_gap) +plt.plot(x, y_gap) -y_gap_fill=np.copy(y_gap) -y_gap_fill[nan_mask]= 0 +y_gap_fill = np.copy(y_gap) +y_gap_fill[nan_mask] = 0 # %% import ICEsat2_SI_tools.generalized_FT as gFT + imp.reload(gFT) -#plt.plot(data) +# plt.plot(data) # %% # choose wavenumber range -T_max = 40 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 -x= np.array(Gi['dist']) +T_max = 40 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 +x = np.array(Gi["dist"]) dx = np.diff(x).mean() -min_datapoint = 2*np.pi/k_0/dx +min_datapoint = 2 * np.pi / k_0 / dx -Lpoints = int(np.round(min_datapoint) * 10 ) -Lmeters =Lpoints * dx +Lpoints = int(np.round(min_datapoint) * 10) +Lmeters = Lpoints * dx -#plt.plot(np.diff(np.array(Gi['dist']))) -print('L number of gridpoint:', Lpoints) +# plt.plot(np.diff(np.array(Gi['dist']))) +print("L number of gridpoint:", Lpoints) -print('L length in km:', Lmeters/1e3) -print('approx number windows', 2* Gi['dist'].iloc[-1] /Lmeters-1 ) +print("L length in km:", Lmeters / 1e3) +print("approx number windows", 2 * Gi["dist"].iloc[-1] / Lmeters - 1) -T_min=5 -lambda_min = 9.81 * T_min**2/ (2 *np.pi) -flim = 1/T_min +T_min = 5 +lambda_min = 9.81 * T_min**2 / (2 * np.pi) +flim = 1 / T_min oversample = 2 -dlambda = (Lpoints *dx ) * oversample -dk = 2 * np.pi/ dlambda -kk =np.arange(0, 1/lambda_min, 1/dlambda) * 2*np.pi -#kk.shape -#kk = kk[k_0<=kk] +dlambda = (Lpoints * dx) * oversample +dk = 2 * np.pi / dlambda +kk = np.arange(0, 1 / lambda_min, 1 / dlambda) * 2 * np.pi +# kk.shape +# kk = kk[k_0<=kk] k = kk[1:] kk.shape dk = np.diff(kk).mean() -print('2 M = ', kk.size *2 ) +print("2 M = ", kk.size * 2) k = kk # %% -#k_full = np.copy(np.fft.rfftfreq(x.size, d=dx) * 2* np.pi) -#1/np.diff(k_full/2/np.pi).mean() -#k = k_full[1:201] +# k_full = np.copy(np.fft.rfftfreq(x.size, d=dx) * 2* np.pi) +# 1/np.diff(k_full/2/np.pi).mean() +# k = k_full[1:201] x_gap = x[~nan_mask] @@ -149,11 +154,11 @@ NN = x_gap.size data_var = y_gap[~nan_mask].var() -print('2 MM= ', 2*MM, 'NN=', NN) +print("2 MM= ", 2 * MM, "NN=", NN) -#plt.plot( x_gap , np.cos(np.outer(x_gap, k[0:10])), '.' ) -G = np.vstack([ np.cos(np.outer(x_gap, k)).T , np.sin(np.outer(x_gap, k)).T ] ).T -G_full = np.vstack([ np.cos(np.outer(x, k)).T , np.sin(np.outer(x, k)).T ] ).T +# plt.plot( x_gap , np.cos(np.outer(x_gap, k[0:10])), '.' ) +G = np.vstack([np.cos(np.outer(x_gap, k)).T, np.sin(np.outer(x_gap, k)).T]).T +G_full = np.vstack([np.cos(np.outer(x, k)).T, np.sin(np.outer(x, k)).T]).T G.shape k.size @@ -162,69 +167,70 @@ imp.reload(gFT) -k_fft = np.fft.rfftfreq(x.size, d=dx) * 2* np.pi -f_weight= np.sqrt(9.81 * k_fft) / (2 *np.pi) +k_fft = np.fft.rfftfreq(x.size, d=dx) * 2 * np.pi +f_weight = np.sqrt(9.81 * k_fft) / (2 * np.pi) data_weight = spec.Z_to_power(np.fft.rfft(y_gap_fill), np.diff(f_weight).mean(), x.size) imp.reload(gFT) -S = gFT.get_prior_spec(f_weight, data_weight ) +S = gFT.get_prior_spec(f_weight, data_weight) pars = S.set_parameters() -S.params['gamma'].set(value =1 ,vary= False) -f= np.sqrt(9.81 * k) / (2 *np.pi) +S.params["gamma"].set(value=1, vary=False) +f = np.sqrt(9.81 * k) / (2 * np.pi) -weight = S.create_weight(freq = f, plot_flag= True) -weight = weight + weight.max()* 0.05 # add pemnalty floor -weight = weight/weight.max() # add pemnalty floor -weight = weight *data_var +weight = S.create_weight(freq=f, plot_flag=True) +weight = weight + weight.max() * 0.05 # add pemnalty floor +weight = weight / weight.max() # add pemnalty floor +weight = weight * data_var -plt.title('FFT(data) and fitted Pior model') +plt.title("FFT(data) and fitted Pior model") plt.show() -#weight[weight< 0] =0.001 -plt.plot([kk[0], kk[0]], [0, 1/weight.min()], 'k') -plt.plot([kk[-1], kk[-1]], [0, 1/weight.min()], 'k') +# weight[weight< 0] =0.001 +plt.plot([kk[0], kk[0]], [0, 1 / weight.min()], "k") +plt.plot([kk[-1], kk[-1]], [0, 1 / weight.min()], "k") -plt.plot(kk, 1/weight) -#plt.plot(kk, weight) -plt.title('penalty') +plt.plot(kk, 1 / weight) +# plt.plot(kk, weight) +plt.title("penalty") -plt.plot(f, (1/f)**(1/4)) +plt.plot(f, (1 / f) ** (1 / 4)) -#next(iter(S.params.items()))[1].value +# next(iter(S.params.items()))[1].value -#plt.plot( np.arange(MM, 0, -1)/MM ) -#weights = np.concatenate([ np.arange(MM, 0, -1)/MM , np.arange(MM, 0, -1)/MM ]) -weights = np.concatenate([ weight , weight ]) +# plt.plot( np.arange(MM, 0, -1)/MM ) +# weights = np.concatenate([ np.arange(MM, 0, -1)/MM , np.arange(MM, 0, -1)/MM ]) +weights = np.concatenate([weight, weight]) weights.shape # weights = np.concatenate([ np.arange(MM, 0, -1)/MM , np.arange(MM, 0, -1)/MM ]) *0+1 # weights.shape -#weights = np.concatenate([ np.exp(-4 *np.arange(0, MM)/MM), np.exp(-4 *np.arange(0, MM)/MM) ]) +# weights = np.concatenate([ np.exp(-4 *np.arange(0, MM)/MM), np.exp(-4 *np.arange(0, MM)/MM) ]) P = np.diag(weights) -P_1d = weights -#plt.plot(1/weights) +P_1d = weights +# plt.plot(1/weights) P.shape # uncertainty of the data -data_uncertainty = np.array(Gi['heights_c_std'])[~nan_mask] +data_uncertainty = np.array(Gi["heights_c_std"])[~nan_mask] noise_amp = 0.1 -R = np.diag( noise_amp * data_uncertainty/data_uncertainty.std() ) +R = np.diag(noise_amp * data_uncertainty / data_uncertainty.std()) R_1d = np.diag(R) -#np.array(Gi['heights_c_std'])[~nan_mask].std() +# np.array(Gi['heights_c_std'])[~nan_mask].std() from numpy import linalg + inv = linalg.inv -plt.plot( - 1/np.diag(P) ) +plt.plot(-1 / np.diag(P)) # Hessian_int # %timeit Hess =(G.T @ inv(R) @ G ) + inv(P) # %timeit b_hat = inv( Hess) @ G.T @ inv(R) @ y[~nan_mask] -Hess =(G.T @ inv(R) @ G ) + inv(P) -b_hat = inv( Hess) @ G.T @ inv(R) @ y[~nan_mask] +Hess = (G.T @ inv(R) @ G) + inv(P) +b_hat = inv(Hess) @ G.T @ inv(R) @ y[~nan_mask] # %timeit G_T_R_inv = G.T * (1/R_noise) # %timeit Hess = (G_T_R_inv @ G ) + np.diag(1/P_1d) @@ -234,93 +240,108 @@ # %% -model_error_k = np.diag(inv( Hess)) -model_error_real = ((G**2) @ inv( Hess)).sum(1) +model_error_k = np.diag(inv(Hess)) +model_error_real = ((G**2) @ inv(Hess)).sum(1) residual = y_gap[~nan_mask] - (b_hat * G).sum(1) -normalized_residual = residual.var() /np.diag(R).var() +normalized_residual = residual.var() / np.diag(R).var() data_var = y_gap[~nan_mask].var() model_var = (b_hat * G).sum(1).var() residual.var() -print('normlized residual ', normalized_residual) -print('data variance=', data_var,'model variance', model_var, ' residual variance', residual.var()) -print('sum', data_var-model_var- residual.var()) +print("normlized residual ", normalized_residual) +print( + "data variance=", + data_var, + "model variance", + model_var, + " residual variance", + residual.var(), +) +print("sum", data_var - model_var - residual.var()) # %% -plt.plot(k, gFT.power_from_model(b_hat, dk, MM, NN, x.size) ) -plt.title('Power(b_hat)') +plt.plot(k, gFT.power_from_model(b_hat, dk, MM, NN, x.size)) +plt.title("Power(b_hat)") plt.show() -plt.title('model error') -plt.plot(k, np.sqrt(model_error_k[0:MM]**2 + model_error_k[MM:]**2 ) ) +plt.title("model error") +plt.plot(k, np.sqrt(model_error_k[0:MM] ** 2 + model_error_k[MM:] ** 2)) plt.show() -#plt.xlim(0, 0.01) +# plt.xlim(0, 0.01) # %% -M.figure_axis_xy(10,5, view_scale =0.6) +M.figure_axis_xy(10, 5, view_scale=0.6) -plt.subplot(2,1, 1) +plt.subplot(2, 1, 1) -plt.plot( x, y_gap, '-k', linewidth= 2) -#plt.plot( x, y, '-k', linewidth= 2) +plt.plot(x, y_gap, "-k", linewidth=2) +# plt.plot( x, y, '-k', linewidth= 2) x_gap.size -k.size *2 +k.size * 2 -plt.plot( x_gap , (b_hat * G).sum(1), 'r-', markersize= 3 , label='model') -#plt.plot( x[~nan_mask] , (b_hat * G_full).sum(1)[~nan_mask], '-', color='orange', linewidth = 0.5 , label='prediction') -plt.plot( x[nan_mask] , (b_hat * G_full).sum(1)[nan_mask], '-', color='orange', linewidth = 0.5 , label='prediction') +plt.plot(x_gap, (b_hat * G).sum(1), "r-", markersize=3, label="model") +# plt.plot( x[~nan_mask] , (b_hat * G_full).sum(1)[~nan_mask], '-', color='orange', linewidth = 0.5 , label='prediction') +plt.plot( + x[nan_mask], + (b_hat * G_full).sum(1)[nan_mask], + "-", + color="orange", + linewidth=0.5, + label="prediction", +) -plt.plot(x_gap, np.sqrt(model_error_real) , 'b.', markersize= 1 , label= 'sqrt(model error)') +plt.plot( + x_gap, np.sqrt(model_error_real), "b.", markersize=1, label="sqrt(model error)" +) -#plt.xlim(1000,28000) +# plt.xlim(1000,28000) plt.ylim(-1, 1) plt.legend() # % -plt.subplot(2,1, 2) +plt.subplot(2, 1, 2) -#plt.plot( x, (y - (b_hat * G).sum(1)) , '.k', markersize= 0.8 ) -plt.plot( x_gap,residual, '.k', markersize= 0.8 , label= 'residual') +# plt.plot( x, (y - (b_hat * G).sum(1)) , '.k', markersize= 0.8 ) +plt.plot(x_gap, residual, ".k", markersize=0.8, label="residual") -plt.plot( x_gap, np.diag(R) , '-b', markersize= 0.8, label= 'given data uncertainty' ) +plt.plot(x_gap, np.diag(R), "-b", markersize=0.8, label="given data uncertainty") plt.legend() -#plt.xlim(1000,28000) +# plt.xlim(1000,28000) plt.ylim(-1, 1) # %% Auto correlation function of the Prior - -plt.plot( (G @ P @ G.T)[0,0:200] ) -plt.title('Prior auto correlation') +plt.plot((G @ P @ G.T)[0, 0:200]) +plt.title("Prior auto correlation") # %% -plt.plot( (G * np.diag(P ) ).sum(1) ) -plt.plot( np.abs( (np.diag(P ) * G).sum(1) )[0:100] ) +plt.plot((G * np.diag(P)).sum(1)) +plt.plot(np.abs((np.diag(P) * G).sum(1))[0:100]) # %% -M.figure_axis_xy(7,3, view_scale =0.6) +M.figure_axis_xy(7, 3, view_scale=0.6) -Z = b_hat[0:MM] - b_hat[MM:] *1j -#Z_err = model_error_k[0:MM] - model_error_k[MM:] *1j +Z = b_hat[0:MM] - b_hat[MM:] * 1j +# Z_err = model_error_k[0:MM] - model_error_k[MM:] *1j plt.subplot(1, 2, 1) -plt.title('abs(Z) model') +plt.title("abs(Z) model") -plt.plot(k, weight/weight.max()*abs(Z).max()) +plt.plot(k, weight / weight.max() * abs(Z).max()) -plt.plot(k, abs(Z) ) -plt.xlabel('k ( 2 pi/ lambda) ') +plt.plot(k, abs(Z)) +plt.xlabel("k ( 2 pi/ lambda) ") plt.subplot(1, 2, 2) -plt.title('model error') +plt.title("model error") plt.plot(k, abs(Z_err)) -plt.xlabel('k ( 2 pi/ lambda) ') +plt.xlabel("k ( 2 pi/ lambda) ") # %% diff --git a/analysis/SB03_posterior_phase_fitting.py b/analysis/SB03_posterior_phase_fitting.py index 697aaa6b..97bf54b5 100644 --- a/analysis/SB03_posterior_phase_fitting.py +++ b/analysis/SB03_posterior_phase_fitting.py @@ -1,16 +1,16 @@ - import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -22,84 +22,94 @@ import spicke_remover import datetime import concurrent.futures as futures -#import s3fs -# %% -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False -track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False - - - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -save_path = mconfig['paths']['work'] + '/B03_spectra_'+hemis+'/' -save_name = 'B03_'+track_name - -plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/B_spectra/' +# import s3fs +# %% +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False +track_name, batch_key, test_flag = "20190219073735_08070210_004_01", "SH_batch02", False + + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +save_path = mconfig["paths"]["work"] + "/B03_spectra_" + hemis + "/" +save_name = "B03_" + track_name + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/B_spectra/" +) MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -#Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +# Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data -load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # +load_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # # %% l1 = 100 -k1= 2* np.pi /l1 +k1 = 2 * np.pi / l1 l2 = 65 -k2= 2* np.pi /l2 +k2 = 2 * np.pi / l2 -x=np.arange(0, 1000, 0.5) -y=np.arange(0, 1000, 0.5) +x = np.arange(0, 1000, 0.5) +y = np.arange(0, 1000, 0.5) -z = np.sin(k1) + np.sin(k2* x) +z = np.sin(k1) + np.sin(k2 * x) # %% font_for_pres() -plt.semilogx(k, abs(Z) ) -ax =plt.gca() +plt.semilogx(k, abs(Z)) +ax = plt.gca() ax.axvline(k1) ax.axvline(k2) # %% -F = M.figure_axis_xy(4,4) -#kk = np.array([k1, k2]) +F = M.figure_axis_xy(4, 4) +# kk = np.array([k1, k2]) kk = np.array([k1, k2]) -kk_pos= np.array([abs(k - ki).argmin() for ki in kk]) +kk_pos = np.array([abs(k - ki).argmin() for ki in kk]) -Zi = Z[kk_pos]/abs(Z[kk_pos]) -plt.plot(Zi.real , Zi.imag, '.k' ) +Zi = Z[kk_pos] / abs(Z[kk_pos]) +plt.plot(Zi.real, Zi.imag, ".k") -b = np.arange(-3*l1, l1*6, 2) +b = np.arange(-3 * l1, l1 * 6, 2) -p = np.exp( np.outer(b, kk) *1j ) -#p = np.exp( b.*kk *1j ) +p = np.exp(np.outer(b, kk) * 1j) +# p = np.exp( b.*kk *1j ) -Z2= p * Zi -plt.plot( Z2.real ,Z2.imag, '.r' ) +Z2 = p * Zi +plt.plot(Z2.real, Z2.imag, ".r") -plt.axis('equal') +plt.axis("equal") plt.grid() -F.ax.axhline(0, color='black') -F.ax.axvline(0, color='black') +F.ax.axhline(0, color="black") +F.ax.axvline(0, color="black") # %% # np.angle(Zi[0]-Zi[1], deg=True) @@ -107,8 +117,8 @@ # # np.angle(Zi[0]-Zi[1], deg=True) - np.angle(Zi[1]-Zi[0], deg=True) -plt.plot( np.angle(Zi) - np.angle(Z2) ) -plt.plot( np.angle(Z2) - np.angle(Zi) ) +plt.plot(np.angle(Zi) - np.angle(Z2)) +plt.plot(np.angle(Z2) - np.angle(Zi)) np.angle(Zi[1]) - np.angle(Zi[0]) Zi[0].dot @@ -116,13 +126,15 @@ np.angle(Zi[0], deg=True) np.angle(Zi[1], deg=True) -#@vectorize + +# @vectorize def cdot(a, b): - return (a.real*b.real + a.imag*b.imag) + return a.real * b.real + a.imag * b.imag + -plt.plot(np.arccos(cdot(Zi, Z2))**2) +plt.plot(np.arccos(cdot(Zi, Z2)) ** 2) -plt.plot( b, (np.arccos(cdot(Zi, Z2))**2).sum(1) ) +plt.plot(b, (np.arccos(cdot(Zi, Z2)) ** 2).sum(1)) plt.grid() # np.arccos( (Zi[0]* Zi[1]).real + (Zi[0]* Zi[1]).imag) diff --git a/analysis/SB03_swell_phase_fitting.py b/analysis/SB03_swell_phase_fitting.py index 83116278..8a16324c 100644 --- a/analysis/SB03_swell_phase_fitting.py +++ b/analysis/SB03_swell_phase_fitting.py @@ -1,16 +1,16 @@ - import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -22,95 +22,124 @@ import spicke_remover import datetime import concurrent.futures as futures -#import s3fs -# %% -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False -track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False - - - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -save_path = mconfig['paths']['work'] + '/B03_spectra_'+hemis+'/' -save_name = 'B03_'+track_name - -plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/B_spectra/' +# import s3fs +# %% +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False +track_name, batch_key, test_flag = "20190219073735_08070210_004_01", "SH_batch02", False + + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +save_path = mconfig["paths"]["work"] + "/B03_spectra_" + hemis + "/" +save_name = "B03_" + track_name + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/B_spectra/" +) MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -#Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +# Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data -load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # +load_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # col.colormaps2(21) # %% -T_max = 40 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 -x= np.array(Gd['gt1r']['dist']) +T_max = 40 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 +x = np.array(Gd["gt1r"]["dist"]) dx = np.diff(x).mean() -xlims = x[0], x[-1] -min_datapoint = 1/k_0/dx +xlims = x[0], x[-1] +min_datapoint = 1 / k_0 / dx Lpoints = int(np.round(min_datapoint) * 20) -Lmeters =Lpoints * dx +Lmeters = Lpoints * dx -def get_stacil_data(stancil, G3, key = 'dist' ): + +def get_stacil_data(stancil, G3, key="dist"): mask = (G3[key] >= stancil[0]) & (G3[key] < stancil[2]) return G3[np.array(mask)] -def make_slopes(G3, key = 'heights_c_weighted_mean', spreed =10, verbose= False, photon_min =5): - dd = np.copy(G3['heights_c_weighted_mean']) - dd = np.gradient(dd) - dd, _ = spicke_remover.spicke_remover(dd, spreed=spreed, verbose=verbose) - dd_nans = (np.isnan(dd) ) + (G3['N_photos'] <= photon_min) +def make_slopes( + G3, key="heights_c_weighted_mean", spreed=10, verbose=False, photon_min=5 +): + + dd = np.copy(G3["heights_c_weighted_mean"]) + dd = np.gradient(dd) + dd, _ = spicke_remover.spicke_remover(dd, spreed=spreed, verbose=verbose) + dd_nans = (np.isnan(dd)) + (G3["N_photos"] <= photon_min) return dd, dd_nans -stancil_iter = spec.create_chunk_boundaries_unit_lengths(Lmeters, xlims, ov= None , iter_flag=False) + +stancil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeters, xlims, ov=None, iter_flag=False +) # %% -sys.path.append(mconfig['paths']['analysis']) +sys.path.append(mconfig["paths"]["analysis"]) import SB04_2d_wavefield_emulator as WaveEmulator # test wave model -k_abs = 2 * np.pi/100 +k_abs = 2 * np.pi / 100 imp.reload(WaveEmulator) -kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar(k_abs,0, size=1, dk= k_abs/10, amp_std=k_abs/20 ) +kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar( + k_abs, 0, size=1, dk=k_abs / 10, amp_std=k_abs / 20 +) -x=np.arange(-250, 250, 0.5) -y=np.arange(-200, 200, 0.5) +x = np.arange(-250, 250, 0.5) +y = np.arange(-200, 200, 0.5) -def wavemodel(XX, YY, ks, ls, amps, group_phase = 0, amp_height= 1): + +def wavemodel(XX, YY, ks, ls, amps, group_phase=0, amp_height=1): import numpy as np - G = np.vstack([ np.cos(np.outer(XX, ks) + np.outer(YY, ls) ).T , np.sin(np.outer(XX, ks) + np.outer(YY, ls) ).T ] ).T + G = np.vstack( + [ + np.cos(np.outer(XX, ks) + np.outer(YY, ls)).T, + np.sin(np.outer(XX, ks) + np.outer(YY, ls)).T, + ] + ).T - #phase1 = np.random.rand(1, amp_list.size) * np.pi*2 - #phase = np.arange(0, amp_list.size) * np.pi/2 + # phase1 = np.random.rand(1, amp_list.size) * np.pi*2 + # phase = np.arange(0, amp_list.size) * np.pi/2 - b = np.hstack([ np.cos(group_phase)*amps, np.sin(group_phase) *amps]).squeeze() * amp_height - z_model = (G @ b) + b = ( + np.hstack([np.cos(group_phase) * amps, np.sin(group_phase) * amps]).squeeze() + * amp_height + ) + z_model = G @ b return z_model + Nx, Ny = x.size, y.size XX, YY = np.meshgrid(x, y) XX, YY = XX.reshape(XX.size), YY.reshape(YY.size) @@ -121,274 +150,348 @@ def wavemodel(XX, YY, ks, ls, amps, group_phase = 0, amp_height= 1): # %% select data Gd.keys() + def get_beam(G3, key): G1 = G3[key] - G1['slopes'], _ = make_slopes(G1) - G1['beam']= key + G1["slopes"], _ = make_slopes(G1) + G1["beam"] = key return G1 -B1= get_beam(Gd, 'gt3r') -B2= get_beam(Gd, 'gt3l') -dist_y_mean = np.sqrt( (B1['x']-B2['x'])**2 + (B1['y']-B2['y'])**2 ).mean() +B1 = get_beam(Gd, "gt3r") +B2 = get_beam(Gd, "gt3l") + +dist_y_mean = np.sqrt((B1["x"] - B2["x"]) ** 2 + (B1["y"] - B2["y"]) ** 2).mean() -B1['dist_y'] = 0 -B2['dist_y'] = dist_y_mean +B1["dist_y"] = 0 +B2["dist_y"] = dist_y_mean -GG = pd.concat( [B1, B2] ) -#GG = get_beam(Gd, 'gt3r') +GG = pd.concat([B1, B2]) +# GG = get_beam(Gd, 'gt3r') stancil = stancil_iter[:, 30] -GG = get_stacil_data(stancil, GG).sort_values(['beam','dist']) +GG = get_stacil_data(stancil, GG).sort_values(["beam", "dist"]) # %% -GG['x_prime'] = GG['x'] - GG['x'].min() -GG['y_prime'] = GG['y'] - GG['y'].max() +GG["x_prime"] = GG["x"] - GG["x"].min() +GG["y_prime"] = GG["y"] - GG["y"].max() # plt.plot(GG['x_prime'], GG['y_prime'] , '.', markersize = 0.5) # plt.axis('equal') # %% get peak wavenumber, power and spreed G4 = GG + + def get_fft(G4, beam, dx): - G4[G4['beam'] == beam] - ys = np.copy(G4['slopes']) - ys[np.isnan(ys)] =0 - k_fft = np.fft.rfftfreq(ys.size, d=dx) * 2* np.pi + G4[G4["beam"] == beam] + ys = np.copy(G4["slopes"]) + ys[np.isnan(ys)] = 0 + k_fft = np.fft.rfftfreq(ys.size, d=dx) * 2 * np.pi return np.fft.rfft(ys), k_fft -B1_fft, k_fft = get_fft(GG, 'gt3r', dx) -B2_fft, k_fft = get_fft(GG, 'gt3l', dx) -Z_fft = (B1_fft+ B2_fft)/2 -Z_fft_rm = M.runningmean(abs(Z_fft), 2, tailcopy=True) +B1_fft, k_fft = get_fft(GG, "gt3r", dx) +B2_fft, k_fft = get_fft(GG, "gt3l", dx) + +Z_fft = (B1_fft + B2_fft) / 2 +Z_fft_rm = M.runningmean(abs(Z_fft), 2, tailcopy=True) Z_max_pos = Z_fft_rm[~np.isnan(Z_fft_rm)].argmax() -k_fft_max = k_fft[Z_max_pos] -Z_max = Z_fft[Z_max_pos] +k_fft_max = k_fft[Z_max_pos] +Z_max = Z_fft[Z_max_pos] font_for_pres() -plt.plot(k_fft, abs(Z_fft) ) -plt.plot(k_fft, Z_fft_rm ) -plt.plot(k_fft_max, Z_fft_rm[Z_max_pos], '.', markersize= 20) +plt.plot(k_fft, abs(Z_fft)) +plt.plot(k_fft, Z_fft_rm) +plt.plot(k_fft_max, Z_fft_rm[Z_max_pos], ".", markersize=20) + + +def gaus(x, x_0, amp, sigma_g): + return amp * np.exp(-0.5 * ((x - x_0) / sigma_g) ** 2) -def gaus(x, x_0, amp, sigma_g ): - return amp* np.exp(-0.5 * ( (x-x_0)/sigma_g)**2) -dk =0.002 -plt.plot(k_fft, gaus(k_fft, k_fft_max, Z_max , dk ), 'r', linewidth = 2 ) +dk = 0.002 +plt.plot(k_fft, gaus(k_fft, k_fft_max, Z_max, dk), "r", linewidth=2) plt.xlim(0, 0.1) # %% -k_prime_abs = k_fft_max#2 * np.pi/ 200 +k_prime_abs = k_fft_max # 2 * np.pi/ 200 imp.reload(WaveEmulator) alpha = 60 -for group_phase in np.linspace(0, 2* np.pi,4): +for group_phase in np.linspace(0, 2 * np.pi, 4): - akpha_rad= alpha *np.pi/180 - k_abs = k_prime_abs / np.cos(akpha_rad) + akpha_rad = alpha * np.pi / 180 + k_abs = k_prime_abs / np.cos(akpha_rad) print(k_abs, k_prime_abs) - kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar(k_abs, akpha_rad , size=0 , dk= dk , plot_flag= False) + kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar( + k_abs, akpha_rad, size=0, dk=dk, plot_flag=False + ) - print('k mean ', kk.mean(), 'l mean ', ll.mean() ) + print("k mean ", kk.mean(), "l mean ", ll.mean()) # x2 = np.arange( GG['x_prime'].min(), GG['x_prime'].max(), 1) # y2 = x2 * 0 # x2, y2 = np.concatenate([x2, x2]), np.concatenate([y2, y2+60]) # #z_model = wavemodel( x2, y2 , kk, ll, amps) N = GG.shape[0] - #abs(Z_max)**2/ N + # abs(Z_max)**2/ N # normalized version - #amp_Z = abs(Z_max)/np.sqrt(2)/2 + # amp_Z = abs(Z_max)/np.sqrt(2)/2 # dimensional version - amp_Z = 2* abs(Z_max)**2 /N - GG['z_model'] = wavemodel( GG['dist'],GG['dist_y'], kk, ll, amps, group_phase= group_phase, amp_height= amp_Z) - - M.figure_axis_xy(6, 5.5, view_scale = 0.5) - - beam = 'gt3l' + amp_Z = 2 * abs(Z_max) ** 2 / N + GG["z_model"] = wavemodel( + GG["dist"], + GG["dist_y"], + kk, + ll, + amps, + group_phase=group_phase, + amp_height=amp_Z, + ) + + M.figure_axis_xy(6, 5.5, view_scale=0.5) + + beam = "gt3l" plt.subplot(2, 1, 1) - plt.title(beam +' ' + str(alpha) ) - #plt.plot(GG['x_prime'], GG['z_model'], '.' ) + plt.title(beam + " " + str(alpha)) + # plt.plot(GG['x_prime'], GG['z_model'], '.' ) - GGsel =GG[GG['beam'] == beam] - plt.plot(GGsel['dist'], GGsel['z_model'], c=col.gray) + GGsel = GG[GG["beam"] == beam] + plt.plot(GGsel["dist"], GGsel["z_model"], c=col.gray) # diumensional version - plt.plot(GGsel['dist'], GGsel['slopes'] , '-', c=col.cascade2) + plt.plot(GGsel["dist"], GGsel["slopes"], "-", c=col.cascade2) # normalized version - #plt.plot(GGsel['dist'], GGsel['slopes']/GGsel['slopes'].std() , '-', c=col.cascade2) + # plt.plot(GGsel['dist'], GGsel['slopes']/GGsel['slopes'].std() , '-', c=col.cascade2) - beam = 'gt3r' + beam = "gt3r" plt.subplot(2, 1, 2) - plt.title(beam +' ' + str(alpha) ) - GGsel =GG[GG['beam'] == beam] - plt.plot(GGsel['dist'], GGsel['z_model'], c=col.gray) - plt.plot(GGsel['dist'], GGsel['slopes'] , '-', c=col.rascade2) + plt.title(beam + " " + str(alpha)) + GGsel = GG[GG["beam"] == beam] + plt.plot(GGsel["dist"], GGsel["z_model"], c=col.gray) + plt.plot(GGsel["dist"], GGsel["slopes"], "-", c=col.rascade2) plt.show() # %% -#abs(Z_max)**2/ N +# abs(Z_max)**2/ N # normalized version -#amp_Z = abs(Z_max)/np.sqrt(2)/2 +# amp_Z = abs(Z_max)/np.sqrt(2)/2 # dimensional version -amp_Z = 2* abs(Z_max)**2 /N +amp_Z = 2 * abs(Z_max) ** 2 / N -def get_z_model(x_positions, y_position, K_prime, K_amp, alpha_rad, group_phase, dk, size =0): - K_abs = K_prime / np.cos(alpha_rad) - kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar(K_abs, alpha_rad , size=size , dk= dk , plot_flag= False) +def get_z_model( + x_positions, y_position, K_prime, K_amp, alpha_rad, group_phase, dk, size=0 +): - return wavemodel( x_positions,y_position, kk, ll, amps, group_phase= group_phase, amp_height= K_amp) + K_abs = K_prime / np.cos(alpha_rad) + kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar( + K_abs, alpha_rad, size=size, dk=dk, plot_flag=False + ) + return wavemodel( + x_positions, y_position, kk, ll, amps, group_phase=group_phase, amp_height=K_amp + ) -def objective_func(pars, x, y, z, test_flag= False ): +def objective_func(pars, x, y, z, test_flag=False): - z_model = get_z_model(x, y, pars['K_prime'], pars['K_amp'],pars['alpha'],pars['group_phase'], pars['dk'], size=pars['model_size']) - cost =( abs(z - z_model) )**2 /z.std()**2 + z_model = get_z_model( + x, + y, + pars["K_prime"], + pars["K_amp"], + pars["alpha"], + pars["group_phase"], + pars["dk"], + size=pars["model_size"], + ) + cost = (abs(z - z_model)) ** 2 / z.std() ** 2 if test_flag: return z_model else: return cost -def plot_instance(GG3, data_key, model_key, non_dim=False ): +def plot_instance(GG3, data_key, model_key, non_dim=False): import itertools - F = M.figure_axis_xy(8, 5, view_scale = 0.5) - beam_list = list(set(GG3['beam'])) + F = M.figure_axis_xy(8, 5, view_scale=0.5) + beam_list = list(set(GG3["beam"])) - col_list = itertools.cycle([col.cascade2, col.rascade2, col.cascade1, col.rascade1, col.cascade3, col.rascade3]) + col_list = itertools.cycle( + [ + col.cascade2, + col.rascade2, + col.cascade1, + col.rascade1, + col.cascade3, + col.rascade3, + ] + ) subz = len(beam_list) - for beam, i in zip(beam_list, np.arange(1, subz+1, 1)): + for beam, i in zip(beam_list, np.arange(1, subz + 1, 1)): plt.subplot(subz, 1, i) - plt.title(beam ) - #plt.plot(GG['x_prime'], GG['z_model'], '.' ) + plt.title(beam) + # plt.plot(GG['x_prime'], GG['z_model'], '.' ) - GGsel =GG3[GG3['beam'] == beam] - plt.plot(GGsel['dist'], GGsel[model_key], c=col.gray) + GGsel = GG3[GG3["beam"] == beam] + plt.plot(GGsel["dist"], GGsel[model_key], c=col.gray) if non_dim: # normalized version - plt.plot(GGsel['dist'], GGsel['slopes']/GGsel['slopes'].std() , '-', c=next(col_list)) + plt.plot( + GGsel["dist"], + GGsel["slopes"] / GGsel["slopes"].std(), + "-", + c=next(col_list), + ) else: # diumensional version - plt.plot(GGsel['dist'], GGsel[data_key] , '-', c=next(col_list)) + plt.plot(GGsel["dist"], GGsel[data_key], "-", c=next(col_list)) return F - -#GG = pd.concat( [B1, B2] ) -GG = pd.concat( [B1, B2] ) +# GG = pd.concat( [B1, B2] ) +GG = pd.concat([B1, B2]) stancil = stancil_iter[:, 30] -GG = get_stacil_data(stancil, GG).sort_values(['beam','dist']) -#GG['slopes'] = zm -slopes1 = GG[GG['beam'] == 'gt3r']['slopes'] -GG['slopes'] = np.concatenate([slopes1, slopes1]) - +GG = get_stacil_data(stancil, GG).sort_values(["beam", "dist"]) +# GG['slopes'] = zm +slopes1 = GG[GG["beam"] == "gt3r"]["slopes"] +GG["slopes"] = np.concatenate([slopes1, slopes1]) # %% import lmfit as LM + params = LM.Parameters() -params.add('K_prime', k_prime_abs , vary=False , min=k_prime_abs*0.5, max=k_prime_abs*1.5) -params.add('K_amp', amp_Z , vary=False , min=amp_Z*.5, max=amp_Z*5) -params.add('alpha', np.pi/4 , vary=True , min=-0.95 * np.pi /2, max=0.95 * np.pi /2) -params.add('group_phase', 0 , vary=True , min=0, max= 2*np.pi) -params.add('dk', dk , vary=False , min=1e-4, max= 0.05) -params.add('model_size', 0 , vary=False , min=0, max= 2) +params.add( + "K_prime", k_prime_abs, vary=False, min=k_prime_abs * 0.5, max=k_prime_abs * 1.5 +) +params.add("K_amp", amp_Z, vary=False, min=amp_Z * 0.5, max=amp_Z * 5) +params.add("alpha", np.pi / 4, vary=True, min=-0.95 * np.pi / 2, max=0.95 * np.pi / 2) +params.add("group_phase", 0, vary=True, min=0, max=2 * np.pi) +params.add("dk", dk, vary=False, min=1e-4, max=0.05) +params.add("model_size", 0, vary=False, min=0, max=2) # test: this should be 0 -zm =get_z_model( GG['dist'],GG['dist_y'], k_prime_abs, amp_Z, np.pi/4, 0 , dk) -objective_func(params, GG['dist'],GG['dist_y'], zm) - -objective_func(params, GG['dist'],GG['dist_y'], GG['slopes']) - -fitting_args = (GG['dist'],GG['dist_y'], GG['slopes']) -#fitting_kargs = {'size' :1} - -fitter = LM.minimize(objective_func, params, args=fitting_args, method='brute', Ns=100, workers=4, max_nfev=None) -GG['z_model'] = objective_func(fitter.params, GG['dist'],GG['dist_y'], GG['slopes'] , test_flag= True) -plot_instance(GG, 'slopes', 'z_model' ) +zm = get_z_model(GG["dist"], GG["dist_y"], k_prime_abs, amp_Z, np.pi / 4, 0, dk) +objective_func(params, GG["dist"], GG["dist_y"], zm) + +objective_func(params, GG["dist"], GG["dist_y"], GG["slopes"]) + +fitting_args = (GG["dist"], GG["dist_y"], GG["slopes"]) +# fitting_kargs = {'size' :1} + +fitter = LM.minimize( + objective_func, + params, + args=fitting_args, + method="brute", + Ns=100, + workers=4, + max_nfev=None, +) +GG["z_model"] = objective_func( + fitter.params, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=True +) +plot_instance(GG, "slopes", "z_model") fitter # %% params2 = fitter.params.copy() -params2['model_size'].value = 0 -params2['K_prime'].vary = True -#params2['dk'].vary = False -#params2['dk'].value = params2['K_prime']/20 - -#params2['alpha'].vary = False -params2['group_phase'].vary = True -#params2['K_amp'].vary = True - - -fitter2 = LM.minimize(objective_func, params2, args=fitting_args, method='dual_annealing', max_nfev=None) -#fitter3 = LM.minimize(objective_func, params2, args=fitting_args, method='emcee', nwalkers=100, steps=1500) -GG['z_model'] = objective_func(fitter2.params, GG['dist'],GG['dist_y'], GG['slopes'] , test_flag= True) -plot_instance(GG, 'slopes', 'z_model' ) +params2["model_size"].value = 0 +params2["K_prime"].vary = True +# params2['dk'].vary = False +# params2['dk'].value = params2['K_prime']/20 + +# params2['alpha'].vary = False +params2["group_phase"].vary = True +# params2['K_amp'].vary = True + + +fitter2 = LM.minimize( + objective_func, params2, args=fitting_args, method="dual_annealing", max_nfev=None +) +# fitter3 = LM.minimize(objective_func, params2, args=fitting_args, method='emcee', nwalkers=100, steps=1500) +GG["z_model"] = objective_func( + fitter2.params, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=True +) +plot_instance(GG, "slopes", "z_model") fitter2 # %% rotation test -#GG = pd.concat( [B1, B2] ) -GG = pd.concat( [B1, B2] ) +# GG = pd.concat( [B1, B2] ) +GG = pd.concat([B1, B2]) stancil = stancil_iter[:, 30] -GG = get_stacil_data(stancil, GG).sort_values(['beam','dist']) -#GG['slopes'] = zm -slopes1 = GG[GG['beam'] == 'gt3r']['slopes'] -GG['slopes'] = np.concatenate([slopes1, np.roll(slopes1, int( (0 *np.pi/4) * (2*np.pi/ k_prime_abs)/dx ))]) +GG = get_stacil_data(stancil, GG).sort_values(["beam", "dist"]) +# GG['slopes'] = zm +slopes1 = GG[GG["beam"] == "gt3r"]["slopes"] +GG["slopes"] = np.concatenate( + [slopes1, np.roll(slopes1, int((0 * np.pi / 4) * (2 * np.pi / k_prime_abs) / dx))] +) import lmfit as LM + params = LM.Parameters() -params.add('K_prime', k_prime_abs , vary=False , min=k_prime_abs*0.5, max=k_prime_abs*1.5) -params.add('K_amp', amp_Z , vary=False , min=amp_Z*.5, max=amp_Z*5) -params.add('alpha', np.pi/4 , vary=True , min=-0.95 * np.pi /2, max=0.95 * np.pi /2) -params.add('group_phase', 0 , vary=True , min=0, max= 2*np.pi) -params.add('dk', dk , vary=False , min=1e-4, max= 0.05) -params.add('model_size', 0 , vary=False , min=0, max= 2) +params.add( + "K_prime", k_prime_abs, vary=False, min=k_prime_abs * 0.5, max=k_prime_abs * 1.5 +) +params.add("K_amp", amp_Z, vary=False, min=amp_Z * 0.5, max=amp_Z * 5) +params.add("alpha", np.pi / 4, vary=True, min=-0.95 * np.pi / 2, max=0.95 * np.pi / 2) +params.add("group_phase", 0, vary=True, min=0, max=2 * np.pi) +params.add("dk", dk, vary=False, min=1e-4, max=0.05) +params.add("model_size", 0, vary=False, min=0, max=2) # test: this should be 0 -zm =get_z_model( GG['dist'],GG['dist_y'], k_prime_abs, amp_Z, np.pi/4, 0 , dk) -objective_func(params, GG['dist'],GG['dist_y'], zm).sum() - -objective_func(params, GG['dist'],GG['dist_y'], GG['slopes']).sum() - -fitting_args = (GG['dist'],GG['dist_y'], GG['slopes']) -#fitting_kargs = {'size' :1} -fitter = LM.minimize(objective_func, params, args=fitting_args, method='dual_annealing', max_nfev=None) -GG['z_model'] = objective_func(fitter.params, GG['dist'],GG['dist_y'], GG['slopes'] , test_flag= True) -plot_instance(GG, 'slopes', 'z_model' ) +zm = get_z_model(GG["dist"], GG["dist_y"], k_prime_abs, amp_Z, np.pi / 4, 0, dk) +objective_func(params, GG["dist"], GG["dist_y"], zm).sum() + +objective_func(params, GG["dist"], GG["dist_y"], GG["slopes"]).sum() + +fitting_args = (GG["dist"], GG["dist_y"], GG["slopes"]) +# fitting_kargs = {'size' :1} +fitter = LM.minimize( + objective_func, params, args=fitting_args, method="dual_annealing", max_nfev=None +) +GG["z_model"] = objective_func( + fitter.params, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=True +) +plot_instance(GG, "slopes", "z_model") fitter # %% params2 = fitter.params.copy() -params2['K_prime'].vary = True -GG['slopes_smth'] = M.runningmean_wrap_around(np.array(GG['slopes']), 2) -fitter2 = LM.minimize(objective_func, params, args=fitting_args, method='dual_annealing', max_nfev=None) -#fitter3 = LM.minimize(objective_func, params2, args=fitting_args, method='emcee', nwalkers=100, steps=1500) -GG['z_model'] = objective_func(fitter2.params, GG['dist'],GG['dist_y'], GG['slopes_smth'] , test_flag= True) -plot_instance(GG, 'slopes_smth', 'z_model' ) +params2["K_prime"].vary = True +GG["slopes_smth"] = M.runningmean_wrap_around(np.array(GG["slopes"]), 2) +fitter2 = LM.minimize( + objective_func, params, args=fitting_args, method="dual_annealing", max_nfev=None +) +# fitter3 = LM.minimize(objective_func, params2, args=fitting_args, method='emcee', nwalkers=100, steps=1500) +GG["z_model"] = objective_func( + fitter2.params, GG["dist"], GG["dist_y"], GG["slopes_smth"], test_flag=True +) +plot_instance(GG, "slopes_smth", "z_model") fitter2 # %% diff --git a/analysis/SB03_swell_phase_fitting_fake.py b/analysis/SB03_swell_phase_fitting_fake.py index 6c3eb997..de6f5456 100644 --- a/analysis/SB03_swell_phase_fitting_fake.py +++ b/analysis/SB03_swell_phase_fitting_fake.py @@ -1,16 +1,16 @@ - import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -22,130 +22,149 @@ import spicke_remover import datetime import concurrent.futures as futures -#import s3fs -# %% -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False -track_name, batch_key, test_flag = '20190601093502_09790310_004_01', 'SH_batch02', False - - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -save_path = mconfig['paths']['work'] + '/B03_spectra_'+hemis+'/' -save_name = 'B03_'+track_name - -#plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/B_spectra/' -plot_path = mconfig['paths']['plot'] + '/phase_fitting_fake/' +# import s3fs +# %% +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +track_name, batch_key, test_flag = "20190601093502_09790310_004_01", "SH_batch02", False + + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +save_path = mconfig["paths"]["work"] + "/B03_spectra_" + hemis + "/" +save_name = "B03_" + track_name + +# plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/B_spectra/' +plot_path = mconfig["paths"]["plot"] + "/phase_fitting_fake/" MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -#Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +# Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data -load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # +load_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # col.colormaps2(21) # %% -T_max = 40 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 -x= np.array(Gd['gt1r']['dist']) +T_max = 40 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 +x = np.array(Gd["gt1r"]["dist"]) dx = np.diff(x).mean() -xlims = x[0], x[-1] -min_datapoint = 1/k_0/dx +xlims = x[0], x[-1] +min_datapoint = 1 / k_0 / dx Lpoints = int(np.round(min_datapoint) * 20) * 0.5 -Lmeters =Lpoints * dx +Lmeters = Lpoints * dx -xg = np.arange(Lpoints) *dx +xg = np.arange(Lpoints) * dx yg = np.array([0, 300]) XX, YY = np.meshgrid(xg, yg) Nx, Ny = xg.size, yg.size # %% -sys.path.append(mconfig['paths']['analysis']) +sys.path.append(mconfig["paths"]["analysis"]) import SB04_2d_wavefield_emulator as WaveEmulator # test wave model -T_max = 14 #sec -k_abs = (2 * np.pi/ T_max)**2 / 9.81 +T_max = 14 # sec +k_abs = (2 * np.pi / T_max) ** 2 / 9.81 print(k_abs) -angle =angle_true = 30 * np.pi/180 +angle = angle_true = 30 * np.pi / 180 imp.reload(WaveEmulator) -#kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar(k_abs, angle, size=20, dk= 0.005 , plot_flag=False, random = True) -kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar(k_abs, angle, size=200, dk= 0.003 , plot_flag=False, random = True) -#kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar(k_abs, angle, size=2, dk= 0.005 , plot_flag=False, random = True) +# kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar(k_abs, angle, size=20, dk= 0.005 , plot_flag=False, random = True) +kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar( + k_abs, angle, size=200, dk=0.003, plot_flag=False, random=True +) +# kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar(k_abs, angle, size=2, dk= 0.005 , plot_flag=False, random = True) -k_noise, l_noise, amp_noise, stancil_shape = WaveEmulator.get_stancils_polar(0.8, 0 * np.pi/180, size=2, dk = 1, mesh = True , plot_flag= False, random = True) -#k_noise, l_noise, amp_noise, stancil_shape = WaveEmulator.get_stancils_polar(0.8, 0 * np.pi/180, size=20, dk = 0.9, mesh = True , plot_flag= False, random = True) -amp_noise = (amp_noise *0+1) * 0 +k_noise, l_noise, amp_noise, stancil_shape = WaveEmulator.get_stancils_polar( + 0.8, 0 * np.pi / 180, size=2, dk=1, mesh=True, plot_flag=False, random=True +) +# k_noise, l_noise, amp_noise, stancil_shape = WaveEmulator.get_stancils_polar(0.8, 0 * np.pi/180, size=20, dk = 0.9, mesh = True , plot_flag= False, random = True) +amp_noise = (amp_noise * 0 + 1) * 0 k_all = np.concatenate([kk, k_noise]) l_all = np.concatenate([ll, l_noise]) amp_all = np.concatenate([amps, amp_noise]) -phase = (np.random.random(amp_all.size) ) *2*np.pi -G = np.vstack([ np.cos(np.outer(XX, k_all) + np.outer(YY, l_all)).T , np.sin(np.outer(XX, k_all) + np.outer(YY, l_all)).T ] ).T - -b = np.hstack([ np.cos(phase)*amp_all, np.sin(phase) *amp_all]).squeeze() -z_model = (G @ b) -z_model += np.random.normal(0, 1,z_model.size) * z_model.std()/2 +phase = (np.random.random(amp_all.size)) * 2 * np.pi +G = np.vstack( + [ + np.cos(np.outer(XX, k_all) + np.outer(YY, l_all)).T, + np.sin(np.outer(XX, k_all) + np.outer(YY, l_all)).T, + ] +).T + +b = np.hstack([np.cos(phase) * amp_all, np.sin(phase) * amp_all]).squeeze() +z_model = G @ b +z_model += np.random.normal(0, 1, z_model.size) * z_model.std() / 2 z_model = z_model.reshape(Ny, Nx) -GG0 = pd.DataFrame(columns = ['x', 'y', 'slopes']) -GG0['x_prime'] = XX.reshape(Nx*Ny) -GG0['y_prime'] = YY.reshape(Nx*Ny) -GG0['slopes'] = z_model.reshape(Nx*Ny) -GG0['dist'] = np.sqrt( (GG0['x_prime']) **2 + (GG0['y_prime'])**2) -GG= GG0 -#plt.plot(GG['x_prime'], GG['slopes'] , '-', markersize = 0.8) -plt.plot(GG['x_prime'], GG['slopes'] + GG0['y_prime']/10 , '-', markersize = 0.8) +GG0 = pd.DataFrame(columns=["x", "y", "slopes"]) +GG0["x_prime"] = XX.reshape(Nx * Ny) +GG0["y_prime"] = YY.reshape(Nx * Ny) +GG0["slopes"] = z_model.reshape(Nx * Ny) +GG0["dist"] = np.sqrt((GG0["x_prime"]) ** 2 + (GG0["y_prime"]) ** 2) +GG = GG0 +# plt.plot(GG['x_prime'], GG['slopes'] , '-', markersize = 0.8) +plt.plot(GG["x_prime"], GG["slopes"] + GG0["y_prime"] / 10, "-", markersize=0.8) plt.grid() -key_name = 'Lp'+ str(int(Lpoints)) + '_size' + str(int(kk.size)) + '_angle' + str(int(np.round(angle *180/np.pi, 0))) - -M.save_anyfig(plt.gcf(), name= key_name+ '_realization', path= plot_path ) +key_name = ( + "Lp" + + str(int(Lpoints)) + + "_size" + + str(int(kk.size)) + + "_angle" + + str(int(np.round(angle * 180 / np.pi, 0))) +) + +M.save_anyfig(plt.gcf(), name=key_name + "_realization", path=plot_path) plt.show() # plt.plot(GG['x_prime']/1e3, GG['y_prime']/1e3) # plt.show() # plt.axis('equal') + # %% get peak wavenumber, power and spreed -def get_fft(y , dx): +def get_fft(y, dx): ys = np.copy(y) - ys[np.isnan(ys)] =0 - k_fft = np.fft.rfftfreq(ys.size, d=dx) * 2* np.pi + ys[np.isnan(ys)] = 0 + k_fft = np.fft.rfftfreq(ys.size, d=dx) * 2 * np.pi return np.fft.rfft(ys), k_fft -B1_fft, k_fft = get_fft(GG0[ GG0['y_prime'] == 0 ]['slopes'], dx) -B2_fft, k_fft = get_fft(GG0[ GG0['y_prime'] == 300 ]['slopes'], dx) +B1_fft, k_fft = get_fft(GG0[GG0["y_prime"] == 0]["slopes"], dx) +B2_fft, k_fft = get_fft(GG0[GG0["y_prime"] == 300]["slopes"], dx) -Z_fft = (B1_fft+ B2_fft)/2 -#Z_fft = Z_fft[1:] -Z_fft_rm = M.runningmean(abs(Z_fft), 2, tailcopy=True) +Z_fft = (B1_fft + B2_fft) / 2 +# Z_fft = Z_fft[1:] +Z_fft_rm = M.runningmean(abs(Z_fft), 2, tailcopy=True) -k_fft = k_fft[~np.isnan(Z_fft_rm)] -Z_fft = Z_fft[~np.isnan(Z_fft_rm)] -Z_fft_rm = Z_fft_rm[~np.isnan(Z_fft_rm)] +k_fft = k_fft[~np.isnan(Z_fft_rm)] +Z_fft = Z_fft[~np.isnan(Z_fft_rm)] +Z_fft_rm = Z_fft_rm[~np.isnan(Z_fft_rm)] Z_max_pos = Z_fft_rm.argmax() -k_fft_max = k_fft[Z_max_pos] -Z_max = Z_fft_rm[Z_max_pos] +k_fft_max = k_fft[Z_max_pos] +Z_max = Z_fft_rm[Z_max_pos] # Z_sort = Z_fft_rm.argsort()[::-1] @@ -154,48 +173,54 @@ def get_fft(y , dx): # Z_fft_sel = Z_fft_rm[Z_sort][np.cumsum(Z_fft_rm[Z_sort])/np.sum(Z_fft_rm) <0.2] # # plt.plot(K_fft_sel, Z_fft_sel, '.') -#plt.plot(k_fft, Z_fft_rm ) +# plt.plot(k_fft, Z_fft_rm ) # % font_for_pres() -plt.plot(k_fft, abs(Z_fft) ) -plt.plot(k_fft, Z_fft_rm ) -treshold = np.nanmean(Z_fft_rm) + np.nanstd(Z_fft_rm) *3 +plt.plot(k_fft, abs(Z_fft)) +plt.plot(k_fft, Z_fft_rm) +treshold = np.nanmean(Z_fft_rm) + np.nanstd(Z_fft_rm) * 3 # k_fft_max_list = k_fft[Z_fft_rm > treshold] # Z_max_list = Z_fft_rm[Z_fft_rm > treshold] dk = np.diff(k_fft).mean() -k_interp = np.arange(k_fft[0], k_fft[-1], dk/4) -Z_interp = np.interp( k_interp, k_fft, abs(Z_fft_rm) ) +k_interp = np.arange(k_fft[0], k_fft[-1], dk / 4) +Z_interp = np.interp(k_interp, k_fft, abs(Z_fft_rm)) mask_interp = abs(Z_interp) > treshold -plt.plot(k_interp[mask_interp], Z_interp[mask_interp], '.') +plt.plot(k_interp[mask_interp], Z_interp[mask_interp], ".") -k_list_interp , Z_list_interp = k_interp[mask_interp], Z_interp[mask_interp] +k_list_interp, Z_list_interp = k_interp[mask_interp], Z_interp[mask_interp] mask = abs(Z_fft) > treshold -k_fft_max_list = k_fft[mask] -Z_max_list = abs(Z_fft)[mask] +k_fft_max_list = k_fft[mask] +Z_max_list = abs(Z_fft)[mask] -plt.plot(k_fft_max_list, Z_max_list, '.r', markersize= 20) +plt.plot(k_fft_max_list, Z_max_list, ".r", markersize=20) -def gaus(x, x_0, amp, sigma_g ): - return amp* np.exp(-0.5 * ( (x-x_0)/sigma_g)**2) +def gaus(x, x_0, amp, sigma_g): + return amp * np.exp(-0.5 * ((x - x_0) / sigma_g) ** 2) -k_sigma =0.003 -plt.plot(k_fft, gaus(k_fft, k_fft_max, Z_max , k_sigma ), 'r', linewidth = 2 ) +k_sigma = 0.003 +plt.plot(k_fft, gaus(k_fft, k_fft_max, Z_max, k_sigma), "r", linewidth=2) -k_list_gauss = np.arange(k_fft_max - 1.5* k_sigma, k_fft_max + 1.5* k_sigma+ np.diff(k_fft).mean(), np.diff(k_fft).mean()/2) -Z_max_list_gauss =gaus(k_list_gauss, k_fft_max, Z_max , k_sigma ) -plt.plot(k_list_gauss, Z_max_list_gauss, 'b.', linewidth = 2 ) + +k_list_gauss = np.arange( + k_fft_max - 1.5 * k_sigma, + k_fft_max + 1.5 * k_sigma + np.diff(k_fft).mean(), + np.diff(k_fft).mean() / 2, +) +Z_max_list_gauss = gaus(k_list_gauss, k_fft_max, Z_max, k_sigma) +plt.plot(k_list_gauss, Z_max_list_gauss, "b.", linewidth=2) plt.xlim(0, 0.09) -M.save_anyfig(plt.gcf(), name= key_name+ '_fft', path= plot_path ) +M.save_anyfig(plt.gcf(), name=key_name + "_fft", path=plot_path) # %% -def get_wavenumbers_polar( amp, angle_rad): + +def get_wavenumbers_polar(amp, angle_rad): """ inputs: @@ -206,192 +231,283 @@ def get_wavenumbers_polar( amp, angle_rad): wavenumber k,l """ import numpy as np + k0 = amp * np.cos(angle_rad) l0 = amp * np.sin(angle_rad) return k0, l0 -def wavemodel(XX, YY, ks, ls, amps, group_phase = 0): + +def wavemodel(XX, YY, ks, ls, amps, group_phase=0): import numpy as np - G = np.vstack([ np.cos(np.outer(XX, ks) + np.outer(YY, ls) ).T , np.sin(np.outer(XX, ks) + np.outer(YY, ls) ).T ] ).T + G = np.vstack( + [ + np.cos(np.outer(XX, ks) + np.outer(YY, ls)).T, + np.sin(np.outer(XX, ks) + np.outer(YY, ls)).T, + ] + ).T - #phase1 = np.random.rand(1, amp_list.size) * np.pi*2 - #phase = np.arange(0, amp_list.size) * np.pi/2 + # phase1 = np.random.rand(1, amp_list.size) * np.pi*2 + # phase = np.arange(0, amp_list.size) * np.pi/2 - b = np.hstack([ np.cos(group_phase) * amps , np.sin(group_phase) * amps ]).squeeze() - z_model = (G @ b) + b = np.hstack([np.cos(group_phase) * amps, np.sin(group_phase) * amps]).squeeze() + z_model = G @ b return z_model + # %% -def get_z_model(x_positions, y_position, K_prime, K_amp, alpha_rad, group_phase): - K_abs = K_prime / np.cos(alpha_rad) +def get_z_model(x_positions, y_position, K_prime, K_amp, alpha_rad, group_phase): + + K_abs = K_prime / np.cos(alpha_rad) k = K_abs * np.cos(alpha_rad) l = K_abs * np.sin(alpha_rad) - return wavemodel( x_positions,y_position, k, l, np.array(K_amp ), group_phase= group_phase) + return wavemodel( + x_positions, y_position, k, l, np.array(K_amp), group_phase=group_phase + ) +def objective_func(pars, x, y, z, test_flag=False, alpha_prior=None): -def objective_func(pars, x, y, z, test_flag= False , alpha_prior= None ): - - z_model = get_z_model(x, y, pars['K_prime'], pars['K_amp'], pars['alpha'],pars['group_phase']) + z_model = get_z_model( + x, y, pars["K_prime"], pars["K_amp"], pars["alpha"], pars["group_phase"] + ) if alpha_prior is not None: - penalties = np.array([ abs(alpha_prior - pars['alpha']) / (5 * np.pi/ 180) ]) + penalties = np.array([abs(alpha_prior - pars["alpha"]) / (5 * np.pi / 180)]) else: - penalties = np.array([0]) + penalties = np.array([0]) - cost =( abs(z - z_model) )**2 /z.std()**2 + cost = (abs(z - z_model)) ** 2 / z.std() ** 2 if test_flag: return z_model else: - return np.concatenate([cost , 2 * penalties]) + return np.concatenate([cost, 2 * penalties]) + + +# objective_func(fitter.params, *fitting_args , test_flag= False, alpha_prior = -np.pi/4) -#objective_func(fitter.params, *fitting_args , test_flag= False, alpha_prior = -np.pi/4) def plot_brute_force(fitter_brute): clevel = np.linspace(0.4, 1.2, 30) - plt.contourf(fitter_brute.brute_grid[1,:,:], fitter_brute.brute_grid[0,:,:], fitter_brute.brute_Jout/fitter_brute.brute_Jout.mean() , clevel, cmap= plt.cm.YlGnBu_r ) + plt.contourf( + fitter_brute.brute_grid[1, :, :], + fitter_brute.brute_grid[0, :, :], + fitter_brute.brute_Jout / fitter_brute.brute_Jout.mean(), + clevel, + cmap=plt.cm.YlGnBu_r, + ) plt.colorbar() - plt.scatter(fitter_brute.brute_grid[1,:,:], fitter_brute.brute_grid[0,:,:], s=0.2, alpha= 0.4, color='black') - plt.plot(fitter_brute.brute_x0[1], fitter_brute.brute_x0[0], '+r', markersize=20, label= 'Brute force') - - plt.xlabel('Phase (rad)') - plt.ylabel('Angle (rad)') + plt.scatter( + fitter_brute.brute_grid[1, :, :], + fitter_brute.brute_grid[0, :, :], + s=0.2, + alpha=0.4, + color="black", + ) + plt.plot( + fitter_brute.brute_x0[1], + fitter_brute.brute_x0[0], + "+r", + markersize=20, + label="Brute force", + ) + + plt.xlabel("Phase (rad)") + plt.ylabel("Angle (rad)") plt.legend() -def plot_instance(GG3, data_key, model_key, non_dim=False , view_scale = 0.3, fitter=None, title_str= None): + +def plot_instance( + GG3, data_key, model_key, non_dim=False, view_scale=0.3, fitter=None, title_str=None +): import itertools - F = M.figure_axis_xy(5,6, view_scale = view_scale, container = True) + + F = M.figure_axis_xy(5, 6, view_scale=view_scale, container=True) plt.suptitle(title_str) - gs = GridSpec(4, 2, wspace=0.3, hspace=1.2)#figure=fig, + gs = GridSpec(4, 2, wspace=0.3, hspace=1.2) # figure=fig, - #beam_list = list(set(GG3['beam'])) + # beam_list = list(set(GG3['beam'])) beam_list = [0, 300] - col_list = itertools.cycle([col.cascade2, col.rascade2, col.cascade1, col.rascade1, col.cascade3, col.rascade3]) - + col_list = itertools.cycle( + [ + col.cascade2, + col.rascade2, + col.cascade1, + col.rascade1, + col.cascade3, + col.rascade3, + ] + ) subz = len(beam_list) - for beam, pos in zip(beam_list, [ gs[0, :] , gs[1, :] ] ): + for beam, pos in zip(beam_list, [gs[0, :], gs[1, :]]): F.ax2 = F.fig.add_subplot(pos) - plt.title( 'y=' + str(beam) , loc='left') - #plt.plot(GG['x_prime'], GG['z_model'], '.' ) + plt.title("y=" + str(beam), loc="left") + # plt.plot(GG['x_prime'], GG['z_model'], '.' ) - GGsel =GG3[GG3['y_prime'] == beam] - plt.plot(GGsel['dist'], GGsel[model_key], c=col.gray, linewidth = 1) + GGsel = GG3[GG3["y_prime"] == beam] + plt.plot(GGsel["dist"], GGsel[model_key], c=col.gray, linewidth=1) if non_dim: # normalized version - plt.plot(GGsel['dist'], GGsel['slopes']/GGsel['slopes'].std() , '-', linewidth = 0.5, c=next(col_list)) + plt.plot( + GGsel["dist"], + GGsel["slopes"] / GGsel["slopes"].std(), + "-", + linewidth=0.5, + c=next(col_list), + ) else: # diumensional version - plt.plot(GGsel['dist'], GGsel[data_key] , '-', c=next(col_list), linewidth = 0.5) - plt.ylabel('slope (m/m)') + plt.plot( + GGsel["dist"], GGsel[data_key], "-", c=next(col_list), linewidth=0.5 + ) + plt.ylabel("slope (m/m)") - plt.xlabel('meter') + plt.xlabel("meter") F.ax3 = F.fig.add_subplot(gs[2:, :]) if fitter is not None: - plt.title('Brute-force costs', loc='left') + plt.title("Brute-force costs", loc="left") plot_brute_force(fitter) return F -#plot_instance(GG, 'slopes', 'z_model' , view_scale= 0.7, fitter= fitter_brute, title_str='sdgjfds') + +# plot_instance(GG, 'slopes', 'z_model' , view_scale= 0.7, fitter= fitter_brute, title_str='sdgjfds') # %% # dimensional version N = GG.shape[0] -amp_Z = 2* abs(Z_max)**2 /N +amp_Z = 2 * abs(Z_max) ** 2 / N import lmfit as LM + params0 = LM.Parameters() -params0.add('alpha', 0 , vary=True , min=-0.95 * np.pi /2, max=0.95 * np.pi /2) -params0.add('group_phase', 0 , vary=True , min=0, max= 2*np.pi) +params0.add("alpha", 0, vary=True, min=-0.95 * np.pi / 2, max=0.95 * np.pi / 2) +params0.add("group_phase", 0, vary=True, min=0, max=2 * np.pi) # test: this should be 0 -np.isnan(GG['slopes']).sum() -fitting_args = (GG['x_prime'],GG['y_prime'], GG['slopes']/GG['slopes'].std(), ) -fitting_kargs = {'alpha_prior': None} - -N_data = GG['x_prime'].size -angle_list = list() -#fitting_kargs = {'size' :1} -L = pd.DataFrame(index=['alpha', 'group_phase', 'K_prime', 'K_amp'] ) +np.isnan(GG["slopes"]).sum() +fitting_args = ( + GG["x_prime"], + GG["y_prime"], + GG["slopes"] / GG["slopes"].std(), +) +fitting_kargs = {"alpha_prior": None} + +N_data = GG["x_prime"].size +angle_list = list() +# fitting_kargs = {'size' :1} +L = pd.DataFrame(index=["alpha", "group_phase", "K_prime", "K_amp"]) Z_indexes = np.argsort(Z_max_list_gauss)[::-1] cost_stack = dict() -#for k_prime_max,Z_max in zip(k_list_gauss[Z_indexes],Z_max_list_gauss[Z_indexes] ): +# for k_prime_max,Z_max in zip(k_list_gauss[Z_indexes],Z_max_list_gauss[Z_indexes] ): N_grid = 90 -#for k_prime_max,Z_max in zip(k_fft_max_list,Z_max_list): -for k_prime_max,Z_max in zip(k_list_interp , Z_list_interp): +# for k_prime_max,Z_max in zip(k_fft_max_list,Z_max_list): +for k_prime_max, Z_max in zip(k_list_interp, Z_list_interp): print(k_prime_max) amp_enhancement = 1 - amp_Z = 1 #amp_enhancement * abs(Z_max)**2 /N + amp_Z = 1 # amp_enhancement * abs(Z_max)**2 /N params = params0.copy() - params.add('K_prime', k_prime_max , vary=False , min=k_prime_max*0.5, max=k_prime_max*1.5) - params.add('K_amp', amp_Z , vary=False , min=amp_Z*.5 , max=amp_Z*5) - - #fitter = LM.minimize(objective_func, params, args=fitting_args, method='dual_annealing',max_nfev=None) - fitter_brute = LM.minimize(objective_func, params, \ - args=fitting_args,kws=fitting_kargs , method='brute', Ns=N_grid, ) - fitter = LM.minimize(objective_func, fitter_brute.params, \ - args=fitting_args,kws=fitting_kargs , method='differential_evolution',max_nfev=None) - - GG['z_model'] = objective_func(fitter.params, *fitting_args , test_flag= True) - angle_list.append(fitter.params['alpha'].value) - #fitting_kargs = {'alpha_prior': np.mean(angle_list )} - fitting_kargs = {'alpha_prior': None} - F = plot_instance(GG, 'slopes', 'z_model', fitter = fitter_brute, view_scale = 0.5, non_dim=True ) - - plt.plot(fitter.params['group_phase'].value, fitter.params['alpha'].value, '.r', markersize=20) + params.add( + "K_prime", k_prime_max, vary=False, min=k_prime_max * 0.5, max=k_prime_max * 1.5 + ) + params.add("K_amp", amp_Z, vary=False, min=amp_Z * 0.5, max=amp_Z * 5) + + # fitter = LM.minimize(objective_func, params, args=fitting_args, method='dual_annealing',max_nfev=None) + fitter_brute = LM.minimize( + objective_func, + params, + args=fitting_args, + kws=fitting_kargs, + method="brute", + Ns=N_grid, + ) + fitter = LM.minimize( + objective_func, + fitter_brute.params, + args=fitting_args, + kws=fitting_kargs, + method="differential_evolution", + max_nfev=None, + ) + + GG["z_model"] = objective_func(fitter.params, *fitting_args, test_flag=True) + angle_list.append(fitter.params["alpha"].value) + # fitting_kargs = {'alpha_prior': np.mean(angle_list )} + fitting_kargs = {"alpha_prior": None} + F = plot_instance( + GG, "slopes", "z_model", fitter=fitter_brute, view_scale=0.5, non_dim=True + ) + + plt.plot( + fitter.params["group_phase"].value, + fitter.params["alpha"].value, + ".r", + markersize=20, + ) print(fitting_kargs) print(fitter.params.pretty_print()) plt.show() - cost_stack[k_prime_max] = xr.DataArray( fitter_brute.brute_Jout/N_data, dims= ('angle', 'phase'), coords = {'angle':np.linspace(-np.pi/2, np.pi/2, N_grid), 'phase':np.linspace(0, 2* np.pi, N_grid) } ) - cost_stack[k_prime_max].coords['k'] = np.array(k_prime_max) #( ('k'), np.array(k_prime_max) ) - - - L[k_prime_max] = fitter.params.valuesdict().values() + cost_stack[k_prime_max] = xr.DataArray( + fitter_brute.brute_Jout / N_data, + dims=("angle", "phase"), + coords={ + "angle": np.linspace(-np.pi / 2, np.pi / 2, N_grid), + "phase": np.linspace(0, 2 * np.pi, N_grid), + }, + ) + cost_stack[k_prime_max].coords["k"] = np.array( + k_prime_max + ) # ( ('k'), np.array(k_prime_max) ) - #F.save_light(path= plot_path, name = key_name + '_fit_k' + str(k_prime_max)) + L[k_prime_max] = fitter.params.valuesdict().values() -cost_stack = xr.concat(cost_stack.values(), dim='k' ).sortby('k') + # F.save_light(path= plot_path, name = key_name + '_fit_k' + str(k_prime_max)) -L = L.T.sort_values('K_prime') +cost_stack = xr.concat(cost_stack.values(), dim="k").sortby("k") +L = L.T.sort_values("K_prime") # %% -cost_stack2 = cost_stack #/ cost_stack.mean('angle').mean('phase').mean() -cost_stack_rolled = xr.DataArray(coords = cost_stack.coords) +cost_stack2 = cost_stack # / cost_stack.mean('angle').mean('phase').mean() +cost_stack_rolled = xr.DataArray(coords=cost_stack.coords) shift_list = list() for kindex in L.index: - #.sel( phase= L[i]['group_phase'], method ='nearest').plot() + # .sel( phase= L[i]['group_phase'], method ='nearest').plot() ii = abs(cost_stack.k - kindex).argmin().data - shift = int( -abs(cost_stack.phase - L['group_phase'][kindex]).argmin().data + cost_stack.phase.size/2 ) - #shift = int(- abs(cost_stack.phase - L['group_phase'][kindex]).argmin().data ) + shift = int( + -abs(cost_stack.phase - L["group_phase"][kindex]).argmin().data + + cost_stack.phase.size / 2 + ) + # shift = int(- abs(cost_stack.phase - L['group_phase'][kindex]).argmin().data ) shift_list.append(shift) - cost_stack_rolled[ii, :,:]= cost_stack.sel(k= kindex).roll(phase = shift, roll_coords='phase').data + cost_stack_rolled[ii, :, :] = ( + cost_stack.sel(k=kindex).roll(phase=shift, roll_coords="phase").data + ) # M.figure_axis_xy(7, 3, view_scale= 0.5) # plt.subplot(1, 2,1) @@ -402,123 +518,154 @@ def plot_instance(GG3, data_key, model_key, non_dim=False , view_scale = 0.3, fi # cost_stack_rolled[ii, :,:].plot(cmap =plt.cm.Blues_r) # plt.show() -cost_stack_rolled['phase'] = cost_stack_rolled['phase'] - np.pi +cost_stack_rolled["phase"] = cost_stack_rolled["phase"] - np.pi -#weights = xr.DataArray(Z_max_list_gauss, dims ='k', coords = {'k': k_list_gauss}) -weights = xr.DataArray(Z_max_list, dims ='k', coords = {'k': k_fft_max_list}) -weights2 = 1/ cost_stack_rolled.sel(phase=0, method = 'nearest').min('angle') -#plt.plot(weights) -data_normed = cost_stack_rolled #/cost_stack_rolled.std(['angle', 'phase']) -cost_wmean = (weights * data_normed /weights.sum() ).sum('k') +# weights = xr.DataArray(Z_max_list_gauss, dims ='k', coords = {'k': k_list_gauss}) +weights = xr.DataArray(Z_max_list, dims="k", coords={"k": k_fft_max_list}) +weights2 = 1 / cost_stack_rolled.sel(phase=0, method="nearest").min("angle") +# plt.plot(weights) +data_normed = cost_stack_rolled # /cost_stack_rolled.std(['angle', 'phase']) +cost_wmean = (weights * data_normed / weights.sum()).sum("k") # %% cross correlation -A1 = GG[GG['y_prime'] == 0] -A2 = GG[GG['y_prime'] == 300] +A1 = GG[GG["y_prime"] == 0] +A2 = GG[GG["y_prime"] == 300] + def autocrosscorr_func_1d(dd1, dd2): "takes data s as xarray" - #print(dim) - xp1=(dd1-dd1.mean()) - xp2=(dd2-dd2.mean()) - var=xp2.var() - corr=np.correlate(xp1,xp2,'full')[len(xp1)-1:]/var/len(xp1) + # print(dim) + xp1 = dd1 - dd1.mean() + xp2 = dd2 - dd2.mean() + var = xp2.var() + corr = np.correlate(xp1, xp2, "full")[len(xp1) - 1 :] / var / len(xp1) return corr -ymean = abs(A1['y_prime'].mean() - A2['y_prime'].mean()) -angle_rad = np.concatenate([ -np.arctan2(A2['x_prime'], ymean)[::-1], np.arctan2(A2['x_prime'], ymean) ]) -cross_corr = np.concatenate([autocrosscorr_func_1d(A1['slopes'][::-1], A2['slopes'][::-1])[::-1], autocrosscorr_func_1d(A1['slopes'], A2['slopes']) ]) -#plt.plot( angle_rad, cross_corr ) +ymean = abs(A1["y_prime"].mean() - A2["y_prime"].mean()) +angle_rad = np.concatenate( + [-np.arctan2(A2["x_prime"], ymean)[::-1], np.arctan2(A2["x_prime"], ymean)] +) +cross_corr = np.concatenate( + [ + autocrosscorr_func_1d(A1["slopes"][::-1], A2["slopes"][::-1])[::-1], + autocrosscorr_func_1d(A1["slopes"], A2["slopes"]), + ] +) + +# plt.plot( angle_rad, cross_corr ) # %% -F = M.figure_axis_xy(6, 6.5, view_scale= 0.5) +F = M.figure_axis_xy(6, 6.5, view_scale=0.5) ax = plt.subplot(3, 1, 1) -plt.title( 'Normalized weighted mean cost at maximum phase', loc='left') -dn_unweighted = data_normed.mean('k') +plt.title("Normalized weighted mean cost at maximum phase", loc="left") +dn_unweighted = data_normed.mean("k") + +plt.plot( + data_normed.angle * 180 / np.pi, + M.normalize(dn_unweighted.sel(phase=0, method="nearest")), + label="mean", +) +plt.plot( + cost_wmean.angle * 180 / np.pi, + M.normalize(cost_wmean.sel(phase=0, method="nearest")), + label="weighted mean", +) -plt.plot(data_normed.angle* 180/np.pi, M.normalize(dn_unweighted.sel(phase=0, method = 'nearest') ) , label='mean') -plt.plot(cost_wmean.angle* 180/np.pi , M.normalize( cost_wmean.sel(phase=0, method = 'nearest') ) , label='weighted mean' ) def get_min_values(field2d, x, y): min_pos = np.unravel_index(field2d.argmin(), field2d.shape) - return x[min_pos[0]] , y[min_pos[1]] + return x[min_pos[0]], y[min_pos[1]] + -angle_mean, phase_mean = get_min_values(cost_wmean,cost_wmean.angle, cost_wmean.phase ) +angle_mean, phase_mean = get_min_values(cost_wmean, cost_wmean.angle, cost_wmean.phase) phase_mean, angle_mean -angle_mean.data * 180 /np.pi +angle_mean.data * 180 / np.pi -ax.axvline(angle_true* 180 /np.pi, linewidth = 1, color = 'black', label ='True') +ax.axvline(angle_true * 180 / np.pi, linewidth=1, color="black", label="True") plt.legend() -plt.xlabel('Angle (deg)') -plt.ylabel('normalized cost') +plt.xlabel("Angle (deg)") +plt.ylabel("normalized cost") plt.xlim(-80, 80) ax = plt.subplot(3, 1, 2) -plt.title( 'minus Lagged cross-correlation', loc='left') -plt.plot( angle_rad* 180 /np.pi, - cross_corr ) +plt.title("minus Lagged cross-correlation", loc="left") +plt.plot(angle_rad * 180 / np.pi, -cross_corr) -ax.axvline(angle_true* 180 /np.pi, linewidth = 1, color = 'black', label ='True') +ax.axvline(angle_true * 180 / np.pi, linewidth=1, color="black", label="True") plt.legend() -plt.xlabel('Angle (deg)') -plt.ylabel('corr') +plt.xlabel("Angle (deg)") +plt.ylabel("corr") plt.xlim(-80, 80) ax = plt.subplot(3, 1, 3) -plt.plot( angle_rad* 180 /np.pi, 1/np.cos(angle_rad) , c='k' ) -ax.axvline(angle_true* 180 /np.pi, linewidth = 1, color = 'black', label ='True') +plt.plot(angle_rad * 180 / np.pi, 1 / np.cos(angle_rad), c="k") +ax.axvline(angle_true * 180 / np.pi, linewidth=1, color="black", label="True") -plt.title( 'Angle correction factor for wavenumber (1/cos($\\alpha$))', loc='left') -plt.xlabel('Angle (deg)') -plt.ylabel('k') +plt.title("Angle correction factor for wavenumber (1/cos($\\alpha$))", loc="left") +plt.xlabel("Angle (deg)") +plt.ylabel("k") plt.grid() plt.xlim(-80, 80) plt.ylim(0.9, 3) -F.save_light(path= plot_path, name = key_name + '_rolled_pahse_weighted_mean') +F.save_light(path=plot_path, name=key_name + "_rolled_pahse_weighted_mean") # %% -F = M.figure_axis_xy(5, 4.5, view_scale = 0.5) +F = M.figure_axis_xy(5, 4.5, view_scale=0.5) # L.T['alpha'].hist(bins = 60) # F.ax.axvline(alpha_true, linewidth = 0.8, color = 'black') -xlims= [L.index.min()* 0.7, L.index.max()* 1.6] +xlims = [L.index.min() * 0.7, L.index.max() * 1.6] ax0 = plt.subplot(2, 1, 1) -plt.plot( L.index, L['alpha'], '.r', markersize = 10 , zorder=12) +plt.plot(L.index, L["alpha"], ".r", markersize=10, zorder=12) -plt.text( np.mean(xlims), angle_true + 0.08, 'True $\\alpha$=' + str(np.round(angle_true, 2))) -ax0.axhline(angle_true, linewidth = 1, color = 'black') +plt.text( + np.mean(xlims), angle_true + 0.08, "True $\\alpha$=" + str(np.round(angle_true, 2)) +) +ax0.axhline(angle_true, linewidth=1, color="black") -ax0.axhline(angle_mean, linewidth = 1, color = 'blue') -plt.text( np.mean(xlims)/1.5, angle_mean.data - 0.2, 'mean $\\alpha$=' + str(np.round(angle_mean.data , 2))) +ax0.axhline(angle_mean, linewidth=1, color="blue") +plt.text( + np.mean(xlims) / 1.5, + angle_mean.data - 0.2, + "mean $\\alpha$=" + str(np.round(angle_mean.data, 2)), +) -for ang, aamp in zip(np.arctan2(ll, kk), kk ): - #ax0.axhline(ang, linewidth = 0.8,alpha=0.3, color = 'black') - plt.plot(aamp, ang, '.k', markersize= 9) +for ang, aamp in zip(np.arctan2(ll, kk), kk): + # ax0.axhline(ang, linewidth = 0.8,alpha=0.3, color = 'black') + plt.plot(aamp, ang, ".k", markersize=9) plt.xlim(xlims[0], xlims[1]) -plt.ylim(-np.pi/2, np.pi/2) +plt.ylim(-np.pi / 2, np.pi / 2) plt.subplot(2, 1, 2) -plt.plot(k_fft, abs(Z_fft)/abs(Z_fft).mean() , label = 'normalized Power') -plt.plot(k_fft, Z_fft_rm/Z_fft_rm.mean() , label = 'smoothed') -plt.plot(k_list_interp,k_list_interp * 0 + k_list_interp.mean(), '.r', markersize= 10, zorder=12) -#weights2 = 1/ cost_stack_rolled.sel(phase=0, method = 'nearest').min('angle') -(weights/Z_fft_rm.mean()).plot(label = 'weights') -#plt.plot(k_list_gauss,k_list_gauss * 0 +Z_max_list_gauss.mean(), '.r', markersize= 10) -#plt.plot(k_fft_max_list,Z_max_list, '.r', markersize= 10) +plt.plot(k_fft, abs(Z_fft) / abs(Z_fft).mean(), label="normalized Power") +plt.plot(k_fft, Z_fft_rm / Z_fft_rm.mean(), label="smoothed") +plt.plot( + k_list_interp, + k_list_interp * 0 + k_list_interp.mean(), + ".r", + markersize=10, + zorder=12, +) +# weights2 = 1/ cost_stack_rolled.sel(phase=0, method = 'nearest').min('angle') +(weights / Z_fft_rm.mean()).plot(label="weights") +# plt.plot(k_list_gauss,k_list_gauss * 0 +Z_max_list_gauss.mean(), '.r', markersize= 10) +# plt.plot(k_fft_max_list,Z_max_list, '.r', markersize= 10) plt.legend() plt.xlim(xlims[0], xlims[1]) -F.save_light(path= plot_path, name = key_name + 'single_results') +F.save_light(path=plot_path, name=key_name + "single_results") diff --git a/analysis/SB03_swell_phase_fitting_iter_freq.py b/analysis/SB03_swell_phase_fitting_iter_freq.py index 8fe42b39..c74b798a 100644 --- a/analysis/SB03_swell_phase_fitting_iter_freq.py +++ b/analysis/SB03_swell_phase_fitting_iter_freq.py @@ -1,16 +1,16 @@ - import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -22,90 +22,118 @@ import spicke_remover import datetime import concurrent.futures as futures -#import s3fs -# %% -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False -track_name, batch_key, test_flag = '20190601093502_09790310_004_01', 'SH_batch02', False - - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -save_path = mconfig['paths']['work'] + '/B03_spectra_'+hemis+'/' -save_name = 'B03_'+track_name - -plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/B_spectra/' +# import s3fs +# %% +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +track_name, batch_key, test_flag = "20190601093502_09790310_004_01", "SH_batch02", False + + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +save_path = mconfig["paths"]["work"] + "/B03_spectra_" + hemis + "/" +save_name = "B03_" + track_name + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/B_spectra/" +) MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -#Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +# Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data -load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # +load_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # col.colormaps2(21) # %% -T_max = 40 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 -x= np.array(Gd['gt1r']['dist']) +T_max = 40 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 +x = np.array(Gd["gt1r"]["dist"]) dx = np.diff(x).mean() -xlims = x[0], x[-1] -min_datapoint = 1/k_0/dx +xlims = x[0], x[-1] +min_datapoint = 1 / k_0 / dx Lpoints = int(np.round(min_datapoint) * 10) -Lmeters =Lpoints * dx +Lmeters = Lpoints * dx -def get_stacil_data(stancil, G3, key = 'dist' ): + +def get_stacil_data(stancil, G3, key="dist"): mask = (G3[key] >= stancil[0]) & (G3[key] < stancil[2]) return G3[np.array(mask)] -def make_slopes(G3, key = 'heights_c_weighted_mean', spreed =10, verbose= False, photon_min =5): - dd = np.copy(G3['heights_c_weighted_mean']) - dd = np.gradient(dd) - dd, _ = spicke_remover.spicke_remover(dd, spreed=spreed, verbose=verbose) - dd_nans = (np.isnan(dd) ) + (G3['N_photos'] <= photon_min) +def make_slopes( + G3, key="heights_c_weighted_mean", spreed=10, verbose=False, photon_min=5 +): + + dd = np.copy(G3["heights_c_weighted_mean"]) + dd = np.gradient(dd) + dd, _ = spicke_remover.spicke_remover(dd, spreed=spreed, verbose=verbose) + dd_nans = (np.isnan(dd)) + (G3["N_photos"] <= photon_min) return dd, dd_nans -stancil_iter = spec.create_chunk_boundaries_unit_lengths(Lmeters, xlims, ov= None , iter_flag=False) + +stancil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeters, xlims, ov=None, iter_flag=False +) # %% -sys.path.append(mconfig['paths']['analysis']) +sys.path.append(mconfig["paths"]["analysis"]) import SB04_2d_wavefield_emulator as WaveEmulator # test wave model -k_abs = 2 * np.pi/100 +k_abs = 2 * np.pi / 100 imp.reload(WaveEmulator) -kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar(k_abs,0, size=1, dk= k_abs/10, amp_std=k_abs/20 , plot_flag=False) +kk, ll, amps, mesh_shape = WaveEmulator.get_stancils_polar( + k_abs, 0, size=1, dk=k_abs / 10, amp_std=k_abs / 20, plot_flag=False +) -def wavemodel(XX, YY, ks, ls, amps, group_phase = 0, amp_height= 1): +def wavemodel(XX, YY, ks, ls, amps, group_phase=0, amp_height=1): import numpy as np - G = np.vstack([ np.cos(np.outer(XX, ks) + np.outer(YY, ls) ).T , np.sin(np.outer(XX, ks) + np.outer(YY, ls) ).T ] ).T + G = np.vstack( + [ + np.cos(np.outer(XX, ks) + np.outer(YY, ls)).T, + np.sin(np.outer(XX, ks) + np.outer(YY, ls)).T, + ] + ).T - #phase1 = np.random.rand(1, amp_list.size) * np.pi*2 - #phase = np.arange(0, amp_list.size) * np.pi/2 + # phase1 = np.random.rand(1, amp_list.size) * np.pi*2 + # phase = np.arange(0, amp_list.size) * np.pi/2 - b = np.hstack([ np.cos(group_phase)*amps, np.sin(group_phase) *amps]).squeeze() * amp_height - z_model = (G @ b) + b = ( + np.hstack([np.cos(group_phase) * amps, np.sin(group_phase) * amps]).squeeze() + * amp_height + ) + z_model = G @ b return z_model @@ -113,66 +141,70 @@ def wavemodel(XX, YY, ks, ls, amps, group_phase = 0, amp_height= 1): # %% select data Gd.keys() + def get_beam(G3, key): G1 = G3[key] - G1['slopes'], _ = make_slopes(G1) - G1['beam']= key + G1["slopes"], _ = make_slopes(G1) + G1["beam"] = key return G1 -B1= get_beam(Gd, 'gt1r') -B2= get_beam(Gd, 'gt1l') -dist_y_mean = np.sqrt( (B1['x']-B2['x'])**2 + (B1['y']-B2['y'])**2 ).mean() +B1 = get_beam(Gd, "gt1r") +B2 = get_beam(Gd, "gt1l") + +dist_y_mean = np.sqrt((B1["x"] - B2["x"]) ** 2 + (B1["y"] - B2["y"]) ** 2).mean() dist_y_mean -B1['dist_y'] = 0 -B2['dist_y'] = dist_y_mean +B1["dist_y"] = 0 +B2["dist_y"] = dist_y_mean -GG0 = pd.concat( [B1, B2] ) +GG0 = pd.concat([B1, B2]) # GG = get_beam(Gd, 'gt3r') -#GG0['slopes'].plot() -#plt.plot( GG0['dist'] , GG0['slopes']) +# GG0['slopes'].plot() +# plt.plot( GG0['dist'] , GG0['slopes']) -#plt.plot( GG0['x']/1e3 , GG0['y']/1e3) -plt.plot( GG0['x']/1e3 , GG0['y']/1e3) +# plt.plot( GG0['x']/1e3 , GG0['y']/1e3) +plt.plot(GG0["x"] / 1e3, GG0["y"] / 1e3) # %% -stancil = stancil_iter[:, stancil_iter[0,:] > 0.9 *1e6][:, 0] -#stancil = stancil_iter[:, 200] -#stancil_iter.shape -GG = get_stacil_data(stancil, GG0).sort_values(['beam','dist']) -GG = GG.loc[~np.isnan(GG['slopes'])] - -GG['x_prime'] = GG['x'] - GG['x'].min() -GG['y_prime'] = GG['y'] - GG['y'].max() -plt.plot(GG['dist'], GG['slopes'] , '-', markersize = 0.8) +stancil = stancil_iter[:, stancil_iter[0, :] > 0.9 * 1e6][:, 0] +# stancil = stancil_iter[:, 200] +# stancil_iter.shape +GG = get_stacil_data(stancil, GG0).sort_values(["beam", "dist"]) +GG = GG.loc[~np.isnan(GG["slopes"])] + +GG["x_prime"] = GG["x"] - GG["x"].min() +GG["y_prime"] = GG["y"] - GG["y"].max() +plt.plot(GG["dist"], GG["slopes"], "-", markersize=0.8) plt.show() -plt.plot(GG['x_prime']/1e3, GG['y_prime']/1e3) +plt.plot(GG["x_prime"] / 1e3, GG["y_prime"] / 1e3) plt.show() # plt.axis('equal') # %% get peak wavenumber, power and spreed + def get_fft(G4, beam, dx): - G4 = G4[G4['beam'] == beam] - ys = np.copy(G4['slopes']) - ys[np.isnan(ys)] =0 - k_fft = np.fft.rfftfreq(ys.size, d=dx) * 2* np.pi + G4 = G4[G4["beam"] == beam] + ys = np.copy(G4["slopes"]) + ys[np.isnan(ys)] = 0 + k_fft = np.fft.rfftfreq(ys.size, d=dx) * 2 * np.pi return np.fft.rfft(ys), k_fft -B1_fft, k_fft = get_fft(GG, 'gt1l', dx) -B2_fft, k_fft = get_fft(GG, 'gt1r', dx) -Z_fft = (B1_fft+ B2_fft)/2 -Z_fft_rm = M.runningmean(abs(Z_fft), 4, tailcopy=True) +B1_fft, k_fft = get_fft(GG, "gt1l", dx) +B2_fft, k_fft = get_fft(GG, "gt1r", dx) + +Z_fft = (B1_fft + B2_fft) / 2 +Z_fft_rm = M.runningmean(abs(Z_fft), 4, tailcopy=True) -k_fft = k_fft[~np.isnan(Z_fft_rm)] -Z_fft_rm = Z_fft_rm[~np.isnan(Z_fft_rm)] +k_fft = k_fft[~np.isnan(Z_fft_rm)] +Z_fft_rm = Z_fft_rm[~np.isnan(Z_fft_rm)] Z_max_pos = Z_fft_rm.argmax() -k_fft_max = k_fft[Z_max_pos] -Z_max = Z_fft_rm[Z_max_pos] +k_fft_max = k_fft[Z_max_pos] +Z_max = Z_fft_rm[Z_max_pos] # Z_sort = Z_fft_rm.argsort()[::-1] # k_fft[Z_sort].shape @@ -180,38 +212,44 @@ def get_fft(G4, beam, dx): # Z_fft_sel = Z_fft_rm[Z_sort][np.cumsum(Z_fft_rm[Z_sort])/np.sum(Z_fft_rm) <0.2] # # plt.plot(K_fft_sel, Z_fft_sel, '.') -plt.plot(k_fft, Z_fft_rm ) +plt.plot(k_fft, Z_fft_rm) # %% font_for_pres() -plt.plot(k_fft, abs(Z_fft)[:-1] ) -plt.plot(k_fft, Z_fft_rm ) -treshold = np.nanmean(Z_fft_rm) + np.nanstd(Z_fft_rm) *2 -k_fft_max_list = k_fft[Z_fft_rm > treshold] -Z_max_list = Z_fft_rm[Z_fft_rm > treshold] +plt.plot(k_fft, abs(Z_fft)[:-1]) +plt.plot(k_fft, Z_fft_rm) +treshold = np.nanmean(Z_fft_rm) + np.nanstd(Z_fft_rm) * 2 +k_fft_max_list = k_fft[Z_fft_rm > treshold] +Z_max_list = Z_fft_rm[Z_fft_rm > treshold] + +plt.plot(k_fft_max, Z_fft_rm[Z_max_pos], ".", markersize=20) +plt.plot(k_fft_max_list, Z_max_list, ".", markersize=20) +2 * np.pi / 0.15 -plt.plot(k_fft_max, Z_fft_rm[Z_max_pos], '.', markersize= 20) -plt.plot(k_fft_max_list, Z_max_list, '.', markersize= 20) -2*np.pi/.15 +# plt.plot(k_fft_max_list,Z_max_list, '.', markersize= 20) -#plt.plot(k_fft_max_list,Z_max_list, '.', markersize= 20) +def gaus(x, x_0, amp, sigma_g): + return amp * np.exp(-0.5 * ((x - x_0) / sigma_g) ** 2) -def gaus(x, x_0, amp, sigma_g ): - return amp* np.exp(-0.5 * ( (x-x_0)/sigma_g)**2) -k_sigma =0.02 -plt.plot(k_fft, gaus(k_fft, k_fft_max, Z_max , k_sigma ), 'r', linewidth = 2 ) +k_sigma = 0.02 +plt.plot(k_fft, gaus(k_fft, k_fft_max, Z_max, k_sigma), "r", linewidth=2) -k_list = np.arange(k_fft_max - 1* k_sigma, k_fft_max + 1* k_sigma+ np.diff(k_fft).mean(), np.diff(k_fft).mean()) -plt.plot(k_list, gaus(k_list, k_fft_max, Z_max , k_sigma ), 'b.', linewidth = 2 ) +k_list = np.arange( + k_fft_max - 1 * k_sigma, + k_fft_max + 1 * k_sigma + np.diff(k_fft).mean(), + np.diff(k_fft).mean(), +) +plt.plot(k_list, gaus(k_list, k_fft_max, Z_max, k_sigma), "b.", linewidth=2) -#plt.xlim(0, 0.09) +# plt.xlim(0, 0.09) # %% -def get_wavenumbers_polar( amp, angle_rad): + +def get_wavenumbers_polar(amp, angle_rad): """ inputs: @@ -222,88 +260,113 @@ def get_wavenumbers_polar( amp, angle_rad): wavenumber k,l """ import numpy as np + k0 = amp * np.cos(angle_rad) l0 = amp * np.sin(angle_rad) return k0, l0 -def wavemodel(XX, YY, ks, ls, amps, group_phase = 0): + +def wavemodel(XX, YY, ks, ls, amps, group_phase=0): import numpy as np - G = np.vstack([ np.cos(np.outer(XX, ks) + np.outer(YY, ls) ).T , np.sin(np.outer(XX, ks) + np.outer(YY, ls) ).T ] ).T + G = np.vstack( + [ + np.cos(np.outer(XX, ks) + np.outer(YY, ls)).T, + np.sin(np.outer(XX, ks) + np.outer(YY, ls)).T, + ] + ).T - #phase1 = np.random.rand(1, amp_list.size) * np.pi*2 - #phase = np.arange(0, amp_list.size) * np.pi/2 + # phase1 = np.random.rand(1, amp_list.size) * np.pi*2 + # phase = np.arange(0, amp_list.size) * np.pi/2 - b = np.hstack([ np.cos(group_phase) * amps , np.sin(group_phase) * amps ]).squeeze() - z_model = (G @ b) + b = np.hstack([np.cos(group_phase) * amps, np.sin(group_phase) * amps]).squeeze() + z_model = G @ b return z_model + # %% -def get_z_model(x_positions, y_position, K_prime, K_amp, alpha_rad, group_phase): - K_abs = K_prime / np.cos(alpha_rad) +def get_z_model(x_positions, y_position, K_prime, K_amp, alpha_rad, group_phase): + + K_abs = K_prime / np.cos(alpha_rad) k = K_abs * np.cos(alpha_rad) l = K_abs * np.sin(alpha_rad) - return wavemodel( x_positions,y_position, k, l, np.array(K_amp ), group_phase= group_phase) + return wavemodel( + x_positions, y_position, k, l, np.array(K_amp), group_phase=group_phase + ) +def objective_func(pars, x, y, z, test_flag=False): -def objective_func(pars, x, y, z, test_flag= False ): - - z_model = get_z_model(x, y, pars['K_prime'], pars['K_amp'], pars['alpha'],pars['group_phase']) - cost =( abs(z - z_model) )**2 /z.std()**2 + z_model = get_z_model( + x, y, pars["K_prime"], pars["K_amp"], pars["alpha"], pars["group_phase"] + ) + cost = (abs(z - z_model)) ** 2 / z.std() ** 2 if test_flag: return z_model else: return cost -def plot_instance(GG3, data_key, model_key, non_dim=False , view_scale = 0.3): +def plot_instance(GG3, data_key, model_key, non_dim=False, view_scale=0.3): import itertools - F = M.figure_axis_xy(8, 5, view_scale = view_scale) - beam_list = list(set(GG3['beam'])) + F = M.figure_axis_xy(8, 5, view_scale=view_scale) + beam_list = list(set(GG3["beam"])) - col_list = itertools.cycle([col.cascade2, col.rascade2, col.cascade1, col.rascade1, col.cascade3, col.rascade3]) + col_list = itertools.cycle( + [ + col.cascade2, + col.rascade2, + col.cascade1, + col.rascade1, + col.cascade3, + col.rascade3, + ] + ) subz = len(beam_list) - for beam, i in zip(beam_list, np.arange(1, subz+1, 1)): + for beam, i in zip(beam_list, np.arange(1, subz + 1, 1)): plt.subplot(subz, 1, i) - plt.title(beam ) - #plt.plot(GG['x_prime'], GG['z_model'], '.' ) + plt.title(beam) + # plt.plot(GG['x_prime'], GG['z_model'], '.' ) - GGsel =GG3[GG3['beam'] == beam] - plt.plot(GGsel['dist'], GGsel[model_key], c=col.gray) + GGsel = GG3[GG3["beam"] == beam] + plt.plot(GGsel["dist"], GGsel[model_key], c=col.gray) if non_dim: # normalized version - plt.plot(GGsel['dist'], GGsel['slopes']/GGsel['slopes'].std() , '-', c=next(col_list)) + plt.plot( + GGsel["dist"], + GGsel["slopes"] / GGsel["slopes"].std(), + "-", + c=next(col_list), + ) else: # diumensional version - plt.plot(GGsel['dist'], GGsel[data_key] , '-', c=next(col_list)) + plt.plot(GGsel["dist"], GGsel[data_key], "-", c=next(col_list)) return F - -#GG = pd.concat( [B1, B2] ) -#GG = pd.concat( [B1, B2] ) +# GG = pd.concat( [B1, B2] ) +# GG = pd.concat( [B1, B2] ) # stancil = stancil_iter[:, 200] # GG = get_stacil_data(stancil, GG).sort_values(['beam','dist']) # GG = GG.loc[~np.isnan(GG['slopes'])] # np.isnan(GG['slopes']).sum() -#GG['slopes'] = zm -#slopes1 = GG[GG['beam'] == 'gt3r']['slopes'] -#GG['slopes'] = np.concatenate([slopes1, slopes1]) +# GG['slopes'] = zm +# slopes1 = GG[GG['beam'] == 'gt3r']['slopes'] +# GG['slopes'] = np.concatenate([slopes1, slopes1]) # alpha_true = 75 * np.pi/180 # 0.95 * (np.pi/2) # dy = 60 # meters @@ -312,128 +375,153 @@ def plot_instance(GG3, data_key, model_key, non_dim=False , view_scale = 0.3): # %% # normalized version -#amp_Z = abs(Z_max)/np.sqrt(2)/2 +# amp_Z = abs(Z_max)/np.sqrt(2)/2 # dimensional version N = GG.shape[0] -amp_Z = 2* abs(Z_max)**2 /N +amp_Z = 2 * abs(Z_max) ** 2 / N import lmfit as LM + params0 = LM.Parameters() -params0.add('alpha', 0 , vary=True , min=-0.95 * np.pi /2, max=0.95 * np.pi /2) -params0.add('group_phase', 0 , vary=True , min=0, max= 2*np.pi) +params0.add("alpha", 0, vary=True, min=-0.95 * np.pi / 2, max=0.95 * np.pi / 2) +params0.add("group_phase", 0, vary=True, min=0, max=2 * np.pi) # test: this should be 0 -np.isnan(GG['slopes']).sum() -fitting_args = (GG['dist'],GG['dist_y'], GG['slopes']) -#fitting_kargs = {'size' :1} -L = pd.DataFrame(index=['alpha', 'group_phase', 'K_prime', 'K_amp'] ) +np.isnan(GG["slopes"]).sum() +fitting_args = (GG["dist"], GG["dist_y"], GG["slopes"]) +# fitting_kargs = {'size' :1} +L = pd.DataFrame(index=["alpha", "group_phase", "K_prime", "K_amp"]) -for k_prime_max,Z_max in zip(k_list, gaus(k_list, k_fft_max, Z_max , k_sigma )): - #for k_prime_max,Z_max in zip(k_fft_max_list,Z_max_list): +for k_prime_max, Z_max in zip(k_list, gaus(k_list, k_fft_max, Z_max, k_sigma)): + # for k_prime_max,Z_max in zip(k_fft_max_list,Z_max_list): print(k_prime_max) amp_enhancement = 4 - amp_Z = amp_enhancement * abs(Z_max)**2 /N + amp_Z = amp_enhancement * abs(Z_max) ** 2 / N params = params0.copy() - params.add('K_prime', k_prime_max , vary=False , min=k_prime_max*0.5, max=k_prime_max*1.5) - params.add('K_amp', amp_Z , vary=False , min=amp_Z*.5 , max=amp_Z*5) - - fitter = LM.minimize(objective_func, params, args=fitting_args, method='dual_annealing',max_nfev=None) - #fitter = LM.minimize(objective_func, params, args=fitting_args, method='brute', Ns=120, ) - GG['z_model'] = objective_func(fitter.params, GG['dist'],GG['dist_y'], GG['slopes'] , test_flag= True) - plot_instance(GG, 'slopes', 'z_model' ) + params.add( + "K_prime", k_prime_max, vary=False, min=k_prime_max * 0.5, max=k_prime_max * 1.5 + ) + params.add("K_amp", amp_Z, vary=False, min=amp_Z * 0.5, max=amp_Z * 5) + + fitter = LM.minimize( + objective_func, + params, + args=fitting_args, + method="dual_annealing", + max_nfev=None, + ) + # fitter = LM.minimize(objective_func, params, args=fitting_args, method='brute', Ns=120, ) + GG["z_model"] = objective_func( + fitter.params, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=True + ) + plot_instance(GG, "slopes", "z_model") print(fitter.params.pretty_print()) plt.show() - L[k_prime_max] = fitter.params.valuesdict().values() + L[k_prime_max] = fitter.params.valuesdict().values() # %% - -F = M.figure_axis_xy(5, 4.5, view_scale = 0.5) +F = M.figure_axis_xy(5, 4.5, view_scale=0.5) # L.T['alpha'].hist(bins = 60) # F.ax.axvline(alpha_true, linewidth = 0.8, color = 'black') -xlims= [L.T.index.min()* 0.9, L.T.index.max()* 1.1] +xlims = [L.T.index.min() * 0.9, L.T.index.max() * 1.1] ax0 = plt.subplot(2, 1, 1) -plt.plot( L.T.index, L.T['alpha'], '.r', markersize = 10) -#ax0.axhline(alpha_true, linewidth = 0.8, color = 'black') +plt.plot(L.T.index, L.T["alpha"], ".r", markersize=10) +# ax0.axhline(alpha_true, linewidth = 0.8, color = 'black') plt.xlim(xlims[0], xlims[1]) plt.subplot(2, 1, 2) -plt.plot(k_fft, Z_fft_rm ) -#plt.plot(k_fft_max, Z_fft_rm[Z_max_pos], '.', markersize= 20) -plt.plot(k_fft_max_list,Z_max_list, '.r', markersize= 10) +plt.plot(k_fft, Z_fft_rm) +# plt.plot(k_fft_max, Z_fft_rm[Z_max_pos], '.', markersize= 20) +plt.plot(k_fft_max_list, Z_max_list, ".r", markersize=10) plt.xlim(xlims[0], xlims[1]) # %% construct spectrum L -amp_Z = 2* abs(Z_max)**2 /N - -GG['z_model'] = get_z_model(GG['dist'],GG['dist_y'], L.T['K_prime'], L.T['K_amp']/amp_enhancement, L.T['alpha'],L.T['group_phase']) -plot_instance(GG, 'slopes', 'z_model', view_scale = 0.8) +amp_Z = 2 * abs(Z_max) ** 2 / N + +GG["z_model"] = get_z_model( + GG["dist"], + GG["dist_y"], + L.T["K_prime"], + L.T["K_amp"] / amp_enhancement, + L.T["alpha"], + L.T["group_phase"], +) +plot_instance(GG, "slopes", "z_model", view_scale=0.8) # %% params_big = LM.Parameters() -L2 = L.T.sort_values('K_prime') +L2 = L.T.sort_values("K_prime") N_wavenumbers = L2.shape[0] -L2['num'] = np.arange(0, L2.shape[0]) +L2["num"] = np.arange(0, L2.shape[0]) for p in L2.T.items(): - numi= str(int(p[1]['num'])) - ki = p[1]['K_prime'] - params_big.add('K_prime_' + numi , ki , vary=False , min=ki*0.5 , max=ki*1.5) + numi = str(int(p[1]["num"])) + ki = p[1]["K_prime"] + params_big.add("K_prime_" + numi, ki, vary=False, min=ki * 0.5, max=ki * 1.5) for p in L2.T.items(): - numi= str(int(p[1]['num'])) - ampi = p[1]['K_amp']/amp_enhancement - params_big.add('K_amp_' + numi , ampi , vary=False , min=ampi*.5 , max=ampi*5) + numi = str(int(p[1]["num"])) + ampi = p[1]["K_amp"] / amp_enhancement + params_big.add("K_amp_" + numi, ampi, vary=False, min=ampi * 0.5, max=ampi * 5) for p in L2.T.items(): - numi= str(int(p[1]['num'])) - grpi = p[1]['group_phase'] - params_big.add('group_phase_' + numi , grpi , vary=False , min=0 , max= 2*np.pi) + numi = str(int(p[1]["num"])) + grpi = p[1]["group_phase"] + params_big.add("group_phase_" + numi, grpi, vary=False, min=0, max=2 * np.pi) for p in L2.T.items(): - numi= str(int(p[1]['num'])) - grpi = p[1]['alpha'] - params_big.add('alpha_' + numi , grpi , vary=False , min=-0.95 * np.pi /2, max=0.95 * np.pi /2) + numi = str(int(p[1]["num"])) + grpi = p[1]["alpha"] + params_big.add( + "alpha_" + numi, grpi, vary=False, min=-0.95 * np.pi / 2, max=0.95 * np.pi / 2 + ) # def get_max_angle(a): # hist = np.histogram(a, np.linspace(- np.pi/2, np.pi/2 , int(np.round(a.size/2)) ) ) # return hist[1][hist[0].argmax()] # # -params_big.add('angle_shift', 0 , vary=False )#, min=-0.95 * np.pi /2, max=0.95 * np.pi /2) +params_big.add( + "angle_shift", 0, vary=False +) # , min=-0.95 * np.pi /2, max=0.95 * np.pi /2) # %% -#params_big - -def objective_func_big(params_big, x, y, z, test_flag= False ): - - pardict_val = np.array(list(params_big.valuesdict().values())) - K_prime_array = pardict_val[0:N_wavenumbers] - K_amp_array = pardict_val[N_wavenumbers:N_wavenumbers*2] - group_phase_array = pardict_val[N_wavenumbers*2:N_wavenumbers*3] - #alpha = pardict_val[-1] - alpha = pardict_val[N_wavenumbers*3:N_wavenumbers*4] + pardict_val[-1] - - #print(pardict_val[-1]) - #print(alpha) - z_model = get_z_model(GG['dist'],GG['dist_y'], K_prime_array, K_amp_array, alpha, group_phase_array) - cost = ( abs(z - z_model) )**2 /z.std()**2 - #alpha_cost = (alpha - alpha.mean())**2 / (2* np.pi/180)**2 +# params_big + + +def objective_func_big(params_big, x, y, z, test_flag=False): + + pardict_val = np.array(list(params_big.valuesdict().values())) + K_prime_array = pardict_val[0:N_wavenumbers] + K_amp_array = pardict_val[N_wavenumbers : N_wavenumbers * 2] + group_phase_array = pardict_val[N_wavenumbers * 2 : N_wavenumbers * 3] + # alpha = pardict_val[-1] + alpha = pardict_val[N_wavenumbers * 3 : N_wavenumbers * 4] + pardict_val[-1] + + # print(pardict_val[-1]) + # print(alpha) + z_model = get_z_model( + GG["dist"], GG["dist_y"], K_prime_array, K_amp_array, alpha, group_phase_array + ) + cost = (abs(z - z_model)) ** 2 / z.std() ** 2 + # alpha_cost = (alpha - alpha.mean())**2 / (2* np.pi/180)**2 if test_flag: return z_model else: return cost + def turn_vary_key_on(pard, key): for pi in pard.keys(): @@ -444,57 +532,83 @@ def turn_vary_key_on(pard, key): return pard -GG['z_model'] = objective_func_big(params_big, GG['dist'], GG['dist_y'], GG['slopes'], test_flag= True) -plot_instance(GG, 'slopes', 'z_model', view_scale = 0.6) +GG["z_model"] = objective_func_big( + params_big, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=True +) +plot_instance(GG, "slopes", "z_model", view_scale=0.6) # %% -#help(LM.minimize) +# help(LM.minimize) # params_big = turn_vary_key_on(params_big, 'group_phase') # fitter = LM.minimize(objective_func_big, params_big, args=fitting_args, method='ampgo',max_nfev=10000) -#fitter - -params2 = turn_vary_key_on(params_big, 'K_prime') -fitter = LM.minimize(objective_func_big, params2, args=fitting_args, method='least_squares',max_nfev=10000) - -params2 = turn_vary_key_on(fitter.params, 'K_amp') -fitter = LM.minimize(objective_func_big, params2, args=fitting_args, method='least_squares',max_nfev=10000) - -#print('angle_shift') +# fitter + +params2 = turn_vary_key_on(params_big, "K_prime") +fitter = LM.minimize( + objective_func_big, + params2, + args=fitting_args, + method="least_squares", + max_nfev=10000, +) + +params2 = turn_vary_key_on(fitter.params, "K_amp") +fitter = LM.minimize( + objective_func_big, + params2, + args=fitting_args, + method="least_squares", + max_nfev=10000, +) + +# print('angle_shift') # params2 = turn_vary_key_on(fitter.params, 'angle_shift') # fitter = LM.minimize(objective_func_big, params2, args=fitting_args, method='least_squares',max_nfev=10000) -#fitter = LM.minimize(objective_func_big, fitter.params, args=fitting_args, method='emcee') +# fitter = LM.minimize(objective_func_big, fitter.params, args=fitting_args, method='emcee') # %% -GG['z_model'] = objective_func_big(fitter.params, GG['dist'],GG['dist_y'], GG['slopes'], test_flag= True) -plot_instance(GG, 'slopes', 'z_model', view_scale = 0.6) +GG["z_model"] = objective_func_big( + fitter.params, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=True +) +plot_instance(GG, "slopes", "z_model", view_scale=0.6) # %% -z_mdoel_init = objective_func_big(params_big, GG['dist'],GG['dist_y'], GG['slopes'], test_flag= True) -objective_func_big(params_big, GG['dist'],GG['dist_y'], GG['slopes'], test_flag= False).sum() -z_mdoel_fit = objective_func_big(fitter.params, GG['dist'],GG['dist_y'], GG['slopes'], test_flag= True) -objective_func_big(fitter.params, GG['dist'],GG['dist_y'], GG['slopes'], test_flag= False).sum() +z_mdoel_init = objective_func_big( + params_big, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=True +) +objective_func_big( + params_big, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=False +).sum() +z_mdoel_fit = objective_func_big( + fitter.params, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=True +) +objective_func_big( + fitter.params, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=False +).sum() plt.plot(z_mdoel_init) plt.plot(z_mdoel_fit) # %% -#fitter.params.valuesdict() +# fitter.params.valuesdict() + def params_to_Dataframe(pard): - pardict_val = np.array(list(pard.valuesdict().values())) + pardict_val = np.array(list(pard.valuesdict().values())) pardict = dict() - pardict['K_prime_array'] = pardict_val[0:N_wavenumbers] - pardict['K_amp_array'] = pardict_val[N_wavenumbers:N_wavenumbers*2] - pardict['group_phase_array'] = pardict_val[N_wavenumbers*2:N_wavenumbers*3] - pardict['alpha'] = pardict_val[N_wavenumbers*3:N_wavenumbers*4] + pardict["K_prime_array"] = pardict_val[0:N_wavenumbers] + pardict["K_amp_array"] = pardict_val[N_wavenumbers : N_wavenumbers * 2] + pardict["group_phase_array"] = pardict_val[N_wavenumbers * 2 : N_wavenumbers * 3] + pardict["alpha"] = pardict_val[N_wavenumbers * 3 : N_wavenumbers * 4] return pd.DataFrame.from_dict(pardict) + Fits = params_to_Dataframe(fitter.params) -plt.plot( k_fft_max_list, abs( (Fits['alpha']) ), '.' ) +plt.plot(k_fft_max_list, abs((Fits["alpha"])), ".") -Fits['alpha'] +Fits["alpha"] plt.hist(fitter.flatchain, 30) diff --git a/analysis/SB03_swell_phase_fitting_iter_with_prior.py b/analysis/SB03_swell_phase_fitting_iter_with_prior.py index a781c331..6d0e8251 100644 --- a/analysis/SB03_swell_phase_fitting_iter_with_prior.py +++ b/analysis/SB03_swell_phase_fitting_iter_with_prior.py @@ -1,16 +1,16 @@ - import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -25,51 +25,62 @@ col.colormaps2(21) -#import s3fs +# import s3fs # %% -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190601093502_09790310_004_01', 'SH_batch02', False -track_name, batch_key, test_flag = '20190502021224_05160312_004_01', 'SH_batch02', False - - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -save_path = mconfig['paths']['work'] + '/B03_spectra_'+hemis+'/' -save_name = 'B03_'+track_name - -plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/B_spectra/' +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190601093502_09790310_004_01', 'SH_batch02', False +track_name, batch_key, test_flag = "20190502021224_05160312_004_01", "SH_batch02", False + + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +save_path = mconfig["paths"]["work"] + "/B03_spectra_" + hemis + "/" +save_name = "B03_" + track_name + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/B_spectra/" +) MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -beam_groups = mconfig['beams']['groups'] +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +beam_groups = mconfig["beams"]["groups"] -#Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data +# Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data -load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' -G_binned = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # +load_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" +G_binned = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # -load_path = mconfig['paths']['work'] +'/B02_spectra_'+hemis+'/' -Gx = xr.load_dataset(load_path+ '/B02_'+track_name + '_gFT_x.nc' ) # +load_path = mconfig["paths"]["work"] + "/B02_spectra_" + hemis + "/" +Gx = xr.load_dataset(load_path + "/B02_" + track_name + "_gFT_x.nc") # -Gk = xr.load_dataset(load_path+ '/B02_'+track_name + '_gFT_k.nc' ) # +Gk = xr.load_dataset(load_path + "/B02_" + track_name + "_gFT_k.nc") # # %% load prior information -load_path = mconfig['paths']['work'] +'/A02_prior_'+hemis+'/' -#track_name = '20190208104534_06410210_004_01' -Prior = MT.load_pandas_table_dict('/A02b_'+track_name, load_path)['priors_hindcast'] +load_path = mconfig["paths"]["work"] + "/A02_prior_" + hemis + "/" +# track_name = '20190208104534_06410210_004_01' +Prior = MT.load_pandas_table_dict("/A02b_" + track_name, load_path)["priors_hindcast"] # # import numpy as np @@ -87,8 +98,8 @@ # %% select data for b in Gx.beam: - B = Gx.sel(beam= b) - plt.plot(B['x_coord'], B['y_coord'], '.') + B = Gx.sel(beam=b) + plt.plot(B["x_coord"], B["y_coord"], ".") plt.grid() # %% @@ -106,20 +117,20 @@ # isolate case xi = 2 group = beam_groups[0] -GGx = Gx.sel(beam= group).isel(x = 2) -GGk = Gk.sel(beam= group).isel(x = 2) +GGx = Gx.sel(beam=group).isel(x=2) +GGk = Gk.sel(beam=group).isel(x=2) # %% -GGk.gFT_PSD_model.mean('beam').plot() +GGk.gFT_PSD_model.mean("beam").plot() # %% -data_plot_karg = {'linewidth':2, 'color':'k', 'alpha' :0.5} -model_plot_karg = {'linewidth':1, 'color':'r', 'alpha' :1} +data_plot_karg = {"linewidth": 2, "color": "k", "alpha": 0.5} +model_plot_karg = {"linewidth": 1, "color": "r", "alpha": 1} -G_x = GGx.isel(beam = 0) -y_offset= 0.5 -plt.plot(G_x.eta,G_x.y_model , **model_plot_karg) -plt.plot(G_x.eta,G_x.y_data, **data_plot_karg) +G_x = GGx.isel(beam=0) +y_offset = 0.5 +plt.plot(G_x.eta, G_x.y_model, **model_plot_karg) +plt.plot(G_x.eta, G_x.y_data, **data_plot_karg) plt.xlim(-500, 1000) @@ -129,8 +140,8 @@ # m = 3 # variance_frac = 0.5 -def define_wavenumber_weights_tot_var(dd, m = 3, variance_frac = 0.33, verbose=False): +def define_wavenumber_weights_tot_var(dd, m=3, variance_frac=0.33, verbose=False): """ return peaks of a power spectrum dd that in the format such that they can be used as weights for the frequencies based fitting @@ -147,65 +158,67 @@ def define_wavenumber_weights_tot_var(dd, m = 3, variance_frac = 0.33, verbose=F dd_rm smoothed version of dd positions postions where of significant data in array """ - dd_rm = M.runningmean(dd, m, tailcopy=True) - k = dd.k[~np.isnan(dd_rm)].data - dd_rm = dd_rm[~np.isnan(dd_rm)] + dd_rm = M.runningmean(dd, m, tailcopy=True) + k = dd.k[~np.isnan(dd_rm)].data + dd_rm = dd_rm[~np.isnan(dd_rm)] orders = dd_rm.argsort()[::-1] - var_mask = dd_rm[orders].cumsum()/dd_rm.sum() < variance_frac - + var_mask = dd_rm[orders].cumsum() / dd_rm.sum() < variance_frac pos_cumsum = orders[var_mask] - k_list = k[pos_cumsum] + k_list = k[pos_cumsum] dd_list = dd_rm[pos_cumsum] if verbose: - plt.plot(dd.k, dd, '-k', markersize= 20) - plt.plot(k, dd_rm, '-b', markersize= 20) + plt.plot(dd.k, dd, "-k", markersize=20) + plt.plot(k, dd_rm, "-b", markersize=20) - #print(k_list, dd_list) - plt.plot(k_list, dd_list, '.r', markersize= 10, zorder=12) + # print(k_list, dd_list) + plt.plot(k_list, dd_list, ".r", markersize=10, zorder=12) return var_mask[orders.argsort()], k, dd_rm, pos_cumsum -def define_wavenumber_weights_threshold(dd, m = 3, Nstd= 2, verbose=False): +def define_wavenumber_weights_threshold(dd, m=3, Nstd=2, verbose=False): - dd_rm = M.runningmean(dd, m, tailcopy=True) - k = dd.k[~np.isnan(dd_rm)] - dd_rm = dd_rm[~np.isnan(dd_rm)] - - treshold = np.nanmean(dd_rm) + np.nanstd(dd_rm) *Nstd - mask = dd_rm > treshold + dd_rm = M.runningmean(dd, m, tailcopy=True) + k = dd.k[~np.isnan(dd_rm)] + dd_rm = dd_rm[~np.isnan(dd_rm)] + treshold = np.nanmean(dd_rm) + np.nanstd(dd_rm) * Nstd + mask = dd_rm > treshold if verbose: - plt.plot(dd.k, dd, '-k', markersize= 20) - plt.plot(k, dd_rm, '-b', markersize= 20) + plt.plot(dd.k, dd, "-k", markersize=20) + plt.plot(k, dd_rm, "-b", markersize=20) - k_list = k[mask] - dd_list = dd_rm[mask] - #print(k_list, dd_list) - plt.plot(k_list, dd_list, '.r', markersize= 10, zorder=12) + k_list = k[mask] + dd_list = dd_rm[mask] + # print(k_list, dd_list) + plt.plot(k_list, dd_list, ".r", markersize=10, zorder=12) return mask, k, dd_rm, np.arange(0, mask.size)[mask] -#mask, k, weights, positions = define_wavenumber_weights_threshold( Gi.mean('dist_y')['gFT_PSD_data'], 3 , verbose= True) -#plt.plot(k[mask], weights[mask], 'g*', markersize=20) + +# mask, k, weights, positions = define_wavenumber_weights_threshold( Gi.mean('dist_y')['gFT_PSD_data'], 3 , verbose= True) +# plt.plot(k[mask], weights[mask], 'g*', markersize=20) # plt.show() -mask, k, weights, positions = define_wavenumber_weights_tot_var( GGk.mean('beam')['gFT_PSD_data'], m= 3, variance_frac = 0.33 , verbose= True) -#plt.xlim(k_list.min()*.9, k_list.max()*1.1) +mask, k, weights, positions = define_wavenumber_weights_tot_var( + GGk.mean("beam")["gFT_PSD_data"], m=3, variance_frac=0.33, verbose=True +) +# plt.xlim(k_list.min()*.9, k_list.max()*1.1) # group wavenumbers import scipy.ndimage -k_group_mask =scipy.ndimage.measurements.label(mask ) +k_group_mask = scipy.ndimage.measurements.label(mask) # %% -def get_wavenumbers_polar( amp, angle_rad): + +def get_wavenumbers_polar(amp, angle_rad): """ inputs: @@ -216,73 +229,98 @@ def get_wavenumbers_polar( amp, angle_rad): wavenumber k,l """ import numpy as np + k0 = amp * np.cos(angle_rad) l0 = amp * np.sin(angle_rad) return k0, l0 -def wavemodel(XX, YY, ks, ls, amps, group_phase = 0): + +def wavemodel(XX, YY, ks, ls, amps, group_phase=0): import numpy as np - G = np.vstack([ np.cos(np.outer(XX, ks) + np.outer(YY, ls) ).T , np.sin(np.outer(XX, ks) + np.outer(YY, ls) ).T ] ).T + G = np.vstack( + [ + np.cos(np.outer(XX, ks) + np.outer(YY, ls)).T, + np.sin(np.outer(XX, ks) + np.outer(YY, ls)).T, + ] + ).T - #phase1 = np.random.rand(1, amp_list.size) * np.pi*2 - #phase = np.arange(0, amp_list.size) * np.pi/2 + # phase1 = np.random.rand(1, amp_list.size) * np.pi*2 + # phase = np.arange(0, amp_list.size) * np.pi/2 - b = np.hstack([ np.cos(group_phase) * amps , np.sin(group_phase) * amps ]).squeeze() - z_model = (G @ b) + b = np.hstack([np.cos(group_phase) * amps, np.sin(group_phase) * amps]).squeeze() + z_model = G @ b return z_model -def get_z_model(x_positions, y_position, K_prime, K_amp, alpha_rad, group_phase): +def get_z_model(x_positions, y_position, K_prime, K_amp, alpha_rad, group_phase): - K_abs = K_prime / np.cos(alpha_rad) + K_abs = K_prime / np.cos(alpha_rad) k = K_abs * np.cos(alpha_rad) l = K_abs * np.sin(alpha_rad) - return wavemodel( x_positions,y_position, k, l, np.array(K_amp ), group_phase= group_phase) - - + return wavemodel( + x_positions, y_position, k, l, np.array(K_amp), group_phase=group_phase + ) -def plot_instance(z_model, fargs , key, non_dim=False, title_str= None ,fitter=None, view_scale = 0.3): +def plot_instance( + z_model, fargs, key, non_dim=False, title_str=None, fitter=None, view_scale=0.3 +): x_concat, y_concat, z_concat = fargs - - F = M.figure_axis_xy(5,6, view_scale = view_scale, container = True) + F = M.figure_axis_xy(5, 6, view_scale=view_scale, container=True) plt.suptitle(title_str) - gs = GridSpec(4, 2, wspace=0.3, hspace=1.2)#figure=fig, + gs = GridSpec(4, 2, wspace=0.3, hspace=1.2) # figure=fig, # y_offset= 0.5 # plt.plot(Gm.eta, Gm.y_model_normed+y_offset * Gm.dist_y/np.diff(Gm.dist_y), **model_plot_karg) # plt.plot(Gm.eta, Gm.y_model_normed+y_offset * Gm.dist_y/np.diff(Gm.dist_y), **data_plot_karg) # plt.xlim(-1000, 1000) - import itertools - col_list = itertools.cycle([col.cascade2, col.rascade2, col.cascade1, col.rascade1, col.cascade3, col.rascade3]) + + col_list = itertools.cycle( + [ + col.cascade2, + col.rascade2, + col.cascade1, + col.rascade1, + col.cascade3, + col.rascade3, + ] + ) beam_list = list(set(y_concat)) - for y_pos, pos in zip(beam_list, [ gs[0, :] , gs[1, :] ] ): + for y_pos, pos in zip(beam_list, [gs[0, :], gs[1, :]]): F.ax2 = F.fig.add_subplot(pos) - plt.title( str(y_pos) ) - #plt.plot(GG['x_prime'], GG['z_model'], '.' ) - - - plt.plot(x_concat[y_concat == y_pos], z_concat[y_concat == y_pos] , c=col.gray, linewidth = 1) - plt.plot(x_concat[y_concat == y_pos], z_model[y_concat == y_pos] , '-', c=next(col_list)) + plt.title(str(y_pos)) + # plt.plot(GG['x_prime'], GG['z_model'], '.' ) + + plt.plot( + x_concat[y_concat == y_pos], + z_concat[y_concat == y_pos], + c=col.gray, + linewidth=1, + ) + plt.plot( + x_concat[y_concat == y_pos], + z_model[y_concat == y_pos], + "-", + c=next(col_list), + ) plt.xlim(x_concat[y_concat == y_pos][0], x_concat[y_concat == y_pos][-1]) - - plt.xlabel('meter') + plt.xlabel("meter") F.ax3 = F.fig.add_subplot(gs[2:, :]) if fitter is not None: - plt.title('Brute-force costs', loc='left') + plt.title("Brute-force costs", loc="left") plot_brute_force(fitter) return F @@ -291,24 +329,39 @@ def plot_instance(z_model, fargs , key, non_dim=False, title_str= None ,fitter=N def plot_brute_force(fitter_brute): clevel = np.linspace(-2.2, 2.2, 30) - dd = (fitter_brute.brute_Jout- fitter_brute.brute_Jout.mean())/fitter_brute.brute_Jout.std() - plt.contourf(fitter_brute.brute_grid[1,:,:], fitter_brute.brute_grid[0,:,:], dd , clevel, cmap= plt.cm.YlGnBu_r ) + dd = ( + fitter_brute.brute_Jout - fitter_brute.brute_Jout.mean() + ) / fitter_brute.brute_Jout.std() + plt.contourf( + fitter_brute.brute_grid[1, :, :], + fitter_brute.brute_grid[0, :, :], + dd, + clevel, + cmap=plt.cm.YlGnBu_r, + ) plt.colorbar() - #plt.scatter(fitter_brute.brute_grid[1,:,:], fitter_brute.brute_grid[0,:,:], s=0.2, alpha= 0.4, color='black') - plt.plot(fitter_brute.brute_x0[1], fitter_brute.brute_x0[0], '+r', markersize=20, label= 'Brute force') - - plt.xlabel('Phase (rad)') - plt.ylabel('Angle (rad)') + # plt.scatter(fitter_brute.brute_grid[1,:,:], fitter_brute.brute_grid[0,:,:], s=0.2, alpha= 0.4, color='black') + plt.plot( + fitter_brute.brute_x0[1], + fitter_brute.brute_x0[0], + "+r", + markersize=20, + label="Brute force", + ) + + plt.xlabel("Phase (rad)") + plt.ylabel("Angle (rad)") plt.legend() - # %% normalize data -key = 'y_data' -amp_Z = G_x[key +'_normed']= (GGx[key] - GGx[key].mean(['eta']) )/GGx[key].std(['eta']) -N = G_x[key +'_normed'].shape[0] +key = "y_data" +amp_Z = G_x[key + "_normed"] = (GGx[key] - GGx[key].mean(["eta"])) / GGx[key].std( + ["eta"] +) +N = G_x[key + "_normed"].shape[0] eta_2d = GGx.eta + GGx.x_coord - GGx.x_coord.mean() nu_2d = GGx.eta * 0 + GGx.y_coord - GGx.y_coord.mean() @@ -326,7 +379,7 @@ def plot_brute_force(fitter_brute): # z_concat = np.concatenate([amp_Z.isel(dist_y= 0).data, amp_Z.isel(dist_y= 1).data]) # test -#plt.plot(x_concat, 1*y_concat +z_concat) +# plt.plot(x_concat, 1*y_concat +z_concat) # plt.plot(G_x_model.eta, G_x_model.y_data/G_x_model.y_data.std() +y_offset * G_x_model.dist_y/np.diff(G_x_model.dist_y), **data_plot_karg) # plt.xlim(-1000, 1000) @@ -335,46 +388,58 @@ def plot_brute_force(fitter_brute): # plt.plot(G_x_model.eta,G_x_model.y_data+y_offset * G_x_model.dist_y/np.diff(G_x_model.dist_y), **data_plot_karg) # plt.xlim(-1000, 1000) # test: this should be 0 -x_concat= x_concat[~np.isnan(z_concat)] -y_concat= y_concat[~np.isnan(z_concat)] -z_concat= z_concat[~np.isnan(z_concat)] +x_concat = x_concat[~np.isnan(z_concat)] +y_concat = y_concat[~np.isnan(z_concat)] +z_concat = z_concat[~np.isnan(z_concat)] if np.isnan(z_concat).sum() != 0: - raise ValueError('There are still nans') + raise ValueError("There are still nans") # %% -def objective_func(pars, x, y, z, test_flag= False , prior= None ): +def objective_func(pars, x, y, z, test_flag=False, prior=None): - z_model = get_z_model(x, y, pars['K_prime'], pars['K_amp'], pars['alpha'],pars['group_phase']) + z_model = get_z_model( + x, y, pars["K_prime"], pars["K_amp"], pars["alpha"], pars["group_phase"] + ) if prior is not None: - a_0, a_std = prior['alpha'] - penalties = np.array([ (abs(a_0 - pars['alpha'] )**2 / a_std) ]) + a_0, a_std = prior["alpha"] + penalties = np.array([(abs(a_0 - pars["alpha"]) ** 2 / a_std)]) else: - penalties = np.array([0]) + penalties = np.array([0]) - cost =( abs(z - z_model) )**2 /z.std()**2 + cost = (abs(z - z_model)) ** 2 / z.std() ** 2 if test_flag: return z_model else: - return np.concatenate([cost , 2 * penalties]) + return np.concatenate([cost, 2 * penalties]) import lmfit as LM -params0 = LM.Parameters() -d_alpha, d_phase= np.pi/90, 2 *np.pi/90 -alpha_grid = np.arange(-0.9 * np.pi /2, 0.9 * np.pi /2+ d_alpha, d_alpha) -phase_grid = np.arange(0 , 2*np.pi+d_phase, d_phase) +params0 = LM.Parameters() -params0.add('alpha', 0 , vary=True , min=alpha_grid[0], max=alpha_grid[-1], brute_step= d_alpha) -params0.add('group_phase', 0 , vary=True , min=phase_grid[0], max= phase_grid[-1]+d_phase, brute_step= d_phase) +d_alpha, d_phase = np.pi / 90, 2 * np.pi / 90 +alpha_grid = np.arange(-0.9 * np.pi / 2, 0.9 * np.pi / 2 + d_alpha, d_alpha) +phase_grid = np.arange(0, 2 * np.pi + d_phase, d_phase) + +params0.add( + "alpha", 0, vary=True, min=alpha_grid[0], max=alpha_grid[-1], brute_step=d_alpha +) +params0.add( + "group_phase", + 0, + vary=True, + min=phase_grid[0], + max=phase_grid[-1] + d_phase, + brute_step=d_phase, +) fitting_args = (x_concat, y_concat, z_concat) -Prior2 = Prior.loc[['ptp0','ptp1','ptp2','ptp3','ptp4','ptp5']]['mean'] +Prior2 = Prior.loc[["ptp0", "ptp1", "ptp2", "ptp3", "ptp4", "ptp5"]]["mean"] # dominat_period = Prior2[Prior2.max() ==Prior2] # dominant_dir = Prior.loc[list(dominat_period.index)[0].replace('ptp', 'pdp' )]['mean'] # dominant_dir_spread = Prior.loc[list(dominat_period.index)[0].replace('ptp', 'pspr' )]['mean'] @@ -384,101 +449,143 @@ def objective_func(pars, x, y, z, test_flag= False , prior= None ): import ICEsat2_SI_tools.wave_tools as waves -dominat_period = Prior2[Prior2.max() ==Prior2] -aa = Prior.loc[['pdp0','pdp1','pdp2','pdp3','pdp4','pdp5']]['mean'].astype('float') -dominant_dir = waves.get_ave_amp_angle(aa *0+1,aa )[1] -dominant_dir_spread = Prior.loc[['pspr0','pspr1','pspr2','pspr3','pspr4','pspr5']]['mean'].median() +dominat_period = Prior2[Prior2.max() == Prior2] +aa = Prior.loc[["pdp0", "pdp1", "pdp2", "pdp3", "pdp4", "pdp5"]]["mean"].astype("float") +dominant_dir = waves.get_ave_amp_angle(aa * 0 + 1, aa)[1] +dominant_dir_spread = Prior.loc[["pspr0", "pspr1", "pspr2", "pspr3", "pspr4", "pspr5"]][ + "mean" +].median() -#prior_sel= {'alpha': ( dominant_dir *np.pi/180 , dominant_dir_spread *np.pi/180) } # to radiens -prior_sel= {'alpha': ( 1 , dominant_dir_spread *np.pi/180) } # to radiens +# prior_sel= {'alpha': ( dominant_dir *np.pi/180 , dominant_dir_spread *np.pi/180) } # to radiens +prior_sel = {"alpha": (1, dominant_dir_spread * np.pi / 180)} # to radiens -#prior_sel= {'alpha': ( Prior.loc['dp']['mean'] *np.pi/180 , Prior.loc['spr']['mean'] *np.pi/180) } +# prior_sel= {'alpha': ( Prior.loc['dp']['mean'] *np.pi/180 , Prior.loc['spr']['mean'] *np.pi/180) } -#fitting_kargs = {'prior': None} -fitting_kargs = {'prior': prior_sel } -angle_list = list() -cost_list = list() +# fitting_kargs = {'prior': None} +fitting_kargs = {"prior": prior_sel} +angle_list = list() +cost_list = list() cost_stack = dict() -#fitting_kargs = {'size' :1} -L = pd.DataFrame(index=['alpha', 'group_phase', 'K_prime', 'K_amp'] ) +# fitting_kargs = {'size' :1} +L = pd.DataFrame(index=["alpha", "group_phase", "K_prime", "K_amp"]) -k_list, weight_list = k[mask], weights[mask] +k_list, weight_list = k[mask], weights[mask] N_grid = 30 N_data = x_concat.size -#k_prime_max, Z_max = k_list[0], weight_list[0] -#k_list[:4] -for k_prime_max,Z_max in zip(k_list[::4], weight_list[::4]): +# k_prime_max, Z_max = k_list[0], weight_list[0] +# k_list[:4] +for k_prime_max, Z_max in zip(k_list[::4], weight_list[::4]): - print('#-------------------------------------------------------------#') + print("#-------------------------------------------------------------#") print(k_prime_max) amp_enhancement = 1 - amp_Z = 0.5 #amp_enhancement * abs(Z_max)**2 /N + amp_Z = 0.5 # amp_enhancement * abs(Z_max)**2 /N params = params0.copy() - params.add('K_prime', k_prime_max , vary=False , min=k_prime_max*0.5, max=k_prime_max*1.5) - params.add('K_amp', amp_Z , vary=False , min=amp_Z*.0 , max=amp_Z*5) - - #fitter = LM.minimize(objective_func, params, args=fitting_args, method='dual_annealing',max_nfev=None) - fitter_brute = LM.minimize(objective_func, params, \ - args=fitting_args, kws=fitting_kargs , method='brute', workers=3 ) + params.add( + "K_prime", k_prime_max, vary=False, min=k_prime_max * 0.5, max=k_prime_max * 1.5 + ) + params.add("K_amp", amp_Z, vary=False, min=amp_Z * 0.0, max=amp_Z * 5) + + # fitter = LM.minimize(objective_func, params, args=fitting_args, method='dual_annealing',max_nfev=None) + fitter_brute = LM.minimize( + objective_func, + params, + args=fitting_args, + kws=fitting_kargs, + method="brute", + workers=3, + ) print(LM.report_fit(fitter_brute)) - params['K_amp'].vary = False - fitter = LM.minimize(objective_func, params, \ - args=fitting_args, kws=fitting_kargs , method='dual_annealing',max_nfev=None) + params["K_amp"].vary = False + fitter = LM.minimize( + objective_func, + params, + args=fitting_args, + kws=fitting_kargs, + method="dual_annealing", + max_nfev=None, + ) print(LM.report_fit(fitter)) - z_model = objective_func(fitter.params, *fitting_args , test_flag= True) - angle_list.append(fitter.params['alpha'].value) - cost_list.append( (fitter.residual**2).sum()/(z_concat**2).sum() ) + z_model = objective_func(fitter.params, *fitting_args, test_flag=True) + angle_list.append(fitter.params["alpha"].value) + cost_list.append((fitter.residual**2).sum() / (z_concat**2).sum()) # add prior: - #fitting_kargs = {'prior': None} - #fitter=None, F = plot_instance(GG, 'slopes', 'z_model', fitter = fitter_brute, view_scale = 0.5, non_dim=True ) - F = plot_instance(z_model, fitting_args, 'y_data_normed' , fitter = fitter_brute ,title_str = str(k_prime_max), view_scale = 0.6) - F.ax3.axhline(prior_sel['alpha'][0], color='k', linewidth = 1.5) - #F.ax3.axhspan(prior_sel['alpha'][0]- prior_sel['alpha'][1], prior_sel['alpha'][0]+ prior_sel['alpha'][1], color='gray', alpha=0.3) - F.ax3.axhline(prior_sel['alpha'][0]- prior_sel['alpha'][1], color='k', linewidth = 0.7) - F.ax3.axhline(prior_sel['alpha'][0]+ prior_sel['alpha'][1], color='k', linewidth = 0.7) - plt.plot(fitter.params['group_phase'].value, fitter.params['alpha'].value, '.r', markersize=20) + # fitting_kargs = {'prior': None} + # fitter=None, F = plot_instance(GG, 'slopes', 'z_model', fitter = fitter_brute, view_scale = 0.5, non_dim=True ) + F = plot_instance( + z_model, + fitting_args, + "y_data_normed", + fitter=fitter_brute, + title_str=str(k_prime_max), + view_scale=0.6, + ) + F.ax3.axhline(prior_sel["alpha"][0], color="k", linewidth=1.5) + # F.ax3.axhspan(prior_sel['alpha'][0]- prior_sel['alpha'][1], prior_sel['alpha'][0]+ prior_sel['alpha'][1], color='gray', alpha=0.3) + F.ax3.axhline( + prior_sel["alpha"][0] - prior_sel["alpha"][1], color="k", linewidth=0.7 + ) + F.ax3.axhline( + prior_sel["alpha"][0] + prior_sel["alpha"][1], color="k", linewidth=0.7 + ) + plt.plot( + fitter.params["group_phase"].value, + fitter.params["alpha"].value, + ".r", + markersize=20, + ) print(fitting_kargs) print(fitter.params.pretty_print()) plt.show() - #cost_stack[k_prime_max] = xr.DataArray( fitter_brute.brute_Jout/N_data, dims= ('angle', 'phase'), coords = {'angle':np.linspace(-np.pi/2, np.pi/2, N_grid), 'phase':np.linspace(0, 2* np.pi, N_grid) } ) - cost_stack[k_prime_max] = xr.DataArray( fitter_brute.brute_Jout/N_data, dims= ('angle', 'phase'), coords = {'angle':alpha_grid, 'phase':phase_grid } ) - - cost_stack[k_prime_max].coords['k'] = np.array(k_prime_max) #( ('k'), np.array(k_prime_max) ) + # cost_stack[k_prime_max] = xr.DataArray( fitter_brute.brute_Jout/N_data, dims= ('angle', 'phase'), coords = {'angle':np.linspace(-np.pi/2, np.pi/2, N_grid), 'phase':np.linspace(0, 2* np.pi, N_grid) } ) + cost_stack[k_prime_max] = xr.DataArray( + fitter_brute.brute_Jout / N_data, + dims=("angle", "phase"), + coords={"angle": alpha_grid, "phase": phase_grid}, + ) + cost_stack[k_prime_max].coords["k"] = np.array( + k_prime_max + ) # ( ('k'), np.array(k_prime_max) ) - L[k_prime_max] = fitter.params.valuesdict().values() + L[k_prime_max] = fitter.params.valuesdict().values() - #F.save_light(path= plot_path, name = key_name + '_fit_k' + str(k_prime_max)) + # F.save_light(path= plot_path, name = key_name + '_fit_k' + str(k_prime_max)) -cost_stack = xr.concat(cost_stack.values(), dim='k' ).sortby('k') -L = L.T.sort_values('K_prime') +cost_stack = xr.concat(cost_stack.values(), dim="k").sortby("k") +L = L.T.sort_values("K_prime") # %% -cost_stack2 = cost_stack #/ cost_stack.mean('angle').mean('phase').mean() -cost_stack_rolled = xr.DataArray(coords = cost_stack.coords) +cost_stack2 = cost_stack # / cost_stack.mean('angle').mean('phase').mean() +cost_stack_rolled = xr.DataArray(coords=cost_stack.coords) shift_list = list() for kindex in L.index: - #.sel( phase= L[i]['group_phase'], method ='nearest').plot() + # .sel( phase= L[i]['group_phase'], method ='nearest').plot() ii = abs(cost_stack.k - kindex).argmin().data - shift = int( -abs(cost_stack.phase - L['group_phase'][kindex]).argmin().data + cost_stack.phase.size/2 ) - #shift = int(- abs(cost_stack.phase - L['group_phase'][kindex]).argmin().data ) + shift = int( + -abs(cost_stack.phase - L["group_phase"][kindex]).argmin().data + + cost_stack.phase.size / 2 + ) + # shift = int(- abs(cost_stack.phase - L['group_phase'][kindex]).argmin().data ) shift_list.append(shift) - cost_stack_rolled[ii, :,:]= cost_stack.sel(k= kindex).roll(phase = shift, roll_coords='phase').data + cost_stack_rolled[ii, :, :] = ( + cost_stack.sel(k=kindex).roll(phase=shift, roll_coords="phase").data + ) # M.figure_axis_xy(7, 3, view_scale= 0.5) # plt.subplot(1, 2,1) @@ -489,87 +596,122 @@ def objective_func(pars, x, y, z, test_flag= False , prior= None ): # cost_stack_rolled[ii, :,:].plot(cmap =plt.cm.Blues_r) # plt.show() -cost_stack_rolled['phase'] = cost_stack_rolled['phase'] - np.pi +cost_stack_rolled["phase"] = cost_stack_rolled["phase"] - np.pi # %% -#weights = xr.DataArray(Z_max_list_gauss, dims ='k', coords = {'k': k_list_gauss}) -weights_costs = xr.DataArray(weight_list, dims ='k', coords = {'k': k_list}) -weights_costs = weights_costs/weights_costs.max() -weights_costs = weights_costs.sel(k= cost_stack_rolled.k) +# weights = xr.DataArray(Z_max_list_gauss, dims ='k', coords = {'k': k_list_gauss}) +weights_costs = xr.DataArray(weight_list, dims="k", coords={"k": k_list}) +weights_costs = weights_costs / weights_costs.max() +weights_costs = weights_costs.sel(k=cost_stack_rolled.k) -weight_k = (1/weights_costs.k) -weight_k = weight_k/weight_k.max() +weight_k = 1 / weights_costs.k +weight_k = weight_k / weight_k.max() -weights_sum= weights_costs * weight_k.data**2 +weights_sum = weights_costs * weight_k.data**2 plt.plot(weights_sum) -data_normed = cost_stack_rolled#/cost_stack_rolled.std(['angle', 'phase']) -cost_wmean = (weights_sum * data_normed /weights_sum.sum() ).interp(phase= 0)#.sum('k') - -#cost_wmean = data_normed.sel(phase = 0 , method='nearest') -#plt.plot( cost_wmean.T ) +data_normed = cost_stack_rolled # /cost_stack_rolled.std(['angle', 'phase']) +cost_wmean = (weights_sum * data_normed / weights_sum.sum()).interp( + phase=0 +) # .sum('k') -best_guess = cost_wmean.angle[cost_wmean.mean('k').argmin()].data +# cost_wmean = data_normed.sel(phase = 0 , method='nearest') +# plt.plot( cost_wmean.T ) -best_guess * 180/ np.pi +best_guess = cost_wmean.angle[cost_wmean.mean("k").argmin()].data +best_guess * 180 / np.pi # %% 2nd step optimization -F = M.figure_axis_xy(5, 3, view_scale= 0.7, container = True) +F = M.figure_axis_xy(5, 3, view_scale=0.7, container=True) -gs = GridSpec(1,5, wspace=0.25, hspace=.2)#figure=fig, +gs = GridSpec(1, 5, wspace=0.25, hspace=0.2) # figure=fig, ax0 = F.fig.add_subplot(gs[0, -1]) ax0.tick_params(labelleft=False) -klims = k_list.min()*0.8 , k_list.max()*1.2 +klims = k_list.min() * 0.8, k_list.max() * 1.2 col_dict = col.rels for g in group: - plt.plot( GGk.sel(beam=g)['gFT_PSD_data'].data,GGk.k, '-k', color= col_dict[g], markersize= 20, linewidth = 0.8) - -plt.plot(weights, k, '-b', linewidth=0.9) -plt.plot(weight_list, k_list, '.r', markersize= 5, zorder=12) + plt.plot( + GGk.sel(beam=g)["gFT_PSD_data"].data, + GGk.k, + "-k", + color=col_dict[g], + markersize=20, + linewidth=0.8, + ) + +plt.plot(weights, k, "-b", linewidth=0.9) +plt.plot(weight_list, k_list, ".r", markersize=5, zorder=12) plt.ylim(klims) -ax1 = F.fig.add_subplot(gs[0 , 0:-1]) - -plt.scatter( np.array(angle_list)* 180 /np.pi, np.array(k_list), s= (np.array(weight_list)*4e1)**3 , c=col.rascade1, label ='min per freq.') - -cost_dd = cost_wmean.mean('k') -plt.plot(cost_wmean.angle * 180 /np.pi, k_list.mean() * cost_dd/cost_dd.mean() , c=col.rascade3, label ='weighted mean BF') - -lflag= 'paritions ww3' +ax1 = F.fig.add_subplot(gs[0, 0:-1]) + +plt.scatter( + np.array(angle_list) * 180 / np.pi, + np.array(k_list), + s=(np.array(weight_list) * 4e1) ** 3, + c=col.rascade1, + label="min per freq.", +) + +cost_dd = cost_wmean.mean("k") +plt.plot( + cost_wmean.angle * 180 / np.pi, + k_list.mean() * cost_dd / cost_dd.mean(), + c=col.rascade3, + label="weighted mean BF", +) + +lflag = "paritions ww3" for i in np.arange(6): - i_dir, i_period = Prior.loc['pdp'+ str(i)]['mean'], Prior.loc['ptp'+ str(i)]['mean'] - i_k = (2 * np.pi/ i_period)**2 / 9.81 - i_dir = [i_dir -360 if i_dir > 180 else i_dir][0] - i_dir = [i_dir +360 if i_dir < -180 else i_dir][0] - - plt.plot(i_dir, i_k, '.', markersize = 10, color= col.green, label= lflag) + i_dir, i_period = ( + Prior.loc["pdp" + str(i)]["mean"], + Prior.loc["ptp" + str(i)]["mean"], + ) + i_k = (2 * np.pi / i_period) ** 2 / 9.81 + i_dir = [i_dir - 360 if i_dir > 180 else i_dir][0] + i_dir = [i_dir + 360 if i_dir < -180 else i_dir][0] + + plt.plot(i_dir, i_k, ".", markersize=10, color=col.green, label=lflag) lflag = None -ax1.axvline( best_guess * 180/ np.pi , color=col.blue, linewidth = 1.5, label ='best guess fitting') - -ax1.axvline( (prior_sel['alpha'][0]) * 180 /np.pi, color='k', linewidth = 1.5, label ='prior') -ax1.axvline( (prior_sel['alpha'][0]- prior_sel['alpha'][1]) * 180 /np.pi, color='k', linewidth = 0.7, label ='prior uncertrainty') -ax1.axvline( (prior_sel['alpha'][0]+ prior_sel['alpha'][1]) * 180 /np.pi , color='k', linewidth = 0.7) +ax1.axvline( + best_guess * 180 / np.pi, color=col.blue, linewidth=1.5, label="best guess fitting" +) + +ax1.axvline( + (prior_sel["alpha"][0]) * 180 / np.pi, color="k", linewidth=1.5, label="prior" +) +ax1.axvline( + (prior_sel["alpha"][0] - prior_sel["alpha"][1]) * 180 / np.pi, + color="k", + linewidth=0.7, + label="prior uncertrainty", +) +ax1.axvline( + (prior_sel["alpha"][0] + prior_sel["alpha"][1]) * 180 / np.pi, + color="k", + linewidth=0.7, +) plt.legend() -plt.xlabel('Angle (deg)') -plt.ylabel('wavenumber (deg)') -plt.xlim(- 125, 125) +plt.xlabel("Angle (deg)") +plt.ylabel("wavenumber (deg)") +plt.xlim(-125, 125) plt.ylim(klims) # _%%%%%%%%%%%%%%%%%%%%%%%%%% upto here - # %% -#plt.plot( k_list, cost_list, '.') +# %% +# plt.plot( k_list, cost_list, '.') ## %% cross correlation @@ -590,130 +732,158 @@ def objective_func(pars, x, y, z, test_flag= False , prior= None ): # angle_rad = np.concatenate([ -np.arctan2(A2['x_prime'], ymean)[::-1], np.arctan2(A2['x_prime'], ymean) ]) # cross_corr = np.concatenate([autocrosscorr_func_1d(A1['slopes'][::-1], A2['slopes'][::-1])[::-1], autocrosscorr_func_1d(A1['slopes'], A2['slopes']) ]) -#plt.plot( angle_rad, cross_corr ) +# plt.plot( angle_rad, cross_corr ) # %% -F = M.figure_axis_xy(6, 6.5, view_scale= 0.5) +F = M.figure_axis_xy(6, 6.5, view_scale=0.5) ax = plt.subplot(3, 1, 1) -plt.title( 'Normalized weighted mean cost at maximum phase', loc='left') -dn_unweighted = data_normed.mean('k') +plt.title("Normalized weighted mean cost at maximum phase", loc="left") +dn_unweighted = data_normed.mean("k") + +plt.plot( + data_normed.angle * 180 / np.pi, + M.normalize(dn_unweighted.sel(phase=0, method="nearest")), + label="mean", +) +plt.plot( + cost_wmean.angle * 180 / np.pi, + M.normalize(cost_wmean.sel(phase=0, method="nearest")), + label="weighted mean", +) -plt.plot(data_normed.angle* 180/np.pi, M.normalize(dn_unweighted.sel(phase=0, method = 'nearest') ) , label='mean') -plt.plot(cost_wmean.angle* 180/np.pi , M.normalize(cost_wmean.sel(phase=0, method = 'nearest') ) , label='weighted mean' ) def get_min_values(field2d, x, y): min_pos = np.unravel_index(field2d.argmin(), field2d.shape) - return x[min_pos[0]] , y[min_pos[1]] - -angle_mean, phase_mean = get_min_values(cost_wmean,cost_wmean.angle, cost_wmean.phase ) + return x[min_pos[0]], y[min_pos[1]] -phase_mean, angle_mean -angle_mean.data * 180 /np.pi -#ax.axvline(angle_true* 180 /np.pi, linewidth = 1, color = 'black', label ='True') +angle_mean, phase_mean = get_min_values(cost_wmean, cost_wmean.angle, cost_wmean.phase) -ax.axvline( (prior_sel['alpha'][0]) * 180 /np.pi, color='k', linewidth = 1.5) -ax.axvline( (prior_sel['alpha'][0]- prior_sel['alpha'][1]) * 180 /np.pi, color='k', linewidth = 0.7) -ax.axvline( (prior_sel['alpha'][0]+ prior_sel['alpha'][1]) * 180 /np.pi , color='k', linewidth = 0.7) +phase_mean, angle_mean +angle_mean.data * 180 / np.pi + +# ax.axvline(angle_true* 180 /np.pi, linewidth = 1, color = 'black', label ='True') + +ax.axvline((prior_sel["alpha"][0]) * 180 / np.pi, color="k", linewidth=1.5) +ax.axvline( + (prior_sel["alpha"][0] - prior_sel["alpha"][1]) * 180 / np.pi, + color="k", + linewidth=0.7, +) +ax.axvline( + (prior_sel["alpha"][0] + prior_sel["alpha"][1]) * 180 / np.pi, + color="k", + linewidth=0.7, +) plt.legend() -plt.xlabel('Angle (deg)') -plt.ylabel('normalized cost') +plt.xlabel("Angle (deg)") +plt.ylabel("normalized cost") plt.xlim(-80, 80) # %% - - - - - -F = M.figure_axis_xy(5, 4.5, view_scale = 0.5) +F = M.figure_axis_xy(5, 4.5, view_scale=0.5) # L.T['alpha'].hist(bins = 60) # F.ax.axvline(alpha_true, linewidth = 0.8, color = 'black') -xlims= [L.T.index.min()* 0.9, L.T.index.max()* 1.1] +xlims = [L.T.index.min() * 0.9, L.T.index.max() * 1.1] ax0 = plt.subplot(2, 1, 1) -plt.plot( L.T.index, L.T['alpha'], '.r', markersize = 10) -#ax0.axhline(alpha_true, linewidth = 0.8, color = 'black') +plt.plot(L.T.index, L.T["alpha"], ".r", markersize=10) +# ax0.axhline(alpha_true, linewidth = 0.8, color = 'black') plt.xlim(xlims[0], xlims[1]) plt.subplot(2, 1, 2) -plt.plot(k_fft, Z_fft_rm ) -#plt.plot(k_fft_max, Z_fft_rm[Z_max_pos], '.', markersize= 20) -plt.plot(k_fft_max_list,Z_max_list, '.r', markersize= 10) +plt.plot(k_fft, Z_fft_rm) +# plt.plot(k_fft_max, Z_fft_rm[Z_max_pos], '.', markersize= 20) +plt.plot(k_fft_max_list, Z_max_list, ".r", markersize=10) plt.xlim(xlims[0], xlims[1]) # %% construct spectrum L -amp_Z = 2* abs(Z_max)**2 /N - -GG['z_model'] = get_z_model(GG['dist'],GG['dist_y'], L.T['K_prime'], L.T['K_amp']/amp_enhancement, L.T['alpha'],L.T['group_phase']) -plot_instance(GG, 'slopes', 'z_model', view_scale = 0.8) +amp_Z = 2 * abs(Z_max) ** 2 / N + +GG["z_model"] = get_z_model( + GG["dist"], + GG["dist_y"], + L.T["K_prime"], + L.T["K_amp"] / amp_enhancement, + L.T["alpha"], + L.T["group_phase"], +) +plot_instance(GG, "slopes", "z_model", view_scale=0.8) # %% params_big = LM.Parameters() -L2 = L.T.sort_values('K_prime') +L2 = L.T.sort_values("K_prime") N_wavenumbers = L2.shape[0] -L2['num'] = np.arange(0, L2.shape[0]) +L2["num"] = np.arange(0, L2.shape[0]) for p in L2.T.items(): - numi= str(int(p[1]['num'])) - ki = p[1]['K_prime'] - params_big.add('K_prime_' + numi , ki , vary=False , min=ki*0.5 , max=ki*1.5) + numi = str(int(p[1]["num"])) + ki = p[1]["K_prime"] + params_big.add("K_prime_" + numi, ki, vary=False, min=ki * 0.5, max=ki * 1.5) for p in L2.T.items(): - numi= str(int(p[1]['num'])) - ampi = p[1]['K_amp']/amp_enhancement - params_big.add('K_amp_' + numi , ampi , vary=False , min=ampi*.5 , max=ampi*5) + numi = str(int(p[1]["num"])) + ampi = p[1]["K_amp"] / amp_enhancement + params_big.add("K_amp_" + numi, ampi, vary=False, min=ampi * 0.5, max=ampi * 5) for p in L2.T.items(): - numi= str(int(p[1]['num'])) - grpi = p[1]['group_phase'] - params_big.add('group_phase_' + numi , grpi , vary=False , min=0 , max= 2*np.pi) + numi = str(int(p[1]["num"])) + grpi = p[1]["group_phase"] + params_big.add("group_phase_" + numi, grpi, vary=False, min=0, max=2 * np.pi) for p in L2.T.items(): - numi= str(int(p[1]['num'])) - grpi = p[1]['alpha'] - params_big.add('alpha_' + numi , grpi , vary=False , min=-0.95 * np.pi /2, max=0.95 * np.pi /2) + numi = str(int(p[1]["num"])) + grpi = p[1]["alpha"] + params_big.add( + "alpha_" + numi, grpi, vary=False, min=-0.95 * np.pi / 2, max=0.95 * np.pi / 2 + ) # def get_max_angle(a): # hist = np.histogram(a, np.linspace(- np.pi/2, np.pi/2 , int(np.round(a.size/2)) ) ) # return hist[1][hist[0].argmax()] # # -params_big.add('angle_shift', 0 , vary=False )#, min=-0.95 * np.pi /2, max=0.95 * np.pi /2) +params_big.add( + "angle_shift", 0, vary=False +) # , min=-0.95 * np.pi /2, max=0.95 * np.pi /2) # %% -#params_big - -def objective_func_big(params_big, x, y, z, test_flag= False ): - - pardict_val = np.array(list(params_big.valuesdict().values())) - K_prime_array = pardict_val[0:N_wavenumbers] - K_amp_array = pardict_val[N_wavenumbers:N_wavenumbers*2] - group_phase_array = pardict_val[N_wavenumbers*2:N_wavenumbers*3] - #alpha = pardict_val[-1] - alpha = pardict_val[N_wavenumbers*3:N_wavenumbers*4] + pardict_val[-1] - - #print(pardict_val[-1]) - #print(alpha) - z_model = get_z_model(GG['dist'],GG['dist_y'], K_prime_array, K_amp_array, alpha, group_phase_array) - cost = ( abs(z - z_model) )**2 /z.std()**2 - #alpha_cost = (alpha - alpha.mean())**2 / (2* np.pi/180)**2 +# params_big + + +def objective_func_big(params_big, x, y, z, test_flag=False): + + pardict_val = np.array(list(params_big.valuesdict().values())) + K_prime_array = pardict_val[0:N_wavenumbers] + K_amp_array = pardict_val[N_wavenumbers : N_wavenumbers * 2] + group_phase_array = pardict_val[N_wavenumbers * 2 : N_wavenumbers * 3] + # alpha = pardict_val[-1] + alpha = pardict_val[N_wavenumbers * 3 : N_wavenumbers * 4] + pardict_val[-1] + + # print(pardict_val[-1]) + # print(alpha) + z_model = get_z_model( + GG["dist"], GG["dist_y"], K_prime_array, K_amp_array, alpha, group_phase_array + ) + cost = (abs(z - z_model)) ** 2 / z.std() ** 2 + # alpha_cost = (alpha - alpha.mean())**2 / (2* np.pi/180)**2 if test_flag: return z_model else: return cost + def turn_vary_key_on(pard, key): for pi in pard.keys(): @@ -724,57 +894,83 @@ def turn_vary_key_on(pard, key): return pard -GG['z_model'] = objective_func_big(params_big, GG['dist'], GG['dist_y'], GG['slopes'], test_flag= True) -plot_instance(GG, 'slopes', 'z_model', view_scale = 0.6) +GG["z_model"] = objective_func_big( + params_big, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=True +) +plot_instance(GG, "slopes", "z_model", view_scale=0.6) # %% -#help(LM.minimize) +# help(LM.minimize) # params_big = turn_vary_key_on(params_big, 'group_phase') # fitter = LM.minimize(objective_func_big, params_big, args=fitting_args, method='ampgo',max_nfev=10000) -#fitter - -params2 = turn_vary_key_on(params_big, 'K_prime') -fitter = LM.minimize(objective_func_big, params2, args=fitting_args, method='least_squares',max_nfev=10000) - -params2 = turn_vary_key_on(fitter.params, 'K_amp') -fitter = LM.minimize(objective_func_big, params2, args=fitting_args, method='least_squares',max_nfev=10000) - -#print('angle_shift') +# fitter + +params2 = turn_vary_key_on(params_big, "K_prime") +fitter = LM.minimize( + objective_func_big, + params2, + args=fitting_args, + method="least_squares", + max_nfev=10000, +) + +params2 = turn_vary_key_on(fitter.params, "K_amp") +fitter = LM.minimize( + objective_func_big, + params2, + args=fitting_args, + method="least_squares", + max_nfev=10000, +) + +# print('angle_shift') # params2 = turn_vary_key_on(fitter.params, 'angle_shift') # fitter = LM.minimize(objective_func_big, params2, args=fitting_args, method='least_squares',max_nfev=10000) -#fitter = LM.minimize(objective_func_big, fitter.params, args=fitting_args, method='emcee') +# fitter = LM.minimize(objective_func_big, fitter.params, args=fitting_args, method='emcee') # %% -GG['z_model'] = objective_func_big(fitter.params, GG['dist'],GG['dist_y'], GG['slopes'], test_flag= True) -plot_instance(GG, 'slopes', 'z_model', view_scale = 0.6) +GG["z_model"] = objective_func_big( + fitter.params, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=True +) +plot_instance(GG, "slopes", "z_model", view_scale=0.6) # %% -z_mdoel_init = objective_func_big(params_big, GG['dist'],GG['dist_y'], GG['slopes'], test_flag= True) -objective_func_big(params_big, GG['dist'],GG['dist_y'], GG['slopes'], test_flag= False).sum() -z_mdoel_fit = objective_func_big(fitter.params, GG['dist'],GG['dist_y'], GG['slopes'], test_flag= True) -objective_func_big(fitter.params, GG['dist'],GG['dist_y'], GG['slopes'], test_flag= False).sum() +z_mdoel_init = objective_func_big( + params_big, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=True +) +objective_func_big( + params_big, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=False +).sum() +z_mdoel_fit = objective_func_big( + fitter.params, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=True +) +objective_func_big( + fitter.params, GG["dist"], GG["dist_y"], GG["slopes"], test_flag=False +).sum() plt.plot(z_mdoel_init) plt.plot(z_mdoel_fit) # %% -#fitter.params.valuesdict() +# fitter.params.valuesdict() + def params_to_Dataframe(pard): - pardict_val = np.array(list(pard.valuesdict().values())) + pardict_val = np.array(list(pard.valuesdict().values())) pardict = dict() - pardict['K_prime_array'] = pardict_val[0:N_wavenumbers] - pardict['K_amp_array'] = pardict_val[N_wavenumbers:N_wavenumbers*2] - pardict['group_phase_array'] = pardict_val[N_wavenumbers*2:N_wavenumbers*3] - pardict['alpha'] = pardict_val[N_wavenumbers*3:N_wavenumbers*4] + pardict["K_prime_array"] = pardict_val[0:N_wavenumbers] + pardict["K_amp_array"] = pardict_val[N_wavenumbers : N_wavenumbers * 2] + pardict["group_phase_array"] = pardict_val[N_wavenumbers * 2 : N_wavenumbers * 3] + pardict["alpha"] = pardict_val[N_wavenumbers * 3 : N_wavenumbers * 4] return pd.DataFrame.from_dict(pardict) + Fits = params_to_Dataframe(fitter.params) -plt.plot( k_fft_max_list, abs( (Fits['alpha']) ), '.' ) +plt.plot(k_fft_max_list, abs((Fits["alpha"])), ".") -Fits['alpha'] +Fits["alpha"] plt.hist(fitter.flatchain, 30) diff --git a/analysis/SB03_swell_phase_fitting_posterior_direct.py b/analysis/SB03_swell_phase_fitting_posterior_direct.py index bf1bee9b..c66552f1 100644 --- a/analysis/SB03_swell_phase_fitting_posterior_direct.py +++ b/analysis/SB03_swell_phase_fitting_posterior_direct.py @@ -1,16 +1,16 @@ - import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -25,39 +25,50 @@ col.colormaps2(21) -#import s3fs +# import s3fs # %% -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190601093502_09790310_004_01', 'SH_batch02', False -track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False - - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -save_path = mconfig['paths']['work'] + '/B03_spectra_'+hemis+'/' -save_name = 'B03_'+track_name - -plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/B_spectra/' +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190601093502_09790310_004_01', 'SH_batch02', False +track_name, batch_key, test_flag = "20190219073735_08070210_004_01", "SH_batch02", False + + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +save_path = mconfig["paths"]["work"] + "/B03_spectra_" + hemis + "/" +save_name = "B03_" + track_name + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/B_spectra/" +) MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -#Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +# Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data -load_path = mconfig['paths']['work'] +'/B02_spectra_'+hemis+'/' -Gd = xr.load_dataset(load_path+ '/B02_'+track_name + '_gFT_k.nc' ) # -Gdi = Gd.sel(beam = 'gt1r').isel(x = 1).sel(k =slice(0, 0.06)) +load_path = mconfig["paths"]["work"] + "/B02_spectra_" + hemis + "/" +Gd = xr.load_dataset(load_path + "/B02_" + track_name + "_gFT_k.nc") # +Gdi = Gd.sel(beam="gt1r").isel(x=1).sel(k=slice(0, 0.06)) Gdi.gFT_PSD_data.plot() # %% @@ -80,112 +91,172 @@ # %% select data Gd.keys() for b in Gd.beam: - B = Gd.sel(beam= b) - plt.plot(B['lon'], B['lat']) + B = Gd.sel(beam=b) + plt.plot(B["lon"], B["lat"]) -B1 = Gd.sel(beam = 'gt2r') -B2 = Gd.sel(beam = 'gt1r') +B1 = Gd.sel(beam="gt2r") +B2 = Gd.sel(beam="gt1r") -dist_y_mean = np.sqrt( (B1['x_coord']-B2['x_coord'])**2 + (B1['y_coord']-B2['y_coord'])**2 ).mean() +dist_y_mean = np.sqrt( + (B1["x_coord"] - B2["x_coord"]) ** 2 + (B1["y_coord"] - B2["y_coord"]) ** 2 +).mean() dist_y_mean -B1.coords['dist_y'] = 0 -B2.coords['dist_y'] = dist_y_mean +B1.coords["dist_y"] = 0 +B2.coords["dist_y"] = dist_y_mean -GG0 = xr.concat( [B1.expand_dims('dist_y'), B2.expand_dims('dist_y')] , dim= 'dist_y') +GG0 = xr.concat([B1.expand_dims("dist_y"), B2.expand_dims("dist_y")], dim="dist_y") -#plt.plot( GG0['x']/1e3 , GG0['y']/1e3) -plt.plot( GG0['x_coord']/1e3 , GG0['y_coord']/1e3, '^') +# plt.plot( GG0['x']/1e3 , GG0['y']/1e3) +plt.plot(GG0["x_coord"] / 1e3, GG0["y_coord"] / 1e3, "^") # %% -GG0.gFT_PSD_model.isel(dist_y= 1).rolling(x=2, k =30).mean().plot() -GG0.Z_hat_imag.isel(dist_y= 1).rolling(x=2, k =30).mean().plot() -GG0.Z_hat_real.isel(dist_y= 1).rolling(x=2, k =30).mean().plot() - - +GG0.gFT_PSD_model.isel(dist_y=1).rolling(x=2, k=30).mean().plot() +GG0.Z_hat_imag.isel(dist_y=1).rolling(x=2, k=30).mean().plot() +GG0.Z_hat_real.isel(dist_y=1).rolling(x=2, k=30).mean().plot() k_smooth = 5 -some_angle = np.arctan2( GG0.Z_hat_imag.isel(dist_y= 1).rolling(k =k_smooth).mean() , GG0.Z_hat_real.isel(dist_y= 1 ).rolling(k =k_smooth).mean() ) -some_angle.plot(cmap =col.circle_big) - -plt.plot(some_angle.k, some_angle.sel(x = slice(2.025e6, 2.075e6) ), c='gray', alpha = 0.3) - +some_angle = np.arctan2( + GG0.Z_hat_imag.isel(dist_y=1).rolling(k=k_smooth).mean(), + GG0.Z_hat_real.isel(dist_y=1).rolling(k=k_smooth).mean(), +) +some_angle.plot(cmap=col.circle_big) +plt.plot(some_angle.k, some_angle.sel(x=slice(2.025e6, 2.075e6)), c="gray", alpha=0.3) # %% -#GG0.sel(x = 2e6, method= 'nearest').isel( dist_y= 0, k = slice(100, 300 )).Z_hat_imag.plot() -GG0.sel(x = 2e6 , method= 'nearest').sel(k = slice(0.02, 0.05 )).isel( dist_y= 0).Z_hat_imag.plot() -GG0.sel(x = 2e6, method= 'nearest').sel(k = slice(0.02, 0.05 )).isel( dist_y= 1).Z_hat_imag.plot() +# GG0.sel(x = 2e6, method= 'nearest').isel( dist_y= 0, k = slice(100, 300 )).Z_hat_imag.plot() +GG0.sel(x=2e6, method="nearest").sel(k=slice(0.02, 0.05)).isel( + dist_y=0 +).Z_hat_imag.plot() +GG0.sel(x=2e6, method="nearest").sel(k=slice(0.02, 0.05)).isel( + dist_y=1 +).Z_hat_imag.plot() # %% -k_smooth= 5 -GG0.sel(x = 2.05e6 , method= 'nearest').rolling(k =k_smooth).mean().sel(k = slice(0.02, 0.06 )).isel( dist_y= 0).gFT_PSD_model.plot() -GG0.sel(x = 2.05e6, method= 'nearest').rolling(k =k_smooth).mean().sel(k = slice(0.02, 0.06 )).isel( dist_y= 1).gFT_PSD_model.plot() +k_smooth = 5 +GG0.sel(x=2.05e6, method="nearest").rolling(k=k_smooth).mean().sel( + k=slice(0.02, 0.06) +).isel(dist_y=0).gFT_PSD_model.plot() +GG0.sel(x=2.05e6, method="nearest").rolling(k=k_smooth).mean().sel( + k=slice(0.02, 0.06) +).isel(dist_y=1).gFT_PSD_model.plot() # %% -klim= 0, 0.06 -Gi = GG0.sel(x = 2.05e6 , method= 'nearest').isel( dist_y= 0).sel(k = slice(klim[0] , klim[1]))#.rolling(k=10 ).mean() -Gi2 = GG0.sel(x = 2.05e6 , method= 'nearest').isel( dist_y= 0).sel(k = slice(klim[0] , klim[1]))#.rolling(k=5 ).mean() - -#Gi.gFT_PSD_model.plot() - -def plot_Z(X, Y, cmap= plt.cm.jet, line_c= 'k', k_wave=None, shift=0, head = 'o', head_size=30, alpha = 1 , **kargs): - - all_cols = np.array(plt.cm.jet(np.arange(0, Y.size ))) - - def plot_q(xi, yi, c, kk = None, **kargs): - #print(xi, yi) - #xi, yi =np.sign(xi) * np.log(abs(xi)), np.sign(yi) *np.log(abs(yi)) - xi, yi =np.sign(xi) * np.sqrt(abs(xi)), np.sign(yi) *np.sqrt(abs(yi)) - - #plt.plot([shift, xi+shift], [0, yi], '-', color = c, alpha = 0.5* alpha, **kargs) - plt.plot([shift, xi+shift], [0, yi], '-', color =line_c, linewidth =1.5, alpha = 0.5* alpha, **kargs, zorder= 0) - plt.scatter([xi+shift], [yi], head_size, marker= head ,color = c, alpha = alpha, **kargs, edgecolors= 'black') +klim = 0, 0.06 +Gi = ( + GG0.sel(x=2.05e6, method="nearest").isel(dist_y=0).sel(k=slice(klim[0], klim[1])) +) # .rolling(k=10 ).mean() +Gi2 = ( + GG0.sel(x=2.05e6, method="nearest").isel(dist_y=0).sel(k=slice(klim[0], klim[1])) +) # .rolling(k=5 ).mean() + +# Gi.gFT_PSD_model.plot() + + +def plot_Z( + X, + Y, + cmap=plt.cm.jet, + line_c="k", + k_wave=None, + shift=0, + head="o", + head_size=30, + alpha=1, + **kargs, +): + + all_cols = np.array(plt.cm.jet(np.arange(0, Y.size))) + + def plot_q(xi, yi, c, kk=None, **kargs): + # print(xi, yi) + # xi, yi =np.sign(xi) * np.log(abs(xi)), np.sign(yi) *np.log(abs(yi)) + xi, yi = np.sign(xi) * np.sqrt(abs(xi)), np.sign(yi) * np.sqrt(abs(yi)) + + # plt.plot([shift, xi+shift], [0, yi], '-', color = c, alpha = 0.5* alpha, **kargs) + plt.plot( + [shift, xi + shift], + [0, yi], + "-", + color=line_c, + linewidth=1.5, + alpha=0.5 * alpha, + **kargs, + zorder=0, + ) + plt.scatter( + [xi + shift], + [yi], + head_size, + marker=head, + color=c, + alpha=alpha, + **kargs, + edgecolors="black", + ) if kk is not None: - plt.text(xi+shift, yi, ' '+str(int(kk)) ) - + plt.text(xi + shift, yi, " " + str(int(kk))) if k_wave is None: - for xi,yi,cc in zip(X, Y, all_cols): + for xi, yi, cc in zip(X, Y, all_cols): if ~np.isnan(xi): plot_q(xi, yi, cc[0:3], **kargs) else: - for xi,yi,cc,kk in zip(X, Y, all_cols, k_wave): + for xi, yi, cc, kk in zip(X, Y, all_cols, k_wave): if ~np.isnan(xi): - plot_q(xi, yi, cc[0:3],kk= kk, **kargs) + plot_q(xi, yi, cc[0:3], kk=kk, **kargs) -weight = np.copy(Gi.gFT_PSD_model ) -#weight = np.copy(Gi.gFT_PSD_model.sel(k = slice(klim[0] , klim[1])) ) -weight[weight < 0.015] =np.nan -weight =1e3* weight/np.nanmean(weight) -# col.colormaps2( Gi.k.size ) -#all_cols = np.array(col.circle_medium_triple(np.arange(0, Gi.k.size ))) +weight = np.copy(Gi.gFT_PSD_model) +# weight = np.copy(Gi.gFT_PSD_model.sel(k = slice(klim[0] , klim[1])) ) +weight[weight < 0.015] = np.nan +weight = 1e3 * weight / np.nanmean(weight) -plot_Z(Gi.gFT_cos_coeff.data * weight,Gi.gFT_sin_coeff.data *weight, k_wave = np.arange(Gi.k.size), line_c = 'k', head='o', head_size=60, shift= 0) - -#Gi2.gFT_PSD_model.rolling(k=3 ).mean().plot() +# col.colormaps2( Gi.k.size ) +# all_cols = np.array(col.circle_medium_triple(np.arange(0, Gi.k.size ))) + +plot_Z( + Gi.gFT_cos_coeff.data * weight, + Gi.gFT_sin_coeff.data * weight, + k_wave=np.arange(Gi.k.size), + line_c="k", + head="o", + head_size=60, + shift=0, +) + +# Gi2.gFT_PSD_model.rolling(k=3 ).mean().plot() # Gi2.gFT_PSD_model.plot() -weight = np.copy(Gi2.gFT_PSD_model ) -weight[weight < 0.015] =np.nan -weight = 1e3 *weight/np.nanmean(weight) +weight = np.copy(Gi2.gFT_PSD_model) +weight[weight < 0.015] = np.nan +weight = 1e3 * weight / np.nanmean(weight) -plot_Z(Gi2.gFT_cos_coeff.data * weight,Gi2.gFT_sin_coeff.data *weight , k_wave = np.arange(Gi2.k.size) , shift= 0, line_c = 'r', head='+', head_size=120 , alpha = 0.99) +plot_Z( + Gi2.gFT_cos_coeff.data * weight, + Gi2.gFT_sin_coeff.data * weight, + k_wave=np.arange(Gi2.k.size), + shift=0, + line_c="r", + head="+", + head_size=120, + alpha=0.99, +) plt.grid() -plt.axis('equal') +plt.axis("equal") -#llim= 8 -#plt.xlim(- llim, llim) -#plt.ylim(- llim, llim) +# llim= 8 +# plt.xlim(- llim, llim) +# plt.ylim(- llim, llim) -#GG0.sel(x = 2.05e6 , method= 'nearest').gFT_cos_coeff +# GG0.sel(x = 2.05e6 , method= 'nearest').gFT_cos_coeff diff --git a/analysis/SB04_2d_wavefield_emulator.py b/analysis/SB04_2d_wavefield_emulator.py index 7f2a2037..5df8a9b1 100644 --- a/analysis/SB04_2d_wavefield_emulator.py +++ b/analysis/SB04_2d_wavefield_emulator.py @@ -1,12 +1,11 @@ - import os, sys """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -if __name__ == '__main__': - exec(open(os.environ['PYTHONSTARTUP']).read()) +if __name__ == "__main__": + exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) import ICEsat2_SI_tools.convert_GPS_time as cGPS @@ -20,122 +19,130 @@ import datetime import concurrent.futures as futures - track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment - - track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False - + track_name, batch_key, test_flag = io.init_from_input( + sys.argv + ) # loads standard experiment + track_name, batch_key, test_flag = ( + "20190219073735_08070210_004_01", + "SH_batch02", + False, + ) + # print(track_name, batch_key, test_flag) + hemis, batch = batch_key.split("_") + ATlevel = "ATL03" - #print(track_name, batch_key, test_flag) - hemis, batch = batch_key.split('_') - ATlevel= 'ATL03' + save_path = mconfig["paths"]["work"] + "/B03_spectra_" + hemis + "/" + save_name = "B03_" + track_name - save_path = mconfig['paths']['work'] + '/B03_spectra_'+hemis+'/' - save_name = 'B03_'+track_name - - plot_path = mconfig['paths']['plot'] + '/phase_fitting_fake/2D_fake/' + plot_path = mconfig["paths"]["plot"] + "/phase_fitting_fake/2D_fake/" MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) - bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' - -if __name__ == '__main__': - all_beams = mconfig['beams']['all_beams'] - high_beams = mconfig['beams']['high_beams'] - low_beams = mconfig['beams']['low_beams'] - - - load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' - Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # - load_path = mconfig['paths']['work'] + '/B02_spectra_'+hemis+'/' - Gpars = io.load_pandas_table_dict('B02_'+ track_name + '_params' , load_path) # - Gspec = xr.open_dataset(load_path + 'B02_'+ track_name + '_LS.nc' ) # - Gspec['Y_model_hat'] = Gspec.Y_model_hat_real + Gspec.Y_model_hat_imag *1j - Gspec = Gspec.drop('Y_model_hat_real').drop('Y_model_hat_imag') - - dk = Gspec.k.diff('k').mean().data + bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" + +if __name__ == "__main__": + all_beams = mconfig["beams"]["all_beams"] + high_beams = mconfig["beams"]["high_beams"] + low_beams = mconfig["beams"]["low_beams"] + + load_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" + Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # + load_path = mconfig["paths"]["work"] + "/B02_spectra_" + hemis + "/" + Gpars = io.load_pandas_table_dict("B02_" + track_name + "_params", load_path) # + Gspec = xr.open_dataset(load_path + "B02_" + track_name + "_LS.nc") # + Gspec["Y_model_hat"] = Gspec.Y_model_hat_real + Gspec.Y_model_hat_imag * 1j + Gspec = Gspec.drop("Y_model_hat_real").drop("Y_model_hat_imag") + + dk = Gspec.k.diff("k").mean().data Lpoints = Gspec.Lpoints - Gspec = Gspec.sel(k = slice(0.000125, 0.025)).isel(x =slice(4, 30)) - - Gspec.coords['f'] = (('k'), np.sqrt(Gspec.k.data * 9.81)/ 2/np.pi ) - Gspec.coords['T'] = 1/Gspec.coords['f'] - Gspec=Gspec.swap_dims({'k': 'f'}) + Gspec = Gspec.sel(k=slice(0.000125, 0.025)).isel(x=slice(4, 30)) + Gspec.coords["f"] = (("k"), np.sqrt(Gspec.k.data * 9.81) / 2 / np.pi) + Gspec.coords["T"] = 1 / Gspec.coords["f"] + Gspec = Gspec.swap_dims({"k": "f"}) - A, B = Gspec.sel(beam= 'gt2r').Y_model_hat , Gspec.sel(beam= 'gt2l').Y_model_hat + A, B = Gspec.sel(beam="gt2r").Y_model_hat, Gspec.sel(beam="gt2l").Y_model_hat - r_ave_kargs={'x':2, 'f':10, 'center':True, 'min_periods':2} - r_ave_kargs2={'f':10, 'center':True, 'min_periods':2} + r_ave_kargs = {"x": 2, "f": 10, "center": True, "min_periods": 2} + r_ave_kargs2 = {"f": 10, "center": True, "min_periods": 2} - S_aa = (A*A.conj()).real - S_bb = (B*B.conj()).real + S_aa = (A * A.conj()).real + S_bb = (B * B.conj()).real - co_spec = (A.conj() *B).rolling(**r_ave_kargs).mean() + co_spec = (A.conj() * B).rolling(**r_ave_kargs).mean() np.log(abs(co_spec)).plot(levels=np.arange(-2, 3, 0.1)) (abs(co_spec)).plot(levels=np.exp(np.arange(-3, 2, 0.1))) - abs(co_spec).mean('x').plot() -if __name__ == '__main__': + abs(co_spec).mean("x").plot() +if __name__ == "__main__": L1 = 50 - k1 = 2* np.pi /L1 - l1 = 2* np.pi /L1 + k1 = 2 * np.pi / L1 + l1 = 2 * np.pi / L1 L2 = 65 - k2 = 2* np.pi /L2 + k2 = 2 * np.pi / L2 - x=np.arange(-250, 250, 0.5) - y=np.arange(-200, 200, 0.5) - Nx, Ny= x.size, y.size + x = np.arange(-250, 250, 0.5) + y = np.arange(-200, 200, 0.5) + Nx, Ny = x.size, y.size XX, YY = np.meshgrid(x, y) - XX, YY = XX.reshape(XX.size), YY.reshape(YY.size) + XX, YY = XX.reshape(XX.size), YY.reshape(YY.size) alpha = 35 - kk, ll = np.cos(alpha * np.pi/180) * np.array([0.9*k1, k1, 1.1* k1]), np.sin(alpha * np.pi/180) * np.array([0.9* k1, 1*k1, 1.1* k1]) + kk, ll = np.cos(alpha * np.pi / 180) * np.array([0.9 * k1, k1, 1.1 * k1]), np.sin( + alpha * np.pi / 180 + ) * np.array([0.9 * k1, 1 * k1, 1.1 * k1]) M_k, M_l = kk.size, ll.size kk_mesh, ll_mesh = np.meshgrid(kk, ll) kk_mesh, ll_mesh = kk_mesh.reshape(kk_mesh.size), ll_mesh.reshape(ll_mesh.size) G = np.cos(np.outer(XX, kk_mesh) + np.outer(YY, ll_mesh)).T G.shape - plt.contourf(x, y, G.sum(0).reshape(Ny, Nx) ) - plt.axis('equal') + plt.contourf(x, y, G.sum(0).reshape(Ny, Nx)) + plt.axis("equal") - # %% radial coordincates +# %% radial coordincates -def gaus_2d(x, y, pos_tuple, sigma_g ): - gx = np.exp(-0.5 * (x - pos_tuple[0])**2 /sigma_g**2 ) - gy = np.exp(-0.5 * (y - pos_tuple[1])**2 /sigma_g**2 ) - return np.outer(gx , gy).T -if __name__ == '__main__': +def gaus_2d(x, y, pos_tuple, sigma_g): + gx = np.exp(-0.5 * (x - pos_tuple[0]) ** 2 / sigma_g**2) + gy = np.exp(-0.5 * (y - pos_tuple[1]) ** 2 / sigma_g**2) + return np.outer(gx, gy).T - k_range = np.linspace(0, 0.1, 30) - l_range = np.linspace(-0.1, .1 , 60) + +if __name__ == "__main__": + + k_range = np.linspace(0, 0.1, 30) + l_range = np.linspace(-0.1, 0.1, 60) kk, ll = np.meshgrid(k_range, l_range) - gaus_lk = gaus_2d( k_range, l_range, [0.02, 0.0] , 0.01) + gaus_lk = gaus_2d(k_range, l_range, [0.02, 0.0], 0.01) - M.figure_axis_xy(4, 4, view_scale= 0.5) - plt.contourf(k_range, l_range, gaus_lk ) - plt.axis('equal') + M.figure_axis_xy(4, 4, view_scale=0.5) + plt.contourf(k_range, l_range, gaus_lk) + plt.axis("equal") k_0 = 0.03 l_0 = 0 dk = 0.01 -stancil_size =0 -def get_stancils_kl(k_0, l_0, size =1 , dk= 0.01, mesh = True): +stancil_size = 0 + + +def get_stancils_kl(k_0, l_0, size=1, dk=0.01, mesh=True): import numpy as np + """ size is the stancil half width. if 0 the stancil is 1, if one the stancil is 3 and so on. """ - if size ==0: + if size == 0: stancil_k = np.array(k_0) stancil_l = np.array(l_0) else: - stancil_k = (np.arange(-size, size +1 , 1) *dk + k_0 ) - stancil_l = (np.arange(-size, size +1 , 1) *dk + l_0 ) + stancil_k = np.arange(-size, size + 1, 1) * dk + k_0 + stancil_l = np.arange(-size, size + 1, 1) * dk + l_0 if mesh: stancil_k_mesh, stancil_l_mesh = np.meshgrid(stancil_k, stancil_l) @@ -145,8 +152,9 @@ def get_stancils_kl(k_0, l_0, size =1 , dk= 0.01, mesh = True): return stancil_k_mesh, stancil_l_mesh -def get_rand_stancils(a_mean, size =1 , dk= 0.01): +def get_rand_stancils(a_mean, size=1, dk=0.01): import numpy as np + """ size is here the total stancil size. dk is the 2d std of the gaussian @@ -156,32 +164,44 @@ def get_rand_stancils(a_mean, size =1 , dk= 0.01): stancil_k = np.array(a_mean) else: - stancil_k = np.random.normal(a_mean, dk,size-1) - stancil_k = np.insert(stancil_k ,0, a_mean ) + stancil_k = np.random.normal(a_mean, dk, size - 1) + stancil_k = np.insert(stancil_k, 0, a_mean) return stancil_k -def gaus_2d_mesh(XX,YY, pos_tuple, sigma_g ): - #grid = ( (XX - pos_tuple[0]) * (YY - pos_tuple[1]) ) +def gaus_2d_mesh(XX, YY, pos_tuple, sigma_g): + # grid = ( (XX - pos_tuple[0]) * (YY - pos_tuple[1]) ) import numpy as np - gx = np.exp(-0.5 * (XX - pos_tuple[0])**2 /sigma_g**2 ) - gy = np.exp(-0.5 * (YY - pos_tuple[1])**2 /sigma_g**2 ) - return (gx * gy).T -if __name__ == '__main__': + gx = np.exp(-0.5 * (XX - pos_tuple[0]) ** 2 / sigma_g**2) + gy = np.exp(-0.5 * (YY - pos_tuple[1]) ** 2 / sigma_g**2) + return (gx * gy).T - k_mesh, l_mesh = get_stancils_kl(k_0, l_0, size= stancil_size, dk= dk) - amp_mesh =gaus_2d_mesh( k_mesh, l_mesh , [k_0, l_0] , dk) +if __name__ == "__main__": - stancil_k_mesh, stancil_l_mesh = k_mesh.reshape(k_mesh.size), l_mesh.reshape(l_mesh.size) + k_mesh, l_mesh = get_stancils_kl(k_0, l_0, size=stancil_size, dk=dk) + amp_mesh = gaus_2d_mesh(k_mesh, l_mesh, [k_0, l_0], dk) + + stancil_k_mesh, stancil_l_mesh = k_mesh.reshape(k_mesh.size), l_mesh.reshape( + l_mesh.size + ) stancil_amp_mesh = amp_mesh.reshape(amp_mesh.size) plt.contourf(k_mesh, l_mesh, amp_mesh) - plt.axis('equal') - - -def get_stancils_polar( amp, angle_rad, size=1, dk = 0.01, mesh = True, plot_flag = True, amp_std= None, random=True): + plt.axis("equal") + + +def get_stancils_polar( + amp, + angle_rad, + size=1, + dk=0.01, + mesh=True, + plot_flag=True, + amp_std=None, + random=True, +): """ inputs: @@ -196,6 +216,7 @@ def get_stancils_polar( amp, angle_rad, size=1, dk = 0.01, mesh = True, plot_fl list of k wave numbers, list of l wave numbers, list of relative amplitudes, shape of the stancil """ import numpy as np + k0 = amp * np.cos(angle_rad) l0 = amp * np.sin(angle_rad) @@ -206,64 +227,90 @@ def get_stancils_polar( amp, angle_rad, size=1, dk = 0.01, mesh = True, plot_fl if random: - k_mesh = get_rand_stancils(k0, size= size, dk= dk) - l_mesh = get_rand_stancils(l0, size= size, dk= dk) + k_mesh = get_rand_stancils(k0, size=size, dk=dk) + l_mesh = get_rand_stancils(l0, size=size, dk=dk) - amp_mesh = get_rand_stancils(0, size= size, dk= 0.2) - amp_mesh= np.ones(amp_mesh.size) - abs(amp_mesh) + amp_mesh = get_rand_stancils(0, size=size, dk=0.2) + amp_mesh = np.ones(amp_mesh.size) - abs(amp_mesh) stancil_k_mesh, stancil_l_mesh, stancil_amp_mesh = k_mesh, l_mesh, amp_mesh else: - k_mesh, l_mesh = get_stancils_kl(k0, l0, size= size, dk= dk, mesh= mesh) - amp_mesh = gaus_2d_mesh( k_mesh, l_mesh , [k0, l0] , amp_std) - - stancil_k_mesh, stancil_l_mesh = k_mesh.reshape(k_mesh.size), l_mesh.reshape(l_mesh.size) - stancil_amp_mesh = amp_mesh.reshape(amp_mesh.size) + k_mesh, l_mesh = get_stancils_kl(k0, l0, size=size, dk=dk, mesh=mesh) + amp_mesh = gaus_2d_mesh(k_mesh, l_mesh, [k0, l0], amp_std) - amp_mesh = amp_mesh/amp_mesh.sum() + stancil_k_mesh, stancil_l_mesh = k_mesh.reshape(k_mesh.size), l_mesh.reshape( + l_mesh.size + ) + stancil_amp_mesh = amp_mesh.reshape(amp_mesh.size) + amp_mesh = amp_mesh / amp_mesh.sum() - #print(k_mesh, l_mesh, amp_mesh) + # print(k_mesh, l_mesh, amp_mesh) if plot_flag: import matplotlib.pyplot as plt + if size == 1: - plt.plot(k_mesh, l_mesh, '.', markersize= amp_mesh*3, color= 'black') + plt.plot(k_mesh, l_mesh, ".", markersize=amp_mesh * 3, color="black") else: if random: - plt.scatter(k_mesh, l_mesh, amp_mesh*3, color= 'black') + plt.scatter(k_mesh, l_mesh, amp_mesh * 3, color="black") else: - plt.contour(k_mesh, l_mesh, amp_mesh, colors= 'black', linewidths= 1) - + plt.contour(k_mesh, l_mesh, amp_mesh, colors="black", linewidths=1) return stancil_k_mesh, stancil_l_mesh, stancil_amp_mesh, k_mesh.shape -if __name__ == '__main__': +if __name__ == "__main__": font_for_pres() - angle =45 - amp =1 + angle = 45 + amp = 1 phase = 0 k_abs = 0.1 for dk in np.arange(0.005, 0.03, 0.002): for size in [2, 5, 10, 30, 50, 100, 200]: - F = M.figure_axis_xy(8, 3, view_scale = 0.5, container =False) - plt.suptitle('k_abs=' + str(k_abs) +' angle=' + str(angle) + ' size=' + str(size) +' dk=' + str(dk) ) + F = M.figure_axis_xy(8, 3, view_scale=0.5, container=False) + plt.suptitle( + "k_abs=" + + str(k_abs) + + " angle=" + + str(angle) + + " size=" + + str(size) + + " dk=" + + str(dk) + ) ax = plt.subplot(1, 2, 1) - k_list, l_list, amp_weights, stancil_shape = get_stancils_polar(k_abs, angle * np.pi/180, size=size, dk = dk, mesh = True , plot_flag= True, random = True) - circle1 = plt.Circle((k_list[0], l_list[0]), dk, color='b', fill=False) + k_list, l_list, amp_weights, stancil_shape = get_stancils_polar( + k_abs, + angle * np.pi / 180, + size=size, + dk=dk, + mesh=True, + plot_flag=True, + random=True, + ) + circle1 = plt.Circle((k_list[0], l_list[0]), dk, color="b", fill=False) ax.add_patch(circle1) - k_noise, l_noise, amp_noise, stancil_shape = get_stancils_polar(0.8, 0 * np.pi/180, size=20, dk = 0.3, mesh = True , plot_flag= True, random = True) - amp_noise = (amp_noise *0+1) * 0 + k_noise, l_noise, amp_noise, stancil_shape = get_stancils_polar( + 0.8, + 0 * np.pi / 180, + size=20, + dk=0.3, + mesh=True, + plot_flag=True, + random=True, + ) + amp_noise = (amp_noise * 0 + 1) * 0 plt.xlim(0, 0.1) - #plt.axis('equal') + # plt.axis('equal') plt.ylim(-0.1, 0.1) - plt.xlabel('k wavenumber') - plt.ylabel('l wavenumber') + plt.xlabel("k wavenumber") + plt.ylabel("l wavenumber") # % Derive real space model plt.subplot(1, 2, 2) @@ -272,15 +319,24 @@ def get_stancils_polar( amp, angle_rad, size=1, dk = 0.01, mesh = True, plot_fl l_all = np.concatenate([l_list, l_noise]) amp_all = np.concatenate([amp_weights, amp_noise]) amp_all.shape - G = np.vstack([ np.cos(np.outer(XX, k_all) + np.outer(YY, l_all)).T , np.sin(np.outer(XX, k_all) + np.outer(YY, l_all)).T ] ).T - - b = np.hstack([ np.cos(phase)*amp_all, np.sin(phase) *amp_all]).squeeze() * amp + G = np.vstack( + [ + np.cos(np.outer(XX, k_all) + np.outer(YY, l_all)).T, + np.sin(np.outer(XX, k_all) + np.outer(YY, l_all)).T, + ] + ).T + + b = ( + np.hstack([np.cos(phase) * amp_all, np.sin(phase) * amp_all]).squeeze() + * amp + ) z_model = (G @ b).reshape(Ny, Nx) z_model.shape - plt.contourf(x, y, z_model ) - plt.axis('equal') + plt.contourf(x, y, z_model) + plt.axis("equal") - F.save_light(path = plot_path, name = 'fake_2d_dk' +str(dk) +'_s' + str(int(size))) + F.save_light( + path=plot_path, name="fake_2d_dk" + str(dk) + "_s" + str(int(size)) + ) plt.show() - diff --git a/analysis/X01_track_visualization.py b/analysis/X01_track_visualization.py index 7106089d..5dc7dd9d 100644 --- a/analysis/X01_track_visualization.py +++ b/analysis/X01_track_visualization.py @@ -1,16 +1,16 @@ - import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -22,137 +22,155 @@ import spicke_remover import datetime import concurrent.futures as futures -#import s3fs -# %% -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False -track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False - - - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -save_path = mconfig['paths']['work'] + '/B03_spectra_'+hemis+'/' -save_name = 'B03_'+track_name - -plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/B_spectra/' +# import s3fs +# %% +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False +track_name, batch_key, test_flag = "20190219073735_08070210_004_01", "SH_batch02", False + + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +save_path = mconfig["paths"]["work"] + "/B03_spectra_" + hemis + "/" +save_name = "B03_" + track_name + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/B_spectra/" +) MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -#Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +# Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data -load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) #\ +load_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # \ # %% -Gi = Gd['gt1r'][['dist', 'heights_c_weighted_mean', 'heights_c_std', 'N_photos', 'y','x']] -x = Gi['dist'] -xlims = x.iloc[0], x.iloc[-1] +Gi = Gd["gt1r"][ + ["dist", "heights_c_weighted_mean", "heights_c_std", "N_photos", "y", "x"] +] +x = Gi["dist"] +xlims = x.iloc[0], x.iloc[-1] - - -#Gi['slopes'] = dd +# Gi['slopes'] = dd # %% -T_max = 40 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 -x= np.array(Gi['dist']) +T_max = 40 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 +x = np.array(Gi["dist"]) dx = np.diff(x).mean() -min_datapoint = 1/k_0/dx +min_datapoint = 1 / k_0 / dx Lpoints = int(np.round(min_datapoint) * 20) -Lmeters =Lpoints * dx +Lmeters = Lpoints * dx -stancil_iter = spec.create_chunk_boundaries_unit_lengths(Lmeters, xlims, ov= None , iter_flag=False) +stancil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeters, xlims, ov=None, iter_flag=False +) # %% - - - # %% -def plot_beam(x,y, z, alpha_mean= 0.5, levels_std = 1, point_size= None ): +def plot_beam(x, y, z, alpha_mean=0.5, levels_std=1, point_size=None): if alpha_mean is False: alphas = 1 else: - alphas= (abs(z/z.std()/2) )**(1/2) - alphas= alphas*alpha_mean - alphas[alphas>=1] = 1 - alphas[alphas<0] = 0 + alphas = (abs(z / z.std() / 2)) ** (1 / 2) + alphas = alphas * alpha_mean + alphas[alphas >= 1] = 1 + alphas[alphas < 0] = 0 alphas[np.isnan(alphas)] = 0 if type(levels_std) is not np.array: - levels = np.linspace(-z.std()*levels_std, z.std()*levels_std, 30) + levels = np.linspace(-z.std() * levels_std, z.std() * levels_std, 30) else: levels = levels_std if point_size is None: - psize = abs(z*400)#**(1.5) + psize = abs(z * 400) # **(1.5) else: point_size = point_size - plt.scatter(x,y, s =psize ,vmin=levels[0], vmax=levels[-1], c= z, marker='o', cmap= plt.cm.coolwarm, alpha= alphas, edgecolors= 'none' ) - + plt.scatter( + x, + y, + s=psize, + vmin=levels[0], + vmax=levels[-1], + c=z, + marker="o", + cmap=plt.cm.coolwarm, + alpha=alphas, + edgecolors="none", + ) -def get_stacil_data(stancil, G3, key = 'dist' ): +def get_stacil_data(stancil, G3, key="dist"): mask = (G3[key] >= stancil[0]) & (G3[key] < stancil[2]) return G3[np.array(mask)] -def make_slopes(G3, key = 'heights_c_weighted_mean', spreed =10, verbose= False, photon_min =5): - dd = np.copy(G3['heights_c_weighted_mean']) - dd = np.gradient(dd) - dd, _ = spicke_remover.spicke_remover(dd, spreed=spreed, verbose=verbose) - dd_nans = (np.isnan(dd) ) + (G3['N_photos'] <= photon_min) +def make_slopes( + G3, key="heights_c_weighted_mean", spreed=10, verbose=False, photon_min=5 +): + + dd = np.copy(G3["heights_c_weighted_mean"]) + dd = np.gradient(dd) + dd, _ = spicke_remover.spicke_remover(dd, spreed=spreed, verbose=verbose) + dd_nans = (np.isnan(dd)) + (G3["N_photos"] <= photon_min) return dd, dd_nans -y_positions = { -'gt1l': 1, -'gt1r': 1.1, -'gt2l': 2, -'gt2r': 2.1, -'gt3l': 3, -'gt3r': 3.1 -} + +y_positions = {"gt1l": 1, "gt1r": 1.1, "gt2l": 2, "gt2r": 2.1, "gt3l": 3, "gt3r": 3.1} Gd.keys() -F = M.figure_axis_xy(8, 2, view_scale= 0.5) +F = M.figure_axis_xy(8, 2, view_scale=0.5) -stancil_iter = spec.create_chunk_boundaries_unit_lengths(Lmeters, xlims, ov= None , iter_flag=False) -#stancil = stancil_iter[:, 30] +stancil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeters, xlims, ov=None, iter_flag=False +) +# stancil = stancil_iter[:, 30] stancil_iter.shape -for stancil in stancil_iter[:,6:10].T: +for stancil in stancil_iter[:, 6:10].T: print(stancil) - for k,I in Gd.items(): + for k, I in Gd.items(): GG = get_stacil_data(stancil, I) slopes, nans = make_slopes(GG) - GG['slopes'] = slopes - #x,y, z = GG['lons'],GG['lats'] , GG['slopes'] - x,y, z = GG['dist'], GG['dist']*0 + y_positions[k] , GG['slopes'] - - plot_beam(x,y, z) - plt.plot(x.iloc[0], y.iloc[0], 'o', color='orange', markersize=5, alpha = 1) - plt.plot(x.iloc[-1], y.iloc[-1], '|', color='green', markersize=20, alpha = 1) - #plt.axis('equal') + GG["slopes"] = slopes + # x,y, z = GG['lons'],GG['lats'] , GG['slopes'] + x, y, z = GG["dist"], GG["dist"] * 0 + y_positions[k], GG["slopes"] + + plot_beam(x, y, z) + plt.plot(x.iloc[0], y.iloc[0], "o", color="orange", markersize=5, alpha=1) + plt.plot(x.iloc[-1], y.iloc[-1], "|", color="green", markersize=20, alpha=1) + # plt.axis('equal') diff --git a/analysis/_B02_make_spectra.py b/analysis/_B02_make_spectra.py index 888db747..43eedaba 100644 --- a/analysis/_B02_make_spectra.py +++ b/analysis/_B02_make_spectra.py @@ -1,15 +1,16 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -21,347 +22,454 @@ import spicke_remover import datetime -#import s3fs +# import s3fs # %% -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190205231558_06030212_004_01', 'SH_batch02', False - - - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' -load_file = load_path + 'processed_' + ATlevel + '_' + track_name + '.h5' - -save_path = mconfig['paths']['work'] + '/B02_spectra_'+hemis+'/' -save_name = 'B02_'+track_name - -plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/B_spectra/' +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190205231558_06030212_004_01', 'SH_batch02', False + + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +load_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" +load_file = load_path + "processed_" + ATlevel + "_" + track_name + ".h5" + +save_path = mconfig["paths"]["work"] + "/B02_spectra_" + hemis + "/" +save_name = "B02_" + track_name + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/B_spectra/" +) MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +Gfilt = io.load_pandas_table_dict( + track_name + "_B01_regridded", load_path +) # rhis is the rar photon data +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # # %% test amount of nans in the data -nan_fraction= list() +nan_fraction = list() for I in Gd.values(): - nan_fraction.append( np.sum(np.isnan(I['heights_c_std'])) / I['heights_c_std'].shape[0] ) + nan_fraction.append( + np.sum(np.isnan(I["heights_c_std"])) / I["heights_c_std"].shape[0] + ) if np.array(nan_fraction).mean() > 0.95: - print('nan fraction > 95%, pass this track, add to bad tracks') - MT.json_save(track_name, bad_track_path, {'nan_fraction': np.array(nan_fraction).mean(), 'date': str(datetime.date.today()) }) - print('exit.') + print("nan fraction > 95%, pass this track, add to bad tracks") + MT.json_save( + track_name, + bad_track_path, + { + "nan_fraction": np.array(nan_fraction).mean(), + "date": str(datetime.date.today()), + }, + ) + print("exit.") exit() # %% test LS with an even grid where missing values are set to 0 imp.reload(spec) print(Gd.keys()) -Gi =Gd[ list(Gd.keys())[0] ] # to select a test beam +Gi = Gd[list(Gd.keys())[0]] # to select a test beam # derive spectal limits # Longest deserved period: -T_max = 40 #sec -k_0 = (2 * np.pi/ T_max)**2 / 9.81 -x= np.array(Gi['dist']) +T_max = 40 # sec +k_0 = (2 * np.pi / T_max) ** 2 / 9.81 +x = np.array(Gi["dist"]) dx = np.diff(x).mean() -min_datapoint = 1/k_0/dx +min_datapoint = 1 / k_0 / dx Lpoints = int(np.round(min_datapoint) * 20) -Lmeters =Lpoints * dx - -#plt.plot(np.diff(np.array(Gi['dist']))) -print('L number of gridpoint:', Lpoints) -print('L length in meters:', Lmeters) -print('approx number windows', 2* Gi['dist'].iloc[-1] /Lmeters-1 ) - - -S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( x, Lpoints, dx= dx, method='fft', minimum_frequency=None, maximum_frequency=None, samples_per_peak=0.005) +Lmeters = Lpoints * dx + +# plt.plot(np.diff(np.array(Gi['dist']))) +print("L number of gridpoint:", Lpoints) +print("L length in meters:", Lmeters) +print("approx number windows", 2 * Gi["dist"].iloc[-1] / Lmeters - 1) + + +S_pwelch_k, S_pwelch_dk = spec.calc_freq_LS( + x, + Lpoints, + dx=dx, + method="fft", + minimum_frequency=None, + maximum_frequency=None, + samples_per_peak=0.005, +) S_pwelch_k.shape -hkey= 'heights_c_weighted_mean' +hkey = "heights_c_weighted_mean" -G_LS= dict() -G_fft= dict() +G_LS = dict() +G_fft = dict() imp.reload(spec) # %% -k =all_beams[0] +k = all_beams[0] for k in all_beams: - x = Gd[k]['dist'] - xlims = x.iloc[0], x.iloc[-1] - dd = np.copy(Gd[k][hkey]) + x = Gd[k]["dist"] + xlims = x.iloc[0], x.iloc[-1] + dd = np.copy(Gd[k][hkey]) print(xlims) - # %% + # %% F = M.figure_axis_xy(6, 3) plt.subplot(2, 1, 1) - plt.plot(x, dd, 'gray', label='displacement (m) ') + plt.plot(x, dd, "gray", label="displacement (m) ") # compute slope spectra !! - dd = np.gradient(dd) - dd, _ = spicke_remover.spicke_remover(dd, spreed=10, verbose=False) - dd_nans = (np.isnan(dd) ) + (Gd[k]['N_photos'] <= 5) + dd = np.gradient(dd) + dd, _ = spicke_remover.spicke_remover(dd, spreed=10, verbose=False) + dd_nans = (np.isnan(dd)) + (Gd[k]["N_photos"] <= 5) dd_no_nans = dd[~dd_nans] - x_no_nans = x[~dd_nans] + x_no_nans = x[~dd_nans] # outlyers = abs(dd) > dd.std() *5 # dd= dd[~outlyers] # x= x[~outlyers] - plt.plot(x_no_nans, dd_no_nans, 'black', label='slope (m/m)') + plt.plot(x_no_nans, dd_no_nans, "black", label="slope (m/m)") plt.legend() - print('LS') - S = spec.wavenumber_spectrogram_LS( np.array(x_no_nans), np.array(dd_no_nans), Lmeters, waven_method = S_pwelch_k[1:] , ov=None, window=None, kjumps=2) - G_ls_i = S.cal_spectrogram(xlims= xlims) - S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True) + print("LS") + S = spec.wavenumber_spectrogram_LS( + np.array(x_no_nans), + np.array(dd_no_nans), + Lmeters, + waven_method=S_pwelch_k[1:], + ov=None, + window=None, + kjumps=2, + ) + G_ls_i = S.cal_spectrogram(xlims=xlims) + S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.parceval(add_attrs=True) # assign beam coordinate - G_ls_i.coords['beam'] = str(k)#(('beam'), str(k)) - G_ls_i = G_ls_i.expand_dims(dim = 'beam', axis = 2) + G_ls_i.coords["beam"] = str(k) # (('beam'), str(k)) + G_ls_i = G_ls_i.expand_dims(dim="beam", axis=2) # repack such that all coords are associated with beam - G_ls_i.coords['N_per_stancil'] = (('x', 'beam' ), np.expand_dims(G_ls_i['N_per_stancil'], 1)) - G_ls_i.coords['mean_El'] = (('k', 'beam' ), np.expand_dims(G_ls_i['mean_El'], 1)) - G_ls_i.coords['mean_Eu'] = (('k', 'beam' ), np.expand_dims(G_ls_i['mean_Eu'], 1)) + G_ls_i.coords["N_per_stancil"] = ( + ("x", "beam"), + np.expand_dims(G_ls_i["N_per_stancil"], 1), + ) + G_ls_i.coords["mean_El"] = (("k", "beam"), np.expand_dims(G_ls_i["mean_El"], 1)) + G_ls_i.coords["mean_Eu"] = (("k", "beam"), np.expand_dims(G_ls_i["mean_Eu"], 1)) # calculate number data points def get_stancil_nans(stancil): x_mask = (stancil[0] < x) & (x <= stancil[-1]) - idata = Gd[k]['N_photos'][x_mask] + idata = Gd[k]["N_photos"][x_mask] return stancil[1], idata.sum() - photon_list = np.array(list(dict(map( get_stancil_nans, copy.copy(S.stancil_iter) )).values())) - G_ls_i.coords['N_photons'] = (('x', 'beam' ), np.expand_dims(photon_list, 1)) + photon_list = np.array( + list(dict(map(get_stancil_nans, copy.copy(S.stancil_iter))).values()) + ) + G_ls_i.coords["N_photons"] = (("x", "beam"), np.expand_dims(photon_list, 1)) G_LS[k] = G_ls_i plt.subplot(2, 1, 2) - GG =G_ls_i.squeeze() - plt.plot(GG.k, GG.mean('x'), 'k', label='mean LS') - + GG = G_ls_i.squeeze() + plt.plot(GG.k, GG.mean("x"), "k", label="mean LS") # standard FFT - print('FFT') - dd[dd_nans] = 0 + print("FFT") + dd[dd_nans] = 0 S = spec.wavenumber_spectrogram(x, dd, Lpoints) G = S.cal_spectrogram() - S.mean_spectral_error() # add x-mean spectal error estimate to xarray - S.parceval(add_attrs= True) + S.mean_spectral_error() # add x-mean spectal error estimate to xarray + S.parceval(add_attrs=True) # assign beam coordinate - G.coords['beam'] = str(k)#(('beam'), str(k)) - G = G.expand_dims(dim = 'beam', axis = 2) - G.coords['mean_El'] = (('k', 'beam' ), np.expand_dims(G['mean_El'], 1)) - G.coords['mean_Eu'] = (('k', 'beam' ), np.expand_dims(G['mean_Eu'], 1)) - G.coords['x'] = G.coords['x'] * dx # adjust x-coodinate definition + G.coords["beam"] = str(k) # (('beam'), str(k)) + G = G.expand_dims(dim="beam", axis=2) + G.coords["mean_El"] = (("k", "beam"), np.expand_dims(G["mean_El"], 1)) + G.coords["mean_Eu"] = (("k", "beam"), np.expand_dims(G["mean_Eu"], 1)) + G.coords["x"] = G.coords["x"] * dx # adjust x-coodinate definition stancil_iter = spec.create_chunk_boundaries(int(Lpoints), dd_nans.size) + def get_stancil_nans(stancil): - idata = dd_nans[stancil[0]:stancil[-1]] + idata = dd_nans[stancil[0] : stancil[-1]] return stancil[1], idata.size - idata.sum() - N_list = np.array(list(dict(map( get_stancil_nans, stancil_iter )).values())) + N_list = np.array(list(dict(map(get_stancil_nans, stancil_iter)).values())) # repack such that all coords are associated with beam - G.coords['N_per_stancil'] = (('x', 'beam' ), np.expand_dims(N_list, 1)) - + G.coords["N_per_stancil"] = (("x", "beam"), np.expand_dims(N_list, 1)) G_fft[k] = G - GG =G.squeeze() + GG = G.squeeze() - plt.plot(G.k, GG[:, GG['N_per_stancil'] > 10 ].mean('x'), 'darkblue', label='mean FFT') - #plt.plot(G.k, GG.mean('x'), 'lightblue', label='mean FFT') + plt.plot( + G.k, GG[:, GG["N_per_stancil"] > 10].mean("x"), "darkblue", label="mean FFT" + ) + # plt.plot(G.k, GG.mean('x'), 'lightblue', label='mean FFT') plt.legend() plt.show() - F.save_light(path=plot_path, name = 'B02_control_'+k+'_' + track_name) - #print('saved as '+'B02_control_'+k+'_' + track_name) - #print(np.isinf(G).sum().data) - + F.save_light(path=plot_path, name="B02_control_" + k + "_" + track_name) + # print('saved as '+'B02_control_'+k+'_' + track_name) + # print(np.isinf(G).sum().data) # %% + def dict_weighted_mean(Gdict, weight_key): """ returns the weighted meean of a dict of xarray, data_arrays weight_key must be in the xr.DataArrays """ - #Gdict = G_LS + # Gdict = G_LS # weight_key='N_per_stancil' - akey = list( Gdict.keys() )[0] + akey = list(Gdict.keys())[0] GSUM = Gdict[akey].copy() - GSUM.data = np.zeros(GSUM.shape) + GSUM.data = np.zeros(GSUM.shape) N_per_stancil = GSUM.N_per_stancil * 0 - N_photons = np.zeros(GSUM.N_per_stancil.size) + N_photons = np.zeros(GSUM.N_per_stancil.size) - counter= 0 - for k,I in Gdict.items(): - I =I.squeeze() - GSUM += I.where( ~np.isnan(I), 0) * I[weight_key] #.sel(x=GSUM.x) - N_per_stancil += I[weight_key] - if 'N_photons' in GSUM.coords: - N_photons += I['N_photons'] - counter+=1 + counter = 0 + for k, I in Gdict.items(): + I = I.squeeze() + GSUM += I.where(~np.isnan(I), 0) * I[weight_key] # .sel(x=GSUM.x) + N_per_stancil += I[weight_key] + if "N_photons" in GSUM.coords: + N_photons += I["N_photons"] + counter += 1 - GSUM = GSUM / N_per_stancil /counter + GSUM = GSUM / N_per_stancil / counter - if 'N_photons' in GSUM.coords: - GSUM.coords['N_photons'] = (('x', 'beam'), np.expand_dims(N_photons, 1) ) + if "N_photons" in GSUM.coords: + GSUM.coords["N_photons"] = (("x", "beam"), np.expand_dims(N_photons, 1)) - GSUM['beam'] = ['weighted_mean'] - GSUM.name='power_spec' + GSUM["beam"] = ["weighted_mean"] + GSUM.name = "power_spec" return GSUM -G_LS_wmean = dict_weighted_mean(G_LS, 'N_per_stancil') -G_fft_wmean = dict_weighted_mean(G_fft, 'N_per_stancil') + +G_LS_wmean = dict_weighted_mean(G_LS, "N_per_stancil") +G_fft_wmean = dict_weighted_mean(G_fft, "N_per_stancil") # %% plot -def plot_wavenumber_spectrogram(ax, Gi, clev, title= None, plot_photon_density=True ): +def plot_wavenumber_spectrogram(ax, Gi, clev, title=None, plot_photon_density=True): - x_lambda= 1/Gi.k - plt.pcolormesh(Gi.x/1e3, x_lambda , Gi, cmap=plt.cm.ocean_r , vmin = clev[0], vmax = clev[-1]) + x_lambda = 1 / Gi.k + plt.pcolormesh( + Gi.x / 1e3, x_lambda, Gi, cmap=plt.cm.ocean_r, vmin=clev[0], vmax=clev[-1] + ) - ax.set_yscale('log') + ax.set_yscale("log") # plt.colorbar(orientation='vertical', pad=0.06, label='Spectral Power (m^2/m)') if plot_photon_density: - plt.plot(Gi.x/1e3, x_lambda[-1] + (Gi.N_per_stancil/Gi.N_per_stancil.max() ) * 10 , c='black', linewidth= 0.8, label='NAN-density' ) - plt.fill_between(Gi.x/1e3, x_lambda[-1] + (Gi.N_per_stancil/Gi.N_per_stancil.max() ) * 10, 0, color='gray', alpha = 0.3) - ax.axhline(30, color='black', linewidth=0.3) - - #plt.xlabel('Distance from the Ice Edge (km)') + plt.plot( + Gi.x / 1e3, + x_lambda[-1] + (Gi.N_per_stancil / Gi.N_per_stancil.max()) * 10, + c="black", + linewidth=0.8, + label="NAN-density", + ) + plt.fill_between( + Gi.x / 1e3, + x_lambda[-1] + (Gi.N_per_stancil / Gi.N_per_stancil.max()) * 10, + 0, + color="gray", + alpha=0.3, + ) + ax.axhline(30, color="black", linewidth=0.3) + + # plt.xlabel('Distance from the Ice Edge (km)') plt.ylim(x_lambda[-1], x_lambda[0]) - plt.title(title, loc='left') + plt.title(title, loc="left") + -#Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() -#Gmean = G_LS_wmean.rolling(x=2, min_periods= 1, center=True).mean() +# Gplot = G.rolling(x=5, min_periods= 1, center=True).mean() +# Gmean = G_LS_wmean.rolling(x=2, min_periods= 1, center=True).mean() Gmean = G_LS_wmean.rolling(k=5, center=True).mean() -#Gmean = Gmean.where(~np.isnan(Gmean), 0) +# Gmean = Gmean.where(~np.isnan(Gmean), 0) -k_max_range = Gmean.k[Gmean.mean('x').argmax().data].data* 0.75, Gmean.k[Gmean.mean('x').argmax().data].data* 1, Gmean.k[Gmean.mean('x').argmax().data].data* 1.25 +k_max_range = ( + Gmean.k[Gmean.mean("x").argmax().data].data * 0.75, + Gmean.k[Gmean.mean("x").argmax().data].data * 1, + Gmean.k[Gmean.mean("x").argmax().data].data * 1.25, +) font_for_print() -F = M.figure_axis_xy(6.5, 5.6, container= True, view_scale =1) +F = M.figure_axis_xy(6.5, 5.6, container=True, view_scale=1) -plt.suptitle('LS and FFT Slope Spectrograms\n' + track_name, y = 0.98) -gs = GridSpec(3,3, wspace=0.2, hspace=.5)#figure=fig, -#clev=np.arange(0, 6, 0.1)*3 +plt.suptitle("LS and FFT Slope Spectrograms\n" + track_name, y=0.98) +gs = GridSpec(3, 3, wspace=0.2, hspace=0.5) # figure=fig, +# clev=np.arange(0, 6, 0.1)*3 -#%matplotlib inline +# %matplotlib inline -clev = M.clevels( [Gmean.quantile(0.01).data, Gmean.quantile(0.99).data * 1.2], 31)* 1 -xlims= Gmean.x[0]/1e3, Gmean.x[-1]/1e3 +clev = M.clevels([Gmean.quantile(0.01).data, Gmean.quantile(0.99).data * 1.2], 31) * 1 +xlims = Gmean.x[0] / 1e3, Gmean.x[-1] / 1e3 -for pos, k, pflag in zip([gs[0, 0],gs[0, 1],gs[0, 2] ], high_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[0, 0], gs[0, 1], gs[0, 2]], high_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G_LS[k].squeeze()#.rolling(k=10, x=2, min_periods= 1, center=True).mean() - #Gplot.mean('x').plot() + Gplot = G_LS[k].squeeze() # .rolling(k=10, x=2, min_periods= 1, center=True).mean() + # Gplot.mean('x').plot() # #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) # - plot_wavenumber_spectrogram(ax0, Gplot, clev, title =k, plot_photon_density=True ) + plot_wavenumber_spectrogram(ax0, Gplot, clev, title=k, plot_photon_density=True) plt.xlim(xlims) # if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -for pos, k, pflag in zip([gs[1, 0],gs[1, 1],gs[1, 2] ], low_beams, [True, False, False] ): +for pos, k, pflag in zip( + [gs[1, 0], gs[1, 1], gs[1, 2]], low_beams, [True, False, False] +): ax0 = F.fig.add_subplot(pos) - Gplot = G_LS[k].squeeze()#.rolling(k=10, x=2, min_periods= 1, center=True).mean() - #Gplot.mean('x').plot() + Gplot = G_LS[k].squeeze() # .rolling(k=10, x=2, min_periods= 1, center=True).mean() + # Gplot.mean('x').plot() # #plt.pcolormesh(G.x/1e3, 1/G.k , G, norm=colors.LogNorm(vmin=G.min()*1e6, vmax=G.max()), cmap='PuBu')#, vmin = clev[0], vmax = clev[-1]) # - plot_wavenumber_spectrogram(ax0, Gplot, clev, title =k, plot_photon_density=True ) + plot_wavenumber_spectrogram(ax0, Gplot, clev, title=k, plot_photon_density=True) plt.xlim(xlims) # if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() -pos, k, pflag = gs[2, 0], 'Power(weighted mean) \n10 $\log_{10}( (m/m)^2 m )$', True +pos, k, pflag = gs[2, 0], "Power(weighted mean) \n10 $\log_{10}( (m/m)^2 m )$", True ax0 = F.fig.add_subplot(pos) -Gplot = G_LS_wmean.squeeze().rolling(k=5, min_periods= 1, center=True).median().rolling(x=3, min_periods= 1, center=True).median() +Gplot = ( + G_LS_wmean.squeeze() + .rolling(k=5, min_periods=1, center=True) + .median() + .rolling(x=3, min_periods=1, center=True) + .median() +) dd = 10 * np.log10(Gplot) -dd= dd.where(~np.isinf(dd), np.nan ) -clev_log = M.clevels( [dd.quantile(0.01).data, dd.quantile(0.98).data * 1.2], 31)* 1 -plot_wavenumber_spectrogram(ax0, dd, clev_log , title =k, plot_photon_density= True) +dd = dd.where(~np.isinf(dd), np.nan) +clev_log = M.clevels([dd.quantile(0.01).data, dd.quantile(0.98).data * 1.2], 31) * 1 +plot_wavenumber_spectrogram(ax0, dd, clev_log, title=k, plot_photon_density=True) plt.xlim(xlims) # plt.plot(Gplot.x/1e3, 10* nan_list +20 , c='black', label='NAN-density' ) # ax0.axhline(30, color='black', linewidth=0.5) -ax0.axhline(1/k_max_range[0], color='red', linestyle= '--', linewidth= 0.5) -ax0.axhline(1/k_max_range[1], color='red', linestyle= '-', linewidth= 0.5) -ax0.axhline(1/k_max_range[2], color='red', linestyle= '--', linewidth= 0.5) +ax0.axhline(1 / k_max_range[0], color="red", linestyle="--", linewidth=0.5) +ax0.axhline(1 / k_max_range[1], color="red", linestyle="-", linewidth=0.5) +ax0.axhline(1 / k_max_range[2], color="red", linestyle="--", linewidth=0.5) if pflag: - plt.ylabel('Wave length\n(meters)') + plt.ylabel("Wave length\n(meters)") plt.legend() pos = gs[2, 1] ax0 = F.fig.add_subplot(pos) -plt.title('Photons density ($m^{-1}$)', loc='left') - -for k,I in G_LS.items(): - plt.plot(Gplot.x/1e3, I.N_photons/Lmeters, label=k, linewidth=0.8) -plt.plot(Gplot.x/1e3, G_LS_wmean.N_photons/3/Lmeters , c='black', label='ave Photons' , linewidth=0.8) +plt.title("Photons density ($m^{-1}$)", loc="left") + +for k, I in G_LS.items(): + plt.plot(Gplot.x / 1e3, I.N_photons / Lmeters, label=k, linewidth=0.8) +plt.plot( + Gplot.x / 1e3, + G_LS_wmean.N_photons / 3 / Lmeters, + c="black", + label="ave Photons", + linewidth=0.8, +) plt.xlim(xlims) -plt.xlabel('Distance from the Ice Edge (km)') +plt.xlabel("Distance from the Ice Edge (km)") pos = gs[2, 2] ax0 = F.fig.add_subplot(pos) -ax0.set_yscale('log') - -plt.title('Peak Spectal Power', loc='left') - -for k,I in G_LS.items(): - plt.scatter(I.x.data/1e3, I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k').data *1e3 , s=0.5, marker='.', color='red', alpha= 0.3) - -for k,I in G_fft.items(): - I= I.squeeze() - I= I[:,I.N_per_stancil >= I.N_per_stancil.max().data*0.9] - plt.scatter(I.x/1e3, I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3 , s=0.5, marker='.', c='blue', alpha= 0.3) - - -Gplot= G_fft_wmean.squeeze() -Gplot = Gplot[:,Gplot.N_per_stancil >= Gplot.N_per_stancil.max().data*0.9] -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3, '.', markersize=1.5 , c='blue', label= 'FFT') - -Gplot= G_LS_wmean -plt.plot(Gplot.x/1e3, Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate('k') *1e3 , '.' , markersize=1.5, c='red', label= 'LS') - -plt.ylabel('1e-3 $(m)^2~m$') +ax0.set_yscale("log") + +plt.title("Peak Spectal Power", loc="left") + +for k, I in G_LS.items(): + plt.scatter( + I.x.data / 1e3, + I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k").data * 1e3, + s=0.5, + marker=".", + color="red", + alpha=0.3, + ) + +for k, I in G_fft.items(): + I = I.squeeze() + I = I[:, I.N_per_stancil >= I.N_per_stancil.max().data * 0.9] + plt.scatter( + I.x / 1e3, + I.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + s=0.5, + marker=".", + c="blue", + alpha=0.3, + ) + + +Gplot = G_fft_wmean.squeeze() +Gplot = Gplot[:, Gplot.N_per_stancil >= Gplot.N_per_stancil.max().data * 0.9] +plt.plot( + Gplot.x / 1e3, + Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + ".", + markersize=1.5, + c="blue", + label="FFT", +) + +Gplot = G_LS_wmean +plt.plot( + Gplot.x / 1e3, + Gplot.sel(k=slice(k_max_range[0], k_max_range[2])).integrate("k") * 1e3, + ".", + markersize=1.5, + c="red", + label="LS", +) + +plt.ylabel("1e-3 $(m)^2~m$") plt.legend() -#plt.ylim(Gplot.min()*1.4, Gplot.max()*1.4 ) +# plt.ylim(Gplot.min()*1.4, Gplot.max()*1.4 ) plt.xlim(xlims) -F.save_light(path=plot_path, name = 'B02_specs_' + track_name +'_L'+str(Lmeters)) +F.save_light(path=plot_path, name="B02_specs_" + track_name + "_L" + str(Lmeters)) # %% repack data @@ -370,20 +478,21 @@ def repack_attributes(DD): for k in attr_dim_list: for ka in list(DD[k].attrs.keys()): I = DD[k] - I.coords[ka] = ( 'beam', np.expand_dims(I.attrs[ka], 1) ) + I.coords[ka] = ("beam", np.expand_dims(I.attrs[ka], 1)) return DD -G_LS[G_LS_wmean.beam.data[0]] =G_LS_wmean -G_fft[G_fft_wmean.beam.data[0]] =G_fft_wmean + +G_LS[G_LS_wmean.beam.data[0]] = G_LS_wmean +G_fft[G_fft_wmean.beam.data[0]] = G_fft_wmean G_LS = repack_attributes(G_LS) G_fft = repack_attributes(G_fft) # %% save results -G_LS_DS = xr.merge(G_LS.values()) -G_LS_DS['name'] = 'LS_power_spectra' -G_LS_DS.to_netcdf(save_path+save_name+'_LS.nc') +G_LS_DS = xr.merge(G_LS.values()) +G_LS_DS["name"] = "LS_power_spectra" +G_LS_DS.to_netcdf(save_path + save_name + "_LS.nc") -G_fft_DS = xr.merge(G_fft.values()) -G_fft_DS['name']= 'FFT_power_spectra' -G_fft_DS.to_netcdf(save_path+save_name+'_FFT.nc') +G_fft_DS = xr.merge(G_fft.values()) +G_fft_DS["name"] = "FFT_power_spectra" +G_fft_DS.to_netcdf(save_path + save_name + "_FFT.nc") diff --git a/analysis/load_ICEsat2.py b/analysis/load_ICEsat2.py index 21eaf30e..a91453fc 100644 --- a/analysis/load_ICEsat2.py +++ b/analysis/load_ICEsat2.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ There @@ -8,14 +9,15 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -30,12 +32,13 @@ import convert_GPS_time as cGPS -#import s3fs +# import s3fs # %% # Python reader based on Pandas. Other reader examples available in readers.py -def save_pandas_table(table_dict, name , save_path): + +def save_pandas_table(table_dict, name, save_path): if not os.path.exists(save_path): os.makedirs(save_path) @@ -43,181 +46,213 @@ def save_pandas_table(table_dict, name , save_path): import warnings from pandas import HDFStore from pandas.io.pytables import PerformanceWarning - warnings.filterwarnings('ignore',category=PerformanceWarning) - with HDFStore(save_path+'/'+name+'.h5') as store: - for name,table in table_dict.items(): - store[name]=table + warnings.filterwarnings("ignore", category=PerformanceWarning) + with HDFStore(save_path + "/" + name + ".h5") as store: + for name, table in table_dict.items(): + store[name] = table -def getATL03data(fileT, numpy=0, beam='gt1l', maxElev=1e6): +def getATL03data(fileT, numpy=0, beam="gt1l", maxElev=1e6): # Add in a proper description of the function here # Open the file - ATL03 = h5py.File(fileT, 'r') + ATL03 = h5py.File(fileT, "r") - lons=ATL03[beam+'/heights/lon_ph'][:] - lats=ATL03[beam+'/heights/lat_ph'][:] + lons = ATL03[beam + "/heights/lon_ph"][:] + lats = ATL03[beam + "/heights/lat_ph"][:] # Along track distance from equator i think. - along_track_distance=ATL03[beam+'/heights/dist_ph_along'][:] + along_track_distance = ATL03[beam + "/heights/dist_ph_along"][:] # Nathan says it's the number of seconds since the GPS epoch on midnight Jan. 6, 1980 - delta_time=ATL03[beam+'/heights/delta_time'][:] + delta_time = ATL03[beam + "/heights/delta_time"][:] # #Add this value to delta time parameters to compute full gps_seconds - atlas_epoch=ATL03['/ancillary_data/atlas_sdp_gps_epoch'][:] + atlas_epoch = ATL03["/ancillary_data/atlas_sdp_gps_epoch"][:] # Conversion of delta_time to a calendar date temp = cGPS.convert_GPS_time(atlas_epoch[0] + delta_time, OFFSET=0.0) # Express delta_time relative to start time of granule - delta_time_granule=delta_time-delta_time[0] + delta_time_granule = delta_time - delta_time[0] - year = temp['year'][:].astype('int') - month = temp['month'][:].astype('int') - day = temp['day'][:].astype('int') - hour = temp['hour'][:].astype('int') - minute = temp['minute'][:].astype('int') - second = temp['second'][:].astype('int') + year = temp["year"][:].astype("int") + month = temp["month"][:].astype("int") + day = temp["day"][:].astype("int") + hour = temp["hour"][:].astype("int") + minute = temp["minute"][:].astype("int") + second = temp["second"][:].astype("int") # Primary variables of interest # Photon height - heights=ATL03[beam+'/heights/h_ph'][:] + heights = ATL03[beam + "/heights/h_ph"][:] print(heights.shape) # Flag for signal confidence # column index: 0=Land; 1=Ocean; 2=SeaIce; 3=LandIce; 4=InlandWater # values: - #-- -1: Events not associated with a specific surface type - #-- 0: noise - #-- 1: buffer but algorithm classifies as background - #-- 2: low - #-- 3: medium - #-- 4: high - signal_confidence=ATL03[beam+'/heights/signal_conf_ph'][:,2] + # -- -1: Events not associated with a specific surface type + # -- 0: noise + # -- 1: buffer but algorithm classifies as background + # -- 2: low + # -- 3: medium + # -- 4: high + signal_confidence = ATL03[beam + "/heights/signal_conf_ph"][:, 2] print(signal_confidence.shape) # Add photon rate and background rate to the reader here ATL03.close() - if (numpy==1): + if numpy == 1: # list the variables you want to output here.. return along_track_dist, elev else: - dF = pd.DataFrame({'heights':heights, 'lons':lons, 'lats':lats, 'signal_confidence':signal_confidence, - 'delta_time':delta_time_granule,'along_track_distance':along_track_distance, 'year':year, 'month':month, 'day':day, 'hour':hour, 'second':second}) + dF = pd.DataFrame( + { + "heights": heights, + "lons": lons, + "lats": lats, + "signal_confidence": signal_confidence, + "delta_time": delta_time_granule, + "along_track_distance": along_track_distance, + "year": year, + "month": month, + "day": day, + "hour": hour, + "second": second, + } + ) # Filter out high elevation values - #dF = dF[(dF['signal_confidence']>2)] + # dF = dF[(dF['signal_confidence']>2)] # Reset row indexing - #dF=dF.reset_index(drop=True) + # dF=dF.reset_index(drop=True) return dF -bpath=base_path + 'data/' -path=base_path + 'data/data1/processed_ATL03_20190515060436_07170312_002_02.h5' +bpath = base_path + "data/" +path = base_path + "data/data1/processed_ATL03_20190515060436_07170312_002_02.h5" # %% -T =getATL03data(path, beam= 'gt2l') +T = getATL03data(path, beam="gt2l") -Tsel= T[(T['signal_confidence']>3) & (T['heights']<100) ] +Tsel = T[(T["signal_confidence"] > 3) & (T["heights"] < 100)] -T2=Tsel[ (Tsel['delta_time']>5) & (Tsel['delta_time']<24) ] +T2 = Tsel[(Tsel["delta_time"] > 5) & (Tsel["delta_time"] < 24)] # %% M.figure_axis_xy(10, 3, view_scale=0.8) -#plt.plot(Tsel['lats'], Tsel['heights'], '.') -plt.scatter(T2['lats'], T2['heights'],0.2, marker='.') +# plt.plot(Tsel['lats'], Tsel['heights'], '.') +plt.scatter(T2["lats"], T2["heights"], 0.2, marker=".") # %% -r_e= 6.3710E+6 -dx= r_e*2*np.pi/360.0 -deglon_in_m= np.cos(T2['lats']*np.pi/180.0)*dx +r_e = 6.3710e6 +dx = r_e * 2 * np.pi / 360.0 +deglon_in_m = np.cos(T2["lats"] * np.pi / 180.0) * dx + def derive_axis(TT): - TT['x']=(TT['lats'].max() - TT['lats']) *dx - TT['y']=(TT['lons'] - TT['lons'].min()) *deglon_in_m - TT['l']=np.sqrt(TT['x']**2 + TT['y']**2) - TT['l']= TT['l']- TT['l'].min() - TT=TT.sort_values(by='l') + TT["x"] = (TT["lats"].max() - TT["lats"]) * dx + TT["y"] = (TT["lons"] - TT["lons"].min()) * deglon_in_m + TT["l"] = np.sqrt(TT["x"] ** 2 + TT["y"] ** 2) + TT["l"] = TT["l"] - TT["l"].min() + TT = TT.sort_values(by="l") return TT + def reduce_to_height_distance(TT, key, dx=1): from scipy.interpolate import interp1d - x1 = np.arange(TT['l'].min(),TT['l'].max(), dx) - y1 = np.interp(x1, TT['l'], TT[key] ) + x1 = np.arange(TT["l"].min(), TT["l"].max(), dx) + y1 = np.interp(x1, TT["l"], TT[key]) return x1, y1, TT + def poly_corrent(x, y, poly_order=7, plot_flag=False): - z = np.polyfit(x , y , poly_order) + z = np.polyfit(x, y, poly_order) p = np.poly1d(z) if plot_flag: - plt.plot(x,y, '.', markersize=0.2,) - plt.plot(x, p(x), '-', markersize=0.2,) - + plt.plot( + x, + y, + ".", + markersize=0.2, + ) + plt.plot( + x, + p(x), + "-", + markersize=0.2, + ) return y - p(x) # %% T2 = derive_axis(T2) -T2['heights_adjusted']= poly_corrent(T2['l'], T2['heights']) -T2['heights_smth'] = T2['heights_adjusted'].rolling(100).median() -x1, y1, T3 = reduce_to_height_distance(T2,'heights_smth', dx=0.1) -point_dentisty = len(T2['l'])/ T2['l'].max() # point/ meter +T2["heights_adjusted"] = poly_corrent(T2["l"], T2["heights"]) +T2["heights_smth"] = T2["heights_adjusted"].rolling(100).median() +x1, y1, T3 = reduce_to_height_distance(T2, "heights_smth", dx=0.1) +point_dentisty = len(T2["l"]) / T2["l"].max() # point/ meter -x1/1e3 -T2['l'] +x1 / 1e3 +T2["l"] # %% F = M.figure_axis_xy(5, 6.5, view_scale=0.8) -plt.subplot(2,1 , 1) -plt.title('Filtered Photon Heights in Sea Ice') -plt.plot(T3['l'], T3['heights_adjusted'], 'ko', markersize=1,alpha=0.5) +plt.subplot(2, 1, 1) +plt.title("Filtered Photon Heights in Sea Ice") +plt.plot(T3["l"], T3["heights_adjusted"], "ko", markersize=1, alpha=0.5) -plt.plot(T3['l'], T3['heights_smth'], 'bo', markersize=1, alpha=0.5) +plt.plot(T3["l"], T3["heights_smth"], "bo", markersize=1, alpha=0.5) -plt.plot(x1, y1, '-r', markersize=2) +plt.plot(x1, y1, "-r", markersize=2) plt.xlim(3400, 5000) -plt.xlabel('Meters from the Sea Ice Edge') -plt.ylabel('Height Anomalie (meters)') +plt.xlabel("Meters from the Sea Ice Edge") +plt.ylabel("Height Anomalie (meters)") # %% F = M.figure_axis_xy(4.5, 3, view_scale=0.8) -#plt.subplot(2,1, 2) -plt.plot(T3['l']/1e3, T3['heights_adjusted'], 'ko', markersize=1,alpha=0.5, label='rar data') +# plt.subplot(2,1, 2) +plt.plot( + T3["l"] / 1e3, + T3["heights_adjusted"], + "ko", + markersize=1, + alpha=0.5, + label="rar data", +) -#plt.plot(T3['l']/1e3, T3['heights_smth'], 'bo', markersize=1, alpha=0.5) +# plt.plot(T3['l']/1e3, T3['heights_smth'], 'bo', markersize=1, alpha=0.5) -plt.plot(x1/1e3, y1, '-r', linewidth=1, label='best estimate') -plt.xlim(3400/1e3, 7000/1e3) -#plt.xlim(10000, 10500) -plt.xlabel('km from the Sea Ice Edge') -plt.ylabel('Height Anomalie (meters)') +plt.plot(x1 / 1e3, y1, "-r", linewidth=1, label="best estimate") +plt.xlim(3400 / 1e3, 7000 / 1e3) +# plt.xlim(10000, 10500) +plt.xlabel("km from the Sea Ice Edge") +plt.ylabel("Height Anomalie (meters)") plt.legend() -F.save_pup(path=bpath, name='filtered_photon_heights_pup') -F.save_light(path=bpath, name='filtered_photon_heights_pup_lght') -#plt.ylim(5, 12) +F.save_pup(path=bpath, name="filtered_photon_heights_pup") +F.save_light(path=bpath, name="filtered_photon_heights_pup_lght") +# plt.ylim(5, 12) # %% -Gnew =xr.DataArray(y1, coords={'dist': x1}, dims='dist') -Gnew.name='filtered_photon_heights' -Gnew.to_netcdf(bpath+'/data1/filtered_photon_heights.nc') +Gnew = xr.DataArray(y1, coords={"dist": x1}, dims="dist") +Gnew.name = "filtered_photon_heights" +Gnew.to_netcdf(bpath + "/data1/filtered_photon_heights.nc") -save_pandas_table({'T':T2}, 'data1/20190515060436_07170312_002_02_filted' , bpath) +save_pandas_table({"T": T2}, "data1/20190515060436_07170312_002_02_filted", bpath) diff --git a/analysis/load_ICEsat2_AL07.py b/analysis/load_ICEsat2_AL07.py index e167a260..4116b1be 100644 --- a/analysis/load_ICEsat2_AL07.py +++ b/analysis/load_ICEsat2_AL07.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ There @@ -8,14 +9,15 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -30,12 +32,13 @@ import convert_GPS_time as cGPS -#import s3fs +# import s3fs # %% # Python reader based on Pandas. Other reader examples available in readers.py -def save_pandas_table(table_dict, name , save_path): + +def save_pandas_table(table_dict, name, save_path): if not os.path.exists(save_path): os.makedirs(save_path) @@ -43,196 +46,244 @@ def save_pandas_table(table_dict, name , save_path): import warnings from pandas import HDFStore from pandas.io.pytables import PerformanceWarning - warnings.filterwarnings('ignore',category=PerformanceWarning) - with HDFStore(save_path+'/'+name+'.h5') as store: - for name,table in table_dict.items(): - store[name]=table + warnings.filterwarnings("ignore", category=PerformanceWarning) + with HDFStore(save_path + "/" + name + ".h5") as store: + for name, table in table_dict.items(): + store[name] = table -def getATL07data(fileT, numpy=0, beam='gt1r', maxElev=1e6): - # Add in a proper description of the function here +def getATL07data(fileT, numpy=0, beam="gt1r", maxElev=1e6): + # Add in a proper description of the function here # Open the file - ATL07 = h5py.File(fileT, 'r') + ATL07 = h5py.File(fileT, "r") - lons=ATL07[beam+'/sea_ice_segments/longitude'][:] - lats=ATL07[beam+'/sea_ice_segments/latitude'][:] + lons = ATL07[beam + "/sea_ice_segments/longitude"][:] + lats = ATL07[beam + "/sea_ice_segments/latitude"][:] # Along track distance from the equator crossing to the segment center. # I removed the first point so it's relative to the start of the beam - along_track_distance=ATL07[beam+'/sea_ice_segments/seg_dist_x'][:] - ATL07[beam+'/sea_ice_segments/seg_dist_x'][0] + along_track_distance = ( + ATL07[beam + "/sea_ice_segments/seg_dist_x"][:] + - ATL07[beam + "/sea_ice_segments/seg_dist_x"][0] + ) # Height segment ID (10 km segments) - height_segment_id=ATL07[beam+'/sea_ice_segments/height_segment_id'][:] + height_segment_id = ATL07[beam + "/sea_ice_segments/height_segment_id"][:] # Nathan says it's the number of seconds since the GPS epoch on midnight Jan. 6, 1980 - delta_time=ATL07[beam+'/sea_ice_segments/delta_time'][:] + delta_time = ATL07[beam + "/sea_ice_segments/delta_time"][:] # #Add this value to delta time parameters to compute full gps_seconds - atlas_epoch=ATL07['/ancillary_data/atlas_sdp_gps_epoch'][:] + atlas_epoch = ATL07["/ancillary_data/atlas_sdp_gps_epoch"][:] # Conversion of delta_time to a calendar date temp = cGPS.convert_GPS_time(atlas_epoch[0] + delta_time, OFFSET=0.0) - year = temp['year'][:].astype('int') - month = temp['month'][:].astype('int') - day = temp['day'][:].astype('int') - hour = temp['hour'][:].astype('int') - minute = temp['minute'][:].astype('int') - second = temp['second'][:].astype('int') - + year = temp["year"][:].astype("int") + month = temp["month"][:].astype("int") + day = temp["day"][:].astype("int") + hour = temp["hour"][:].astype("int") + minute = temp["minute"][:].astype("int") + second = temp["second"][:].astype("int") # Primary variables of interest # Beam segment height - elev=ATL07[beam+'/sea_ice_segments/heights/height_segment_height'][:] + elev = ATL07[beam + "/sea_ice_segments/heights/height_segment_height"][:] # Flag for potential leads, 0=sea ice, 1 = sea surface - ssh_flag=ATL07[beam+'/sea_ice_segments/heights/height_segment_ssh_flag'][:] + ssh_flag = ATL07[beam + "/sea_ice_segments/heights/height_segment_ssh_flag"][:] - #Quality metrics for each segment include confidence level in the surface height estimate, which is based on the number of photons, the background noise rate, and the error measure provided by the surface-finding algorithm. + # Quality metrics for each segment include confidence level in the surface height estimate, which is based on the number of photons, the background noise rate, and the error measure provided by the surface-finding algorithm. # Height quality flag, 1 for good fit, 0 for bad - quality=ATL07[beam+'/sea_ice_segments/heights/height_segment_quality'][:] - - elev_rms = ATL07[beam+'/sea_ice_segments/heights/height_segment_rms'][:] #RMS difference between modeled and observed photon height distribution - seg_length = ATL07[beam+'/sea_ice_segments/heights/height_segment_length_seg'][:] # Along track length of segment - height_confidence = ATL07[beam+'/sea_ice_segments/heights/height_segment_confidence'][:] # Height segment confidence flag - reflectance = ATL07[beam+'/sea_ice_segments/heights/height_segment_asr_calc'][:] # Apparent surface reflectance - ssh_flag = ATL07[beam+'/sea_ice_segments/heights/height_segment_ssh_flag'][:] # Flag for potential leads, 0=sea ice, 1 = sea surface - seg_type = ATL07[beam+'/sea_ice_segments/heights/height_segment_type'][:] # 0 = Cloud covered - gauss_width = ATL07[beam+'/sea_ice_segments/heights/height_segment_w_gaussian'][:] # Width of Gaussian fit - + quality = ATL07[beam + "/sea_ice_segments/heights/height_segment_quality"][:] + + elev_rms = ATL07[beam + "/sea_ice_segments/heights/height_segment_rms"][ + : + ] # RMS difference between modeled and observed photon height distribution + seg_length = ATL07[beam + "/sea_ice_segments/heights/height_segment_length_seg"][ + : + ] # Along track length of segment + height_confidence = ATL07[ + beam + "/sea_ice_segments/heights/height_segment_confidence" + ][ + : + ] # Height segment confidence flag + reflectance = ATL07[beam + "/sea_ice_segments/heights/height_segment_asr_calc"][ + : + ] # Apparent surface reflectance + ssh_flag = ATL07[beam + "/sea_ice_segments/heights/height_segment_ssh_flag"][ + : + ] # Flag for potential leads, 0=sea ice, 1 = sea surface + seg_type = ATL07[beam + "/sea_ice_segments/heights/height_segment_type"][ + : + ] # 0 = Cloud covered + gauss_width = ATL07[beam + "/sea_ice_segments/heights/height_segment_w_gaussian"][ + : + ] # Width of Gaussian fit # Geophysical corrections # NOTE: All of these corrections except ocean tides, DAC, and geoid undulations are applied to the ATL03 photon heights. # AVISO dynamic Atmospheric Correction (DAC) including inverted barometer (IB) effect (±5cm) - dac = ATL07[beam+'/sea_ice_segments/geophysical/height_segment_dac'][:] + dac = ATL07[beam + "/sea_ice_segments/geophysical/height_segment_dac"][:] # Solid Earth Tides (±40 cm, max) - earth = ATL07[beam+'/sea_ice_segments/geophysical/height_segment_earth'][:] + earth = ATL07[beam + "/sea_ice_segments/geophysical/height_segment_earth"][:] # Geoid (-105 to +90 m, max) - geoid = ATL07[beam+'/sea_ice_segments/geophysical/height_segment_geoid'][:] + geoid = ATL07[beam + "/sea_ice_segments/geophysical/height_segment_geoid"][:] # Local displacement due to Ocean Loading (-6 to 0 cm) - loadTide = ATL07[beam+'/sea_ice_segments/geophysical/height_segment_load'][:] + loadTide = ATL07[beam + "/sea_ice_segments/geophysical/height_segment_load"][:] # Ocean Tides including diurnal and semi-diurnal (harmonic analysis), # and longer period tides (dynamic and self-consistent equilibrium) (±5 m) - oceanTide = ATL07[beam+'/sea_ice_segments/geophysical/height_segment_ocean'][:] + oceanTide = ATL07[beam + "/sea_ice_segments/geophysical/height_segment_ocean"][:] # Deformation due to centrifugal effect from small variations in polar motion # (Solid Earth Pole Tide) (±1.5 cm, the ocean pole tide ±2mm amplitude is considered negligible) - poleTide = ATL07[beam+'/sea_ice_segments/geophysical/height_segment_pole'][:] + poleTide = ATL07[beam + "/sea_ice_segments/geophysical/height_segment_pole"][:] # Mean sea surface (±2 m) # Taken from ICESat and CryoSat-2, see Kwok and Morison [2015]) - mss = ATL07[beam+'/sea_ice_segments/geophysical/height_segment_mss'][:] + mss = ATL07[beam + "/sea_ice_segments/geophysical/height_segment_mss"][:] - photon_rate = ATL07[beam+'/sea_ice_segments/stats/photon_rate'][:] + photon_rate = ATL07[beam + "/sea_ice_segments/stats/photon_rate"][:] ATL07.close() - - if (numpy==1): + if numpy == 1: # list the variables you want to output here.. return along_track_dist, elev else: - dF = pd.DataFrame({'elev':elev, 'lons':lons, 'lats':lats, 'ssh_flag':ssh_flag, - 'quality_flag':quality, 'delta_time':delta_time,'along_track_distance':along_track_distance, 'height_segment_id':height_segment_id, 'photon_rate':photon_rate, 'year':year, 'month':month, 'day':day, 'hour':hour, 'second':second}) + dF = pd.DataFrame( + { + "elev": elev, + "lons": lons, + "lats": lats, + "ssh_flag": ssh_flag, + "quality_flag": quality, + "delta_time": delta_time, + "along_track_distance": along_track_distance, + "height_segment_id": height_segment_id, + "photon_rate": photon_rate, + "year": year, + "month": month, + "day": day, + "hour": hour, + "second": second, + } + ) # Filter out high elevation values - dF = dF[(dF['elev']5) & (T['delta_time']<24) ] +T = getATL07data(path, beam="gt2l") +T["delta_time"] = T["delta_time"] - T["delta_time"].min() +T = T[(T["delta_time"] > 5) & (T["delta_time"] < 24)] # %% -plt.plot(T['ssh_flag']) +plt.plot(T["ssh_flag"]) + +r_e = 6.3710e6 +dx = r_e * 2 * np.pi / 360.0 +deglon_in_m = np.cos(T["lats"] * np.pi / 180.0) * dx -r_e= 6.3710E+6 -dx= r_e*2*np.pi/360.0 -deglon_in_m= np.cos(T['lats']*np.pi/180.0)*dx def derive_axis(TT): - TT['x']=(TT['lats'].max() - TT['lats']) *dx - TT['y']=(TT['lons'] - TT['lons'].min()) *deglon_in_m - TT['l']=np.sqrt(TT['x']**2 + TT['y']**2) - TT['l']= TT['l']- TT['l'].min() - TT=TT.sort_values(by='l') + TT["x"] = (TT["lats"].max() - TT["lats"]) * dx + TT["y"] = (TT["lons"] - TT["lons"].min()) * deglon_in_m + TT["l"] = np.sqrt(TT["x"] ** 2 + TT["y"] ** 2) + TT["l"] = TT["l"] - TT["l"].min() + TT = TT.sort_values(by="l") return TT + def reduce_to_height_distance(TT, key, dx=1): from scipy.interpolate import interp1d - x1 = np.arange(TT['l'].min(),TT['l'].max(), dx) - y1 = np.interp(x1, TT['l'], TT[key] ) + x1 = np.arange(TT["l"].min(), TT["l"].max(), dx) + y1 = np.interp(x1, TT["l"], TT[key]) return x1, y1, TT + def poly_corrent(x, y, poly_order=7, plot_flag=False): - z = np.polyfit(x , y , poly_order) + z = np.polyfit(x, y, poly_order) p = np.poly1d(z) if plot_flag: - plt.plot(x,y, '.', markersize=0.2,) - plt.plot(x, p(x), '-', markersize=0.2,) - + plt.plot( + x, + y, + ".", + markersize=0.2, + ) + plt.plot( + x, + p(x), + "-", + markersize=0.2, + ) return y - p(x) # %% M.figure_axis_xy(5, 3, view_scale=0.8) -plt.plot(T['lats'], T['elev'], '.', markersize=1) +plt.plot(T["lats"], T["elev"], ".", markersize=1) # %% T2 = derive_axis(T) -T3=T2.groupby('ssh_flag') +T3 = T2.groupby("ssh_flag") -mssh = T3['elev'].get_group(1).mean() -#T2['heights_adjusted']= poly_corrent(T2['l'], T2['heights']) -#T2['heights_smth'] = T2['heights_adjusted'].rolling(100).median() -#x1, y1, T3 = reduce_to_height_distance(T2,'heights_smth', dx=0.1) -#point_dentisty = len(T2['l'])/ T2['l'].max() # point/ meter +mssh = T3["elev"].get_group(1).mean() +# T2['heights_adjusted']= poly_corrent(T2['l'], T2['heights']) +# T2['heights_smth'] = T2['heights_adjusted'].rolling(100).median() +# x1, y1, T3 = reduce_to_height_distance(T2,'heights_smth', dx=0.1) +# point_dentisty = len(T2['l'])/ T2['l'].max() # point/ meter # %% F = M.figure_axis_xy(5, 6.5, view_scale=0.8) -plt.subplot(2,1 , 1) -plt.title('Freeboard') -plt.plot(T3['l'].get_group(0), T3['elev'].get_group(0)-mssh, 'ko', markersize=1,alpha=0.5) +plt.subplot(2, 1, 1) +plt.title("Freeboard") +plt.plot( + T3["l"].get_group(0), T3["elev"].get_group(0) - mssh, "ko", markersize=1, alpha=0.5 +) -plt.plot(T3['l'].get_group(1), T3['elev'].get_group(1), 'g+', markersize=1,alpha=0.5) +plt.plot(T3["l"].get_group(1), T3["elev"].get_group(1), "g+", markersize=1, alpha=0.5) -#plt.plot(T3['l'], T3['heights_smth'], 'bo', markersize=1, alpha=0.5) +# plt.plot(T3['l'], T3['heights_smth'], 'bo', markersize=1, alpha=0.5) -#plt.plot(x1, y1, '-r', markersize=2) -#plt.xlim(150000, 9000) +# plt.plot(x1, y1, '-r', markersize=2) +# plt.xlim(150000, 9000) plt.xlim(3400, 5000) -plt.xlabel('Meters from the Sea Ice Edge') -plt.ylabel('Height Anomalie (meters)') +plt.xlabel("Meters from the Sea Ice Edge") +plt.ylabel("Height Anomalie (meters)") -plt.subplot(2,1, 2) -plt.plot(T3['l'].get_group(0), T3['elev'].get_group(0)-mssh, 'ko', markersize=1,alpha=0.5) +plt.subplot(2, 1, 2) +plt.plot( + T3["l"].get_group(0), T3["elev"].get_group(0) - mssh, "ko", markersize=1, alpha=0.5 +) -plt.plot(T3['l'].get_group(1), T3['elev'].get_group(1), 'g+', markersize=1,alpha=0.5) +plt.plot(T3["l"].get_group(1), T3["elev"].get_group(1), "g+", markersize=1, alpha=0.5) -#plt.plot(x1, y1, '-r', linewidth=0.5) -#plt.xlim(3400, 7000) -#plt.xlim(10000, 10500) -plt.xlabel('Meters from the Sea Ice Edge') -plt.ylabel('Height Anomalie (meters)') +# plt.plot(x1, y1, '-r', linewidth=0.5) +# plt.xlim(3400, 7000) +# plt.xlim(10000, 10500) +plt.xlabel("Meters from the Sea Ice Edge") +plt.ylabel("Height Anomalie (meters)") -F.save_pup(path=bpath, name='freeboard_heights') -F.save_light(path=bpath, name='freeboard_heights_lght') -#plt.ylim(5, 12) +F.save_pup(path=bpath, name="freeboard_heights") +F.save_light(path=bpath, name="freeboard_heights_lght") +# plt.ylim(5, 12) # %% @@ -241,17 +292,19 @@ def poly_corrent(x, y, poly_order=7, plot_flag=False): # # Gnew.to_netcdf(bpath+'filtered_photon_heights.nc') -save_pandas_table({'T':T2}, 'freeboard_heights_filted' , bpath) +save_pandas_table({"T": T2}, "freeboard_heights_filted", bpath) # %% -mean_freeboard=(T3['elev'].get_group(0)-mssh).rolling(1000).mean() -std_freeboard=(T3['elev'].get_group(0)-mssh).rolling(1000).std() -xx=T3['l'].get_group(0)/1e3 +mean_freeboard = (T3["elev"].get_group(0) - mssh).rolling(1000).mean() +std_freeboard = (T3["elev"].get_group(0) - mssh).rolling(1000).std() +xx = T3["l"].get_group(0) / 1e3 F = M.figure_axis_xy(5, 1.6) -plt.title('Average Free Board heigth', loc='left') -plt.fill_between(xx, mean_freeboard-std_freeboard, mean_freeboard+std_freeboard , color='gray') -plt.plot(xx , mean_freeboard, 'k' ) +plt.title("Average Free Board heigth", loc="left") +plt.fill_between( + xx, mean_freeboard - std_freeboard, mean_freeboard + std_freeboard, color="gray" +) +plt.plot(xx, mean_freeboard, "k") plt.xlim(10, 130) -F.save_pup(path=bpath, name='freeboard_heights_pup') +F.save_pup(path=bpath, name="freeboard_heights_pup") diff --git a/analysis_db/A01b_ALT10_variance_tester.py b/analysis_db/A01b_ALT10_variance_tester.py index e4d75572..58bace6a 100644 --- a/analysis_db/A01b_ALT10_variance_tester.py +++ b/analysis_db/A01b_ALT10_variance_tester.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,7 +9,7 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) import datetime @@ -17,7 +18,10 @@ import imp import ICEsat2_SI_tools.convert_GPS_time as cGPS import ICEsat2_SI_tools.io as io -from spectral_estimates import create_chunk_boundaries_unit_lengths, create_chunk_boundaries +from spectral_estimates import ( + create_chunk_boundaries_unit_lengths, + create_chunk_boundaries, +) import spectral_estimates as spec import m_tools_ph3 as MT import filter_regrid as regrid @@ -26,69 +30,83 @@ import piecewise_regression -#import s3fs -#processed_ATL03_20190605061807_10380310_004_01.h5 - -#imp.reload(io) -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207234532_06340210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190215184558_07530210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +# import s3fs +# processed_ATL03_20190605061807_10380310_004_01.h5 -#track_name, batch_key, test_flag = '20190101101716_00610201_005_02', 'SH_batch04', False -#track_name, batch_key, test_flag = '20190102130012_00780201_005_02', 'SH_batch04', False -#track_name, batch_key, test_flag = '20190101040007_00570201_005_02', 'SH_batch04', False -#track_name, batch_key, test_flag = '20190101084259_00600201_005_02', 'SH_batch04', False -#track_name, batch_key, test_flag = '20190128235320_04820201_005_02', 'SH_batch04', False +# imp.reload(io) +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207234532_06340210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190215184558_07530210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190101101716_00610201_005_02', 'SH_batch04', False +# track_name, batch_key, test_flag = '20190102130012_00780201_005_02', 'SH_batch04', False +# track_name, batch_key, test_flag = '20190101040007_00570201_005_02', 'SH_batch04', False +# track_name, batch_key, test_flag = '20190101084259_00600201_005_02', 'SH_batch04', False +# track_name, batch_key, test_flag = '20190128235320_04820201_005_02', 'SH_batch04', False # equatorward track -#track_name, batch_key, test_flag = '20190208154150_06440212_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190208154150_06440212_004_01', 'SH_batch02', False # poleward track -#track_name, batch_key, test_flag = '20190209150245_06590210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190209150245_06590210_004_01', 'SH_batch02', False # -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL10-02' if hemis == 'SH' else 'ATL10-01' +ATlevel = "ATL10-02" if hemis == "SH" else "ATL10-01" -load_path = mconfig['paths']['scratch'] +'/'+ batch_key +'/' -load_file = load_path + ATlevel+'_'+track_name+'.h5' +load_path = mconfig["paths"]["scratch"] + "/" + batch_key + "/" +load_file = load_path + ATlevel + "_" + track_name + ".h5" -save_path = mconfig['paths']['work'] +'/'+ batch_key +'/'+'/A01b_ID_'+hemis+'/' +save_path = mconfig["paths"]["work"] + "/" + batch_key + "/" + "/A01b_ID_" + hemis + "/" -plot_path = mconfig['paths']['plot']+ '/'+hemis+'/'+batch_key+'/'+track_name +'/A01b/' -#bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/A01b/" +) +# bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' MT.mkdirs_r(save_path) # set pars # define parameters: -#Nphoton_min = 5 # mininum need photons per stancil to return results +# Nphoton_min = 5 # mininum need photons per stancil to return results -plot_flag = False +plot_flag = False # %% # test which beams exist: -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] # low_beams = mconfig['beams']['low_beams'] try: - f = h5py.File(load_file, 'r') + f = h5py.File(load_file, "r") except: - print('file not found, exit') - MT.json_save(name='A01b_success_'+track_name, path=save_path, data= {'reason':'ALT10 file not found, exit'}) + print("file not found, exit") + MT.json_save( + name="A01b_success_" + track_name, + path=save_path, + data={"reason": "ALT10 file not found, exit"}, + ) exit() -beams = [b if b in f.keys() else None for b in all_beams] +beams = [b if b in f.keys() else None for b in all_beams] imp.reload(regrid) -#track_poleward = regrid.track_pole_ward_file(f, product='ALT10') +# track_poleward = regrid.track_pole_ward_file(f, product='ALT10') # print('poleward track is' , track_poleward) # ATL03 = h5py.File(load_file, 'r') # ATL03['orbit_info'].keys() @@ -96,62 +114,70 @@ # # ATL03['gt1l/freeboard_beam_segment/height_segments'].keys() + # %% -def cut_rear_data(xx0, dd0, N_seg= 20): +def cut_rear_data(xx0, dd0, N_seg=20): """ returns masks that cuts large variance in the back of the data """ - rear_mask = xx0*0 > -1 # True + rear_mask = xx0 * 0 > -1 # True nsize0 = rear_mask.size cut_flag = True dd_old = -1 - print('inital length' , nsize0) + print("inital length", nsize0) - #@jit(nopython=True, parallel= False) + # @jit(nopython=True, parallel= False) def adjust_length(var_list, rear_mask, cut_flag): - #var_list = var_list if track_poleward else var_list[::-1] + # var_list = var_list if track_poleward else var_list[::-1] - if var_list[0:3].mean()*2 < var_list[-1]: - #print('cut last '+ str(100/N_seg) +'% of data') - rear_mask[int(nsize* (N_seg-1) / N_seg):] = False + if var_list[0:3].mean() * 2 < var_list[-1]: + # print('cut last '+ str(100/N_seg) +'% of data') + rear_mask[int(nsize * (N_seg - 1) / N_seg) :] = False else: - cut_flag = False + cut_flag = False - #rear_mask = rear_mask if track_poleward else rear_mask[::-1] + # rear_mask = rear_mask if track_poleward else rear_mask[::-1] return rear_mask, cut_flag def get_var(sti): - return dd[sti[0]: sti[1]].var() + return dd[sti[0] : sti[1]].var() while cut_flag: - dd= dd0[rear_mask] + dd = dd0[rear_mask] nsize = dd.size - print('new length', nsize) - if (nsize/N_seg) < 1: + print("new length", nsize) + if (nsize / N_seg) < 1: break - stencil_iter = create_chunk_boundaries( int(nsize/N_seg), nsize,ov =0, iter_flag=True ) + stencil_iter = create_chunk_boundaries( + int(nsize / N_seg), nsize, ov=0, iter_flag=True + ) var_list = np.array(list(map(get_var, stencil_iter))) - #print(k, var_list) + # print(k, var_list) rear_mask, cut_flag = adjust_length(var_list, rear_mask, cut_flag) if nsize == dd_old: - print('--- lengthen segments') - N_seg -=1 - #cut_flag = False + print("--- lengthen segments") + N_seg -= 1 + # cut_flag = False dd_old = nsize return rear_mask -def get_breakingpoints(xx, dd ,Lmeter= 3000): + +def get_breakingpoints(xx, dd, Lmeter=3000): nsize = dd.size - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter, [ xx.min(), xx.max()],ov =Lmeter*3/4, iter_flag= True) - iter_x = spec.create_chunk_boundaries_unit_lengths( Lmeter, [ xx.min(), xx.max()],ov =Lmeter*3/4, iter_flag= False)[1,:] + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter, [xx.min(), xx.max()], ov=Lmeter * 3 / 4, iter_flag=True + ) + iter_x = spec.create_chunk_boundaries_unit_lengths( + Lmeter, [xx.min(), xx.max()], ov=Lmeter * 3 / 4, iter_flag=False + )[1, :] def get_var(sti): mask = (sti[0] < xx) & (xx <= sti[1]) @@ -159,62 +185,63 @@ def get_var(sti): var_list = np.array(list(map(get_var, stencil_iter))) - x2, y2 = iter_x/1e3, var_list + x2, y2 = iter_x / 1e3, var_list - x2= x2[~np.isnan(y2)] - y2= y2[~np.isnan(y2)] + x2 = x2[~np.isnan(y2)] + y2 = y2[~np.isnan(y2)] - convergence_flag =True - n_breakpoints= 1 + convergence_flag = True + n_breakpoints = 1 while convergence_flag: pw_fit = piecewise_regression.Fit(x2, y2, n_breakpoints=1) - print('n_breakpoints', n_breakpoints, pw_fit.get_results()['converged']) - convergence_flag = not pw_fit.get_results()['converged'] + print("n_breakpoints", n_breakpoints, pw_fit.get_results()["converged"]) + convergence_flag = not pw_fit.get_results()["converged"] n_breakpoints += 1 if n_breakpoints == 4: convergence_flag = False pw_results = pw_fit.get_results() - if pw_results['converged']: - if pw_results['estimates']['alpha1']['estimate'] < 0: - print('decay at the front') - print('n_breakpoints',pw_fit.n_breakpoints ) + if pw_results["converged"]: + if pw_results["estimates"]["alpha1"]["estimate"] < 0: + print("decay at the front") + print("n_breakpoints", pw_fit.n_breakpoints) - breakpoint = pw_results['estimates']['breakpoint1']['estimate'] - return pw_results['estimates']['alpha1']['estimate'], pw_fit, breakpoint + breakpoint = pw_results["estimates"]["breakpoint1"]["estimate"] + return pw_results["estimates"]["alpha1"]["estimate"], pw_fit, breakpoint else: return np.nan, pw_fit, False -DD_slope = pd.DataFrame(index =beams, columns= ['TF1', 'TF2']) -DD_data = pd.DataFrame(index =beams, columns= ['TF1', 'TF2']) -DD_region = pd.DataFrame(index =beams, columns= ['TF1', 'TF2']) -plot_flag= True +DD_slope = pd.DataFrame(index=beams, columns=["TF1", "TF2"]) +DD_data = pd.DataFrame(index=beams, columns=["TF1", "TF2"]) +DD_region = pd.DataFrame(index=beams, columns=["TF1", "TF2"]) + +plot_flag = True for k in beams: - #k = beams[1] - #imp.reload(io) + # k = beams[1] + # imp.reload(io) print(k) try: - T_freeboard, T_leads = io.getATL10_beam(load_file, beam= k) + T_freeboard, T_leads = io.getATL10_beam(load_file, beam=k) except: - print('failed to load beam') + print("failed to load beam") slope_test = False - data_flag = False - #return data_flag, slope_test - print('break -------', k, TF, data_flag, slope_test) + data_flag = False + # return data_flag, slope_test + print("break -------", k, TF, data_flag, slope_test) continue ###### for SH tracks # split tracks: - long_diffs=T_freeboard['freeboard']['longitude'].diff() + long_diffs = T_freeboard["freeboard"]["longitude"].diff() isplit = abs(long_diffs).argmax() if abs(long_diffs[isplit]) < 90: TF1 = T_freeboard - TF2 = None + TF2 = None else: TF1 = T_freeboard.iloc[0:isplit] - TF2 = T_freeboard.iloc[isplit:] + TF2 = T_freeboard.iloc[isplit:] def pole_ward_table(T): """ @@ -222,77 +249,82 @@ def pole_ward_table(T): hdf5_file is a an HFD5 object in read mode """ - time = T['time']['delta_time'] - lat = T['ref']['latitude'] - print('1st lat =' + str(abs(lat.iloc[time.argmin()])) , ';last lat =' + str(abs(lat.iloc[time.argmax()])) ) + time = T["time"]["delta_time"] + lat = T["ref"]["latitude"] + print( + "1st lat =" + str(abs(lat.iloc[time.argmin()])), + ";last lat =" + str(abs(lat.iloc[time.argmax()])), + ) return abs(lat.iloc[time.argmax()]) > abs(lat.iloc[time.argmin()]) - TF1_poleward = pole_ward_table(TF1) if TF2 is not None: - TF2_poleward = pole_ward_table(TF2) + TF2_poleward = pole_ward_table(TF2) if TF1_poleward & TF2_poleward: - raise ValueError('both parts are acending or decending') + raise ValueError("both parts are acending or decending") else: TF2_poleward = not TF1_poleward # flip the beam section that is not poleward if TF1_poleward & (TF2 is not None): - print('TF2 is ', TF2_poleward) - TF2 = TF2.sort_values(('ref','seg_dist_x'), ascending=False).reset_index() + print("TF2 is ", TF2_poleward) + TF2 = TF2.sort_values(("ref", "seg_dist_x"), ascending=False).reset_index() # Tsel2 = T_leads.sort_values('lead_dist_x', ascending=False).reset_index() # Tsel2['x'] = abs(Tsel2['lead_dist_x'] -Tsel2['lead_dist_x'][0]) else: - print('TF1 is ', TF2_poleward) - TF1 = TF1.sort_values(('ref','seg_dist_x'), ascending=False).reset_index() + print("TF1 is ", TF2_poleward) + TF1 = TF1.sort_values(("ref", "seg_dist_x"), ascending=False).reset_index() # TF1['x'] = abs(TF1['ref']['seg_dist_x'] -TF1['ref']['seg_dist_x'][0]) # Tsel2 = T_leads#['lead_dist_x'] # Tsel2['x'] = abs(Tsel2['lead_dist_x'] -Tsel2['lead_dist_x'][0]) # create local x axis - TF1['x'] = abs(TF1['ref']['seg_dist_x'] -TF1['ref']['seg_dist_x'].iloc[0]) + TF1["x"] = abs(TF1["ref"]["seg_dist_x"] - TF1["ref"]["seg_dist_x"].iloc[0]) if TF2 is not None: - TF2['x'] = abs(TF2['ref']['seg_dist_x'] -TF2['ref']['seg_dist_x'].iloc[0]) + TF2["x"] = abs(TF2["ref"]["seg_dist_x"] - TF2["ref"]["seg_dist_x"].iloc[0]) # define Region - - for TF,Tsel,TF_polward in zip(['TF1', 'TF2'], [TF1, TF2], [TF1_poleward, TF2_poleward]): - - if (hemis == 'SH') & TF_polward: - region = '10' - elif (hemis == 'SH') & (not TF_polward): - region = '12' - elif (hemis == 'NH') & (TF_polward): - region = ('03','04') - elif (hemis == 'NH') & (not TF_polward): - region = ('05','04') + for TF, Tsel, TF_polward in zip( + ["TF1", "TF2"], [TF1, TF2], [TF1_poleward, TF2_poleward] + ): + + if (hemis == "SH") & TF_polward: + region = "10" + elif (hemis == "SH") & (not TF_polward): + region = "12" + elif (hemis == "NH") & (TF_polward): + region = ("03", "04") + elif (hemis == "NH") & (not TF_polward): + region = ("05", "04") else: - region =False + region = False # cut high sigma values - if (Tsel is None): + if Tsel is None: slope_test = False - data_flag = False - #return data_flag, slope_test - print('break -------', k, TF, data_flag, slope_test) + data_flag = False + # return data_flag, slope_test + print("break -------", k, TF, data_flag, slope_test) continue - Tsel = Tsel[Tsel['ref']['beam_fb_sigma'] < 1e2] - if (Tsel.size <= 50): - #print('too small table, skip ') + Tsel = Tsel[Tsel["ref"]["beam_fb_sigma"] < 1e2] + if Tsel.size <= 50: + # print('too small table, skip ') Tsel = None if Tsel is None: slope_test = False - data_flag = False - #return data_flag, slope_test - print('break -------', k, TF, data_flag, slope_test) + data_flag = False + # return data_flag, slope_test + print("break -------", k, TF, data_flag, slope_test) continue else: - data_flag =Tsel.shape[0]/abs(Tsel['x'].max() - Tsel['x'].min()) # datapoints per meters + data_flag = Tsel.shape[0] / abs( + Tsel["x"].max() - Tsel["x"].min() + ) # datapoints per meters # plt.plot(Tsel['x']/1e3, Tsel['ref']['beam_fb_sigma'], '.k') # plt.plot(Tsel['x']/1e3, Tsel['ref']['beam_fb_height'], '.r') @@ -301,84 +333,111 @@ def pole_ward_table(T): # plt.ylim(-1, 5) # % curt data in the back - xx0, dd0 = np.array(Tsel['x']), np.array(Tsel['freeboard']['height_segment_height']) + xx0, dd0 = np.array(Tsel["x"]), np.array( + Tsel["freeboard"]["height_segment_height"] + ) rear_mask = cut_rear_data(xx0, dd0) - #print('density post cutting', len(xx0[rear_mask])/abs(xx0[rear_mask].max() - xx0[rear_mask].min()) ) - if len(xx0[rear_mask]) < 500: # if cutted data is too short + # print('density post cutting', len(xx0[rear_mask])/abs(xx0[rear_mask].max() - xx0[rear_mask].min()) ) + if len(xx0[rear_mask]) < 500: # if cutted data is too short slope_test = False - data_flag = False - #return data_flag, slope_test - print('break -------', k, TF, data_flag, slope_test) + data_flag = False + # return data_flag, slope_test + print("break -------", k, TF, data_flag, slope_test) continue # estmiate slope at the beginning - slope_test, pw_fit, breakpoint = get_breakingpoints(xx0[rear_mask], dd0[rear_mask], Lmeter= 3000) + slope_test, pw_fit, breakpoint = get_breakingpoints( + xx0[rear_mask], dd0[rear_mask], Lmeter=3000 + ) if plot_flag: plt.figure() - plt.plot(xx0[rear_mask]/1e3, dd0[rear_mask], '.k', markersize= 0.4) + plt.plot(xx0[rear_mask] / 1e3, dd0[rear_mask], ".k", markersize=0.4) pw_fit.plot() - plt.title(k +' '+ TF + ', data=' +str(data_flag) + ', slope='+str(slope_test) + '\n' + track_name , loc= 'left') - M.save_anyfig(plt.gcf(), name='A01b_'+track_name+'_'+k +'_'+ TF , path=plot_path) + plt.title( + k + + " " + + TF + + ", data=" + + str(data_flag) + + ", slope=" + + str(slope_test) + + "\n" + + track_name, + loc="left", + ) + M.save_anyfig( + plt.gcf(), + name="A01b_" + track_name + "_" + k + "_" + TF, + path=plot_path, + ) plt.close() - #plt.show() + # plt.show() DD_slope.loc[k, TF] = slope_test DD_data.loc[k, TF] = data_flag DD_region.loc[k, TF] = region - print('result-------', k, TF, data_flag, slope_test) + print("result-------", k, TF, data_flag, slope_test) -#DD_data -DD_slope_mask = DD_slope <0 +# DD_data +DD_slope_mask = DD_slope < 0 if (DD_slope_mask.sum() > 1).sum() > 0: - print('download data') + print("download data") else: - print('no suffcient data, quit()') - ll_name = ATlevel+'_stats_'+track_name+'_fail' - DD_merge = pd.concat({'density_Nperm':DD_data , 'slopes':DD_slope}, axis=1) - DD_merge.to_html(save_path+ll_name+'.html') - #DD_merge.columns = ['-'.join(col).strip() for col in DD_merge.columns.values] - MT.save_pandas_table({'T':DD_merge},ll_name, save_path) - #DD_merge.columns = ['-'.join(col).strip() for col in DD_merge.columns.values] - #MT.json_save(name=ll_name, path=save_path, data= DD_merge.where(pd.notnull(DD_merge), 0).T.to_dict()) - MT.json_save(name='A01b_success_'+track_name, path=save_path, data= DD_slope.where(pd.notnull(DD_slope), 0).to_dict()) + print("no suffcient data, quit()") + ll_name = ATlevel + "_stats_" + track_name + "_fail" + DD_merge = pd.concat({"density_Nperm": DD_data, "slopes": DD_slope}, axis=1) + DD_merge.to_html(save_path + ll_name + ".html") + # DD_merge.columns = ['-'.join(col).strip() for col in DD_merge.columns.values] + MT.save_pandas_table({"T": DD_merge}, ll_name, save_path) + # DD_merge.columns = ['-'.join(col).strip() for col in DD_merge.columns.values] + # MT.json_save(name=ll_name, path=save_path, data= DD_merge.where(pd.notnull(DD_merge), 0).T.to_dict()) + MT.json_save( + name="A01b_success_" + track_name, + path=save_path, + data=DD_slope.where(pd.notnull(DD_slope), 0).to_dict(), + ) exit() -region_list = DD_region[DD_slope_mask].to_numpy().flatten().astype('float') +region_list = DD_region[DD_slope_mask].to_numpy().flatten().astype("float") region_list = list(set(region_list[~np.isnan(region_list)])) region_list = [str(int(i)) for i in region_list] -print('region(s) ', region_list) +print("region(s) ", region_list) -ALT03_list= list() +ALT03_list = list() for i in region_list: - track_segs = track_name.split('_') - track_segs[1] = track_segs[1][0:-2]+ i # correct cycle number - track_segs[3] = '01' # correct version number - ALT03_trackname = '_'.join(track_segs) + track_segs = track_name.split("_") + track_segs[1] = track_segs[1][0:-2] + i # correct cycle number + track_segs[3] = "01" # correct version number + ALT03_trackname = "_".join(track_segs) print(ALT03_trackname) ALT03_list.append(ALT03_trackname) # DD_data # DD_region -print('data density N/meter') +print("data density N/meter") print(DD_data) -print('slopes') +print("slopes") print(DD_slope) -#DD_slope.to_html() +# DD_slope.to_html() for ll in ALT03_list: - ll_name = 'ALT03_stats_'+ll - DD_merge = pd.concat({'density_Nperm':DD_data , 'slopes':DD_slope}, axis=1) - DD_merge.to_html(save_path+ll_name+'.html') - #DD_merge.columns = ['-'.join(col).strip() for col in DD_merge.columns.values] - MT.save_pandas_table({'T':DD_merge},ll_name, save_path) - #MT.json_save(name=ll_name, path=save_path, data= DD_merge.where(pd.notnull(DD_merge), 0).T.to_dict()) - - #DD_merge.to_json(save_path+ll_name+'.json', orient="records", lines=True) - -MT.json_save(name='A01b_success_'+track_name, path=save_path, data= DD_slope.where(pd.notnull(DD_slope), 0).to_dict()) + ll_name = "ALT03_stats_" + ll + DD_merge = pd.concat({"density_Nperm": DD_data, "slopes": DD_slope}, axis=1) + DD_merge.to_html(save_path + ll_name + ".html") + # DD_merge.columns = ['-'.join(col).strip() for col in DD_merge.columns.values] + MT.save_pandas_table({"T": DD_merge}, ll_name, save_path) + # MT.json_save(name=ll_name, path=save_path, data= DD_merge.where(pd.notnull(DD_merge), 0).T.to_dict()) + + # DD_merge.to_json(save_path+ll_name+'.json', orient="records", lines=True) + +MT.json_save( + name="A01b_success_" + track_name, + path=save_path, + data=DD_slope.where(pd.notnull(DD_slope), 0).to_dict(), +) diff --git a/analysis_db/A01c_mergefiles.py b/analysis_db/A01c_mergefiles.py index 6a49b529..06fcf37b 100644 --- a/analysis_db/A01c_mergefiles.py +++ b/analysis_db/A01c_mergefiles.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,7 +9,7 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) import datetime @@ -17,7 +18,10 @@ import imp import ICEsat2_SI_tools.convert_GPS_time as cGPS import ICEsat2_SI_tools.io as io -from spectral_estimates import create_chunk_boundaries_unit_lengths, create_chunk_boundaries +from spectral_estimates import ( + create_chunk_boundaries_unit_lengths, + create_chunk_boundaries, +) import spectral_estimates as spec import m_tools_ph3 as MT import filter_regrid as regrid @@ -27,106 +31,126 @@ def get_size(x): from pympler import asizeof - ss = asizeof.asizeof(x)/1e6 + + ss = asizeof.asizeof(x) / 1e6 return str(ss) -all_beams = mconfig['beams']['all_beams'] -#imp.reload(io) -ID_name, batch_key, ID_flag = io.init_from_input(sys.argv) # loads standard experiment +all_beams = mconfig["beams"]["all_beams"] + +# imp.reload(io) +ID_name, batch_key, ID_flag = io.init_from_input(sys.argv) # loads standard experiment # SH example -#ID_name, batch_key, ID_flag = 'SH_20190101_00550210', 'SH_batch04', True -#ID_name, batch_key, ID_flag = 'SH_20190102_00770210', 'SH_batch04', True +# ID_name, batch_key, ID_flag = 'SH_20190101_00550210', 'SH_batch04', True +# ID_name, batch_key, ID_flag = 'SH_20190102_00770210', 'SH_batch04', True # NH example -#ID_name, batch_key, ID_flag = 'NH_20190301_09560203', 'NH_batch05', True # poleward false -#ID_name, batch_key, ID_flag = 'NH_20190301_09560205', 'NH_batch05', True # poleward false +# ID_name, batch_key, ID_flag = 'NH_20190301_09560203', 'NH_batch05', True # poleward false +# ID_name, batch_key, ID_flag = 'NH_20190301_09560205', 'NH_batch05', True # poleward false # ID example -#ID_name, batch_key, ID_flag = '20190101015140_00550210_005_01', 'SH_batch04', False +# ID_name, batch_key, ID_flag = '20190101015140_00550210_005_01', 'SH_batch04', False -#imp.reload(io) -ID, track_names, hemis, batch = io.init_data(ID_name, batch_key, ID_flag, mconfig['paths']['work'] ) +# imp.reload(io) +ID, track_names, hemis, batch = io.init_data( + ID_name, batch_key, ID_flag, mconfig["paths"]["work"] +) ID, track_names, hemis, batch # %% -load_path_data = mconfig['paths']['scratch'] +'/'+ batch_key +'/' -save_path_data = mconfig['paths']['scratch'] +'/'+ batch_key +'/' -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +load_path_data = mconfig["paths"]["scratch"] + "/" + batch_key + "/" +save_path_data = mconfig["paths"]["scratch"] + "/" + batch_key + "/" +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" + -#imp.reload(io) +# imp.reload(io) def ATL03_loader(k): """ returns tables with merged ATL03 data """ - print('beam ' + k) + print("beam " + k) # concat and resize if there are more then 2 tracks involved - if len(track_names) > 1 : - print('tracklist found') - T_lists, seg_list,Tsel_c_list = list(), list(), list() + if len(track_names) > 1: + print("tracklist found") + T_lists, seg_list, Tsel_c_list = list(), list(), list() for ATLi in track_names: try: - T0, seg0 = io.getATL03_beam(load_path_data + ATLi +'.h5', beam= k) + T0, seg0 = io.getATL03_beam(load_path_data + ATLi + ".h5", beam=k) T_lists.append(T0) seg_list.append(seg0) - Tsel_c = io.getATL03_height_correction(load_path_data + ATLi +'.h5', beam= k) + Tsel_c = io.getATL03_height_correction( + load_path_data + ATLi + ".h5", beam=k + ) Tsel_c_list.append(Tsel_c) except: - print(ATLi, ' not found!! continue with single file!') - - if len(T_lists) == 0: # in photons per meter - print('no files found, this track is classified as bad track and pushed to bad list') - MT.json_save(ID_name, bad_track_path, {'A01c': 'no files found, this track is classified as bad track and pushed to bad list','track_names':track_names , 'date': str(datetime.date.today()) }) - print('exit.') + print(ATLi, " not found!! continue with single file!") + + if len(T_lists) == 0: # in photons per meter + print( + "no files found, this track is classified as bad track and pushed to bad list" + ) + MT.json_save( + ID_name, + bad_track_path, + { + "A01c": "no files found, this track is classified as bad track and pushed to bad list", + "track_names": track_names, + "date": str(datetime.date.today()), + }, + ) + print("exit.") exit() Ti = pd.concat(T_lists) Si = pd.concat(seg_list) - Ti_c = pd.concat(Tsel_c_list) + Ti_c = pd.concat(Tsel_c_list) - if ID['pars']['poleward']: - Ti = Ti[ Ti['delta_time'] <= ID['pars']['end']['delta_time'] ] - Si = Si[ Si['delta_time'] <= ID['pars']['end']['delta_time'] ] - Ti_c = Ti_c[ Ti_c['delta_time'] <= ID['pars']['end']['delta_time'] ] + if ID["pars"]["poleward"]: + Ti = Ti[Ti["delta_time"] <= ID["pars"]["end"]["delta_time"]] + Si = Si[Si["delta_time"] <= ID["pars"]["end"]["delta_time"]] + Ti_c = Ti_c[Ti_c["delta_time"] <= ID["pars"]["end"]["delta_time"]] else: - Ti = Ti[ Ti['delta_time'] >= ID['pars']['end']['delta_time'] ] - Si = Si[ Si['delta_time'] >= ID['pars']['end']['delta_time'] ] - Ti_c = Ti_c[ Ti_c['delta_time'] >= ID['pars']['end']['delta_time'] ] + Ti = Ti[Ti["delta_time"] >= ID["pars"]["end"]["delta_time"]] + Si = Si[Si["delta_time"] >= ID["pars"]["end"]["delta_time"]] + Ti_c = Ti_c[Ti_c["delta_time"] >= ID["pars"]["end"]["delta_time"]] else: - print('only 1 track found') - T0, seg0 = io.getATL03_beam(load_path_data + track_names[0] +'.h5', beam= k) - Tsel_c = io.getATL03_height_correction(load_path_data + track_names[0] +'.h5', beam= k) + print("only 1 track found") + T0, seg0 = io.getATL03_beam(load_path_data + track_names[0] + ".h5", beam=k) + Tsel_c = io.getATL03_height_correction( + load_path_data + track_names[0] + ".h5", beam=k + ) Ti = T0 Si = seg0 - Ti_c =Tsel_c - + Ti_c = Tsel_c - print('mask and correct with dem_h') - Ti = Ti[Ti['mask_seaice']] # only take sea ice points, no ocean points - #T = T.drop(labels=[ 'year', 'month', 'day', 'hour', 'minute', 'second', 'ph_id_count', 'mask_seaice'], axis= 1) - Ti = Ti.drop(labels=[ 'ph_id_count', 'mask_seaice'], axis= 1) + print("mask and correct with dem_h") + Ti = Ti[Ti["mask_seaice"]] # only take sea ice points, no ocean points + # T = T.drop(labels=[ 'year', 'month', 'day', 'hour', 'minute', 'second', 'ph_id_count', 'mask_seaice'], axis= 1) + Ti = Ti.drop(labels=["ph_id_count", "mask_seaice"], axis=1) - #Ti = Ti[Ti['mask_seaice']] # only take sea ice points, no ocean points - #T = T.drop(labels=[ 'year', 'month', 'day', 'hour', 'minute', 'second', 'ph_id_count', 'mask_seaice'], axis= 1) - #Ti = Ti.drop(labels=[ 'ph_id_count'], axis= 1) + # Ti = Ti[Ti['mask_seaice']] # only take sea ice points, no ocean points + # T = T.drop(labels=[ 'year', 'month', 'day', 'hour', 'minute', 'second', 'ph_id_count', 'mask_seaice'], axis= 1) + # Ti = Ti.drop(labels=[ 'ph_id_count'], axis= 1) - print( 'T MB ' + get_size(Ti) ) + print("T MB " + get_size(Ti)) # filter: - Tsel = Ti[ (Ti['heights']<100) & (Ti['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] + Tsel = Ti[ + (Ti["heights"] < 100) & (Ti["heights"] > -100) + ] # & (T['delta_time']>5) & (T['delta_time']<24) ] # if len(Tsel) == 0: # ho = MT.add_line_var(ho, 'no photons found', '') # #Tsel= T[(T['signal_confidence'] ==-1 ) & (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] - #Tsel_c = io.getATL03_height_correction(load_file_str) - Ti_c = Ti_c[Ti_c['dem_h'] < 1e5] # cute out weird references + # Tsel_c = io.getATL03_height_correction(load_file_str) + Ti_c = Ti_c[Ti_c["dem_h"] < 1e5] # cute out weird references # needs only dem_h and heihgts - Tsel = regrid.correct_heights(Tsel, Ti_c).reset_index(drop=True)# correct height - print('height corrected') + Tsel = regrid.correct_heights(Tsel, Ti_c).reset_index(drop=True) # correct height + print("height corrected") return k, Tsel, Si @@ -137,15 +161,15 @@ def ATL03_loader(k): A = list() for bb in all_beams: - A.append( ATL03_loader(bb) ) + A.append(ATL03_loader(bb)) # %% reorganize loaderd data -TT, SS, TCC =dict(), dict(), dict() +TT, SS, TCC = dict(), dict(), dict() for Ai in A: - k = Ai[0] + k = Ai[0] TT[k] = Ai[1] SS[k] = Ai[2] - #TCC[k] = Ai[3] + # TCC[k] = Ai[3] # Write data @@ -153,10 +177,9 @@ def ATL03_loader(k): # MT.save_pandas_table(SS, 'A01c_ATL03_'+ ID_name+'_seg', save_path_data) imp.reload(io) -io.write_track_to_HDF5(TT, 'A01c_ATL03_'+ ID_name+'_corrected', save_path_data) -io.write_track_to_HDF5(SS, 'A01c_ATL03_'+ ID_name+'_seg' , save_path_data) -#io.write_track_to_HDF5(TCC, 'A01c_ATL03_'+ ID_name+'_c', save_path_data) - +io.write_track_to_HDF5(TT, "A01c_ATL03_" + ID_name + "_corrected", save_path_data) +io.write_track_to_HDF5(SS, "A01c_ATL03_" + ID_name + "_seg", save_path_data) +# io.write_track_to_HDF5(TCC, 'A01c_ATL03_'+ ID_name+'_c', save_path_data) # %% for testing diff --git a/analysis_db/A02a_sentinal1_prior.py b/analysis_db/A02a_sentinal1_prior.py index 09760983..29a65634 100644 --- a/analysis_db/A02a_sentinal1_prior.py +++ b/analysis_db/A02a_sentinal1_prior.py @@ -1,16 +1,16 @@ - import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -22,82 +22,103 @@ import spicke_remover import datetime import concurrent.futures as futures -#import s3fs + +# import s3fs col.colormaps2(21) # %% -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False - - - - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -save_path = mconfig['paths']['work'] + '/B03_spectra_'+hemis+'/' -save_name = 'B03_'+track_name - -plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/A02a_sentenial_prior/' +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False + + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +save_path = mconfig["paths"]["work"] + "/B03_spectra_" + hemis + "/" +save_name = "B03_" + track_name + +plot_path = ( + mconfig["paths"]["plot"] + + "/" + + hemis + + "/" + + batch_key + + "/" + + track_name + + "/A02a_sentenial_prior/" +) MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -#Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +# Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data -load_path_WAVE_GLO = mconfig['paths']['work'] +'/CMEMS_WAVE_GLO_L3/WAVE_GLO_WAV_L3_SPC_REP_OBSERVATIONS/' -file_name_base = 'dataset-wav-sar-l3-spc-rep-global-' +load_path_WAVE_GLO = ( + mconfig["paths"]["work"] + + "/CMEMS_WAVE_GLO_L3/WAVE_GLO_WAV_L3_SPC_REP_OBSERVATIONS/" +) +file_name_base = "dataset-wav-sar-l3-spc-rep-global-" -load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' -Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # +load_path = mconfig["paths"]["work"] + "/B01_regrid_" + hemis + "/" +Gd = io.load_pandas_table_dict(track_name + "_B01_binned", load_path) # # %% G1 = dict() for b in all_beams: # find 1st point - G1[b] = Gd[b].iloc[abs(Gd[b]['lats']).argmin()] + G1[b] = Gd[b].iloc[abs(Gd[b]["lats"]).argmin()] G1 = pd.DataFrame.from_dict(G1).T -dlon_deg = 2.5 # degree range aroud 1st point -dlat_deg = 10 # degree range aroud 1st point -dtime = 24 # in hours - -lon_range = G1['lons'].min() - dlon_deg , G1['lons'].max() + dlon_deg -lat_range = G1['lats'].min() - dlat_deg , G1['lats'].max() + dlat_deg -timestamp = pd.to_datetime(G1[['year', 'month', 'day', 'hour', 'minute', 'second']]).mean() -time_range= np.datetime64(timestamp) - np.timedelta64(dtime, 'h') , np.datetime64(timestamp) + np.timedelta64(dtime, 'h') +dlon_deg = 2.5 # degree range aroud 1st point +dlat_deg = 10 # degree range aroud 1st point +dtime = 24 # in hours + +lon_range = G1["lons"].min() - dlon_deg, G1["lons"].max() + dlon_deg +lat_range = G1["lats"].min() - dlat_deg, G1["lats"].max() + dlat_deg +timestamp = pd.to_datetime( + G1[["year", "month", "day", "hour", "minute", "second"]] +).mean() +time_range = np.datetime64(timestamp) - np.timedelta64(dtime, "h"), np.datetime64( + timestamp +) + np.timedelta64(dtime, "h") print(time_range) # create timestamp according to fiels on ftp server: -time_stamps_search = np.arange(time_range[0].astype('datetime64[3h]') - np.timedelta64(12*24, 'h') , time_range[1].astype('datetime64[3h]') + np.timedelta64(3, 'h'), np.timedelta64(3, 'h')) -time_stamps_search_str = [str(t).replace('-', '') for t in time_stamps_search] +time_stamps_search = np.arange( + time_range[0].astype("datetime64[3h]") - np.timedelta64(12 * 24, "h"), + time_range[1].astype("datetime64[3h]") + np.timedelta64(3, "h"), + np.timedelta64(3, "h"), +) +time_stamps_search_str = [str(t).replace("-", "") for t in time_stamps_search] del G1 # %% import glob -if time_range[0].astype('M8[M]') != time_range[1].astype('M8[M]'): # spanning two years - MM_str = str(time_range[0].astype('M8[M]')).replace('-', '') - f_list = glob.glob(load_path_WAVE_GLO+'/*'+MM_str+'*.nc') +if time_range[0].astype("M8[M]") != time_range[1].astype("M8[M]"): # spanning two years + MM_str = str(time_range[0].astype("M8[M]")).replace("-", "") + f_list = glob.glob(load_path_WAVE_GLO + "/*" + MM_str + "*.nc") - MM_str = str(time_range[-1].astype('M8[M]')).replace('-', '') - f_list = glob.glob(load_path_WAVE_GLO+'/*'+MM_str+'*.nc') + MM_str = str(time_range[-1].astype("M8[M]")).replace("-", "") + f_list = glob.glob(load_path_WAVE_GLO + "/*" + MM_str + "*.nc") f_list = f_list + f_list else: - MM_str = str(time_range[0].astype('M8[M]')).replace('-', '') - f_list = glob.glob(load_path_WAVE_GLO+'/*'+MM_str+'*.nc') + MM_str = str(time_range[0].astype("M8[M]")).replace("-", "") + f_list = glob.glob(load_path_WAVE_GLO + "/*" + MM_str + "*.nc") # %% @@ -106,15 +127,21 @@ files_to_use = list() for f in f_list: for ts_stamp in time_stamps_search_str: - fn = f.split('/')[-1] - if (file_name_base + 's1a_'+ ts_stamp in fn): - files_to_use.append([ 's1a_'+ ts_stamp, f.split('/')[-1] , f ] ) - elif (file_name_base + 's1b_'+ ts_stamp in fn): - files_to_use.append([ 's1b_'+ ts_stamp, f.split('/')[-1] , f ] ) + fn = f.split("/")[-1] + if file_name_base + "s1a_" + ts_stamp in fn: + files_to_use.append(["s1a_" + ts_stamp, f.split("/")[-1], f]) + elif file_name_base + "s1b_" + ts_stamp in fn: + files_to_use.append(["s1b_" + ts_stamp, f.split("/")[-1], f]) def draw_range(lon_range, lat_range, *args, **kargs): - plt.plot( [lon_range[0], lon_range[1], lon_range[1], lon_range[0], lon_range[0]] , [lat_range[0], lat_range[0], lat_range[1], lat_range[1], lat_range[0]] , *args, **kargs) + plt.plot( + [lon_range[0], lon_range[1], lon_range[1], lon_range[0], lon_range[0]], + [lat_range[0], lat_range[0], lat_range[1], lat_range[1], lat_range[0]], + *args, + **kargs, + ) + def check_in_mask(I, lon_range, lat_range, time_range): lon_flag = (lon_range[0] < I.longitude.data) & (I.longitude.data < lon_range[1]) @@ -125,7 +152,8 @@ def check_in_mask(I, lon_range, lat_range, time_range): # %% -#Go through all files and find tracks that fit goeloation and time +# Go through all files and find tracks that fit goeloation and time + # ID, file_name, path = files_to_use[9] # #for ID, file_name, path in files_to_use[0:50]: @@ -137,7 +165,7 @@ def get_possible_tracks_from_file(ff): D = dict() FWi = xr.open_mfdataset(path) - #print(FWi.load().time.min().data, FWi.load().time.max().data ) + # print(FWi.load().time.min().data, FWi.load().time.max().data ) # M.figure_axis_xy(9, 2.5, view_scale= 0.5) # plt.suptitle(ID) @@ -147,136 +175,170 @@ def get_possible_tracks_from_file(ff): # # ax2 = plt.subplot(1, 2, 2) # draw_range(time_range, [FWi.obs[0], FWi.obs[-1]], c='black') - #print(ID) + # print(ID) for ob in FWi.obs: - F3 = FWi.sel(obs = ob) - #ax2.plot(F3['time'][~np.isnan(F3['longitude'])], F3['latitude'][~np.isnan(F3['longitude'])]*0 +ob ) + F3 = FWi.sel(obs=ob) + # ax2.plot(F3['time'][~np.isnan(F3['longitude'])], F3['latitude'][~np.isnan(F3['longitude'])]*0 +ob ) nmask = ~np.isnan(F3.latitude) F3 = F3.isel(propag_time=nmask) - #check last point - if (F3.propag_time.size > 0): + # check last point + if F3.propag_time.size > 0: if check_in_mask(F3.isel(propag_time=-1), lon_range, lat_range, time_range): - print(ID, ' match') - D[ID+'_obs'+str(ob.data)] = F3.load() + print(ID, " match") + D[ID + "_obs" + str(ob.data)] = F3.load() return D - -#futures.ProcessPoolExecutor +# futures.ProcessPoolExecutor len(files_to_use) -with futures.ProcessPoolExecutor(max_workers= 7) as executor_sub: - D_nested = list(executor_sub.map(get_possible_tracks_from_file, files_to_use )) +with futures.ProcessPoolExecutor(max_workers=7) as executor_sub: + D_nested = list(executor_sub.map(get_possible_tracks_from_file, files_to_use)) D = {} for d in D_nested: - D.update(d) + D.update(d) # D[next(iter(D.keys()))].load()['VPED'] # D[next(iter(D.keys()))].load()['VTPK'] # D[next(iter(D.keys()))].load()['VAVH'] -#Di = D[next(iter(D.keys()))].load() +# Di = D[next(iter(D.keys()))].load() # %% # make pandas table with obs track end postitions -key_list = next(iter(D.values())).isel(propag_time = -1).keys() -Tend = pd.DataFrame(index = key_list) +key_list = next(iter(D.values())).isel(propag_time=-1).keys() +Tend = pd.DataFrame(index=key_list) + def format_to_list(II): - k,I = II[0], II[1] + k, I = II[0], II[1] dlist = list() for kk in key_list: - dlist.append(I.isel(propag_time = -1)[kk].data) + dlist.append(I.isel(propag_time=-1)[kk].data) return k, dlist -for i in list(map( format_to_list, D.items())): - Tend[i[0]] = i[1] -#Tend['s1a_20190209T09_obs105']['latitude'] +for i in list(map(format_to_list, D.items())): + Tend[i[0]] = i[1] -timetemp = Tend.T['time'].apply( np.datetime64 ) -Tend = Tend.T.astype('float') -Tend['time'] = timetemp +# Tend['s1a_20190209T09_obs105']['latitude'] +timetemp = Tend.T["time"].apply(np.datetime64) +Tend = Tend.T.astype("float") +Tend["time"] = timetemp # %% -weight = np.array(1e2/abs(Tend['time'] - np.datetime64(timestamp)).astype('m8[h]') )**2 -weight= (weight -weight.min()) -weight= weight/weight.max() +weight = ( + np.array(1e2 / abs(Tend["time"] - np.datetime64(timestamp)).astype("m8[h]")) ** 2 +) +weight = weight - weight.min() +weight = weight / weight.max() + def weighted_mean_and_var(a): - am =(a * weight).sum()/weight.sum() - av = ((a-am)**2 * weight).sum()/weight.sum() + am = (a * weight).sum() / weight.sum() + av = ((a - am) ** 2 * weight).sum() / weight.sum() return am, av -Tfinal = Tend[['VAVH', 'VPED','latitude','VTPK','longitude']] + +Tfinal = Tend[["VAVH", "VPED", "latitude", "VTPK", "longitude"]] # rescale variables accorinding to documenation: -Tfinal['VAVH'][Tfinal['VAVH'] == -999] = np.nan -Tfinal['VAVH'] = Tfinal['VAVH'] * 0.001 -Tfinal['VTPK'] = Tfinal['VTPK'] * 0.01 -Tfinal['VPED'] = Tfinal['VPED'] * 0.1 - -Tfinal = Tfinal.rename(columns= {'VPED': 'incident_angle', 'VTPK':'peak_period' , 'VAVH':'Hs'}) -wmeans = Tfinal.apply(weighted_mean_and_var, axis= 0) -wmeans.index = ['w_mean', 'w_var'] +Tfinal["VAVH"][Tfinal["VAVH"] == -999] = np.nan +Tfinal["VAVH"] = Tfinal["VAVH"] * 0.001 +Tfinal["VTPK"] = Tfinal["VTPK"] * 0.01 +Tfinal["VPED"] = Tfinal["VPED"] * 0.1 + +Tfinal = Tfinal.rename( + columns={"VPED": "incident_angle", "VTPK": "peak_period", "VAVH": "Hs"} +) +wmeans = Tfinal.apply(weighted_mean_and_var, axis=0) +wmeans.index = ["w_mean", "w_var"] print(wmeans) -print(np.sqrt(wmeans.T['w_var'])) +print(np.sqrt(wmeans.T["w_var"])) # %% import itertools -col_iter= itertools.cycle(col.circle_big(np.arange(0, 21))) + +col_iter = itertools.cycle(col.circle_big(np.arange(0, 21))) GridSpec font_for_print() -F = M.figure_axis_xy(7.5, 3, view_scale= 0.7, container = True) +F = M.figure_axis_xy(7.5, 3, view_scale=0.7, container=True) plt.suptitle(track_name) -gs = GridSpec(1,3, wspace=0.2, hspace=.7)#figure=fig, +gs = GridSpec(1, 3, wspace=0.2, hspace=0.7) # figure=fig, ax1 = F.fig.add_subplot(gs[0, 0]) -plt.title('Geolocation displacement', loc= 'left') +plt.title("Geolocation displacement", loc="left") -draw_range(lon_range, lat_range, c='black', linewidth = 2) -draw_range( [lon_range[0]+dlon_deg, lon_range[1] -dlon_deg], [lat_range[0]+dlat_deg, lat_range[1] -dlat_deg], c='red', linewidth = 4) +draw_range(lon_range, lat_range, c="black", linewidth=2) +draw_range( + [lon_range[0] + dlon_deg, lon_range[1] - dlon_deg], + [lat_range[0] + dlat_deg, lat_range[1] - dlat_deg], + c="red", + linewidth=4, +) ax2 = F.fig.add_subplot(gs[0, 1]) -plt.title('time displacement', loc= 'left') +plt.title("time displacement", loc="left") kkk = 0 -for k,I in D.items(): +for k, I in D.items(): cc = next(col_iter) - ax1.plot(I['longitude'], I['latitude'], '-', color = cc , linewidth= 0.7) - - angle = I.VPED[~np.isnan(I.latitude)]/10 - pos = I['longitude'], I['latitude'] - ax1.quiver(pos[0][::5], pos[1][::5], -np.sin( angle *np.pi/180)[::5], - np.cos( angle *np.pi/180)[::5] , scale=20, zorder =12, alpha = 0.6, color = 'black' ) - - ax2.plot(I['time'], I['latitude']*0 +kkk , color = cc ) - kkk+=1 - -draw_range(time_range, [-1, kkk+1], c='black', linewidth = 2) + ax1.plot(I["longitude"], I["latitude"], "-", color=cc, linewidth=0.7) + + angle = I.VPED[~np.isnan(I.latitude)] / 10 + pos = I["longitude"], I["latitude"] + ax1.quiver( + pos[0][::5], + pos[1][::5], + -np.sin(angle * np.pi / 180)[::5], + -np.cos(angle * np.pi / 180)[::5], + scale=20, + zorder=12, + alpha=0.6, + color="black", + ) + + ax2.plot(I["time"], I["latitude"] * 0 + kkk, color=cc) + kkk += 1 + +draw_range(time_range, [-1, kkk + 1], c="black", linewidth=2) plt.gcf().autofmt_xdate() -ax2.axvline(timestamp, color ='red', linewidth = 4) +ax2.axvline(timestamp, color="red", linewidth=4) ax3 = F.fig.add_subplot(gs[0, 2]) -plt.title('nearest points and angles', loc= 'left') -#plt.plot( Tend['longitude'] , Tend['latitude'] , '.') - -angle = Tfinal['incident_angle'] -pos = np.array(Tfinal['longitude']), np.array(Tfinal['latitude']) -plt.quiver(pos[0], pos[1], -np.sin( angle *np.pi/180), - np.cos( angle *np.pi/180) , scale=10, zorder =12, alpha = weight) -plt.scatter( Tend['longitude'] , Tend['latitude'] ,color = 'black' , s= 6, alpha = weight ) - -draw_range( [lon_range[0]+dlon_deg, lon_range[1] -dlon_deg], [lat_range[0]+dlat_deg, lat_range[1] -dlat_deg], c='red', linewidth = 2) -F.save_pup(path = plot_path , name = track_name ) +plt.title("nearest points and angles", loc="left") +# plt.plot( Tend['longitude'] , Tend['latitude'] , '.') + +angle = Tfinal["incident_angle"] +pos = np.array(Tfinal["longitude"]), np.array(Tfinal["latitude"]) +plt.quiver( + pos[0], + pos[1], + -np.sin(angle * np.pi / 180), + -np.cos(angle * np.pi / 180), + scale=10, + zorder=12, + alpha=weight, +) +plt.scatter(Tend["longitude"], Tend["latitude"], color="black", s=6, alpha=weight) + +draw_range( + [lon_range[0] + dlon_deg, lon_range[1] - dlon_deg], + [lat_range[0] + dlat_deg, lat_range[1] - dlat_deg], + c="red", + linewidth=2, +) +F.save_pup(path=plot_path, name=track_name) diff --git a/analysis_db/A02b_WW3_hindcast_prior.py b/analysis_db/A02b_WW3_hindcast_prior.py index 3e0732c8..adbab8be 100644 --- a/analysis_db/A02b_WW3_hindcast_prior.py +++ b/analysis_db/A02b_WW3_hindcast_prior.py @@ -1,15 +1,15 @@ - import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -23,88 +23,101 @@ import spicke_remover import datetime import concurrent.futures as futures -#import s3fs + +# import s3fs col.colormaps2(21) # %% -track_name, batch_key, ID_flag = io.init_from_input(sys.argv) # loads standard experiment -#track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False -#track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190217194220_07840212_004_01', 'SH_batch02', False -#track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False -#track_name, batch_key, ID_flag = 'NH_20190301_09600205', 'NH_batch05', True - -#track_name, batch_key, ID_flag = 'NH_20210228_10231005', 'NH_batch07', True +track_name, batch_key, ID_flag = io.init_from_input( + sys.argv +) # loads standard experiment +# track_name, batch_key, test_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190601094826_09790312_004_01', 'SH_batch01', False +# track_name, batch_key, test_flag = '20190207111114_06260210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190217194220_07840212_004_01', 'SH_batch02', False +# track_name, batch_key, test_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +# track_name, batch_key, ID_flag = 'NH_20190301_09600205', 'NH_batch05', True + +# track_name, batch_key, ID_flag = 'NH_20210228_10231005', 'NH_batch07', True track_name_short = track_name[0:-16] -ID, track_names, hemis, batch = io.init_data(track_name, batch_key, ID_flag, mconfig['paths']['work'] ) - -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' - -save_path = mconfig['paths']['work'] + batch_key+'/A02_prior/' -plot_path = mconfig['paths']['plot'] + '/'+hemis+'/'+batch_key+'/' + track_name + '/' -save_name = 'A02_'+track_name -plot_name = 'A02_'+track_name_short +ID, track_names, hemis, batch = io.init_data( + track_name, batch_key, ID_flag, mconfig["paths"]["work"] +) + +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" + +save_path = mconfig["paths"]["work"] + batch_key + "/A02_prior/" +plot_path = ( + mconfig["paths"]["plot"] + "/" + hemis + "/" + batch_key + "/" + track_name + "/" +) +save_name = "A02_" + track_name +plot_name = "A02_" + track_name_short MT.mkdirs_r(plot_path) MT.mkdirs_r(save_path) -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" # %% -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] -low_beams = mconfig['beams']['low_beams'] -#Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] +low_beams = mconfig["beams"]["low_beams"] +# Gfilt = io.load_pandas_table_dict(track_name + '_B01_regridded', load_path) # rhis is the rar photon data -load_path_WAVE_GLO = mconfig['paths']['work'] +'/GLOBMULTI_ERA5_GLOBCUR_01/' -file_name_base = 'LOPS_WW3-GLOB-30M_' +load_path_WAVE_GLO = mconfig["paths"]["work"] + "/GLOBMULTI_ERA5_GLOBCUR_01/" +file_name_base = "LOPS_WW3-GLOB-30M_" -load_path = mconfig['paths']['work'] +batch_key+'/B01_regrid/' -#Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # -Gd = h5py.File(load_path +'/'+track_name + '_B01_binned.h5', 'r') +load_path = mconfig["paths"]["work"] + batch_key + "/B01_regrid/" +# Gd = io.load_pandas_table_dict(track_name + '_B01_binned' , load_path) # +Gd = h5py.File(load_path + "/" + track_name + "_B01_binned.h5", "r") # %% G1 = dict() for b in all_beams: # find 1st point - Gi = io.get_beam_hdf_store(Gd[b]) - G1[b] = Gi.iloc[abs(Gi['lats']).argmin()] + Gi = io.get_beam_hdf_store(Gd[b]) + G1[b] = Gi.iloc[abs(Gi["lats"]).argmin()] Gd.close() G1 = pd.DataFrame.from_dict(G1).T -if hemis == 'SH': +if hemis == "SH": # DEFINE SEARCH REGION AND SIZE OF BOXES FOR AVERAGES - dlon_deg = 1 # degree range aroud 1st point - dlat_deg = 30, 5 # degree range aroud 1st point - dlat_deg_prior = 2, 1 # degree range aroud 1st point + dlon_deg = 1 # degree range aroud 1st point + dlat_deg = 30, 5 # degree range aroud 1st point + dlat_deg_prior = 2, 1 # degree range aroud 1st point - dtime = 4 # in hours + dtime = 4 # in hours - lon_range = G1['lons'].min() - dlon_deg , G1['lons'].max() + dlon_deg - lat_range = np.sign(G1['lats'].min())*78 , G1['lats'].max() + dlat_deg[1] - lat_range_prior = G1['lats'].min() - dlat_deg_prior[0] , G1['lats'].max() + dlat_deg_prior[1] + lon_range = G1["lons"].min() - dlon_deg, G1["lons"].max() + dlon_deg + lat_range = np.sign(G1["lats"].min()) * 78, G1["lats"].max() + dlat_deg[1] + lat_range_prior = ( + G1["lats"].min() - dlat_deg_prior[0], + G1["lats"].max() + dlat_deg_prior[1], + ) else: # DEFINE SEARCH REGION AND SIZE OF BOXES FOR AVERAGES - dlon_deg = 2 # lon degree range aroud 1st point - dlat_deg = 20, 20 # lat degree range aroud 1st point - dlat_deg_prior = 2, 1 # degree range aroud 1st point + dlon_deg = 2 # lon degree range aroud 1st point + dlat_deg = 20, 20 # lat degree range aroud 1st point + dlat_deg_prior = 2, 1 # degree range aroud 1st point - dtime = 4 # in hours + dtime = 4 # in hours - #ID['pars']['end'] - lon_range = G1['lons'].min() - dlon_deg , G1['lons'].max() + dlon_deg - lat_range = G1['lats'].min() - dlat_deg[0] , G1['lats'].max() + dlat_deg[1] - lat_range_prior = G1['lats'].min() - dlat_deg_prior[0] , G1['lats'].max() + dlat_deg_prior[1] + # ID['pars']['end'] + lon_range = G1["lons"].min() - dlon_deg, G1["lons"].max() + dlon_deg + lat_range = G1["lats"].min() - dlat_deg[0], G1["lats"].max() + dlat_deg[1] + lat_range_prior = ( + G1["lats"].min() - dlat_deg_prior[0], + G1["lats"].max() + dlat_deg_prior[1], + ) # lon_range = ID['pars']['start']['longitude'] - dlon_deg , ID['pars']['start']['longitude'] + dlon_deg # lat_range = ID['pars']['start']['latitude'] - dlat_deg[0] , ID['pars']['start']['latitude'] + dlat_deg[1] @@ -112,11 +125,13 @@ # load .json to get time -#ID['tracks']['ATL07'].split('_')[1] -timestamp = pd.to_datetime(ID['tracks']['ATL07'].split('_')[1]) -#timestamp = pd.to_datetime(G1[['year', 'month', 'day', 'hour', 'minute', 'second']]).mean() -time_range = np.datetime64(timestamp) - np.timedelta64(dtime, 'h') , np.datetime64(timestamp) + np.timedelta64(dtime, 'h') -#print(time_range) +# ID['tracks']['ATL07'].split('_')[1] +timestamp = pd.to_datetime(ID["tracks"]["ATL07"].split("_")[1]) +# timestamp = pd.to_datetime(G1[['year', 'month', 'day', 'hour', 'minute', 'second']]).mean() +time_range = np.datetime64(timestamp) - np.timedelta64(dtime, "h"), np.datetime64( + timestamp +) + np.timedelta64(dtime, "h") +# print(time_range) # create timestamp according to fiels on ftp server: # time_stamps_search = np.arange(time_range[0].astype('datetime64[3h]') - np.timedelta64(12*24, 'h') , time_range[1].astype('datetime64[3h]') + np.timedelta64(3, 'h'), np.timedelta64(3, 'h')) @@ -125,20 +140,21 @@ # %% import glob -if time_range[0].astype('M8[M]') != time_range[1].astype('M8[M]'): # spanning two years - MM_str = str(time_range[0].astype('M8[M]')).replace('-', '_') - f_list1 = glob.glob(load_path_WAVE_GLO+'/*'+MM_str+'_'+hemis+'*.nc') +if time_range[0].astype("M8[M]") != time_range[1].astype("M8[M]"): # spanning two years + MM_str = str(time_range[0].astype("M8[M]")).replace("-", "_") + f_list1 = glob.glob(load_path_WAVE_GLO + "/*" + MM_str + "_" + hemis + "*.nc") - MM_str = str(time_range[-1].astype('M8[M]')).replace('-', '_') - f_list2 = glob.glob(load_path_WAVE_GLO+'/*'+MM_str+'_'+hemis+'*.nc') + MM_str = str(time_range[-1].astype("M8[M]")).replace("-", "_") + f_list2 = glob.glob(load_path_WAVE_GLO + "/*" + MM_str + "_" + hemis + "*.nc") f_list = f_list1 + f_list2 else: - MM_str = str(time_range[0].astype('M8[M]')).replace('-', '_') - f_list = glob.glob(load_path_WAVE_GLO+'/*'+MM_str+'_'+hemis+'*.nc') + MM_str = str(time_range[0].astype("M8[M]")).replace("-", "_") + f_list = glob.glob(load_path_WAVE_GLO + "/*" + MM_str + "_" + hemis + "*.nc") print(f_list) -def sel_data(I, lon_range, lat_range, timestamp = None): + +def sel_data(I, lon_range, lat_range, timestamp=None): """ this method returns the selected data inthe lon-lat box at an interpolated timestamp """ @@ -146,13 +162,16 @@ def sel_data(I, lon_range, lat_range, timestamp = None): lat_flag = (lat_range[0] < I.latitude.data) & (I.latitude.data < lat_range[1]) time_flag = (time_range[0] < I.time.data) & (I.time.data < time_range[1]) if timestamp is None: - I = I.isel(latitude = lat_flag, longitude = lon_flag) + I = I.isel(latitude=lat_flag, longitude=lon_flag) else: - I = I.interp(time=np.datetime64(timestamp)).isel(latitude = lat_flag, longitude = lon_flag) + I = I.interp(time=np.datetime64(timestamp)).isel( + latitude=lat_flag, longitude=lon_flag + ) - #I = I.isel(latitude = lat_flag, longitude = lon_flag) + # I = I.isel(latitude = lat_flag, longitude = lon_flag) return I + # # % # Gww3 = xr.open_mfdataset(load_path_WAVE_GLO+'/*'+'2019_*_'+hemis+'*.nc') # @@ -169,253 +188,382 @@ def sel_data(I, lon_range, lat_range, timestamp = None): try: # load file and load WW# data Gww3 = xr.open_mfdataset(f_list) - G_beam = sel_data(Gww3 , lon_range, lat_range , timestamp).load() - G_prior = sel_data(G_beam , lon_range, lat_range_prior) - + G_beam = sel_data(Gww3, lon_range, lat_range, timestamp).load() + G_prior = sel_data(G_beam, lon_range, lat_range_prior) - if hemis == 'SH': + if hemis == "SH": # % create Ice mask ice_mask = (G_beam.ice > 0) | np.isnan(G_beam.ice) - #G1.mean()['lats'] + # G1.mean()['lats'] # mask_at_points = (ice_mask.sum('longitude') == ice_mask.shape[1]).sel(latitude =slice(G1['lats'].min(), G1['lats'].max())) # if (mask_at_points.sum().data == mask_at_points.size): # print('all points in ice mask') # lat_range_prior # lat_range_prior = lat_range_prior, lat_range_prior[1] + 2 - #ice_mask.sel(latitude=G1.mean()['lats'], longitude =G1.mean()['lons'], method ='nearest') + # ice_mask.sel(latitude=G1.mean()['lats'], longitude =G1.mean()['lons'], method ='nearest') # mask all latitudes that are completely full with sea ice. lats = list(ice_mask.latitude.data) - lats.sort(reverse= True) - #(ice_mask.sum('longitude') == ice_mask.longitude.size).sel(latitude = lats) + lats.sort(reverse=True) + # (ice_mask.sum('longitude') == ice_mask.longitude.size).sel(latitude = lats) # find 1st latitude that is completely full with sea ice. - ice_lat_pos = next((i for i, j in enumerate( (ice_mask.sum('longitude') == ice_mask.longitude.size).sel(latitude = lats)) if j), None) + ice_lat_pos = next( + ( + i + for i, j in enumerate( + (ice_mask.sum("longitude") == ice_mask.longitude.size).sel( + latitude=lats + ) + ) + if j + ), + None, + ) # recreate lat mask based on this criteria lat_mask = lats < lats[ice_lat_pos] - lat_mask = xr.DataArray( lat_mask.repeat(ice_mask.longitude.size ).reshape(ice_mask.shape), dims = ice_mask.dims, coords = ice_mask.coords ) - lat_mask['latitude'] =lats + lat_mask = xr.DataArray( + lat_mask.repeat(ice_mask.longitude.size).reshape(ice_mask.shape), + dims=ice_mask.dims, + coords=ice_mask.coords, + ) + lat_mask["latitude"] = lats # combine ice mask and new lat mask ice_mask = ice_mask + lat_mask else: - ice_mask =np.isnan(G_beam.ice) + ice_mask = np.isnan(G_beam.ice) - #G_beam.hs.sum('longitude').plot() + # G_beam.hs.sum('longitude').plot() lats = ice_mask.latitude - #lats.sort(reverse= True) - + # lats.sort(reverse= True) # find closed latituyde with with non-nan data - ice_lat_pos = abs(lats.where(ice_mask.sum('longitude') > 4, np.nan) - np.array(lat_range).mean()).argmin().data - - #redefine lat-range - lat_range = lats[ice_lat_pos].data - 2 , lats[ice_lat_pos].data + 2 + ice_lat_pos = ( + abs( + lats.where(ice_mask.sum("longitude") > 4, np.nan) + - np.array(lat_range).mean() + ) + .argmin() + .data + ) + + # redefine lat-range + lat_range = lats[ice_lat_pos].data - 2, lats[ice_lat_pos].data + 2 lat_flag2 = (lat_range[0] < lats.data) & (lats.data < lat_range[1]) # find 1st latitude that has non-zero HS . - #ice_lat_pos = next((i for i, j in enumerate( (G_beam.hs.sum('longitude') != 0).sel(latitude = lats)) if j), None) + # ice_lat_pos = next((i for i, j in enumerate( (G_beam.hs.sum('longitude') != 0).sel(latitude = lats)) if j), None) # recreate lat mask based on this criteria - #lat_mask = lats < lats[ice_lat_pos] - lat_mask = xr.DataArray( lat_flag2.repeat(ice_mask.longitude.size ).reshape(ice_mask.shape), dims = ice_mask.dims, coords = ice_mask.coords ) - lat_mask['latitude'] =lats + # lat_mask = lats < lats[ice_lat_pos] + lat_mask = xr.DataArray( + lat_flag2.repeat(ice_mask.longitude.size).reshape(ice_mask.shape), + dims=ice_mask.dims, + coords=ice_mask.coords, + ) + lat_mask["latitude"] = lats # combine ice mask and new lat mask - #ice_mask = ~lat_mask - + # ice_mask = ~lat_mask # plot 1st figure def draw_range(lon_range, lat_range, *args, **kargs): - plt.plot( [lon_range[0], lon_range[1], lon_range[1], lon_range[0], lon_range[0]] , [lat_range[0], lat_range[0], lat_range[1], lat_range[1], lat_range[0]] , *args, **kargs) - - dir_clev= np.arange(0, 380, 20) - f_clev = np.arange(1/40, 1/5, 0.01) - fvar = ['ice', 'dir', 'dp', 'spr', 'fp', 'hs'] - fcmap =[plt.cm.Blues_r, col.circle_medium_triple, col.circle_medium_triple, plt.cm.Blues, plt.cm.Blues, plt.cm.Blues ] + plt.plot( + [lon_range[0], lon_range[1], lon_range[1], lon_range[0], lon_range[0]], + [lat_range[0], lat_range[0], lat_range[1], lat_range[1], lat_range[0]], + *args, + **kargs, + ) + + dir_clev = np.arange(0, 380, 20) + f_clev = np.arange(1 / 40, 1 / 5, 0.01) + fvar = ["ice", "dir", "dp", "spr", "fp", "hs"] + fcmap = [ + plt.cm.Blues_r, + col.circle_medium_triple, + col.circle_medium_triple, + plt.cm.Blues, + plt.cm.Blues, + plt.cm.Blues, + ] fpos = [0, 1, 2, 3, 4, 5] - clevs = [np.arange(0, 1, 0.2), dir_clev, dir_clev, np.arange(0, 90, 10), f_clev, np.arange(.5, 9, 0.5) ] + clevs = [ + np.arange(0, 1, 0.2), + dir_clev, + dir_clev, + np.arange(0, 90, 10), + f_clev, + np.arange(0.5, 9, 0.5), + ] font_for_print() - #plt.rc('pcolor', shading = 'auto') - F = M.figure_axis_xy(4, 3.5, view_scale= 0.9, container = True) - plt.suptitle(track_name_short + ' | ' +file_name_base[0:-1].replace('_', ' '), y = 1.3) - lon, lat= G_beam.longitude, G_beam.latitude + # plt.rc('pcolor', shading = 'auto') + F = M.figure_axis_xy(4, 3.5, view_scale=0.9, container=True) + plt.suptitle( + track_name_short + " | " + file_name_base[0:-1].replace("_", " "), y=1.3 + ) + lon, lat = G_beam.longitude, G_beam.latitude - gs = GridSpec(9,6, wspace=0.1, hspace=0.4)#figure=fig, - #pos0,pos1,pos2 = gs[0:3, 0],gs[3, 0],gs[4, 0]#,gs[3, 0] + gs = GridSpec(9, 6, wspace=0.1, hspace=0.4) # figure=fig, + # pos0,pos1,pos2 = gs[0:3, 0],gs[3, 0],gs[4, 0]#,gs[3, 0] for fv, fp, fc, cl in zip(fvar, fpos, fcmap, clevs): - ax1 = F.fig.add_subplot(gs[0:7, fp]) #plt.subplot(1, 6, fp) - if fp ==0: - ax1.spines['bottom'].set_visible(False) - ax1.spines['left'].set_visible(False) + ax1 = F.fig.add_subplot(gs[0:7, fp]) # plt.subplot(1, 6, fp) + if fp == 0: + ax1.spines["bottom"].set_visible(False) + ax1.spines["left"].set_visible(False) ax1.tick_params(labelbottom=True, bottom=True) else: - ax1.axis('off') - - plt.plot(G1['lons'], G1['lats'], '.r', markersize=5) - draw_range(lon_range, lat_range_prior, c='red', linewidth = 1, zorder=12) - draw_range(lon_range, lat_range, c='blue', linewidth = 0.7, zorder=10) - #G_beam.ice.plot(cmap=plt.cm.Blues_r, ) - if fv != 'ice': - #.where(~ice_mask, np.nan) - cm = plt.pcolor(lon, lat,G_beam[fv], vmin=cl[0], vmax=cl[-1] , cmap=fc) + ax1.axis("off") + + plt.plot(G1["lons"], G1["lats"], ".r", markersize=5) + draw_range(lon_range, lat_range_prior, c="red", linewidth=1, zorder=12) + draw_range(lon_range, lat_range, c="blue", linewidth=0.7, zorder=10) + # G_beam.ice.plot(cmap=plt.cm.Blues_r, ) + if fv != "ice": + # .where(~ice_mask, np.nan) + cm = plt.pcolor(lon, lat, G_beam[fv], vmin=cl[0], vmax=cl[-1], cmap=fc) if G_beam.ice.shape[0] > 1: - plt.contour(lon, lat,G_beam.ice, colors= 'black', linewidths = 0.6) + plt.contour(lon, lat, G_beam.ice, colors="black", linewidths=0.6) else: - cm =plt.pcolor(lon, lat,G_beam[fv], vmin=cl[0], vmax=cl[-1], cmap=fc) + cm = plt.pcolor(lon, lat, G_beam[fv], vmin=cl[0], vmax=cl[-1], cmap=fc) - #plt.title(fv, loc='center') - plt.title(G_beam[fv].long_name.replace(' ', '\n') +'\n'+ fv, loc='left') - ax1.axis('equal') + # plt.title(fv, loc='center') + plt.title(G_beam[fv].long_name.replace(" ", "\n") + "\n" + fv, loc="left") + ax1.axis("equal") - ax2 = F.fig.add_subplot(gs[-1, fp]) #plt.subplot(1, 6, fp) - #plt.axis('off') - cbar = plt.colorbar(cm, cax= ax2, orientation = 'horizontal', aspect= 1, fraction=1) - cl_ticks = np.linspace(cl[0], cl[-1] , 3) + ax2 = F.fig.add_subplot(gs[-1, fp]) # plt.subplot(1, 6, fp) + # plt.axis('off') + cbar = plt.colorbar(cm, cax=ax2, orientation="horizontal", aspect=1, fraction=1) + cl_ticks = np.linspace(cl[0], cl[-1], 3) - cbar.set_ticks( np.round( cl_ticks, 3) ) - #[str(i) for i in np.round( cl_ticks, 1) ] - cbar.set_ticklabels( np.round( cl_ticks, 2)) - - F.save_pup(path= plot_path, name =plot_name+'_hindcast_data') + cbar.set_ticks(np.round(cl_ticks, 3)) + # [str(i) for i in np.round( cl_ticks, 1) ] + cbar.set_ticklabels(np.round(cl_ticks, 2)) + F.save_pup(path=plot_path, name=plot_name + "_hindcast_data") # % derive prior: - #G_beam_masked['dir'] + # G_beam_masked['dir'] G_beam_masked = G_beam.where(~ice_mask, np.nan) ice_mask_prior = ice_mask.sel(latitude=G_prior.latitude) G_prior_masked = G_prior.where(~ice_mask_prior, np.nan) - def test_nan_frac(imask): "test if False is less then 0.3" - return ((~imask).sum()/imask.size).data < 0.3 + return ((~imask).sum() / imask.size).data < 0.3 while test_nan_frac(ice_mask_prior): print(lat_range_prior) lat_range_prior = lat_range_prior[0] + 0.5, lat_range_prior[1] + 0.5 - G_prior = sel_data(G_beam , lon_range, lat_range_prior) + G_prior = sel_data(G_beam, lon_range, lat_range_prior) ice_mask_prior = ice_mask.sel(latitude=G_prior.latitude) G_prior_masked = G_prior.where(~ice_mask_prior, np.nan) - - ### make pandas table with obs track end postitions key_list = list(G_prior_masked.keys()) # define directional and amplitude pairs # pack as (amp, angle) key_list_pairs = { - 'mean': ( 'hs', 'dir'), - 'peak': ( 'hs', 'dp'), - 'partion0': ('phs0', 'pdp0'), - 'partion1': ('phs1', 'pdp1'), - 'partion2': ('phs2', 'pdp2'), - 'partion3': ('phs3', 'pdp3'), - 'partion4': ('phs4', 'pdp4') } - - - key_list_pairs2 =list() + "mean": ("hs", "dir"), + "peak": ("hs", "dp"), + "partion0": ("phs0", "pdp0"), + "partion1": ("phs1", "pdp1"), + "partion2": ("phs2", "pdp2"), + "partion3": ("phs3", "pdp3"), + "partion4": ("phs4", "pdp4"), + } + + key_list_pairs2 = list() for k in key_list_pairs.values(): key_list_pairs2.append(k[0]) key_list_pairs2.append(k[1]) key_list_scaler = set(key_list) - set(key_list_pairs2) - ### derive angle avearge - Tend = pd.DataFrame(index =key_list, columns = ['mean', 'std', 'name'] ) + Tend = pd.DataFrame(index=key_list, columns=["mean", "std", "name"]) for k, pair in key_list_pairs.items(): - ave_amp, ave_deg, std_amp, std_deg = waves.get_ave_amp_angle(G_prior_masked[pair[0]].data, G_prior_masked[pair[1]].data) + ave_amp, ave_deg, std_amp, std_deg = waves.get_ave_amp_angle( + G_prior_masked[pair[0]].data, G_prior_masked[pair[1]].data + ) Tend.loc[pair[0]] = ave_amp, std_amp, G_prior_masked[pair[0]].long_name Tend.loc[pair[1]] = ave_deg, std_deg, G_prior_masked[pair[1]].long_name - #key_list_pairs[k] = {pair[0]:ave_amp, pair[1]:ave_deg } + # key_list_pairs[k] = {pair[0]:ave_amp, pair[1]:ave_deg } for k in key_list_scaler: - Tend.loc[k] = G_prior_masked[k].mean().data, G_prior_masked[k].std().data, G_prior_masked[k].long_name - + Tend.loc[k] = ( + G_prior_masked[k].mean().data, + G_prior_masked[k].std().data, + G_prior_masked[k].long_name, + ) Tend = Tend.T - Tend['lon'] = [ice_mask_prior.longitude.mean().data ,ice_mask_prior.longitude.std().data , 'lontigude'] - Tend['lat'] = [ice_mask_prior.latitude[ice_mask_prior.sum('longitude') ==0].mean().data ,ice_mask_prior.latitude[ice_mask_prior.sum('longitude') ==0].std().data , 'latitude'] + Tend["lon"] = [ + ice_mask_prior.longitude.mean().data, + ice_mask_prior.longitude.std().data, + "lontigude", + ] + Tend["lat"] = [ + ice_mask_prior.latitude[ice_mask_prior.sum("longitude") == 0].mean().data, + ice_mask_prior.latitude[ice_mask_prior.sum("longitude") == 0].std().data, + "latitude", + ] Tend = Tend.T Prior = dict() - Prior['incident_angle'] = {'value': Tend['mean']['dp'].astype('float') , 'name': Tend['name']['dp']} - Prior['spread'] = {'value': Tend['mean']['spr'].astype('float') , 'name': Tend['name']['spr']} - Prior['Hs'] = {'value': Tend['mean']['hs'].astype('float') , 'name': Tend['name']['hs']} - Prior['peak_period'] = {'value': 1/Tend['mean']['fp'].astype('float') , 'name': '1/' +Tend['name']['fp']} - - Prior['center_lon'] = {'value': Tend['mean']['lon'].astype('float') , 'name': Tend['name']['lon']} - Prior['center_lat'] = {'value': Tend['mean']['lat'].astype('float') , 'name': Tend['name']['lat']} - + Prior["incident_angle"] = { + "value": Tend["mean"]["dp"].astype("float"), + "name": Tend["name"]["dp"], + } + Prior["spread"] = { + "value": Tend["mean"]["spr"].astype("float"), + "name": Tend["name"]["spr"], + } + Prior["Hs"] = { + "value": Tend["mean"]["hs"].astype("float"), + "name": Tend["name"]["hs"], + } + Prior["peak_period"] = { + "value": 1 / Tend["mean"]["fp"].astype("float"), + "name": "1/" + Tend["name"]["fp"], + } + + Prior["center_lon"] = { + "value": Tend["mean"]["lon"].astype("float"), + "name": Tend["name"]["lon"], + } + Prior["center_lat"] = { + "value": Tend["mean"]["lat"].astype("float"), + "name": Tend["name"]["lat"], + } def plot_prior(Prior, axx): - angle = Prior['incident_angle']['value'] # incident direction in degrees from North clockwise (Meerological convention) + angle = Prior["incident_angle"][ + "value" + ] # incident direction in degrees from North clockwise (Meerological convention) # use - angle_plot = - angle -90 - axx.quiver(Prior['center_lon']['value'], Prior['center_lat']['value'], - np.cos( angle_plot *np.pi/180), - np.sin( angle_plot *np.pi/180) , scale=4.5, zorder =12, width=0.1 ,headlength = 4.5, minshaft=2, alpha = 0.6, color = 'black' ) - axx.plot(Prior['center_lon']['value'], Prior['center_lat']['value'] , '.', markersize= 6, zorder =12, alpha = 1, color = 'black' ) - tstring= ' ' +str(np.round( Prior['peak_period']['value'], 1) )+'sec \n ' + str( np.round(Prior['Hs']['value'], 1) )+'m\n ' + str(np.round(angle, 1)) +'deg' - plt.text(lon_range[1], Prior['center_lat']['value'], tstring) - + angle_plot = -angle - 90 + axx.quiver( + Prior["center_lon"]["value"], + Prior["center_lat"]["value"], + -np.cos(angle_plot * np.pi / 180), + -np.sin(angle_plot * np.pi / 180), + scale=4.5, + zorder=12, + width=0.1, + headlength=4.5, + minshaft=2, + alpha=0.6, + color="black", + ) + axx.plot( + Prior["center_lon"]["value"], + Prior["center_lat"]["value"], + ".", + markersize=6, + zorder=12, + alpha=1, + color="black", + ) + tstring = ( + " " + + str(np.round(Prior["peak_period"]["value"], 1)) + + "sec \n " + + str(np.round(Prior["Hs"]["value"], 1)) + + "m\n " + + str(np.round(angle, 1)) + + "deg" + ) + plt.text(lon_range[1], Prior["center_lat"]["value"], tstring) font_for_print() - F = M.figure_axis_xy(2, 4.5, view_scale= 0.9, container = False) + F = M.figure_axis_xy(2, 4.5, view_scale=0.9, container=False) ax1 = F.ax - lon, lat= G_beam.longitude, G_beam.latitude - #gs = GridSpec(1,6, wspace=0.1, hspace=0.4)#figure=fig, - #pos0,pos1,pos2 = gs[0:3, 0],gs[3, 0],gs[4, 0]#,gs[3, 0] - ax1.spines['bottom'].set_visible(False) - ax1.spines['left'].set_visible(False) + lon, lat = G_beam.longitude, G_beam.latitude + # gs = GridSpec(1,6, wspace=0.1, hspace=0.4)#figure=fig, + # pos0,pos1,pos2 = gs[0:3, 0],gs[3, 0],gs[4, 0]#,gs[3, 0] + ax1.spines["bottom"].set_visible(False) + ax1.spines["left"].set_visible(False) ax1.tick_params(labelbottom=True, bottom=True) plot_prior(Prior, ax1) str_list = list() for i in np.arange(0, 6): - str_list.append(' ' + str(np.round(Tend.loc['ptp'+str(i)]['mean'], 1)) +'sec\n ' + str(np.round(Tend.loc['phs'+str(i)]['mean'], 1)) +'m '+str(np.round(Tend.loc['pdp'+str(i)]['mean'], 1))+'d') - - plt.text(lon_range[1], lat_range[0], '\n '.join(str_list) ) - - for vv in zip(['pdp0','pdp1','pdp2','pdp3','pdp4','pdp5'],['phs0','phs1','phs3','phs4','phs5']) : - - angle_plot = - Tend.loc[vv[0]]['mean'] -90 - vsize = (1/Tend.loc[vv[1]]['mean'])**(1/2) *5 + str_list.append( + " " + + str(np.round(Tend.loc["ptp" + str(i)]["mean"], 1)) + + "sec\n " + + str(np.round(Tend.loc["phs" + str(i)]["mean"], 1)) + + "m " + + str(np.round(Tend.loc["pdp" + str(i)]["mean"], 1)) + + "d" + ) + + plt.text(lon_range[1], lat_range[0], "\n ".join(str_list)) + + for vv in zip( + ["pdp0", "pdp1", "pdp2", "pdp3", "pdp4", "pdp5"], + ["phs0", "phs1", "phs3", "phs4", "phs5"], + ): + + angle_plot = -Tend.loc[vv[0]]["mean"] - 90 + vsize = (1 / Tend.loc[vv[1]]["mean"]) ** (1 / 2) * 5 print(vsize) - ax1.quiver(Prior['center_lon']['value'], Prior['center_lat']['value'], - np.cos( angle_plot *np.pi/180), - np.sin( angle_plot *np.pi/180) , scale=vsize, zorder =5, width=0.1 ,headlength = 4.5, minshaft=4, alpha = 0.6, color = 'green' ) - - plt.plot(G1['lons'], G1['lats'], '.r', markersize=5) - draw_range(lon_range, lat_range_prior, c='red', linewidth = 1, zorder=11) - draw_range(lon_range, lat_range, c='blue', linewidth = 0.7, zorder=10) - - #G_beam.ice.plot(cmap=plt.cm.Blues_r, ) - fv ='ice' - if fv != 'ice': - plt.pcolor(lon, lat,G_beam[fv].where(~ice_mask, np.nan), cmap=fc) - plt.contour(lon, lat,G_beam.ice, colors= 'black', linewidths = 0.6) + ax1.quiver( + Prior["center_lon"]["value"], + Prior["center_lat"]["value"], + -np.cos(angle_plot * np.pi / 180), + -np.sin(angle_plot * np.pi / 180), + scale=vsize, + zorder=5, + width=0.1, + headlength=4.5, + minshaft=4, + alpha=0.6, + color="green", + ) + + plt.plot(G1["lons"], G1["lats"], ".r", markersize=5) + draw_range(lon_range, lat_range_prior, c="red", linewidth=1, zorder=11) + draw_range(lon_range, lat_range, c="blue", linewidth=0.7, zorder=10) + + # G_beam.ice.plot(cmap=plt.cm.Blues_r, ) + fv = "ice" + if fv != "ice": + plt.pcolor(lon, lat, G_beam[fv].where(~ice_mask, np.nan), cmap=fc) + plt.contour(lon, lat, G_beam.ice, colors="black", linewidths=0.6) else: - plt.pcolor(lon, lat,G_beam[fv], cmap=fc) - - #plt.title(fv, loc='center') - plt.title('Prior\n' + file_name_base[0:-1].replace('_', ' ') +'\n'+ track_name_short + '\nIncident angle', loc='left') - ax1.axis('equal') + plt.pcolor(lon, lat, G_beam[fv], cmap=fc) + # plt.title(fv, loc='center') + plt.title( + "Prior\n" + + file_name_base[0:-1].replace("_", " ") + + "\n" + + track_name_short + + "\nIncident angle", + loc="left", + ) + ax1.axis("equal") - F.save_pup(path= plot_path, name =plot_name+'_hindcast_prior') + F.save_pup(path=plot_path, name=plot_name + "_hindcast_prior") - MT.save_pandas_table({'priors_hindcast':Tend}, save_name, save_path) + MT.save_pandas_table({"priors_hindcast": Tend}, save_name, save_path) - target_name = 'A02_'+track_name+'_hindcast_success' + target_name = "A02_" + track_name + "_hindcast_success" except: - target_name = 'A02_'+track_name+'hindcast_fail' + target_name = "A02_" + track_name + "hindcast_fail" -MT.json_save(target_name, save_path, str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M")) ) +MT.json_save( + target_name, save_path, str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M")) +) diff --git a/analysis_db/B01_SL_load_batch.py b/analysis_db/B01_SL_load_batch.py index 888a47f4..d581aa04 100644 --- a/analysis_db/B01_SL_load_batch.py +++ b/analysis_db/B01_SL_load_batch.py @@ -1,10 +1,10 @@ - # %% """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3.11 """ import os, sys + exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) @@ -23,22 +23,22 @@ import h5py, imp, copy -xr.set_options(display_style='text') +xr.set_options(display_style="text") # %matplotlib inline # %matplotlib widget plot_flag = True -load_path_RGT = mconfig['paths']['analysis'] +'../analysis_db/support_files/' +load_path_RGT = mconfig["paths"]["analysis"] + "../analysis_db/support_files/" -batch_key = 'SH_testSL' -save_path = mconfig['paths']['work'] +'/'+batch_key+'/B01_regrid/' +batch_key = "SH_testSL" +save_path = mconfig["paths"]["work"] + "/" + batch_key + "/B01_regrid/" MT.mkdirs_r(save_path) # %% Configure SL Session # -icesat2.init("slideruleearth.io", True) -asset = 'nsidc-s3' +icesat2.init("slideruleearth.io", True) +asset = "nsidc-s3" # %% Select region and retrive batch of tracks @@ -46,25 +46,29 @@ latR, lonR = [-67.2, -64.3], [140.0, 155.0] # Northern Hemisphere test -#latR, lonR = [65.0, 75.0], [140.0, 155.0] +# latR, lonR = [65.0, 75.0], [140.0, 155.0] -# init polygon opbjects +# init polygon opbjects poly = sct.create_polygons(latR, lonR) # %% load ground track data in domain: -#Gtrack = gpd.read_file(load_path_RGT + 'IS2_mission_points_NH_RGT_all.shp', mask=poly['shapely']) -Gtrack = gpd.read_file(load_path_RGT + 'IS2_mission_points_SH_RGT_all.shp', mask=poly['shapely']) +# Gtrack = gpd.read_file(load_path_RGT + 'IS2_mission_points_NH_RGT_all.shp', mask=poly['shapely']) +Gtrack = gpd.read_file( + load_path_RGT + "IS2_mission_points_SH_RGT_all.shp", mask=poly["shapely"] +) Gtrack_lowest = sct.get_RGT_start_points(Gtrack) -# plot the ground tracks in geographic location -#if plot_flag: -m = sct.plot_polygon(poly['list'], basemap=basemaps.Esri.WorldImagery, zoom=4) -#m.add_layer(poly['shapely'], c ='red') -geo_data = GeoData(geo_dataframe = Gtrack[::5], - #style={'color': 'red', 'radius':1}, - point_style={'radius': 0.1, 'color': 'red', 'fillOpacity': 0.1}, - name = 'rgt') +# plot the ground tracks in geographic location +# if plot_flag: +m = sct.plot_polygon(poly["list"], basemap=basemaps.Esri.WorldImagery, zoom=4) +# m.add_layer(poly['shapely'], c ='red') +geo_data = GeoData( + geo_dataframe=Gtrack[::5], + # style={'color': 'red', 'radius':1}, + point_style={"radius": 0.1, "color": "red", "fillOpacity": 0.1}, + name="rgt", +) m.add_layer(geo_data) m @@ -72,37 +76,45 @@ # Generate ATL06-type segments using the ATL03-native photon classification # Use the ocean classification for photons with a confidence parmeter to 2 or higher (low confidence or better) -params={'srt': 1, # Ocean classification - 'len': 25, # 10-meter segments - 'ats':3, # require that each segment contain photons separated by at least 5 m - 'res':10, # return one photon every 10 m - 'track': 0, # return all ground tracks - 'pass_invalid': True, - 'cnf': 2, # require classification confidence of 2 or more - #'iterations':10, # iterate the fit +params = { + "srt": 1, # Ocean classification + "len": 25, # 10-meter segments + "ats": 3, # require that each segment contain photons separated by at least 5 m + "res": 10, # return one photon every 10 m + "track": 0, # return all ground tracks + "pass_invalid": True, + "cnf": 2, # require classification confidence of 2 or more + #'iterations':10, # iterate the fit } # YAPC alternatibe -params_yapc={'srt': 1, - 'len': 20, - 'ats':3, - 'res':10, - 'track': 0, - 'pass_invalid': True, - 'cnf': -2, - 'iterations':10,} +params_yapc = { + "srt": 1, + "len": 20, + "ats": 3, + "res": 10, + "track": 0, + "pass_invalid": True, + "cnf": -2, + "iterations": 10, +} # # "yapc": dict(knn=0, win_h=6, win_x=11, min_ph=4, score=100), # use the YAPC photon classifier; these are the recommended parameters, but the results might be more specific with a smaller win_h value, or a higher score cutoff # "yapc": dict(knn=0, win_h=3, win_x=11, min_ph=4, score=50), # use the YAPC photon classifier; these are the recommended parameters, but the results might be more specific with a smaller win_h value, or a higher score cutoff # add domain -params['poly'] = poly['list'] -params['t0'] = '2019-05-01T00:00:00' -params['t1'] = '2019-05-20T00:00:00' -#params['t1'] = '2019-06-10T00:00:00' +params["poly"] = poly["list"] +params["t0"] = "2019-05-01T00:00:00" +params["t1"] = "2019-05-20T00:00:00" +# params['t1'] = '2019-06-10T00:00:00' # get granuale list -release = '005' -granules_list = icesat2.cmr(polygon=params['poly'] , time_start=params['t0'], time_end=params['t1'], version=release) +release = "005" +granules_list = icesat2.cmr( + polygon=params["poly"], + time_start=params["t0"], + time_end=params["t1"], + version=release, +) # %% download data from Sliderule gdf = icesat2.atl06p(params, asset="nsidc-s3", resources=granules_list) @@ -114,11 +126,12 @@ if plot_flag: # sample data gdf_sample = gdf.sample(n=12000, replace=False, random_state=1) - fig, axx = sct.plot_data_in_domain(gdf_sample, poly['list']) + fig, axx = sct.plot_data_in_domain(gdf_sample, poly["list"]) plt.show() # %% main routine for defining the x coordinate and sacing table data + def make_B01_dict(table_data, split_by_beam=True, to_hdf5=False): """ converts a GeoDataFrame from Sliderule to GeoDataFrames for each beam witht the correct columns and names @@ -131,61 +144,66 @@ def make_B01_dict(table_data, split_by_beam=True, to_hdf5=False): else: table_data: GeoDataFrame with the data for all beams in one table """ - #table_data = copy.copy(B01b_atl06_native) - - table_data.rename(columns= { - 'n_fit_photons':'N_photos', - 'w_surface_window_final':'signal_confidence', - 'across':'y', - #'spot':'beam', - } - , inplace=True) - - table_data['lons'] = table_data['geometry'].x - table_data['lats'] = table_data['geometry'].y - - drop_columns = ['cycle','gt','rgt', 'pflags'] + # table_data = copy.copy(B01b_atl06_native) + + table_data.rename( + columns={ + "n_fit_photons": "N_photos", + "w_surface_window_final": "signal_confidence", + "across": "y", + #'spot':'beam', + }, + inplace=True, + ) + + table_data["lons"] = table_data["geometry"].x + table_data["lats"] = table_data["geometry"].y + + drop_columns = ["cycle", "gt", "rgt", "pflags"] if to_hdf5: - drop_columns.append('geometry') + drop_columns.append("geometry") table_data.drop(columns=drop_columns, inplace=True) if split_by_beam: B01b = dict() # this is not tested - for spot,beam in zip([1,2,3, 4, 5, 6],['gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r']) : - ii = table_data.spot==spot + for spot, beam in zip( + [1, 2, 3, 4, 5, 6], ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] + ): + ii = table_data.spot == spot B01b[beam] = table_data[ii] return B01b else: return table_data + imp.reload(sct) -T= dict() +T = dict() for rgt in RGT_common: - tmp = gdf[gdf['rgt']==rgt] + tmp = gdf[gdf["rgt"] == rgt] # define reference point and then define 'x' table_data = copy.copy(tmp) - # the reference point is defined as the most equatorward point of the polygon. + # the reference point is defined as the most equatorward point of the polygon. # It's distance from the equator is subtracted from the distance of each photon. table_data = sct.define_x_coordinate_with_RGT(table_data, Gtrack_lowest) # fake 'across' - table_data['across'] = table_data['x']*0 +table_data['spot'] + table_data["across"] = table_data["x"] * 0 + table_data["spot"] # add spike remover # renames columns and splits beams - Ti = make_B01_dict(table_data, split_by_beam=True, to_hdf5=True) - #T[rgt] = - #Ti['gt1l'].drop('geometry', axis=1, inplace=True) + Ti = make_B01_dict(table_data, split_by_beam=True, to_hdf5=True) + # T[rgt] = + # Ti['gt1l'].drop('geometry', axis=1, inplace=True) ID_name = sct.create_ID_name(tmp.iloc[0]) - print( ID_name ) - io.write_track_to_HDF5(Ti, ID_name + '_B01_binned' , save_path) # regridding heights + print(ID_name) + io.write_track_to_HDF5(Ti, ID_name + "_B01_binned", save_path) # regridding heights -print('done') +print("done") # %% diff --git a/analysis_db/B01_filter_regrid_segments.py b/analysis_db/B01_filter_regrid_segments.py index 8df6a89e..9738d0d0 100644 --- a/analysis_db/B01_filter_regrid_segments.py +++ b/analysis_db/B01_filter_regrid_segments.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,7 +9,7 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) import datetime @@ -17,7 +18,10 @@ import imp import ICEsat2_SI_tools.convert_GPS_time as cGPS import ICEsat2_SI_tools.io as io -from spectral_estimates import create_chunk_boundaries_unit_lengths, create_chunk_boundaries +from spectral_estimates import ( + create_chunk_boundaries_unit_lengths, + create_chunk_boundaries, +) import spectral_estimates as spec import m_tools_ph3 as MT import filter_regrid as regrid @@ -27,207 +31,221 @@ import concurrent.futures as futures # memory test -#from guppy import hpy +# from guppy import hpy def get_size(x): from pympler import asizeof - ss = asizeof.asizeof(x)/1e6 + + ss = asizeof.asizeof(x) / 1e6 return str(ss) -#get_size(hemis) +# get_size(hemis) -#import s3fs -#processed_ATL03_20190605061807_10380310_004_01.h5 +# import s3fs +# processed_ATL03_20190605061807_10380310_004_01.h5 -#imp.reload(io) -ID_name, batch_key, ID_flag = io.init_from_input(sys.argv) # loads standard experiment -#ID_name, batch_key, ID_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#ID_name, batch_key, ID_flag = '20190207234532_06340210_004_01', 'SH_batch02', False -#ID_name, batch_key, ID_flag = '20190215184558_07530210_004_01', 'SH_batch02', False -#ID_name, batch_key, ID_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +# imp.reload(io) +ID_name, batch_key, ID_flag = io.init_from_input(sys.argv) # loads standard experiment +# ID_name, batch_key, ID_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# ID_name, batch_key, ID_flag = '20190207234532_06340210_004_01', 'SH_batch02', False +# ID_name, batch_key, ID_flag = '20190215184558_07530210_004_01', 'SH_batch02', False +# ID_name, batch_key, ID_flag = '20190219073735_08070210_004_01', 'SH_batch02', False -#ID_name, batch_key, ID_flag = '20190224023410_08800212_004_01', 'SH_batch02', False -#ID_name, batch_key, ID_flag = '20190101020504_00550212_005_01', 'SH_batch04', False +# ID_name, batch_key, ID_flag = '20190224023410_08800212_004_01', 'SH_batch02', False +# ID_name, batch_key, ID_flag = '20190101020504_00550212_005_01', 'SH_batch04', False # NH -#ID_name, batch_key, ID_flag = 'NH_20190301_09560205', 'NH_batch05', True # poleward false -#ID_name, batch_key, ID_flag = 'NH_20190301_09570203', 'NH_batch05', True # poleward false -#ID_name, batch_key, test_flag = 'NH_20190301_09580203', 'NH_batch05', True +# ID_name, batch_key, ID_flag = 'NH_20190301_09560205', 'NH_batch05', True # poleward false +# ID_name, batch_key, ID_flag = 'NH_20190301_09570203', 'NH_batch05', True # poleward false +# ID_name, batch_key, test_flag = 'NH_20190301_09580203', 'NH_batch05', True # SH -#ID_name, batch_key, ID_flag = 'SH_20190101_00550210', 'SH_batch04', True -#ID_name, batch_key, ID_flag = 'SH_20190101_00570212', 'SH_batch04', True +# ID_name, batch_key, ID_flag = 'SH_20190101_00550210', 'SH_batch04', True +# ID_name, batch_key, ID_flag = 'SH_20190101_00570212', 'SH_batch04', True # equatorward track -#ID_name, batch_key, ID_flag = '20190208154150_06440212_004_01', 'SH_batch02', False +# ID_name, batch_key, ID_flag = '20190208154150_06440212_004_01', 'SH_batch02', False # poleward track -#ID_name, batch_key, ID_flag = '20190209150245_06590210_004_01', 'SH_batch02', False -#conner +# ID_name, batch_key, ID_flag = '20190209150245_06590210_004_01', 'SH_batch02', False +# conner # for plotting -#rack_name, batch_key, ID_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +# rack_name, batch_key, ID_flag = '20190219073735_08070210_004_01', 'SH_batch02', False -ID, _, hemis, batch = io.init_data(ID_name, batch_key, ID_flag, mconfig['paths']['work'], ) +ID, _, hemis, batch = io.init_data( + ID_name, + batch_key, + ID_flag, + mconfig["paths"]["work"], +) -#ID_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' +# ID_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" -load_path = mconfig['paths']['scratch'] +'/'+ batch_key +'/' -load_file = 'A01c_'+ATlevel+'_'+ID_name -load_file_str = load_path + load_file+'.h5' +load_path = mconfig["paths"]["scratch"] + "/" + batch_key + "/" +load_file = "A01c_" + ATlevel + "_" + ID_name +load_file_str = load_path + load_file + ".h5" -save_path = mconfig['paths']['work'] +'/'+batch_key+'/B01_regrid/' +save_path = mconfig["paths"]["work"] + "/" + batch_key + "/B01_regrid/" -plot_path = mconfig['paths']['plot']+ '/'+hemis+'/'+batch_key+'/'+ID_name +'/B01/' -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +plot_path = ( + mconfig["paths"]["plot"] + "/" + hemis + "/" + batch_key + "/" + ID_name + "/B01/" +) +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" MT.mkdirs_r(save_path) # set pars # define parameters: -Lmeter = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 -Nphoton_min = 5 # mininum need photons per stancil to return results +Lmeter = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 +Nphoton_min = 5 # mininum need photons per stancil to return results -Lmeter_large= 100e3 # stancil width for testing photon density. stancils do not overlab. -minium_photon_density = 0.02 # minimum photon density per meter in Lmeter_large chunk to be counted as real signal +Lmeter_large = ( + 100e3 # stancil width for testing photon density. stancils do not overlab. +) +minium_photon_density = 0.02 # minimum photon density per meter in Lmeter_large chunk to be counted as real signal -plot_flag = True +plot_flag = True Nworkers_process = 6 # number of threads for parallel processing # outer loop # %% # test which beams exist: -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] # low_beams = mconfig['beams']['low_beams'] -if __name__ == '__main__': +if __name__ == "__main__": if ID_flag: - track_poleward = ID['pars']['poleward'] + track_poleward = ID["pars"]["poleward"] beams = all_beams - print('take poleward from ID file') + print("take poleward from ID file") else: - f = h5py.File(load_file_str, 'r') - beams = [b if b in f.keys() else None for b in all_beams] + f = h5py.File(load_file_str, "r") + beams = [b if b in f.keys() else None for b in all_beams] imp.reload(regrid) - track_poleward = regrid.track_pole_ward_file(f) - print('take poleward from ATL03 file') - print('poleward track is ' , track_poleward) + track_poleward = regrid.track_pole_ward_file(f) + print("take poleward from ATL03 file") + print("poleward track is ", track_poleward) # %% # open HDF5 file for all tracks # ATL03 = h5py.File(save_path_data + '/A01c_ATL03_'+ ID_name+'_2.h5', 'r') + def get_ATL03_beam(ATL03_k): - DD = pd.DataFrame()#columns = ATL03.keys()) + DD = pd.DataFrame() # columns = ATL03.keys()) for ikey in ATL03_k.keys(): DD[ikey] = ATL03_k[ikey] return DD -#for k in beams: + +# for k in beams: def load_data_and_cut(k): - #print(k) - #k =all_beams[1] + # print(k) + # k =all_beams[1] - Tsel = get_ATL03_beam(ATL03[k]) + Tsel = get_ATL03_beam(ATL03[k]) seg = get_ATL03_beam(ATL03_seg[k]) - #Tsel_c = get_ATL03_beam(ATL03_c[k]) + # Tsel_c = get_ATL03_beam(ATL03_c[k]) - #imp.reload(io) - #T, seg = io.getATL03_beam(load_file_str, beam= k) + # imp.reload(io) + # T, seg = io.getATL03_beam(load_file_str, beam= k) - print('loaded') - #T = T[T['mask_seaice']] # only take sea ice points, no ocean points - #T = T.drop(labels=[ 'year', 'month', 'day', 'hour', 'minute', 'second', 'ph_id_count', 'mask_seaice'], axis= 1) - #T = T.drop(labels=[ 'ph_id_count', 'mask_seaice'], axis= 1) - print( 'T MB ' + get_size(Tsel) ) + print("loaded") + # T = T[T['mask_seaice']] # only take sea ice points, no ocean points + # T = T.drop(labels=[ 'year', 'month', 'day', 'hour', 'minute', 'second', 'ph_id_count', 'mask_seaice'], axis= 1) + # T = T.drop(labels=[ 'ph_id_count', 'mask_seaice'], axis= 1) + print("T MB " + get_size(Tsel)) ho = k # ho = MT.add_line_var(ho, 'size', str(T.shape[0])) # ho = MT.add_line_var(ho, 'by confidence levels:' + str(np.arange(0, 5)), [ (T['signal_confidence'] == i).sum() for i in np.arange(0, 5) ]) # filter: - #Tsel = T[ (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] + # Tsel = T[ (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] # if len(Tsel) == 0: # ho = MT.add_line_var(ho, 'no photons found', '') # #Tsel= T[(T['signal_confidence'] ==-1 ) & (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] - #Tsel_c = io.getATL03_height_correction(load_file_str) + # Tsel_c = io.getATL03_height_correction(load_file_str) # Tsel_c = Tsel_c[Tsel_c['dem_h'] < 1e5] # cute out weird references # # needs only dem_h and heihgts # Tsel = regrid.correct_heights(Tsel, Tsel_c).reset_index(drop=True)# correct height - #print('height corrected') - + # print('height corrected') ### cut data at the rear that has too much variance # cut last segments of data until variance is similar - rear_mask = np.array(Tsel.index) > -1 # True + rear_mask = np.array(Tsel.index) > -1 # True nsize0 = Tsel.shape[0] - N_seg= 20 + N_seg = 20 cut_flag = True dd_old = -1 - dd0 = np.array(Tsel['heights_c']) - print('inital length' , nsize0) + dd0 = np.array(Tsel["heights_c"]) + print("inital length", nsize0) - @jit(nopython=True, parallel= False) + @jit(nopython=True, parallel=False) def adjust_length(var_list, rear_mask, cut_flag, track_poleward): var_list = var_list if track_poleward else var_list[::-1] - if var_list[0:3].mean()*10 < var_list[-1]: - #print('cut last '+ str(100/N_seg) +'% of data') - rear_mask[int(nsize* (N_seg-1) / N_seg):] = False + if var_list[0:3].mean() * 10 < var_list[-1]: + # print('cut last '+ str(100/N_seg) +'% of data') + rear_mask[int(nsize * (N_seg - 1) / N_seg) :] = False else: - cut_flag = False + cut_flag = False rear_mask = rear_mask if track_poleward else rear_mask[::-1] return rear_mask, cut_flag - #@jit(nopython=True, parallel= True) + # @jit(nopython=True, parallel= True) def get_var(sti): - return dd[sti[0]: sti[1]].var() - + return dd[sti[0] : sti[1]].var() while cut_flag: - dd= dd0[rear_mask] + dd = dd0[rear_mask] nsize = dd.size - print('new length', nsize) + print("new length", nsize) - stencil_iter = create_chunk_boundaries( int(nsize/N_seg), nsize,ov =0, iter_flag=True ) + stencil_iter = create_chunk_boundaries( + int(nsize / N_seg), nsize, ov=0, iter_flag=True + ) var_list = np.array(list(map(get_var, stencil_iter))) - #print(k, var_list) - rear_mask, cut_flag = adjust_length(var_list, rear_mask, cut_flag, track_poleward) + # print(k, var_list) + rear_mask, cut_flag = adjust_length( + var_list, rear_mask, cut_flag, track_poleward + ) if nsize == dd_old: - print('--- lengthen segments') - N_seg -=1 - #cut_flag = False + print("--- lengthen segments") + N_seg -= 1 + # cut_flag = False dd_old = nsize - print( 'Tsel MB ' + get_size(Tsel) ) + print("Tsel MB " + get_size(Tsel)) return k, rear_mask, Tsel, seg, ho -#load_data_and_cut(all_beams[1]) +# load_data_and_cut(all_beams[1]) # %% -if __name__ == '__main__': - ATL03 = h5py.File(load_path +'/'+load_file +'_corrected.h5', 'r') - ATL03_seg = h5py.File(load_path +'/'+load_file +'_seg.h5', 'r') +if __name__ == "__main__": + ATL03 = h5py.File(load_path + "/" + load_file + "_corrected.h5", "r") + ATL03_seg = h5py.File(load_path + "/" + load_file + "_seg.h5", "r") - hist = 'Beam stats' - B = dict() - #B1save = dict() - SEG = dict() + hist = "Beam stats" + B = dict() + # B1save = dict() + SEG = dict() k = beams[0] # A = list() @@ -235,35 +253,40 @@ def get_var(sti): # A.append(load_data_and_cut(k)) with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: - A = list( executor.map(load_data_and_cut, all_beams) ) + A = list(executor.map(load_data_and_cut, all_beams)) - print( 'A MB ' + get_size(A) ) + print("A MB " + get_size(A)) ATL03.close() ATL03_seg.close() - #ATL03_c.close() - - for I in A: # collect returns from from mapping - k, rear_mask, Tsel, seg, ho = I - - Tsel['process_mask'] = rear_mask - #B1save[k] = Tsel - B[k] = Tsel[rear_mask].drop(columns='process_mask') - SEG[k] = seg - - ho = MT.add_line_var(ho, 'cutted ', str( np.round( 100 *(rear_mask.size -rear_mask.sum() ) / rear_mask.size, 0)) + '% in the back of the track' ) - ho = MT.add_line_var(ho, 'selected size', str(Tsel.shape[0])) - #ho = MT.add_line_var(ho, 'final size ', str(Tsel_c.shape[0])) + # ATL03_c.close() + + for I in A: # collect returns from from mapping + k, rear_mask, Tsel, seg, ho = I + + Tsel["process_mask"] = rear_mask + # B1save[k] = Tsel + B[k] = Tsel[rear_mask].drop(columns="process_mask") + SEG[k] = seg + + ho = MT.add_line_var( + ho, + "cutted ", + str(np.round(100 * (rear_mask.size - rear_mask.sum()) / rear_mask.size, 0)) + + "% in the back of the track", + ) + ho = MT.add_line_var(ho, "selected size", str(Tsel.shape[0])) + # ho = MT.add_line_var(ho, 'final size ', str(Tsel_c.shape[0])) print(ho) - hist = MT.write_log(hist, ho) + hist = MT.write_log(hist, ho) - print('done with 1st loop') + print("done with 1st loop") del Tsel del A - #print( 'B_save MB ' + get_size(B1save) ) - print( 'B MB ' + get_size(B) ) - print( 'SEG MB ' + get_size(SEG) ) + # print( 'B_save MB ' + get_size(B1save) ) + print("B MB " + get_size(B)) + print("SEG MB " + get_size(SEG)) # %% define x- coodindate @@ -271,122 +294,143 @@ def get_var(sti): # this is on the equatorward side for poleward track, or # on the poleward side for equatorward tracks. - total_segment_dist_x_min= list() - for k,I in SEG.items(): - total_segment_dist_x_min.append( I['segment_dist_x'].min() ) + total_segment_dist_x_min = list() + for k, I in SEG.items(): + total_segment_dist_x_min.append(I["segment_dist_x"].min()) total_segment_dist_x_min = min(total_segment_dist_x_min) + # %% def make_x_coorindate(k): """ Returns the "true" along track coordindate but finding the correpsonding segment length also adds the segment_ID to the main table T """ - print(k, ' make coodindate') - T, seg = B[k], SEG[k] + print(k, " make coodindate") + T, seg = B[k], SEG[k] # make sure data is strictly ordered by delta_time - T = T.sort_values('delta_time').reset_index(drop=True) - seg = seg.sort_values('delta_time').reset_index(drop=True) + T = T.sort_values("delta_time").reset_index(drop=True) + seg = seg.sort_values("delta_time").reset_index(drop=True) # find positions where segmetn length is reset # shifts segment length postions in time - delta_onehalf = seg['delta_time'].diff()/2 - delta_onehalf.iloc[0] = delta_onehalf.iloc[1] - seg['delta_half_time'] = seg['delta_time'] - delta_onehalf - 1e-5 + delta_onehalf = seg["delta_time"].diff() / 2 + delta_onehalf.iloc[0] = delta_onehalf.iloc[1] + seg["delta_half_time"] = seg["delta_time"] - delta_onehalf - 1e-5 # cur phontos that are not in segmentns - T = T[ (T['delta_time'] > seg['delta_half_time'].iloc[0]) & (T['delta_time'] < seg['delta_half_time'].iloc[-1])] - bin_labels = np.digitize( T['delta_time'], seg['delta_half_time'], right = True ) + T = T[ + (T["delta_time"] > seg["delta_half_time"].iloc[0]) + & (T["delta_time"] < seg["delta_half_time"].iloc[-1]) + ] + bin_labels = np.digitize(T["delta_time"], seg["delta_half_time"], right=True) # select relevant data - SS = seg['segment_dist_x'] - SS_sid = seg['segment_id'] + SS = seg["segment_dist_x"] + SS_sid = seg["segment_id"] - repeats = np.bincount(bin_labels, minlength =SS.shape[0]) + repeats = np.bincount(bin_labels, minlength=SS.shape[0]) # check if repeats sumup if repeats.sum() != T.shape[0]: - print('repeats do not sum up') + print("repeats do not sum up") # repeat segment dat accoridng to photons - SS = SS.repeat(repeats) + SS = SS.repeat(repeats) SS.index = T.index - SS_sid = SS_sid.repeat(repeats) + SS_sid = SS_sid.repeat(repeats) SS_sid.index = T.index # define new coordinate - T['x'] = SS + T['along_track_distance'] - T['segment_id'] = SS_sid + T["x"] = SS + T["along_track_distance"] + T["segment_id"] = SS_sid # find bad photons def find_anomalie_photons(Ti2, segi): - x_interp = np.interp(Ti2['delta_time'], segi['delta_time'], segi['segment_dist_x'] ) + x_interp = np.interp( + Ti2["delta_time"], segi["delta_time"], segi["segment_dist_x"] + ) - diff_x = x_interp - Ti2['x'] - diff_x = abs(diff_x-diff_x.mean()) - return diff_x > 3 *diff_x.std() , x_interp + diff_x = x_interp - Ti2["x"] + diff_x = abs(diff_x - diff_x.mean()) + return diff_x > 3 * diff_x.std(), x_interp fail_mask, x_interp = find_anomalie_photons(T, seg) - print('weird photon fraction:' , sum(fail_mask)/ fail_mask.size) + print("weird photon fraction:", sum(fail_mask) / fail_mask.size) # aply fail mask - #T= T[~fail_mask] - T['x'][fail_mask] = x_interp[fail_mask] + # T= T[~fail_mask] + T["x"][fail_mask] = x_interp[fail_mask] return k, T -if __name__ == '__main__': + +if __name__ == "__main__": with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: - A = list( executor.map(make_x_coorindate, all_beams) ) - - B= dict() - for I in A: # collect returns from from mapping - k = I[0] - B[ k ] = I[1][::-1] - - if not track_poleward: # invert x- coordinate if there is an equatorward track - print('invert table') - B[k] = B[k].reset_index(drop=True) - B[k]['x_true'] = B[k]['x'] - B[k]['x'] = abs(B[k]['x'] - B[k]['x'].iloc[0]) + A = list(executor.map(make_x_coorindate, all_beams)) + + B = dict() + for I in A: # collect returns from from mapping + k = I[0] + B[k] = I[1][::-1] + + if not track_poleward: # invert x- coordinate if there is an equatorward track + print("invert table") + B[k] = B[k].reset_index(drop=True) + B[k]["x_true"] = B[k]["x"] + B[k]["x"] = abs(B[k]["x"] - B[k]["x"].iloc[0]) else: - B[k]['x_true'] = B[k]['x'] - print('no table invert needed') + B[k]["x_true"] = B[k]["x"] + print("no table invert needed") - dist_list = np.array([np.nan, np.nan]) + dist_list = np.array([np.nan, np.nan]) for k in B.keys(): - dist_list = np.vstack([ dist_list, [ B[k]['x'].iloc[0] , B[k]['x'].iloc[-1] ] ]) + dist_list = np.vstack([dist_list, [B[k]["x"].iloc[0], B[k]["x"].iloc[-1]]]) - print( 'B MB ' + get_size(B) ) + print("B MB " + get_size(B)) del A del SEG - #del T + # del T # test ave photon density abnd quit if necessary - track_dist_bounds = [ np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0) ] - length_meter = abs(track_dist_bounds[1] - track_dist_bounds[0]) - #length_meter = (abs(lat_lims[1]) - abs(lat_lims[0])) * 110e3 - p_densities_r = list() - p_densities_l = list() + track_dist_bounds = [np.nanmin(dist_list[:, 0], 0), np.nanmax(dist_list[:, 1], 0)] + length_meter = abs(track_dist_bounds[1] - track_dist_bounds[0]) + # length_meter = (abs(lat_lims[1]) - abs(lat_lims[0])) * 110e3 + p_densities_r = list() + p_densities_l = list() for k, I in B.items(): - if 'r' in k: - p_densities_r.append( I.shape[0] /length_meter) + if "r" in k: + p_densities_r.append(I.shape[0] / length_meter) else: - p_densities_l.append( I.shape[0] /length_meter) - - if (np.array(p_densities_l).mean() < 0.5) & (np.array(p_densities_r).mean() < 0.5): # in photons per meter - print('photon density too low, this track is classified as bad track and pushed to bad list') - MT.json_save(ID_name, bad_track_path, {'densities': [ np.array(p_densities_r).mean(), np.array(p_densities_l).mean()] , 'date': str(datetime.date.today()) }) - print('exit.') + p_densities_l.append(I.shape[0] / length_meter) + + if (np.array(p_densities_l).mean() < 0.5) & ( + np.array(p_densities_r).mean() < 0.5 + ): # in photons per meter + print( + "photon density too low, this track is classified as bad track and pushed to bad list" + ) + MT.json_save( + ID_name, + bad_track_path, + { + "densities": [ + np.array(p_densities_r).mean(), + np.array(p_densities_l).mean(), + ], + "date": str(datetime.date.today()), + }, + ) + print("exit.") exit() # %% save corrected data and delete from cash -#io.save_pandas_table(B1save, ID_name + '_B01_corrected' , save_path) # all photos but heights adjusted and with -#io.save_pandas_table(B, ID_name + '_B01_new_coords' , save_path) # all photos but heights adjusted and with distance coordinate -#del B1save +# io.save_pandas_table(B1save, ID_name + '_B01_corrected' , save_path) # all photos but heights adjusted and with +# io.save_pandas_table(B, ID_name + '_B01_new_coords' , save_path) # all photos but heights adjusted and with distance coordinate +# del B1save # for testing # T2 = B['gt1r'] @@ -402,248 +446,336 @@ def find_anomalie_photons(Ti2, segi): # %% -if __name__ == '__main__': - F = M.figure_axis_xy(4, 3, view_scale = 0.7) +if __name__ == "__main__": + F = M.figure_axis_xy(4, 3, view_scale=0.7) - for k,I in B.items(): - plt.plot( I['lats'] , I['x'] , '.' , markersize = 0.2) - #plt.xlim(3e6, 3.25e6) - plt.xlabel('lats') - plt.ylabel('x') - F.save_light(path= plot_path, name='B01_ALT03_'+ID_name+'_tracks_check_lat_x') + for k, I in B.items(): + plt.plot(I["lats"], I["x"], ".", markersize=0.2) + # plt.xlim(3e6, 3.25e6) + plt.xlabel("lats") + plt.ylabel("x") + F.save_light(path=plot_path, name="B01_ALT03_" + ID_name + "_tracks_check_lat_x") # % - F = M.figure_axis_xy(4, 3, view_scale = 0.7) - for k,I in B.items(): - plt.plot( I['delta_time'] , I['lats'], '.' , markersize = 0.3) + F = M.figure_axis_xy(4, 3, view_scale=0.7) + for k, I in B.items(): + plt.plot(I["delta_time"], I["lats"], ".", markersize=0.3) - plt.xlabel('delta time') - plt.ylabel('lat') - F.save_light(path= plot_path, name='B01_ALT03_'+ID_name+'_tracks_check_time_lat') + plt.xlabel("delta time") + plt.ylabel("lat") + F.save_light(path=plot_path, name="B01_ALT03_" + ID_name + "_tracks_check_time_lat") - F = M.figure_axis_xy(4, 3, view_scale = 0.7) - for k,I in B.items(): - plt.plot( I['delta_time'] , I['x'], '.' , markersize = 0.3) + F = M.figure_axis_xy(4, 3, view_scale=0.7) + for k, I in B.items(): + plt.plot(I["delta_time"], I["x"], ".", markersize=0.3) - plt.xlabel('delta time') - plt.ylabel('x') + plt.xlabel("delta time") + plt.ylabel("x") - F.save_light(path= plot_path, name='B01_ALT03_'+ID_name+'_tracks_check_time_x') + F.save_light(path=plot_path, name="B01_ALT03_" + ID_name + "_tracks_check_time_x") # %% ##### 1.) derive common axis for beams and filter out low density area at the beginning -print('filter out low density area at the beginning') +print("filter out low density area at the beginning") + def derive_axis_and_boundaries(key): - #key = 'gt3r' + # key = 'gt3r' print(key) - #T2 # = regrid.derive_axis(B[key], lat_lims).sort_values('dist') - T2 = B[key]#['x'] + # T2 # = regrid.derive_axis(B[key], lat_lims).sort_values('dist') + T2 = B[key] # ['x'] - x0 = get_better_lower_boundary(Lmeter_large, np.array(T2['x'])) + x0 = get_better_lower_boundary(Lmeter_large, np.array(T2["x"])) - print( 'cut off ' , 100 * (1 - T2[T2['x'] > x0].shape[0]/T2.shape[0]) , '% off all data points at the beginning' ) - T2 = T2[T2['x'] >x0] # cut off low density start + print( + "cut off ", + 100 * (1 - T2[T2["x"] > x0].shape[0] / T2.shape[0]), + "% off all data points at the beginning", + ) + T2 = T2[T2["x"] > x0] # cut off low density start + + return key, T2, [T2["x"].min(), T2["x"].max()] - return key, T2, [T2['x'].min(), T2['x'].max()] def get_better_lower_boundary(Lmeter_large, dd): - #T2 = regrid.derive_axis(B[key], lat_lims) - #dd = np.array(B['gt1l']['x']) + # T2 = regrid.derive_axis(B[key], lat_lims) + # dd = np.array(B['gt1l']['x']) - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=False + ) if stencil_iter.shape[1] == 0: while stencil_iter.shape[1] == 0: - Lmeter_large = int(Lmeter_large/2) - print( 'new Lmeter_large' + str(Lmeter_large)) - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) + Lmeter_large = int(Lmeter_large / 2) + print("new Lmeter_large" + str(Lmeter_large)) + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=False + ) - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= True) + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=True + ) def get_density(sti): - return sti[0], sum( (sti[0] <= dd) & (dd < sti[-1]) ) / Lmeter_large + return sti[0], sum((sti[0] <= dd) & (dd < sti[-1])) / Lmeter_large - with futures.ThreadPoolExecutor(max_workers= Nworkers_process) as executor_sub: - #var_list = np.array(list(map(get_density, stencil_iter))) + with futures.ThreadPoolExecutor(max_workers=Nworkers_process) as executor_sub: + # var_list = np.array(list(map(get_density, stencil_iter))) var_list = np.array(list(executor_sub.map(get_density, stencil_iter))) var_list = np.array(var_list) - #var_list[:,0] = np.random.rand(10) - #sort var_list + # var_list[:,0] = np.random.rand(10) + # sort var_list print(var_list) - var_list = var_list[var_list[:,0].argsort(), :] - #print(var_list) - if sum(var_list[:,1] > minium_photon_density) > 1: - first_stencil = next((i for i, j in enumerate(var_list[:,1] > minium_photon_density) if j), None) #- 1 - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) + var_list = var_list[var_list[:, 0].argsort(), :] + # print(var_list) + if sum(var_list[:, 1] > minium_photon_density) > 1: + first_stencil = next( + (i for i, j in enumerate(var_list[:, 1] > minium_photon_density) if j), None + ) # - 1 + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=False + ) return stencil_iter[0, first_stencil] else: - #first_stencil = next((i for i, j in enumerate(var_list[:,1] > 0) if j), None)# - 1 - print('no sufficient photon density found. return short stencil') + # first_stencil = next((i for i, j in enumerate(var_list[:,1] > 0) if j), None)# - 1 + print("no sufficient photon density found. return short stencil") # first_stencil= len(var_list[:,1]) -1 # stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) # return stencil_iter[0, first_stencil] # #print(first_stencil) - return var_list[-1,0]#[-1,0] + return var_list[-1, 0] # [-1,0] + -if __name__ == '__main__': +if __name__ == "__main__": with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: - A = list( executor.map(derive_axis_and_boundaries, all_beams) ) + A = list(executor.map(derive_axis_and_boundaries, all_beams)) # for k in all_beams: # A = derive_axis_and_boundaries(k) - B2 = dict() - dist_list = np.array([np.nan, np.nan]) + B2 = dict() + dist_list = np.array([np.nan, np.nan]) for I in A: - k = I[0] - B2[k] = I[1] - #B2[k]['dist'] = B2[k]['x'] - dist_list = np.vstack([dist_list,I[2] ]) + k = I[0] + B2[k] = I[1] + # B2[k]['dist'] = B2[k]['x'] + dist_list = np.vstack([dist_list, I[2]]) del A del B - track_dist_bounds = [ np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0) ] + track_dist_bounds = [np.nanmin(dist_list[:, 0], 0), np.nanmax(dist_list[:, 1], 0)] - print( 'B2 MB ' + get_size(B2) ) + print("B2 MB " + get_size(B2)) # %% -if __name__ == '__main__': - xscale= 1e3 - F= M.figure_axis_xy(5, 3, view_scale= 0.6) - for k,I in B2.items(): - plt.plot( I['x']/xscale , I['across_track_distance']/xscale , '.' , markersize = 0.3) - #plt.xlim(3e6, 3.25e6) +if __name__ == "__main__": + xscale = 1e3 + F = M.figure_axis_xy(5, 3, view_scale=0.6) + for k, I in B2.items(): + plt.plot( + I["x"] / xscale, I["across_track_distance"] / xscale, ".", markersize=0.3 + ) + # plt.xlim(3e6, 3.25e6) for k in high_beams: Ii = B2[k].iloc[0] - plt.text(Ii.x/xscale+ 5, Ii.across_track_distance/xscale , str(Ii[[ 'lats', 'lons'] ]).split('Name')[0] ) + plt.text( + Ii.x / xscale + 5, + Ii.across_track_distance / xscale, + str(Ii[["lats", "lons"]]).split("Name")[0], + ) Ii = B2[k].iloc[-1] - plt.text(Ii.x/xscale+ 5, Ii.across_track_distance/xscale , str(Ii[[ 'lats', 'lons'] ]).split('Name')[0], ha ='right' ) - - F.ax.axvline(track_dist_bounds[0]/xscale, color='gray', zorder= 2) - F.ax.axvline(track_dist_bounds[1]/xscale, color='gray', zorder= 2) - F.ax.axhline(0, color='gray', zorder= 2) - - plt.title('B01 filter and regrid | ' + ID_name +'\npoleward '+str(track_poleward)+' \n \n', loc='left') - plt.xlabel('along track distance (km)') - plt.ylabel('across track distance (km)') - - F.save_light(path= plot_path +'../', name='B01_ALT03_'+ID_name+'_tracks_all') + plt.text( + Ii.x / xscale + 5, + Ii.across_track_distance / xscale, + str(Ii[["lats", "lons"]]).split("Name")[0], + ha="right", + ) + + F.ax.axvline(track_dist_bounds[0] / xscale, color="gray", zorder=2) + F.ax.axvline(track_dist_bounds[1] / xscale, color="gray", zorder=2) + F.ax.axhline(0, color="gray", zorder=2) + + plt.title( + "B01 filter and regrid | " + + ID_name + + "\npoleward " + + str(track_poleward) + + " \n \n", + loc="left", + ) + plt.xlabel("along track distance (km)") + plt.ylabel("across track distance (km)") + + F.save_light(path=plot_path + "../", name="B01_ALT03_" + ID_name + "_tracks_all") # %% # for testing -#track_dist_bounds[1] = track_dist_bounds[0] + (track_dist_bounds[1] - track_dist_bounds[0])/20 -#track_dist_bounds = ts_s[0] ,ts_s[0] + (ts_s[1] - ts_s[0]) /6 +# track_dist_bounds[1] = track_dist_bounds[0] + (track_dist_bounds[1] - track_dist_bounds[0])/20 +# track_dist_bounds = ts_s[0] ,ts_s[0] + (ts_s[1] - ts_s[0]) /6 # %% ##### 2.) regridding and averaging -print('regrid') -def regridding_wrapper(I): - key, Ti = I - print(key, Ti.shape,2* Ti.shape[0]/Lmeter) - stencil_iter = create_chunk_boundaries_unit_lengths( Lmeter, track_dist_bounds, iter_flag=False ) - Bi = regrid.get_stencil_stats_shift( Ti, stencil_iter, 'heights_c', 'x' , stancil_width= Lmeter/2, Nphoton_min=Nphoton_min) +print("regrid") - #print( 'Bi MB ' + get_size(Bi) ) - print(key, 'done') + +def regridding_wrapper(I): + key, Ti = I + print(key, Ti.shape, 2 * Ti.shape[0] / Lmeter) + stencil_iter = create_chunk_boundaries_unit_lengths( + Lmeter, track_dist_bounds, iter_flag=False + ) + Bi = regrid.get_stencil_stats_shift( + Ti, + stencil_iter, + "heights_c", + "x", + stancil_width=Lmeter / 2, + Nphoton_min=Nphoton_min, + ) + + # print( 'Bi MB ' + get_size(Bi) ) + print(key, "done") return key, Bi -if __name__ == '__main__': + +if __name__ == "__main__": # ---define start and end position and same in Json file - with futures.ProcessPoolExecutor(max_workers = Nworkers_process) as executor: - B3 = dict( executor.map( regridding_wrapper, B2.items() ) ) + with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: + B3 = dict(executor.map(regridding_wrapper, B2.items())) - print( 'B3 MB ' + get_size(B3) ) - print( 'I MB ' + get_size(I) ) + print("B3 MB " + get_size(B3)) + print("I MB " + get_size(I)) - #I = B3['gt2l'].copy() + # I = B3['gt2l'].copy() D_info = dict() - for k,I in B3.items(): + for k, I in B3.items(): # reset x coordinate - I['median_dist'] = I['x_median'] - track_dist_bounds[0] #- Lmeter/2 - I['dist'] = I['x'] - track_dist_bounds[0] #- Lmeter/2 - #I['index'] = I['x'] + I["median_dist"] = I["x_median"] - track_dist_bounds[0] # - Lmeter/2 + I["dist"] = I["x"] - track_dist_bounds[0] # - Lmeter/2 + # I['index'] = I['x'] # rename y coordinate - I = I.rename(columns={'across_track_distance': 'y'}) + I = I.rename(columns={"across_track_distance": "y"}) # find starting and end position - Di_s = dict(I[I['segment_id'] == I['segment_id'].iloc[0] ].mean()[['lons', 'lats', 'segment_id', 'delta_time']]) - Di_s['across_track_distance_0'] =track_dist_bounds[0] - - Di_e = dict(I[I['segment_id'] == I['segment_id'].iloc[-1] ].mean()[['lons', 'lats', 'segment_id', 'delta_time']]) - Di_e['across_track_distance_0'] =track_dist_bounds[0] - - D_info[k] = {'start':Di_s, 'end':Di_e , 'poleward': str(track_poleward) } + Di_s = dict( + I[I["segment_id"] == I["segment_id"].iloc[0]].mean()[ + ["lons", "lats", "segment_id", "delta_time"] + ] + ) + Di_s["across_track_distance_0"] = track_dist_bounds[0] + + Di_e = dict( + I[I["segment_id"] == I["segment_id"].iloc[-1]].mean()[ + ["lons", "lats", "segment_id", "delta_time"] + ] + ) + Di_e["across_track_distance_0"] = track_dist_bounds[0] + + D_info[k] = {"start": Di_s, "end": Di_e, "poleward": str(track_poleward)} # reorder indexes - column_names = ['x', 'y', 'x_median', 'median_dist', 'lons', 'lats' ,'heights_c_weighted_mean', 'heights_c_median', 'heights_c_std', 'N_photos', ] - vars_ad = set(list(I[I['segment_id'] == I['segment_id'].iloc[0] ].mean().index)) - set(column_names) - I = I.reindex(columns=column_names + list(vars_ad)) + column_names = [ + "x", + "y", + "x_median", + "median_dist", + "lons", + "lats", + "heights_c_weighted_mean", + "heights_c_median", + "heights_c_std", + "N_photos", + ] + vars_ad = set( + list(I[I["segment_id"] == I["segment_id"].iloc[0]].mean().index) + ) - set(column_names) + I = I.reindex(columns=column_names + list(vars_ad)) B3[k] = I - # save Json - MT.json_save(ID_name + '_B01_stats',save_path, D_info, verbose= True ) + MT.json_save(ID_name + "_B01_stats", save_path, D_info, verbose=True) # saving data # io.save_pandas_table(B2, ID_name + '_B01_regridded' , save_path) # all photos but heights adjusted and with distance coordinate # io.save_pandas_table(B3, ID_name + '_B01_binned' , save_path) # regridding heights - io.write_track_to_HDF5(B2, ID_name + '_B01_regridded' , save_path) # all photos but heights adjusted and with distance coordinate - io.write_track_to_HDF5(B3, ID_name + '_B01_binned' , save_path) # regridding heights + io.write_track_to_HDF5( + B2, ID_name + "_B01_regridded", save_path + ) # all photos but heights adjusted and with distance coordinate + io.write_track_to_HDF5(B3, ID_name + "_B01_binned", save_path) # regridding heights # %% plotting just for checking -if __name__ == '__main__': +if __name__ == "__main__": - key = 'gt2r' + key = "gt2r" if plot_flag: MT.mkdirs_r(plot_path) Ti2 = B3[key] - T2 = B2[key] + T2 = B2[key] dl = 25000 - x_key= 'x' + x_key = "x" - latlims = (Ti2['x'].iloc[0] , Ti2['x'].iloc[-1] ) - #chunk_list = np.arange(latlims[0],latlims[1], dl ) - #chunk_list = sample( list(np.arange(latlims[0],latlims[1],dl )[0:80]) ,10) - chunk_list = np.arange(latlims[0],latlims[1], dl )[::10] - chunk_list = np.append( chunk_list, latlims[1]- dl-1) + latlims = (Ti2["x"].iloc[0], Ti2["x"].iloc[-1]) + # chunk_list = np.arange(latlims[0],latlims[1], dl ) + # chunk_list = sample( list(np.arange(latlims[0],latlims[1],dl )[0:80]) ,10) + chunk_list = np.arange(latlims[0], latlims[1], dl)[::10] + chunk_list = np.append(chunk_list, latlims[1] - dl - 1) for ll in chunk_list: F = M.figure_axis_xy(7, 3, view_scale=0.8) - plt.plot( T2[x_key], T2['heights_c'], 'k.', markersize= 0.5, alpha =0.8 ) - #plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') - - plt.plot(Ti2[x_key], Ti2['heights_c_weighted_mean'] -0.5, '.-', color='blue', linewidth=0.5, markersize=2,alpha=0.9, label='x-gauss weighted mean +1') - plt.plot(Ti2[x_key], Ti2['heights_c_median'] +0.5, 'r.-', linewidth=0.5, markersize=2,alpha=0.9, label='median') - - #plt.plot(Ti2['x'], Ti2['heights_c_mode']-1, 'g.-', linewidth=0.5, markersize=2,alpha=0.9, label='mode - 1') - - plt.plot(Ti2['x'], Ti2['heights_c_std'] - 1.8, 'k-', linewidth=0.5,alpha=1) - plt.fill_between( Ti2['x'], Ti2['heights_c_std'] -1.8 , y2=-1.8, color='gray',alpha=1) + plt.plot(T2[x_key], T2["heights_c"], "k.", markersize=0.5, alpha=0.8) + # plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') + + plt.plot( + Ti2[x_key], + Ti2["heights_c_weighted_mean"] - 0.5, + ".-", + color="blue", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="x-gauss weighted mean +1", + ) + plt.plot( + Ti2[x_key], + Ti2["heights_c_median"] + 0.5, + "r.-", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="median", + ) + + # plt.plot(Ti2['x'], Ti2['heights_c_mode']-1, 'g.-', linewidth=0.5, markersize=2,alpha=0.9, label='mode - 1') + + plt.plot(Ti2["x"], Ti2["heights_c_std"] - 1.8, "k-", linewidth=0.5, alpha=1) + plt.fill_between( + Ti2["x"], Ti2["heights_c_std"] - 1.8, y2=-1.8, color="gray", alpha=1 + ) # plt.plot( ALT03['delta_time'], ALT03['heights_c'], 'k.', markersize= 0.3, alpha =0.2 ) # plt.plot(ALT07['time']['delta_time'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') plt.legend(loc=1) - plt.xlim(ll, ll+dl) + plt.xlim(ll, ll + dl) plt.ylim(-4, 4) - plt.xlabel('Meters from the Sea Ice Edge') - plt.ylabel('Height Anomalie (meters)') - F.ax.axhline(y =-1.8, color='black', linewidth=0.5) - F.save_light(path= plot_path, name='ALT03_filt_compare'+ str(ll)) + plt.xlabel("Meters from the Sea Ice Edge") + plt.ylabel("Height Anomalie (meters)") + F.ax.axhline(y=-1.8, color="black", linewidth=0.5) + F.save_light(path=plot_path, name="ALT03_filt_compare" + str(ll)) diff --git a/analysis_db/B01_filter_regrid_segments_tester2.py b/analysis_db/B01_filter_regrid_segments_tester2.py index 4001df63..f2e937c7 100644 --- a/analysis_db/B01_filter_regrid_segments_tester2.py +++ b/analysis_db/B01_filter_regrid_segments_tester2.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,7 +9,7 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) import datetime @@ -17,7 +18,10 @@ import imp import ICEsat2_SI_tools.convert_GPS_time as cGPS import ICEsat2_SI_tools.io as io -from spectral_estimates import create_chunk_boundaries_unit_lengths, create_chunk_boundaries +from spectral_estimates import ( + create_chunk_boundaries_unit_lengths, + create_chunk_boundaries, +) import spectral_estimates as spec import m_tools_ph3 as MT import filter_regrid as regrid @@ -27,207 +31,221 @@ import concurrent.futures as futures # memory test -#from guppy import hpy +# from guppy import hpy def get_size(x): from pympler import asizeof - ss = asizeof.asizeof(x)/1e6 + + ss = asizeof.asizeof(x) / 1e6 return str(ss) -#get_size(hemis) +# get_size(hemis) -#import s3fs -#processed_ATL03_20190605061807_10380310_004_01.h5 +# import s3fs +# processed_ATL03_20190605061807_10380310_004_01.h5 -#imp.reload(io) -ID_name, batch_key, ID_flag = io.init_from_input(sys.argv) # loads standard experiment -#ID_name, batch_key, ID_flag = '20190605061807_10380310_004_01', 'SH_batch01', False -#ID_name, batch_key, ID_flag = '20190207234532_06340210_004_01', 'SH_batch02', False -#ID_name, batch_key, ID_flag = '20190215184558_07530210_004_01', 'SH_batch02', False -#ID_name, batch_key, ID_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +# imp.reload(io) +ID_name, batch_key, ID_flag = io.init_from_input(sys.argv) # loads standard experiment +# ID_name, batch_key, ID_flag = '20190605061807_10380310_004_01', 'SH_batch01', False +# ID_name, batch_key, ID_flag = '20190207234532_06340210_004_01', 'SH_batch02', False +# ID_name, batch_key, ID_flag = '20190215184558_07530210_004_01', 'SH_batch02', False +# ID_name, batch_key, ID_flag = '20190219073735_08070210_004_01', 'SH_batch02', False -#ID_name, batch_key, ID_flag = '20190224023410_08800212_004_01', 'SH_batch02', False -#ID_name, batch_key, ID_flag = '20190101020504_00550212_005_01', 'SH_batch04', False +# ID_name, batch_key, ID_flag = '20190224023410_08800212_004_01', 'SH_batch02', False +# ID_name, batch_key, ID_flag = '20190101020504_00550212_005_01', 'SH_batch04', False # NH -#ID_name, batch_key, ID_flag = 'NH_20190301_09560205', 'NH_batch05', True # poleward false -#ID_name, batch_key, ID_flag = 'NH_20190301_09570203', 'NH_batch05', True # poleward false -#ID_name, batch_key, test_flag = 'NH_20190301_09580203', 'NH_batch05', True +# ID_name, batch_key, ID_flag = 'NH_20190301_09560205', 'NH_batch05', True # poleward false +# ID_name, batch_key, ID_flag = 'NH_20190301_09570203', 'NH_batch05', True # poleward false +# ID_name, batch_key, test_flag = 'NH_20190301_09580203', 'NH_batch05', True # SH -#ID_name, batch_key, ID_flag = 'SH_20190101_00550210', 'SH_batch04', True -ID_name, batch_key, ID_flag = 'SH_20190224_08800210', 'SH_publish', True +# ID_name, batch_key, ID_flag = 'SH_20190101_00550210', 'SH_batch04', True +ID_name, batch_key, ID_flag = "SH_20190224_08800210", "SH_publish", True # equatorward track -#ID_name, batch_key, ID_flag = '20190208154150_06440212_004_01', 'SH_batch02', False +# ID_name, batch_key, ID_flag = '20190208154150_06440212_004_01', 'SH_batch02', False # poleward track -#ID_name, batch_key, ID_flag = '20190209150245_06590210_004_01', 'SH_batch02', False -#conner +# ID_name, batch_key, ID_flag = '20190209150245_06590210_004_01', 'SH_batch02', False +# conner # for plotting -#rack_name, batch_key, ID_flag = '20190219073735_08070210_004_01', 'SH_batch02', False +# rack_name, batch_key, ID_flag = '20190219073735_08070210_004_01', 'SH_batch02', False -ID, _, hemis, batch = io.init_data(ID_name, batch_key, ID_flag, mconfig['paths']['work'], ) +ID, _, hemis, batch = io.init_data( + ID_name, + batch_key, + ID_flag, + mconfig["paths"]["work"], +) -#ID_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL03' +# ID_name= '20190605061807_10380310_004_01' +ATlevel = "ATL03" -load_path = mconfig['paths']['scratch'] +'/'+ batch_key +'/' -load_file = 'A01c_'+ATlevel+'_'+ID_name -load_file_str = load_path + load_file+'.h5' +load_path = mconfig["paths"]["scratch"] + "/" + batch_key + "/" +load_file = "A01c_" + ATlevel + "_" + ID_name +load_file_str = load_path + load_file + ".h5" -save_path = mconfig['paths']['work'] +'/'+batch_key+'/B01_regrid/' +save_path = mconfig["paths"]["work"] + "/" + batch_key + "/B01_regrid/" -plot_path = mconfig['paths']['plot']+ '/'+hemis+'/'+batch_key+'/'+ID_name +'/B01/' -bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +plot_path = ( + mconfig["paths"]["plot"] + "/" + hemis + "/" + batch_key + "/" + ID_name + "/B01/" +) +bad_track_path = mconfig["paths"]["work"] + "bad_tracks/" + batch_key + "/" MT.mkdirs_r(save_path) # set pars # define parameters: -Lmeter = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 -Nphoton_min = 5 # mininum need photons per stancil to return results +Lmeter = 20 # stencil length in units of 'dist'; likely in meters the resulting resolution is L/2 +Nphoton_min = 5 # mininum need photons per stancil to return results -Lmeter_large= 100e3 # stancil width for testing photon density. stancils do not overlab. -minium_photon_density = 0.02 # minimum photon density per meter in Lmeter_large chunk to be counted as real signal +Lmeter_large = ( + 100e3 # stancil width for testing photon density. stancils do not overlab. +) +minium_photon_density = 0.02 # minimum photon density per meter in Lmeter_large chunk to be counted as real signal -plot_flag = True +plot_flag = True Nworkers_process = 6 # number of threads for parallel processing # outer loop # %% # test which beams exist: -all_beams = mconfig['beams']['all_beams'] -high_beams = mconfig['beams']['high_beams'] +all_beams = mconfig["beams"]["all_beams"] +high_beams = mconfig["beams"]["high_beams"] # low_beams = mconfig['beams']['low_beams'] -if __name__ == '__main__': +if __name__ == "__main__": if ID_flag: - track_poleward = ID['pars']['poleward'] + track_poleward = ID["pars"]["poleward"] beams = all_beams - print('take poleward from ID file') + print("take poleward from ID file") else: - f = h5py.File(load_file_str, 'r') - beams = [b if b in f.keys() else None for b in all_beams] + f = h5py.File(load_file_str, "r") + beams = [b if b in f.keys() else None for b in all_beams] imp.reload(regrid) - track_poleward = regrid.track_pole_ward_file(f) - print('take poleward from ATL03 file') - print('poleward track is ' , track_poleward) + track_poleward = regrid.track_pole_ward_file(f) + print("take poleward from ATL03 file") + print("poleward track is ", track_poleward) # %% # open HDF5 file for all tracks # ATL03 = h5py.File(save_path_data + '/A01c_ATL03_'+ ID_name+'_2.h5', 'r') + def get_ATL03_beam(ATL03_k): - DD = pd.DataFrame()#columns = ATL03.keys()) + DD = pd.DataFrame() # columns = ATL03.keys()) for ikey in ATL03_k.keys(): DD[ikey] = ATL03_k[ikey] return DD -#for k in beams: + +# for k in beams: def load_data_and_cut(k): - #print(k) - #k =all_beams[1] + # print(k) + # k =all_beams[1] - Tsel = get_ATL03_beam(ATL03[k]) + Tsel = get_ATL03_beam(ATL03[k]) seg = get_ATL03_beam(ATL03_seg[k]) - #Tsel_c = get_ATL03_beam(ATL03_c[k]) + # Tsel_c = get_ATL03_beam(ATL03_c[k]) - #imp.reload(io) - #T, seg = io.getATL03_beam(load_file_str, beam= k) + # imp.reload(io) + # T, seg = io.getATL03_beam(load_file_str, beam= k) - print('loaded') - #T = T[T['mask_seaice']] # only take sea ice points, no ocean points - #T = T.drop(labels=[ 'year', 'month', 'day', 'hour', 'minute', 'second', 'ph_id_count', 'mask_seaice'], axis= 1) - #T = T.drop(labels=[ 'ph_id_count', 'mask_seaice'], axis= 1) - print( 'T MB ' + get_size(Tsel) ) + print("loaded") + # T = T[T['mask_seaice']] # only take sea ice points, no ocean points + # T = T.drop(labels=[ 'year', 'month', 'day', 'hour', 'minute', 'second', 'ph_id_count', 'mask_seaice'], axis= 1) + # T = T.drop(labels=[ 'ph_id_count', 'mask_seaice'], axis= 1) + print("T MB " + get_size(Tsel)) ho = k # ho = MT.add_line_var(ho, 'size', str(T.shape[0])) # ho = MT.add_line_var(ho, 'by confidence levels:' + str(np.arange(0, 5)), [ (T['signal_confidence'] == i).sum() for i in np.arange(0, 5) ]) # filter: - #Tsel = T[ (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] + # Tsel = T[ (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] # if len(Tsel) == 0: # ho = MT.add_line_var(ho, 'no photons found', '') # #Tsel= T[(T['signal_confidence'] ==-1 ) & (T['heights']<100) & (T['heights'] > -100) ]# & (T['delta_time']>5) & (T['delta_time']<24) ] - #Tsel_c = io.getATL03_height_correction(load_file_str) + # Tsel_c = io.getATL03_height_correction(load_file_str) # Tsel_c = Tsel_c[Tsel_c['dem_h'] < 1e5] # cute out weird references # # needs only dem_h and heihgts # Tsel = regrid.correct_heights(Tsel, Tsel_c).reset_index(drop=True)# correct height - #print('height corrected') - + # print('height corrected') ### cut data at the rear that has too much variance # cut last segments of data until variance is similar - rear_mask = np.array(Tsel.index) > -1 # True + rear_mask = np.array(Tsel.index) > -1 # True nsize0 = Tsel.shape[0] - N_seg= 20 + N_seg = 20 cut_flag = True dd_old = -1 - dd0 = np.array(Tsel['heights_c']) - print('inital length' , nsize0) + dd0 = np.array(Tsel["heights_c"]) + print("inital length", nsize0) - @jit(nopython=True, parallel= False) + @jit(nopython=True, parallel=False) def adjust_length(var_list, rear_mask, cut_flag, track_poleward): var_list = var_list if track_poleward else var_list[::-1] - if var_list[0:3].mean()*10 < var_list[-1]: - #print('cut last '+ str(100/N_seg) +'% of data') - rear_mask[int(nsize* (N_seg-1) / N_seg):] = False + if var_list[0:3].mean() * 10 < var_list[-1]: + # print('cut last '+ str(100/N_seg) +'% of data') + rear_mask[int(nsize * (N_seg - 1) / N_seg) :] = False else: - cut_flag = False + cut_flag = False rear_mask = rear_mask if track_poleward else rear_mask[::-1] return rear_mask, cut_flag - #@jit(nopython=True, parallel= True) + # @jit(nopython=True, parallel= True) def get_var(sti): - return dd[sti[0]: sti[1]].var() - + return dd[sti[0] : sti[1]].var() while cut_flag: - dd= dd0[rear_mask] + dd = dd0[rear_mask] nsize = dd.size - print('new length', nsize) + print("new length", nsize) - stencil_iter = create_chunk_boundaries( int(nsize/N_seg), nsize,ov =0, iter_flag=True ) + stencil_iter = create_chunk_boundaries( + int(nsize / N_seg), nsize, ov=0, iter_flag=True + ) var_list = np.array(list(map(get_var, stencil_iter))) - #print(k, var_list) - rear_mask, cut_flag = adjust_length(var_list, rear_mask, cut_flag, track_poleward) + # print(k, var_list) + rear_mask, cut_flag = adjust_length( + var_list, rear_mask, cut_flag, track_poleward + ) if nsize == dd_old: - print('--- lengthen segments') - N_seg -=1 - #cut_flag = False + print("--- lengthen segments") + N_seg -= 1 + # cut_flag = False dd_old = nsize - print( 'Tsel MB ' + get_size(Tsel) ) + print("Tsel MB " + get_size(Tsel)) return k, rear_mask, Tsel, seg, ho -#load_data_and_cut(all_beams[1]) +# load_data_and_cut(all_beams[1]) # %% -if __name__ == '__main__': - ATL03 = h5py.File(load_path +'/'+load_file +'_corrected.h5', 'r') - ATL03_seg = h5py.File(load_path +'/'+load_file +'_seg.h5', 'r') +if __name__ == "__main__": + ATL03 = h5py.File(load_path + "/" + load_file + "_corrected.h5", "r") + ATL03_seg = h5py.File(load_path + "/" + load_file + "_seg.h5", "r") - hist = 'Beam stats' - B = dict() - #B1save = dict() - SEG = dict() + hist = "Beam stats" + B = dict() + # B1save = dict() + SEG = dict() k = beams[0] # A = list() @@ -235,35 +253,40 @@ def get_var(sti): # A.append(load_data_and_cut(k)) with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: - A = list( executor.map(load_data_and_cut, all_beams) ) + A = list(executor.map(load_data_and_cut, all_beams)) - print( 'A MB ' + get_size(A) ) + print("A MB " + get_size(A)) ATL03.close() ATL03_seg.close() - #ATL03_c.close() - - for I in A: # collect returns from from mapping - k, rear_mask, Tsel, seg, ho = I - - Tsel['process_mask'] = rear_mask - #B1save[k] = Tsel - B[k] = Tsel[rear_mask].drop(columns='process_mask') - SEG[k] = seg - - ho = MT.add_line_var(ho, 'cutted ', str( np.round( 100 *(rear_mask.size -rear_mask.sum() ) / rear_mask.size, 0)) + '% in the back of the track' ) - ho = MT.add_line_var(ho, 'selected size', str(Tsel.shape[0])) - #ho = MT.add_line_var(ho, 'final size ', str(Tsel_c.shape[0])) + # ATL03_c.close() + + for I in A: # collect returns from from mapping + k, rear_mask, Tsel, seg, ho = I + + Tsel["process_mask"] = rear_mask + # B1save[k] = Tsel + B[k] = Tsel[rear_mask].drop(columns="process_mask") + SEG[k] = seg + + ho = MT.add_line_var( + ho, + "cutted ", + str(np.round(100 * (rear_mask.size - rear_mask.sum()) / rear_mask.size, 0)) + + "% in the back of the track", + ) + ho = MT.add_line_var(ho, "selected size", str(Tsel.shape[0])) + # ho = MT.add_line_var(ho, 'final size ', str(Tsel_c.shape[0])) print(ho) - hist = MT.write_log(hist, ho) + hist = MT.write_log(hist, ho) - print('done with 1st loop') + print("done with 1st loop") del Tsel del A - #print( 'B_save MB ' + get_size(B1save) ) - print( 'B MB ' + get_size(B) ) - print( 'SEG MB ' + get_size(SEG) ) + # print( 'B_save MB ' + get_size(B1save) ) + print("B MB " + get_size(B)) + print("SEG MB " + get_size(SEG)) # %% define x- coodindate @@ -271,122 +294,133 @@ def get_var(sti): # this is on the equatorward side for poleward track, or # on the poleward side for equatorward tracks. - total_segment_dist_x_min= list() - for k,I in SEG.items(): - total_segment_dist_x_min.append( I['segment_dist_x'].min() ) + total_segment_dist_x_min = list() + for k, I in SEG.items(): + total_segment_dist_x_min.append(I["segment_dist_x"].min()) total_segment_dist_x_min = min(total_segment_dist_x_min) + # %% def make_x_coorindate(k): """ Returns the "true" along track coordindate but finding the correpsonding segment length also adds the segment_ID to the main table T """ - print(k, ' make coodindate') - T, seg = B[k], SEG[k] + print(k, " make coodindate") + T, seg = B[k], SEG[k] # make sure data is strictly ordered by delta_time - T = T.sort_values('delta_time').reset_index(drop=True) - seg = seg.sort_values('delta_time').reset_index(drop=True) + T = T.sort_values("delta_time").reset_index(drop=True) + seg = seg.sort_values("delta_time").reset_index(drop=True) # find positions where segmetn length is reset # shifts segment length postions in time - delta_onehalf = seg['delta_time'].diff()/2 - delta_onehalf.iloc[0] = delta_onehalf.iloc[1] - seg['delta_half_time'] = seg['delta_time'] - delta_onehalf - 1e-5 + delta_onehalf = seg["delta_time"].diff() / 2 + delta_onehalf.iloc[0] = delta_onehalf.iloc[1] + seg["delta_half_time"] = seg["delta_time"] - delta_onehalf - 1e-5 # cut phontos that are not in segmentns - T = T[ (T['delta_time'] > seg['delta_half_time'].iloc[0]) & (T['delta_time'] < seg['delta_half_time'].iloc[-1])] - bin_labels = np.digitize( T['delta_time'], seg['delta_half_time'], right = True ) + T = T[ + (T["delta_time"] > seg["delta_half_time"].iloc[0]) + & (T["delta_time"] < seg["delta_half_time"].iloc[-1]) + ] + bin_labels = np.digitize(T["delta_time"], seg["delta_half_time"], right=True) # select relevant data - SS = seg['segment_dist_x'] - SS_sid = seg['segment_id'] + SS = seg["segment_dist_x"] + SS_sid = seg["segment_id"] - repeats = np.bincount(bin_labels, minlength =SS.shape[0]) + repeats = np.bincount(bin_labels, minlength=SS.shape[0]) # check if repeats sumup if repeats.sum() != T.shape[0]: - print('repeats do not sum up') + print("repeats do not sum up") # repeat segment dat accoridng to photons - SS = SS.repeat(repeats) + SS = SS.repeat(repeats) SS.index = T.index - SS_sid = SS_sid.repeat(repeats) + SS_sid = SS_sid.repeat(repeats) SS_sid.index = T.index # define new coordinate - T['x'] = SS + T['along_track_distance'] - T['segment_id'] = SS_sid + T["x"] = SS + T["along_track_distance"] + T["segment_id"] = SS_sid # find bad photons def find_anomalie_photons(Ti2, segi): - x_interp = np.interp(Ti2['delta_time'], segi['delta_time'], segi['segment_dist_x'] ) + x_interp = np.interp( + Ti2["delta_time"], segi["delta_time"], segi["segment_dist_x"] + ) - diff_x = x_interp - Ti2['x'] - diff_x = abs(diff_x-diff_x.mean()) - return diff_x > 3 *diff_x.std() , x_interp + diff_x = x_interp - Ti2["x"] + diff_x = abs(diff_x - diff_x.mean()) + return diff_x > 3 * diff_x.std(), x_interp fail_mask, x_interp = find_anomalie_photons(T, seg) - print('weird photon fraction:' , sum(fail_mask)/ fail_mask.size) + print("weird photon fraction:", sum(fail_mask) / fail_mask.size) # aply fail mask - #T= T[~fail_mask] - T['x'][fail_mask] = x_interp[fail_mask] + # T= T[~fail_mask] + T["x"][fail_mask] = x_interp[fail_mask] return k, T -if __name__ == '__main__': + +if __name__ == "__main__": with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: - A = list( executor.map(make_x_coorindate, all_beams) ) - - B= dict() - for I in A: # collect returns from from mapping - k = I[0] - B[ k ] = I[1][::-1] - - if not track_poleward: # invert x- coordinate if there is an equatorward track - print('invert table') - B[k] = B[k].reset_index(drop=True) - B[k]['x_true'] = B[k]['x'] - B[k]['x'] = abs(B[k]['x'] - B[k]['x'].iloc[0]) + A = list(executor.map(make_x_coorindate, all_beams)) + + B = dict() + for I in A: # collect returns from from mapping + k = I[0] + B[k] = I[1][::-1] + + if not track_poleward: # invert x- coordinate if there is an equatorward track + print("invert table") + B[k] = B[k].reset_index(drop=True) + B[k]["x_true"] = B[k]["x"] + B[k]["x"] = abs(B[k]["x"] - B[k]["x"].iloc[0]) else: - B[k]['x_true'] = B[k]['x'] - print('no table invert needed') + B[k]["x_true"] = B[k]["x"] + print("no table invert needed") - dist_list = np.array([np.nan, np.nan]) + dist_list = np.array([np.nan, np.nan]) for k in B.keys(): - dist_list = np.vstack([ dist_list, [ B[k]['x'].iloc[0] , B[k]['x'].iloc[-1] ] ]) + dist_list = np.vstack([dist_list, [B[k]["x"].iloc[0], B[k]["x"].iloc[-1]]]) - print( 'B MB ' + get_size(B) ) + print("B MB " + get_size(B)) del A del SEG - #del T + # del T # test ave photon density abnd quit if necessary - track_dist_bounds = [ np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0) ] - length_meter = abs(track_dist_bounds[1] - track_dist_bounds[0]) - #length_meter = (abs(lat_lims[1]) - abs(lat_lims[0])) * 110e3 - p_densities_r = list() - p_densities_l = list() + track_dist_bounds = [np.nanmin(dist_list[:, 0], 0), np.nanmax(dist_list[:, 1], 0)] + length_meter = abs(track_dist_bounds[1] - track_dist_bounds[0]) + # length_meter = (abs(lat_lims[1]) - abs(lat_lims[0])) * 110e3 + p_densities_r = list() + p_densities_l = list() for k, I in B.items(): - if 'r' in k: - p_densities_r.append( I.shape[0] /length_meter) + if "r" in k: + p_densities_r.append(I.shape[0] / length_meter) else: - p_densities_l.append( I.shape[0] /length_meter) - - if (np.array(p_densities_l).mean() < 0.5) & (np.array(p_densities_r).mean() < 0.5): # in photons per meter - print('photon density too low, this track is classified as bad track and pushed to bad list') - #MT.json_save(ID_name, bad_track_path, {'densities': [ np.array(p_densities_r).mean(), np.array(p_densities_l).mean()] , 'date': str(datetime.date.today()) }) - print('exit.') + p_densities_l.append(I.shape[0] / length_meter) + + if (np.array(p_densities_l).mean() < 0.5) & ( + np.array(p_densities_r).mean() < 0.5 + ): # in photons per meter + print( + "photon density too low, this track is classified as bad track and pushed to bad list" + ) + # MT.json_save(ID_name, bad_track_path, {'densities': [ np.array(p_densities_r).mean(), np.array(p_densities_l).mean()] , 'date': str(datetime.date.today()) }) + print("exit.") exit() # %% save corrected data and delete from cash -#io.save_pandas_table(B1save, ID_name + '_B01_corrected' , save_path) # all photos but heights adjusted and with -#io.save_pandas_table(B, ID_name + '_B01_new_coords' , save_path) # all photos but heights adjusted and with distance coordinate -#del B1save +# io.save_pandas_table(B1save, ID_name + '_B01_corrected' , save_path) # all photos but heights adjusted and with +# io.save_pandas_table(B, ID_name + '_B01_new_coords' , save_path) # all photos but heights adjusted and with distance coordinate +# del B1save # for testing # T2 = B['gt1r'] @@ -402,250 +436,348 @@ def find_anomalie_photons(Ti2, segi): # %% -if __name__ == '__main__': - F = M.figure_axis_xy(4, 3, view_scale = 0.7) +if __name__ == "__main__": + F = M.figure_axis_xy(4, 3, view_scale=0.7) - for k,I in B.items(): - plt.plot( I['lats'] , I['x'] , '.' , markersize = 0.2) - #plt.xlim(3e6, 3.25e6) - plt.xlabel('lats') - plt.ylabel('x') - F.save_light(path= plot_path, name='B01_ALT03_'+ID_name+'_tracks_check_lat_x') + for k, I in B.items(): + plt.plot(I["lats"], I["x"], ".", markersize=0.2) + # plt.xlim(3e6, 3.25e6) + plt.xlabel("lats") + plt.ylabel("x") + F.save_light(path=plot_path, name="B01_ALT03_" + ID_name + "_tracks_check_lat_x") # % - F = M.figure_axis_xy(4, 3, view_scale = 0.7) - for k,I in B.items(): - plt.plot( I['delta_time'] , I['lats'], '.' , markersize = 0.3) + F = M.figure_axis_xy(4, 3, view_scale=0.7) + for k, I in B.items(): + plt.plot(I["delta_time"], I["lats"], ".", markersize=0.3) - plt.xlabel('delta time') - plt.ylabel('lat') - F.save_light(path= plot_path, name='B01_ALT03_'+ID_name+'_tracks_check_time_lat') + plt.xlabel("delta time") + plt.ylabel("lat") + F.save_light(path=plot_path, name="B01_ALT03_" + ID_name + "_tracks_check_time_lat") - F = M.figure_axis_xy(4, 3, view_scale = 0.7) - for k,I in B.items(): - plt.plot( I['delta_time'] , I['x'], '.' , markersize = 0.3) + F = M.figure_axis_xy(4, 3, view_scale=0.7) + for k, I in B.items(): + plt.plot(I["delta_time"], I["x"], ".", markersize=0.3) - plt.xlabel('delta time') - plt.ylabel('x') + plt.xlabel("delta time") + plt.ylabel("x") - F.save_light(path= plot_path, name='B01_ALT03_'+ID_name+'_tracks_check_time_x') + F.save_light(path=plot_path, name="B01_ALT03_" + ID_name + "_tracks_check_time_x") # %% ##### 1.) derive common axis for beams and filter out low density area at the beginning -print('filter out low density area at the beginning') +print("filter out low density area at the beginning") + def derive_axis_and_boundaries(key): - #key = 'gt3r' + # key = 'gt3r' print(key) - #T2 # = regrid.derive_axis(B[key], lat_lims).sort_values('dist') - T2 = B[key]#['x'] + # T2 # = regrid.derive_axis(B[key], lat_lims).sort_values('dist') + T2 = B[key] # ['x'] - x0 = get_better_lower_boundary(Lmeter_large, np.array(T2['x'])) + x0 = get_better_lower_boundary(Lmeter_large, np.array(T2["x"])) - print( 'cut off ' , 100 * (1 - T2[T2['x'] > x0].shape[0]/T2.shape[0]) , '% off all data points at the beginning' ) - T2 = T2[T2['x'] >x0] # cut off low density start + print( + "cut off ", + 100 * (1 - T2[T2["x"] > x0].shape[0] / T2.shape[0]), + "% off all data points at the beginning", + ) + T2 = T2[T2["x"] > x0] # cut off low density start + + return key, T2, [T2["x"].min(), T2["x"].max()] - return key, T2, [T2['x'].min(), T2['x'].max()] def get_better_lower_boundary(Lmeter_large, dd): - #T2 = regrid.derive_axis(B[key], lat_lims) - #dd = np.array(B['gt1l']['x']) + # T2 = regrid.derive_axis(B[key], lat_lims) + # dd = np.array(B['gt1l']['x']) - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=False + ) if stencil_iter.shape[1] == 0: while stencil_iter.shape[1] == 0: - Lmeter_large = int(Lmeter_large/2) - print( 'new Lmeter_large' + str(Lmeter_large)) - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) + Lmeter_large = int(Lmeter_large / 2) + print("new Lmeter_large" + str(Lmeter_large)) + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=False + ) - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= True) + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=True + ) def get_density(sti): - return sti[0], sum( (sti[0] <= dd) & (dd < sti[-1]) ) / Lmeter_large + return sti[0], sum((sti[0] <= dd) & (dd < sti[-1])) / Lmeter_large - with futures.ThreadPoolExecutor(max_workers= Nworkers_process) as executor_sub: - #var_list = np.array(list(map(get_density, stencil_iter))) + with futures.ThreadPoolExecutor(max_workers=Nworkers_process) as executor_sub: + # var_list = np.array(list(map(get_density, stencil_iter))) var_list = np.array(list(executor_sub.map(get_density, stencil_iter))) var_list = np.array(var_list) - #var_list[:,0] = np.random.rand(10) - #sort var_list + # var_list[:,0] = np.random.rand(10) + # sort var_list print(var_list) - var_list = var_list[var_list[:,0].argsort(), :] - #print(var_list) - if sum(var_list[:,1] > minium_photon_density) > 1: - first_stencil = next((i for i, j in enumerate(var_list[:,1] > minium_photon_density) if j), None) #- 1 - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) + var_list = var_list[var_list[:, 0].argsort(), :] + # print(var_list) + if sum(var_list[:, 1] > minium_photon_density) > 1: + first_stencil = next( + (i for i, j in enumerate(var_list[:, 1] > minium_photon_density) if j), None + ) # - 1 + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter_large, [dd.min(), dd.max()], ov=0, iter_flag=False + ) return stencil_iter[0, first_stencil] else: - #first_stencil = next((i for i, j in enumerate(var_list[:,1] > 0) if j), None)# - 1 - print('no sufficient photon density found. return short stencil') + # first_stencil = next((i for i, j in enumerate(var_list[:,1] > 0) if j), None)# - 1 + print("no sufficient photon density found. return short stencil") # first_stencil= len(var_list[:,1]) -1 # stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter_large, [ dd.min(), dd.max()],ov =0, iter_flag= False) # return stencil_iter[0, first_stencil] # #print(first_stencil) - return var_list[-1,0]#[-1,0] + return var_list[-1, 0] # [-1,0] + -if __name__ == '__main__': +if __name__ == "__main__": with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: - A = list( executor.map(derive_axis_and_boundaries, all_beams) ) + A = list(executor.map(derive_axis_and_boundaries, all_beams)) # for k in all_beams: # A = derive_axis_and_boundaries(k) - B2 = dict() - dist_list = np.array([np.nan, np.nan]) + B2 = dict() + dist_list = np.array([np.nan, np.nan]) for I in A: - k = I[0] - B2[k] = I[1] - #B2[k]['dist'] = B2[k]['x'] - dist_list = np.vstack([dist_list,I[2] ]) + k = I[0] + B2[k] = I[1] + # B2[k]['dist'] = B2[k]['x'] + dist_list = np.vstack([dist_list, I[2]]) del A del B - track_dist_bounds = [ np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0) ] + track_dist_bounds = [np.nanmin(dist_list[:, 0], 0), np.nanmax(dist_list[:, 1], 0)] - print( 'B2 MB ' + get_size(B2) ) + print("B2 MB " + get_size(B2)) # %% -if __name__ == '__main__': - xscale= 1e3 - F= M.figure_axis_xy(5, 3, view_scale= 0.6) - for k,I in B2.items(): - plt.plot( I['x']/xscale , I['across_track_distance']/xscale , '.' , markersize = 0.3) - #plt.xlim(3e6, 3.25e6) +if __name__ == "__main__": + xscale = 1e3 + F = M.figure_axis_xy(5, 3, view_scale=0.6) + for k, I in B2.items(): + plt.plot( + I["x"] / xscale, I["across_track_distance"] / xscale, ".", markersize=0.3 + ) + # plt.xlim(3e6, 3.25e6) for k in high_beams: Ii = B2[k].iloc[0] - plt.text(Ii.x/xscale+ 5, Ii.across_track_distance/xscale , str(Ii[[ 'lats', 'lons'] ]).split('Name')[0] ) + plt.text( + Ii.x / xscale + 5, + Ii.across_track_distance / xscale, + str(Ii[["lats", "lons"]]).split("Name")[0], + ) Ii = B2[k].iloc[-1] - plt.text(Ii.x/xscale+ 5, Ii.across_track_distance/xscale , str(Ii[[ 'lats', 'lons'] ]).split('Name')[0], ha ='right' ) - - F.ax.axvline(track_dist_bounds[0]/xscale, color='gray', zorder= 2) - F.ax.axvline(track_dist_bounds[1]/xscale, color='gray', zorder= 2) - F.ax.axhline(0, color='gray', zorder= 2) - - plt.title('B01 filter and regrid | ' + ID_name +'\npoleward '+str(track_poleward)+' \n \n', loc='left') - plt.xlabel('along track distance (km)') - plt.ylabel('across track distance (km)') - - F.save_light(path= plot_path +'../', name='B01_ALT03_'+ID_name+'_tracks_all') + plt.text( + Ii.x / xscale + 5, + Ii.across_track_distance / xscale, + str(Ii[["lats", "lons"]]).split("Name")[0], + ha="right", + ) + + F.ax.axvline(track_dist_bounds[0] / xscale, color="gray", zorder=2) + F.ax.axvline(track_dist_bounds[1] / xscale, color="gray", zorder=2) + F.ax.axhline(0, color="gray", zorder=2) + + plt.title( + "B01 filter and regrid | " + + ID_name + + "\npoleward " + + str(track_poleward) + + " \n \n", + loc="left", + ) + plt.xlabel("along track distance (km)") + plt.ylabel("across track distance (km)") + + F.save_light(path=plot_path + "../", name="B01_ALT03_" + ID_name + "_tracks_all") # %% # for testing -#track_dist_bounds[1] = track_dist_bounds[0] + (track_dist_bounds[1] - track_dist_bounds[0])/20 -#track_dist_bounds = ts_s[0] ,ts_s[0] + (ts_s[1] - ts_s[0]) /6 +# track_dist_bounds[1] = track_dist_bounds[0] + (track_dist_bounds[1] - track_dist_bounds[0])/20 +# track_dist_bounds = ts_s[0] ,ts_s[0] + (ts_s[1] - ts_s[0]) /6 # %% ##### 2.) regridding and averaging imp.reload(regrid) -print('regrid') -def regridding_wrapper(I): - key, Ti = I - print(key, Ti.shape,2* Ti.shape[0]/Lmeter) - stencil_iter = create_chunk_boundaries_unit_lengths( Lmeter, track_dist_bounds, iter_flag=False ) - Bi = regrid.get_stencil_stats_shift( Ti, stencil_iter, 'heights_c', 'x' , stancil_width= Lmeter/2, Nphoton_min=Nphoton_min) +print("regrid") - #print( 'Bi MB ' + get_size(Bi) ) - print(key, 'done') + +def regridding_wrapper(I): + key, Ti = I + print(key, Ti.shape, 2 * Ti.shape[0] / Lmeter) + stencil_iter = create_chunk_boundaries_unit_lengths( + Lmeter, track_dist_bounds, iter_flag=False + ) + Bi = regrid.get_stencil_stats_shift( + Ti, + stencil_iter, + "heights_c", + "x", + stancil_width=Lmeter / 2, + Nphoton_min=Nphoton_min, + ) + + # print( 'Bi MB ' + get_size(Bi) ) + print(key, "done") return key, Bi -if __name__ == '__main__': + +if __name__ == "__main__": # ---define start and end position and same in Json file - with futures.ProcessPoolExecutor(max_workers = Nworkers_process) as executor: - B3 = dict( executor.map( regridding_wrapper, B2.items() ) ) + with futures.ProcessPoolExecutor(max_workers=Nworkers_process) as executor: + B3 = dict(executor.map(regridding_wrapper, B2.items())) - print( 'B3 MB ' + get_size(B3) ) - print( 'I MB ' + get_size(I) ) + print("B3 MB " + get_size(B3)) + print("I MB " + get_size(I)) - #I = B3['gt2l'].copy() + # I = B3['gt2l'].copy() D_info = dict() - for k,I in B3.items(): + for k, I in B3.items(): # reset x coordinate - I['median_dist'] = I['x_median'] - track_dist_bounds[0] #- Lmeter/2 - I['dist'] = I['x'] - track_dist_bounds[0] #- Lmeter/2 - #I['index'] = I['x'] + I["median_dist"] = I["x_median"] - track_dist_bounds[0] # - Lmeter/2 + I["dist"] = I["x"] - track_dist_bounds[0] # - Lmeter/2 + # I['index'] = I['x'] # rename y coordinate - I = I.rename(columns={'across_track_distance': 'y'}) + I = I.rename(columns={"across_track_distance": "y"}) # find starting and end position - Di_s = dict(I[I['segment_id'] == I['segment_id'].iloc[0] ].mean()[['lons', 'lats', 'segment_id', 'delta_time']]) - Di_s['across_track_distance_0'] =track_dist_bounds[0] - - Di_e = dict(I[I['segment_id'] == I['segment_id'].iloc[-1] ].mean()[['lons', 'lats', 'segment_id', 'delta_time']]) - Di_e['across_track_distance_0'] =track_dist_bounds[0] - - D_info[k] = {'start':Di_s, 'end':Di_e , 'poleward': str(track_poleward) } + Di_s = dict( + I[I["segment_id"] == I["segment_id"].iloc[0]].mean()[ + ["lons", "lats", "segment_id", "delta_time"] + ] + ) + Di_s["across_track_distance_0"] = track_dist_bounds[0] + + Di_e = dict( + I[I["segment_id"] == I["segment_id"].iloc[-1]].mean()[ + ["lons", "lats", "segment_id", "delta_time"] + ] + ) + Di_e["across_track_distance_0"] = track_dist_bounds[0] + + D_info[k] = {"start": Di_s, "end": Di_e, "poleward": str(track_poleward)} # reorder indexes - column_names = ['x', 'y', 'x_median', 'median_dist', 'lons', 'lats' ,'heights_c_weighted_mean', 'heights_c_median', 'heights_c_std', 'N_photos', ] - vars_ad = set(list(I[I['segment_id'] == I['segment_id'].iloc[0] ].mean().index)) - set(column_names) - I = I.reindex(columns=column_names + list(vars_ad)) + column_names = [ + "x", + "y", + "x_median", + "median_dist", + "lons", + "lats", + "heights_c_weighted_mean", + "heights_c_median", + "heights_c_std", + "N_photos", + ] + vars_ad = set( + list(I[I["segment_id"] == I["segment_id"].iloc[0]].mean().index) + ) - set(column_names) + I = I.reindex(columns=column_names + list(vars_ad)) B3[k] = I - # save Json - MT.json_save(ID_name + '_B01_stats',save_path, D_info, verbose= True ) + MT.json_save(ID_name + "_B01_stats", save_path, D_info, verbose=True) # saving data # io.save_pandas_table(B2, ID_name + '_B01_regridded' , save_path) # all photos but heights adjusted and with distance coordinate # io.save_pandas_table(B3, ID_name + '_B01_binned' , save_path) # regridding heights - #io.write_track_to_HDF5(B2, ID_name + '_B01_regridded' , save_path) # all photos but heights adjusted and with distance coordinate - #io.write_track_to_HDF5(B3, ID_name + '_B01_binned' , save_path) # regridding heights + # io.write_track_to_HDF5(B2, ID_name + '_B01_regridded' , save_path) # all photos but heights adjusted and with distance coordinate + # io.write_track_to_HDF5(B3, ID_name + '_B01_binned' , save_path) # regridding heights -#B3[key].T +# B3[key].T # %% plotting just for checking font_for_print() -if __name__ == '__main__': +if __name__ == "__main__": - key = 'gt1r' + key = "gt1r" if plot_flag: MT.mkdirs_r(plot_path) Ti2 = B3[key] - T2 = B2[key] + T2 = B2[key] - dl = 8*1e3 - x_key= 'x' + dl = 8 * 1e3 + x_key = "x" - latlims = (Ti2['x'].iloc[0] , Ti2['x'].iloc[-1] ) - #chunk_list = np.arange(latlims[0],latlims[1], dl ) - #chunk_list = sample( list(np.arange(latlims[0],latlims[1],dl )[0:80]) ,10) - chunk_list = np.arange(latlims[0],latlims[1], dl )[::1][0:20] - #chunk_list = np.append( chunk_list, latlims[1]- dl-1) + latlims = (Ti2["x"].iloc[0], Ti2["x"].iloc[-1]) + # chunk_list = np.arange(latlims[0],latlims[1], dl ) + # chunk_list = sample( list(np.arange(latlims[0],latlims[1],dl )[0:80]) ,10) + chunk_list = np.arange(latlims[0], latlims[1], dl)[::1][0:20] + # chunk_list = np.append( chunk_list, latlims[1]- dl-1) for ll in chunk_list: F = M.figure_axis_xy(7, 2, view_scale=0.8) - plt.plot( T2[x_key]/1e3, T2['heights_c'], 'k.', markersize= 0.5, alpha =0.8 ) - #plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') - - plt.plot(Ti2[x_key]/1e3, Ti2['heights_c_weighted_mean'] +1, '.-', color='blue', linewidth=0.5, markersize=2,alpha=0.9, label='x-gauss weighted mean +1') - plt.plot(Ti2[x_key]/1e3, Ti2['heights_c_median'] +2, 'r.-', linewidth=0.5, markersize=2,alpha=0.9, label='median + 2') - - plt.plot(Ti2['x']/1e3, Ti2['heights_c_mode']+3, 'g.-', linewidth=0.5, markersize=2,alpha=0.9, label='mode + 3') - - plt.plot(Ti2['x']/1e3, Ti2['heights_c_std'] - 2, 'k-', linewidth=0.5,alpha=1) - plt.fill_between( Ti2['x']/1e3, Ti2['heights_c_std'] -2 , y2=-2, color='gray',alpha=1) + plt.plot(T2[x_key] / 1e3, T2["heights_c"], "k.", markersize=0.5, alpha=0.8) + # plt.plot( ALT07['ref']['latitude'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') + + plt.plot( + Ti2[x_key] / 1e3, + Ti2["heights_c_weighted_mean"] + 1, + ".-", + color="blue", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="x-gauss weighted mean +1", + ) + plt.plot( + Ti2[x_key] / 1e3, + Ti2["heights_c_median"] + 2, + "r.-", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="median + 2", + ) + + plt.plot( + Ti2["x"] / 1e3, + Ti2["heights_c_mode"] + 3, + "g.-", + linewidth=0.5, + markersize=2, + alpha=0.9, + label="mode + 3", + ) + + plt.plot( + Ti2["x"] / 1e3, Ti2["heights_c_std"] - 2, "k-", linewidth=0.5, alpha=1 + ) + plt.fill_between( + Ti2["x"] / 1e3, Ti2["heights_c_std"] - 2, y2=-2, color="gray", alpha=1 + ) # plt.plot( ALT03['delta_time'], ALT03['heights_c'], 'k.', markersize= 0.3, alpha =0.2 ) # plt.plot(ALT07['time']['delta_time'] , ALT07['heights']['height_segment_height'] , 'r.', markersize=0.8, alpha = 1, label ='ALT07 seg. heights') plt.legend(loc=1, ncol=3) - plt.xlim(ll/1e3, (ll+dl)/1e3) + plt.xlim(ll / 1e3, (ll + dl) / 1e3) plt.ylim(-2, 6) - plt.xlabel('Distance from the Ice Edge (km)') - plt.ylabel('Height Anomalie (m)') - #F.ax.axhline(y =-1.8, color='black', linewidth=0.5) - F.save_light(path= plot_path, name='ALT03_filt_compare'+ str(round(ll/1e3))) + plt.xlabel("Distance from the Ice Edge (km)") + plt.ylabel("Height Anomalie (m)") + # F.ax.axhline(y =-1.8, color='black', linewidth=0.5) + F.save_light( + path=plot_path, name="ALT03_filt_compare" + str(round(ll / 1e3)) + ) diff --git a/analysis_db/S01_derive_tracklist_from_file.py b/analysis_db/S01_derive_tracklist_from_file.py index 61eddade..dcab0683 100644 --- a/analysis_db/S01_derive_tracklist_from_file.py +++ b/analysis_db/S01_derive_tracklist_from_file.py @@ -1,19 +1,20 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This script takes in a list of """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") -#import m_general as M -#import m_tools as MT +# import m_general as M +# import m_tools as MT import numpy as np import os import pandas as pd @@ -22,67 +23,74 @@ import icepyx as ipx # %% -path = mconfig['paths']['analysis']+'../track_lists/' +path = mconfig["paths"]["analysis"] + "../track_lists/" # batch = 'Batch02_alex' -with open(path+ 'alex_ATL07_filelist.txt', 'r') as f: +with open(path + "alex_ATL07_filelist.txt", "r") as f: contents = f.readlines() -batch = 'batch03' -with open(path+ 'batch03_ATL07_filelist.txt', 'r') as f: +batch = "batch03" +with open(path + "batch03_ATL07_filelist.txt", "r") as f: contents = f.readlines() -h5_files= list() +h5_files = list() for l in contents: - if '.h5' in l: + if ".h5" in l: h5_files.append(l) file_instances = list() for h in h5_files: - #h.split('.')[0].split('_') - file_instances.append( h.split('.')[0].split('_')[1:4] ) + # h.split('.')[0].split('_') + file_instances.append(h.split(".")[0].split("_")[1:4]) -MT.json_save(batch+'_tracks_components', path, file_instances) +MT.json_save(batch + "_tracks_components", path, file_instances) -#file_instances +# file_instances ## make dataframe and derive ID that is need to compare the data: D = pd.DataFrame(file_instances) + def str2dt64(s): - return np.datetime64(s[0:4]+'-'+s[4:6]+'-'+s[6:8]) + return np.datetime64(s[0:4] + "-" + s[4:6] + "-" + s[6:8]) -D['date'] = D[0].apply(lambda row: str2dt64(row[0:8]) ) -dmin, dmax = D['date'].min(), D['date'].max() # needed for icspyx modules +D["date"] = D[0].apply(lambda row: str2dt64(row[0:8])) -D['RGT'] = D[1].apply(lambda row: row[0:4]) -D['cycle'] = D[1].apply(lambda row: row[4:6]) -D['segment'] = D[1].apply(lambda row: row[6:8]) -#D['segment'].hist() +dmin, dmax = D["date"].min(), D["date"].max() # needed for icspyx modules -D['id'] = D[0]+'_'+D[1] -#D['id_compare'] = D[0]+'_'+ -D['id_compare'] = D['RGT']+D['cycle'] +D["RGT"] = D[1].apply(lambda row: row[0:4]) +D["cycle"] = D[1].apply(lambda row: row[4:6]) +D["segment"] = D[1].apply(lambda row: row[6:8]) +# D['segment'].hist() + +D["id"] = D[0] + "_" + D[1] +# D['id_compare'] = D[0]+'_'+ +D["id_compare"] = D["RGT"] + D["cycle"] # len(D['id_compare']) # len(set(D['id_compare'])) # %% -dx= 100 +dx = 100 all_wanted_tracks = list() for x in np.arange(0, int(len(D)), dx): - Dsub = D[x:x+dx] + Dsub = D[x : x + dx] - print('set ', x) + print("set ", x) # % login to earth data .. - date_range =[str(dmin).split(' ')[0],str(dmax).split(' ')[0]] - region_a = ipx.Query('ATL03',[180, -70, -180, -55],date_range, \ - start_time='00:00:00', end_time='23:59:59', \ - tracks = list(Dsub['RGT'])) - - region_a.earthdata_login('mhell','mhell@ucsd.edu') + date_range = [str(dmin).split(" ")[0], str(dmax).split(" ")[0]] + region_a = ipx.Query( + "ATL03", + [180, -70, -180, -55], + date_range, + start_time="00:00:00", + end_time="23:59:59", + tracks=list(Dsub["RGT"]), + ) + + region_a.earthdata_login("mhell", "mhell@ucsd.edu") # pw # @[49[4tK\-qBWB%5 @@ -91,15 +99,15 @@ def str2dt64(s): region_a.avail_granules(ids=True) # % compare availabe ID's with the wanted ID's - gran_list = [i['producer_granule_id'] for i in region_a.granules.avail] - sub_set= list() - for id_wanted in Dsub['id_compare']: + gran_list = [i["producer_granule_id"] for i in region_a.granules.avail] + sub_set = list() + for id_wanted in Dsub["id_compare"]: sub_set.append([i for i in gran_list if id_wanted in i]) - all_possible_tracks = [item for sublist in sub_set for item in sublist] - print( len(all_possible_tracks), ' matching granules found') + all_possible_tracks = [item for sublist in sub_set for item in sublist] + print(len(all_possible_tracks), " matching granules found") [all_wanted_tracks.append(i) for i in all_possible_tracks] # %% save clean file list -MT.json_save(batch+'_ATL03_A00', path, all_wanted_tracks) +MT.json_save(batch + "_ATL03_A00", path, all_wanted_tracks) diff --git a/analysis_db/SB01_SL_kml_orbits_to_geopandas.py b/analysis_db/SB01_SL_kml_orbits_to_geopandas.py index ea49c624..05932a14 100644 --- a/analysis_db/SB01_SL_kml_orbits_to_geopandas.py +++ b/analysis_db/SB01_SL_kml_orbits_to_geopandas.py @@ -10,18 +10,19 @@ from ICEsat2_SI_tools.read_ground_tracks import * -load_path ='/Users/Shared/Projects/2021_ICESat2_tracks/data/groundtracks/originals/' -#save_path ='/Users/Shared/Projects/2021_ICESat2_tracks/analysis_db/support_files/' -save_path = mconfig['paths']['analysis'] +'../analysis_db/support_files/' +load_path = "/Users/Shared/Projects/2021_ICESat2_tracks/data/groundtracks/originals/" +# save_path ='/Users/Shared/Projects/2021_ICESat2_tracks/analysis_db/support_files/' +save_path = mconfig["paths"]["analysis"] + "../analysis_db/support_files/" -#for filename, hemis in zip(['ICESat2_groundtracks_EasternHem_small.zip' , 'ICESat2groundtracksWesternHem.zip'], ['EAST', 'WEST'] ): -for filename, hemis in zip(['arcticallorbits.zip' , 'antarcticaallorbits.zip'], ['NH', 'SH'] ): - loadfile = load_path + filename - save_basename = 'IS2_mission_points_'+hemis+'_RGT_' +# for filename, hemis in zip(['ICESat2_groundtracks_EasternHem_small.zip' , 'ICESat2groundtracksWesternHem.zip'], ['EAST', 'WEST'] ): +for filename, hemis in zip( + ["arcticallorbits.zip", "antarcticaallorbits.zip"], ["NH", "SH"] +): + loadfile = load_path + filename + save_basename = "IS2_mission_points_" + hemis + "_RGT_" G = ICESat2_mission_points(loadfile) - G[ (G['GT']=='GT7')].to_file(save_path+save_basename +'all.shp') - + G[(G["GT"] == "GT7")].to_file(save_path + save_basename + "all.shp") # %% diff --git a/analysis_fake_data/A01_1D_spectra_and_ice.py b/analysis_fake_data/A01_1D_spectra_and_ice.py index 2fab1afd..4c0c0bd1 100644 --- a/analysis_fake_data/A01_1D_spectra_and_ice.py +++ b/analysis_fake_data/A01_1D_spectra_and_ice.py @@ -1,6 +1,7 @@ # %% import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,19 +9,20 @@ """ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -#matplotlib inline -#import m_general as M -#import m_tools as MT +# matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -38,24 +40,23 @@ import JONSWAP_gamma - -plot_path = mconfig['paths']['plot']+ 'proposal/' +plot_path = mconfig["paths"]["plot"] + "proposal/" # %% -f = np.arange(0.001, 0.2,0.001) +f = np.arange(0.001, 0.2, 0.001) spec_power = JONSWAP_gamma.JONSWAP_default(f, 2e6, 15) plt.plot(f, spec_power) # %% -amps = (spec_power * np.gradient(f)) **0.5 +amps = (spec_power * np.gradient(f)) ** 0.5 -2/f[amps.argmax()] +2 / f[amps.argmax()] t = np.arange(0, 500, 0.1) -tt, ww = np.meshgrid(2* np.pi * f, t) -phi = np.random.random(len(amps))*2*np.pi +tt, ww = np.meshgrid(2 * np.pi * f, t) +phi = np.random.random(len(amps)) * 2 * np.pi -instance = amps* np.cos( ww * tt + phi ) +instance = amps * np.cos(ww * tt + phi) # %% fake ice @@ -64,99 +65,102 @@ noise = stats.norm(0).rvs(size=t.size) data = np.zeros(len(t)) -alpha1= 0.98 +alpha1 = 0.98 xx = 1.0 -for i in np.arange(1,len(t)): - xx = data[i] = alpha1 * xx + noise[i] +for i in np.arange(1, len(t)): + xx = data[i] = alpha1 * xx + noise[i] -data = data/ data.std() +data = data / data.std() ice_mask = abs(data) > 0.8 ice = np.where(ice_mask, 1, np.nan) -SIC = ice_mask.sum()/ice.size +SIC = ice_mask.sum() / ice.size print(SIC) plt.plot(ice[0:200]) # % -ice_height = 0.04 +ice_height = 0.04 font_for_print() -ylims = -0.05, ice_height* 2.5 -F = M.figure_axis_xy(6, 3, view_scale=0.9, container = True) -plt.suptitle("Illustration of signal decomposition", y = 1) +ylims = -0.05, ice_height * 2.5 +F = M.figure_axis_xy(6, 3, view_scale=0.9, container=True) +plt.suptitle("Illustration of signal decomposition", y=1) -gs = GridSpec(3,1, wspace=0, hspace=0.5)#figure=fig, +gs = GridSpec(3, 1, wspace=0, hspace=0.5) # figure=fig, ax1 = F.fig.add_subplot(gs[0, :]) ax1.tick_params(labelbottom=False) -plt.title('Total observed mean surface height ' + str(int(np.round(SIC, 1)*100)) + '% SIC', loc = 'left') +plt.title( + "Total observed mean surface height " + str(int(np.round(SIC, 1) * 100)) + "% SIC", + loc="left", +) -#ax1 =plt.subplot(3, 1, 1) +# ax1 =plt.subplot(3, 1, 1) -#plt.plot(t, instance.mean(1) * 5 , linewidth = 0.5, color='gray') +# plt.plot(t, instance.mean(1) * 5 , linewidth = 0.5, color='gray') total_y = instance.mean(1) * 5 + ice * ice_height -plt.plot(t, total_y, '.', c=col.cascade2,markersize = 0.5 ) -plt.fill_between(t, total_y, color=col.cascade2, alpha = 0.6) +plt.plot(t, total_y, ".", c=col.cascade2, markersize=0.5) +plt.fill_between(t, total_y, color=col.cascade2, alpha=0.6) plt.xlim(0, 300) plt.ylim(ylims[0], ylims[1]) ax2 = F.fig.add_subplot(gs[1, :]) -ax2.tick_params(labelbottom=False)#, labelleft=False) -plt.title('Sea ice freeboard without waves', loc = 'left') +ax2.tick_params(labelbottom=False) # , labelleft=False) +plt.title("Sea ice freeboard without waves", loc="left") -plt.plot(t, ice * ice_height, '.', c=col.cascade2,markersize = 0.5 ) -plt.fill_between(t, ice * ice_height, color=col.cascade2, alpha = 0.6) +plt.plot(t, ice * ice_height, ".", c=col.cascade2, markersize=0.5) +plt.fill_between(t, ice * ice_height, color=col.cascade2, alpha=0.6) plt.xlim(0, 300) plt.ylim(ylims[0], ylims[1]) ax3 = F.fig.add_subplot(gs[2, :]) -plt.title('Waves only', loc = 'left') +plt.title("Waves only", loc="left") -plt.plot(t, instance.mean(1) * 5 , '-', c=col.black,linewidth = 0.8 ) +plt.plot(t, instance.mean(1) * 5, "-", c=col.black, linewidth=0.8) plt.xlim(0, 300) -plt.ylim(ylims[0]*1.4, ylims[1]) -plt.xlabel('meters') +plt.ylim(ylims[0] * 1.4, ylims[1]) +plt.xlabel("meters") -ax1.axhline(0, linewidth =0.7, color = col.gridcolor) -ax2.axhline(0, linewidth =0.7, color = col.gridcolor) -ax3.axhline(0, linewidth =0.7, color = col.gridcolor) +ax1.axhline(0, linewidth=0.7, color=col.gridcolor) +ax2.axhline(0, linewidth=0.7, color=col.gridcolor) +ax3.axhline(0, linewidth=0.7, color=col.gridcolor) -F.save_pup(path = plot_path, name = "fake_height_model") +F.save_pup(path=plot_path, name="fake_height_model") # %% ICesat 2 track density? N_tracks = 1387 # 91 day repeat cycle, means 91/2 revisit, because of acending and descending. -Dtime = 91 * 24 # in hours +Dtime = 91 * 24 # in hours -91/2 -#*60 *60 # hours -#tacks on a 91-day repetitio cycle. +91 / 2 +# *60 *60 # hours +# tacks on a 91-day repetitio cycle. # circumface at 65 N -re= np.float(mconfig['constants']['radius'])/1e3 -circumface = np.cos(np.pi * 65/ 180) * 2* np.pi * re # km +re = np.float(mconfig["constants"]["radius"]) / 1e3 +circumface = np.cos(np.pi * 65 / 180) * 2 * np.pi * re # km -dx = circumface/N_tracks # km -dt = Dtime/N_tracks +dx = circumface / N_tracks # km +dt = Dtime / N_tracks dx, dt # study area width 40 deg lon: -area_width = circumface * 40/360 +area_width = circumface * 40 / 360 -sub_area_width = circumface * 5/360 +sub_area_width = circumface * 5 / 360 # time to pass study area once: dt * area_width / dx / 24 -sub_area_width/dx +sub_area_width / dx # %% diff --git a/analysis_fake_data/A01_1d_fake_surface_obs.py b/analysis_fake_data/A01_1d_fake_surface_obs.py index eb55e10a..265c1bb1 100644 --- a/analysis_fake_data/A01_1d_fake_surface_obs.py +++ b/analysis_fake_data/A01_1d_fake_surface_obs.py @@ -1,6 +1,7 @@ # %% import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -9,15 +10,16 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -#%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -32,18 +34,21 @@ import m_spectrum_ph3 as spec import JONSWAP_gamma -sys.path.append(base_path +'analysis_fake_data/') +sys.path.append(base_path + "analysis_fake_data/") import wave_realizations as WR + font_for_pres() # %% -f = np.arange(0.001, 0.2,0.001) -f_std =0.01 +f = np.arange(0.001, 0.2, 0.001) +f_std = 0.01 spec_power = JONSWAP_gamma.JONSWAP_default(f, 2e6, 10) -spec_power_gauss = np.max(spec_power) * np.exp( - ( (f -f[spec_power.argmax()])/ f_std )**2 ) -plt.plot(f, spec_power_gauss, label= 'gauss E= ' + str(np.trapz(spec_power_gauss, f))) -plt.plot(f, spec_power, label= 'JONSWAP E= ' + str(np.trapz(spec_power, f))) +spec_power_gauss = np.max(spec_power) * np.exp( + -(((f - f[spec_power.argmax()]) / f_std) ** 2) +) +plt.plot(f, spec_power_gauss, label="gauss E= " + str(np.trapz(spec_power_gauss, f))) +plt.plot(f, spec_power, label="JONSWAP E= " + str(np.trapz(spec_power, f))) plt.legend() @@ -63,17 +68,17 @@ plt.subplot(211) plt.title plt.plot(f, amps, linewidth=0.5) -plt.title('Spectrum', loc='left') -plt.xlabel('frequency [Hz]') -plt.ylabel('amplitude [m]') +plt.title("Spectrum", loc="left") +plt.xlabel("frequency [Hz]") +plt.ylabel("amplitude [m]") plt.subplot(212) -plt.plot(t, instance, label='realization') -plt.title('Realization', loc='left') +plt.plot(t, instance, label="realization") +plt.title("Realization", loc="left") plt.legend() -plt.ylabel('height [m]') -plt.xlabel('time [s]') +plt.ylabel("height [m]") +plt.xlabel("time [s]") WR.test_variance_conservations(f, spec_power_gauss, instance) @@ -81,8 +86,8 @@ # %% test wave number version imp.reload(WR) -x= np.arange(0, 1e4, 0.5) -k =(2 * np.pi * f)**2 / 9.81 +x = np.arange(0, 1e4, 0.5) +k = (2 * np.pi * f) ** 2 / 9.81 amps = WR.amplitude_from_wavenumber_spectrum(spec_power_gauss, k) instance = WR.space_realization_from_spectrum(x, k, spec_power_gauss) @@ -92,55 +97,55 @@ plt.subplot(211) plt.title plt.plot(f, amps, linewidth=0.5) -plt.title('Spectrum', loc='left') -plt.xlabel('wavenumber [k= 2pi/lambda]') -plt.ylabel('amplitude [m]') +plt.title("Spectrum", loc="left") +plt.xlabel("wavenumber [k= 2pi/lambda]") +plt.ylabel("amplitude [m]") plt.subplot(212) -plt.plot(x, instance, label='realization') -plt.title('Realization', loc='left') +plt.plot(x, instance, label="realization") +plt.title("Realization", loc="left") plt.legend() -plt.ylabel('height [m]') -plt.xlabel('space [m]') +plt.ylabel("height [m]") +plt.xlabel("space [m]") -WR.test_variance_conservations(k, spec_power_gauss, instance, wave_number=True) +WR.test_variance_conservations(k, spec_power_gauss, instance, wave_number=True) # %% -C= -3 -k = 2* np.pi / 200 +C = -3 +k = 2 * np.pi / 200 x = np.arange(0, 2000, 2) -phi = np.pi *3/2 -instance = C* np.cos(k * x + phi ) -plt.plot(x, instance, '.') +phi = np.pi * 3 / 2 +instance = C * np.cos(k * x + phi) +plt.plot(x, instance, ".") -A = C * np.cos(phi) -B = - C * np.sin(phi) +A = C * np.cos(phi) +B = -C * np.sin(phi) -instance2 = A * np.cos(k * x) + B * np.sin(k * x) -plt.plot(x, instance2) +instance2 = A * np.cos(k * x) + B * np.sin(k * x) +plt.plot(x, instance2) R = np.sqrt(A**2 + B**2) phi2 = np.arctan2(-B, A) -instance3 = R* np.cos(k * x + phi2 ) -plt.plot(x, instance3, '+') +instance3 = R * np.cos(k * x + phi2) +plt.plot(x, instance3, "+") # %% -C= -3 -k = 2* np.pi / 200 +C = -3 +k = 2 * np.pi / 200 x = np.arange(0, 4000, 2) -phi = np.pi *3/2 -instance = C* np.cos(k * x + phi ) -plt.plot(x, instance, '-') +phi = np.pi * 3 / 2 +instance = C * np.cos(k * x + phi) +plt.plot(x, instance, "-") -instance2 = C* np.cos(k*1.05 * x + phi + np.pi*1.5 ) -plt.plot(x, instance2, '-') +instance2 = C * np.cos(k * 1.05 * x + phi + np.pi * 1.5) +plt.plot(x, instance2, "-") -plt.plot(x, instance + instance2, '-k') +plt.plot(x, instance + instance2, "-k") diff --git a/analysis_fake_data/S01_1D_fake_data_variance_conservation.py b/analysis_fake_data/S01_1D_fake_data_variance_conservation.py index 1fc3025a..edff728b 100644 --- a/analysis_fake_data/S01_1D_fake_data_variance_conservation.py +++ b/analysis_fake_data/S01_1D_fake_data_variance_conservation.py @@ -1,29 +1,32 @@ # %% import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file test the wave realizations and the spectral estimates and the variance conservation between GFT, FFT, and real space. """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.spectral_estimates as spec import imp import copy import spicke_remover -#import datetime + +# import datetime import generalized_FT as gFT from scipy.ndimage.measurements import label -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'analysis_fake_data/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "analysis_fake_data/") import wave_realizations as WR import JONSWAP_gamma -#import s3fs + +# import s3fs # %% # load_path = mconfig['paths']['work'] +'/B01_regrid_'+hemis+'/' # load_file = load_path + 'processed_' + ATlevel + '_' + track_name + '.h5' @@ -40,81 +43,89 @@ # %% create fake 1D data # define spectral power and its reaalization imp.reload(WR) -f = np.arange(0.001, 0.2,0.001) # frequency range -x_obs= np.arange(0, 3e5, 0.5) # space range -k =(2 * np.pi * f)**2 / 9.81 # wavenumber range +f = np.arange(0.001, 0.2, 0.001) # frequency range +x_obs = np.arange(0, 3e5, 0.5) # space range +k = (2 * np.pi * f) ** 2 / 9.81 # wavenumber range -f_std =0.005 +f_std = 0.005 spec_power = JONSWAP_gamma.JONSWAP_default(f, 1e6, 10) -spec_power_gauss = np.max(spec_power) * np.exp( - ( (f -f[spec_power.argmax()])/ f_std )**2 ) +spec_power_gauss = np.max(spec_power) * np.exp( + -(((f - f[spec_power.argmax()]) / f_std) ** 2) +) amps = WR.amplitude_from_wavenumber_spectrum(spec_power_gauss, k) instance = WR.space_realization_from_spectrum(x_obs, k, spec_power_gauss) -WR.test_variance_conservations(f, spec_power_gauss, instance ) +WR.test_variance_conservations(f, spec_power_gauss, instance) M.figure_axis_xy(5.5, 5.5, view_scale=0.9) plt.subplot(311) plt.plot(k, amps, linewidth=0.5) -plt.title('Amplitude Spectrum', loc='left') -plt.xlabel('wavenumber [k= 2pi/lambda]') -plt.ylabel('amplitude [m]') +plt.title("Amplitude Spectrum", loc="left") +plt.xlabel("wavenumber [k= 2pi/lambda]") +plt.ylabel("amplitude [m]") plt.subplot(312) -plt.plot(x_obs/1e3, instance, label='realization') -plt.title('Realization', loc='left') +plt.plot(x_obs / 1e3, instance, label="realization") +plt.title("Realization", loc="left") plt.legend() -plt.ylabel('height [m]') -plt.xlabel('space [km]') +plt.ylabel("height [m]") +plt.xlabel("space [km]") plt.subplot(313) -plt.plot(x_obs[0:12000], instance[0:12000], label='realization') -plt.title('Realization', loc='left') +plt.plot(x_obs[0:12000], instance[0:12000], label="realization") +plt.title("Realization", loc="left") plt.legend() -plt.ylabel('height [m]') -plt.xlabel('space [m]') +plt.ylabel("height [m]") +plt.xlabel("space [m]") -WR.test_variance_conservations(k, spec_power_gauss, instance, wave_number=True) +WR.test_variance_conservations(k, spec_power_gauss, instance, wave_number=True) # %% Make standard DFT estimate and test variance conservation import spectral_estimates as spec -instance2= instance[0:3000] + +instance2 = instance[0:3000] x2 = x_obs[0:3000] -Lpoints= len(instance2) +Lpoints = len(instance2) kk, dkk = spec.calc_freq_fft(x2, Lpoints) -kk, dkk = 2 * np.pi *kk, 2 * np.pi * dkk +kk, dkk = 2 * np.pi * kk, 2 * np.pi * dkk spec_FFT = spec.calc_spectrum_fft(instance2, dkk, len(instance2)) -S = spec.wavenumber_spectrogram(x2, instance2, 12000 , window='hann') +S = spec.wavenumber_spectrogram(x2, instance2, 12000, window="hann") -spec_FFT_welch =S.cal_spectrogram().mean('x') +spec_FFT_welch = S.cal_spectrogram().mean("x") S.parceval() M.figure_axis_xy(5.5, 5.5, view_scale=0.9) -#plt.plot(kk, spec_FFT) -plt.plot( spec_FFT_welch.k, spec_FFT_welch ) +# plt.plot(kk, spec_FFT) +plt.plot(spec_FFT_welch.k, spec_FFT_welch) -print('variance of the initial spectrum', np.trapz(spec_power_gauss, k)) -plt.plot(k, spec_power_gauss, linewidth=0.8, color='k', label='Input data') +print("variance of the initial spectrum", np.trapz(spec_power_gauss, k)) +plt.plot(k, spec_power_gauss, linewidth=0.8, color="k", label="Input data") plt.xlim(0, 0.05) # %% GFT inverse of a single case -instance2= instance[0:3000] +instance2 = instance[0:3000] x2 = x_obs[0:3000] -Lpoints= len(instance2) +Lpoints = len(instance2) Lmeters = x2[-1] - x2[0] kk, dkk = spec.calc_freq_fft(x2, Lpoints) -kk, dkk = 2 * np.pi *kk, 2 * np.pi * dkk +kk, dkk = 2 * np.pi * kk, 2 * np.pi * dkk -dk_GFT = dkk* 1 -kk_GFT= np.arange(kk[0], kk[-1], dk_GFT) +dk_GFT = dkk * 1 +kk_GFT = np.arange(kk[0], kk[-1], dk_GFT) import generalized_FT as gFT + G = gFT.generalized_Fourier(x2, instance2, kk_GFT) -weight = ( np.interp(kk_GFT, k, spec_power_gauss) /spec_power_gauss.max()) * 0.8 * instance2.var() +weight = ( + (np.interp(kk_GFT, k, spec_power_gauss) / spec_power_gauss.max()) + * 0.8 + * instance2.var() +) err = instance2 * 0 + 0.1 G.define_problem(weight, err) @@ -125,18 +136,18 @@ M.figure_axis_xy(5.5, 5.5, view_scale=0.9) plt.subplot(211) -plt.plot(x2, instance2, label='realization') -plt.plot(x2, GFT_model, label='GFT model') +plt.plot(x2, instance2, label="realization") +plt.plot(x2, GFT_model, label="GFT model") # %% test of different spectral estimate calculations font_for_pres() MM = len(kk_GFT) Nx = len(x2) -Mtwo = 2 *MM +Mtwo = 2 * MM -Z = (p_hat[0:MM] - p_hat[MM:] *1j) * (Nx/2+1) +Z = (p_hat[0:MM] - p_hat[MM:] * 1j) * (Nx / 2 + 1) -DFT = np.fft.rfft(instance2) +DFT = np.fft.rfft(instance2) M.figure_axis_xy(5.5, 5.5, view_scale=0.6) plt.subplot(211) @@ -144,181 +155,195 @@ plt.plot(kk[0:25], DFT[0:25].real) plt.plot(kk[0:25], DFT[0:25].imag) -plt.plot(kk_GFT[0:50], Z[0:50].real, '-*') -plt.plot(kk_GFT[0:50], Z[0:50].imag, '-*') +plt.plot(kk_GFT[0:50], Z[0:50].real, "-*") +plt.plot(kk_GFT[0:50], Z[0:50].imag, "-*") plt.subplot(212) -spec_DFT = 2.*(DFT*DFT.conj()).real / dkk /Nx**2 -plt.plot(kk[0:50], spec_DFT[0:50], label='DFT') +spec_DFT = 2.0 * (DFT * DFT.conj()).real / dkk / Nx**2 +plt.plot(kk[0:50], spec_DFT[0:50], label="DFT") -spec_GFT = ( 2.*(Z*Z.conj()).real / dk_GFT /Nx**2 ) #/ ( Nx / Mtwo ) -plt.plot(kk_GFT[0:100], spec_GFT[0:100], '*', label='GFT') +spec_GFT = 2.0 * (Z * Z.conj()).real / dk_GFT / Nx**2 # / ( Nx / Mtwo ) +plt.plot(kk_GFT[0:100], spec_GFT[0:100], "*", label="GFT") -dk_fake = 2 * np.pi/ Lmeters -spec_GFT2 = 2.*(Z*Z.conj()).real / dk_fake /Nx**2 -plt.plot(kk_GFT[0:100], spec_GFT2[0:100], '-+', label='GFT') +dk_fake = 2 * np.pi / Lmeters +spec_GFT2 = 2.0 * (Z * Z.conj()).real / dk_fake / Nx**2 +plt.plot(kk_GFT[0:100], spec_GFT2[0:100], "-+", label="GFT") -Z = (p_hat[0:MM] - p_hat[MM:] *1j) * (MM/2+1) -#dk_fake = 2 * np.pi/ Lmeters -spec_GFT3 = 2.*(Z*Z.conj()).real / dk_GFT /MM**2 -plt.plot(kk_GFT[0:100], spec_GFT3[0:100], '--', label='GFT') +Z = (p_hat[0:MM] - p_hat[MM:] * 1j) * (MM / 2 + 1) +# dk_fake = 2 * np.pi/ Lmeters +spec_GFT3 = 2.0 * (Z * Z.conj()).real / dk_GFT / MM**2 +plt.plot(kk_GFT[0:100], spec_GFT3[0:100], "--", label="GFT") # test variance conservation -#print('variance of the initial spectrum', np.trapz(spec_power_gauss, k)) -print('variance of the data', instance2.var()) -print('variance of the DFT spectrum', np.trapz(spec_DFT, kk)) -print('-----------') -print('variance of the GFT model', GFT_model.var()) -print('variance of the GFT spectrum', np.trapz(spec_GFT, kk_GFT)) -print('variance of the GFT2 spectrum', np.trapz(spec_GFT2, kk_GFT)) -print('variance of the GFT3 spectrum', np.trapz(spec_GFT3, kk_GFT)) -print('2M', str(2* MM) , 'N', str(Nx) ) +# print('variance of the initial spectrum', np.trapz(spec_power_gauss, k)) +print("variance of the data", instance2.var()) +print("variance of the DFT spectrum", np.trapz(spec_DFT, kk)) +print("-----------") +print("variance of the GFT model", GFT_model.var()) +print("variance of the GFT spectrum", np.trapz(spec_GFT, kk_GFT)) +print("variance of the GFT2 spectrum", np.trapz(spec_GFT2, kk_GFT)) +print("variance of the GFT3 spectrum", np.trapz(spec_GFT3, kk_GFT)) +print("2M", str(2 * MM), "N", str(Nx)) # %% Define limits of the GFT wave numbers + # derive spectal limits # Longest deserved period: def define_minimal_datapoints_from_period(T0, x): - k_0 = (2 * np.pi/ T0)**2 / 9.81 - dx = np.diff(x).mean() - lambda_0 = 2 * np.pi/ k_0 - min_datapoint = lambda_0/dx + k_0 = (2 * np.pi / T0) ** 2 / 9.81 + dx = np.diff(x).mean() + lambda_0 = 2 * np.pi / k_0 + min_datapoint = lambda_0 / dx return int(np.ceil(min_datapoint)), k_0, dx + def define_minimal_datapoints_from_wavenumber(k0, x): - lambda_min = 2 * np.pi/ k0 - dx = np.diff(x).mean() - min_datapoint = lambda_min/dx + lambda_min = 2 * np.pi / k0 + dx = np.diff(x).mean() + min_datapoint = lambda_min / dx return int(np.ceil(min_datapoint)), k0, dx + min_datapoint, k_0, dx = define_minimal_datapoints_from_period(20, x_obs) -Lpoints = min_datapoint * 10 -Lmeters = Lpoints * dx +Lpoints = min_datapoint * 10 +Lmeters = Lpoints * dx -#plt.plot(np.diff(np.array(Gi['dist']))) -print('L number of gridpoint:', Lpoints) -print('L length in km:', Lmeters/1e3) -print('approx number windows', 2* x_obs[-1] /Lmeters-1 ) +# plt.plot(np.diff(np.array(Gi['dist']))) +print("L number of gridpoint:", Lpoints) +print("L length in km:", Lmeters / 1e3) +print("approx number windows", 2 * x_obs[-1] / Lmeters - 1) -T_min = 8 -lambda_min = 9.81 * T_min**2/ (2 *np.pi) -flim = 1/T_min +T_min = 8 +lambda_min = 9.81 * T_min**2 / (2 * np.pi) +flim = 1 / T_min -#2* np.pi/dx +# 2* np.pi/dx -oversample = 4 -dlambda = Lmeters * oversample -dk = 2 * np.pi/ dlambda -kk = np.arange(0, 1/lambda_min, 1/dlambda) * 2*np.pi -kk = kk[k_0<=kk] -#dk = np.diff(kk).mean() +oversample = 4 +dlambda = Lmeters * oversample +dk = 2 * np.pi / dlambda +kk = np.arange(0, 1 / lambda_min, 1 / dlambda) * 2 * np.pi +kk = kk[k_0 <= kk] +# dk = np.diff(kk).mean() -dk_adjustment = (np.pi *2 ) / (dk * Lmeters ) +dk_adjustment = (np.pi * 2) / (dk * Lmeters) -print('oversample ', oversample) -print('2 M = ', kk.size *2 ) -print('N = ', Lpoints) -#print('2M / N = ', (kk.size *2 ) / Lpoints ) -print('N / 2M = ', Lpoints / (kk.size *2 ) ) -print('2M / N ', (kk.size *2 ) / Lmeters ) -print('Lmeter / 2M ', Lmeters / (kk.size *2 ) ) -print('dk = ', dk) -print('dk_adjustment = ', dk_adjustment) +print("oversample ", oversample) +print("2 M = ", kk.size * 2) +print("N = ", Lpoints) +# print('2M / N = ', (kk.size *2 ) / Lpoints ) +print("N / 2M = ", Lpoints / (kk.size * 2)) +print("2M / N ", (kk.size * 2) / Lmeters) +print("Lmeter / 2M ", Lmeters / (kk.size * 2)) +print("dk = ", dk) +print("dk_adjustment = ", dk_adjustment) imp.reload(spec) -x = x_obs#[0:Lpoints*4] -dd = instance#[0:Lpoints*4] +x = x_obs # [0:Lpoints*4] +dd = instance # [0:Lpoints*4] -xlims = x[0], x[-1] +xlims = x[0], x[-1] (xlims[1] - xlims[0]) / Lmeters -dd_error = dd * 0 + 1 #np.copy(Gd[k]['heights_c_std']) -#dd_error[np.isnan(dd_error)] = 100 +dd_error = dd * 0 + 1 # np.copy(Gd[k]['heights_c_std']) +# dd_error[np.isnan(dd_error)] = 100 np.std(dd_error) -#sum(np.isnan(dd_error)) -#plt.hist(1/dd_weight, bins=40) +# sum(np.isnan(dd_error)) +# plt.hist(1/dd_weight, bins=40) # F = M.figure_axis_xy(6, 3) # plt.subplot(2, 1, 1) # plt.plot(x, dd, 'gray', label='displacement (m) ') # compute slope spectra !! -dd = dd #np.gradient(dd) -dd, _ = spicke_remover.spicke_remover(dd, spreed=10, verbose=False) -dd_nans = (np.isnan(dd) )#)# + (Gd[k]['N_photos'] <= 5) +dd = dd # np.gradient(dd) +dd, _ = spicke_remover.spicke_remover(dd, spreed=10, verbose=False) +dd_nans = np.isnan(dd) # )# + (Gd[k]['N_photos'] <= 5) # using gappy data -dd_no_nans = dd[~dd_nans] # windowing is applied here -x_no_nans = x[~dd_nans] +dd_no_nans = dd[~dd_nans] # windowing is applied here +x_no_nans = x[~dd_nans] dd_error_no_nans = dd_error[~dd_nans] -#plt.subplot(2, 1, 2) +# plt.subplot(2, 1, 2) -#plt.plot(x_no_nans, dd_no_nans, 'black', label='slope (m/m)') -#plt.legend() -#plt.show() +# plt.plot(x_no_nans, dd_no_nans, 'black', label='slope (m/m)') +# plt.legend() +# plt.show() -#xlims = xlims[0], xlims[0] + (xlims[1] -xlims[0])/2 +# xlims = xlims[0], xlims[0] + (xlims[1] -xlims[0])/2 # %% -print('gFT') +print("gFT") font_for_print() -#S_pwelch_k2 = np.arange(S_pwelch_k[1], S_pwelch_k[-1], S_pwelch_dk*2 ) +# S_pwelch_k2 = np.arange(S_pwelch_k[1], S_pwelch_k[-1], S_pwelch_dk*2 ) imp.reload(gFT) -S = gFT.wavenumber_spectrogram_gFT( np.array(x_no_nans), np.array(dd_no_nans), Lmeters, dx, kk, data_error = dd_error_no_nans , ov=None) +S = gFT.wavenumber_spectrogram_gFT( + np.array(x_no_nans), + np.array(dd_no_nans), + Lmeters, + dx, + kk, + data_error=dd_error_no_nans, + ov=None, +) -GG, GG_x, Params = S.cal_spectrogram(xlims= xlims, max_nfev = None, plot_flag = True) +GG, GG_x, Params = S.cal_spectrogram(xlims=xlims, max_nfev=None, plot_flag=True) M.figure_axis_xy(4.5, 4.5) -ave_data_var=GG_x.y_data.var('eta').mean().data +ave_data_var = GG_x.y_data.var("eta").mean().data font_for_pres() # np.nanmean( (GG.gFT_PSD_model.sum('k') *dk) ) # (dk_adjustment* GG.gFT_PSD_model.mean('x')).plot(marker = '+', label='GFT model') # (dk_adjustment* GG.gFT_PSD_data.mean('x')).plot(label='GFT data') -(GG.gFT_PSD_model.mean('x')).plot(marker = '+', label='GFT model') -(GG.gFT_PSD_data.mean('x')).plot(label='GFT data') +(GG.gFT_PSD_model.mean("x")).plot(marker="+", label="GFT model") +(GG.gFT_PSD_data.mean("x")).plot(label="GFT data") # GG.gFT_PSD_data.median('x').rolling(k= 5).mean().plot(label='GFT data') # GG.gFT_PSD_model.median('x').rolling(k= 5).mean().plot(label='GFT model') -plt.plot(k, spec_power_gauss, linewidth=0.8, color='k', label='Input data') -plt.title('Power Spectrum', loc='left') -plt.xlabel('wavenumber [k= 2pi/lambda]') -plt.ylabel('amplitude [m^2/k]') +plt.plot(k, spec_power_gauss, linewidth=0.8, color="k", label="Input data") +plt.title("Power Spectrum", loc="left") +plt.xlabel("wavenumber [k= 2pi/lambda]") +plt.ylabel("amplitude [m^2/k]") plt.xlim(k[0], 0.05) plt.legend() -print('data energy / mean model energy = ' , np.array(dd_no_nans).var() / ( GG.gFT_PSD_model.sum('k') *dk ).mean('x').data ) +print( + "data energy / mean model energy = ", + np.array(dd_no_nans).var() / (GG.gFT_PSD_model.sum("k") * dk).mean("x").data, +) -#plt.plot( spec_power_gauss / GG.gFT_PSD_model.median('x').interp(k= k) ) +# plt.plot( spec_power_gauss / GG.gFT_PSD_model.median('x').interp(k= k) ) GG.spec_adjust # %% -GGi = GG.isel(x= 1) -GGxi = GG_x.isel(x= 1) -spec_man = GGi.gFT_cos_coeff**2 + GGi.gFT_sin_coeff **2 +GGi = GG.isel(x=1) +GGxi = GG_x.isel(x=1) +spec_man = GGi.gFT_cos_coeff**2 + GGi.gFT_sin_coeff**2 np.trapz(spec_man, GGi.k) -GGxi.y_data.var('eta').data / (GGi.gFT_PSD_model.sum('k') *dk ).data +GGxi.y_data.var("eta").data / (GGi.gFT_PSD_model.sum("k") * dk).data + # %% def linear_gap_fill(F, key_lead, key_int): - """ F pd.DataFrame key_lead key in F that determined the independent coordindate @@ -326,8 +351,8 @@ def linear_gap_fill(F, key_lead, key_int): """ y_g = np.array(F[key_int]) - nans, x2= np.isnan(y_g), lambda z: z.nonzero()[0] - y_g[nans]= np.interp(x2(nans), x2(~nans), y_g[~nans]) + nans, x2 = np.isnan(y_g), lambda z: z.nonzero()[0] + y_g[nans] = np.interp(x2(nans), x2(~nans), y_g[~nans]) return y_g @@ -335,64 +360,74 @@ def linear_gap_fill(F, key_lead, key_int): font_for_pres() # plot_data_model=True # if plot_data_model: -for i in np.arange(1,2,1): - c1= 'blue' - c2= 'red' +for i in np.arange(1, 2, 1): + c1 = "blue" + c2 = "red" - GGi = GG.isel(x= i) + GGi = GG.isel(x=i) - xi_1=GG_x.x[i] - xi_2=GG_x.x[i+1] - #if k%2 ==0: + xi_1 = GG_x.x[i] + xi_2 = GG_x.x[i + 1] + # if k%2 ==0: F = M.figure_axis_xy(16, 2) eta = GG_x.eta # gFT model y_model = GG_x.squeeze().y_model[:, i] - plt.plot(eta +xi_1, y_model ,'-', c=c1, linewidth=0.8, alpha=1, zorder=12) - y_model = GG_x.squeeze().y_model[:, i+1] - plt.plot(eta +xi_2, y_model,'-', c=c2, linewidth=0.8, alpha=1, zorder=12) + plt.plot(eta + xi_1, y_model, "-", c=c1, linewidth=0.8, alpha=1, zorder=12) + y_model = GG_x.squeeze().y_model[:, i + 1] + plt.plot(eta + xi_2, y_model, "-", c=c2, linewidth=0.8, alpha=1, zorder=12) # iterpolated model in gaps - FT = gFT.generalized_Fourier(eta +xi_1, None,GG.k ) + FT = gFT.generalized_Fourier(eta + xi_1, None, GG.k) _ = FT.get_H() - FT.p_hat=np.concatenate([ GGi.gFT_cos_coeff, GGi.gFT_sin_coeff ]) + FT.p_hat = np.concatenate([GGi.gFT_cos_coeff, GGi.gFT_sin_coeff]) # FT.ydata_std = GG_x.y_data.std() # FT.ydata_mean = GG_x.y_data.mean() - plt.plot(eta +xi_1, FT.model() ,'-', c='orange', linewidth=0.8, alpha=1,zorder= 2) + plt.plot(eta + xi_1, FT.model(), "-", c="orange", linewidth=0.8, alpha=1, zorder=2) - FT = gFT.generalized_Fourier(eta +xi_2, None,GG.k ) + FT = gFT.generalized_Fourier(eta + xi_2, None, GG.k) _ = FT.get_H() - FT.p_hat=np.concatenate([ GGi.gFT_cos_coeff, GGi.gFT_sin_coeff ]) - plt.plot(eta +xi_2, FT.model() ,'-', c='orange', linewidth=0.8, alpha=1,zorder= 2) - + FT.p_hat = np.concatenate([GGi.gFT_cos_coeff, GGi.gFT_sin_coeff]) + plt.plot(eta + xi_2, FT.model(), "-", c="orange", linewidth=0.8, alpha=1, zorder=2) # oringial data - #plt.plot(x, dd, '-', c='k',linewidth=3, alpha =0.6, zorder=11) - #plt.plot(x, dd, '.', c='k',markersize=3, alpha =0.5) + # plt.plot(x, dd, '-', c='k',linewidth=3, alpha =0.6, zorder=11) + # plt.plot(x, dd, '.', c='k',markersize=3, alpha =0.5) # oringal data from model_output y_data = GG_x.y_data[:, i] - plt.plot(eta +xi_1, y_data ,'-', c='k',linewidth=3, alpha =0.3, zorder=11) - y_data = GG_x.y_data[:, i+1] - plt.plot(eta +xi_2, y_data,'-', c='k',linewidth=3, alpha =0.3, zorder=11) - - - #plt.plot(x[~dd_nans], dd_error[~dd_nans], '.', c='green',linewidth=1, alpha =0.5) - - F.ax.axvline(xi_1 + eta[0].data , linewidth=4, color=c1, alpha=0.5) - F.ax.axvline(xi_1 + eta[-1].data, linewidth=4, color=c1, alpha=0.5) - F.ax.axvline(xi_2 + eta[0].data , linewidth=4, color=c2, alpha=0.5) - F.ax.axvline(xi_2 + eta[-1].data, linewidth=4, color=c2, alpha=0.5) - - ylims= -np.nanstd(dd)*2, np.nanstd(dd)*2 - plt.text(xi_1 + eta[0].data, ylims[-1], ' N='+ str(GG.sel(x=xi_1, method='nearest').N_per_stancil.data) + ' N/2M= '+ str(GG.sel(x=xi_1, method='nearest').N_per_stancil.data/2/kk.size) ) - plt.text(xi_2 + eta[0].data, ylims[-1], ' N='+ str(GG.sel(x=xi_2, method='nearest').N_per_stancil.data) + ' N/2M= '+ str(GG.sel(x=xi_2, method='nearest').N_per_stancil.data/2/kk.size) ) - plt.xlim(xi_1 + eta[0].data*1.2, xi_2 + eta[-1].data*1.2 ) - #plt.xlim(xi_1, xi_2 ) - + plt.plot(eta + xi_1, y_data, "-", c="k", linewidth=3, alpha=0.3, zorder=11) + y_data = GG_x.y_data[:, i + 1] + plt.plot(eta + xi_2, y_data, "-", c="k", linewidth=3, alpha=0.3, zorder=11) + + # plt.plot(x[~dd_nans], dd_error[~dd_nans], '.', c='green',linewidth=1, alpha =0.5) + + F.ax.axvline(xi_1 + eta[0].data, linewidth=4, color=c1, alpha=0.5) + F.ax.axvline(xi_1 + eta[-1].data, linewidth=4, color=c1, alpha=0.5) + F.ax.axvline(xi_2 + eta[0].data, linewidth=4, color=c2, alpha=0.5) + F.ax.axvline(xi_2 + eta[-1].data, linewidth=4, color=c2, alpha=0.5) + + ylims = -np.nanstd(dd) * 2, np.nanstd(dd) * 2 + plt.text( + xi_1 + eta[0].data, + ylims[-1], + " N=" + + str(GG.sel(x=xi_1, method="nearest").N_per_stancil.data) + + " N/2M= " + + str(GG.sel(x=xi_1, method="nearest").N_per_stancil.data / 2 / kk.size), + ) + plt.text( + xi_2 + eta[0].data, + ylims[-1], + " N=" + + str(GG.sel(x=xi_2, method="nearest").N_per_stancil.data) + + " N/2M= " + + str(GG.sel(x=xi_2, method="nearest").N_per_stancil.data / 2 / kk.size), + ) + plt.xlim(xi_1 + eta[0].data * 1.2, xi_2 + eta[-1].data * 1.2) + # plt.xlim(xi_1, xi_2 ) plt.ylim(ylims[0], ylims[-1]) plt.show() - diff --git a/analysis_fake_data/wave_realizations.py b/analysis_fake_data/wave_realizations.py index 79130b8e..23bc49b8 100644 --- a/analysis_fake_data/wave_realizations.py +++ b/analysis_fake_data/wave_realizations.py @@ -1,4 +1,5 @@ -import numpy as np +import numpy as np + def amplitudes_from_spectrum(spec, freq): """ @@ -9,7 +10,8 @@ def amplitudes_from_spectrum(spec, freq): returns: amps: amplitude (m) per frequency """ - return ( 2 * spec * np.gradient(freq)) **0.5 + return (2 * spec * np.gradient(freq)) ** 0.5 + def amplitude_from_wavenumber_spectrum(spec, wavenumber): """ @@ -18,9 +20,10 @@ def amplitude_from_wavenumber_spectrum(spec, wavenumber): spec: spectrum power (m^2/Hz) wavenumber: wavenumber (2 pi /m) """ - return ( 2* spec * np.gradient(wavenumber)) **0.5 + return (2 * spec * np.gradient(wavenumber)) ** 0.5 + -def time_realization_from_spectrum( T, f, spec_power): +def time_realization_from_spectrum(T, f, spec_power): """ returns a realization of a random process with a given spectrum input: @@ -31,11 +34,12 @@ def time_realization_from_spectrum( T, f, spec_power): returns: instance: realization of a random process with a given spectrum """ - tt, ww = np.meshgrid(2* np.pi * f, T) - phi = np.random.random(len(spec_power))*2*np.pi - return (amplitudes_from_spectrum(spec_power, f)* np.cos( ww * tt + phi )).sum(1) + tt, ww = np.meshgrid(2 * np.pi * f, T) + phi = np.random.random(len(spec_power)) * 2 * np.pi + return (amplitudes_from_spectrum(spec_power, f) * np.cos(ww * tt + phi)).sum(1) -def space_realization_from_spectrum( x, k, spec_power): + +def space_realization_from_spectrum(x, k, spec_power): """ returns a realization of a random process with a given spectrum input: @@ -43,11 +47,13 @@ def space_realization_from_spectrum( x, k, spec_power): k: wavenumber vector (2 pi /m) spec_power: spectrum power (m^2/Hz) returns: - instance: realization of a random process with a given spectrum + instance: realization of a random process with a given spectrum """ kk, xx = np.meshgrid(k, x) - phi = np.random.random(len(spec_power))*2*np.pi - return (amplitude_from_wavenumber_spectrum(spec_power, k)* np.cos( kk * xx + phi )).sum(1) + phi = np.random.random(len(spec_power)) * 2 * np.pi + return ( + amplitude_from_wavenumber_spectrum(spec_power, k) * np.cos(kk * xx + phi) + ).sum(1) def test_variance_conservations(f, spec, realization, wave_number=False): @@ -63,4 +69,11 @@ def test_variance_conservations(f, spec, realization, wave_number=False): spectral_energy = np.trapz(spec, f) else: spectral_energy = np.trapz(spec, f) / (2 * np.pi) - return print("Energy of realization", np.var(realization), "\nEnergy of spectrum", spectral_energy, "\nDifference ", np.round(np.var(realization) - spectral_energy , 5 )) + return print( + "Energy of realization", + np.var(realization), + "\nEnergy of spectrum", + spectral_energy, + "\nDifference ", + np.round(np.var(realization) - spectral_energy, 5), + ) diff --git a/config/2021_IceSAT2_startup_local.py b/config/2021_IceSAT2_startup_local.py index 5b27eb7e..020ef2ab 100644 --- a/config/2021_IceSAT2_startup_local.py +++ b/config/2021_IceSAT2_startup_local.py @@ -1,10 +1,12 @@ import os -#os.environ["DISPLAY"] = "localhost:10.0" + +# os.environ["DISPLAY"] = "localhost:10.0" # 14, 16, work -#standart libraries: +# standart libraries: import numpy as np import matplotlib -#matplotlib.use('Agg') + +# matplotlib.use('Agg') import matplotlib.pyplot as plt import pandas as pd @@ -12,8 +14,9 @@ import string import xarray as xr -#xr.set_options(display_width=80, display_style='text') -xr.set_options(display_style='text') + +# xr.set_options(display_width=80, display_style='text') +xr.set_options(display_style="text") import sys import imp @@ -21,136 +24,146 @@ import string # my own libraries: -#import m_general as M +# import m_general as M -#import AA_plot_base as AA + +# import AA_plot_base as AA def json_load(name, path, verbose=False): import json - full_name= (os.path.join(path,name+ '.json')) - with open(full_name, 'r') as ifile: - data=json.load(ifile) + full_name = os.path.join(path, name + ".json") + + with open(full_name, "r") as ifile: + data = json.load(ifile) if verbose: - print('loaded from: ',full_name) + print("loaded from: ", full_name) return data -mconfig=json_load('config_local','/Users/Shared/Projects/2021_IceSAT2_tracks/config/') + +mconfig = json_load( + "config_local", "/Users/Shared/Projects/2021_IceSAT2_tracks/config/" +) # add project depenent libraries -sys.path.append(mconfig['paths']['local_script']) -sys.path.append(mconfig['paths']['local_script'] +'/ICEsat2_SI_tools/') +sys.path.append(mconfig["paths"]["local_script"]) +sys.path.append(mconfig["paths"]["local_script"] + "/ICEsat2_SI_tools/") import m_colormanager_ph3 as M_color import m_tools_ph3 as MT import m_general_ph3 as M -#load colorscheme -col=M_color.color(path=mconfig['paths']['analysis']+'../config/', name='color_def') +# load colorscheme +col = M_color.color(path=mconfig["paths"]["analysis"] + "../config/", name="color_def") -lstrings =iter([i+') ' for i in list(string.ascii_lowercase)]) +lstrings = iter([i + ") " for i in list(string.ascii_lowercase)]) # define journal fig sizes -fig_sizes = mconfig['fig_sizes']['AMS'] +fig_sizes = mconfig["fig_sizes"]["AMS"] SMALL_SIZE = 8 MEDIUM_SIZE = 10 BIGGER_SIZE = 12 -#csfont = {'fontname':'Comic Sans MS'} -legend_properties = {'weight':'bold'} -#font.family: sans-serif -#font.sans-serif: Helvetica Neue - -#import matplotlib.font_manager as font_manager -#font_dirs = ['/home/mhell/HelveticaNeue/', ] -#font_files = font_manager.findSystemFonts(fontpaths=font_dirs) -#font_list = font_manager.createFontList(font_files) -#font_manager.fontManager.ttflist.extend(font_list) - -plt.rc('font', size=SMALL_SIZE, serif='Helvetica Neue', weight='normal') # controls default text sizes -#plt.rc('font', size=SMALL_SIZE, serif='DejaVu Sans', weight='light') -plt.rc('text', usetex='false') -plt.rc('axes', titlesize=MEDIUM_SIZE, labelweight='normal') # fontsize of the axes title -plt.rc('axes', labelsize=SMALL_SIZE, labelweight='normal') #, family='bold') # fontsize of the x and y labels -plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels -plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels -plt.rc('legend', fontsize=SMALL_SIZE, frameon=False) # legend fontsize -plt.rc('figure', titlesize=MEDIUM_SIZE, titleweight='bold', autolayout=True) #, family='bold') # fontsize of the figure title - -#figure.autolayout : False -#matplotlib.rcParams['pdf.fonttype'] = 42 -#matplotlib.rcParams['ps.fonttype'] = 42 - - -plt.rc('path', simplify=True) - -plt.rcParams['figure.figsize'] = (10, 8)#(20.0, 10.0) #inline -#plt.rcParams['pcolor.shading'] = 'auto' -#rcParams['pcolor.shading'] = 'auto' -#plt.rc('pcolor', shading = 'auto') +# csfont = {'fontname':'Comic Sans MS'} +legend_properties = {"weight": "bold"} +# font.family: sans-serif +# font.sans-serif: Helvetica Neue + +# import matplotlib.font_manager as font_manager +# font_dirs = ['/home/mhell/HelveticaNeue/', ] +# font_files = font_manager.findSystemFonts(fontpaths=font_dirs) +# font_list = font_manager.createFontList(font_files) +# font_manager.fontManager.ttflist.extend(font_list) + +plt.rc( + "font", size=SMALL_SIZE, serif="Helvetica Neue", weight="normal" +) # controls default text sizes +# plt.rc('font', size=SMALL_SIZE, serif='DejaVu Sans', weight='light') +plt.rc("text", usetex="false") +plt.rc( + "axes", titlesize=MEDIUM_SIZE, labelweight="normal" +) # fontsize of the axes title +plt.rc( + "axes", labelsize=SMALL_SIZE, labelweight="normal" +) # , family='bold') # fontsize of the x and y labels +plt.rc("xtick", labelsize=SMALL_SIZE) # fontsize of the tick labels +plt.rc("ytick", labelsize=SMALL_SIZE) # fontsize of the tick labels +plt.rc("legend", fontsize=SMALL_SIZE, frameon=False) # legend fontsize +plt.rc( + "figure", titlesize=MEDIUM_SIZE, titleweight="bold", autolayout=True +) # , family='bold') # fontsize of the figure title + +# figure.autolayout : False +# matplotlib.rcParams['pdf.fonttype'] = 42 +# matplotlib.rcParams['ps.fonttype'] = 42 + + +plt.rc("path", simplify=True) + +plt.rcParams["figure.figsize"] = (10, 8) # (20.0, 10.0) #inline +# plt.rcParams['pcolor.shading'] = 'auto' +# rcParams['pcolor.shading'] = 'auto' +# plt.rc('pcolor', shading = 'auto') ### TICKS # see http://matplotlib.org/api/axis_api.html#matplotlib.axis.Tick -#xtick.top : False # draw ticks on the top side -#xtick.bottom : True # draw ticks on the bottom side -#xtick.major.size : 3.5 # major tick size in points -#xtick.minor.size : 2 # minor tick size in points -#xtick.major.width : .8 # major tick width in points -#xtick.minor.width : 0.6 # minor tick width in points -#xtick.major.pad : 3.5 # distance to major tick label in points -#xtick.minor.pad : 3.4 # distance to the minor tick label in points -#xtick.color : k # color of the tick labels -#xtick.labelsize : medium # fontsize of the tick labels -#xtick.direction : out # direction: in, out, or inout -#xtick.minor.visible : False # visibility of minor ticks on x-axis -#xtick.major.top : True # draw x axis top major ticks -#xtick.major.bottom : True # draw x axis bottom major ticks -#xtick.minor.top : True # draw x axis top minor ticks -#xtick.minor.bottom : True # draw x axis bottom minor ticks - -#ytick.left : True # draw ticks on the left side -#ytick.right : False # draw ticks on the right side -#ytick.major.size : 3.5 # major tick size in points -#ytick.minor.size : 2 # minor tick size in points -#ytick.major.width : 0.8 # major tick width in points -#ytick.minor.width : 0.6 # minor tick width in points -#ytick.major.pad : 3.5 # distance to major tick label in points -#ytick.minor.pad : 3.4 # distance to the minor tick label in points -#ytick.color : k # color of the tick labels -#ytick.labelsize : medium # fontsize of the tick labels -#ytick.direction : out # direction: in, out, or inout -#ytick.minor.visible : False # visibility of minor ticks on y-axis -#ytick.major.left : True # draw y axis left major ticks -#ytick.major.right : True # draw y axis right major ticks -#ytick.minor.left : True # draw y axis left minor ticks -#ytick.minor.right : True # draw y axis right minor ticks - - -plt.rc('xtick.major', size= 4, width=1 ) -plt.rc('ytick.major', size= 3.8, width=1 ) - -#axes.facecolor : white # axes background color -#axes.edgecolor : black # axes edge color -#axes.linewidth : 0.8 # edge linewidth -#axes.grid : False # display grid or not -#axes.titlesize : large # fontsize of the axes title -#axes.titlepad : 6.0 # pad between axes and title in points -#axes.labelsize : medium # fontsize of the x any y labels -#axes.labelpad : 4.0 # space between label and axis -#axes.labelweight : normal # weight of the x and y labels -#axes.labelcolor : black - -plt.rc('axes', labelsize= MEDIUM_SIZE, labelweight='normal') - - +# xtick.top : False # draw ticks on the top side +# xtick.bottom : True # draw ticks on the bottom side +# xtick.major.size : 3.5 # major tick size in points +# xtick.minor.size : 2 # minor tick size in points +# xtick.major.width : .8 # major tick width in points +# xtick.minor.width : 0.6 # minor tick width in points +# xtick.major.pad : 3.5 # distance to major tick label in points +# xtick.minor.pad : 3.4 # distance to the minor tick label in points +# xtick.color : k # color of the tick labels +# xtick.labelsize : medium # fontsize of the tick labels +# xtick.direction : out # direction: in, out, or inout +# xtick.minor.visible : False # visibility of minor ticks on x-axis +# xtick.major.top : True # draw x axis top major ticks +# xtick.major.bottom : True # draw x axis bottom major ticks +# xtick.minor.top : True # draw x axis top minor ticks +# xtick.minor.bottom : True # draw x axis bottom minor ticks + +# ytick.left : True # draw ticks on the left side +# ytick.right : False # draw ticks on the right side +# ytick.major.size : 3.5 # major tick size in points +# ytick.minor.size : 2 # minor tick size in points +# ytick.major.width : 0.8 # major tick width in points +# ytick.minor.width : 0.6 # minor tick width in points +# ytick.major.pad : 3.5 # distance to major tick label in points +# ytick.minor.pad : 3.4 # distance to the minor tick label in points +# ytick.color : k # color of the tick labels +# ytick.labelsize : medium # fontsize of the tick labels +# ytick.direction : out # direction: in, out, or inout +# ytick.minor.visible : False # visibility of minor ticks on y-axis +# ytick.major.left : True # draw y axis left major ticks +# ytick.major.right : True # draw y axis right major ticks +# ytick.minor.left : True # draw y axis left minor ticks +# ytick.minor.right : True # draw y axis right minor ticks + + +plt.rc("xtick.major", size=4, width=1) +plt.rc("ytick.major", size=3.8, width=1) + +# axes.facecolor : white # axes background color +# axes.edgecolor : black # axes edge color +# axes.linewidth : 0.8 # edge linewidth +# axes.grid : False # display grid or not +# axes.titlesize : large # fontsize of the axes title +# axes.titlepad : 6.0 # pad between axes and title in points +# axes.labelsize : medium # fontsize of the x any y labels +# axes.labelpad : 4.0 # space between label and axis +# axes.labelweight : normal # weight of the x and y labels +# axes.labelcolor : black + +plt.rc("axes", labelsize=MEDIUM_SIZE, labelweight="normal") # axes.spines.left : True # display axis spines # axes.spines.bottom : True # axes.spines.top : True # axes.spines.right : True -plt.rc('axes.spines', top= False, right=False ) - +plt.rc("axes.spines", top=False, right=False) def font_for_print(): @@ -158,76 +171,88 @@ def font_for_print(): SMALL_SIZE = 6 MEDIUM_SIZE = 8 BIGGER_SIZE = 10 - #csfont = {'fontname':'Comic Sans MS'} - legend_properties = {'weight':'bold'} - #font.family: sans-serif - #font.sans-serif: Helvetica Neue - - #import matplotlib.font_manager as font_manager - #font_dirs = ['/home/mhell/HelveticaNeue/', ] - #font_files = font_manager.findSystemFonts(fontpaths=font_dirs) - #font_list = font_manager.createFontList(font_files) - #font_manager.fontManager.ttflist.extend(font_list) - - plt.rc('font', size=SMALL_SIZE, serif='Helvetica Neue', weight='normal') # controls default text sizes - #plt.rc('font', size=SMALL_SIZE, serif='DejaVu Sans', weight='light') - plt.rc('text', usetex='false') - plt.rc('axes', titlesize=MEDIUM_SIZE, labelweight='normal') # fontsize of the axes title - plt.rc('axes', labelsize=SMALL_SIZE, labelweight='normal') #, family='bold') # fontsize of the x and y labels - plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels - plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels - plt.rc('legend', fontsize=SMALL_SIZE, frameon=False) # legend fontsize - plt.rc('figure', titlesize=MEDIUM_SIZE, titleweight='bold', autolayout=True) #, family='bold') # fontsize of the figure title - - #figure.autolayout : False - #matplotlib.rcParams['pdf.fonttype'] = 42 - #matplotlib.rcParams['ps.fonttype'] = 42 + # csfont = {'fontname':'Comic Sans MS'} + legend_properties = {"weight": "bold"} + # font.family: sans-serif + # font.sans-serif: Helvetica Neue + + # import matplotlib.font_manager as font_manager + # font_dirs = ['/home/mhell/HelveticaNeue/', ] + # font_files = font_manager.findSystemFonts(fontpaths=font_dirs) + # font_list = font_manager.createFontList(font_files) + # font_manager.fontManager.ttflist.extend(font_list) + + plt.rc( + "font", size=SMALL_SIZE, serif="Helvetica Neue", weight="normal" + ) # controls default text sizes + # plt.rc('font', size=SMALL_SIZE, serif='DejaVu Sans', weight='light') + plt.rc("text", usetex="false") + plt.rc( + "axes", titlesize=MEDIUM_SIZE, labelweight="normal" + ) # fontsize of the axes title + plt.rc( + "axes", labelsize=SMALL_SIZE, labelweight="normal" + ) # , family='bold') # fontsize of the x and y labels + plt.rc("xtick", labelsize=SMALL_SIZE) # fontsize of the tick labels + plt.rc("ytick", labelsize=SMALL_SIZE) # fontsize of the tick labels + plt.rc("legend", fontsize=SMALL_SIZE, frameon=False) # legend fontsize + plt.rc( + "figure", titlesize=MEDIUM_SIZE, titleweight="bold", autolayout=True + ) # , family='bold') # fontsize of the figure title + + # figure.autolayout : False + # matplotlib.rcParams['pdf.fonttype'] = 42 + # matplotlib.rcParams['ps.fonttype'] = 42 + + # plt.rc('xtick.major', size= 4, width=1 ) + # plt.rc('ytick.major', size= 3.8, width=1 ) + + plt.rc("axes", labelsize=SMALL_SIZE, labelweight="normal") - #plt.rc('xtick.major', size= 4, width=1 ) - #plt.rc('ytick.major', size= 3.8, width=1 ) - - - plt.rc('axes', labelsize= SMALL_SIZE, labelweight='normal') - def font_for_pres(): SMALL_SIZE = 10 MEDIUM_SIZE = 12 BIGGER_SIZE = 14 - #csfont = {'fontname':'Comic Sans MS'} - legend_properties = {'weight':'bold'} - #font.family: sans-serif - #font.sans-serif: Helvetica Neue - - #import matplotlib.font_manager as font_manager - #font_dirs = ['/home/mhell/HelveticaNeue/', ] - #font_files = font_manager.findSystemFonts(fontpaths=font_dirs) - #font_list = font_manager.createFontList(font_files) - #font_manager.fontManager.ttflist.extend(font_list) - - plt.rc('font', size=SMALL_SIZE, serif='Helvetica Neue', weight='normal') # controls default text sizes - #plt.rc('font', size=SMALL_SIZE, serif='DejaVu Sans', weight='light') - plt.rc('text', usetex='false') - plt.rc('axes', titlesize=MEDIUM_SIZE, labelweight='normal') # fontsize of the axes title - plt.rc('axes', labelsize=SMALL_SIZE, labelweight='normal') #, family='bold') # fontsize of the x and y labels - plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels - plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels - plt.rc('legend', fontsize=SMALL_SIZE, frameon=False) # legend fontsize - plt.rc('figure', titlesize=MEDIUM_SIZE, titleweight='bold', autolayout=True) #, family='bold') # fontsize of the figure title - - #figure.autolayout : False - #matplotlib.rcParams['pdf.fonttype'] = 42 - #matplotlib.rcParams['ps.fonttype'] = 42 - - - #plt.rc('xtick.major', size= 4, width=1 ) - #plt.rc('ytick.major', size= 3.8, width=1 ) - - - plt.rc('axes', labelsize= SMALL_SIZE, labelweight='normal') - + # csfont = {'fontname':'Comic Sans MS'} + legend_properties = {"weight": "bold"} + # font.family: sans-serif + # font.sans-serif: Helvetica Neue + + # import matplotlib.font_manager as font_manager + # font_dirs = ['/home/mhell/HelveticaNeue/', ] + # font_files = font_manager.findSystemFonts(fontpaths=font_dirs) + # font_list = font_manager.createFontList(font_files) + # font_manager.fontManager.ttflist.extend(font_list) + + plt.rc( + "font", size=SMALL_SIZE, serif="Helvetica Neue", weight="normal" + ) # controls default text sizes + # plt.rc('font', size=SMALL_SIZE, serif='DejaVu Sans', weight='light') + plt.rc("text", usetex="false") + plt.rc( + "axes", titlesize=MEDIUM_SIZE, labelweight="normal" + ) # fontsize of the axes title + plt.rc( + "axes", labelsize=SMALL_SIZE, labelweight="normal" + ) # , family='bold') # fontsize of the x and y labels + plt.rc("xtick", labelsize=SMALL_SIZE) # fontsize of the tick labels + plt.rc("ytick", labelsize=SMALL_SIZE) # fontsize of the tick labels + plt.rc("legend", fontsize=SMALL_SIZE, frameon=False) # legend fontsize + plt.rc( + "figure", titlesize=MEDIUM_SIZE, titleweight="bold", autolayout=True + ) # , family='bold') # fontsize of the figure title + + # figure.autolayout : False + # matplotlib.rcParams['pdf.fonttype'] = 42 + # matplotlib.rcParams['ps.fonttype'] = 42 + + # plt.rc('xtick.major', size= 4, width=1 ) + # plt.rc('ytick.major', size= 3.8, width=1 ) + + plt.rc("axes", labelsize=SMALL_SIZE, labelweight="normal") # add project depenent libraries -#sys.path.append(config['paths']['local_script']) +# sys.path.append(config['paths']['local_script']) diff --git a/config/generate_colors.py b/config/generate_colors.py index e65f9b5c..dc335870 100644 --- a/config/generate_colors.py +++ b/config/generate_colors.py @@ -1,63 +1,63 @@ -#%matplotlib inline +# %matplotlib inline -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) import m_colormanager_ph3 as M_color -mconfig['paths'] -path=mconfig['paths']['config'] -A = M_color.ase_to_json(path+'color_def.ase') +mconfig["paths"] +path = mconfig["paths"]["config"] +A = M_color.ase_to_json(path + "color_def.ase") -B=dict() -for i in A[0]['swatches']: - B[i['name']] = i['data']['values'] - print(i['name'] + ' ' + str(i['data']['values'])) +B = dict() +for i in A[0]["swatches"]: + B[i["name"]] = i["data"]["values"] + print(i["name"] + " " + str(i["data"]["values"])) -rels=dict() +rels = dict() -rels['plus']=B['red'] -rels['minus']=B['blue'] +rels["plus"] = B["red"] +rels["minus"] = B["blue"] -rels['blue']=B['blue'] -rels['lightblue']=B['cascade3'] -rels['darkblue']=B['cascade1'] +rels["blue"] = B["blue"] +rels["lightblue"] = B["cascade3"] +rels["darkblue"] = B["cascade1"] -rels['white']=B['white'] -rels['gridcolor']=B['gridcolor'] -rels['grey']=B['gray'] +rels["white"] = B["white"] +rels["gridcolor"] = B["gridcolor"] +rels["grey"] = B["gray"] -rels['orange']=B['orange'] -rels['red']=B['red'] -rels['green']=B['green'] +rels["orange"] = B["orange"] +rels["red"] = B["red"] +rels["green"] = B["green"] -rels['cascade1']=B['cascade1'] -rels['cascade2']=B['cascade2'] -rels['cascade3']=B['cascade3'] -rels['cascade4']=B['gridcolor'] +rels["cascade1"] = B["cascade1"] +rels["cascade2"] = B["cascade2"] +rels["cascade3"] = B["cascade3"] +rels["cascade4"] = B["gridcolor"] -rels['rascade1']=B['rascade2'] -rels['rascade2']=B['rascade1'] -rels['rascade3']=B['rascade3'] +rels["rascade1"] = B["rascade2"] +rels["rascade2"] = B["rascade1"] +rels["rascade3"] = B["rascade3"] -rels['aug1']=B['orange'] -rels['aug2']=B['green'] +rels["aug1"] = B["orange"] +rels["aug2"] = B["green"] -rels['gt1l']=B['rascade1'] -rels['gt1r']=B['rascade3'] +rels["gt1l"] = B["rascade1"] +rels["gt1r"] = B["rascade3"] -rels['gt2l']=B['green1'] -rels['gt2r']=B['green2'] +rels["gt2l"] = B["green1"] +rels["gt2r"] = B["green2"] -rels['gt3l']=B['cascade1'] -rels['gt3r']=B['blue'] +rels["gt3l"] = B["cascade1"] +rels["gt3r"] = B["blue"] -rels['group1']=B['rascade1'] -rels['group2']=B['green1'] -rels['group3']=B['cascade1'] +rels["group1"] = B["rascade1"] +rels["group2"] = B["green1"] +rels["group3"] = B["cascade1"] -B['rels']=rels +B["rels"] = rels -M_color.json_save('color_def', path, B) +M_color.json_save("color_def", path, B) diff --git a/src/icesat2_tracks/ICEsat2_SI_tools/__init__.py b/src/icesat2_tracks/ICEsat2_SI_tools/__init__.py index a391f31e..8c3cde44 100644 --- a/src/icesat2_tracks/ICEsat2_SI_tools/__init__.py +++ b/src/icesat2_tracks/ICEsat2_SI_tools/__init__.py @@ -1 +1 @@ -"Utility functions for loading files/data and tools for ploting tables and figures " \ No newline at end of file +"Utility functions for loading files/data and tools for ploting tables and figures " diff --git a/src/icesat2_tracks/ICEsat2_SI_tools/angle_optimizer.py b/src/icesat2_tracks/ICEsat2_SI_tools/angle_optimizer.py index 7ec54aa2..4627ead2 100644 --- a/src/icesat2_tracks/ICEsat2_SI_tools/angle_optimizer.py +++ b/src/icesat2_tracks/ICEsat2_SI_tools/angle_optimizer.py @@ -1,13 +1,15 @@ """ This library contains method, and classes used to search for the best angle given x,y data using single frequecy fits. """ + from numba import jit import numpy as np numba_parallel = False -def get_wavenumbers_polar( amp, angle_rad): + +def get_wavenumbers_polar(amp, angle_rad): """ inputs: @@ -18,108 +20,137 @@ def get_wavenumbers_polar( amp, angle_rad): wavenumber k,l """ import numpy as np + k0 = amp * np.cos(angle_rad) l0 = amp * np.sin(angle_rad) return k0, l0 -def wavemodel(XX, YY, ks, ls, amps, group_phase = 0): + +def wavemodel(XX, YY, ks, ls, amps, group_phase=0): import numpy as np - G = np.vstack([ np.cos(np.outer(XX, ks) + np.outer(YY, ls) ).T , np.sin(np.outer(XX, ks) + np.outer(YY, ls) ).T ] ).T + G = np.vstack( + [ + np.cos(np.outer(XX, ks) + np.outer(YY, ls)).T, + np.sin(np.outer(XX, ks) + np.outer(YY, ls)).T, + ] + ).T - #phase1 = np.random.rand(1, amp_list.size) * np.pi*2 - #phase = np.arange(0, amp_list.size) * np.pi/2 + # phase1 = np.random.rand(1, amp_list.size) * np.pi*2 + # phase = np.arange(0, amp_list.size) * np.pi/2 - b = np.hstack([ np.cos(group_phase) * amps , np.sin(group_phase) * amps ]).squeeze() - z_model = (G @ b) + b = np.hstack([np.cos(group_phase) * amps, np.sin(group_phase) * amps]).squeeze() + z_model = G @ b return z_model -@jit(nopython=True, parallel= numba_parallel) -def wavemodel_single_wave(XX, YY, ks, ls, amps, group_phase = 0): - z_model = amps * np.cos(XX * ks + YY * ls + group_phase ) - return z_model#.squeeze() + +@jit(nopython=True, parallel=numba_parallel) +def wavemodel_single_wave(XX, YY, ks, ls, amps, group_phase=0): + z_model = amps * np.cos(XX * ks + YY * ls + group_phase) + return z_model # .squeeze() -def get_z_model(x_positions, y_position, K_prime, K_amp, alpha_rad, group_phase): +def get_z_model(x_positions, y_position, K_prime, K_amp, alpha_rad, group_phase): - K_abs = K_prime / np.cos(alpha_rad) + K_abs = K_prime / np.cos(alpha_rad) k = K_abs * np.cos(alpha_rad) l = K_abs * np.sin(alpha_rad) - return wavemodel( x_positions,y_position, k, l, np.array(K_amp ), group_phase= group_phase) + return wavemodel( + x_positions, y_position, k, l, np.array(K_amp), group_phase=group_phase + ) -@jit(nopython=True, parallel= False) -def get_z_model_single_wave(x_positions, y_position, K_prime, K_amp, alpha_rad, group_phase): - K_abs = K_prime / np.cos(alpha_rad) +@jit(nopython=True, parallel=False) +def get_z_model_single_wave( + x_positions, y_position, K_prime, K_amp, alpha_rad, group_phase +): + + K_abs = K_prime / np.cos(alpha_rad) k = K_abs * np.cos(alpha_rad) l = K_abs * np.sin(alpha_rad) - return wavemodel_single_wave( x_positions,y_position, k, l, K_amp, group_phase= group_phase) - + return wavemodel_single_wave( + x_positions, y_position, k, l, K_amp, group_phase=group_phase + ) -def objective_func(pars, x, y, z, test_flag= False , prior= None, prior_weight = 2 ): +def objective_func(pars, x, y, z, test_flag=False, prior=None, prior_weight=2): "objective function that returns the residual array" - z_model = get_z_model_single_wave(x, y, pars['K_prime'].value, pars['K_amp'].value, pars['alpha'].value,pars['phase'].value) + z_model = get_z_model_single_wave( + x, + y, + pars["K_prime"].value, + pars["K_amp"].value, + pars["alpha"].value, + pars["phase"].value, + ) if prior is not None: - a_0, a_std = prior['alpha'] - penalties = np.array([ (abs(a_0 - pars['alpha'] )**2 / a_std**2) ]) + a_0, a_std = prior["alpha"] + penalties = np.array([(abs(a_0 - pars["alpha"]) ** 2 / a_std**2)]) else: - penalties = np.array([0]) + penalties = np.array([0]) - cost =( abs(z - z_model) )**2 /z.std()**2 + cost = (abs(z - z_model)) ** 2 / z.std() ** 2 if test_flag: return z_model else: - return np.concatenate([cost , prior_weight * penalties]) + return np.concatenate([cost, prior_weight * penalties]) + # def gaussian_prior(x, x0, sigma): # return np.exp(-np.power((x - x0)/sigma, 2.)/2.) -def likelyhood_func(pars, x, y, z, z_error= None, test_flag= False , prior= None , prior_weight = 2): +def likelyhood_func( + pars, x, y, z, z_error=None, test_flag=False, prior=None, prior_weight=2 +): """ ---not well testet --- likelyhood function using log(p) """ # get model - z_model = get_z_model(x, y, pars['K_prime'], pars['K_amp'], pars['alpha'],pars['phase']) + z_model = get_z_model( + x, y, pars["K_prime"], pars["K_amp"], pars["alpha"], pars["phase"] + ) # define cost - cost_sqrt =( abs(z - z_model) )**2 + cost_sqrt = (abs(z - z_model)) ** 2 # estimate total variance if z_error is None: - tot_var = z.std()**2* 0.01 + z_model**2 + tot_var = z.std() ** 2 * 0.01 + z_model**2 else: - tot_var = z_error**2 + z_model**2 + tot_var = z_error**2 + z_model**2 - #cost_sqrt.sum()/tot_var.sum() + # cost_sqrt.sum()/tot_var.sum() # (cost_sqrt/tot_var).sum() # # np.log(tot_var).sum() def simple_log_panelty(x, x0, sigma): - return -np.power((x - x0)/sigma, 2.)/2. + return -np.power((x - x0) / sigma, 2.0) / 2.0 # try ot get prior if prior is not None: - a_0, a_std = prior['alpha'] + a_0, a_std = prior["alpha"] # this corresponds to the the panelty log( p(alpha) ) - penalties = simple_log_panelty(pars['alpha'], a_0,a_std ) + penalties = simple_log_panelty(pars["alpha"], a_0, a_std) else: - penalties = np.array([0]) + penalties = np.array([0]) if test_flag: return z_model else: - return - 0.5 * (cost_sqrt/tot_var + np.log(tot_var) ).sum() + prior_weight * penalties + return ( + -0.5 * (cost_sqrt / tot_var + np.log(tot_var)).sum() + + prior_weight * penalties + ) class sample_with_mcmc: @@ -156,6 +187,7 @@ class sample_with_mcmc: def __init__(self, params): import lmfit as LM + self.LM = LM self.set_parameters(params) @@ -172,7 +204,7 @@ def set_objective_func(self, ofunc): # def test_ojective_func(self, model_func): # return self.objective_func(self.params, self.data, model_func, self.freq) - def set_parameters(self, par_dict, verbose= False): + def set_parameters(self, par_dict, verbose=False): """ defines params object at inital seed for mcmc par_dict should contain: var_name : [min, max, nseed] @@ -181,97 +213,134 @@ def set_parameters(self, par_dict, verbose= False): params = self.LM.Parameters() var_seeds = list() - for k,I in par_dict.items(): - params.add(k, (I[0]+ I[1])/2, vary=True , min=I[0], max=I[1]) - - var_seeds.append( np.linspace(I[0],I[1], I[2])) + for k, I in par_dict.items(): + params.add(k, (I[0] + I[1]) / 2, vary=True, min=I[0], max=I[1]) + var_seeds.append(np.linspace(I[0], I[1], I[2])) if len(var_seeds) > 2: - raise ValueError('nor proframmed for 3d') + raise ValueError("nor proframmed for 3d") - self.nwalkers= int(var_seeds[0].size * var_seeds[1].size) + self.nwalkers = int(var_seeds[0].size * var_seeds[1].size) pxx, pyy = np.meshgrid(var_seeds[0], var_seeds[1]) - self.seeds = np.vstack([pxx.flatten(), pyy.flatten() ]).T + self.seeds = np.vstack([pxx.flatten(), pyy.flatten()]).T self.params = params if verbose: - print('Nwalker: ', self.nwalkers) - print('Seeds: ', self.seeds.shape) + print("Nwalker: ", self.nwalkers) + print("Seeds: ", self.seeds.shape) print(self.params) def test_objective_func(self): - return self.objective_func(self.params, *self.fitting_args, **self.fitting_kargs) + return self.objective_func( + self.params, *self.fitting_args, **self.fitting_kargs + ) - def sample(self, fitting_args= None , method='emcee', steps=100, verbose= True, **kargs): + def sample( + self, fitting_args=None, method="emcee", steps=100, verbose=True, **kargs + ): - fitting_args, fitting_kargs = self.fitting_args, self.fitting_kargs - # TODO: this funciton throws an error in CI. The nan_policy='omit' policiy was added to avoid this issue + fitting_args, fitting_kargs = self.fitting_args, self.fitting_kargs + # TODO: this funciton throws an error in CI. The nan_policy='omit' policiy was added to avoid this issue # according to the guidelines in https://lmfit.github.io/lmfit-py/faq.html#i-get-errors-from-nan-in-my-fit-what-can-i-do - self.fitter = self.LM.minimize(self.objective_func, self.params, method=method, - args=fitting_args, kws=fitting_kargs , - nwalkers=self.nwalkers, steps=steps, pos= self.seeds,nan_policy='omit' , **kargs) + self.fitter = self.LM.minimize( + self.objective_func, + self.params, + method=method, + args=fitting_args, + kws=fitting_kargs, + nwalkers=self.nwalkers, + steps=steps, + pos=self.seeds, + nan_policy="omit", + **kargs, + ) if verbose: print(self.LM.report_fit(self.fitter)) - print('results at self.fitter') + print("results at self.fitter") - def plot_sample(self, **kargs ): + def plot_sample(self, **kargs): import matplotlib.pyplot as plt + chain = self.chain() nwalkers = self.nwalkers for n in np.arange(nwalkers): - plt.plot(chain[:,n,1], chain[:,n,0] , '-', **kargs) - plt.plot(chain[:,n,1], chain[:,n,0] , '.', **kargs) - + plt.plot(chain[:, n, 1], chain[:, n, 0], "-", **kargs) + plt.plot(chain[:, n, 1], chain[:, n, 0], ".", **kargs) - def optimize(self, fitting_args= None , method='dual_annealing', verbose= True): + def optimize(self, fitting_args=None, method="dual_annealing", verbose=True): - fitting_args = self.fitting_args - fitting_kargs = self.fitting_kargs + fitting_args = self.fitting_args + fitting_kargs = self.fitting_kargs - self.fitter_optimize = self.LM.minimize(self.objective_func, self.params, method=method, - args=fitting_args, kws=fitting_kargs ) + self.fitter_optimize = self.LM.minimize( + self.objective_func, + self.params, + method=method, + args=fitting_args, + kws=fitting_kargs, + ) if verbose: print(self.LM.report_fit(self.fitter_optimize)) - print('results at self.fitter_optimize') + print("results at self.fitter_optimize") def plot_optimze(self, **kargs): import matplotlib.pyplot as plt - plt.plot(self.fitter_optimize.params['phase'].value, self.fitter_optimize.params['alpha'].value, '.', **kargs) + plt.plot( + self.fitter_optimize.params["phase"].value, + self.fitter_optimize.params["alpha"].value, + ".", + **kargs, + ) - def brute(self, fitting_args= None , method='brute', verbose= True, N_grid = 30): + def brute(self, fitting_args=None, method="brute", verbose=True, N_grid=30): - fitting_args = self.fitting_args - fitting_kargs = self.fitting_kargs + fitting_args = self.fitting_args + fitting_kargs = self.fitting_kargs - self.fitter_brute = self.LM.minimize(self.objective_func, self.params, method=method, - args=fitting_args, kws=fitting_kargs, Ns=N_grid ) + self.fitter_brute = self.LM.minimize( + self.objective_func, + self.params, + method=method, + args=fitting_args, + kws=fitting_kargs, + Ns=N_grid, + ) if verbose: print(self.LM.report_fit(self.fitter_brute)) - print('results at self.fitter_brute') + print("results at self.fitter_brute") - - def plot_brute(self, clevel = np.linspace(-3.2, 3.2, 30), **kargs): + def plot_brute(self, clevel=np.linspace(-3.2, 3.2, 30), **kargs): import matplotlib.pyplot as plt - fitter_brute =self.fitter_brute - - dd = (fitter_brute.brute_Jout- fitter_brute.brute_Jout.mean())/fitter_brute.brute_Jout.std() - plt.contourf(fitter_brute.brute_grid[1,:,:], fitter_brute.brute_grid[0,:,:], dd , clevel, cmap= plt.cm.YlGnBu_r ) - #plt.scatter(fitter_brute.brute_grid[1,:,:], fitter_brute.brute_grid[0,:,:], s=0.2, alpha= 0.4, color='black') - #plt.plot(fitter_brute.brute_x0[1], fitter_brute.brute_x0[0], **kargs) - x_name, y_name = list(fitter_brute.params.keys())[1], list(fitter_brute.params.keys())[0] + fitter_brute = self.fitter_brute + + dd = ( + fitter_brute.brute_Jout - fitter_brute.brute_Jout.mean() + ) / fitter_brute.brute_Jout.std() + plt.contourf( + fitter_brute.brute_grid[1, :, :], + fitter_brute.brute_grid[0, :, :], + dd, + clevel, + cmap=plt.cm.YlGnBu_r, + ) + # plt.scatter(fitter_brute.brute_grid[1,:,:], fitter_brute.brute_grid[0,:,:], s=0.2, alpha= 0.4, color='black') + # plt.plot(fitter_brute.brute_x0[1], fitter_brute.brute_x0[0], **kargs) + + x_name, y_name = ( + list(fitter_brute.params.keys())[1], + list(fitter_brute.params.keys())[0], + ) plt.xlabel(x_name) plt.ylabel(y_name) # plt.xlabel('Phase (rad)') # plt.ylabel('Angle (rad)') - - def chain(self, burn=None): "return results as nparray contains walk of each walker" if burn is not None: @@ -286,8 +355,9 @@ def flatchain(self, burn=None): else: return self.fitter.flatchain - def get_marginal_dist(self, var, var_dx, burn = None, plot_flag= False, normalize = True): - + def get_marginal_dist( + self, var, var_dx, burn=None, plot_flag=False, normalize=True + ): """ retrurn the marginal distribution from self.params object @@ -301,17 +371,18 @@ def get_marginal_dist(self, var, var_dx, burn = None, plot_flag= False, normaliz """ data = self.flatchain(burn) - #fitter.flatchain.loc[100:][var] - bins = np.arange(self.params[var].min,self.params[var].max+ var_dx,var_dx) + # fitter.flatchain.loc[100:][var] + bins = np.arange(self.params[var].min, self.params[var].max + var_dx, var_dx) y_hist, _ = np.histogram(self.fitter.flatchain.loc[burn:][var], bins) - bins_pos = (bins[0:-1] + np.diff(bins)/2) + bins_pos = bins[0:-1] + np.diff(bins) / 2 if normalize: - y_hist = y_hist/var_dx/y_hist.sum() + y_hist = y_hist / var_dx / y_hist.sum() if plot_flag: import matplotlib.pyplot as plt + plt.stairs(y_hist, bins) return y_hist, bins, bins_pos diff --git a/src/icesat2_tracks/ICEsat2_SI_tools/convert_GPS_time.py b/src/icesat2_tracks/ICEsat2_SI_tools/convert_GPS_time.py index a1e491c8..9d9a7936 100644 --- a/src/icesat2_tracks/ICEsat2_SI_tools/convert_GPS_time.py +++ b/src/icesat2_tracks/ICEsat2_SI_tools/convert_GPS_time.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -u""" +""" convert_GPS_time.py (10/2017) Return the calendar date and time for given GPS time. Based on Tiffany Summerscales's PHP conversion algorithm @@ -33,88 +33,111 @@ from .convert_julian import convert_julian import pdb -#-- PURPOSE: Define GPS leap seconds + +# -- PURPOSE: Define GPS leap seconds def get_leaps(): - leaps = [46828800, 78364801, 109900802, 173059203, 252028804, 315187205, - 346723206, 393984007, 425520008, 457056009, 504489610, 551750411, - 599184012, 820108813, 914803214, 1025136015, 1119744016, 1167264017] - return leaps + leaps = [ + 46828800, + 78364801, + 109900802, + 173059203, + 252028804, + 315187205, + 346723206, + 393984007, + 425520008, + 457056009, + 504489610, + 551750411, + 599184012, + 820108813, + 914803214, + 1025136015, + 1119744016, + 1167264017, + ] + return leaps + -#-- PURPOSE: Test to see if any GPS seconds are leap seconds +# -- PURPOSE: Test to see if any GPS seconds are leap seconds def is_leap(GPS_Time): - leaps = get_leaps() - Flag = np.zeros_like(GPS_Time, dtype=np.bool_) - for leap in leaps: - count = np.count_nonzero(np.floor(GPS_Time) == leap) - if (count > 0): - indices, = np.nonzero(np.floor(GPS_Time) == leap) - Flag[indices] = True - return Flag - -#-- PURPOSE: Count number of leap seconds that have passed for each GPS time + leaps = get_leaps() + Flag = np.zeros_like(GPS_Time, dtype=np.bool_) + for leap in leaps: + count = np.count_nonzero(np.floor(GPS_Time) == leap) + if count > 0: + (indices,) = np.nonzero(np.floor(GPS_Time) == leap) + Flag[indices] = True + return Flag + + +# -- PURPOSE: Count number of leap seconds that have passed for each GPS time def count_leaps(GPS_Time): - leaps = get_leaps() - #-- number of leap seconds prior to GPS_Time - n_leaps = np.zeros_like(GPS_Time, dtype=np.uint) - for i,leap in enumerate(leaps): - count = np.count_nonzero(GPS_Time >= leap) - if (count > 0): - indices, = np.nonzero(GPS_Time >= leap) - # print(indices) - # pdb.set_trace() - n_leaps[indices] += 1 - return n_leaps - -#-- PURPOSE: Convert UNIX Time to GPS Time + leaps = get_leaps() + # -- number of leap seconds prior to GPS_Time + n_leaps = np.zeros_like(GPS_Time, dtype=np.uint) + for i, leap in enumerate(leaps): + count = np.count_nonzero(GPS_Time >= leap) + if count > 0: + (indices,) = np.nonzero(GPS_Time >= leap) + # print(indices) + # pdb.set_trace() + n_leaps[indices] += 1 + return n_leaps + + +# -- PURPOSE: Convert UNIX Time to GPS Time def convert_UNIX_to_GPS(UNIX_Time): - #-- calculate offsets for UNIX times that occur during leap seconds - offset = np.zeros_like(UNIX_Time) - count = np.count_nonzero((UNIX_Time % 1) != 0) - if (count > 0): - indices, = np.nonzero((UNIX_Time % 1) != 0) - UNIX_Time[indices] -= 0.5 - offset[indices] = 1.0 - #-- convert UNIX_Time to GPS without taking into account leap seconds - #-- (UNIX epoch: Jan 1, 1970 00:00:00, GPS epoch: Jan 6, 1980 00:00:00) - GPS_Time = UNIX_Time - 315964800 - leaps = get_leaps() - #-- calculate number of leap seconds prior to GPS_Time - n_leaps = np.zeros_like(GPS_Time, dtype=np.uint) - for i,leap in enumerate(leaps): - count = np.count_nonzero(GPS_Time >= (leap - i)) - if (count > 0): - indices, = np.nonzero(GPS_Time >= (leap - i)) - n_leaps[indices] += 1 - #-- take into account leap seconds and offsets - GPS_Time += n_leaps + offset - return GPS_Time - -#-- PURPOSE: Convert GPS Time to UNIX Time + # -- calculate offsets for UNIX times that occur during leap seconds + offset = np.zeros_like(UNIX_Time) + count = np.count_nonzero((UNIX_Time % 1) != 0) + if count > 0: + (indices,) = np.nonzero((UNIX_Time % 1) != 0) + UNIX_Time[indices] -= 0.5 + offset[indices] = 1.0 + # -- convert UNIX_Time to GPS without taking into account leap seconds + # -- (UNIX epoch: Jan 1, 1970 00:00:00, GPS epoch: Jan 6, 1980 00:00:00) + GPS_Time = UNIX_Time - 315964800 + leaps = get_leaps() + # -- calculate number of leap seconds prior to GPS_Time + n_leaps = np.zeros_like(GPS_Time, dtype=np.uint) + for i, leap in enumerate(leaps): + count = np.count_nonzero(GPS_Time >= (leap - i)) + if count > 0: + (indices,) = np.nonzero(GPS_Time >= (leap - i)) + n_leaps[indices] += 1 + # -- take into account leap seconds and offsets + GPS_Time += n_leaps + offset + return GPS_Time + + +# -- PURPOSE: Convert GPS Time to UNIX Time def convert_GPS_to_UNIX(GPS_Time): - #-- convert GPS_Time to UNIX without taking into account leap seconds - #-- (UNIX epoch: Jan 1, 1970 00:00:00, GPS epoch: Jan 6, 1980 00:00:00) - UNIX_Time = GPS_Time + 315964800 - #-- number of leap seconds prior to GPS_Time - n_leaps = count_leaps(GPS_Time) - UNIX_Time -= n_leaps - #-- check if GPS Time is leap second - Flag = is_leap(GPS_Time) - if Flag.any(): - #-- for leap seconds: add a half second offset - indices, = np.nonzero(Flag) - UNIX_Time[indices] += 0.5 - return UNIX_Time - -#-- PURPOSE: convert from GPS time to calendar dates + # -- convert GPS_Time to UNIX without taking into account leap seconds + # -- (UNIX epoch: Jan 1, 1970 00:00:00, GPS epoch: Jan 6, 1980 00:00:00) + UNIX_Time = GPS_Time + 315964800 + # -- number of leap seconds prior to GPS_Time + n_leaps = count_leaps(GPS_Time) + UNIX_Time -= n_leaps + # -- check if GPS Time is leap second + Flag = is_leap(GPS_Time) + if Flag.any(): + # -- for leap seconds: add a half second offset + (indices,) = np.nonzero(Flag) + UNIX_Time[indices] += 0.5 + return UNIX_Time + + +# -- PURPOSE: convert from GPS time to calendar dates def convert_GPS_time(GPS_Time, OFFSET=0.0): - #-- convert from standard GPS time to UNIX time accounting for leap seconds - #-- and adding the specified offset to GPS_Time - UNIX_Time = convert_GPS_to_UNIX(np.array(GPS_Time) + OFFSET) - #-- calculate Julian date from UNIX time and convert into calendar dates - #-- UNIX time: seconds from 1970-01-01 00:00:00 UTC - julian_date = (UNIX_Time/86400.0) + 2440587.500000 - cal_date = convert_julian(julian_date) - #-- include UNIX times in output - cal_date['UNIX'] = UNIX_Time - #-- return the calendar dates and UNIX time - return cal_date + # -- convert from standard GPS time to UNIX time accounting for leap seconds + # -- and adding the specified offset to GPS_Time + UNIX_Time = convert_GPS_to_UNIX(np.array(GPS_Time) + OFFSET) + # -- calculate Julian date from UNIX time and convert into calendar dates + # -- UNIX time: seconds from 1970-01-01 00:00:00 UTC + julian_date = (UNIX_Time / 86400.0) + 2440587.500000 + cal_date = convert_julian(julian_date) + # -- include UNIX times in output + cal_date["UNIX"] = UNIX_Time + # -- return the calendar dates and UNIX time + return cal_date diff --git a/src/icesat2_tracks/ICEsat2_SI_tools/convert_julian.py b/src/icesat2_tracks/ICEsat2_SI_tools/convert_julian.py index 74b3fd81..b5d6c7bc 100644 --- a/src/icesat2_tracks/ICEsat2_SI_tools/convert_julian.py +++ b/src/icesat2_tracks/ICEsat2_SI_tools/convert_julian.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -u""" +""" convert_julian.py Written by Tyler Sutterley (10/2017) @@ -50,60 +50,62 @@ """ import numpy as np -def convert_julian(JD, ASTYPE=None, FORMAT='dict'): - #-- convert to array if only a single value was imported - if (np.ndim(JD) == 0): - JD = np.array([JD]) - SINGLE_VALUE = True - else: - SINGLE_VALUE = False - JDO = np.floor(JD + 0.5) - C = np.zeros_like(JD) - #-- calculate C for dates before and after the switch to Gregorian - IGREG = 2299161.0 - ind1, = np.nonzero(JDO < IGREG) - C[ind1] = JDO[ind1] + 1524.0 - ind2, = np.nonzero(JDO >= IGREG) - B = np.floor((JDO[ind2] - 1867216.25)/36524.25) - C[ind2] = JDO[ind2] + B - np.floor(B/4.0) + 1525.0 - #-- calculate coefficients for date conversion - D = np.floor((C - 122.1)/365.25) - E = np.floor((365.0 * D) + np.floor(D/4.0)) - F = np.floor((C - E)/30.6001) - #-- calculate day, month, year and hour - DAY = np.floor(C - E + 0.5) - np.floor(30.6001*F) - MONTH = F - 1.0 - 12.0*np.floor(F/14.0) - YEAR = D - 4715.0 - np.floor((7.0+MONTH)/10.0) - HOUR = np.floor(24.0*(JD + 0.5 - JDO)) - #-- calculate minute and second - G = (JD + 0.5 - JDO) - HOUR/24.0 - MINUTE = np.floor(G*1440.0) - SECOND = (G - MINUTE/1440.0) * 86400.0 +def convert_julian(JD, ASTYPE=None, FORMAT="dict"): + # -- convert to array if only a single value was imported + if np.ndim(JD) == 0: + JD = np.array([JD]) + SINGLE_VALUE = True + else: + SINGLE_VALUE = False + + JDO = np.floor(JD + 0.5) + C = np.zeros_like(JD) + # -- calculate C for dates before and after the switch to Gregorian + IGREG = 2299161.0 + (ind1,) = np.nonzero(JDO < IGREG) + C[ind1] = JDO[ind1] + 1524.0 + (ind2,) = np.nonzero(JDO >= IGREG) + B = np.floor((JDO[ind2] - 1867216.25) / 36524.25) + C[ind2] = JDO[ind2] + B - np.floor(B / 4.0) + 1525.0 + # -- calculate coefficients for date conversion + D = np.floor((C - 122.1) / 365.25) + E = np.floor((365.0 * D) + np.floor(D / 4.0)) + F = np.floor((C - E) / 30.6001) + # -- calculate day, month, year and hour + DAY = np.floor(C - E + 0.5) - np.floor(30.6001 * F) + MONTH = F - 1.0 - 12.0 * np.floor(F / 14.0) + YEAR = D - 4715.0 - np.floor((7.0 + MONTH) / 10.0) + HOUR = np.floor(24.0 * (JD + 0.5 - JDO)) + # -- calculate minute and second + G = (JD + 0.5 - JDO) - HOUR / 24.0 + MINUTE = np.floor(G * 1440.0) + SECOND = (G - MINUTE / 1440.0) * 86400.0 - #-- convert all variables to output type (from float) - if ASTYPE is not None: - YEAR = YEAR.astype(ASTYPE) - MONTH = MONTH.astype(ASTYPE) - DAY = DAY.astype(ASTYPE) - HOUR = HOUR.astype(ASTYPE) - MINUTE = MINUTE.astype(ASTYPE) - SECOND = SECOND.astype(ASTYPE) + # -- convert all variables to output type (from float) + if ASTYPE is not None: + YEAR = YEAR.astype(ASTYPE) + MONTH = MONTH.astype(ASTYPE) + DAY = DAY.astype(ASTYPE) + HOUR = HOUR.astype(ASTYPE) + MINUTE = MINUTE.astype(ASTYPE) + SECOND = SECOND.astype(ASTYPE) - #-- if only a single value was imported initially: remove singleton dims - if SINGLE_VALUE: - YEAR = YEAR.item(0) - MONTH = MONTH.item(0) - DAY = DAY.item(0) - HOUR = HOUR.item(0) - MINUTE = MINUTE.item(0) - SECOND = SECOND.item(0) + # -- if only a single value was imported initially: remove singleton dims + if SINGLE_VALUE: + YEAR = YEAR.item(0) + MONTH = MONTH.item(0) + DAY = DAY.item(0) + HOUR = HOUR.item(0) + MINUTE = MINUTE.item(0) + SECOND = SECOND.item(0) - #-- return date variables in output format (default python dictionary) - if (FORMAT == 'dict'): - return dict(year=YEAR, month=MONTH, day=DAY, - hour=HOUR, minute=MINUTE, second=SECOND) - elif (FORMAT == 'tuple'): - return (YEAR, MONTH, DAY, HOUR, MINUTE, SECOND) - elif (FORMAT == 'zip'): - return zip(YEAR, MONTH, DAY, HOUR, MINUTE, SECOND) + # -- return date variables in output format (default python dictionary) + if FORMAT == "dict": + return dict( + year=YEAR, month=MONTH, day=DAY, hour=HOUR, minute=MINUTE, second=SECOND + ) + elif FORMAT == "tuple": + return (YEAR, MONTH, DAY, HOUR, MINUTE, SECOND) + elif FORMAT == "zip": + return zip(YEAR, MONTH, DAY, HOUR, MINUTE, SECOND) diff --git a/src/icesat2_tracks/ICEsat2_SI_tools/filter_regrid.py b/src/icesat2_tracks/ICEsat2_SI_tools/filter_regrid.py index 1c276853..a7f6e4b9 100644 --- a/src/icesat2_tracks/ICEsat2_SI_tools/filter_regrid.py +++ b/src/icesat2_tracks/ICEsat2_SI_tools/filter_regrid.py @@ -1,14 +1,15 @@ - import numpy as np from numba import jit -def correct_heights(T03, T03c, coord = 'delta_time'): +def correct_heights(T03, T03c, coord="delta_time"): """ returns the corrected photon heigts in T03 given SSSH approxiamtion 'dem_h' in T03c """ - T03['heights_c']= T03['heights'] - np.interp( T03[coord],T03c[coord], T03c['dem_h'] ) + T03["heights_c"] = T03["heights"] - np.interp( + T03[coord], T03c[coord], T03c["dem_h"] + ) return T03 @@ -24,32 +25,39 @@ def correct_heights(T03, T03c, coord = 'delta_time'): # """ # return hdf5_file['orbit_info/sc_orient'][:][0] ==0 -def track_pole_ward_file(hdf5_file, product='ALT03'): + +def track_pole_ward_file(hdf5_file, product="ALT03"): """ Returns true if track goes poleward hdf5_file is a an HFD5 object in read mode """ - if product == 'ALT03': - T_lat = hdf5_file['gt1r/geolocation/reference_photon_lat'][:] - T_time = hdf5_file['gt1r/geolocation/delta_time'][:] - elif product == 'ALT10': - T_lat = hdf5_file['gt1r/freeboard_beam_segment/latitude'][:] - T_time = hdf5_file['gt1r/freeboard_beam_segment/delta_time'][:] - #return ( T_lat[T_time.argmax()] - T_lat[T_time.argmin()] ) < 0 - print('1st lat =' + str(abs(T_lat[T_time.argmin()])) , ';last lat =' + str(abs(T_lat[T_time.argmax()])) ) + if product == "ALT03": + T_lat = hdf5_file["gt1r/geolocation/reference_photon_lat"][:] + T_time = hdf5_file["gt1r/geolocation/delta_time"][:] + elif product == "ALT10": + T_lat = hdf5_file["gt1r/freeboard_beam_segment/latitude"][:] + T_time = hdf5_file["gt1r/freeboard_beam_segment/delta_time"][:] + # return ( T_lat[T_time.argmax()] - T_lat[T_time.argmin()] ) < 0 + print( + "1st lat =" + str(abs(T_lat[T_time.argmin()])), + ";last lat =" + str(abs(T_lat[T_time.argmax()])), + ) return abs(T_lat[T_time.argmax()]) > abs(T_lat[T_time.argmin()]) - def track_type(T): """ Returns if track acending or desending T is a pandas table """ - #T = B[k] - #T = B[beams_list[0]] - return (T['lats'].iloc[T['delta_time'].argmax()] - T['lats'].iloc[T['delta_time'].argmin()] ) < 0 + # T = B[k] + # T = B[beams_list[0]] + return ( + T["lats"].iloc[T["delta_time"].argmax()] + - T["lats"].iloc[T["delta_time"].argmin()] + ) < 0 + def lat_min_max_extended(B, beams_list, accent=None): """ @@ -62,35 +70,33 @@ def lat_min_max_extended(B, beams_list, accent=None): returns: min_lat, max_lat, accent min and max latitudes of the beams, (True/False) True if the track is accending """ - #B, beams_list = B , high_beams - accent = regrid.track_type( B[beams_list[0]] ) if accent is None else accent + # B, beams_list = B , high_beams + accent = regrid.track_type(B[beams_list[0]]) if accent is None else accent - if B[beams_list[0]]['lats'].iloc[0] < 0: - hemis = 'SH' + if B[beams_list[0]]["lats"].iloc[0] < 0: + hemis = "SH" else: - hemis = 'NH' + hemis = "NH" - track_pos_start, track_pos_end= list(), list() + track_pos_start, track_pos_end = list(), list() for k in beams_list: - if (hemis == 'SH'): - track_pos_start.append( B[k].loc[B[k]['lats'].argmax()][ ['lats', 'lons']] ) - track_pos_end.append( B[k].loc[B[k]['lats'].argmin()][ ['lats', 'lons']] ) + if hemis == "SH": + track_pos_start.append(B[k].loc[B[k]["lats"].argmax()][["lats", "lons"]]) + track_pos_end.append(B[k].loc[B[k]["lats"].argmin()][["lats", "lons"]]) else: - track_pos_start.append( B[k].loc[B[k]['lats'].argmin()][ ['lats', 'lons']] ) - track_pos_end.append( B[k].loc[B[k]['lats'].argmax()][ ['lats', 'lons']] ) - + track_pos_start.append(B[k].loc[B[k]["lats"].argmin()][["lats", "lons"]]) + track_pos_end.append(B[k].loc[B[k]["lats"].argmax()][["lats", "lons"]]) track_lat_start, track_lat_end = list(), list() track_lon_start, track_lon_end = list(), list() for ll in track_pos_start: - track_lat_start.append(ll['lats']) - track_lon_start.append(ll['lons']) - + track_lat_start.append(ll["lats"]) + track_lon_start.append(ll["lons"]) for ll in track_pos_end: - track_lat_end.append(ll['lats']) - track_lon_end.append(ll['lons']) + track_lat_end.append(ll["lats"]) + track_lon_end.append(ll["lons"]) # track_lat_start.append( B[k]['lats'].min() ) # track_lat_end.append( B[k]['lats'].max() ) @@ -100,19 +106,34 @@ def lat_min_max_extended(B, beams_list, accent=None): if accent: track_lon_start - #track_lat_start.min(), track_lon_right.max() - - if (hemis == 'SH') & accent: - return [max(track_lat_start) , min(track_lat_end)], [max(track_lon_start), min(track_lon_end)], accent # accenting SH mean start is in the top right - elif (hemis == 'SH') & ~accent: - return [max(track_lat_start) , min(track_lat_end)], [min(track_lon_start), max(track_lon_end)], accent # decent SH mean start is in the top left - elif (hemis == 'NH') & accent: - return [min(track_lat_start) , max(track_lat_end)], [min(track_lon_start), max(track_lon_end)], accent # accent NH mean start is in the lower left - elif (hemis == 'NH') & ~accent: - return [min(track_lat_start) , max(track_lat_end)], [max(track_lon_start), min(track_lon_end)], accent # decent NH mean start is in the lower right + # track_lat_start.min(), track_lon_right.max() + + if (hemis == "SH") & accent: + return ( + [max(track_lat_start), min(track_lat_end)], + [max(track_lon_start), min(track_lon_end)], + accent, + ) # accenting SH mean start is in the top right + elif (hemis == "SH") & ~accent: + return ( + [max(track_lat_start), min(track_lat_end)], + [min(track_lon_start), max(track_lon_end)], + accent, + ) # decent SH mean start is in the top left + elif (hemis == "NH") & accent: + return ( + [min(track_lat_start), max(track_lat_end)], + [min(track_lon_start), max(track_lon_end)], + accent, + ) # accent NH mean start is in the lower left + elif (hemis == "NH") & ~accent: + return ( + [min(track_lat_start), max(track_lat_end)], + [max(track_lon_start), min(track_lon_end)], + accent, + ) # decent NH mean start is in the lower right else: - raise ValueError('some defintions went wrong') - + raise ValueError("some defintions went wrong") def lat_min_max(B, beams_list, accent=None): @@ -126,25 +147,26 @@ def lat_min_max(B, beams_list, accent=None): returns: min_lat, max_lat, accent min and max latitudes of the beams, (True/False) True if the track is accending """ - #B, beams_list = B , high_beams - accent = track_type( B[beams_list[0]] ) if accent is None else accent + # B, beams_list = B , high_beams + accent = track_type(B[beams_list[0]]) if accent is None else accent - if B[beams_list[0]]['lats'].iloc[0] < 0: - hemis = 'SH' + if B[beams_list[0]]["lats"].iloc[0] < 0: + hemis = "SH" else: - hemis = 'NH' + hemis = "NH" - track_lat_mins, track_lat_maxs= list(), list() + track_lat_mins, track_lat_maxs = list(), list() for k in beams_list: - track_lat_mins.append( B[k]['lats'].min() ) - track_lat_maxs.append( B[k]['lats'].max() ) + track_lat_mins.append(B[k]["lats"].min()) + track_lat_maxs.append(B[k]["lats"].max()) - if hemis == 'SH': - return max(track_lat_maxs) , min(track_lat_mins), accent + if hemis == "SH": + return max(track_lat_maxs), min(track_lat_mins), accent else: return min(track_lat_mins), max(track_lat_maxs), accent -def derive_axis(TT, lat_lims = None): + +def derive_axis(TT, lat_lims=None): """ returns TT distance along track 'dist' in meters input: @@ -153,43 +175,43 @@ def derive_axis(TT, lat_lims = None): returns: TT with x,y,dist and order by dist """ - #TT, lat_lims = B[key], lat_lims_high + # TT, lat_lims = B[key], lat_lims_high # derive distances in meters - r_e= 6.3710E+6 - dy= r_e*2*np.pi/360.0 - #deglon_in_m= np.cos(T2['lats']*np.pi/180.0)*dy + r_e = 6.3710e6 + dy = r_e * 2 * np.pi / 360.0 + # deglon_in_m= np.cos(T2['lats']*np.pi/180.0)*dy # either use position of the 1st photon or use defined start latitude if lat_lims is None: - TT['y']=(TT['lats'].max() - TT['lats']) *dy + TT["y"] = (TT["lats"].max() - TT["lats"]) * dy else: - TT['y']=(lat_lims[0] - TT['lats']) *dy - - #TT['y'] = (TT['lats']) *dy + TT["y"] = (lat_lims[0] - TT["lats"]) * dy + # TT['y'] = (TT['lats']) *dy - if (lat_lims[2] == True): + if lat_lims[2] == True: # accending track - lon_min = TT['lons'].max() + lon_min = TT["lons"].max() else: # decending track - lon_min = TT['lons'].min() + lon_min = TT["lons"].min() - #print(lon_min) - TT['x'] = (TT['lons'] - lon_min) * np.cos( TT['lats']*np.pi/180.0 ) * dy - #TT['x'] = (TT['lons'] ) * np.cos( TT['lats']*np.pi/180.0 ) * dy - TT['dist'] = np.sqrt(TT['x']**2 + TT['y']**2) + # print(lon_min) + TT["x"] = (TT["lons"] - lon_min) * np.cos(TT["lats"] * np.pi / 180.0) * dy + # TT['x'] = (TT['lons'] ) * np.cos( TT['lats']*np.pi/180.0 ) * dy + TT["dist"] = np.sqrt(TT["x"] ** 2 + TT["y"] ** 2) # set 1st dist to 0, not used if global limits are used if lat_lims is None: - TT['dist']= TT['dist']- TT['dist'].min() + TT["dist"] = TT["dist"] - TT["dist"].min() else: - TT['dist']= TT['dist']#- lat_lims[0] + TT["dist"] = TT["dist"] # - lat_lims[0] - TT=TT.sort_values(by='dist') + TT = TT.sort_values(by="dist") return TT -def reduce_to_height_distance(TT, key, dx=1, lat_lims = None): + +def reduce_to_height_distance(TT, key, dx=1, lat_lims=None): """ interpolates key (photos heights) to regular grid using 'dist' in pandas table TT. dx is the interpolation interval @@ -200,17 +222,18 @@ def reduce_to_height_distance(TT, key, dx=1, lat_lims = None): x1, y1 position, height """ from scipy.interpolate import interp1d + if type(dx) is np.ndarray: x1 = dx else: - x1 = np.arange(0,TT['dist'].max(), dx) - y1 = np.interp(x1, TT['dist'], TT[key] ) + x1 = np.arange(0, TT["dist"].max(), dx) + y1 = np.interp(x1, TT["dist"], TT[key]) return x1, y1 + # this is not need anymore def poly_correct(x, y, poly_order=7, plot_flag=False): - """ subtracts a fitted polynom to y inputs: @@ -220,38 +243,58 @@ def poly_correct(x, y, poly_order=7, plot_flag=False): returns y' y - polynom fit """ - z = np.polyfit(x , y , poly_order) + z = np.polyfit(x, y, poly_order) p = np.poly1d(z) if plot_flag: - plt.plot(x,y, '.', markersize=0.2,) - plt.plot(x, p(x), '-', markersize=0.2,) - #return z + plt.plot( + x, + y, + ".", + markersize=0.2, + ) + plt.plot( + x, + p(x), + "-", + markersize=0.2, + ) + # return z return y - p(x) ### regridding -#@jit(nopython=True) -def get_mode(y, bins = np.arange(-5,5, 0.1)): + +# @jit(nopython=True) +def get_mode(y, bins=np.arange(-5, 5, 0.1)): "returns modes of histogram of y defined by bins" - hist, xbin = np.histogram(y, bins = bins ) + hist, xbin = np.histogram(y, bins=bins) return xbin[hist.argmax()] -@jit(nopython=True, parallel= False) + +@jit(nopython=True, parallel=False) def weighted_mean(x_rel, y): "returns the gaussian weighted mean for stencil" - #@jit(nopython=True, parallel= False) + # @jit(nopython=True, parallel= False) def weight_fnk(x): "returns gaussian weight given the distance to the center x" - return np.exp(- (x/.5)**2 ) + return np.exp(-((x / 0.5) ** 2)) w = weight_fnk(x_rel) - return np.sum(w*y)/np.sum(w) + return np.sum(w * y) / np.sum(w) -# this function is applied to beam: -def get_stencil_stats_shift( T2, stencil_iter, key_var , key_x_coord, stancil_width , Nphoton_min = 5, plot_flag= False): +# this function is applied to beam: +def get_stencil_stats_shift( + T2, + stencil_iter, + key_var, + key_x_coord, + stancil_width, + Nphoton_min=5, + plot_flag=False, +): """ T2 pd.Dataframe with beam data needs at least 'dist' and key stencil_iter np.array that constains the stancil boundaries and center [left boundary, center, right boundary] @@ -272,42 +315,44 @@ def get_stencil_stats_shift( T2, stencil_iter, key_var , key_x_coord, stancil_w """ import pandas as pd - stencil_1 = stencil_iter[:, ::2] - stencil_1half = stencil_iter[:, 1::2] - def calc_stencil_stats(group, key, key_x_coord, stancil_width, stancils): + stencil_1 = stencil_iter[:, ::2] + stencil_1half = stencil_iter[:, 1::2] - "returns stats per stencil" - #import time - #tstart = time.time() - Nphoton = group.shape[0] - istancil = group['x_bins'].iloc[int(Nphoton/2)] - stencil_center = stancils[1, istancil-1] + def calc_stencil_stats(group, key, key_x_coord, stancil_width, stancils): + "returns stats per stencil" + # import time + # tstart = time.time() + Nphoton = group.shape[0] + istancil = group["x_bins"].iloc[int(Nphoton / 2)] + stencil_center = stancils[1, istancil - 1] if Nphoton > Nphoton_min: - x_rel = (group[key_x_coord] - stencil_center)/ stancil_width - y = group[key] + x_rel = (group[key_x_coord] - stencil_center) / stancil_width + y = group[key] - #Tmedian[key+ '_weighted_mean'] + # Tmedian[key+ '_weighted_mean'] key_weighted_mean = weighted_mean(np.array(x_rel), np.array(y)) - key_std = y.std() - key_mode = get_mode(y) + key_std = y.std() + key_mode = get_mode(y) else: - #Nphoton = 0 + # Nphoton = 0 key_weighted_mean = np.nan - #Tmedian[key+ '_mode'] = np.nan - key_std = np.nan - key_mode = np.nan - - #Tweight = pd.DataFrame([key_weighted_mean, key_std, Nphoton], index= [key+ '_weighted_mean', key+ '_std', 'N_photos' ]) - Tweight = pd.Series([key_weighted_mean, key_std, Nphoton, key_mode], index= [key+ '_weighted_mean', key+ '_std', 'N_photos', key+ '_mode' ]) + # Tmedian[key+ '_mode'] = np.nan + key_std = np.nan + key_mode = np.nan + # Tweight = pd.DataFrame([key_weighted_mean, key_std, Nphoton], index= [key+ '_weighted_mean', key+ '_std', 'N_photos' ]) + Tweight = pd.Series( + [key_weighted_mean, key_std, Nphoton, key_mode], + index=[key + "_weighted_mean", key + "_std", "N_photos", key + "_mode"], + ) - #print ( str( istancil) + ' s' + str(time.time() - tstart)) + # print ( str( istancil) + ' s' + str(time.time() - tstart)) return Tweight.T T_sets = list() @@ -315,55 +360,61 @@ def calc_stencil_stats(group, key, key_x_coord, stancil_width, stancils): for stancil_set in [stencil_1, stencil_1half]: # select photons that are in bins - Ti_sel = T2[ (stancil_set[0,0] < T2['x']) & (T2['x'] < stancil_set[2,-1]) ] + Ti_sel = T2[(stancil_set[0, 0] < T2["x"]) & (T2["x"] < stancil_set[2, -1])] # put each photon in a bin - bin_labels = np.searchsorted(stancil_set[0,:], Ti_sel['x']) - #bin_labels2 = np.digitize( Ti_sel['x'], stancil_set[0,:], right = True ) + bin_labels = np.searchsorted(stancil_set[0, :], Ti_sel["x"]) + # bin_labels2 = np.digitize( Ti_sel['x'], stancil_set[0,:], right = True ) - Ti_sel['x_bins'] =bin_labels + Ti_sel["x_bins"] = bin_labels # group data by this bin - Ti_g = Ti_sel.groupby(Ti_sel['x_bins'], dropna= False , as_index = True )#.median() + Ti_g = Ti_sel.groupby( + Ti_sel["x_bins"], dropna=False, as_index=True + ) # .median() # take median of the data Ti_median = Ti_g.median() # apply weighted mean and count photons - args = [ key_var, key_x_coord, stancil_width, stancil_set] + args = [key_var, key_x_coord, stancil_width, stancil_set] - #%timeit -r 1 -n 1 Ti_weight = Ti_g.apply(calc_stencil_stats, *args) - Ti_weight = Ti_g.apply(calc_stencil_stats, *args) + # %timeit -r 1 -n 1 Ti_weight = Ti_g.apply(calc_stencil_stats, *args) + Ti_weight = Ti_g.apply(calc_stencil_stats, *args) - #merge both datasets - T_merged = pd.concat( [Ti_median, Ti_weight], axis= 1) + # merge both datasets + T_merged = pd.concat([Ti_median, Ti_weight], axis=1) # rename columns - T_merged = T_merged.rename(columns={key_var: key_var+'_median', key_x_coord: key_x_coord+ '_median'}) - T_merged[ key_var+ '_median'][ np.isnan(T_merged[key_var+ '_std']) ] = np.nan # replace median calculation with nans + T_merged = T_merged.rename( + columns={key_var: key_var + "_median", key_x_coord: key_x_coord + "_median"} + ) + T_merged[key_var + "_median"][ + np.isnan(T_merged[key_var + "_std"]) + ] = np.nan # replace median calculation with nans # set stancil center an new x-coodinate - T_merged['x'] = stancil_set[1, T_merged.index-1] + T_merged["x"] = stancil_set[1, T_merged.index - 1] T_sets.append(T_merged) # mergeboth stancils - T3 = pd.concat(T_sets ).sort_values(by= 'x').reset_index() + T3 = pd.concat(T_sets).sort_values(by="x").reset_index() if plot_flag: - Ti_1, Ti_1half = T_sets - - plt.plot( Ti_1half.iloc[0:60].x, Ti_1half.iloc[0:60]['heights_c_median'], '.' ) - plt.plot( Ti_1.iloc[0:60].x, Ti_1.iloc[0:60]['heights_c_median'], '.' ) - plt.plot( T3.iloc[0:120].x, T3.iloc[0:120]['heights_c_median'], '-' ) + Ti_1, Ti_1half = T_sets + plt.plot(Ti_1half.iloc[0:60].x, Ti_1half.iloc[0:60]["heights_c_median"], ".") + plt.plot(Ti_1.iloc[0:60].x, Ti_1.iloc[0:60]["heights_c_median"], ".") + plt.plot(T3.iloc[0:120].x, T3.iloc[0:120]["heights_c_median"], "-") return T3 # this function is applied to beam: -#old version -def get_stencil_stats(T2, stencil_iter, key , key_x_coord, stancil_width , Nphoton_min = 5, map_func=None): - +# old version +def get_stencil_stats( + T2, stencil_iter, key, key_x_coord, stancil_width, Nphoton_min=5, map_func=None +): """ T2 pd.DAtaframe with beam data needs at least 'dist' and key stencil_iter iterable that constains the stancil boundaries and center [left boundary, center, right boundary] @@ -396,67 +447,72 @@ def calc_stencil_stats(istencil): "returns stats per stencil" tstart = time.time() - i_mask = (x_data >= istencil[0]) & (x_data < istencil[2]) - Nphoton = sum(i_mask) + i_mask = (x_data >= istencil[0]) & (x_data < istencil[2]) + Nphoton = sum(i_mask) if Nphoton < Nphoton_min: Tmedian = T2[i_mask].median() - Tmedian[key+ '_weighted_mean'] = np.nan - Tmedian[key+ '_mode'] = np.nan - Tmedian['N_photos'] = Nphoton - Tmedian[key+ '_std'] = np.nan + Tmedian[key + "_weighted_mean"] = np.nan + Tmedian[key + "_mode"] = np.nan + Tmedian["N_photos"] = Nphoton + Tmedian[key + "_std"] = np.nan return istencil[1], Tmedian + x_rel = (x_data[i_mask] - istencil[1]) / stancil_width + y = y_data[i_mask] - x_rel = (x_data[i_mask] - istencil[1])/ stancil_width - y = y_data[i_mask] - - Tmedian = T2[i_mask].median() - Tmedian[key+ '_weighted_mean'] = weighted_mean(x_rel, y) - Tmedian[key+ '_mode'] = get_mode(y) - Tmedian['N_photos'] = Nphoton - Tmedian[key+ '_std'] = y.std() - #Tmedian[key+ '_median'][ np.isnan(Tmedian[key+ 'std']) ]= np.nan # replace median calculation with nans - print ( str( istencil[1]) + ' s' + str(time.time() - tstart)) + Tmedian = T2[i_mask].median() + Tmedian[key + "_weighted_mean"] = weighted_mean(x_rel, y) + Tmedian[key + "_mode"] = get_mode(y) + Tmedian["N_photos"] = Nphoton + Tmedian[key + "_std"] = y.std() + # Tmedian[key+ '_median'][ np.isnan(Tmedian[key+ 'std']) ]= np.nan # replace median calculation with nans + print(str(istencil[1]) + " s" + str(time.time() - tstart)) return istencil[1], Tmedian # apply func to all stancils map_func = map if map_func is None else map_func - D_filt = dict(map_func(calc_stencil_stats, stencil_iter)) - - DF_filt = pd.DataFrame.from_dict(D_filt, orient='index') - DF_filt = DF_filt.rename(columns={key: key+'_median', key_x_coord: 'median_'+key_x_coord}) - DF_filt[ key+ '_median'][ np.isnan(DF_filt[key+ '_std']) ] = np.nan # replace median calculation with nans + D_filt = dict(map_func(calc_stencil_stats, stencil_iter)) + + DF_filt = pd.DataFrame.from_dict(D_filt, orient="index") + DF_filt = DF_filt.rename( + columns={key: key + "_median", key_x_coord: "median_" + key_x_coord} + ) + DF_filt[key + "_median"][ + np.isnan(DF_filt[key + "_std"]) + ] = np.nan # replace median calculation with nans DF_filt[key_x_coord] = DF_filt.index - DF_filt = DF_filt.reset_index() + DF_filt = DF_filt.reset_index() return DF_filt + # %% old version # define common dist_grid: -#dx= 5 # 2 * resolution in meters, datapoint +-dx are used to take the mean -#dist_grid = np.arange( np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0), dx ) +# dx= 5 # 2 * resolution in meters, datapoint +-dx are used to take the mean +# dist_grid = np.arange( np.nanmin(dist_list[:, 0], 0) , np.nanmax(dist_list[:, 1], 0), dx ) + # derive bin means def bin_means(T2, dist_grid): - dF_mean = pd.DataFrame(index =T2.columns) - ilim = int(len(dist_grid)) - N_i = list() + dF_mean = pd.DataFrame(index=T2.columns) + ilim = int(len(dist_grid)) + N_i = list() - for i in np.arange(1,ilim-1, 1): - if i % 5000 ==0: + for i in np.arange(1, ilim - 1, 1): + if i % 5000 == 0: print(i) - i_mask=(T2['dist'] >= dist_grid[i-1]) & (T2['dist'] < dist_grid[i+1]) - #if ( (T2['dist'] >= dist_grid[i-1]) & (T2['dist'] < dist_grid[i+1]) ).sum() > 0: + i_mask = (T2["dist"] >= dist_grid[i - 1]) & (T2["dist"] < dist_grid[i + 1]) + # if ( (T2['dist'] >= dist_grid[i-1]) & (T2['dist'] < dist_grid[i+1]) ).sum() > 0: dF_mean[i] = T2[i_mask].mean() - #dF_median[i] = T2[i_mask].median() + # dF_median[i] = T2[i_mask].median() N_i.append(i_mask.sum()) - dF_mean = dF_mean.T - dF_mean['N_photos'] = N_i - dF_mean['dist'] = dist_grid[np.arange(1,ilim-1, 1)] + dF_mean = dF_mean.T + dF_mean["N_photos"] = N_i + dF_mean["dist"] = dist_grid[np.arange(1, ilim - 1, 1)] return dF_mean diff --git a/src/icesat2_tracks/ICEsat2_SI_tools/lanczos.py b/src/icesat2_tracks/ICEsat2_SI_tools/lanczos.py index 3396b57f..7cbbaad9 100644 --- a/src/icesat2_tracks/ICEsat2_SI_tools/lanczos.py +++ b/src/icesat2_tracks/ICEsat2_SI_tools/lanczos.py @@ -1,6 +1,7 @@ import numpy as np -def lanczos_1d(width, dx, a=2): + +def lanczos_1d(width, dx, a=2): """ This is a 1D lanczos Filter for time series analysis. it generates the Filter to be convolved with the timeseries @@ -18,25 +19,26 @@ def lanczos_1d(width, dx, a=2): # a= 1 # Lanczos parameter. the length of the filter is a*width # dx= .1 # deltax of the to be filtered timeseries - r=width/2.0 - xl=a*r - x= np.arange(-xl, xl, dx) - xprime=x/r + r = width / 2.0 + xl = a * r + x = np.arange(-xl, xl, dx) + xprime = x / r # define the filter - L = np.sinc(xprime) * np.sinc(xprime/a) - L = np.where((xprime > -a) & (xprime < a),L, 0) + L = np.sinc(xprime) * np.sinc(xprime / a) + L = np.where((xprime > -a) & (xprime < a), L, 0) - return x, L/L.sum() + return x, L / L.sum() -def lanczos_2d(width , dx, a=2): - x , L =lanczos_1d(width, dx, a=a) - L2d =np.outer(L ,L.T) +def lanczos_2d(width, dx, a=2): + x, L = lanczos_1d(width, dx, a=a) + L2d = np.outer(L, L.T) return x, L2d -def lanczos_filter_1d(x, data, width, a=2 , mode='same', method='direct'): + +def lanczos_filter_1d(x, data, width, a=2, mode="same", method="direct"): """ colvolves the lanzcos filter with data. inputs @@ -52,15 +54,16 @@ def lanczos_filter_1d(x, data, width, a=2 , mode='same', method='direct'): data_lp low-passed data, same size as before. """ import scipy.signal as signal - dx = np.diff(x).mean() - x , L = lanczos_1d(width, dx, a=a) + dx = np.diff(x).mean() + x, L = lanczos_1d(width, dx, a=a) - data_lp= signal.convolve(data, L, mode=mode, method=method)#* + data_lp = signal.convolve(data, L, mode=mode, method=method) # * return data_lp -def lanczos_filter_1d_wrapping(x, data, width, a=2 , mode='wrap'): + +def lanczos_filter_1d_wrapping(x, data, width, a=2, mode="wrap"): """ colvolves the lanzcos filter with data. same as lanczos_filter_1d but can wrap around @@ -78,16 +81,16 @@ def lanczos_filter_1d_wrapping(x, data, width, a=2 , mode='wrap'): data_lp low-passed data, same size as before. """ import scipy.ndimage - dx = np.diff(x).mean() - x , L = lanczos_1d(width, dx, a=a) - data_lp= scipy.ndimage.convolve(data, L, mode=mode)#* + dx = np.diff(x).mean() + x, L = lanczos_1d(width, dx, a=a) - return data_lp + data_lp = scipy.ndimage.convolve(data, L, mode=mode) # * + return data_lp -def lanczos_filter_2d(x, data, width, a=2 , mode='same'): +def lanczos_filter_2d(x, data, width, a=2, mode="same"): """ colvolves the lanzcos filter with data in 3 dimensions. inputs @@ -104,27 +107,28 @@ def lanczos_filter_2d(x, data, width, a=2 , mode='same'): """ import scipy.ndimage.filters as signal - #import scipy.ndimage.filters as signal - dx = abs(np.diff(x).mean()) - x , L2d = lanczos_2d(width, dx, a=a) + # import scipy.ndimage.filters as signal + dx = abs(np.diff(x).mean()) + x, L2d = lanczos_2d(width, dx, a=a) - - data_lp= signal.convolve(data, L2d, mode=mode)#* + data_lp = signal.convolve(data, L2d, mode=mode) # * return data_lp -def lanczos_filter_2d_apply(data, x, width, a=2 , mode='same'): - return lanczos_filter_2d(x, data, width, a=a , mode=mode) -def lanczos_3d(width , dx, a=2): - x , L =lanczos_1d(width, dx, a=a) - L2d =np.outer(L ,L.T) +def lanczos_filter_2d_apply(data, x, width, a=2, mode="same"): + return lanczos_filter_2d(x, data, width, a=a, mode=mode) + + +def lanczos_3d(width, dx, a=2): + x, L = lanczos_1d(width, dx, a=a) + L2d = np.outer(L, L.T) L3d = np.multiply.outer(L2d, L.T) return x, L3d -def lanczos_filter_3d(x, data, width, a=2 , mode='same'): +def lanczos_filter_3d(x, data, width, a=2, mode="same"): """ colvolves the lanzcos filter with data in 3 dimensions. inputs @@ -141,11 +145,10 @@ def lanczos_filter_3d(x, data, width, a=2 , mode='same'): """ import scipy.ndimage.filters as signal - #import scipy.ndimage.filters as signal - dx = abs(np.diff(x).mean()) - x , L3d = lanczos_3d(width, dx, a=a) - + # import scipy.ndimage.filters as signal + dx = abs(np.diff(x).mean()) + x, L3d = lanczos_3d(width, dx, a=a) - data_lp= signal.convolve(data, L3d, mode=mode)#* + data_lp = signal.convolve(data, L3d, mode=mode) # * return data_lp diff --git a/src/icesat2_tracks/ICEsat2_SI_tools/read_ground_tracks.py b/src/icesat2_tracks/ICEsat2_SI_tools/read_ground_tracks.py index e449fef6..1777d1f6 100644 --- a/src/icesat2_tracks/ICEsat2_SI_tools/read_ground_tracks.py +++ b/src/icesat2_tracks/ICEsat2_SI_tools/read_ground_tracks.py @@ -8,130 +8,146 @@ import osgeo.ogr import matplotlib.pyplot as plt + # PURPOSE: read ICESat-2 ground tracks for TRACK and BEAM def read_ICESat2_groundtrack(input_file): # decompress and parse KMZ file input_file = pathlib.Path(input_file).expanduser().absolute() - kmls = zipfile.ZipFile(str(input_file), 'r') + kmls = zipfile.ZipFile(str(input_file), "r") parser = lxml.etree.XMLParser(recover=True, remove_blank_text=True) # list of geodataframes for all RGTs RGTS = [] # for each kml in the zipfile (per RGT) for kml in kmls.filelist: - tree = lxml.etree.parse(kmls.open(kml, 'r'), parser) + tree = lxml.etree.parse(kmls.open(kml, "r"), parser) root = tree.getroot() # create list of rows rows = [] # find documents within kmz file - for document in root.iterfind('.//kml:Document//Folder//Placemark', root.nsmap): + for document in root.iterfind(".//kml:Document//Folder//Placemark", root.nsmap): # extract laser name, satellite track and coordinates of line strings - description = document.find('description', root.nsmap).text + description = document.find("description", root.nsmap).text columns = {} - columns['RGT'], = re.findall(r'RGT\s(\d+)',description) + (columns["RGT"],) = re.findall(r"RGT\s(\d+)", description) # get date and time - date = re.findall(r'\d{2}\-\w{3}\-\d{4}\s\d{2}\:\d{2}\:\d{2}', description).pop() - columns['date'] = geopandas.pd.to_datetime(date, format='%d-%b-%Y %H:%M:%S') - columns['DOY'], = re.findall(r'DOY\s(\d+)', description) - columns['cycle'], = re.findall(r'Cycle\s(\d+)', description) - coords = document.findall('Point/coordinates',root.nsmap) + date = re.findall( + r"\d{2}\-\w{3}\-\d{4}\s\d{2}\:\d{2}\:\d{2}", description + ).pop() + columns["date"] = geopandas.pd.to_datetime(date, format="%d-%b-%Y %H:%M:%S") + (columns["DOY"],) = re.findall(r"DOY\s(\d+)", description) + (columns["cycle"],) = re.findall(r"Cycle\s(\d+)", description) + coords = document.findall("Point/coordinates", root.nsmap) # for each set of coordinates - for i,c in enumerate(coords): - points = np.array([x.split(',')[:2] for x in c.text.split()],dtype='f8') - columns['geometry'], = geopandas.points_from_xy(points[:,0],points[:,1]) + for i, c in enumerate(coords): + points = np.array( + [x.split(",")[:2] for x in c.text.split()], dtype="f8" + ) + (columns["geometry"],) = geopandas.points_from_xy( + points[:, 0], points[:, 1] + ) rows.append(columns) # create geopandas geodataframe for points gdf = geopandas.GeoDataFrame(rows) RGTS.append(gdf) # return the concatenated geodataframe - return geopandas.pd.concat(RGTS).set_index('date') + return geopandas.pd.concat(RGTS).set_index("date") # PURPOSE: read ICESat-2 ground tracks def ICESat2_mission_groundtrack(input_file): # decompress and parse KMZ file input_file = pathlib.Path(input_file).expanduser().absolute() - kmzs = zipfile.ZipFile(str(input_file), 'r') + kmzs = zipfile.ZipFile(str(input_file), "r") parser = lxml.etree.XMLParser(recover=True, remove_blank_text=True) # for each kml in the zipfile (per GT) GTs = [] for kmz in kmzs.filelist: - kmls = zipfile.ZipFile(kmzs.open(kmz, 'r')) + kmls = zipfile.ZipFile(kmzs.open(kmz, "r")) for kml in kmls.filelist: - tree = lxml.etree.parse(kmls.open(kml, 'r'), parser) + tree = lxml.etree.parse(kmls.open(kml, "r"), parser) root = tree.getroot() # find documents within kmz file - for document in root.iterfind('.//kml:Document', root.nsmap): + for document in root.iterfind(".//kml:Document", root.nsmap): # extract laser name, satellite track and coordinates of line strings - name = document.find('name',root.nsmap).text - placemarks = document.findall('Placemark/name',root.nsmap) - coords = document.findall('Placemark/LineString/coordinates',root.nsmap) + name = document.find("name", root.nsmap).text + placemarks = document.findall("Placemark/name", root.nsmap) + coords = document.findall( + "Placemark/LineString/coordinates", root.nsmap + ) # create list of rows rows = [] wkt = [] # for each set of coordinates - for i,c in enumerate(coords): + for i, c in enumerate(coords): columns = {} - columns['Laser'], = re.findall(r'laser(\d+)', name) - columns['GT'], = re.findall(r'GT\d[LR]?', kmz.filename) - columns['RGT'] = int(placemarks[i].text) - coords = document.findall('Point/coordinates',root.nsmap) + (columns["Laser"],) = re.findall(r"laser(\d+)", name) + (columns["GT"],) = re.findall(r"GT\d[LR]?", kmz.filename) + columns["RGT"] = int(placemarks[i].text) + coords = document.findall("Point/coordinates", root.nsmap) # create LineString object linestring = osgeo.ogr.Geometry(osgeo.ogr.wkbLineString) - line = np.array([x.split(',')[:2] for x in c.text.split()],dtype='f8') - for ln,lt in zip(line[:,0],line[:,1]): + line = np.array( + [x.split(",")[:2] for x in c.text.split()], dtype="f8" + ) + for ln, lt in zip(line[:, 0], line[:, 1]): linestring.AddPoint(ln, lt) # convert to wkt and then add to geometry wkt.append(linestring.ExportToWkt()) rows.append(columns) # create geopandas geodataframe for points - gdf = geopandas.GeoDataFrame(rows, - geometry=geopandas.GeoSeries.from_wkt(wkt) + gdf = geopandas.GeoDataFrame( + rows, geometry=geopandas.GeoSeries.from_wkt(wkt) ) GTs.append(gdf) # return the concatenated geodataframe return geopandas.pd.concat(GTs) + # PURPOSE: read ICESat-2 mission ground tracks as points def ICESat2_mission_points(input_file): # decompress and parse KMZ file input_file = pathlib.Path(input_file).expanduser().absolute() - kmzs = zipfile.ZipFile(str(input_file), 'r') + kmzs = zipfile.ZipFile(str(input_file), "r") parser = lxml.etree.XMLParser(recover=True, remove_blank_text=True) # for each kml in the zipfile (per GT) GTs = [] for kmz in kmzs.filelist: - kmls = zipfile.ZipFile(kmzs.open(kmz, 'r')) + kmls = zipfile.ZipFile(kmzs.open(kmz, "r")) for kml in kmls.filelist: - tree = lxml.etree.parse(kmls.open(kml, 'r'), parser) + tree = lxml.etree.parse(kmls.open(kml, "r"), parser) root = tree.getroot() # find documents within kmz file - for document in root.iterfind('.//kml:Document', root.nsmap): + for document in root.iterfind(".//kml:Document", root.nsmap): # extract laser name, satellite track and coordinates of line strings - name = document.find('name', root.nsmap).text - placemarks = document.findall('Placemark/name', root.nsmap) - coords = document.findall('Placemark/LineString/coordinates', root.nsmap) + name = document.find("name", root.nsmap).text + placemarks = document.findall("Placemark/name", root.nsmap) + coords = document.findall( + "Placemark/LineString/coordinates", root.nsmap + ) # create list of rows rows = [] - x = []; y = [] + x = [] + y = [] # for each set of coordinates - for i,c in enumerate(coords): + for i, c in enumerate(coords): # create a line string of coordinates - line = np.array([x.split(',')[:2] for x in c.text.split()], dtype='f8') - for ln,lt in zip(line[:,0], line[:,1]): + line = np.array( + [x.split(",")[:2] for x in c.text.split()], dtype="f8" + ) + for ln, lt in zip(line[:, 0], line[:, 1]): columns = {} - columns['Laser'], = re.findall(r'laser(\d+)', name) - columns['GT'], = re.findall(r'GT\d[LR]?', kmz.filename) - columns['RGT'] = int(placemarks[i].text) + (columns["Laser"],) = re.findall(r"laser(\d+)", name) + (columns["GT"],) = re.findall(r"GT\d[LR]?", kmz.filename) + columns["RGT"] = int(placemarks[i].text) rows.append(columns) x.append(ln) y.append(lt) # create geopandas geodataframe for points - gdf = geopandas.GeoDataFrame(rows, - geometry=geopandas.points_from_xy(x,y) + gdf = geopandas.GeoDataFrame( + rows, geometry=geopandas.points_from_xy(x, y) ) GTs.append(gdf) # return the concatenated and georefernced geodataframe G = geopandas.pd.concat(GTs) - G.geometry.crs = {'init': 'epsg:4326'} + G.geometry.crs = {"init": "epsg:4326"} return G - diff --git a/src/icesat2_tracks/ICEsat2_SI_tools/spectral_estimates.py b/src/icesat2_tracks/ICEsat2_SI_tools/spectral_estimates.py index d46d9ea2..bcc8afde 100644 --- a/src/icesat2_tracks/ICEsat2_SI_tools/spectral_estimates.py +++ b/src/icesat2_tracks/ICEsat2_SI_tools/spectral_estimates.py @@ -2,7 +2,7 @@ # basic functions -def create_chunk_boundaries(L, dsize, ov= None, iter_flag=True): +def create_chunk_boundaries(L, dsize, ov=None, iter_flag=True): """ returns all need chunk boudaries and center position given L, and ov inputs: @@ -13,17 +13,18 @@ def create_chunk_boundaries(L, dsize, ov= None, iter_flag=True): if iter_flag True returns iter else it returns an ndarray """ - ov=int(np.round(L/2)) if ov is None else ov + ov = int(np.round(L / 2)) if ov is None else ov - - xleft = np.arange(0,dsize-int(L-ov),int(L-ov)) - xright = np.arange(int(L-ov)*2,dsize+1,int(L-ov)) - xcenter_pos = np.arange(int(L-ov),dsize-int(L-ov)+1,int(L-ov)) - max_size = min([xleft.size , xcenter_pos.size, xright.size]) + xleft = np.arange(0, dsize - int(L - ov), int(L - ov)) + xright = np.arange(int(L - ov) * 2, dsize + 1, int(L - ov)) + xcenter_pos = np.arange(int(L - ov), dsize - int(L - ov) + 1, int(L - ov)) + max_size = min([xleft.size, xcenter_pos.size, xright.size]) # if xright[max_size-1] < dsize: # print('left out last ' + str(dsize- xright[max_size-1]) + ' data points' ) - #print([xleft[0:max_size], xcenter_pos[0:max_size], xright[0:max_size]]) - position_stancil = np.vstack([xleft[0:max_size], xcenter_pos[0:max_size], xright[0:max_size]]) + # print([xleft[0:max_size], xcenter_pos[0:max_size], xright[0:max_size]]) + position_stancil = np.vstack( + [xleft[0:max_size], xcenter_pos[0:max_size], xright[0:max_size]] + ) if iter_flag is True: return iter(position_stancil.T.tolist()) @@ -31,7 +32,7 @@ def create_chunk_boundaries(L, dsize, ov= None, iter_flag=True): return position_stancil -def create_chunk_boundaries_unit_lengths(L_unit, data_limits, ov= None, iter_flag=True): +def create_chunk_boundaries_unit_lengths(L_unit, data_limits, ov=None, iter_flag=True): """ returns all need chunk boudaries and center position given L, and ov inputs: @@ -42,74 +43,78 @@ def create_chunk_boundaries_unit_lengths(L_unit, data_limits, ov= None, iter_fl if iter_flag True returns iter else it returns an ndarray """ - L= L_unit - ov=np.round(L/2) if ov is None else ov - #print(ov) - dl = (L-ov) - xleft = np.arange(data_limits[0] , data_limits[1]-dl, dl ) - xcenter_pos = np.arange(data_limits[0]+ L/2 , data_limits[1]-dl+1, dl ) - xright = np.arange(data_limits[0] + L , data_limits[1]+1, dl ) - - - max_size = min([xleft.size , xcenter_pos.size, xright.size]) + L = L_unit + ov = np.round(L / 2) if ov is None else ov + # print(ov) + dl = L - ov + xleft = np.arange(data_limits[0], data_limits[1] - dl, dl) + xcenter_pos = np.arange(data_limits[0] + L / 2, data_limits[1] - dl + 1, dl) + xright = np.arange(data_limits[0] + L, data_limits[1] + 1, dl) + + max_size = min([xleft.size, xcenter_pos.size, xright.size]) # if xright[max_size-1] < data_limits[1]: # print('left out last ' + str(data_limits[1]- xright[max_size-1]) + ' data points' ) - #print([xleft[0:max_size], xcenter_pos[0:max_size], xright[0:max_size]]) - position_stancil = np.vstack([xleft[0:max_size], xcenter_pos[0:max_size], xright[0:max_size]]) + # print([xleft[0:max_size], xcenter_pos[0:max_size], xright[0:max_size]]) + position_stancil = np.vstack( + [xleft[0:max_size], xcenter_pos[0:max_size], xright[0:max_size]] + ) if iter_flag is True: return iter(position_stancil.T.tolist()) else: return position_stancil + def Z_to_power(Z, df, N): - """ compute the 1d spectrum of a field phi + """compute the 1d spectrum of a field phi inputs: Z complex fourier coefficients df frequency / or wavenumber step N length of data vector in real space (= L) """ - spec = 2.*(Z*Z.conj()).real / df /N**2 - neven = True if (N%2) else False + spec = 2.0 * (Z * Z.conj()).real / df / N**2 + neven = True if (N % 2) else False # the zeroth frequency should be counted only once - spec[0] = spec[0]/2. + spec[0] = spec[0] / 2.0 if neven: - spec[-1] = spec[-1]/2. + spec[-1] = spec[-1] / 2.0 return spec + # 2nd cal spectra def calc_spectrum_fft(phi, df, N): - """ compute the 1d spectrum of a field phi + """compute the 1d spectrum of a field phi inputs: df frequency / or wavenumber step N length of data vector in real space (= L) neven bool, if True """ - neven = True if (N%2) else False + neven = True if (N % 2) else False phih = np.fft.rfft(phi) # the factor of 2 comes from the symmetry of the Fourier coeffs - spec = 2.*(phih*phih.conj()).real / df /N**2 + spec = 2.0 * (phih * phih.conj()).real / df / N**2 # the zeroth frequency should be counted only once - spec[0] = spec[0]/2. + spec[0] = spec[0] / 2.0 if neven: - spec[-1] = spec[-1]/2. + spec[-1] = spec[-1] / 2.0 return spec -def LS_power_to_PSD( ls_power, L , dff): +def LS_power_to_PSD(ls_power, L, dff): """ returns Power spectral density (unit^2/dfreq) ls_power output of astropy.timeseries.LombScargle.power with normalization='psd' """ - return 2 * ls_power / L /dff + return 2 * ls_power / L / dff -def calc_spectrum_LS( x, y, k, err=None, LS= None, dk =None): + +def calc_spectrum_LS(x, y, k, err=None, LS=None, dk=None): """ returns: Power spectral density of y given postitions x, for wanumbers k @@ -119,16 +124,18 @@ def calc_spectrum_LS( x, y, k, err=None, LS= None, dk =None): """ if LS is None: from astropy.timeseries import LombScargle - LS = LombScargle(x , y, dy = err, fit_mean=False, center_data=True) + + LS = LombScargle(x, y, dy=err, fit_mean=False, center_data=True) else: LS.t = x LS.y = y - LS.dy= err + LS.dy = err + + ls_power = LS.power(k, normalization="psd", assume_regular_frequency="False") - ls_power = LS.power(k, normalization='psd', assume_regular_frequency='False') + dk = np.diff(k).mean() if dk is None else dk + return 2 * ls_power / y.size / dk, LS - dk = np.diff(k).mean() if dk is None else dk - return 2 * ls_power / y.size / dk , LS def reconstruct_data_from_LS(LS, x_real_axis, freq): """ @@ -141,16 +148,18 @@ def reconstruct_data_from_LS(LS, x_real_axis, freq): y_reconstruct = LS.offset() * np.ones(len(x_real_axis)) freq_seq = freq[1:] if freq[0] == 0 else freq - freq_mask= freq <= 1/100 + freq_mask = freq <= 1 / 100 - #print(freq_seq) + # print(freq_seq) for fi in freq_seq: try: theta = LS.model_parameters(fi) except: - theta = [0,0] - y_reconstruct += theta[0] * np.sin(x_real_axis * 2 * np.pi *fi ) + theta[1]* np.cos(x_real_axis * 2 * np.pi *fi) - #y_reconstruct += LS.model(x_real_axis, fi) + theta = [0, 0] + y_reconstruct += theta[0] * np.sin(x_real_axis * 2 * np.pi * fi) + theta[ + 1 + ] * np.cos(x_real_axis * 2 * np.pi * fi) + # y_reconstruct += LS.model(x_real_axis, fi) # for fi in freq_seq[:]: # theta = LS.model_parameters(fi) @@ -159,20 +168,22 @@ def reconstruct_data_from_LS(LS, x_real_axis, freq): return y_reconstruct + def calc_freq_fft(x_grid, N): - """ calculate array of spectral variable (frequency or - wavenumber) in cycles per unit of L """ + """calculate array of spectral variable (frequency or + wavenumber) in cycles per unit of L""" - neven = True if (N%2) else False - #dx=np.diff(x_grid).mean() - dx = np.round(np.median(np.diff(x_grid) ), 1) - df = 1./((N-1)*dx) + neven = True if (N % 2) else False + # dx=np.diff(x_grid).mean() + dx = np.round(np.median(np.diff(x_grid)), 1) + df = 1.0 / ((N - 1) * dx) if neven: - f = df*np.arange(N/2+1) + f = df * np.arange(N / 2 + 1) else: - f = df*np.arange( (N-1)/2. + 1 ) - return f,df + f = df * np.arange((N - 1) / 2.0 + 1) + return f, df + def calc_freq_fft_given_dx(dx, N): """ @@ -182,17 +193,25 @@ def calc_freq_fft_given_dx(dx, N): N number of datapoints used in window """ - neven = True if (N%2) else False - df = 1./((N-1)*dx) + neven = True if (N % 2) else False + df = 1.0 / ((N - 1) * dx) if neven: - f = df*np.arange(N/2+1) + f = df * np.arange(N / 2 + 1) else: - f = df*np.arange( (N-1)/2. + 1 ) - return f,df - - -def calc_freq_LS(x, N, method='fftX2', dx=None, minimum_frequency=None, maximum_frequency=None, samples_per_peak=0.01): + f = df * np.arange((N - 1) / 2.0 + 1) + return f, df + + +def calc_freq_LS( + x, + N, + method="fftX2", + dx=None, + minimum_frequency=None, + maximum_frequency=None, + samples_per_peak=0.01, +): """ calculate array of spectral variable (frequency or wavenumber) in cycles per unit of N (window length in number of data points) @@ -204,42 +223,48 @@ def calc_freq_LS(x, N, method='fftX2', dx=None, minimum_frequency=None, maximum_ minimum_frequency, maximum_frequency only used for LS_auto """ - if method is 'fftX2': - neven = True if (N%2) else False + if method is "fftX2": + neven = True if (N % 2) else False dx = np.diff(x).mean() if dx is None else dx - df = 1./((N-1)*dx) /2 + df = 1.0 / ((N - 1) * dx) / 2 if neven: - f = df*np.arange(df, N+1) + f = df * np.arange(df, N + 1) else: - f = df* np.arange(df, (N-1) + 1 ) + f = df * np.arange(df, (N - 1) + 1) - elif method is 'fft': - neven = True if (N%2) else False + elif method is "fft": + neven = True if (N % 2) else False dx = np.diff(x).mean() if dx is None else dx - df = 1./((N-1)*dx) + df = 1.0 / ((N - 1) * dx) if neven: - f = df*np.arange(N/2+1) + f = df * np.arange(N / 2 + 1) else: - f = df*np.arange( (N-1)/2. + 1 ) + f = df * np.arange((N - 1) / 2.0 + 1) - elif method is 'LS_auto': + elif method is "LS_auto": from astropy.timeseries import LombScargle - f = LombScargle(x , np.random.randn(len(x)), fit_mean=True).autofrequency(minimum_frequency=minimum_frequency, maximum_frequency=maximum_frequency, samples_per_peak=samples_per_peak)##0.1) + + f = LombScargle(x, np.random.randn(len(x)), fit_mean=True).autofrequency( + minimum_frequency=minimum_frequency, + maximum_frequency=maximum_frequency, + samples_per_peak=samples_per_peak, + ) ##0.1) df = np.diff(f).mean() df = np.round(df, 5) - elif method is 'fixed_ratio': + elif method is "fixed_ratio": - neven = True if (N%2) else False + neven = True if (N % 2) else False dx = np.diff(x).mean() if dx is None else dx df = dx / 50 if neven: - f = df*np.arange(df, N +1) + f = df * np.arange(df, N + 1) else: - f = df* np.arange(df, N ) + f = df * np.arange(df, N) + + return f, df - return f ,df def create_window(L, window=None): """ @@ -247,65 +272,66 @@ def create_window(L, window=None): if window is not None it show have a length of N """ if window is None: - win=np.hanning(L) + win = np.hanning(L) else: - win=window + win = window - factor=np.sqrt(L/(win**2).sum()) - win*=factor + factor = np.sqrt(L / (win**2).sum()) + win *= factor return win + def create_weighted_window(data, window=None): """ define window function and weight it to conserve variance if window is not None it show have a length of N """ import scipy.signal.windows as WINDOWS + L = data.size if window is None: - #win=np.hanning(L) - win = WINDOWS.tukey(L, alpha=0.1, sym=True) + # win=np.hanning(L) + win = WINDOWS.tukey(L, alpha=0.1, sym=True) else: - win=window + win = window - factor = np.sqrt( np.var(data) / np.var(( data* win) ) ) - #factor=np.sqrt( L/(win**2).sum()) - win *= factor + factor = np.sqrt(np.var(data) / np.var((data * win))) + # factor=np.sqrt( L/(win**2).sum()) + win *= factor return win -def spec_error(E,sn,ci=.95): - - """ Computes confidence interval for one-dimensional spectral - estimate E (the power spectra). - Parameters - =========== - - sn is the number of spectral realizations; - it can be either an scalar or an array of size(E) - - ci = .95 for 95 % confidence interval +def spec_error(E, sn, ci=0.95): + """Computes confidence interval for one-dimensional spectral + estimate E (the power spectra). - Output - ========== - lower (El) and upper (Eu) bounds on E """ + Parameters + =========== + - sn is the number of spectral realizations; + it can be either an scalar or an array of size(E) + - ci = .95 for 95 % confidence interval + Output + ========== + lower (El) and upper (Eu) bounds on E""" - - def yNlu(sn,yN,ci): - """ compute yN[l] yN[u], that is, the lower and - upper limit of yN """ + def yNlu(sn, yN, ci): + """compute yN[l] yN[u], that is, the lower and + upper limit of yN""" from scipy.special import gammainc + # cdf of chi^2 dist. with 2*sn DOF - cdf = gammainc(sn,sn*yN) + cdf = gammainc(sn, sn * yN) # indices that delimit the wedge of the conf. interval fl = np.abs(cdf - ci).argmin() - fu = np.abs(cdf - 1. + ci).argmin() + fu = np.abs(cdf - 1.0 + ci).argmin() - return yN[fl],yN[fu] + return yN[fl], yN[fu] - dbin = .005 - yN = np.arange(0,2.+dbin,dbin) + dbin = 0.005 + yN = np.arange(0, 2.0 + dbin, dbin) El, Eu = np.empty_like(E), np.empty_like(E) @@ -319,19 +345,19 @@ def yNlu(sn,yN,ci): assert n == E.size, " *** sn has different size than E " for i in range(n): - yNl,yNu = yNlu(sn[i],yN=yN,ci=ci) - El[i] = E[i]/yNl - Eu[i] = E[i]/yNu + yNl, yNu = yNlu(sn[i], yN=yN, ci=ci) + El[i] = E[i] / yNl + Eu[i] = E[i] / yNu else: - yNl,yNu = yNlu(sn,yN=yN,ci=ci) - El = E/yNl - Eu = E/yNu + yNl, yNu = yNlu(sn, yN=yN, ci=ci) + El = E / yNl + Eu = E / yNu return El, Eu -def linear_gap_fill(F, key_lead, key_int): +def linear_gap_fill(F, key_lead, key_int): """ F pd.DataFrame key_lead key in F that determined the independent coordindate @@ -339,13 +365,13 @@ def linear_gap_fill(F, key_lead, key_int): """ y_g = np.array(F[key_int]) - nans, x2= np.isnan(y_g), lambda z: z.nonzero()[0] - y_g[nans]= np.interp(x2(nans), x2(~nans), y_g[~nans]) + nans, x2 = np.isnan(y_g), lambda z: z.nonzero()[0] + y_g[nans] = np.interp(x2(nans), x2(~nans), y_g[~nans]) return y_g -def sub_sample_coords(X, lons, lats, stancils, map_func =None): +def sub_sample_coords(X, lons, lats, stancils, map_func=None): """ X non-nan array of coodinate along beam lons, lats arrays of postion data that should be mapped. must have same size as X @@ -357,20 +383,21 @@ def sub_sample_coords(X, lons, lats, stancils, map_func =None): """ import copy + def get_lon_lat_coords(stancil): - x_mask= (stancil[0] <= X) & (X <= stancil[-1]) - #print(stancil[1]) - if sum(x_mask) ==0: # if there are not enough photos set results to nan + x_mask = (stancil[0] <= X) & (X <= stancil[-1]) + # print(stancil[1]) + if sum(x_mask) == 0: # if there are not enough photos set results to nan return np.array([stancil[1], np.nan, np.nan]) lon_bin, lat_bin = lons[x_mask].mean(), lats[x_mask].mean() - return np.array([stancil[1],lon_bin, lat_bin]) + return np.array([stancil[1], lon_bin, lat_bin]) map_func = map if map_func is None else map_func - #print(map_func) - coord_positions = list(map_func( get_lon_lat_coords, copy.copy(stancils) )) + # print(map_func) + coord_positions = list(map_func(get_lon_lat_coords, copy.copy(stancils))) coord_positions = np.vstack(coord_positions) return coord_positions @@ -394,23 +421,26 @@ def __init__(self, x_grid, data, Lpoints, ov=None, window=None): other arributes are in the .attr dict. """ - self.Lpoints = Lpoints - self.ov = int(Lpoints/2) if ov is None else ov #when not defined in create_chunk_boundaries then L/2 + self.Lpoints = Lpoints + self.ov = ( + int(Lpoints / 2) if ov is None else ov + ) # when not defined in create_chunk_boundaries then L/2 - self.data = data + self.data = data # create subsample k - self.k, self.dk = calc_freq_fft(x_grid, Lpoints) # return 1/ unit of frid points + self.k, self.dk = calc_freq_fft( + x_grid, Lpoints + ) # return 1/ unit of frid points # to get the waveumber units (2 pi/ lambda), multiply by 2 pi - self.k, self.dk = self.k * 2 * np.pi, self.dk * 2 * np.pi + self.k, self.dk = self.k * 2 * np.pi, self.dk * 2 * np.pi # print(self.k[0], self.k[-1]) # print(self.dk) # create window - self.win = create_window(Lpoints) + self.win = create_window(Lpoints) def cal_spectrogram(self, data=None, name=None): - """ defines apply function and calculated all sub-sample sprectra using map """ @@ -418,102 +448,109 @@ def cal_spectrogram(self, data=None, name=None): DATA = self.data if data is None else data Lpoints, dk = self.Lpoints, self.dk - win =self.win + win = self.win def calc_spectrum_apply(stancil): "returns spectrum per stencil, detrends and windows the data" from scipy.signal import detrend - idata = DATA[stancil[0]:stancil[-1]] + idata = DATA[stancil[0] : stancil[-1]] idata = detrend(idata) * win - return stancil[1], calc_spectrum_fft(idata , dk, Lpoints) + return stancil[1], calc_spectrum_fft(idata, dk, Lpoints) # def test_func(i_stancil): # return i_stancil[1], yy[i_stancil[0]:i_stancil[-1]].shape # %% derive L2 stancil - stancil_iter = create_chunk_boundaries(Lpoints, DATA.size, ov= self.ov) + stancil_iter = create_chunk_boundaries(Lpoints, DATA.size, ov=self.ov) # apply func to all stancils - D_specs = dict(map(calc_spectrum_apply,stancil_iter)) + D_specs = dict(map(calc_spectrum_apply, stancil_iter)) chunk_positions = np.array(list(D_specs.keys())) - self.N_stancils = len(chunk_positions) # number of spectal relazations + self.N_stancils = len(chunk_positions) # number of spectal relazations # repack data, create xarray - self.spec_name = 'power_spec' if name is None else name - G =dict() - for xi,I in D_specs.items(): - G[xi] = xr.DataArray(I, dims=['k'], coords={'k': self.k, 'x': xi } , name=self.spec_name) - - self.G = xr.concat(G.values(), dim='x').T#.to_dataset() + self.spec_name = "power_spec" if name is None else name + G = dict() + for xi, I in D_specs.items(): + G[xi] = xr.DataArray( + I, dims=["k"], coords={"k": self.k, "x": xi}, name=self.spec_name + ) + + self.G = xr.concat(G.values(), dim="x").T # .to_dataset() if self.G.k[0] == 0: self.G = self.G[1:, :] - self.G.attrs['ov'] = self.ov - self.G.attrs['L'] = self.Lpoints + self.G.attrs["ov"] = self.ov + self.G.attrs["L"] = self.Lpoints return self.G # cal variance def calc_var(self): - """ Compute total variance from spectragram """ - return self.dk*self.G.mean('x').sum().data # do not consider zeroth frequency + """Compute total variance from spectragram""" + return self.dk * self.G.mean("x").sum().data # do not consider zeroth frequency - def mean_spectral_error(self, mask=None, confidence = 0.95): + def mean_spectral_error(self, mask=None, confidence=0.95): "retrurns spetral error for the x-mean spectral estimate and stores it as coordindate in the dataarray" # make error estimate if mask is not None: - meanspec= self.G.isel(x=mask).mean('x') + meanspec = self.G.isel(x=mask).mean("x") N = int(sum(mask)) else: - meanspec= self.G.mean('x') + meanspec = self.G.mean("x") N = self.N_stancils - El_of_mean, Eu_of_mean = spec_error(meanspec, N , confidence ) - El_of_mean.name = 'El_mean' - Eu_of_mean.name = 'Eu_mean' + El_of_mean, Eu_of_mean = spec_error(meanspec, N, confidence) + El_of_mean.name = "El_mean" + Eu_of_mean.name = "Eu_mean" - self.G.coords['mean_El'] = (('k'), El_of_mean.data) - self.G.coords['mean_Eu'] = (('k'), Eu_of_mean.data) + self.G.coords["mean_El"] = (("k"), El_of_mean.data) + self.G.coords["mean_Eu"] = (("k"), Eu_of_mean.data) - def parceval(self, add_attrs=True ): + def parceval(self, add_attrs=True): "test Parceval theorem" DATA = self.data L = self.Lpoints - # derive mean variances of stancils stancil_iter = create_chunk_boundaries(L, DATA.size) def get_stancil_var_apply(stancil): from scipy.signal import detrend + "returns the variance of yy for stancil" - idata = DATA[stancil[0]:stancil[-1]] - idata = detrend(idata)# * win + idata = DATA[stancil[0] : stancil[-1]] + idata = detrend(idata) # * win return stancil[1], idata.var() - D_vars = dict(map(get_stancil_var_apply,stancil_iter)) + D_vars = dict(map(get_stancil_var_apply, stancil_iter)) - stancil_vars =list() + stancil_vars = list() for I in D_vars.values(): stancil_vars.append(I) - print('Parcevals Theorem:') - print('variance of unweighted timeseries: ',DATA.var()) - print('mean variance of detrended chunks: ', np.array(stancil_vars).mean()) - #print('variance of weighted timeseries: ',self.phi.var() ) - #self.calc_var(self) - print('variance of the pwelch Spectrum: ', self.calc_var()) + print("Parcevals Theorem:") + print("variance of unweighted timeseries: ", DATA.var()) + print("mean variance of detrended chunks: ", np.array(stancil_vars).mean()) + # print('variance of weighted timeseries: ',self.phi.var() ) + # self.calc_var(self) + print("variance of the pwelch Spectrum: ", self.calc_var()) if add_attrs: - self.G.attrs['variance_unweighted_data'] = DATA.var() - self.G.attrs['mean_variance_detrended_chunks'] = np.array(stancil_vars).mean() - self.G.attrs['mean_variance_pwelch_spectrum'] = self.calc_var() + self.G.attrs["variance_unweighted_data"] = DATA.var() + self.G.attrs["mean_variance_detrended_chunks"] = np.array( + stancil_vars + ).mean() + self.G.attrs["mean_variance_pwelch_spectrum"] = self.calc_var() + class wavenumber_spectrogram_LS_even: - def __init__(self, x, data, L, waven_method = 'fftX2' , dy=None , ov=None, window=None, kjumps=1): + def __init__( + self, x, data, L, waven_method="fftX2", dy=None, ov=None, window=None, kjumps=1 + ): """ returns a wavenumber spectrogram with the resolution L-ov this uses Lombscargle @@ -532,29 +569,30 @@ def __init__(self, x, data, L, waven_method = 'fftX2' , dy=None , ov=None, wind other arributes are in the .attr dict. """ from astropy.timeseries import LombScargle - self.L = L - self.ov = int(L/2) if ov is None else ov #when not defined in create_chunk_boundaries then L/2 - self.x = x - self.data = data - self.dy = dy + self.L = L + self.ov = ( + int(L / 2) if ov is None else ov + ) # when not defined in create_chunk_boundaries then L/2 + self.x = x + self.data = data + self.dy = dy # create subsample k - #print(waven_method) + # print(waven_method) if type(waven_method) is str: - self.k, self.dk = calc_freq_LS(x, L, method = waven_method ) + self.k, self.dk = calc_freq_LS(x, L, method=waven_method) elif type(waven_method) is np.ndarray: - self.k, self.dk = waven_method, np.diff(waven_method).mean() + self.k, self.dk = waven_method, np.diff(waven_method).mean() else: - raise ValueError('waven_method is neither string nor an array') + raise ValueError("waven_method is neither string nor an array") - self.k, self.dk = self.k[::kjumps], self.dk*kjumps + self.k, self.dk = self.k[::kjumps], self.dk * kjumps # create window - self.win = None #create_window(L) - - def cal_spectrogram(self, x = None, data=None, name=None, dx=1): + self.win = None # create_window(L) + def cal_spectrogram(self, x=None, data=None, name=None, dx=1): """ defines apply function and calculated all sub-sample sprectra using map dx nominal resolution of the data resolutionif not set, dx= 1 @@ -562,64 +600,71 @@ def cal_spectrogram(self, x = None, data=None, name=None, dx=1): from astropy.timeseries import LombScargle import xarray as xr - X = self.x if x is None else x # all x positions - DATA = self.data if data is None else data # all data points - L, dk = self.L, self.dk - win = self.win + X = self.x if x is None else x # all x positions + DATA = self.data if data is None else data # all data points + L, dk = self.L, self.dk + win = self.win self.dx = dx # init Lomb scargle object with noise as nummy data () - #dy_fake= np.random.randn(len(dy))*0.001 if self.dy is not None else None - self.LS = LombScargle(X[0:L] , np.random.randn(L)*0.001, fit_mean=True) - + # dy_fake= np.random.randn(len(dy))*0.001 if self.dy is not None else None + self.LS = LombScargle(X[0:L], np.random.randn(L) * 0.001, fit_mean=True) def calc_spectrum_apply(stancil): "returns spectrum per stencil, detrends and windows the data" from scipy.signal import detrend - x = X[stancil[0]:stancil[-1]] - #x_mask= (stancil[0] < X) & (X <= stancil[-1]) - #x = X[x_mask] - idata = DATA[stancil[0]:stancil[-1]] - y = detrend(idata)# * win + x = X[stancil[0] : stancil[-1]] + # x_mask= (stancil[0] < X) & (X <= stancil[-1]) + # x = X[x_mask] + idata = DATA[stancil[0] : stancil[-1]] + y = detrend(idata) # * win - LS_PSD, LS_object = calc_spectrum_LS( x, y, self.k, LS= self.LS, dk =self.dk) + LS_PSD, LS_object = calc_spectrum_LS(x, y, self.k, LS=self.LS, dk=self.dk) return stancil[1], LS_PSD # % derive L2 stancil - stancil_iter = create_chunk_boundaries(L, DATA.size, ov= self.ov) + stancil_iter = create_chunk_boundaries(L, DATA.size, ov=self.ov) # apply func to all stancils - D_specs = dict(map(calc_spectrum_apply,stancil_iter)) + D_specs = dict(map(calc_spectrum_apply, stancil_iter)) chunk_positions = np.array(list(D_specs.keys())) - self.N_stancils = len(chunk_positions) # number of spectal relazations + self.N_stancils = len(chunk_positions) # number of spectal relazations # repack data, create xarray - self.spec_name = 'power_spec' if name is None else name - G =dict() - for xi,I in D_specs.items(): - G[xi] = xr.DataArray(I, dims=['k'], coords={'k': self.k, 'x': xi * self.dx } , name=self.spec_name) - - self.G = xr.concat(G.values(), dim='x').T#.to_dataset() + self.spec_name = "power_spec" if name is None else name + G = dict() + for xi, I in D_specs.items(): + G[xi] = xr.DataArray( + I, + dims=["k"], + coords={"k": self.k, "x": xi * self.dx}, + name=self.spec_name, + ) + + self.G = xr.concat(G.values(), dim="x").T # .to_dataset() if self.G.k[0] == 0: self.G = self.G[1:, :] - self.G.attrs['ov'] = self.ov - self.G.attrs['L'] = self.L + self.G.attrs["ov"] = self.ov + self.G.attrs["L"] = self.L return self.G def calc_var(self): return wavenumber_spectrogram.calc_var(self) - def parceval(self, add_attrs=True ): - return wavenumber_spectrogram.parceval(self, add_attrs= add_attrs ) + def parceval(self, add_attrs=True): + return wavenumber_spectrogram.parceval(self, add_attrs=add_attrs) + + def mean_spectral_error(self, confidence=0.95): + return wavenumber_spectrogram.mean_spectral_error(self, confidence=confidence) - def mean_spectral_error(self, confidence = 0.95 ): - return wavenumber_spectrogram.mean_spectral_error(self, confidence= confidence ) class wavenumber_spectrogram_LS: - def __init__(self, x, data, L, dx, dy = None, waven_method = 'fftX2', ov=None, window=None): + def __init__( + self, x, data, L, dx, dy=None, waven_method="fftX2", ov=None, window=None + ): """ returns a wavenumber spectrogram with the resolution L-ov this uses Lombscargle @@ -639,31 +684,42 @@ def __init__(self, x, data, L, dx, dy = None, waven_method = 'fftX2', ov=None, w other arributes are in the .attr dict. """ from astropy.timeseries import LombScargle - self.L = L - self.ov = int(L/2) if ov is None else ov #when not defined in create_chunk_boundaries then L/2 - self.x = x - self.dx = dx - self.data = data - self.error = dy if dy is not None else None - self.Lpoints= int(self.L/self.dx) + self.L = L + self.ov = ( + int(L / 2) if ov is None else ov + ) # when not defined in create_chunk_boundaries then L/2 + self.x = x + self.dx = dx + self.data = data + self.error = dy if dy is not None else None + self.Lpoints = int(self.L / self.dx) # create subsample k - #print(waven_method) + # print(waven_method) if type(waven_method) is str: - self.k, self.dk = calc_freq_LS(x, self.Lpoints, method = waven_method ) + self.k, self.dk = calc_freq_LS(x, self.Lpoints, method=waven_method) elif type(waven_method) is np.ndarray: - self.k, self.dk = waven_method, np.diff(waven_method).mean() + self.k, self.dk = waven_method, np.diff(waven_method).mean() else: - raise ValueError('waven_method is neither string nor an array') + raise ValueError("waven_method is neither string nor an array") - #self.k, self.dk = self.k[::kjumps], self.dk*kjumps + # self.k, self.dk = self.k[::kjumps], self.dk*kjumps # create window - self.win = None #create_window(L) - - def cal_spectrogram(self, x = None, data=None, error=None, name=None, xlims =None, weight_data= True, max_nfev = None, map_func=None): - + self.win = None # create_window(L) + + def cal_spectrogram( + self, + x=None, + data=None, + error=None, + name=None, + xlims=None, + weight_data=True, + max_nfev=None, + map_func=None, + ): """ defines apply function and calculated all sub-sample sprectra using map @@ -687,77 +743,81 @@ def cal_spectrogram(self, x = None, data=None, error=None, name=None, xlims =Non import copy import pandas as pd - X = self.x if x is None else x # all x positions - DATA = self.data if data is None else data # all data points - ERR = self.error if error is None else error # all error for points - L, dk = self.L, self.dk - #win = self.win - self.xlims = ( np.round(X.min()), X.max() ) if xlims is None else xlims + X = self.x if x is None else x # all x positions + DATA = self.data if data is None else data # all data points + ERR = self.error if error is None else error # all error for points + L, dk = self.L, self.dk + # win = self.win + self.xlims = (np.round(X.min()), X.max()) if xlims is None else xlims # init Lomb scargle object with noise as nummy data () - #dy_fake= np.random.randn(len(dy))*0.001 if self.dy is not None else None - #self.LS = LombScargle(X[0:L] , np.random.randn(L)*0.001, fit_mean=True) + # dy_fake= np.random.randn(len(dy))*0.001 if self.dy is not None else None + # self.LS = LombScargle(X[0:L] , np.random.randn(L)*0.001, fit_mean=True) # define window import scipy.signal.windows as WINDOWS - self.win = WINDOWS.tukey(self.Lpoints, alpha=0.1, sym=True) + self.win = WINDOWS.tukey(self.Lpoints, alpha=0.1, sym=True) def calc_spectrum_and_field_apply(stancil): - """ windows the data accoding to stencil and applies LS spectrogram returns: stancil center, spectrum for this stencil, number of datapoints in stancil """ from scipy.signal import detrend - #x = X[stancil[0]:stancil[-1]] - x_mask= (stancil[0] <= X) & (X <= stancil[-1]) + # x = X[stancil[0]:stancil[-1]] + x_mask = (stancil[0] <= X) & (X <= stancil[-1]) - #print(stancil[1]) + # print(stancil[1]) x = X[x_mask] - if x.size < 200: # if there are not enough photos set results to nan - #return stancil[1], self.k*np.nan, np.fft.rfftfreq( int(self.Lpoints), d=self.dx)*np.nan, x.size - return stancil[1], self.k*np.nan, self.k*np.nan, np.nan, x.size + if x.size < 200: # if there are not enough photos set results to nan + # return stancil[1], self.k*np.nan, np.fft.rfftfreq( int(self.Lpoints), d=self.dx)*np.nan, x.size + return stancil[1], self.k * np.nan, self.k * np.nan, np.nan, x.size y = DATA[x_mask] - #make x positions - x_pos = (np.round( (x - stancil[0])/ 10.0 -1 , 0) ).astype('int') + # make x positions + x_pos = (np.round((x - stancil[0]) / 10.0 - 1, 0)).astype("int") # weight data if weight_data: window = self.win[x_pos] - y = y * window * np.sqrt( np.var(y) / np.var(( y* window) ) ) + y = y * window * np.sqrt(np.var(y) / np.var((y * window))) - #make y gridded - x_model = np.arange(stancil[0], stancil[-1], self.dx) - y_gridded = np.copy(x_model) * np.nan + # make y gridded + x_model = np.arange(stancil[0], stancil[-1], self.dx) + y_gridded = np.copy(x_model) * np.nan y_gridded[x_pos] = y - nan_mask =np.isnan(y_gridded) + nan_mask = np.isnan(y_gridded) err = ERR[x_mask] if ERR is not None else None - #print(x.shape, y.shape, self.k, self.LS) - LS_PSD, LS_object = calc_spectrum_LS( x, y, self.k, err=err, LS= None, dk =self.dk) - - y_model = reconstruct_data_from_LS(LS_object, x_model, self.k) + # print(x.shape, y.shape, self.k, self.LS) + LS_PSD, LS_object = calc_spectrum_LS( + x, y, self.k, err=err, LS=None, dk=self.dk + ) + y_model = reconstruct_data_from_LS(LS_object, x_model, self.k) # print(stancil[-1], x_model[-1]) # print(stancil[0], x_model[0]) # print(np.fft.rfft(y_model).size , kk.size) # print(x_model.size, y_gridded.size, y_model.size) # print('--') - P = conserve_variance(np.fft.rfft(y_model), self.k, y_gridded, nan_mask = nan_mask ) + P = conserve_variance( + np.fft.rfft(y_model), self.k, y_gridded, nan_mask=nan_mask + ) P.set_parameters() - #P.test_ojective_func(P.tanh_weight_function, plot_flag=False) + # P.test_ojective_func(P.tanh_weight_function, plot_flag=False) fitter = P.optimize(max_nfev=max_nfev) return stancil[1], LS_PSD, P.best_guess_Z(), fitter.params, x.size # % derive L2 stancil - self.stancil_iter = create_chunk_boundaries_unit_lengths(L, self.xlims, ov= self.ov, iter_flag=True) - #stancil_iter = create_chunk_boundaries_unit_lengths(L, ( np.round(X.min()), X.max() ), ov= self.ov, iter_flag=True) + self.stancil_iter = create_chunk_boundaries_unit_lengths( + L, self.xlims, ov=self.ov, iter_flag=True + ) + # stancil_iter = create_chunk_boundaries_unit_lengths(L, ( np.round(X.min()), X.max() ), ov= self.ov, iter_flag=True) # apply func to all stancils # Spec_returns=list() @@ -767,34 +827,38 @@ def calc_spectrum_and_field_apply(stancil): map_func = map if map_func is None else map_func print(map_func) - Spec_returns = list(map_func( calc_spectrum_and_field_apply, copy.copy(self.stancil_iter) )) + Spec_returns = list( + map_func(calc_spectrum_and_field_apply, copy.copy(self.stancil_iter)) + ) # # linear version - #Spec_returns = list(map( calc_spectrum_and_field_apply, copy.copy(self.stancil_iter) )) + # Spec_returns = list(map( calc_spectrum_and_field_apply, copy.copy(self.stancil_iter) )) # unpack resutls of the mapping: - D_specs = dict() - Y_model = dict() - Pars = dict() - N_per_stancil = list() + D_specs = dict() + Y_model = dict() + Pars = dict() + N_per_stancil = list() for I in Spec_returns: - #print(I[1].shape, I[2].shape) - D_specs[I[0]] = I[1] - Y_model[I[0]] = I[2] - Pars[I[0]] = I[3] + # print(I[1].shape, I[2].shape) + D_specs[I[0]] = I[1] + Y_model[I[0]] = I[2] + Pars[I[0]] = I[3] N_per_stancil.append(I[4]) self.N_per_stancil = N_per_stancil chunk_positions = np.array(list(D_specs.keys())) - self.N_stancils = len(chunk_positions) # number of spectral realizatiobs + self.N_stancils = len(chunk_positions) # number of spectral realizatiobs # repack data, create xarray # 1st LS spectal estimates - self.spec_name = 'LS_spectal_power' if name is None else name - G_LS_power =dict() - for xi,I in D_specs.items(): - G_LS_power[xi] = xr.DataArray(I, dims=['k'], coords={'k': self.k, 'x': xi } , name=self.spec_name) + self.spec_name = "LS_spectal_power" if name is None else name + G_LS_power = dict() + for xi, I in D_specs.items(): + G_LS_power[xi] = xr.DataArray( + I, dims=["k"], coords={"k": self.k, "x": xi}, name=self.spec_name + ) - G_LS_power = xr.concat(G_LS_power.values(), dim='x').T#.to_dataset() + G_LS_power = xr.concat(G_LS_power.values(), dim="x").T # .to_dataset() # 2nd Y_model # G_model =dict() @@ -816,52 +880,52 @@ def calc_spectrum_and_field_apply(stancil): # self.G_model.attrs['Lpoints'] = self.Lpoints # self.G_model.coords['N_per_stancil'] = ( ('x'), N_per_stancil) - - #2nd FFT(Y_model) - G_fft =dict() - Y_model_k_fft = np.fft.rfftfreq( int(self.Lpoints), d=self.dx) - for xi,I in Y_model.items(): + # 2nd FFT(Y_model) + G_fft = dict() + Y_model_k_fft = np.fft.rfftfreq(int(self.Lpoints), d=self.dx) + for xi, I in Y_model.items(): if I.size < Y_model_k_fft.size: I = np.insert(I, -1, I[-1]) - G_fft[xi] = xr.DataArray(I, dims=['k'], coords={'k': Y_model_k_fft, 'x': xi } , name='Y_model_hat') + G_fft[xi] = xr.DataArray( + I, dims=["k"], coords={"k": Y_model_k_fft, "x": xi}, name="Y_model_hat" + ) - G_fft = xr.concat(G_fft.values(), dim='x').T#.to_dataset() + G_fft = xr.concat(G_fft.values(), dim="x").T # .to_dataset() # generate power spec as well self.G = Z_to_power(G_fft, self.dk, self.Lpoints) - self.G.name = 'spectral_power_optm' - + self.G.name = "spectral_power_optm" # merge both datasets self.GG = xr.merge([G_LS_power, G_fft, self.G]) - self.GG.attrs['ov'] = self.ov - self.GG.attrs['L'] = self.L - self.GG.attrs['Lpoints'] = self.Lpoints - self.GG.coords['N_per_stancil'] = ( ('x'), N_per_stancil) - - self.GG.expand_dims(dim='eta') - self.GG.coords['eta'] = ( ('eta'), np.arange(0, self.L + self.dx, self.dx) - self.L/2 ) - self.GG['win'] = ( ('eta'), np.insert(self.win, -1, self.win[-1])) + self.GG.attrs["ov"] = self.ov + self.GG.attrs["L"] = self.L + self.GG.attrs["Lpoints"] = self.Lpoints + self.GG.coords["N_per_stancil"] = (("x"), N_per_stancil) + + self.GG.expand_dims(dim="eta") + self.GG.coords["eta"] = ( + ("eta"), + np.arange(0, self.L + self.dx, self.dx) - self.L / 2, + ) + self.GG["win"] = (("eta"), np.insert(self.win, -1, self.win[-1])) # create dataframe with fitted parameters - PP2= dict() + PP2 = dict() for k, I in Pars.items(): if I is not np.nan: - PP2[k] =I + PP2[k] = I keys = PP2[next(iter(PP2))].keys() - params_dataframe = pd.DataFrame(index =keys) + params_dataframe = pd.DataFrame(index=keys) - for k,I in PP2.items(): + for k, I in PP2.items(): I.values() params_dataframe[k] = list(I.valuesdict().values()) return self.GG, params_dataframe - - - def calc_var(self): Gmean = np.nanmean(self.G, 1) @@ -872,61 +936,67 @@ def calc_var(self): # def parceval(self, add_attrs=True ): # return wavenumber_spectrogram.parceval(self, add_attrs= add_attrs ) - def parceval(self, add_attrs=True, weight_data=False ): + def parceval(self, add_attrs=True, weight_data=False): "test Parceval theorem" import copy + DATA = self.data L = self.L X = self.x # derive mean variances of stancils - #stancil_iter = create_chunk_boundaries_unit_lengths(L, self.xlims, ov= self.ov ) + # stancil_iter = create_chunk_boundaries_unit_lengths(L, self.xlims, ov= self.ov ) def get_stancil_var_apply(stancil): from scipy.signal import detrend + "returns the variance of yy for stancil" - x_mask= (stancil[0] < X) & (X <= stancil[-1]) + x_mask = (stancil[0] < X) & (X <= stancil[-1]) idata = DATA[x_mask] if len(idata) < 1: return stancil[1], np.nan, len(idata) idata = detrend(idata) # weight data - x_pos = (np.round( (X[x_mask] - stancil[0])/ 10.0 , 0) ).astype('int') + x_pos = (np.round((X[x_mask] - stancil[0]) / 10.0, 0)).astype("int") if weight_data: window = self.win[x_pos] - idata = idata * window * np.sqrt( np.var(idata) / np.var(( idata* window) ) ) + idata = ( + idata * window * np.sqrt(np.var(idata) / np.var((idata * window))) + ) return stancil[1], idata.var(), len(idata) - D_vars = list(map(get_stancil_var_apply, copy.copy(self.stancil_iter) )) + D_vars = list(map(get_stancil_var_apply, copy.copy(self.stancil_iter))) - stancil_vars, Nphotons =list(), 0 + stancil_vars, Nphotons = list(), 0 for I in D_vars: - stancil_vars.append(I[1] * I[2]) - Nphotons += I[2] + stancil_vars.append(I[1] * I[2]) + Nphotons += I[2] - stancil_weighted_variance = np.nansum(np.array(stancil_vars))/Nphotons + stancil_weighted_variance = np.nansum(np.array(stancil_vars)) / Nphotons - print('Parcevals Theorem:') - print('variance of timeseries: ', DATA.var()) - print('mean variance of stancils: ', stancil_weighted_variance ) - #print('variance of weighted timeseries: ',self.phi.var() ) - #self.calc_var(self) - print('variance of the optimzed windowed LS Spectrum: ', self.calc_var()) + print("Parcevals Theorem:") + print("variance of timeseries: ", DATA.var()) + print("mean variance of stancils: ", stancil_weighted_variance) + # print('variance of weighted timeseries: ',self.phi.var() ) + # self.calc_var(self) + print("variance of the optimzed windowed LS Spectrum: ", self.calc_var()) if add_attrs: - self.G.attrs['variance_unweighted_data'] = DATA.var() - self.G.attrs['mean_variance_stancils'] = np.nanmean(np.array(stancil_vars) ) - self.G.attrs['mean_variance_LS_pwelch_spectrum'] = self.calc_var() - - - def mean_spectral_error(self, mask=None, confidence = 0.95 ): - return wavenumber_spectrogram.mean_spectral_error(self, mask=mask, confidence= confidence ) + self.G.attrs["variance_unweighted_data"] = DATA.var() + self.G.attrs["mean_variance_stancils"] = np.nanmean(np.array(stancil_vars)) + self.G.attrs["mean_variance_LS_pwelch_spectrum"] = self.calc_var() + def mean_spectral_error(self, mask=None, confidence=0.95): + return wavenumber_spectrogram.mean_spectral_error( + self, mask=mask, confidence=confidence + ) # class for getting standard Pwelch spectrum. old version, deprechiate class wavenumber_pwelch: - def __init__(self,data, x, L, ov=None, window=None, save_chunks=False, plot_chunks=False): + def __init__( + self, data, x, L, ov=None, window=None, save_chunks=False, plot_chunks=False + ): """ returns a wavenumber spectrum using the pwelch method @@ -948,93 +1018,90 @@ def __init__(self,data, x, L, ov=None, window=None, save_chunks=False, plot_chun """ from scipy import signal + self.data = data # field to be analyzed + self.dx = np.diff(x)[0] # sampling interval + self.save_chunks = save_chunks + dsize = data.size - self.data = data # field to be analyzed - self.dx = np.diff(x)[0] # sampling interval - self.save_chunks= save_chunks - dsize = data.size - - ov=int(np.round(L/2)) if ov is None else ov + ov = int(np.round(L / 2)) if ov is None else ov self.n = L if window is None: - win=np.hanning(self.n) + win = np.hanning(self.n) else: - win=window + win = window - factor=np.sqrt(self.n/(win**2).sum()) - win*=factor + factor = np.sqrt(self.n / (win**2).sum()) + win *= factor # test if n is even - if (self.n%2): + if self.n % 2: self.neven = False else: self.neven = True - #calculate freq - self.k = self.calc_freq() - #del(self.df) + # calculate freq + self.k = self.calc_freq() + # del(self.df) - #print(data.size, L, ov, int(L-ov) ) - nbin=int(np.floor(dsize/(L-ov))) - #print(nbin) + # print(data.size, L, ov, int(L-ov) ) + nbin = int(np.floor(dsize / (L - ov))) + # print(nbin) if save_chunks: - chunks=np.empty([int(nbin),int(L)]) - - self.specs=np.empty([int(nbin),self.k.size]) - #print(chunks.shape) - #result_array = np.empty((0, 100)) - #if plot_chunks: - #M.figure_axis_xy() - last_k=0 - k=0 - #print('iter range', np.arange(0,data.size,int(L-ov))) - for i in np.arange(0,dsize-int(L-ov)+1,int(L-ov)): - - if (plot_chunks) and (i >= dsize-6*int(L-ov)): + chunks = np.empty([int(nbin), int(L)]) + + self.specs = np.empty([int(nbin), self.k.size]) + # print(chunks.shape) + # result_array = np.empty((0, 100)) + # if plot_chunks: + # M.figure_axis_xy() + last_k = 0 + k = 0 + # print('iter range', np.arange(0,data.size,int(L-ov))) + for i in np.arange(0, dsize - int(L - ov) + 1, int(L - ov)): + + if (plot_chunks) and (i >= dsize - 6 * int(L - ov)): M.figure_axis_xy() - self.phi=data[int(i):int(i+L)] + self.phi = data[int(i) : int(i + L)] - #self.ii=np.append(self.ii,[i,i+L]) - #print(self.phi.max()) + # self.ii=np.append(self.ii,[i,i+L]) + # print(self.phi.max()) - #print(self.phi.mean()) - #print(self.phi.shape) - #print('i',int(i), int(i+L)) - #print(chunk.size, l) - if int(i+L) <= data.size-1: + # print(self.phi.mean()) + # print(self.phi.shape) + # print('i',int(i), int(i+L)) + # print(chunk.size, l) + if int(i + L) <= data.size - 1: if save_chunks: - chunks[k,:]=self.phi - + chunks[k, :] = self.phi - self.phi=signal.detrend(self.phi)*win + self.phi = signal.detrend(self.phi) * win if plot_chunks: - #MT.stats_format(self.phi, 'chunk '+str(i)) + # MT.stats_format(self.phi, 'chunk '+str(i)) plt.plot(self.phi) - self.specs[k,:]= self.calc_spectrum() - last_k=k - last_used_TS=int(i+L) - #if plot_chunks: + self.specs[k, :] = self.calc_spectrum() + last_k = k + last_used_TS = int(i + L) + # if plot_chunks: # MT.stats_format(self.spec, 'spec '+str(i)) else: if plot_chunks: - print('end of TS is reached') - print('last spec No: '+str(last_k)) - print('spec container: '+str(specs.shape)) - print('last used Timestep: '+str(last_used_TS)) - print('length of TS '+ str(dsize) +'ms') - - k+=1 + print("end of TS is reached") + print("last spec No: " + str(last_k)) + print("spec container: " + str(specs.shape)) + print("last used Timestep: " + str(last_used_TS)) + print("length of TS " + str(dsize) + "ms") + k += 1 if save_chunks: - self.chunks=chunks - #del(chunks) + self.chunks = chunks + # del(chunks) - self.spec_est=self.specs.mean(axis=0) + self.spec_est = self.specs.mean(axis=0) # if prewhite is None: # self.specs=specs[:last_k,:] # self.spec_est=self.specs.mean(axis=0) @@ -1045,69 +1112,74 @@ def __init__(self,data, x, L, ov=None, window=None, save_chunks=False, plot_chun # self.specs=specs[:last_k,:]*(2*np.pi*self.f)**2 # self.spec_est=self.specs.mean(axis=0) - - self.n_spec,_=self.specs.shape + self.n_spec, _ = self.specs.shape self.calc_var() - #self.phi=self.data - #self.phi*=win*np.sqrt(factor) + # self.phi=self.data + # self.phi*=win*np.sqrt(factor) def calc_freq(self): - """ calculate array of spectral variable (frequency or - wavenumber) in cycles per unit of L """ + """calculate array of spectral variable (frequency or + wavenumber) in cycles per unit of L""" - self.df = 1./((self.n-1)*self.dx) + self.df = 1.0 / ((self.n - 1) * self.dx) if self.neven: - f = self.df*np.arange(self.n/2+1) + f = self.df * np.arange(self.n / 2 + 1) else: - f = self.df*np.arange( (self.n-1)/2. + 1 ) + f = self.df * np.arange((self.n - 1) / 2.0 + 1) return f def calc_spectrum(self): - """ compute the 1d spectrum of a field phi """ + """compute the 1d spectrum of a field phi""" self.phih = np.fft.rfft(self.phi) # the factor of 2 comes from the symmetry of the Fourier coeffs - spec = 2.*(self.phih*self.phih.conj()).real / self.df /self.n**2 + spec = 2.0 * (self.phih * self.phih.conj()).real / self.df / self.n**2 # the zeroth frequency should be counted only once - spec[0] = spec[0]/2. + spec[0] = spec[0] / 2.0 if self.neven: - spec[-1] = spec[-1]/2. + spec[-1] = spec[-1] / 2.0 return spec - def error(self, ci=0.95): - self.El, self.Eu =spec_error(self.spec_est,self.n_spec,ci=ci) + self.El, self.Eu = spec_error(self.spec_est, self.n_spec, ci=ci) def parceval(self): - print('Parcevals Theorem:') - print('variance of unweighted timeseries: ',self.data.var()) - print('mean variance of timeseries chunks: ',self.chunks.var(axis=1).mean() if self.save_chunks is True else 'data not saved') - #print('variance of weighted timeseries: ',self.phi.var() ) - #self.calc_var(self) - print('variance of the pwelch Spectrum: ',self.var) + print("Parcevals Theorem:") + print("variance of unweighted timeseries: ", self.data.var()) + print( + "mean variance of timeseries chunks: ", + ( + self.chunks.var(axis=1).mean() + if self.save_chunks is True + else "data not saved" + ), + ) + # print('variance of weighted timeseries: ',self.phi.var() ) + # self.calc_var(self) + print("variance of the pwelch Spectrum: ", self.var) def calc_var(self): - """ Compute total variance from spectrum """ - self.var = self.df* np.nanmean(self.specs[1:], 0).sum() # do not consider zeroth frequency - + """Compute total variance from spectrum""" + self.var = ( + self.df * np.nanmean(self.specs[1:], 0).sum() + ) # do not consider zeroth frequency # %% optimze spectral variance -class conserve_variance: - def __init__(self,Z, freq, data, nan_mask= None): - - """ - """ +class conserve_variance: + def __init__(self, Z, freq, data, nan_mask=None): + """ """ import lmfit as LM - self.LM =LM + + self.LM = LM self.data = data - self.Z = Z + self.Z = Z self.freq = freq self.nan_mask = nan_mask @@ -1115,59 +1187,79 @@ def set_parameters(self): params = self.LM.Parameters() - p_smothed = self.runningmean(np.abs(self.Z ), 20, tailcopy=True) + p_smothed = self.runningmean(np.abs(self.Z), 20, tailcopy=True) f_max = self.freq[p_smothed[~np.isnan(p_smothed)].argmax()] - - lambda_max = 9.81 * 5 **2/ (2* np.pi) - params.add('x_cutoff', 1/lambda_max , min=0, max=1, vary=False) - params.add('x_max_pos', f_max , min=f_max*0.75, max=f_max*5+0.001, vary=False) - params.add('LF_amp', 1 , min=0.5 , max=1.2, vary= True) - params.add('HF_amp', 0.5 , min=0 , max=1.5, vary= True) - params.add('sigma_g', 0.002 , min=0.001 , max=0.05, vary= False) - params.add('Gauss_amp', 0.5 , min=0.01 , max=2, vary= True) - + lambda_max = 9.81 * 5**2 / (2 * np.pi) + params.add("x_cutoff", 1 / lambda_max, min=0, max=1, vary=False) + params.add( + "x_max_pos", f_max, min=f_max * 0.75, max=f_max * 5 + 0.001, vary=False + ) + params.add("LF_amp", 1, min=0.5, max=1.2, vary=True) + params.add("HF_amp", 0.5, min=0, max=1.5, vary=True) + params.add("sigma_g", 0.002, min=0.001, max=0.05, vary=False) + params.add("Gauss_amp", 0.5, min=0.01, max=2, vary=True) self.params = params return params def test_ojective_func(self, weight_func, plot_flag=True): - self.objective_func(self.params, self.data, self.Z, weight_func, self.freq, self.nan_mask, plot_flag=plot_flag) - - def tanh_weight_function(self,ff, params): - return self.tanh_weight(ff, params['x_cutoff'].value, - params['x_max_pos'].value, - params['LF_amp'].value, - params['HF_amp'].value, - params['Gauss_amp'].value, - params['sigma_g'].value ) - - def tanh_weight(self,x, x_cutoff , x_max_pos, LF_amp, HF_amp, Gauss_amp, sigma_g): + self.objective_func( + self.params, + self.data, + self.Z, + weight_func, + self.freq, + self.nan_mask, + plot_flag=plot_flag, + ) + + def tanh_weight_function(self, ff, params): + return self.tanh_weight( + ff, + params["x_cutoff"].value, + params["x_max_pos"].value, + params["LF_amp"].value, + params["HF_amp"].value, + params["Gauss_amp"].value, + params["sigma_g"].value, + ) + + def tanh_weight(self, x, x_cutoff, x_max_pos, LF_amp, HF_amp, Gauss_amp, sigma_g): """ - zdgfsg + zdgfsg """ - HF_amp1 = (LF_amp-HF_amp) - decay = 0.5 - np.tanh( (x-x_cutoff)/sigma_g )/2 - y = decay * HF_amp1 + (1 - HF_amp1) - y = y- y[0] +LF_amp + HF_amp1 = LF_amp - HF_amp + decay = 0.5 - np.tanh((x - x_cutoff) / sigma_g) / 2 + y = decay * HF_amp1 + (1 - HF_amp1) + y = y - y[0] + LF_amp - def gaus(x, x_0, amp, sigma_g ): - return amp* np.exp(-0.5 * ( (x-x_0)/sigma_g)**2) + def gaus(x, x_0, amp, sigma_g): + return amp * np.exp(-0.5 * ((x - x_0) / sigma_g) ** 2) - y += gaus(x, x_max_pos, Gauss_amp, sigma_g ) + y += gaus(x, x_max_pos, Gauss_amp, sigma_g) - #y = y * LF_amp + # y = y * LF_amp return y + def objective_func( + self, + params, + data_x, + Z_results, + weight_func, + freq, + nan_mask=None, + plot_flag=False, + ): - def objective_func(self, params, data_x, Z_results, weight_func, freq, nan_mask = None, plot_flag=False): + alpha = 1e7 - alpha =1e7 def model_real_space(Z, weights, n=None): """ Both inputs must have the same length """ - return np.fft.irfft(Z*weights, n = n) + return np.fft.irfft(Z * weights, n=n) weights = weight_func(freq, params) @@ -1175,10 +1267,10 @@ def model_real_space(Z, weights, n=None): weights = np.insert(weights, -1, weights[-1]) if nan_mask is not None: - model = model_real_space(Z_results, weights, n= data_x.size)[~nan_mask] + model = model_real_space(Z_results, weights, n=data_x.size)[~nan_mask] dd = data_x[~nan_mask][:] else: - model = model_real_space(Z_results, weights, n= data_x.size)[:] + model = model_real_space(Z_results, weights, n=data_x.size)[:] dd = data_x[:] if model.size > dd.size: @@ -1190,102 +1282,156 @@ def model_real_space(Z, weights, n=None): import m_general_ph3 as M from matplotlib.gridspec import GridSpec import matplotlib.pyplot as plt - F= M.figure_axis_xy(10, 4.1 * 2.5, view_scale= 0.5, container = True) - gs = GridSpec(5,1, wspace=0.1, hspace=0.4)#figure=fig, - pos0,pos1,pos2 = gs[0:3, 0],gs[3, 0],gs[4, 0]#,gs[3, 0] + F = M.figure_axis_xy(10, 4.1 * 2.5, view_scale=0.5, container=True) + + gs = GridSpec(5, 1, wspace=0.1, hspace=0.4) # figure=fig, + pos0, pos1, pos2 = gs[0:3, 0], gs[3, 0], gs[4, 0] # ,gs[3, 0] ax1 = F.fig.add_subplot(pos0) - plt.title('Stacked Timeseries', loc='left') + plt.title("Stacked Timeseries", loc="left") - chunk_l= 400 - chunk_iter = create_chunk_boundaries(chunk_l, data_x.size, ov=0, iter_flag = True) + chunk_l = 400 + chunk_iter = create_chunk_boundaries( + chunk_l, data_x.size, ov=0, iter_flag=True + ) - ofsett0= 6 + ofsett0 = 6 ofsett = np.copy(ofsett0) for chi in chunk_iter: - v1= np.round(np.nanvar(dd), 4) - plt.plot(ofsett+ data_x[chi[0]:chi[-1]] , linewidth=3, alpha=0.5 , c='black', label=' org. data (var:'+str(v1)+')') - - v1= np.round(model_real_space(Z_results, weights*0 +1)[~nan_mask[1:]].var(), 4) - plt.plot(ofsett + model_real_space(Z_results, weights*0 +1)[chi[0]:chi[-1]] ,linewidth= 0.8, c='red', label='LS model init (var:'+str(v1)+')') - - v1= np.round(model.var(), 4) - plt.plot(ofsett + model_real_space(Z_results, weights)[chi[0]:chi[-1]],linewidth= 0.8, c='blue', label='LS model weighted (var:'+str(v1)+')') + v1 = np.round(np.nanvar(dd), 4) + plt.plot( + ofsett + data_x[chi[0] : chi[-1]], + linewidth=3, + alpha=0.5, + c="black", + label=" org. data (var:" + str(v1) + ")", + ) + + v1 = np.round( + model_real_space(Z_results, weights * 0 + 1)[~nan_mask[1:]].var(), 4 + ) + plt.plot( + ofsett + + model_real_space(Z_results, weights * 0 + 1)[chi[0] : chi[-1]], + linewidth=0.8, + c="red", + label="LS model init (var:" + str(v1) + ")", + ) + + v1 = np.round(model.var(), 4) + plt.plot( + ofsett + model_real_space(Z_results, weights)[chi[0] : chi[-1]], + linewidth=0.8, + c="blue", + label="LS model weighted (var:" + str(v1) + ")", + ) if ofsett == ofsett0: plt.legend() ofsett -= 1 - plt.ylim(ofsett, ofsett0+1) - plt.xlim(0, chunk_l*2) - + plt.ylim(ofsett, ofsett0 + 1) + plt.xlim(0, chunk_l * 2) ax2 = F.fig.add_subplot(pos1) - #ax2 = plt.subplot(3, 1, 2) - plt.title('Amplitude Weight Function', loc='left') - plt.plot(weights , c='black') - ax2.set_xscale('log') + # ax2 = plt.subplot(3, 1, 2) + plt.title("Amplitude Weight Function", loc="left") + plt.plot(weights, c="black") + ax2.set_xscale("log") ax3 = F.fig.add_subplot(pos2) - plt.title('Initial and tuned |Z|', loc='left') + plt.title("Initial and tuned |Z|", loc="left") - #ax3 = plt.subplot(3, 1, 3) + # ax3 = plt.subplot(3, 1, 3) # v2_fft= np.fft.rfft(data_x) # v2 = np.round( (2.*(v2_fft*v2_fft.conj()).real /data_x.size**2 ).sum(), 4) # plt.plot(abs(v2_fft) , linewidth=2, alpha=0.5 , c='black', label='org data (var: '+str(v2) +')') - v2 = np.round( (4.*(Z_results*Z_results.conj()).real /data_x.size**2 ).sum(), 4) - plt.plot(abs(Z_results), linewidth= 0.8, c='red', label='Z (var: '+str(v2) +')') - plt.plot(M.runningmean(abs(Z_results) , 20, tailcopy=True), linewidth= 1.5, c='red', zorder=12) - - Z2= Z_results* weights - v2 = np.round( (4.*(Z2*Z2.conj()).real /data_x.size**2 ).sum(), 4) - plt.plot(abs(Z2), linewidth= 0.8, c='blue', label='weighted Z(var: '+str(v2) +')') - plt.plot(M.runningmean(abs(Z2) , 20, tailcopy=True), linewidth= 1.5, c='blue', zorder=12) + v2 = np.round( + (4.0 * (Z_results * Z_results.conj()).real / data_x.size**2).sum(), 4 + ) + plt.plot( + abs(Z_results), linewidth=0.8, c="red", label="Z (var: " + str(v2) + ")" + ) + plt.plot( + M.runningmean(abs(Z_results), 20, tailcopy=True), + linewidth=1.5, + c="red", + zorder=12, + ) + + Z2 = Z_results * weights + v2 = np.round((4.0 * (Z2 * Z2.conj()).real / data_x.size**2).sum(), 4) + plt.plot( + abs(Z2), + linewidth=0.8, + c="blue", + label="weighted Z(var: " + str(v2) + ")", + ) + plt.plot( + M.runningmean(abs(Z2), 20, tailcopy=True), + linewidth=1.5, + c="blue", + zorder=12, + ) plt.legend() - plt.ylim( np.percentile(abs(Z_results), 0.5), abs(Z_results).max()*1.3 ) - plt.xlabel('wavenumber k') - ax3.set_xscale('log') - ax3.set_yscale('log') - - fitting_cost =( abs(dd - model) / dd.std() )**2 - variance_cost =( abs(dd.var() - model.var()) / dd.std() ) **2 + plt.ylim(np.percentile(abs(Z_results), 0.5), abs(Z_results).max() * 1.3) + plt.xlabel("wavenumber k") + ax3.set_xscale("log") + ax3.set_yscale("log") - return fitting_cost.sum() , alpha* variance_cost + fitting_cost = (abs(dd - model) / dd.std()) ** 2 + variance_cost = (abs(dd.var() - model.var()) / dd.std()) ** 2 - def optimize(self, fitting_args= None , method='dual_annealing', max_nfev=None): + return fitting_cost.sum(), alpha * variance_cost + def optimize(self, fitting_args=None, method="dual_annealing", max_nfev=None): if fitting_args is None: fitting_args = (self.data, self.Z, self.tanh_weight_function, self.freq) self.weight_func = fitting_args[2] - self.fitter = self.LM.minimize(self.objective_func, self.params, args=fitting_args, kws={'nan_mask':self.nan_mask} , method=method, max_nfev=max_nfev) + self.fitter = self.LM.minimize( + self.objective_func, + self.params, + args=fitting_args, + kws={"nan_mask": self.nan_mask}, + method=method, + max_nfev=max_nfev, + ) return self.fitter def plot_result(self): - self.objective_func(self.fitter.params, self.data, self.Z, self.weight_func, self.freq, self.nan_mask, plot_flag=True) + self.objective_func( + self.fitter.params, + self.data, + self.Z, + self.weight_func, + self.freq, + self.nan_mask, + plot_flag=True, + ) def best_guess_Z(self): return self.Z * self.weight_func(self.freq, self.fitter.params) def runningmean(self, var, m, tailcopy=False): - m=int(m) - s =var.shape - if s[0] <= 2*m: - print('0 Dimension is smaller then averaging length') + m = int(m) + s = var.shape + if s[0] <= 2 * m: + print("0 Dimension is smaller then averaging length") return - rr=np.asarray(var)*np.nan - #print(type(rr)) - var_range=np.arange(m,int(s[0])-m-1,1) - for i in var_range[np.isfinite(var[m:int(s[0])-m-1])]: - #rm.append(var[i-m:i+m].mean()) - rr[int(i)]=np.nanmean(var[i-m:i+m]) + rr = np.asarray(var) * np.nan + # print(type(rr)) + var_range = np.arange(m, int(s[0]) - m - 1, 1) + for i in var_range[np.isfinite(var[m : int(s[0]) - m - 1])]: + # rm.append(var[i-m:i+m].mean()) + rr[int(i)] = np.nanmean(var[i - m : i + m]) if tailcopy: - rr[0:m]=rr[m+1] - rr[-m-1:-1]=rr[-m-2] + rr[0:m] = rr[m + 1] + rr[-m - 1 : -1] = rr[-m - 2] return rr diff --git a/src/icesat2_tracks/ICEsat2_SI_tools/spicke_remover.py b/src/icesat2_tracks/ICEsat2_SI_tools/spicke_remover.py index deb19bb7..cd1c57bf 100644 --- a/src/icesat2_tracks/ICEsat2_SI_tools/spicke_remover.py +++ b/src/icesat2_tracks/ICEsat2_SI_tools/spicke_remover.py @@ -1,7 +1,7 @@ - import numpy as np -def spicke_remover(data, nstd=20.0, spreed=500.0, max_loops=10.0 , verbose=False): + +def spicke_remover(data, nstd=20.0, spreed=500.0, max_loops=10.0, verbose=False): """ This function removes spickes from timeseries based on its std and maximum values inputs: @@ -10,81 +10,91 @@ def spicke_remover(data, nstd=20.0, spreed=500.0, max_loops=10.0 , verbose=False spreed width of data between wich will be linear interpolated around the spike. width is in units of data points (dt) max_loops number of maximum possible loobs until the spike removal is stopped, even it the first creteria is not true """ - datastd=np.nanstd(data) - data2=np.copy(data) + datastd = np.nanstd(data) + data2 = np.copy(data) - peak_remove=True - looper_count=0 - act_flag=False + peak_remove = True + looper_count = 0 + act_flag = False while peak_remove is True: - if nstd* datastd < np.nanmax(np.abs(data2)): - act_flag=True + if nstd * datastd < np.nanmax(np.abs(data2)): + act_flag = True if verbose: - print('true: '+ str(nstd* datastd) +' < '+str( np.nanmax(np.abs(data)) ) ) - data2=spickes_to_mean(data2, nloop=0, spreed=spreed, gaussian=False) - looper_count+=1 + print( + "true: " + + str(nstd * datastd) + + " < " + + str(np.nanmax(np.abs(data))) + ) + data2 = spickes_to_mean(data2, nloop=0, spreed=spreed, gaussian=False) + looper_count += 1 else: if verbose: - print('False: '+ str(nstd* datastd) +' > '+str( np.nanmax(np.abs(data)) ) ) - peak_remove=False + print( + "False: " + + str(nstd * datastd) + + " > " + + str(np.nanmax(np.abs(data))) + ) + peak_remove = False if looper_count > max_loops: - peak_remove=False + peak_remove = False if verbose: - print('stoped by max#') - + print("stoped by max#") if verbose: import matplotlib.pyplot as plt - plt.plot(data, 'r') - plt.plot(data2, 'b') - return data2 , act_flag + plt.plot(data, "r") + plt.plot(data2, "b") + + return data2, act_flag def spickes_to_mean(ts, nloop=None, spreed=1, gaussian=True): from scipy import signal - nloop=0 if nloop is None else nloop - i=0 - tsmean=ts.mean() - b=2*spreed - gaus=signal.gaussian(b, std=b/10) + nloop = 0 if nloop is None else nloop + i = 0 + tsmean = ts.mean() + b = 2 * spreed + gaus = signal.gaussian(b, std=b / 10) while i <= nloop: - #print(i) - #ts.max() - #print(np.where(ts == ts.max())) - tsabs=np.abs(ts) - tmax=np.nanmax(tsabs) - #print(tsabs, tmax) - pa=np.where(tsabs == tmax)[0][0] + # print(i) + # ts.max() + # print(np.where(ts == ts.max())) + tsabs = np.abs(ts) + tmax = np.nanmax(tsabs) + # print(tsabs, tmax) + pa = np.where(tsabs == tmax)[0][0] if gaussian: - tsm=np.mean([ts[pa-spreed],ts[pa+spreed]]) #ts[pa-spreed:pa+spreed] - #print(ts[pa-spreed:pa+spreed].shape) - #print((gaus*(tmax-tsm)).shape) - #print(np.shape(gaus*(tmax-tsm))) - le=int(pa-spreed) - ue=int(pa+spreed) + tsm = np.mean([ts[pa - spreed], ts[pa + spreed]]) # ts[pa-spreed:pa+spreed] + # print(ts[pa-spreed:pa+spreed].shape) + # print((gaus*(tmax-tsm)).shape) + # print(np.shape(gaus*(tmax-tsm))) + le = int(pa - spreed) + ue = int(pa + spreed) - ts[le:ue]=ts[le:ue]-gaus*(tmax-tsm) + ts[le:ue] = ts[le:ue] - gaus * (tmax - tsm) else: - #tsm=np.mean([ts[pa-spreed],ts[pa+spreed]]) #ts[pa-spreed:pa+spreed] - #print(ts[pa-spreed:pa+spreed].shape) - #print((gaus*(tmax-tsm)).shape) - #print(np.shape(gaus*(tmax-tsm))) - #print(len(ts)) - #print(pa+spreed) - if pa+spreed > len(ts): - le= int(pa-spreed) - ts[le:-1]=np.linspace(ts[le],ts[-1],len(ts[le:-1])) + # tsm=np.mean([ts[pa-spreed],ts[pa+spreed]]) #ts[pa-spreed:pa+spreed] + # print(ts[pa-spreed:pa+spreed].shape) + # print((gaus*(tmax-tsm)).shape) + # print(np.shape(gaus*(tmax-tsm))) + # print(len(ts)) + # print(pa+spreed) + if pa + spreed > len(ts): + le = int(pa - spreed) + ts[le:-1] = np.linspace(ts[le], ts[-1], len(ts[le:-1])) else: - le=int(pa-spreed) - ue=int(pa+spreed) - #print(le, ue) - ts[ le : ue ]=np.linspace(ts[le],ts[ue],len(ts[le:ue])) + le = int(pa - spreed) + ue = int(pa + spreed) + # print(le, ue) + ts[le:ue] = np.linspace(ts[le], ts[ue], len(ts[le:ue])) - i=i+1 + i = i + 1 return ts diff --git a/src/icesat2_tracks/ICEsat2_SI_tools/wave_tools.py b/src/icesat2_tracks/ICEsat2_SI_tools/wave_tools.py index 62732c16..fffef62d 100644 --- a/src/icesat2_tracks/ICEsat2_SI_tools/wave_tools.py +++ b/src/icesat2_tracks/ICEsat2_SI_tools/wave_tools.py @@ -1,17 +1,17 @@ -def to_vec(amp, angle, deg = True): +def to_vec(amp, angle, deg=True): "from anlge deg to vect" import numpy as np if deg: - u, v = amp * np.cos(angle * np.pi/180), amp * np.sin(angle * np.pi/180) + u, v = amp * np.cos(angle * np.pi / 180), amp * np.sin(angle * np.pi / 180) else: - u, v = amp * np.cos(angle ), amp * np.sin(angle ) + u, v = amp * np.cos(angle), amp * np.sin(angle) - return u,v + return u, v -def to_deg(u,v, deg = True): +def to_deg(u, v, deg=True): """ from vect to angle, amp angle is -180 to 180 @@ -20,7 +20,7 @@ def to_deg(u,v, deg = True): import numpy as np amp = np.sqrt(u**2 + v**2) - angle = np.arctan2(v, u)# + 2 * np.pi + angle = np.arctan2(v, u) # + 2 * np.pi # nan_mask = np.isnan(angle) # angle= np.where(angle > np.pi, angle - 2 * np.pi , angle) @@ -28,18 +28,20 @@ def to_deg(u,v, deg = True): # angle= np.where(nan_mask, np.nan , angle) if deg: - angle = angle * 180/np.pi + angle = angle * 180 / np.pi return amp, angle -def get_ave_amp_angle(amp, angle, deg = True): + +def get_ave_amp_angle(amp, angle, deg=True): import numpy as np - u,v = to_vec(amp, angle , deg= deg) + + u, v = to_vec(amp, angle, deg=deg) # average angle in vector space - #print(u, v) - _ , ave_deg = to_deg( np.nanmean(u) , np.nanmean(v), deg = deg) - _ , std_deg = to_deg( np.nanstd(u) , np.nanstd(v) , deg = deg) + # print(u, v) + _, ave_deg = to_deg(np.nanmean(u), np.nanmean(v), deg=deg) + _, std_deg = to_deg(np.nanstd(u), np.nanstd(v), deg=deg) - #average amp in angle space + # average amp in angle space ave_amp = np.nanmean(amp) std_amp = np.nanstd(amp) diff --git a/src/icesat2_tracks/__init__.py b/src/icesat2_tracks/__init__.py index 3aa0d7b3..6c8e6b97 100644 --- a/src/icesat2_tracks/__init__.py +++ b/src/icesat2_tracks/__init__.py @@ -1 +1 @@ -__version__ = "0.0.0" \ No newline at end of file +__version__ = "0.0.0" diff --git a/src/icesat2_tracks/analysis/SA01b_download_tester.py b/src/icesat2_tracks/analysis/SA01b_download_tester.py index 75a3cdfc..05d3fd12 100644 --- a/src/icesat2_tracks/analysis/SA01b_download_tester.py +++ b/src/icesat2_tracks/analysis/SA01b_download_tester.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This script opens an ATL07 track and tests if there is sufficient data and maybe waves: @@ -13,7 +14,7 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) sys.path -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) import datetime @@ -24,7 +25,10 @@ import ICEsat2_SI_tools.io as io # Icesat2 Modules -from spectral_estimates import create_chunk_boundaries_unit_lengths, create_chunk_boundaries +from spectral_estimates import ( + create_chunk_boundaries_unit_lengths, + create_chunk_boundaries, +) import spectral_estimates as spec import m_tools_ph3 as MT import filter_regrid as regrid @@ -39,7 +43,9 @@ # #build = True -def nsidc_icesat2_get_associated_file(file_list, product, build=True, username=None, password=None): +def nsidc_icesat2_get_associated_file( + file_list, product, build=True, username=None, password=None +): """ THis method returns assocociated files names and paths for files given in file_list for the "product" ICEsat2 product @@ -57,34 +63,39 @@ def nsidc_icesat2_get_associated_file(file_list, product, build=True, username=N import posixpath import os import icesat2_toolkit.utilities - AUXILIARY=False - #product='ATL03' - DIRECTORY= None - FLATTEN=False - TIMEOUT=120 - MODE=0o775 - file_list = ['ATL07-01_20210301023054_10251001_005_01'] + + AUXILIARY = False + # product='ATL03' + DIRECTORY = None + FLATTEN = False + TIMEOUT = 120 + MODE = 0o775 + file_list = ["ATL07-01_20210301023054_10251001_005_01"] if build and not (username or password): - urs = 'urs.earthdata.nasa.gov' - username,login,password = netrc.netrc().authenticators(urs) - #-- build urllib2 opener and check credentials + urs = "urs.earthdata.nasa.gov" + username, login, password = netrc.netrc().authenticators(urs) + # -- build urllib2 opener and check credentials if build: - #-- build urllib2 opener with credentials + # -- build urllib2 opener with credentials icesat2_toolkit.utilities.build_opener(username, password) - #-- check credentials + # -- check credentials icesat2_toolkit.utilities.check_credentials() parser = lxml.etree.HTMLParser() - #-- remote https server for ICESat-2 Data - HOST = 'https://n5eil01u.ecs.nsidc.org' - #-- regular expression operator for extracting information from files - rx = re.compile(r'(processed_)?(ATL\d{2})(-\d{2})?_(\d{4})(\d{2})(\d{2})' - r'(\d{2})(\d{2})(\d{2})_(\d{4})(\d{2})(\d{2})_(\d{3})_(\d{2})') - #-- regular expression pattern for finding specific files - regex_suffix = '(.*?)$' if AUXILIARY else '(h5)$' - remote_regex_pattern = (r'{0}(-\d{{2}})?_(\d{{4}})(\d{{2}})(\d{{2}})' - r'(\d{{2}})(\d{{2}})(\d{{2}})_({1})({2})({3})_({4})_(\d{{2}})(.*?).{5}') + # -- remote https server for ICESat-2 Data + HOST = "https://n5eil01u.ecs.nsidc.org" + # -- regular expression operator for extracting information from files + rx = re.compile( + r"(processed_)?(ATL\d{2})(-\d{2})?_(\d{4})(\d{2})(\d{2})" + r"(\d{2})(\d{2})(\d{2})_(\d{4})(\d{2})(\d{2})_(\d{3})_(\d{2})" + ) + # -- regular expression pattern for finding specific files + regex_suffix = "(.*?)$" if AUXILIARY else "(h5)$" + remote_regex_pattern = ( + r"{0}(-\d{{2}})?_(\d{{4}})(\d{{2}})(\d{{2}})" + r"(\d{{2}})(\d{{2}})(\d{{2}})_({1})({2})({3})_({4})_(\d{{2}})(.*?).{5}" + ) # rx = re.compile(r'(processed_)?(ATL\d{2})(-\d{2})?_(\d{4})(\d{2})(\d{2})' # r'(\d{2})(\d{2})(\d{2})_(\d{4})(\d{2})(\d{2})_(\d{3})_(\d{2})(.*?).h5$') @@ -93,56 +104,54 @@ def nsidc_icesat2_get_associated_file(file_list, product, build=True, username=N # remote_regex_pattern = (r'{0}(-\d{{2}})?_(\d{{4}})(\d{{2}})(\d{{2}})' # r'(\d{{2}})(\d{{2}})(\d{{2}})_({1})({2})({3})_({4})_(\d{{2}})(.*?).{5}') - #-- build list of remote files, remote modification times and local files + # -- build list of remote files, remote modification times and local files original_files = [] remote_files = [] remote_mtimes = [] local_files = [] - remote_names =[] + remote_names = [] for input_file in file_list: - #print(input_file) - #-- extract parameters from ICESat-2 ATLAS HDF5 file name - SUB,PRD,HEM,YY,MM,DD,HH,MN,SS,TRK,CYC,GRN,RL,VRS = \ - rx.findall(input_file).pop() - #-- get directories from remote directory - product_directory = '{0}.{1}'.format(product,RL) - sd = '{0}.{1}.{2}'.format(YY,MM,DD) - PATH = [HOST,'ATLAS',product_directory,sd] - #-- local and remote data directories - remote_dir=posixpath.join(*PATH) - temp=os.path.dirname(input_file) if (DIRECTORY is None) else DIRECTORY - local_dir=os.path.expanduser(temp) if FLATTEN else os.path.join(temp,sd) - #-- create output directory if not currently existing + # print(input_file) + # -- extract parameters from ICESat-2 ATLAS HDF5 file name + SUB, PRD, HEM, YY, MM, DD, HH, MN, SS, TRK, CYC, GRN, RL, VRS = rx.findall( + input_file + ).pop() + # -- get directories from remote directory + product_directory = "{0}.{1}".format(product, RL) + sd = "{0}.{1}.{2}".format(YY, MM, DD) + PATH = [HOST, "ATLAS", product_directory, sd] + # -- local and remote data directories + remote_dir = posixpath.join(*PATH) + temp = os.path.dirname(input_file) if (DIRECTORY is None) else DIRECTORY + local_dir = os.path.expanduser(temp) if FLATTEN else os.path.join(temp, sd) + # -- create output directory if not currently existing # if not os.access(local_dir, os.F_OK): # os.makedirs(local_dir, MODE) - #-- compile regular expression operator for file parameters - args = (product,TRK,CYC,GRN,RL,regex_suffix) + # -- compile regular expression operator for file parameters + args = (product, TRK, CYC, GRN, RL, regex_suffix) R1 = re.compile(remote_regex_pattern.format(*args), re.VERBOSE) - #-- find associated ICESat-2 data file - #-- find matching files (for granule, release, version, track) - colnames,collastmod,colerror=icesat2_toolkit.utilities.nsidc_list(PATH, - build=False, - timeout=TIMEOUT, - parser=parser, - pattern=R1, - sort=True) + # -- find associated ICESat-2 data file + # -- find matching files (for granule, release, version, track) + colnames, collastmod, colerror = icesat2_toolkit.utilities.nsidc_list( + PATH, build=False, timeout=TIMEOUT, parser=parser, pattern=R1, sort=True + ) print(colnames) - #-- print if file was not found + # -- print if file was not found if not colnames: print(colerror) continue - #-- add to lists - for colname,remote_mtime in zip(colnames,collastmod): - #-- save original file to list (expands if getting auxiliary files) + # -- add to lists + for colname, remote_mtime in zip(colnames, collastmod): + # -- save original file to list (expands if getting auxiliary files) original_files.append(input_file) - #-- remote and local versions of the file - remote_files.append(posixpath.join(remote_dir,colname)) - local_files.append(os.path.join(local_dir,colname)) + # -- remote and local versions of the file + remote_files.append(posixpath.join(remote_dir, colname)) + local_files.append(os.path.join(local_dir, colname)) remote_mtimes.append(remote_mtime) remote_names.append(colname) - return original_files, remote_files, remote_names #product_directory, sd, + return original_files, remote_files, remote_names # product_directory, sd, # def nsidc_icesat2_get_associated_file(file_list, product): @@ -251,4 +260,6 @@ def nsidc_icesat2_get_associated_file(file_list, product, build=True, username=N # # return original_files, remote_files, remote_names #product_directory, sd, -org_files, remote_files, remote_names = nsidc_icesat2_get_associated_file(['ATL07-01_20210301023054_10251001_005_01'], 'ATL03') +org_files, remote_files, remote_names = nsidc_icesat2_get_associated_file( + ["ATL07-01_20210301023054_10251001_005_01"], "ATL03" +) diff --git a/src/icesat2_tracks/analysis_db/A01b_ALT07_SHNH_variance_tester.py b/src/icesat2_tracks/analysis_db/A01b_ALT07_SHNH_variance_tester.py index 21abbaa1..70bd39ce 100644 --- a/src/icesat2_tracks/analysis_db/A01b_ALT07_SHNH_variance_tester.py +++ b/src/icesat2_tracks/analysis_db/A01b_ALT07_SHNH_variance_tester.py @@ -1,6 +1,8 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) from icesat2_tracks.config.IceSAT2_startup import mconfig + """ This script opens an ATL07 track and tests if there is sufficient data and maybe waves: 1) disects each beam such that they start in the MIZ and end at the pole. @@ -11,7 +13,6 @@ """ - from threadpoolctl import threadpool_info, threadpool_limits from pprint import pprint @@ -21,7 +22,10 @@ import imp import icesat2_tracks.ICEsat2_SI_tools.convert_GPS_time as cGPS import icesat2_tracks.ICEsat2_SI_tools.io as io -from icesat2_tracks.ICEsat2_SI_tools.spectral_estimates import create_chunk_boundaries_unit_lengths, create_chunk_boundaries +from icesat2_tracks.ICEsat2_SI_tools.spectral_estimates import ( + create_chunk_boundaries_unit_lengths, + create_chunk_boundaries, +) import icesat2_tracks.ICEsat2_SI_tools.spectral_estimates as spec import icesat2_tracks.ICEsat2_SI_tools.filter_regrid as regrid @@ -32,62 +36,68 @@ import piecewise_regression -#import s3fs -#processed_ATL03_20190605061807_10380310_004_01.h5 +# import s3fs +# processed_ATL03_20190605061807_10380310_004_01.h5 -#imp.reload(io) -track_name, batch_key, test_flag = io.init_from_input(sys.argv) # loads standard experiment +# imp.reload(io) +track_name, batch_key, test_flag = io.init_from_input( + sys.argv +) # loads standard experiment # track NH -#track_name, batch_key, test_flag = '20190301004639_09560201_005_01', 'NH_batch05', False # <-- ! -#track_name, batch_key, test_flag = '20190101180843_00660201_005_01', 'SH_batch04', False +# track_name, batch_key, test_flag = '20190301004639_09560201_005_01', 'NH_batch05', False # <-- ! +# track_name, batch_key, test_flag = '20190101180843_00660201_005_01', 'SH_batch04', False # track SH -#track_name, batch_key, test_flag = '20190101084259_00600201_005_01', 'SH_batch04', False -#track_name, batch_key, test_flag = '20190102130012_00780201_005_01', 'SH_batch04', False -#track_name, batch_key, test_flag = '20190101005132_00550201_005_01', 'SH_batch04', False # <-- ! -#track_name, batch_key, test_flag = '20190101225136_00690201_005_01', 'SH_batch04', False +# track_name, batch_key, test_flag = '20190101084259_00600201_005_01', 'SH_batch04', False +# track_name, batch_key, test_flag = '20190102130012_00780201_005_01', 'SH_batch04', False +# track_name, batch_key, test_flag = '20190101005132_00550201_005_01', 'SH_batch04', False # <-- ! +# track_name, batch_key, test_flag = '20190101225136_00690201_005_01', 'SH_batch04', False -#track_name, batch_key, test_flag = '20190219063727_08070201_005_01', 'SH_publish', False +# track_name, batch_key, test_flag = '20190219063727_08070201_005_01', 'SH_publish', False -#track_name, batch_key, test_flag = '20190208142818_06440201_005_01', 'SH_publish', False +# track_name, batch_key, test_flag = '20190208142818_06440201_005_01', 'SH_publish', False -#print(track_name, batch_key, test_flag) -hemis, batch = batch_key.split('_') -#track_name= '20190605061807_10380310_004_01' +# print(track_name, batch_key, test_flag) +hemis, batch = batch_key.split("_") +# track_name= '20190605061807_10380310_004_01' -ATlevel= 'ATL07-02' if hemis == 'SH' else 'ATL07-01' -load_path = mconfig['paths']['scratch'] +'/'+ batch_key +'/' -load_file = load_path + ATlevel+'_'+track_name+'.h5' +ATlevel = "ATL07-02" if hemis == "SH" else "ATL07-01" +load_path = mconfig["paths"]["scratch"] + "/" + batch_key + "/" +load_file = load_path + ATlevel + "_" + track_name + ".h5" -save_path = mconfig['paths']['work'] +'/'+ batch_key +'/A01b_ID/' -scratch_path = mconfig['paths']['scratch'] +'/'+ batch_key +'/' -plot_path = mconfig['paths']['plot']+ '/'+hemis+'/'+batch_key+'/A01b/' -#bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' +save_path = mconfig["paths"]["work"] + "/" + batch_key + "/A01b_ID/" +scratch_path = mconfig["paths"]["scratch"] + "/" + batch_key + "/" +plot_path = mconfig["paths"]["plot"] + "/" + hemis + "/" + batch_key + "/A01b/" +# bad_track_path =mconfig['paths']['work'] +'bad_tracks/'+ batch_key+'/' MT.mkdirs_r(save_path) -plot_flag = True -test_flag = False #writes dummy variable for download files, instead of downloading -N_process = 4 +plot_flag = True +test_flag = False # writes dummy variable for download files, instead of downloading +N_process = 4 # username= "mhell@ucsd.edu" # pword = "@[49[4tK\-qBWB%5" # %% # test which beams exist: -all_beams = mconfig['beams']['all_beams'] +all_beams = mconfig["beams"]["all_beams"] try: - f = h5py.File(load_file, 'r') + f = h5py.File(load_file, "r") except: - print('file not found, exit') - MT.json_save(name='A01b_'+track_name+'_success', path=save_path, data= {'reason':'ATL07 file not found, exit'}) + print("file not found, exit") + MT.json_save( + name="A01b_" + track_name + "_success", + path=save_path, + data={"reason": "ATL07 file not found, exit"}, + ) exit() -beams = [b if b in f.keys() else None for b in all_beams] +beams = [b if b in f.keys() else None for b in all_beams] imp.reload(regrid) -#track_poleward = regrid.track_pole_ward_file(f, product='ATL10') +# track_poleward = regrid.track_pole_ward_file(f, product='ATL10') # print('poleward track is' , track_poleward) # ATL03 = h5py.File(load_file, 'r') # ATL03['orbit_info'].keys() @@ -95,62 +105,70 @@ # # ATL03['gt1l/freeboard_beam_segment/height_segments'].keys() + # %% -def cut_rear_data(xx0, dd0, N_seg= 20): +def cut_rear_data(xx0, dd0, N_seg=20): """ returns masks that cuts large variance in the back of the data """ - rear_mask = xx0*0 > -1 # True + rear_mask = xx0 * 0 > -1 # True nsize0 = rear_mask.size cut_flag = True dd_old = -1 - print('inital length' , nsize0) + print("inital length", nsize0) - #@jit(nopython=True, parallel= False) + # @jit(nopython=True, parallel= False) def adjust_length(var_list, rear_mask, cut_flag): - #var_list = var_list if track_poleward else var_list[::-1] + # var_list = var_list if track_poleward else var_list[::-1] - if var_list[0:3].mean()*2 < var_list[-1]: - #print('cut last '+ str(100/N_seg) +'% of data') - rear_mask[int(nsize* (N_seg-1) / N_seg):] = False + if var_list[0:3].mean() * 2 < var_list[-1]: + # print('cut last '+ str(100/N_seg) +'% of data') + rear_mask[int(nsize * (N_seg - 1) / N_seg) :] = False else: - cut_flag = False + cut_flag = False - #rear_mask = rear_mask if track_poleward else rear_mask[::-1] + # rear_mask = rear_mask if track_poleward else rear_mask[::-1] return rear_mask, cut_flag def get_var(sti): - return dd[sti[0]: sti[1]].var() + return dd[sti[0] : sti[1]].var() while cut_flag: - dd= dd0[rear_mask] + dd = dd0[rear_mask] nsize = dd.size - print('new length', nsize) - if (nsize/N_seg) < 1: + print("new length", nsize) + if (nsize / N_seg) < 1: break - stencil_iter = create_chunk_boundaries( int(nsize/N_seg), nsize,ov =0, iter_flag=True ) + stencil_iter = create_chunk_boundaries( + int(nsize / N_seg), nsize, ov=0, iter_flag=True + ) var_list = np.array(list(map(get_var, stencil_iter))) - #print(k, var_list) + # print(k, var_list) rear_mask, cut_flag = adjust_length(var_list, rear_mask, cut_flag) if nsize == dd_old: - print('--- lengthen segments') - N_seg -=1 - #cut_flag = False + print("--- lengthen segments") + N_seg -= 1 + # cut_flag = False dd_old = nsize return rear_mask -def get_breakingpoints(xx, dd ,Lmeter= 3000): + +def get_breakingpoints(xx, dd, Lmeter=3000): nsize = dd.size - stencil_iter = spec.create_chunk_boundaries_unit_lengths( Lmeter, [ xx.min(), xx.max()],ov =Lmeter*3/4, iter_flag= True) - iter_x = spec.create_chunk_boundaries_unit_lengths( Lmeter, [ xx.min(), xx.max()],ov =Lmeter*3/4, iter_flag= False)[1,:] + stencil_iter = spec.create_chunk_boundaries_unit_lengths( + Lmeter, [xx.min(), xx.max()], ov=Lmeter * 3 / 4, iter_flag=True + ) + iter_x = spec.create_chunk_boundaries_unit_lengths( + Lmeter, [xx.min(), xx.max()], ov=Lmeter * 3 / 4, iter_flag=False + )[1, :] def get_var(sti): mask = (sti[0] < xx) & (xx <= sti[1]) @@ -158,98 +176,99 @@ def get_var(sti): var_list = np.array(list(map(get_var, stencil_iter))) - x2, y2 = iter_x/1e3, var_list + x2, y2 = iter_x / 1e3, var_list - x2= x2[~np.isnan(y2)] - y2= y2[~np.isnan(y2)] + x2 = x2[~np.isnan(y2)] + y2 = y2[~np.isnan(y2)] - convergence_flag =True - n_breakpoints= 1 + convergence_flag = True + n_breakpoints = 1 while convergence_flag: pw_fit = piecewise_regression.Fit(x2, y2, n_breakpoints=1) - print('n_breakpoints', n_breakpoints, pw_fit.get_results()['converged']) - convergence_flag = not pw_fit.get_results()['converged'] + print("n_breakpoints", n_breakpoints, pw_fit.get_results()["converged"]) + convergence_flag = not pw_fit.get_results()["converged"] n_breakpoints += 1 if n_breakpoints == 4: convergence_flag = False pw_results = pw_fit.get_results() - if pw_results['converged']: - if pw_results['estimates']['alpha1']['estimate'] < 0: - print('decay at the front') - print('n_breakpoints',pw_fit.n_breakpoints ) + if pw_results["converged"]: + if pw_results["estimates"]["alpha1"]["estimate"] < 0: + print("decay at the front") + print("n_breakpoints", pw_fit.n_breakpoints) - breakpoint = pw_results['estimates']['breakpoint1']['estimate'] - return pw_results['estimates']['alpha1']['estimate'], pw_fit, breakpoint + breakpoint = pw_results["estimates"]["breakpoint1"]["estimate"] + return pw_results["estimates"]["alpha1"]["estimate"], pw_fit, breakpoint else: return np.nan, pw_fit, False -DD_slope = pd.DataFrame(index =beams, columns= ['TF1', 'TF2']) -DD_data = pd.DataFrame(index =beams, columns= ['TF1', 'TF2']) -DD_region = pd.DataFrame(index =beams, columns= ['TF1', 'TF2']) -DD_region[:] = (np.nan) + +DD_slope = pd.DataFrame(index=beams, columns=["TF1", "TF2"]) +DD_data = pd.DataFrame(index=beams, columns=["TF1", "TF2"]) +DD_region = pd.DataFrame(index=beams, columns=["TF1", "TF2"]) +DD_region[:] = np.nan # DD_pos_start = pd.DataFrame(index =beams, columns= ['TF1_lon', 'TF1_lat', 'TF2_lon', 'TF2_lat']) # DD_pos_end = pd.DataFrame(index =beams, columns= ['TF1_lon', 'TF1_lat', 'TF2_lon', 'TF2_lat']) -DD_pos_start = pd.DataFrame(index =beams, columns= ['TF1', 'TF2']) -DD_pos_end = pd.DataFrame(index =beams, columns= ['TF1', 'TF2']) +DD_pos_start = pd.DataFrame(index=beams, columns=["TF1", "TF2"]) +DD_pos_end = pd.DataFrame(index=beams, columns=["TF1", "TF2"]) plot_flag = False for k in beams: - #k = 'gt2r'#beams[0] - #imp.reload(io) + # k = 'gt2r'#beams[0] + # imp.reload(io) print(k) try: - T_freeboard = io.getATL07_beam(load_file, beam= k) + T_freeboard = io.getATL07_beam(load_file, beam=k) except: - print('failed to load beam') + print("failed to load beam") slope_test = False - data_density = False - #return data_density, slope_test - print('break -------', k, TF, data_density, slope_test) + data_density = False + # return data_density, slope_test + print("break -------", k, TF, data_density, slope_test) continue # find devide such that each hemisphere is split into two parts, if data is there - if hemis == 'SH': + if hemis == "SH": ###### for SH tracks - lon_list = T_freeboard['ref']['longitude'] - mask1 = (lon_list[0]-5 < lon_list) & (lon_list < lon_list[0]+5) + lon_list = T_freeboard["ref"]["longitude"] + mask1 = (lon_list[0] - 5 < lon_list) & (lon_list < lon_list[0] + 5) mask2 = ~mask1 - tot_size =T_freeboard['ref']['latitude'].shape[0] + tot_size = T_freeboard["ref"]["latitude"].shape[0] TF2, TF1 = None, None - if (sum(mask1) > 1000): + if sum(mask1) > 1000: if mask1.iloc[-1]: TF2 = T_freeboard[mask1] else: TF1 = T_freeboard[mask1] - if (sum(mask2) > 1000): + if sum(mask2) > 1000: TF2 = T_freeboard[mask2] else: ###### for NH tracks from scipy.ndimage.measurements import label - #mask1 = label(T_freeboard['ref']['latitude'] < 88)[0] ==1 - break_point = abs(T_freeboard['ref']['latitude']-90).argmin() - mask1 = T_freeboard['ref']['latitude'].index <= break_point + # mask1 = label(T_freeboard['ref']['latitude'] < 88)[0] ==1 + + break_point = abs(T_freeboard["ref"]["latitude"] - 90).argmin() + mask1 = T_freeboard["ref"]["latitude"].index <= break_point mask2 = ~mask1 - tot_size =T_freeboard['ref']['latitude'].shape[0] + tot_size = T_freeboard["ref"]["latitude"].shape[0] # cut data accordingly - if (sum(mask1)/tot_size < 0.05) or (sum(mask1) < 1000): + if (sum(mask1) / tot_size < 0.05) or (sum(mask1) < 1000): TF1 = None TF2 = T_freeboard[mask2] - elif (sum(mask2)/tot_size < 0.05) or (sum(mask2) < 1000): + elif (sum(mask2) / tot_size < 0.05) or (sum(mask2) < 1000): TF1 = T_freeboard[mask1] TF2 = None else: TF1 = T_freeboard[mask1] TF2 = T_freeboard[mask2] - # check if sub-taable goes equatorward or not, then sort accordingly and define along-track axis def pole_ward_table(T): """ @@ -258,9 +277,12 @@ def pole_ward_table(T): """ if T is None: return None - time = T['time']['delta_time'] - lat = T['ref']['latitude'] - print('1st lat =' + str(abs(lat.iloc[time.argmin()])) , ';last lat =' + str(abs(lat.iloc[time.argmax()])) ) + time = T["time"]["delta_time"] + lat = T["ref"]["latitude"] + print( + "1st lat =" + str(abs(lat.iloc[time.argmin()])), + ";last lat =" + str(abs(lat.iloc[time.argmax()])), + ) return abs(lat.iloc[time.argmax()]) > abs(lat.iloc[time.argmin()]) @@ -274,54 +296,62 @@ def pole_ward_table(T): TF2_poleward = not TF1_poleward if TF1_poleward & TF2_poleward: - raise ValueError('both parts are acending or decending') + raise ValueError("both parts are acending or decending") # assign Region to each subset, hemisphere dependent - for TF,Tsel,TF_poleward in zip(['TF1', 'TF2'], [TF1, TF2], [TF1_poleward, TF2_poleward]): - - print(TF,TF_poleward) - if (hemis == 'SH') & TF_poleward: - region = ('10') # SO region - elif (hemis == 'SH') & (not TF_poleward): - region = ('12') # SO region - elif (hemis == 'NH') & (TF_poleward): - region = ('03','04') # assign subarctic and high-arctic region - elif (hemis == 'NH') & (not TF_poleward): - region = ('05','04') # assign subarctic and high-arctic region + for TF, Tsel, TF_poleward in zip( + ["TF1", "TF2"], [TF1, TF2], [TF1_poleward, TF2_poleward] + ): + + print(TF, TF_poleward) + if (hemis == "SH") & TF_poleward: + region = "10" # SO region + elif (hemis == "SH") & (not TF_poleward): + region = "12" # SO region + elif (hemis == "NH") & (TF_poleward): + region = ("03", "04") # assign subarctic and high-arctic region + elif (hemis == "NH") & (not TF_poleward): + region = ("05", "04") # assign subarctic and high-arctic region else: - region =False + region = False - if (Tsel is None): + if Tsel is None: slope_test = False - data_density = False - #return data_density, slope_test - print('break -------', k, TF, data_density, slope_test) + data_density = False + # return data_density, slope_test + print("break -------", k, TF, data_density, slope_test) continue else: # flip the beam section that is not poleward if not TF_poleward: - print('TF polewards is ', TF_poleward) - Tsel = Tsel.sort_values(('ref','seg_dist_x'), ascending=False).reset_index() + print("TF polewards is ", TF_poleward) + Tsel = Tsel.sort_values( + ("ref", "seg_dist_x"), ascending=False + ).reset_index() # create local x axis - Tsel['x'] = abs(Tsel['ref']['seg_dist_x'] -Tsel['ref']['seg_dist_x'].iloc[0]) + Tsel["x"] = abs( + Tsel["ref"]["seg_dist_x"] - Tsel["ref"]["seg_dist_x"].iloc[0] + ) # ignore bad segments - Tsel = Tsel[Tsel['heights']['height_segment_surface_error_est'] < 1e2] - if (Tsel.size <= 50): - #print('too small table, skip ') + Tsel = Tsel[Tsel["heights"]["height_segment_surface_error_est"] < 1e2] + if Tsel.size <= 50: + # print('too small table, skip ') Tsel = None # if Tsel is None skip this itteration if Tsel is None: slope_test = False - data_density = False - #return data_density, slope_test - print('break -------', k, TF, data_density, slope_test) + data_density = False + # return data_density, slope_test + print("break -------", k, TF, data_density, slope_test) continue else: - data_density =Tsel.shape[0]/abs(Tsel['x'].max() - Tsel['x'].min()) # datapoints per meters + data_density = Tsel.shape[0] / abs( + Tsel["x"].max() - Tsel["x"].min() + ) # datapoints per meters # plt.plot(Tsel['x']/1e3, Tsel['ref']['beam_fb_sigma'], '.k') # plt.plot(Tsel['x']/1e3, Tsel['ref']['beam_fb_height'], '.r') @@ -330,48 +360,77 @@ def pole_ward_table(T): # plt.ylim(-1, 5) # limit number of processes # % cut data in the back: only usefull for SH: - xx0, dd0 = np.array(Tsel['x']), np.array(Tsel['heights']['height_segment_height']) - if hemis is 'SH': + xx0, dd0 = np.array(Tsel["x"]), np.array( + Tsel["heights"]["height_segment_height"] + ) + if hemis is "SH": # cut data around the contiental margin - with threadpool_limits(limits=N_process, user_api='blas'): + with threadpool_limits(limits=N_process, user_api="blas"): rear_mask = cut_rear_data(xx0, dd0) else: # assume all data points are valid for NH ... - rear_mask = np.array(Tsel['x'] > -1) + rear_mask = np.array(Tsel["x"] > -1) - #print('density post cutting', len(xx0[rear_mask])/abs(xx0[rear_mask].max() - xx0[rear_mask].min()) ) + # print('density post cutting', len(xx0[rear_mask])/abs(xx0[rear_mask].max() - xx0[rear_mask].min()) ) # if cutted data is too short, skip loop if len(xx0[rear_mask]) < 500: slope_test = False - data_density = False - #return data_density, slope_test - print('break -------', k, TF, data_density, slope_test) + data_density = False + # return data_density, slope_test + print("break -------", k, TF, data_density, slope_test) continue # estmiate slope at the beginning - with threadpool_limits(limits=N_process, user_api='blas'): - #pprint(threadpool_info()) - slope_test, pw_fit, breakpoint = get_breakingpoints(xx0[rear_mask], dd0[rear_mask], Lmeter= 3000) + with threadpool_limits(limits=N_process, user_api="blas"): + # pprint(threadpool_info()) + slope_test, pw_fit, breakpoint = get_breakingpoints( + xx0[rear_mask], dd0[rear_mask], Lmeter=3000 + ) if plot_flag: plt.figure() - plt.plot(xx0[rear_mask]/1e3, dd0[rear_mask], '.k', markersize= 0.4) + plt.plot(xx0[rear_mask] / 1e3, dd0[rear_mask], ".k", markersize=0.4) pw_fit.plot() - plt.title(k +' '+ TF + ', data=' +str(data_density) + ', slope='+str(slope_test) + '\n' + track_name , loc= 'left') - M.save_anyfig(plt.gcf(), name='A01b_'+track_name+'_'+k +'_'+ TF , path=plot_path) + plt.title( + k + + " " + + TF + + ", data=" + + str(data_density) + + ", slope=" + + str(slope_test) + + "\n" + + track_name, + loc="left", + ) + M.save_anyfig( + plt.gcf(), + name="A01b_" + track_name + "_" + k + "_" + TF, + path=plot_path, + ) plt.close() - #plt.show() + # plt.show() # assign to tables - DD_slope.loc[ k, TF] = slope_test - DD_data.loc[ k, TF] = data_density + DD_slope.loc[k, TF] = slope_test + DD_data.loc[k, TF] = data_density DD_region.loc[k, TF] = region - DD_pos_start.loc[k, TF] = Tsel.iloc[0]['ref']['longitude'] , Tsel.iloc[0]['ref']['latitude'], Tsel.iloc[0]['ref']['seg_dist_x'], Tsel.iloc[0]['time']['delta_time'] - DD_pos_end.loc[k, TF] = Tsel.iloc[-1]['ref']['longitude'] , Tsel.iloc[-1]['ref']['latitude'], Tsel.iloc[-1]['ref']['seg_dist_x'], Tsel.iloc[-1]['time']['delta_time'] + DD_pos_start.loc[k, TF] = ( + Tsel.iloc[0]["ref"]["longitude"], + Tsel.iloc[0]["ref"]["latitude"], + Tsel.iloc[0]["ref"]["seg_dist_x"], + Tsel.iloc[0]["time"]["delta_time"], + ) + DD_pos_end.loc[k, TF] = ( + Tsel.iloc[-1]["ref"]["longitude"], + Tsel.iloc[-1]["ref"]["latitude"], + Tsel.iloc[-1]["ref"]["seg_dist_x"], + Tsel.iloc[-1]["time"]["delta_time"], + ) # DD_pos_start.loc[k, [TF+'_lon', TF+'_lat']] = Tsel.iloc[0]['ref']['longitude'] , Tsel.iloc[0]['ref']['latitude'], Tsel.iloc[0]['ref']['seg_dist_x'], Tsel.iloc[0]['time']['delta_time'] # DD_pos_end.loc[k, [TF+'_lon', TF+'_lat']] = Tsel.iloc[-1]['ref']['longitude'] , Tsel.iloc[-1]['ref']['latitude'], Tsel.iloc[-1]['ref']['seg_dist_x'], Tsel.iloc[-1]['time']['delta_time'] - print('result-------', k, TF, data_density, slope_test) + print("result-------", k, TF, data_density, slope_test) # %% check decisions @@ -382,46 +441,74 @@ def pole_ward_table(T): ddtime_start, ddtime_end = list(), list() for k in all_beams: - if (type(DD_pos_start[Ti][k]) is not tuple): - ddtime_start.append(np.nan) # get latitude - ddtime_end.append(np.nan) # get latitude + if type(DD_pos_start[Ti][k]) is not tuple: + ddtime_start.append(np.nan) # get latitude + ddtime_end.append(np.nan) # get latitude else: - ddtime_start.append(DD_pos_start[Ti][k][1]) # get latitude - ddtime_end.append(DD_pos_end[Ti][k][1]) # get latitude - print('poleward check ', k , abs(DD_pos_start[Ti][k][1]) < abs(DD_pos_end[Ti][k][1]), abs(DD_pos_start[Ti][k][1]) , abs(DD_pos_end[Ti][k][1])) - - #print(ddtime_start, ddtime_end) + ddtime_start.append(DD_pos_start[Ti][k][1]) # get latitude + ddtime_end.append(DD_pos_end[Ti][k][1]) # get latitude + print( + "poleward check ", + k, + abs(DD_pos_start[Ti][k][1]) < abs(DD_pos_end[Ti][k][1]), + abs(DD_pos_start[Ti][k][1]), + abs(DD_pos_end[Ti][k][1]), + ) + + # print(ddtime_start, ddtime_end) TT_start[Ti] = DD_pos_start[Ti].iloc[np.array(ddtime_start).argmin()] TT_end[Ti] = DD_pos_end[Ti].iloc[np.array(ddtime_end).argmax()] try: - print('poleward check sum', abs(TT_start[Ti][1]) < abs(TT_end[Ti][1]), abs(TT_start[Ti][1]) , abs(TT_end[Ti][1])) + print( + "poleward check sum", + abs(TT_start[Ti][1]) < abs(TT_end[Ti][1]), + abs(TT_start[Ti][1]), + abs(TT_end[Ti][1]), + ) except: pass -#if plot_flag: +# if plot_flag: font_for_pres() -F = M.figure_axis_xy(10, 4 ,container = True) -#ax = F.fig.add_subplot(122, projection='polar') +F = M.figure_axis_xy(10, 4, container=True) +# ax = F.fig.add_subplot(122, projection='polar') -for Ti,figp in zip(DD_pos_start, [121, 122]): - ax = F.fig.add_subplot(figp, projection='polar') +for Ti, figp in zip(DD_pos_start, [121, 122]): + ax = F.fig.add_subplot(figp, projection="polar") print(Ti) for k in all_beams: - if (type(DD_pos_start[Ti][k]) is tuple): - plt.scatter( np.deg2rad( DD_pos_start[Ti][k][0]), DD_pos_start[Ti][k][1] ,s=20, color='green')#, label='start') - if (type(DD_pos_end[Ti][k]) is tuple): - plt.scatter( np.deg2rad( DD_pos_end[Ti][k][0]) , DD_pos_end[Ti][k][1] ,s=20, color='red')#, label='end') + if type(DD_pos_start[Ti][k]) is tuple: + plt.scatter( + np.deg2rad(DD_pos_start[Ti][k][0]), + DD_pos_start[Ti][k][1], + s=20, + color="green", + ) # , label='start') + if type(DD_pos_end[Ti][k]) is tuple: + plt.scatter( + np.deg2rad(DD_pos_end[Ti][k][0]), + DD_pos_end[Ti][k][1], + s=20, + color="red", + ) # , label='end') if (type(DD_pos_start[Ti][k]) is tuple) & (type(DD_pos_end[Ti][k]) is tuple): - plt.plot( [np.deg2rad( DD_pos_start[Ti][k][0]), np.deg2rad( DD_pos_end[Ti][k][0])], [DD_pos_start[Ti][k][1], DD_pos_end[Ti][k][1]], color='black', linewidth=0.5 )# , width=0.5, edgecolor='none', facecolor= 'black') + plt.plot( + [np.deg2rad(DD_pos_start[Ti][k][0]), np.deg2rad(DD_pos_end[Ti][k][0])], + [DD_pos_start[Ti][k][1], DD_pos_end[Ti][k][1]], + color="black", + linewidth=0.5, + ) # , width=0.5, edgecolor='none', facecolor= 'black') # plt.quiver( DD_pos_start[Ti][k][0], DD_pos_start[Ti][k][1], DD_pos_end[Ti][k][0]-DD_pos_start[Ti][k][0] , DD_pos_end[Ti][k][1]- DD_pos_start[Ti][k][1], scale=5)# , width=0.5, edgecolor='none', facecolor= 'black') - #plt.ylim(TT_start[Ti][1]-1, TT_end[Ti][1] +1) + # plt.ylim(TT_start[Ti][1]-1, TT_end[Ti][1] +1) -plt.title(track_name + ' '+hemis, loc= 'left') +plt.title(track_name + " " + hemis, loc="left") plt.legend() -M.save_anyfig(plt.gcf(), name='A01b_'+track_name+'_'+ hemis+'_'+k , path=plot_path) +M.save_anyfig( + plt.gcf(), name="A01b_" + track_name + "_" + hemis + "_" + k, path=plot_path +) # %% @@ -431,8 +518,8 @@ def pole_ward_table(T): # plt.scatter(Tsel['x'].iloc[-1], Tsel['ref']['latitude'].iloc[-1],s=50, color='red') -#plt.close() -#plt.show() +# plt.close() +# plt.show() # DD_pos_end # @@ -447,42 +534,52 @@ def pole_ward_table(T): # DD_slope < 0 # if there is at leat one slope pair download data, otherwise write files and exit -if ( (DD_slope_mask.sum() > 1).sum() > 0) : - print('download data') +if (DD_slope_mask.sum() > 1).sum() > 0: + print("download data") else: - print('no suffcient data, quit()') - MT.json_save(name='A01b_'+track_name+'_success', path=save_path, \ - data= {'failed': 'True', 'reason':'no sufficient data' ,'slope': DD_slope.where(pd.notnull(DD_slope), 0).to_dict(), 'density':DD_data.where(pd.notnull(DD_data), 0).to_dict() }) + print("no suffcient data, quit()") + MT.json_save( + name="A01b_" + track_name + "_success", + path=save_path, + data={ + "failed": "True", + "reason": "no sufficient data", + "slope": DD_slope.where(pd.notnull(DD_slope), 0).to_dict(), + "density": DD_data.where(pd.notnull(DD_data), 0).to_dict(), + }, + ) exit() # %% # initiate ID files -ATL03_proposed, ATL03_remote_link, ATL03_remote= [],[],[] +ATL03_proposed, ATL03_remote_link, ATL03_remote = [], [], [] imp.reload(io) -for TF,TF_poleward in zip(['TF1', 'TF2'], [TF1_poleward, TF2_poleward]): +for TF, TF_poleward in zip(["TF1", "TF2"], [TF1_poleward, TF2_poleward]): iregion = DD_region[TF][DD_slope_mask[TF]] - if len(iregion) !=0: - #iregion2 = iregion[0][0] # for testing + if len(iregion) != 0: + # iregion2 = iregion[0][0] # for testing iregion2 = iregion[0] print(iregion2) # create track dict - CID = io.case_ID(hemis+'_'+track_name) + CID = io.case_ID(hemis + "_" + track_name) if type(iregion2) is str: # This is the SH case CID.GRN = iregion2 - CID.set() # reset Case ID - DD= {'case_ID': CID.ID , 'tracks' : {} } + CID.set() # reset Case ID + DD = {"case_ID": CID.ID, "tracks": {}} - CIDr = io.case_ID(hemis+'_'+track_name) + CIDr = io.case_ID(hemis + "_" + track_name) CIDr.GRN = iregion2 - CIDr.RL = '005' - #ATL03_list.append('ATL03_'+CIDr.set_ATL03_trackname()) - #print(CIDr.get_granule() in remote_names) + CIDr.RL = "005" + # ATL03_list.append('ATL03_'+CIDr.set_ATL03_trackname()) + # print(CIDr.get_granule() in remote_names) - #ATL03_dummy= [CIDr.set_dummy()] - ATL03_list = ['ATL03_'+CIDr.set_ATL03_trackname()] + # ATL03_dummy= [CIDr.set_dummy()] + ATL03_list = ["ATL03_" + CIDr.set_ATL03_trackname()] - org_files, remote_files, remote_names = io.nsidc_icesat2_get_associated_file(ATL03_list, 'ATL03') + org_files, remote_files, remote_names = ( + io.nsidc_icesat2_get_associated_file(ATL03_list, "ATL03") + ) # DD['tracks']['ATL03'] = ['ATL03_'+i for i in ATL03_list] # #DD['tracks']['ATL03_dummy'] = ATL03_dummy#['ATL03_'+i for i in ATL03_dummy] @@ -490,68 +587,98 @@ def pole_ward_table(T): # ATL03_remote_link +=remote_files # ATL03_remote +=remote_names - else: # this is the NH case - CID.GRN =iregion2[0] - CID.set() # reset Case ID - DD= {'case_ID': CID.ID , 'tracks' : {} } + else: # this is the NH case + CID.GRN = iregion2[0] + CID.set() # reset Case ID + DD = {"case_ID": CID.ID, "tracks": {}} - ATL03_list= list() + ATL03_list = list() for i in iregion2: - CIDr = io.case_ID(hemis+'_'+track_name) + CIDr = io.case_ID(hemis + "_" + track_name) CIDr.GRN = i - ATL03_list.append('ATL03_'+CIDr.set_ATL03_trackname()) + ATL03_list.append("ATL03_" + CIDr.set_ATL03_trackname()) - org_files, remote_files, remote_names = io.nsidc_icesat2_get_associated_file(ATL03_list, 'ATL03') + org_files, remote_files, remote_names = ( + io.nsidc_icesat2_get_associated_file(ATL03_list, "ATL03") + ) - DD['tracks']['ATL03'] = [rem.split('.')[0] for rem in remote_names] #['ATL03_'+i for i in ATL03_list] - ATL03_proposed += org_files - ATL03_remote_link += remote_files - ATL03_remote += remote_names + DD["tracks"]["ATL03"] = [ + rem.split(".")[0] for rem in remote_names + ] # ['ATL03_'+i for i in ATL03_list] + ATL03_proposed += org_files + ATL03_remote_link += remote_files + ATL03_remote += remote_names - DD['tracks']['ATL10'] = 'ATL10-' +CID.set_ATL10_trackname() - DD['tracks']['ATL07'] = 'ATL07-' +CID.set_ATL10_trackname() + DD["tracks"]["ATL10"] = "ATL10-" + CID.set_ATL10_trackname() + DD["tracks"]["ATL07"] = "ATL07-" + CID.set_ATL10_trackname() # add other pars: - DD['pars'] ={ - 'poleward':TF_poleward, 'region': iregion2, - 'start': {'longitude': TT_start[TF][0], 'latitude': TT_start[TF][1], 'seg_dist_x': TT_start[TF][2], 'delta_time': TT_start[TF][3]}, - 'end': {'longitude': TT_end[TF][0], 'latitude': TT_end[TF][1], 'seg_dist_x': TT_end[TF][2], 'delta_time': TT_end[TF][3]}, - 'beams':list(DD_data[TF].index), 'density':list(DD_data[TF]), 'slope': list(DD_slope[TF]) - } + DD["pars"] = { + "poleward": TF_poleward, + "region": iregion2, + "start": { + "longitude": TT_start[TF][0], + "latitude": TT_start[TF][1], + "seg_dist_x": TT_start[TF][2], + "delta_time": TT_start[TF][3], + }, + "end": { + "longitude": TT_end[TF][0], + "latitude": TT_end[TF][1], + "seg_dist_x": TT_end[TF][2], + "delta_time": TT_end[TF][3], + }, + "beams": list(DD_data[TF].index), + "density": list(DD_data[TF]), + "slope": list(DD_slope[TF]), + } # write .json ID file if files found - - if len(remote_names) !=0: - MT.json_save2(name='A01b_ID_'+CID.ID, path=save_path, data= DD) + if len(remote_names) != 0: + MT.json_save2(name="A01b_ID_" + CID.ID, path=save_path, data=DD) else: - print('no ATL03 track found for CID.ID') - - -ATL03_proposed = list(set(ATL03_proposed)) -ATL03_remote_link = list(set(ATL03_remote_link)) -ATL03_remote = list(set(ATL03_remote)) - - -if len(ATL03_remote) ==0: - print('no ATL03 tracks found! quit()') - MT.json_save(name='A01b_'+track_name+'_success', path=save_path, \ - data= {'failed': 'True', 'reason':'no ATL03 track found' ,'slope': DD_slope.where(pd.notnull(DD_slope), 0).to_dict(), 'density':DD_data.where(pd.notnull(DD_data), 0).to_dict() }) + print("no ATL03 track found for CID.ID") + + +ATL03_proposed = list(set(ATL03_proposed)) +ATL03_remote_link = list(set(ATL03_remote_link)) +ATL03_remote = list(set(ATL03_remote)) + + +if len(ATL03_remote) == 0: + print("no ATL03 tracks found! quit()") + MT.json_save( + name="A01b_" + track_name + "_success", + path=save_path, + data={ + "failed": "True", + "reason": "no ATL03 track found", + "slope": DD_slope.where(pd.notnull(DD_slope), 0).to_dict(), + "density": DD_data.where(pd.notnull(DD_data), 0).to_dict(), + }, + ) exit() -product_directory, sd = ATL03_remote_link[0].split('/')[4], ATL03_remote_link[0].split('/')[5] +product_directory, sd = ( + ATL03_remote_link[0].split("/")[4], + ATL03_remote_link[0].split("/")[5], +) # %% download ATL03 file to scratch folder def ATL03_download_worker(fname): - io.ATL03_download(None,None, scratch_path, product_directory, sd,fname) - print(fname, ' done') + io.ATL03_download(None, None, scratch_path, product_directory, sd, fname) + print(fname, " done") + if test_flag: for rname in remote_names: - MT.save_pandas_table({'dummy_download':DD_slope},rname.split('.')[0], scratch_path) + MT.save_pandas_table( + {"dummy_download": DD_slope}, rname.split(".")[0], scratch_path + ) else: with futures.ThreadPoolExecutor(max_workers=3) as executor: - A = list( executor.map(ATL03_download_worker, ATL03_remote) ) + A = list(executor.map(ATL03_download_worker, ATL03_remote)) # linear version @@ -560,20 +687,27 @@ def ATL03_download_worker(fname): # %% # print results and write files to exit -print('data density N/meter') +print("data density N/meter") print(DD_data) -print('slopes') +print("slopes") print(DD_slope) # write slope data for fun -#DD_merge = pd.concat({'density_Nperm':DD_data , 'slopes':DD_slope}, axis=1) -#DD_merge.to_html(save_path+ll_name+'.html') -#DD_merge.columns = ['-'.join(col).strip() for col in DD_merge.columns.values] -#MT.save_pandas_table({'T':DD_merge},ll_name, save_path) -#MT.json_save(name=ll_name, path=save_path, data= DD_merge.where(pd.notnull(DD_merge), 0).T.to_dict()) -#DD_merge.to_json(save_path+ll_name+'.json', orient="records", lines=True) +# DD_merge = pd.concat({'density_Nperm':DD_data , 'slopes':DD_slope}, axis=1) +# DD_merge.to_html(save_path+ll_name+'.html') +# DD_merge.columns = ['-'.join(col).strip() for col in DD_merge.columns.values] +# MT.save_pandas_table({'T':DD_merge},ll_name, save_path) +# MT.json_save(name=ll_name, path=save_path, data= DD_merge.where(pd.notnull(DD_merge), 0).T.to_dict()) +# DD_merge.to_json(save_path+ll_name+'.json', orient="records", lines=True) # write success file -MT.json_save(name='A01b_'+track_name+'_success', path=save_path, \ -data= {'failed': 'False', 'slope': DD_slope.where(pd.notnull(DD_slope), 0).to_dict(), 'density':DD_data.where(pd.notnull(DD_data), 0).to_dict() }) +MT.json_save( + name="A01b_" + track_name + "_success", + path=save_path, + data={ + "failed": "False", + "slope": DD_slope.where(pd.notnull(DD_slope), 0).to_dict(), + "density": DD_data.where(pd.notnull(DD_data), 0).to_dict(), + }, +) diff --git a/src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py b/src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py index c656a4f4..81683e95 100644 --- a/src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py +++ b/src/icesat2_tracks/analysis_db/B01_SL_load_single_file.py @@ -2,6 +2,7 @@ This file open a ICEsat2 tbeam_stats.pyrack applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3.11 """ + import sys import datetime import copy diff --git a/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py b/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py index bae0de4a..b339b287 100644 --- a/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py +++ b/src/icesat2_tracks/analysis_db/B03_plot_spectra_ov.py @@ -2,6 +2,7 @@ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ + import sys import numpy as np import xarray as xr diff --git a/src/icesat2_tracks/analysis_db/B05_define_angle.py b/src/icesat2_tracks/analysis_db/B05_define_angle.py index f201c207..942bc4df 100644 --- a/src/icesat2_tracks/analysis_db/B05_define_angle.py +++ b/src/icesat2_tracks/analysis_db/B05_define_angle.py @@ -3,13 +3,13 @@ This is python 3 """ -import sys +import sys from icesat2_tracks.config.IceSAT2_startup import ( mconfig, color_schemes, plt, - font_for_print + font_for_print, ) import icesat2_tracks.ICEsat2_SI_tools.iotools as io @@ -21,9 +21,10 @@ import icesat2_tracks.ICEsat2_SI_tools.lanczos as lanczos import icesat2_tracks.local_modules.m_tools_ph3 as MT import icesat2_tracks.local_modules.m_general_ph3 as M - + from matplotlib.gridspec import GridSpec from scipy.ndimage import label + color_schemes.colormaps2(21) col_dict = color_schemes.rels @@ -110,10 +111,12 @@ def weighted_means(data, weights, x_angle, color="k"): # makde dummy variables M_final = xr.full_like( - corrected_marginals.isel(k=0, beam_group=0).drop_vars("beam_group").drop_vars("k"), np.nan + corrected_marginals.isel(k=0, beam_group=0).drop_vars("beam_group").drop_vars("k"), + np.nan, ) M_final_smth = xr.full_like( - corrected_marginals.isel(k=0, beam_group=0).drop_vars("beam_group").drop_vars("k"), np.nan + corrected_marginals.isel(k=0, beam_group=0).drop_vars("beam_group").drop_vars("k"), + np.nan, ) if M_final.shape[0] > M_final.shape[1]: M_final = M_final.T diff --git a/src/icesat2_tracks/analysis_db/B06_correct_separate_var.py b/src/icesat2_tracks/analysis_db/B06_correct_separate_var.py index 14d485df..0a8bb463 100644 --- a/src/icesat2_tracks/analysis_db/B06_correct_separate_var.py +++ b/src/icesat2_tracks/analysis_db/B06_correct_separate_var.py @@ -2,6 +2,7 @@ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ + import os, sys from icesat2_tracks.config.IceSAT2_startup import ( @@ -10,7 +11,7 @@ font_for_pres, font_for_print, lstrings, - fig_sizes + fig_sizes, ) import h5py @@ -86,7 +87,7 @@ def dict_weighted_mean(Gdict, weight_key): N_photons = np.zeros(GSUM.N_per_stancil.size) counter = 0 - for _,I in Gdict.items(): + for _, I in Gdict.items(): I = I.squeeze() print(len(I.x)) if len(I.x) != 0: @@ -754,8 +755,11 @@ def save_table(data, tablename, save_path): os.remove(tabletoremove) io.save_pandas_table(data, tablename, save_path) + B06_ID_name = "B06_" + ID_name -table_names = [B06_ID_name + suffix for suffix in ["_B06_corrected_resid", "_binned_resid"]] +table_names = [ + B06_ID_name + suffix for suffix in ["_B06_corrected_resid", "_binned_resid"] +] data = [B2_v2, B3_v2] for tablename, data in zip(table_names, data): save_table(data, tablename, save_path) diff --git a/src/icesat2_tracks/analysis_db/__init__.py b/src/icesat2_tracks/analysis_db/__init__.py index 468e6aa7..8a4cd3c6 100644 --- a/src/icesat2_tracks/analysis_db/__init__.py +++ b/src/icesat2_tracks/analysis_db/__init__.py @@ -1 +1 @@ -"Logic to download and analyse WaveWatch3 files, plot spectra results" \ No newline at end of file +"Logic to download and analyse WaveWatch3 files, plot spectra results" diff --git a/src/icesat2_tracks/analysis_fake_data/A01_1D_spectra.py b/src/icesat2_tracks/analysis_fake_data/A01_1D_spectra.py index d168038f..d4aed2cb 100644 --- a/src/icesat2_tracks/analysis_fake_data/A01_1D_spectra.py +++ b/src/icesat2_tracks/analysis_fake_data/A01_1D_spectra.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,15 +9,16 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -#%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -33,63 +35,63 @@ # %% -f = np.arange(0.001, 0.2,0.001) +f = np.arange(0.001, 0.2, 0.001) spec_power = JONSWAP_gamma.JONSWAP_default(f, 2e6, 10) JONSWAP_gamma.JONSWAP_default(f, 2e6, 10) plt.plot(f, spec_power) -amps = (spec_power * np.gradient(f)) **0.5 -#plt.plot(f, amps) +amps = (spec_power * np.gradient(f)) ** 0.5 +# plt.plot(f, amps) -2/f[amps.argmax()] +2 / f[amps.argmax()] t = np.arange(0, 1000, 0.001) -tt, ww = np.meshgrid(2* np.pi * f, t) -phi = np.random.random(len(amps))*2*np.pi +tt, ww = np.meshgrid(2 * np.pi * f, t) +phi = np.random.random(len(amps)) * 2 * np.pi -instance = amps* np.cos( ww * tt + phi ) +instance = amps * np.cos(ww * tt + phi) # %% M.figure_axis_xy(6, 2, view_scale=0.9) -plt.plot(t, instance.mean(1) ) +plt.plot(t, instance.mean(1)) # %% -C= -3 -k = 2* np.pi / 200 +C = -3 +k = 2 * np.pi / 200 x = np.arange(0, 2000, 2) -phi = np.pi *3/2 -instance = C* np.cos(k * x + phi ) -plt.plot(x, instance, '.') +phi = np.pi * 3 / 2 +instance = C * np.cos(k * x + phi) +plt.plot(x, instance, ".") -A = C * np.cos(phi) -B = - C * np.sin(phi) +A = C * np.cos(phi) +B = -C * np.sin(phi) -instance2 = A * np.cos(k * x) + B * np.sin(k * x) -plt.plot(x, instance2) +instance2 = A * np.cos(k * x) + B * np.sin(k * x) +plt.plot(x, instance2) R = np.sqrt(A**2 + B**2) phi2 = np.arctan2(-B, A) -instance3 = R* np.cos(k * x + phi2 ) -plt.plot(x, instance3, '+') +instance3 = R * np.cos(k * x + phi2) +plt.plot(x, instance3, "+") # %% -C= -3 -k = 2* np.pi / 200 +C = -3 +k = 2 * np.pi / 200 x = np.arange(0, 4000, 2) -phi = np.pi *3/2 -instance = C* np.cos(k * x + phi ) -plt.plot(x, instance, '-') +phi = np.pi * 3 / 2 +instance = C * np.cos(k * x + phi) +plt.plot(x, instance, "-") -instance2 = C* np.cos(k*1.05 * x + phi + np.pi*1.5 ) -plt.plot(x, instance2, '-') +instance2 = C * np.cos(k * 1.05 * x + phi + np.pi * 1.5) +plt.plot(x, instance2, "-") -plt.plot(x, instance + instance2, '-k') +plt.plot(x, instance + instance2, "-k") # %% diff --git a/src/icesat2_tracks/analysis_fake_data/S02_spectra_estimates_test.py b/src/icesat2_tracks/analysis_fake_data/S02_spectra_estimates_test.py index 9b886521..4b2b49ee 100644 --- a/src/icesat2_tracks/analysis_fake_data/S02_spectra_estimates_test.py +++ b/src/icesat2_tracks/analysis_fake_data/S02_spectra_estimates_test.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,15 +9,16 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -30,29 +32,36 @@ import ICEsat2_SI_tools.spectral_estimates as spec import imp -#import s3fs + +# import s3fs # %% -Lx=10000 +Lx = 10000 xx = np.arange(0, Lx, 0.1) lam = 10 k = 1 / lam print(k) -k2 = 1 / 0.21 +k2 = 1 / 0.21 print(k2) -#yy = np.sin( k * xx * 2 * np.pi ) + np.sin( k2 * xx * 2 * np.pi ) + 0.1 *np.random.randn(len(xx)) +# yy = np.sin( k * xx * 2 * np.pi ) + np.sin( k2 * xx * 2 * np.pi ) + 0.1 *np.random.randn(len(xx)) -decay = np.exp( - 0.2 *(np.arange(len(xx))/Lx)) +decay = np.exp(-0.2 * (np.arange(len(xx)) / Lx)) plt.plot(decay) -yy = np.sin( k * xx * 2 * np.pi ) * decay + np.sin( 10 * k * xx * 2 * np.pi ) * decay + 0.3 *np.random.randn(len(xx)) +yy = ( + np.sin(k * xx * 2 * np.pi) * decay + + np.sin(10 * k * xx * 2 * np.pi) * decay + + 0.3 * np.random.randn(len(xx)) +) plt.plot(yy) # %% test simple pwelch wavenumber spectrum -L=400 +L = 400 imp.reload(specs) -S =specs.wavenumber_pwelch(yy, xx, L, ov=None, window=None, save_chunks=False, plot_chunks=False) +S = specs.wavenumber_pwelch( + yy, xx, L, ov=None, window=None, save_chunks=False, plot_chunks=False +) S.specs.shape S.spec_est.shape @@ -65,15 +74,15 @@ imp.reload(specs) S = specs.wavenumber_spetrogram(xx, yy, L) G = S.cal_spectrogram() -S.mean_spectral_error() # add x-mean spectal error estimate to xarray +S.mean_spectral_error() # add x-mean spectal error estimate to xarray -S.parceval(add_attrs= True) +S.parceval(add_attrs=True) G.attrs plt.plot(G.T) G.plot() # test of frequency is recovered -G.k[G.mean('x').argmax().data].data +G.k[G.mean("x").argmax().data].data # %% lomb-scargle diff --git a/src/icesat2_tracks/analysis_fake_data/S03_spectra_estimates_lomb_scargle.py b/src/icesat2_tracks/analysis_fake_data/S03_spectra_estimates_lomb_scargle.py index b228ed73..fdfcf055 100644 --- a/src/icesat2_tracks/analysis_fake_data/S03_spectra_estimates_lomb_scargle.py +++ b/src/icesat2_tracks/analysis_fake_data/S03_spectra_estimates_lomb_scargle.py @@ -1,5 +1,6 @@ import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. @@ -8,15 +9,16 @@ # exec(open(os.environ['PYTHONSTARTUP']).read()) # exec(open(STARTUP_2019_DP).read()) -base_path='/Users/Shared/Projects/2021_IceSAT2_tracks/' -sys.path.append(base_path +'modules/') -sys.path.append(base_path +'modules/ICEsat2_SI_tools/') +base_path = "/Users/Shared/Projects/2021_IceSAT2_tracks/" +sys.path.append(base_path + "modules/") +sys.path.append(base_path + "modules/ICEsat2_SI_tools/") import matplotlib.pyplot as plt -%matplotlib inline -#import m_general as M -#import m_tools as MT +# %matplotlib inline + +# import m_general as M +# import m_tools as MT import numpy as np import m_general_ph3 as M @@ -30,29 +32,34 @@ import ICEsat2_SI_tools.spectral_estimates as spec import imp -#import s3fs + +# import s3fs # %% -Lx=10000 +Lx = 10000 xx = np.arange(0, Lx, 0.1) lam = 10 k = 1 / lam print(k) -k2 = 1 / 0.21 +k2 = 1 / 0.21 print(k2) -#yy = np.sin( k * xx * 2 * np.pi ) + np.sin( k2 * xx * 2 * np.pi ) + 0.1 *np.random.randn(len(xx)) +# yy = np.sin( k * xx * 2 * np.pi ) + np.sin( k2 * xx * 2 * np.pi ) + 0.1 *np.random.randn(len(xx)) -decay = np.exp( - 0.2 *(np.arange(len(xx))/Lx)) +decay = np.exp(-0.2 * (np.arange(len(xx)) / Lx)) plt.plot(decay) -yy = 2.3 *np.sin( k * xx * 2 * np.pi ) + 4 * np.sin( 10 * k * xx * 2 * np.pi ) # + 0.05 *np.random.randn(len(xx)) +yy = 2.3 * np.sin(k * xx * 2 * np.pi) + 4 * np.sin( + 10 * k * xx * 2 * np.pi +) # + 0.05 *np.random.randn(len(xx)) plt.plot(yy) # %% test simple pwelch wavenumber spectrum -L=1000 +L = 1000 imp.reload(spec) -S =spec.wavenumber_pwelch(yy, xx, L, ov=None, window=np.ones(L), save_chunks=False, plot_chunks=False) +S = spec.wavenumber_pwelch( + yy, xx, L, ov=None, window=np.ones(L), save_chunks=False, plot_chunks=False +) S.specs.shape @@ -62,19 +69,19 @@ plt.plot(S.k, S.spec_est) -#np.sqrt(2 *S.spec_est *S.df).max() -plt.plot(S.k, np.sqrt(2 *S.spec_est *S.df)) -plt.ylabel('wave height(m)') +# np.sqrt(2 *S.spec_est *S.df).max() +plt.plot(S.k, np.sqrt(2 * S.spec_est * S.df)) +plt.ylabel("wave height(m)") # %% test class # imp.reload(specs) -#S = spec.wavenumber_spetrogram(xx, yy, L) +# S = spec.wavenumber_spetrogram(xx, yy, L) # G = S.cal_spectrogram() # S.mean_spectral_error() # add x-mean spectal error estimate to xarray # # S.parceval(add_attrs= True) # test of frequency is recovered -#G.k[G.mean('x').argmax().data].data +# G.k[G.mean('x').argmax().data].data # %% lomb-scargle from astropy.timeseries import LombScargle @@ -83,68 +90,70 @@ spec_fft = spec.calc_spectrum_fft(yy, dk, yy.size) ls = LombScargle(xx, yy) -k_ls, power_ls = ls.autopower(minimum_frequency=k[1], maximum_frequency = k[-1]) +k_ls, power_ls = ls.autopower(minimum_frequency=k[1], maximum_frequency=k[-1]) k_ls2 = k[1:][::3] power2 = ls.power(k_ls2) + def ls_power_to_fft_power(power2, N): """ convert ls output to spectral density amplitudes |unit^2|/dk as used for standard fft """ - return N *power2 + return N * power2 + def ls_power_to_wave_height(power2, N, dk): """ convert ls output to spectral density amplitudes of [units] """ - return np.sqrt(2 * N * power2 *dk ) - + return np.sqrt(2 * N * power2 * dk) plt.plot(k, spec_fft) -plt.plot(k_ls, power_ls * 2/np.gradient(k_ls)[0] ) -plt.plot(k_ls,ls_power_to_fft_power(power_ls, yy.size) ) +plt.plot(k_ls, power_ls * 2 / np.gradient(k_ls)[0]) +plt.plot(k_ls, ls_power_to_fft_power(power_ls, yy.size)) -plt.plot(k_ls2, ls_power_to_wave_height(power2, yy.size, dk ), '.' ) -plt.plot(k, np.sqrt(2 *spec_fft *dk)) +plt.plot(k_ls2, ls_power_to_wave_height(power2, yy.size, dk), ".") +plt.plot(k, np.sqrt(2 * spec_fft * dk)) # %% test wave spectra # make fake data import JONSWAP_gamma -dff= 0.001 #0.0005#0.001 + +dff = 0.001 # 0.0005#0.001 f = np.arange(0.01, 0.2, dff) -dff/ f[0] +dff / f[0] spec_power = JONSWAP_gamma.JONSWAP_default(f, 1e6, 10) -#plt.plot(f, spec_power) +# plt.plot(f, spec_power) # variance -print( 'JONSWAP variance' ,(spec_power * dff).sum() ) +print("JONSWAP variance", (spec_power * dff).sum()) -omega= 2 *np.pi * f +omega = 2 * np.pi * f -dt=0.05# 0.01 +dt = 0.05 # 0.01 t = np.arange(0, 10000, dt) -t.size/40 +t.size / 40 -amps = ( 2 * spec_power * dff )**0.5 +amps = (2 * spec_power * dff) ** 0.5 -ww, tt = np.meshgrid(2* np.pi * f, t) -phi = np.random.random(len(amps))*2*np.pi +ww, tt = np.meshgrid(2 * np.pi * f, t) +phi = np.random.random(len(amps)) * 2 * np.pi -instance = amps* np.sin( ww * tt + phi ) -y = instance.sum(1)# + 0.05* np.random.random(len(t)) +instance = amps * np.sin(ww * tt + phi) +y = instance.sum(1) # + 0.05* np.random.random(len(t)) -print( 'data variance' , y.var() )# +print("data variance", y.var()) # # M.figure_axis_xy(6, 4, view_scale=0.5) -plt.subplot(2,1,1) +plt.subplot(2, 1, 1) plt.plot(f, spec_power) -plt.ylabel( 'spectral power m^2/Hz') +plt.ylabel("spectral power m^2/Hz") -plt.subplot(2,1,2) -plt.plot(t, y) +plt.subplot(2, 1, 2) +plt.plot(t, y) plt.xlim(0, 500) # %% manual FFT test @@ -159,16 +168,16 @@ def ls_power_to_wave_height(power2, N, dk): # plt.xlim(0, 0.2) # %% -min_datapoint= 1/f[0]/dt +min_datapoint = 1 / f[0] / dt # % standard FFT print(t.size) print(min_datapoint) -L= int(min_datapoint * 10) +L = int(min_datapoint * 10) print(L) -S = spec.wavenumber_pwelch( y, t, L) +S = spec.wavenumber_pwelch(y, t, L) f_fft2, df2 = spec.calc_freq_fft(t, t.size) @@ -177,28 +186,34 @@ def ls_power_to_wave_height(power2, N, dk): ls = LombScargle(t, y) # #k_ls2 = k[1:][::3] -f_fft= S.k -f_fft = f_fft[ (f_fft> 0.05) & (f_fft < 0.5) ] -ls_power = ls.power(f_fft ,normalization='psd',assume_regular_frequency='True', method='auto') +f_fft = S.k +f_fft = f_fft[(f_fft > 0.05) & (f_fft < 0.5)] +ls_power = ls.power( + f_fft, normalization="psd", assume_regular_frequency="True", method="auto" +) len(f_fft) -ls_auto_f , ls_auto_power = ls.autopower(minimum_frequency=0.05, maximum_frequency=0.5, -normalization='psd', samples_per_peak=0.1) +ls_auto_f, ls_auto_power = ls.autopower( + minimum_frequency=0.05, + maximum_frequency=0.5, + normalization="psd", + samples_per_peak=0.1, +) len(ls_auto_f) M.figure_axis_xy(5, 4, view_scale=0.8) # orginal -plt.plot(f, spec_power, 'k' , linewidth = 6, alpha=0.3, label = 'original') +plt.plot(f, spec_power, "k", linewidth=6, alpha=0.3, label="original") # -plt.plot(S.k, S.spec_est , label ='Pwelch FFT') -print('pwelch FFT variance', (S.spec_est * S.df ).sum() ) +plt.plot(S.k, S.spec_est, label="Pwelch FFT") +print("pwelch FFT variance", (S.spec_est * S.df).sum()) # plt.plot(f_fft2[1:], spec_fft2[1:], '-',alpha = 0.4, label = 'FFT') # print('pure FFT', (spec_fft2 * np.gradient(f_fft2)[0] ).sum() ) df = np.gradient(S.k)[0] -plt.plot(f_fft, 2 * ls_power/ t.size /dff, '-+' , label= 'LS') -plt.plot(ls_auto_f, 2 * ls_auto_power/ t.size /dff, '-+' , label= 'LS auto') +plt.plot(f_fft, 2 * ls_power / t.size / dff, "-+", label="LS") +plt.plot(ls_auto_f, 2 * ls_auto_power / t.size / dff, "-+", label="LS auto") plt.legend() plt.ylim(0, 21) plt.xlim(0, 0.2) @@ -206,42 +221,44 @@ def ls_power_to_wave_height(power2, N, dk): # %% -t_noise = t + 0.05 *np.random.randn(len(t)) -y_noise_instance = 1 *np.random.randn(len(t)) +t_noise = t + 0.05 * np.random.randn(len(t)) +y_noise_instance = 1 * np.random.randn(len(t)) y_noise = y + y_noise_instance # #k_ls2 = k[1:][::3] -#ls_f_noise= S.k -#ls_f_noise = ls_f_noise[ (ls_f_noise> 0.05) & (ls_f_noise < 0.5) ] -ls_f_noise = np.arange(S.k[1], S.k[-1],np.diff(S.k)[0]/2 ) # -#ls_f_noise = np.arange(0.05, 0.5,np.diff(S.k)[0]/2 ) # increase the spectral resolution to double of the fft one +# ls_f_noise= S.k +# ls_f_noise = ls_f_noise[ (ls_f_noise> 0.05) & (ls_f_noise < 0.5) ] +ls_f_noise = np.arange(S.k[1], S.k[-1], np.diff(S.k)[0] / 2) # +# ls_f_noise = np.arange(0.05, 0.5,np.diff(S.k)[0]/2 ) # increase the spectral resolution to double of the fft one ls_noise = LombScargle(t_noise, y_noise, fit_mean=True) ls_noise.offset() -%time ls_power_noise = ls_noise.power(ls_f_noise, normalization='psd') +# %time ls_power_noise = ls_noise.power(ls_f_noise, normalization='psd') len(ls_f_noise) np.diff(ls_f_noise)[0] ls_noise_auto = LombScargle(t_noise, y_noise, fit_mean=True) -%time ls_auto_f_noise, ls_auto_power_noise = ls_noise_auto.autopower(minimum_frequency=0.05, maximum_frequency=0.25, normalization='psd', samples_per_peak=0.1) +# %time ls_auto_f_noise, ls_auto_power_noise = ls_noise_auto.autopower(minimum_frequency=0.05, maximum_frequency=0.25, normalization='psd', samples_per_peak=0.1) len(ls_auto_f_noise) -#np.diff(ls_auto_f_noise)[0] +# np.diff(ls_auto_f_noise)[0] -plt.plot(f, spec_power, 'k' , linewidth = 2, label = 'orginal') -print( 'JONSWAP variance' ,(spec_power * np.diff(f).mean()).sum() ) +plt.plot(f, spec_power, "k", linewidth=2, label="orginal") +print("JONSWAP variance", (spec_power * np.diff(f).mean()).sum()) -plt.plot(ls_f_noise, 2 * ls_power_noise/ t.size /dff, '-+' , label= 'LS noise') -ls_power_noise_P = (2 * ls_power_noise/ t.size /dff * np.diff(ls_f_noise).mean()) -print( 'LS noise variance' ,ls_power_noise_P.sum() ) +plt.plot(ls_f_noise, 2 * ls_power_noise / t.size / dff, "-+", label="LS noise") +ls_power_noise_P = 2 * ls_power_noise / t.size / dff * np.diff(ls_f_noise).mean() +print("LS noise variance", ls_power_noise_P.sum()) -plt.plot(ls_auto_f_noise, 2 * ls_auto_power_noise / t.size /dff, '-+' , label= 'LS auto noise') +plt.plot( + ls_auto_f_noise, 2 * ls_auto_power_noise / t.size / dff, "-+", label="LS auto noise" +) -print( 'data variance' ,y_noise.var() , 'noise variance ' , y_noise_instance.var() ) -print( 'diff variance' , y_noise.var() - y_noise_instance.var() ) +print("data variance", y_noise.var(), "noise variance ", y_noise_instance.var()) +print("diff variance", y_noise.var() - y_noise_instance.var()) plt.legend() plt.ylim(0, 21) @@ -249,60 +266,71 @@ def ls_power_to_wave_height(power2, N, dk): # %% M.figure_axis_xy(6, 4) -plt.plot(ls_f_noise, ls_power_noise_P.cumsum(),label ='LS noise') -plt.plot(ls_f_noise, ls_f_noise*0 + (spec_power * np.diff(f).mean()).sum(), 'k' ,label ='signal variance') -plt.plot(ls_f_noise, ls_f_noise*0 + y_noise.var() , 'k--' ,label ='total variance') +plt.plot(ls_f_noise, ls_power_noise_P.cumsum(), label="LS noise") +plt.plot( + ls_f_noise, + ls_f_noise * 0 + (spec_power * np.diff(f).mean()).sum(), + "k", + label="signal variance", +) +plt.plot(ls_f_noise, ls_f_noise * 0 + y_noise.var(), "k--", label="total variance") plt.legend() -plt.xlabel('wavenumber') -plt.ylabel('cumulative variance') +plt.xlabel("wavenumber") +plt.ylabel("cumulative variance") # %% test phase -f_sel = ls_f_noise#[ (ls_f_noise> 0.05) & (ls_f_noise < 0.1) ] +f_sel = ls_f_noise # [ (ls_f_noise> 0.05) & (ls_f_noise < 0.1) ] m1 = ls_noise.offset() * np.ones(len(t_fit)) for fi in f_sel: m1 += ls_noise.model(t_fit, fi) -f_sel = ls_auto_f_noise#[ (f_fft> 0) & (f_fft < 0.8) ] +f_sel = ls_auto_f_noise # [ (f_fft> 0) & (f_fft < 0.8) ] m2 = ls_noise_auto.offset() * np.ones(len(t_fit)) for fi in f_sel: m2 += ls_noise_auto.model(t_fit, fi) # alternative way -#design_matrix = ls_noise.design_matrix(best_frequency, t_fit) +# design_matrix = ls_noise.design_matrix(best_frequency, t_fit) # %% -plt.plot( t_fit, m1-m1.mean(), zorder= 12, label='m1 | LS noise') -plt.plot( t_fit, m2-m2.mean(), zorder= 12, label='m2 | LS auto noise') -plt.plot( t_noise[0:2000], y_noise[0:2000], '.') +plt.plot(t_fit, m1 - m1.mean(), zorder=12, label="m1 | LS noise") +plt.plot(t_fit, m2 - m2.mean(), zorder=12, label="m2 | LS auto noise") +plt.plot(t_noise[0:2000], y_noise[0:2000], ".") plt.legend() # %% -plt.plot( t_fit, y_noise[0:2000] -m2, '.') +plt.plot(t_fit, y_noise[0:2000] - m2, ".") # %% define functions: -#def calc_freq_LS -ls_f_noise = np.arange(S.k[1], S.k[-1],np.diff(S.k)[0]/2 ) # -#ls_f_noise = np.arange(0.05, 0.5,np.diff(S.k)[0]/2 ) # increase the spectral resolution to double of the fft one +# def calc_freq_LS +ls_f_noise = np.arange(S.k[1], S.k[-1], np.diff(S.k)[0] / 2) # +# ls_f_noise = np.arange(0.05, 0.5,np.diff(S.k)[0]/2 ) # increase the spectral resolution to double of the fft one ls_noise = LombScargle(t_noise, y_noise, fit_mean=True) ls_noise.offset() -ls_power_noise = ls_noise.power(ls_f_noise, normalization='psd') +ls_power_noise = ls_noise.power(ls_f_noise, normalization="psd") len(ls_f_noise) np.diff(ls_f_noise)[0] ls_noise_auto = LombScargle(t_noise, y_noise, fit_mean=True) -ls_auto_f_noise, ls_auto_power_noise = ls_noise_auto.autopower(minimum_frequency=0.05, maximum_frequency=0.25, normalization='psd', samples_per_peak=0.1) - +ls_auto_f_noise, ls_auto_power_noise = ls_noise_auto.autopower( + minimum_frequency=0.05, + maximum_frequency=0.25, + normalization="psd", + samples_per_peak=0.1, +) -ls_auto_f_noise- ls_auto_f_noise2 +ls_auto_f_noise - ls_auto_f_noise2 ls_noise = LombScargle(t_noise, y_noise, fit_mean=True) -ls_auto_f_noise2 = ls_noise.autofrequency(minimum_frequency=0.05, maximum_frequency=0.25, samples_per_peak=0.1) +ls_auto_f_noise2 = ls_noise.autofrequency( + minimum_frequency=0.05, maximum_frequency=0.25, samples_per_peak=0.1 +) len(ls_auto_f_noise2) @@ -311,10 +339,10 @@ def ls_power_to_wave_height(power2, N, dk): # ls_auto_f_noise3 = ls_noise.autofrequency(minimum_frequency=0.05, maximum_frequency=0.25, samples_per_peak=0.1) -min_datapoint= 1/f[0]/dt +min_datapoint = 1 / f[0] / dt L = 1000 -f3 , df3 = spec.calc_freq_fft(t_noise, L) +f3, df3 = spec.calc_freq_fft(t_noise, L) f3.size # def calc_freq_LS(x, N, method='fftX2', minimum_frequency=None, maximum_frequency=None, samples_per_peak=0.01): @@ -362,9 +390,10 @@ def ls_power_to_wave_height(power2, N, dk): # f4, df4 = calc_freq_LS(t_noise, L, method = 'LS_auto', minimum_frequency=0, maximum_frequency=5) # for tesing: -f3, df3 = spec.calc_freq_LS(t_noise, L, method = 'fftX2') -f4, df4 = spec.calc_freq_LS(t_noise, L, method = 'LS_auto', minimum_frequency=0, maximum_frequency=5) - +f3, df3 = spec.calc_freq_LS(t_noise, L, method="fftX2") +f4, df4 = spec.calc_freq_LS( + t_noise, L, method="LS_auto", minimum_frequency=0, maximum_frequency=5 +) # %% @@ -372,41 +401,50 @@ def ls_power_to_wave_height(power2, N, dk): # dt is the sampling frequency -#f[0] is tje lowerest resolved frequuency whee there is expected to be power -min_datapoint= 1/f[0]/dt +# f[0] is tje lowerest resolved frequuency whee there is expected to be power +min_datapoint = 1 / f[0] / dt #'LS_auto','fixed_ratio' -#L = 10000 -L = int(min_datapoint *10) -#L = int(y_noise.size/2) -S2 = spec.wavenumber_spectrogram_LS(t_noise, y_noise, L, waven_method = 'fixed_ratio' , dy=None , ov=None, window=None, kjumps=1) +# L = 10000 +L = int(min_datapoint * 10) +# L = int(y_noise.size/2) +S2 = spec.wavenumber_spectrogram_LS( + t_noise, + y_noise, + L, + waven_method="fixed_ratio", + dy=None, + ov=None, + window=None, + kjumps=1, +) dx = np.diff(t_noise).mean() -%time G = S2.cal_spectrogram() +# %time G = S2.cal_spectrogram() -print(' L / k legnth ', L/ len(S2.k) ) -print('dk / dx ', S2.dk / dx) +print(" L / k legnth ", L / len(S2.k)) +print("dk / dx ", S2.dk / dx) # # S2.dk # dx # df = -1./((L-1)*dx) +1.0 / ((L - 1) * dx) # # 0.02 # % -plt.plot(f, spec_power, 'k' , linewidth = 2, label = 'orginal', zorder =12) +plt.plot(f, spec_power, "k", linewidth=2, label="orginal", zorder=12) -plt.plot(G.k, G , '-' , label= 'LS noise') -#plt.plot(ls_auto_f_noise, 2 * ls_auto_power_noise / t.size /dff, '-+' , label= 'LS auto noise') +plt.plot(G.k, G, "-", label="LS noise") +# plt.plot(ls_auto_f_noise, 2 * ls_auto_power_noise / t.size /dff, '-+' , label= 'LS auto noise') -#plt.legend() -#plt.ylim(0, 21) +# plt.legend() +# plt.ylim(0, 21) plt.xlim(0, 0.2) -S2.dk*S2.G.mean('x').sel(k= slice(0, 0.5)).sum().data +S2.dk * S2.G.mean("x").sel(k=slice(0, 0.5)).sum().data # %% -#wavenumber_spectrogram_LS.calc_var +# wavenumber_spectrogram_LS.calc_var S2.calc_var() S2.parceval() @@ -414,13 +452,12 @@ def ls_power_to_wave_height(power2, N, dk): S2.mean_spectral_error() -plt.plot(S2.G.k, S2.G.mean('x') , '-' , label= 'LS noise') -plt.plot(S2.G.k, S2.G['mean_El'] , '-' , label= 'LS noise') -plt.plot(S2.G.k, S2.G['mean_Eu'] , '-' , label= 'LS noise') +plt.plot(S2.G.k, S2.G.mean("x"), "-", label="LS noise") +plt.plot(S2.G.k, S2.G["mean_El"], "-", label="LS noise") +plt.plot(S2.G.k, S2.G["mean_Eu"], "-", label="LS noise") plt.xlim(0, 0.20) - -#print( 'JONSWAP variance' ,(spec_power * dff).sum() ) +# print( 'JONSWAP variance' ,(spec_power * dff).sum() ) # %% np.log(G).sel(k=slice(0, 0.5)).plot() diff --git a/src/icesat2_tracks/analysis_fake_data/X03_MCMC_surface_smapling.py b/src/icesat2_tracks/analysis_fake_data/X03_MCMC_surface_smapling.py index 4e3f65a9..261c5379 100644 --- a/src/icesat2_tracks/analysis_fake_data/X03_MCMC_surface_smapling.py +++ b/src/icesat2_tracks/analysis_fake_data/X03_MCMC_surface_smapling.py @@ -1,16 +1,16 @@ - import os, sys -#execfile(os.environ['PYTHONSTARTUP']) + +# execfile(os.environ['PYTHONSTARTUP']) """ This file open a ICEsat2 track applied filters and corections and returns smoothed photon heights on a regular grid in an .nc file. This is python 3 """ -exec(open(os.environ['PYTHONSTARTUP']).read()) +exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2021_IceSAT2).read()) -#%matplotlib inline +# %matplotlib inline import ICEsat2_SI_tools.convert_GPS_time as cGPS import h5py @@ -26,21 +26,29 @@ # %% - Lx, Ly = 20, 10 -x = np.linspace(0,Lx,200) -y = np.linspace(0, Ly,100) +x = np.linspace(0, Lx, 200) +y = np.linspace(0, Ly, 100) + +xx, yy = np.meshgrid(x, y) -xx,yy = np.meshgrid(x, y) def objective_func(params): - sn2 = 0.1**2 - return - cost(params['x'], params['y']) + np.log(sn2) + sn2 = 0.1**2 + return -cost(params["x"], params["y"]) + np.log(sn2) + def cost(x, y): - z = 4+ np.sin(4* 2 * np.pi *x/Lx) + np.sin( 3 * np.pi *x/Lx - np.pi/5) + np.cos(1* 2 * np.pi *y/Ly) + np.sin( 3 * np.pi *y/Ly - np.pi/3) + z = ( + 4 + + np.sin(4 * 2 * np.pi * x / Lx) + + np.sin(3 * np.pi * x / Lx - np.pi / 5) + + np.cos(1 * 2 * np.pi * y / Ly) + + np.sin(3 * np.pi * y / Ly - np.pi / 3) + ) return z**2 + class sample_with_mcmc: """ sample a 2nd surface using mcmc. its make for getting a quick estimate! @@ -49,16 +57,17 @@ class sample_with_mcmc: def __init__(self): import lmfit as LM + self.LM = LM def objective_func(self, params): - sn2 = 0.1**2 - return - cost(params['x'], params['y']) + np.log(sn2) + sn2 = 0.1**2 + return -cost(params["x"], params["y"]) + np.log(sn2) # def test_ojective_func(self, model_func): # return self.objective_func(self.params, self.data, model_func, self.freq) - def set_parameters(self, par_dict, verbose= False): + def set_parameters(self, par_dict, verbose=False): """ defines params object at inital seed for mcmc par_dict should contain: var_name : [min, max, nseed] @@ -67,36 +76,47 @@ def set_parameters(self, par_dict, verbose= False): params = self.LM.Parameters() var_seeds = list() - for k,I in par_dict.items(): - params.add(k, (I[0]+ I[1])/2, vary=True , min=I[0], max=I[1]) - - var_seeds.append( np.linspace(I[0],I[1], I[2])) + for k, I in par_dict.items(): + params.add(k, (I[0] + I[1]) / 2, vary=True, min=I[0], max=I[1]) + var_seeds.append(np.linspace(I[0], I[1], I[2])) if len(var_seeds) > 2: - raise ValueError('nor proframmed for 3d') + raise ValueError("nor proframmed for 3d") - self.nwalkers= int(var_seeds[0].size * var_seeds[1].size) + self.nwalkers = int(var_seeds[0].size * var_seeds[1].size) pxx, pyy = np.meshgrid(var_seeds[0], var_seeds[1]) - self.seeds = np.vstack([pxx.flatten(), pyy.flatten() ]).T + self.seeds = np.vstack([pxx.flatten(), pyy.flatten()]).T self.params = params if verbose: - print('Nwalker: ', self.nwalkers) - print('Seeds: ', self.seeds.shape) + print("Nwalker: ", self.nwalkers) + print("Seeds: ", self.seeds.shape) print(self.params) - def sample(self, fitting_args= None , method='emcee', steps=100, verbose= True): + def sample(self, fitting_args=None, method="emcee", steps=100, verbose=True): - self.fitter = self.LM.minimize(self.objective_func, self.params, method=method, - nwalkers=self.nwalkers, steps=steps, pos= self.seeds) + self.fitter = self.LM.minimize( + self.objective_func, + self.params, + method=method, + nwalkers=self.nwalkers, + steps=steps, + pos=self.seeds, + ) if verbose: print(self.LM.report_fit(self.fitter)) - def optimize(self, fitting_args= None , method='dual_annealing', verbose= True): + def optimize(self, fitting_args=None, method="dual_annealing", verbose=True): - self.fitter_optimize = self.LM.minimize(self.objective_func, self.params, method=method, - nwalkers=self.nwalkers, steps=steps, pos= self.seeds) + self.fitter_optimize = self.LM.minimize( + self.objective_func, + self.params, + method=method, + nwalkers=self.nwalkers, + steps=steps, + pos=self.seeds, + ) if verbose: print(self.LM.report_fit(self.fitter)) @@ -114,93 +134,120 @@ def flatchain(self, burn=None): else: return fitter.flatchain - def get_marginal_dist(self, var, var_dx, burn = None, plot_flag= False, normalize = True): + def get_marginal_dist( + self, var, var_dx, burn=None, plot_flag=False, normalize=True + ): data = self.flatchain(burn) - #fitter.flatchain.loc[100:][var] - bins = np.arange(self.params[var].min,self.params[var].max+ var_dx,var_dx) + # fitter.flatchain.loc[100:][var] + bins = np.arange(self.params[var].min, self.params[var].max + var_dx, var_dx) y_hist, _ = np.histogram(fitter.flatchain.loc[100:][var], bins) - bins_pos = (b[0:-1] + np.diff(b)/2).shape + bins_pos = (b[0:-1] + np.diff(b) / 2).shape if normalize: - y_hist = y_hist/var_dx/y_hist.sum() + y_hist = y_hist / var_dx / y_hist.sum() if plot_flag: import matplotlib.pyplot as plt + plt.stairs(y_hist, bins) return y_hist, bins, bins_pos -par_dict = {'x': [0,Lx, 10], 'y':[0,Ly, 10] } + +par_dict = {"x": [0, Lx, 10], "y": [0, Ly, 10]} SM = sample_with_mcmc() -SM.set_parameters(par_dict, verbose= True) +SM.set_parameters(par_dict, verbose=True) SM.params SM.seeds SM.nwalkers SM.sample() SM.chain().shape -SM.flatchain(burn = 15000) -SM.get_marginal_dist('x', 0.1, plot_flag= True, normalize = True) +SM.flatchain(burn=15000) +SM.get_marginal_dist("x", 0.1, plot_flag=True, normalize=True) -np.log(1/5.5) +np.log(1 / 5.5) -def gaussian(x,x0,sigma): - return np.exp(-np.power((x - x0)/sigma, 2.)/2.) +def gaussian(x, x0, sigma): + return np.exp(-np.power((x - x0) / sigma, 2.0) / 2.0) + def simple_log_panelty(x, x0, sigma): - return -np.power((x - x0)/sigma, 2.)/2. + return -np.power((x - x0) / sigma, 2.0) / 2.0 + # %% -#fitter.flatchain.loc[100:] +# fitter.flatchain.loc[100:] import lmfit as LM -params = LM.Parameters() -#p0= (Lx/2 *np.random.rand(1), Ly/2 *np.random.rand(1)) -p0= [Lx/20 , Ly/2 ] +params = LM.Parameters() -params.add('x', p0[0], vary=True , min=0, max=Lx) -params.add('y', p0[1], vary=True , min=0, max=Ly) +# p0= (Lx/2 *np.random.rand(1), Ly/2 *np.random.rand(1)) +p0 = [Lx / 20, Ly / 2] -#fitting_args = (x_concat, y_concat, z_concat) +params.add("x", p0[0], vary=True, min=0, max=Lx) +params.add("y", p0[1], vary=True, min=0, max=Ly) +# fitting_args = (x_concat, y_concat, z_concat) -p0x = np.linspace(0,Lx, 10) -p0y = np.linspace(0, Ly,15) -nwalkers= p0x.size * p0y.size +p0x = np.linspace(0, Lx, 10) +p0y = np.linspace(0, Ly, 15) +nwalkers = p0x.size * p0y.size pxx, pyy = np.meshgrid(p0x, p0y) -#pos0 = np.repeat(p0, nwalkers).reshape(len(p0), nwalkers, ) +# pos0 = np.repeat(p0, nwalkers).reshape(len(p0), nwalkers, ) -pos0= np.vstack([pxx.flatten(), pyy.flatten() ]) -#%timeit fitter = LM.minimize(objective_func, params, method='dual_annealing',max_nfev=None) -#%timeit fitter = LM.minimize(objective_func, params, method='emcee', nwalkers=nwalkers, steps=200, pos= pos0.T, workers= 3) +pos0 = np.vstack([pxx.flatten(), pyy.flatten()]) +# %timeit fitter = LM.minimize(objective_func, params, method='dual_annealing',max_nfev=None) +# %timeit fitter = LM.minimize(objective_func, params, method='emcee', nwalkers=nwalkers, steps=200, pos= pos0.T, workers= 3) -%timeit fitter = LM.minimize(objective_func, params, method='emcee', nwalkers=nwalkers, steps=100, pos= pos0.T) -fitter = LM.minimize(objective_func, params, method='emcee', nwalkers=nwalkers, steps=100, pos= pos0.T) +# %timeit fitter = LM.minimize(objective_func, params, method='emcee', nwalkers=nwalkers, steps=100, pos= pos0.T) +fitter = LM.minimize( + objective_func, params, method="emcee", nwalkers=nwalkers, steps=100, pos=pos0.T +) print(LM.report_fit(fitter)) -#print(fitter.pretty_print()) -#%timeit -#fitter = LM.minimize(objective_func, params, method='brute', workers=1 , Ns=120) +# print(fitter.pretty_print()) +# %timeit +# fitter = LM.minimize(objective_func, params, method='brute', workers=1 , Ns=120) -cost_2d= cost(xx, yy) +cost_2d = cost(xx, yy) -print( 'print final diff:' , cost_2d.min()- cost(fitter.params['x'].value, fitter.params['y'].value) ) +print( + "print final diff:", + cost_2d.min() - cost(fitter.params["x"].value, fitter.params["y"].value), +) # %% -plt.contourf(x, y, cost(xx, yy) ) -plt.axis('equal') +plt.contourf(x, y, cost(xx, yy)) +plt.axis("equal") for n in np.arange(nwalkers): - plt.plot(fitter.chain[:,n,0], fitter.chain[:,n,1] , '-', markersize= 2, linewidth= 0.8, alpha= 0.2, color= 'white') - plt.plot(fitter.chain[:,n,0], fitter.chain[:,n,1] , '.', markersize= 2, linewidth= 0.8, alpha= 0.2, color= 'white') + plt.plot( + fitter.chain[:, n, 0], + fitter.chain[:, n, 1], + "-", + markersize=2, + linewidth=0.8, + alpha=0.2, + color="white", + ) + plt.plot( + fitter.chain[:, n, 0], + fitter.chain[:, n, 1], + ".", + markersize=2, + linewidth=0.8, + alpha=0.2, + color="white", + ) # # # plt.plot(fitter.params['x'].value, fitter.params['y'].value, '.r', markersize=20) @@ -214,20 +261,19 @@ def simple_log_panelty(x, x0, sigma): # %% fitter.flatchain.shape fitter.chain.shape -burn= 10 -chain_burned= fitter.chain[burn:, :, :] +burn = 10 +chain_burned = fitter.chain[burn:, :, :] cs = chain_burned.shape chain_flat = chain_burned.reshape(cs[0] * cs[1], 2) chain_flat.shape -plt.hist(chain_flat[:, 1], bins= 60) - # %% -var = 'y' +plt.hist(chain_flat[:, 1], bins=60) +# %% +var = "y" var_dx = 0.2 y_hist, _ = np.histogram(fitter.flatchain.loc[100:][var], bins) -bins_pos = (b[0:-1] + np.diff(b)/2).shape - +bins_pos = (b[0:-1] + np.diff(b) / 2).shape y_hist_normed.sum() * var_dx diff --git a/src/icesat2_tracks/config/IceSAT2_startup.py b/src/icesat2_tracks/config/IceSAT2_startup.py index d1e27938..a3eb581e 100644 --- a/src/icesat2_tracks/config/IceSAT2_startup.py +++ b/src/icesat2_tracks/config/IceSAT2_startup.py @@ -9,7 +9,7 @@ ## Read folders and configuration paths config_dir_path = os.path.dirname(__file__) -mconfig=MT.json_load('config',config_dir_path) +mconfig = MT.json_load("config", config_dir_path) ## check folders exist. Create if dont. for folder_name, folder_path in mconfig["paths"].items(): @@ -19,54 +19,80 @@ # add config path mconfig["paths"].update({"config": config_dir_path}) -#load colorscheme -color_schemes=M_color.color(path=mconfig['paths']['config'], name='color_def') +# load colorscheme +color_schemes = M_color.color(path=mconfig["paths"]["config"], name="color_def") -lstrings =iter([i+') ' for i in list(string.ascii_lowercase)]) +lstrings = iter([i + ") " for i in list(string.ascii_lowercase)]) # define journal fig sizes -fig_sizes = mconfig['fig_sizes']['AMS'] +fig_sizes = mconfig["fig_sizes"]["AMS"] SMALL_SIZE = 8 MEDIUM_SIZE = 10 BIGGER_SIZE = 12 -legend_properties = {'weight':'bold'} +legend_properties = {"weight": "bold"} -plt.rc('font', size=SMALL_SIZE, serif='Helvetica Neue', weight='normal') # controls default text sizes -plt.rc('text', usetex='false') -plt.rc('axes', titlesize=MEDIUM_SIZE, labelweight='normal') # fontsize of the axes title -plt.rc('axes', labelsize=SMALL_SIZE, labelweight='normal') # fontsize of the x and y labels -plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels -plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels -plt.rc('legend', fontsize=SMALL_SIZE, frameon=False) # legend fontsize -plt.rc('figure', titlesize=MEDIUM_SIZE, titleweight='bold', autolayout=True) # fontsize of the figure title -plt.rc('path', simplify=True) -plt.rcParams['figure.figsize'] = (10, 8) -plt.rcParams['pcolor.shading'] = 'auto' -plt.rc('pcolor', shading = 'auto') -plt.rc('xtick.major', size= 4, width=1 ) -plt.rc('ytick.major', size= 3.8, width=1 ) -plt.rc('axes', labelsize= MEDIUM_SIZE, labelweight='normal') -plt.rc('axes.spines', top= False, right=False ) +plt.rc( + "font", size=SMALL_SIZE, serif="Helvetica Neue", weight="normal" +) # controls default text sizes +plt.rc("text", usetex="false") +plt.rc( + "axes", titlesize=MEDIUM_SIZE, labelweight="normal" +) # fontsize of the axes title +plt.rc( + "axes", labelsize=SMALL_SIZE, labelweight="normal" +) # fontsize of the x and y labels +plt.rc("xtick", labelsize=SMALL_SIZE) # fontsize of the tick labels +plt.rc("ytick", labelsize=SMALL_SIZE) # fontsize of the tick labels +plt.rc("legend", fontsize=SMALL_SIZE, frameon=False) # legend fontsize +plt.rc( + "figure", titlesize=MEDIUM_SIZE, titleweight="bold", autolayout=True +) # fontsize of the figure title +plt.rc("path", simplify=True) +plt.rcParams["figure.figsize"] = (10, 8) +plt.rcParams["pcolor.shading"] = "auto" +plt.rc("pcolor", shading="auto") +plt.rc("xtick.major", size=4, width=1) +plt.rc("ytick.major", size=3.8, width=1) +plt.rc("axes", labelsize=MEDIUM_SIZE, labelweight="normal") +plt.rc("axes.spines", top=False, right=False) -def font_for_print(SMALL_SIZE = 6, MEDIUM_SIZE = 8): - plt.rc('font', size=SMALL_SIZE, serif='Helvetica Neue', weight='normal') # controls default text sizes - plt.rc('text', usetex='false') - plt.rc('axes', titlesize=MEDIUM_SIZE, labelweight='normal') # fontsize of the axes title - plt.rc('axes', labelsize=SMALL_SIZE, labelweight='normal') #, family='bold') # fontsize of the x and y labels - plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels - plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels - plt.rc('legend', fontsize=SMALL_SIZE, frameon=False) # legend fontsize - plt.rc('figure', titlesize=MEDIUM_SIZE, titleweight='bold', autolayout=True) #, family='bold') # fontsize of the figure title - plt.rc('axes', labelsize= SMALL_SIZE, labelweight='normal') -def font_for_pres(SMALL_SIZE = 10, MEDIUM_SIZE = 12): - plt.rc('font', size=SMALL_SIZE, serif='Helvetica Neue', weight='normal') # controls default text sizes - plt.rc('text', usetex='false') - plt.rc('axes', titlesize=MEDIUM_SIZE, labelweight='normal') # fontsize of the axes title - plt.rc('axes', labelsize=SMALL_SIZE, labelweight='normal') #, family='bold') # fontsize of the x and y labels - plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels - plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels - plt.rc('legend', fontsize=SMALL_SIZE, frameon=False) # legend fontsize - plt.rc('figure', titlesize=MEDIUM_SIZE, titleweight='bold', autolayout=True) # fontsize of the figure title - plt.rc('axes', labelsize= SMALL_SIZE, labelweight='normal') +def font_for_print(SMALL_SIZE=6, MEDIUM_SIZE=8): + plt.rc( + "font", size=SMALL_SIZE, serif="Helvetica Neue", weight="normal" + ) # controls default text sizes + plt.rc("text", usetex="false") + plt.rc( + "axes", titlesize=MEDIUM_SIZE, labelweight="normal" + ) # fontsize of the axes title + plt.rc( + "axes", labelsize=SMALL_SIZE, labelweight="normal" + ) # , family='bold') # fontsize of the x and y labels + plt.rc("xtick", labelsize=SMALL_SIZE) # fontsize of the tick labels + plt.rc("ytick", labelsize=SMALL_SIZE) # fontsize of the tick labels + plt.rc("legend", fontsize=SMALL_SIZE, frameon=False) # legend fontsize + plt.rc( + "figure", titlesize=MEDIUM_SIZE, titleweight="bold", autolayout=True + ) # , family='bold') # fontsize of the figure title + plt.rc("axes", labelsize=SMALL_SIZE, labelweight="normal") + + +def font_for_pres(SMALL_SIZE=10, MEDIUM_SIZE=12): + plt.rc( + "font", size=SMALL_SIZE, serif="Helvetica Neue", weight="normal" + ) # controls default text sizes + plt.rc("text", usetex="false") + plt.rc( + "axes", titlesize=MEDIUM_SIZE, labelweight="normal" + ) # fontsize of the axes title + plt.rc( + "axes", labelsize=SMALL_SIZE, labelweight="normal" + ) # , family='bold') # fontsize of the x and y labels + plt.rc("xtick", labelsize=SMALL_SIZE) # fontsize of the tick labels + plt.rc("ytick", labelsize=SMALL_SIZE) # fontsize of the tick labels + plt.rc("legend", fontsize=SMALL_SIZE, frameon=False) # legend fontsize + plt.rc( + "figure", titlesize=MEDIUM_SIZE, titleweight="bold", autolayout=True + ) # fontsize of the figure title + plt.rc("axes", labelsize=SMALL_SIZE, labelweight="normal") diff --git a/src/icesat2_tracks/config/__init__.py b/src/icesat2_tracks/config/__init__.py index c318732e..64d3f348 100644 --- a/src/icesat2_tracks/config/__init__.py +++ b/src/icesat2_tracks/config/__init__.py @@ -1 +1 @@ -"""Configuration file containing project paths""" \ No newline at end of file +"""Configuration file containing project paths""" diff --git a/src/icesat2_tracks/config/generate_colors.py b/src/icesat2_tracks/config/generate_colors.py index b6a792b6..870c8a93 100644 --- a/src/icesat2_tracks/config/generate_colors.py +++ b/src/icesat2_tracks/config/generate_colors.py @@ -1,59 +1,59 @@ -#%matplotlib inline +# %matplotlib inline from icesat2_tracks.local_modules import m_colormanager_ph3 as M_color -mconfig['paths'] -path=mconfig['paths']['config'] -A = M_color.ase_to_json(path+'color_def.ase') +mconfig["paths"] +path = mconfig["paths"]["config"] +A = M_color.ase_to_json(path + "color_def.ase") -B=dict() -for i in A[0]['swatches']: - B[i['name']] = i['data']['values'] - print(i['name'] + ' ' + str(i['data']['values'])) +B = dict() +for i in A[0]["swatches"]: + B[i["name"]] = i["data"]["values"] + print(i["name"] + " " + str(i["data"]["values"])) -rels=dict() +rels = dict() -rels['plus']=B['red'] -rels['minus']=B['blue'] +rels["plus"] = B["red"] +rels["minus"] = B["blue"] -rels['blue']=B['blue'] -rels['lightblue']=B['cascade3'] -rels['darkblue']=B['cascade1'] +rels["blue"] = B["blue"] +rels["lightblue"] = B["cascade3"] +rels["darkblue"] = B["cascade1"] -rels['white']=B['white'] -rels['gridcolor']=B['gridcolor'] -rels['grey']=B['gray'] +rels["white"] = B["white"] +rels["gridcolor"] = B["gridcolor"] +rels["grey"] = B["gray"] -rels['orange']=B['orange'] -rels['red']=B['red'] -rels['green']=B['green'] +rels["orange"] = B["orange"] +rels["red"] = B["red"] +rels["green"] = B["green"] -rels['cascade1']=B['cascade1'] -rels['cascade2']=B['cascade2'] -rels['cascade3']=B['cascade3'] -rels['cascade4']=B['gridcolor'] +rels["cascade1"] = B["cascade1"] +rels["cascade2"] = B["cascade2"] +rels["cascade3"] = B["cascade3"] +rels["cascade4"] = B["gridcolor"] -rels['rascade1']=B['rascade2'] -rels['rascade2']=B['rascade1'] -rels['rascade3']=B['rascade3'] +rels["rascade1"] = B["rascade2"] +rels["rascade2"] = B["rascade1"] +rels["rascade3"] = B["rascade3"] -rels['aug1']=B['orange'] -rels['aug2']=B['green'] +rels["aug1"] = B["orange"] +rels["aug2"] = B["green"] -rels['gt1l']=B['rascade1'] -rels['gt1r']=B['rascade3'] +rels["gt1l"] = B["rascade1"] +rels["gt1r"] = B["rascade3"] -rels['gt2l']=B['green1'] -rels['gt2r']=B['green2'] +rels["gt2l"] = B["green1"] +rels["gt2r"] = B["green2"] -rels['gt3l']=B['cascade1'] -rels['gt3r']=B['blue'] +rels["gt3l"] = B["cascade1"] +rels["gt3r"] = B["blue"] -rels['group1']=B['rascade1'] -rels['group2']=B['green1'] -rels['group3']=B['cascade1'] +rels["group1"] = B["rascade1"] +rels["group2"] = B["green1"] +rels["group3"] = B["cascade1"] -B['rels']=rels +B["rels"] = rels -M_color.json_save('color_def', path, B) +M_color.json_save("color_def", path, B) diff --git a/src/icesat2_tracks/icesat2_tools_scripts/nsidc_icesat2_associated2.py b/src/icesat2_tracks/icesat2_tools_scripts/nsidc_icesat2_associated2.py index 73a9f603..0854178f 100644 --- a/src/icesat2_tracks/icesat2_tools_scripts/nsidc_icesat2_associated2.py +++ b/src/icesat2_tracks/icesat2_tools_scripts/nsidc_icesat2_associated2.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -u""" +""" nsidc_icesat2_associated.py Written by Tyler Sutterley (07/2021) @@ -87,257 +87,322 @@ import multiprocessing as mp import icesat2_toolkit.utilities -#-- PURPOSE: download the ICESat-2 elevation data from NSIDC matching an file -def nsidc_icesat2_associated(file_list, PRODUCT, DIRECTORY=None, - AUXILIARY=False, FLATTEN=False, PROCESSES=0, TIMEOUT=None, - RETRY=1, MODE=0o775): - #-- compile HTML parser for lxml +# -- PURPOSE: download the ICESat-2 elevation data from NSIDC matching an file +def nsidc_icesat2_associated( + file_list, + PRODUCT, + DIRECTORY=None, + AUXILIARY=False, + FLATTEN=False, + PROCESSES=0, + TIMEOUT=None, + RETRY=1, + MODE=0o775, +): + + # -- compile HTML parser for lxml parser = lxml.etree.HTMLParser() - #-- remote https server for ICESat-2 Data - HOST = 'https://n5eil01u.ecs.nsidc.org' - #-- regular expression operator for extracting information from files - rx = re.compile(r'(processed_)?(ATL\d{2})-(\d{2})?_(\d{4})(\d{2})(\d{2})' - r'(\d{2})(\d{2})(\d{2})_(\d{4})(\d{2})(\d{2})_(\d{3})_(\d{2})(.*?).h5$') - #-- regular expression pattern for finding specific files - regex_suffix = '(.*?)$' if AUXILIARY else '(h5)$' - remote_regex_pattern = (r'{0}-({1})_(\d{{4}})(\d{{2}})(\d{{2}})' - r'(\d{{2}})(\d{{2}})(\d{{2}})_({2})({3})({4})_({5})_(\d{{2}})(.*?).{6}') + # -- remote https server for ICESat-2 Data + HOST = "https://n5eil01u.ecs.nsidc.org" + # -- regular expression operator for extracting information from files + rx = re.compile( + r"(processed_)?(ATL\d{2})-(\d{2})?_(\d{4})(\d{2})(\d{2})" + r"(\d{2})(\d{2})(\d{2})_(\d{4})(\d{2})(\d{2})_(\d{3})_(\d{2})(.*?).h5$" + ) + # -- regular expression pattern for finding specific files + regex_suffix = "(.*?)$" if AUXILIARY else "(h5)$" + remote_regex_pattern = ( + r"{0}-({1})_(\d{{4}})(\d{{2}})(\d{{2}})" + r"(\d{{2}})(\d{{2}})(\d{{2}})_({2})({3})({4})_({5})_(\d{{2}})(.*?).{6}" + ) - #-- build list of remote files, remote modification times and local files + # -- build list of remote files, remote modification times and local files original_files = [] remote_files = [] remote_mtimes = [] local_files = [] - #-- for each input file + # -- for each input file for input_file in file_list: - #-- extract parameters from ICESat-2 ATLAS HDF5 file name - SUB,PRD,HEM,YY,MM,DD,HH,MN,SS,TRK,CYC,GRN,RL,VRS,AUX = \ - rx.findall(input_file).pop() - #-- get directories from remote directory - product_directory = '{0}.{1}'.format(PRODUCT,RL) - sd = '{0}.{1}.{2}'.format(YY,MM,DD) - PATH = [HOST,'ATLAS',product_directory,sd] - #-- local and remote data directories - remote_dir=posixpath.join(*PATH) - temp=os.path.dirname(input_file) if (DIRECTORY is None) else DIRECTORY - local_dir=os.path.expanduser(temp) if FLATTEN else os.path.join(temp,sd) - #-- create output directory if not currently existing + # -- extract parameters from ICESat-2 ATLAS HDF5 file name + SUB, PRD, HEM, YY, MM, DD, HH, MN, SS, TRK, CYC, GRN, RL, VRS, AUX = rx.findall( + input_file + ).pop() + # -- get directories from remote directory + product_directory = "{0}.{1}".format(PRODUCT, RL) + sd = "{0}.{1}.{2}".format(YY, MM, DD) + PATH = [HOST, "ATLAS", product_directory, sd] + # -- local and remote data directories + remote_dir = posixpath.join(*PATH) + temp = os.path.dirname(input_file) if (DIRECTORY is None) else DIRECTORY + local_dir = os.path.expanduser(temp) if FLATTEN else os.path.join(temp, sd) + # -- create output directory if not currently existing if not os.access(local_dir, os.F_OK): os.makedirs(local_dir, MODE) - #-- compile regular expression operator for file parameters - args = (PRODUCT,HEM,TRK,CYC,GRN,RL,regex_suffix) + # -- compile regular expression operator for file parameters + args = (PRODUCT, HEM, TRK, CYC, GRN, RL, regex_suffix) R1 = re.compile(remote_regex_pattern.format(*args), re.VERBOSE) - #-- find associated ICESat-2 data file - #-- find matching files (for granule, release, version, track) - colnames,collastmod,colerror=icesat2_toolkit.utilities.nsidc_list(PATH, - build=False, - timeout=TIMEOUT, - parser=parser, - pattern=R1, - sort=True) - #-- print if file was not found + # -- find associated ICESat-2 data file + # -- find matching files (for granule, release, version, track) + colnames, collastmod, colerror = icesat2_toolkit.utilities.nsidc_list( + PATH, build=False, timeout=TIMEOUT, parser=parser, pattern=R1, sort=True + ) + # -- print if file was not found if not colnames: print(colerror) continue - #-- add to lists - for colname,remote_mtime in zip(colnames,collastmod): - #-- save original file to list (expands if getting auxiliary files) + # -- add to lists + for colname, remote_mtime in zip(colnames, collastmod): + # -- save original file to list (expands if getting auxiliary files) original_files.append(input_file) - #-- remote and local versions of the file - remote_files.append(posixpath.join(remote_dir,colname)) - local_files.append(os.path.join(local_dir,colname)) + # -- remote and local versions of the file + remote_files.append(posixpath.join(remote_dir, colname)) + local_files.append(os.path.join(local_dir, colname)) remote_mtimes.append(remote_mtime) - #-- download in series if PROCESSES = 0 - if (PROCESSES == 0): - #-- download each associated ICESat-2 data file - for i,input_file in enumerate(original_files): - #-- download associated ICESat-2 files with NSIDC server - args = (remote_files[i],remote_mtimes[i],local_files[i]) - kwds = dict(TIMEOUT=TIMEOUT,RETRY=RETRY,MODE=MODE) - out = http_pull_file(*args,**kwds) - #-- print the output string - print('{0}\n{1}'.format(input_file,out)) + # -- download in series if PROCESSES = 0 + if PROCESSES == 0: + # -- download each associated ICESat-2 data file + for i, input_file in enumerate(original_files): + # -- download associated ICESat-2 files with NSIDC server + args = (remote_files[i], remote_mtimes[i], local_files[i]) + kwds = dict(TIMEOUT=TIMEOUT, RETRY=RETRY, MODE=MODE) + out = http_pull_file(*args, **kwds) + # -- print the output string + print("{0}\n{1}".format(input_file, out)) else: - #-- set multiprocessing start method + # -- set multiprocessing start method ctx = mp.get_context("fork") - #-- download in parallel with multiprocessing Pool + # -- download in parallel with multiprocessing Pool pool = ctx.Pool(processes=PROCESSES) - #-- download each associated ICESat-2 data file + # -- download each associated ICESat-2 data file output = [] - for i,input_file in enumerate(original_files): - #-- download associated ICESat-2 files with NSIDC server - args = (remote_files[i],remote_mtimes[i],local_files[i]) - kwds = dict(TIMEOUT=TIMEOUT,RETRY=RETRY,MODE=MODE) - out=pool.apply_async(multiprocess_sync,args=args,kwds=kwds) - output.append('{0}\n{1}'.format(input_file,out)) - #-- start multiprocessing jobs - #-- close the pool - #-- prevents more tasks from being submitted to the pool + for i, input_file in enumerate(original_files): + # -- download associated ICESat-2 files with NSIDC server + args = (remote_files[i], remote_mtimes[i], local_files[i]) + kwds = dict(TIMEOUT=TIMEOUT, RETRY=RETRY, MODE=MODE) + out = pool.apply_async(multiprocess_sync, args=args, kwds=kwds) + output.append("{0}\n{1}".format(input_file, out)) + # -- start multiprocessing jobs + # -- close the pool + # -- prevents more tasks from being submitted to the pool pool.close() - #-- exit the completed processes + # -- exit the completed processes pool.join() - #-- print the output string + # -- print the output string for out in output: print(out.get()) -#-- PURPOSE: wrapper for running the sync program in multiprocessing mode + +# -- PURPOSE: wrapper for running the sync program in multiprocessing mode def multiprocess_sync(*args, **kwds): try: output = http_pull_file(*args, **kwds) except: - #-- if there has been an error exception - #-- print the type, value, and stack trace of the - #-- current exception being handled - print('process id {0:d} failed'.format(os.getpid())) + # -- if there has been an error exception + # -- print the type, value, and stack trace of the + # -- current exception being handled + print("process id {0:d} failed".format(os.getpid())) traceback.print_exc() else: return output -#-- PURPOSE: pull file from a remote host checking if file exists locally -#-- and if the remote file is newer than the local file -def http_pull_file(remote_file,remote_mtime,local_file, - TIMEOUT=None, RETRY=1, MODE=0o775): - #-- Printing files transferred - output = '{0} -->\n\t{1}\n'.format(remote_file,local_file) - #-- chunked transfer encoding size + +# -- PURPOSE: pull file from a remote host checking if file exists locally +# -- and if the remote file is newer than the local file +def http_pull_file( + remote_file, remote_mtime, local_file, TIMEOUT=None, RETRY=1, MODE=0o775 +): + # -- Printing files transferred + output = "{0} -->\n\t{1}\n".format(remote_file, local_file) + # -- chunked transfer encoding size CHUNK = 16 * 1024 - #-- attempt to download a file up to a set number of times - retry_download(remote_file, LOCAL=local_file, TIMEOUT=TIMEOUT, - RETRY=RETRY, CHUNK=CHUNK) - #-- keep remote modification time of file and local access time + # -- attempt to download a file up to a set number of times + retry_download( + remote_file, LOCAL=local_file, TIMEOUT=TIMEOUT, RETRY=RETRY, CHUNK=CHUNK + ) + # -- keep remote modification time of file and local access time os.utime(local_file, (os.stat(local_file).st_atime, remote_mtime)) os.chmod(local_file, MODE) - #-- return the output string + # -- return the output string return output -#-- PURPOSE: Try downloading a file up to a set number of times + +# -- PURPOSE: Try downloading a file up to a set number of times def retry_download(remote_file, LOCAL=None, TIMEOUT=None, RETRY=1, CHUNK=0): - #-- attempt to download up to the number of retries + # -- attempt to download up to the number of retries retry_counter = 0 - while (retry_counter < RETRY): - #-- attempt to retrieve file from https server + while retry_counter < RETRY: + # -- attempt to retrieve file from https server try: - #-- Create and submit request. - #-- There are a range of exceptions that can be thrown here - #-- including HTTPError and URLError. - request=icesat2_toolkit.utilities.urllib2.Request(remote_file) - response=icesat2_toolkit.utilities.urllib2.urlopen(request, - timeout=TIMEOUT) - #-- get the length of the remote file - remote_length = int(response.headers['content-length']) - #-- copy contents to file using chunked transfer encoding - #-- transfer should work with ascii and binary data formats - with open(LOCAL, 'wb') as f: + # -- Create and submit request. + # -- There are a range of exceptions that can be thrown here + # -- including HTTPError and URLError. + request = icesat2_toolkit.utilities.urllib2.Request(remote_file) + response = icesat2_toolkit.utilities.urllib2.urlopen( + request, timeout=TIMEOUT + ) + # -- get the length of the remote file + remote_length = int(response.headers["content-length"]) + # -- copy contents to file using chunked transfer encoding + # -- transfer should work with ascii and binary data formats + with open(LOCAL, "wb") as f: shutil.copyfileobj(response, f, CHUNK) local_length = os.path.getsize(LOCAL) except: pass else: - #-- check that downloaded file matches original length - if (local_length == remote_length): + # -- check that downloaded file matches original length + if local_length == remote_length: break - #-- add to retry counter + # -- add to retry counter retry_counter += 1 - #-- check if maximum number of retries were reached - if (retry_counter == RETRY): - raise TimeoutError('Maximum number of retries reached') + # -- check if maximum number of retries were reached + if retry_counter == RETRY: + raise TimeoutError("Maximum number of retries reached") + -#-- Main program that calls nsidc_icesat2_associated() +# -- Main program that calls nsidc_icesat2_associated() def main(): - #-- Read the system arguments listed after the program + # -- Read the system arguments listed after the program parser = argparse.ArgumentParser( description="""Program to acquire ICESat-2 datafiles from the NSIDC server that is associated with an input file """ ) - #-- ICESat-2 Products + # -- ICESat-2 Products PRODUCTS = {} - PRODUCTS['ATL03'] = 'Global Geolocated Photon Data' - PRODUCTS['ATL04'] = 'Normalized Relative Backscatter' - PRODUCTS['ATL06'] = 'Land Ice Height' - PRODUCTS['ATL07'] = 'Sea Ice Height' - PRODUCTS['ATL08'] = 'Land and Vegetation Height' - PRODUCTS['ATL09'] = 'Atmospheric Layer Characteristics' - PRODUCTS['ATL10'] = 'Sea Ice Freeboard' - PRODUCTS['ATL12'] = 'Ocean Surface Height' - PRODUCTS['ATL13'] = 'Inland Water Surface Height' - #-- command line parameters - parser.add_argument('file', - type=lambda p: os.path.abspath(os.path.expanduser(p)), nargs='+', - help='ICESat-2 products to associate') - #-- NASA Earthdata credentials - parser.add_argument('--user','-U', - type=str, default=os.environ.get('EARTHDATA_USERNAME'), - help='Username for NASA Earthdata Login') - parser.add_argument('--netrc','-N', + PRODUCTS["ATL03"] = "Global Geolocated Photon Data" + PRODUCTS["ATL04"] = "Normalized Relative Backscatter" + PRODUCTS["ATL06"] = "Land Ice Height" + PRODUCTS["ATL07"] = "Sea Ice Height" + PRODUCTS["ATL08"] = "Land and Vegetation Height" + PRODUCTS["ATL09"] = "Atmospheric Layer Characteristics" + PRODUCTS["ATL10"] = "Sea Ice Freeboard" + PRODUCTS["ATL12"] = "Ocean Surface Height" + PRODUCTS["ATL13"] = "Inland Water Surface Height" + # -- command line parameters + parser.add_argument( + "file", + type=lambda p: os.path.abspath(os.path.expanduser(p)), + nargs="+", + help="ICESat-2 products to associate", + ) + # -- NASA Earthdata credentials + parser.add_argument( + "--user", + "-U", + type=str, + default=os.environ.get("EARTHDATA_USERNAME"), + help="Username for NASA Earthdata Login", + ) + parser.add_argument( + "--netrc", + "-N", type=lambda p: os.path.abspath(os.path.expanduser(p)), - default=os.path.join(os.path.expanduser('~'),'.netrc'), - help='Path to .netrc file for authentication') - #-- working data directory - parser.add_argument('--directory','-D', + default=os.path.join(os.path.expanduser("~"), ".netrc"), + help="Path to .netrc file for authentication", + ) + # -- working data directory + parser.add_argument( + "--directory", + "-D", type=lambda p: os.path.abspath(os.path.expanduser(p)), default=os.getcwd(), - help='Working data directory') - #-- ICESat-2 parameters - #-- ICESat-2 data product - parser.add_argument('--product','-p', - metavar='PRODUCTS', type=str, - choices=PRODUCTS.keys(), default='ATL06', - help='Associated ICESat-2 data product to download') - #-- download auxiliary files - parser.add_argument('--auxiliary','-a', - default=False, action='store_true', - help='Sync ICESat-2 auxiliary files for each HDF5 file') - #-- output subdirectories - parser.add_argument('--flatten','-F', - default=False, action='store_true', - help='Do not create subdirectories') - #-- run download in series if processes is 0 - parser.add_argument('--np','-P', - metavar='PROCESSES', type=int, default=0, - help='Number of processes to use in downloading files') - #-- connection timeout and number of retry attempts - parser.add_argument('--timeout','-T', - type=int, default=120, - help='Timeout in seconds for blocking operations') - parser.add_argument('--retry','-R', - type=int, default=5, - help='Connection retry attempts') - #-- permissions mode of the local directories and files (number in octal) - parser.add_argument('--mode','-M', - type=lambda x: int(x,base=8), default=0o775, - help='permissions mode of output files') + help="Working data directory", + ) + # -- ICESat-2 parameters + # -- ICESat-2 data product + parser.add_argument( + "--product", + "-p", + metavar="PRODUCTS", + type=str, + choices=PRODUCTS.keys(), + default="ATL06", + help="Associated ICESat-2 data product to download", + ) + # -- download auxiliary files + parser.add_argument( + "--auxiliary", + "-a", + default=False, + action="store_true", + help="Sync ICESat-2 auxiliary files for each HDF5 file", + ) + # -- output subdirectories + parser.add_argument( + "--flatten", + "-F", + default=False, + action="store_true", + help="Do not create subdirectories", + ) + # -- run download in series if processes is 0 + parser.add_argument( + "--np", + "-P", + metavar="PROCESSES", + type=int, + default=0, + help="Number of processes to use in downloading files", + ) + # -- connection timeout and number of retry attempts + parser.add_argument( + "--timeout", + "-T", + type=int, + default=120, + help="Timeout in seconds for blocking operations", + ) + parser.add_argument( + "--retry", "-R", type=int, default=5, help="Connection retry attempts" + ) + # -- permissions mode of the local directories and files (number in octal) + parser.add_argument( + "--mode", + "-M", + type=lambda x: int(x, base=8), + default=0o775, + help="permissions mode of output files", + ) args = parser.parse_args() - #-- NASA Earthdata hostname - HOST = 'urs.earthdata.nasa.gov' - #-- get authentication + # -- NASA Earthdata hostname + HOST = "urs.earthdata.nasa.gov" + # -- get authentication try: - args.user,_,PASSWORD = netrc.netrc(args.netrc).authenticators(HOST) + args.user, _, PASSWORD = netrc.netrc(args.netrc).authenticators(HOST) except: - #-- check that NASA Earthdata credentials were entered + # -- check that NASA Earthdata credentials were entered if not args.user: - prompt = 'Username for {0}: '.format(HOST) + prompt = "Username for {0}: ".format(HOST) args.user = builtins.input(prompt) - #-- enter password securely from command-line - prompt = 'Password for {0}@{1}: '.format(args.user,HOST) + # -- enter password securely from command-line + prompt = "Password for {0}@{1}: ".format(args.user, HOST) PASSWORD = getpass.getpass(prompt) - #-- build a urllib opener for NSIDC - #-- Add the username and password for NASA Earthdata Login system - icesat2_toolkit.utilities.build_opener(args.user,PASSWORD) + # -- build a urllib opener for NSIDC + # -- Add the username and password for NASA Earthdata Login system + icesat2_toolkit.utilities.build_opener(args.user, PASSWORD) - #-- check internet connection before attempting to run program - #-- check NASA earthdata credentials before attempting to run program + # -- check internet connection before attempting to run program + # -- check NASA earthdata credentials before attempting to run program if icesat2_toolkit.utilities.check_credentials(): - nsidc_icesat2_associated(args.file, args.product, - DIRECTORY=args.directory, AUXILIARY=args.auxiliary, - FLATTEN=args.flatten, PROCESSES=args.np, TIMEOUT=args.timeout, - RETRY=args.retry, MODE=args.mode) + nsidc_icesat2_associated( + args.file, + args.product, + DIRECTORY=args.directory, + AUXILIARY=args.auxiliary, + FLATTEN=args.flatten, + PROCESSES=args.np, + TIMEOUT=args.timeout, + RETRY=args.retry, + MODE=args.mode, + ) + -#-- run main program -if __name__ == '__main__': - main() \ No newline at end of file +# -- run main program +if __name__ == "__main__": + main() diff --git a/src/icesat2_tracks/local_modules/JONSWAP_gamma.py b/src/icesat2_tracks/local_modules/JONSWAP_gamma.py index 8b83cacc..6c2e4e00 100644 --- a/src/icesat2_tracks/local_modules/JONSWAP_gamma.py +++ b/src/icesat2_tracks/local_modules/JONSWAP_gamma.py @@ -1,51 +1,56 @@ import os -if __name__ == '__main__': - exec(open(os.environ['PYTHONSTARTUP']).read()) +if __name__ == "__main__": + + exec(open(os.environ["PYTHONSTARTUP"]).read()) exec(open(STARTUP_2019_DP).read()) - #%matplotlib inline + # %matplotlib inline import sys, imp - #import pickle + + # import pickle from lmfit import minimize, Parameters import copy import matplotlib.pyplot as plt import numpy as np + # %% def normalize_time(time): """ returns a time vector from 0 to 1 of the length of time and the time step dt in dimentional units """ - time=np.copy(time) - dt=np.diff(time).mean()#G.dt_periodogram - time=(time-time[0])/dt#np.arange(0,time.size, 1) - return (time)/(time[-1]) , dt + time = np.copy(time) + dt = np.diff(time).mean() # G.dt_periodogram + time = (time - time[0]) / dt # np.arange(0,time.size, 1) + return (time) / (time[-1]), dt + -def gaussian(x,x0,sigma): - return np.exp(-np.power((x - x0)/sigma, 2.)/2.) +def gaussian(x, x0, sigma): + return np.exp(-np.power((x - x0) / sigma, 2.0) / 2.0) -#%% generate some data_ano -if __name__ == '__main__': + +# %% generate some data_ano +if __name__ == "__main__": from scipy.stats import gamma - f=np.arange(1/1000.0, .2, 0.001) - time=np.arange(0, 1, 0.001) + f = np.arange(1 / 1000.0, 0.2, 0.001) + time = np.arange(0, 1, 0.001) tt, ff = np.meshgrid(time, f) - fake_data= np.sin(tt*np.pi)**4 * gaussian( ff, .05, 0.03) + fake_data = np.sin(tt * np.pi) ** 4 * gaussian(ff, 0.05, 0.03) - plt.contour(tt, ff, fake_data, colors='k') + plt.contour(tt, ff, fake_data, colors="k") # %% basic functions -def gamma_time_normlized_amp(time, gammapar=2, loc=.2, scale=0.1): +def gamma_time_normlized_amp(time, gammapar=2, loc=0.2, scale=0.1): from scipy.stats import gamma - np.seterr(divide='ignore', invalid='ignore') + np.seterr(divide="ignore", invalid="ignore") """ configured for normalized time scale (0, 1) gammapar > 1 @@ -53,13 +58,14 @@ def gamma_time_normlized_amp(time, gammapar=2, loc=.2, scale=0.1): loc is position 0 ..1 scale >0 .. \approx 0.3 """ - gamma_mod=gamma(gammapar, loc = loc, scale = scale).pdf(time) - return gamma_mod/gamma_mod.max() + gamma_mod = gamma(gammapar, loc=loc, scale=scale).pdf(time) + return gamma_mod / gamma_mod.max() + -def gamma_time_normlized_amp_shifted(time, gammapar=2, loc=.2, scale=0.1): +def gamma_time_normlized_amp_shifted(time, gammapar=2, loc=0.2, scale=0.1): from scipy.stats import gamma - np.seterr(divide='ignore', invalid='ignore') + np.seterr(divide="ignore", invalid="ignore") """ configured for normalized time scale (0, 1) @@ -69,40 +75,45 @@ def gamma_time_normlized_amp_shifted(time, gammapar=2, loc=.2, scale=0.1): loc is position 0 ..1 scale >0 .. \approx 0.3 """ - gamma_mod = gamma(gammapar, loc = loc[0,:], scale = scale).pdf(time[0,:]) - dloc = (loc[0,:]- time[0,:][gamma_mod.argmax()]) - #print(dloc) - gamma_mod = gamma(gammapar, loc =loc + dloc , scale = scale).pdf(time) - return gamma_mod/gamma_mod.max() + gamma_mod = gamma(gammapar, loc=loc[0, :], scale=scale).pdf(time[0, :]) + dloc = loc[0, :] - time[0, :][gamma_mod.argmax()] + # print(dloc) + gamma_mod = gamma(gammapar, loc=loc + dloc, scale=scale).pdf(time) + return gamma_mod / gamma_mod.max() -if __name__ == '__main__': + +if __name__ == "__main__": plt.plot(time, gamma_time_normlized_amp(time)) - plt.plot(time, gamma_time_normlized_amp_shifted(time, gammapar=3, loc=.2, scale=0.1) ) + plt.plot( + time, gamma_time_normlized_amp_shifted(time, gammapar=3, loc=0.2, scale=0.1) + ) # %% JONSWAP -def JONSWAP_bulk(f, floc=0.04, famp=1e-2, gamma=3.3, peak_std=1e-1): + +def JONSWAP_bulk(f, floc=0.04, famp=1e-2, gamma=3.3, peak_std=1e-1): """ see Ocean Surface waves - S. R. Massel eq.3.69 and eq.3.81 """ - B=0.74 - g=9.81 - w=f*2*np.pi - wp=floc*2*np.pi - stretch=5/4 + B = 0.74 + g = 9.81 + w = f * 2 * np.pi + wp = floc * 2 * np.pi + stretch = 5 / 4 - #peak_std=1e-01 #0.00555 - #scale_std=1e-2 - alpha=famp - #stretch=5/4 - - delta=np.exp(-(w-wp)**2 / ( 2*peak_std**2 *wp**2 ) ) - peak_factor=gamma**delta - return alpha * w**(-5) * np.exp(-stretch* (w/wp)**-4)*peak_factor # units of m^2 / Hz - #np.exp(-B *(g/(w*U))**4) + # peak_std=1e-01 #0.00555 + # scale_std=1e-2 + alpha = famp + # stretch=5/4 + delta = np.exp(-((w - wp) ** 2) / (2 * peak_std**2 * wp**2)) + peak_factor = gamma**delta + return ( + alpha * w ** (-5) * np.exp(-stretch * (w / wp) ** -4) * peak_factor + ) # units of m^2 / Hz + # np.exp(-B *(g/(w*U))**4) def pierson_moskowitz_default(f, U): @@ -111,40 +122,45 @@ def pierson_moskowitz_default(f, U): """ - g=9.81 - wp=0.879*g / U - w=2.*np.pi*f - sigma=0.04 *g / wp**2.0 - alpha=5.0* (wp**2.0 *sigma / g)**2.0 + g = 9.81 + wp = 0.879 * g / U + w = 2.0 * np.pi * f + sigma = 0.04 * g / wp**2.0 + alpha = 5.0 * (wp**2.0 * sigma / g) ** 2.0 + + return alpha * w ** (-5.0) * g**2.0 * np.exp(-5.0 / 4.0 * (w / wp) ** -4) # - return alpha * w**(-5.0) * g**2.0 *np.exp(-5./4.0 * (w/wp)**-4)# -def pierson_moskowitz_fetch_limit(f, X, U): +def pierson_moskowitz_fetch_limit(f, X, U): """ see Ocean Surface waves - S. R. Massel eq.3.81 - eq.3.84 """ - g=9.81 # m s*-2 - w=2.*np.pi*f # rad/sec + g = 9.81 # m s*-2 + w = 2.0 * np.pi * f # rad/sec - alpha=0.076*(g *X /U**2)**(-0.22) #non-dimentional - wp= 7.* np.pi * (g /U) * (g * X /U**2)**(-0.33) - print('wp='+str(wp)) + alpha = 0.076 * (g * X / U**2) ** (-0.22) # non-dimentional + wp = 7.0 * np.pi * (g / U) * (g * X / U**2) ** (-0.33) + print("wp=" + str(wp)) - sigma_p=0.07 - sigma_pp=0.09 - sigma= np.array([[sigma_p if i else sigma_pp][0] for i in list(w<=wp) ]) + sigma_p = 0.07 + sigma_pp = 0.09 + sigma = np.array([[sigma_p if i else sigma_pp][0] for i in list(w <= wp)]) - gamma=3.3 - delta=np.exp(-(w-wp)**2 / ( 2.*sigma**2. *wp**2. ) ) - peak_factor=gamma**delta + gamma = 3.3 + delta = np.exp(-((w - wp) ** 2) / (2.0 * sigma**2.0 * wp**2.0)) + peak_factor = gamma**delta + + return ( + alpha * g**2.0 * w ** (-5.0) * np.exp(-5.0 / 4.0 * (w / wp) ** -4) + ) # Hz**-5 m**2 /s**4 = m**2 sec - return alpha * g**2.0 * w**(-5.) * np.exp(-5./4.0 * (w/wp)**-4)# Hz**-5 m**2 /s**4 = m**2 sec def JONSWAP_default_alt(f, f_max, U, gamma=3.3): return JONSWAP_default(f, X(f_max, U), U, gamma=gamma) + def JONSWAP_default(f, X, U, gamma=3.3): """ see Ocean Surface waves - S. R. Massel eq.3.81 - eq.3.84 @@ -154,63 +170,66 @@ def JONSWAP_default(f, X, U, gamma=3.3): gamma is defined in the JONSWAP spectrum as 3.3. However, The JONSWAP spectrum is a psectrum for a fully deleoped sea, and likely not observed in a remote location. varying gamma to less then its JONSWAP value will mimic the attenuation of the peak with travel time to the limit of gamma=1, which is the PM spectrums """ - g = 9.81 # m s**-2 - w = 2.*np.pi*f # rad/sec + g = 9.81 # m s**-2 + w = 2.0 * np.pi * f # rad/sec - alpha=0.076*(g *X /U**2)**(-0.22) #non-dimentional - wp= 7.* np.pi * (g /U) * (g * X /U**2)**(-0.33) - #print('wp='+str(wp)) - #print('nu='+str(wp*U/(g))) + alpha = 0.076 * (g * X / U**2) ** (-0.22) # non-dimentional + wp = 7.0 * np.pi * (g / U) * (g * X / U**2) ** (-0.33) + # print('wp='+str(wp)) + # print('nu='+str(wp*U/(g))) - sigma_p=0.07 - sigma_pp=0.09 - sigma= np.array([[sigma_p if i else sigma_pp][0] for i in list(w<=wp) ]) + sigma_p = 0.07 + sigma_pp = 0.09 + sigma = np.array([[sigma_p if i else sigma_pp][0] for i in list(w <= wp)]) - delta=np.exp(-(w-wp)**2 / ( 2.*sigma**2. *wp**2. ) ) - peak_factor=gamma**delta + delta = np.exp(-((w - wp) ** 2) / (2.0 * sigma**2.0 * wp**2.0)) + peak_factor = gamma**delta + + return ( + alpha * g**2.0 * w ** (-5.0) * np.exp(-5.0 / 4.0 * (w / wp) ** -4) * peak_factor + ) # Hz**-5 m**2 /s**4 = m**2 sec ? - return alpha * g**2.0 * w**(-5.) * np.exp(-5./4.0 * (w/wp)**-4)*peak_factor # Hz**-5 m**2 /s**4 = m**2 sec ? """ add function for X_tilde(X, U10), alpha(f_max, U10) and f_max(U10, X_tilde) or f_max(U, X)""" -g=9.81 +g = 9.81 + def X_tilde(X, U10): - return g*X/U10**2 + return g * X / U10**2 + def alpha(f_max, U10): - return 0.033*(f_max*U10/g)**0.67 + return 0.033 * (f_max * U10 / g) ** 0.67 + def f_max(U10, X): - return 3.5*g/U10*X_tilde(X,U10)**(-0.33) + return 3.5 * g / U10 * X_tilde(X, U10) ** (-0.33) + def X(f_max, U10): - return 3.5**3 * g**2 /(U10 *f_max**3) + return 3.5**3 * g**2 / (U10 * f_max**3) + # %% -if __name__ == '__main__': +if __name__ == "__main__": M.figure_axis_xy() - plt.plot(f, JONSWAP_bulk(f, gamma=3.3), label='JONSWAP bulk (RIS version)') - plt.plot(f, JONSWAP_bulk(f, gamma=1), label='JONSWAP bulk --> PM (RIS version)') + plt.plot(f, JONSWAP_bulk(f, gamma=3.3), label="JONSWAP bulk (RIS version)") + plt.plot(f, JONSWAP_bulk(f, gamma=1), label="JONSWAP bulk --> PM (RIS version)") - plt.plot(f, pierson_moskowitz_default(f, 15), label='PM default') - plt.plot(f, pierson_moskowitz_fetch_limit(f, 5e5, 15), label='PM fetch limited') - plt.plot(f, JONSWAP_default(f, 5e5, 15, 3.3), label='JONSWAP default') + plt.plot(f, pierson_moskowitz_default(f, 15), label="PM default") + plt.plot(f, pierson_moskowitz_fetch_limit(f, 5e5, 15), label="PM fetch limited") + plt.plot(f, JONSWAP_default(f, 5e5, 15, 3.3), label="JONSWAP default") plt.legend() - # %% -def gamma_time_JONSWAP_default(time, f, - slope_t, intersectT, - tgammapar, tscale, - f_max, - U10, - gamma_peak, - plot=False): +def gamma_time_JONSWAP_default( + time, f, slope_t, intersectT, tgammapar, tscale, f_max, U10, gamma_peak, plot=False +): """ This method calculated a 2D shaped function given the parameters: inputs: @@ -236,67 +255,62 @@ def gamma_time_JONSWAP_default(time, f, 2d function with the shape of [time,freq] """ - #intersectf=intersect-intersect/slopet - intersectF=-intersectT*slope_t - pfreq=time*slope_t+intersectF - #print('intersect F=' + str(intersectF)) - - #intersectf=intersect#-intersect/slope - slopeF=1/slope_t - pfreq_forgamma=f*slopeF+intersectT - - #rint(pfreq.shape) - tt, line=np.meshgrid(time, pfreq_forgamma) - #print(line) - #print(tt) - func_t= gamma_time_normlized_amp_shifted(tt, gammapar=tgammapar, loc=line, scale=tscale) - #func_t_temp= (tamp*np.exp(- (time-t_center )**4 / tsigma )) - #print(func_t.shape) + # intersectf=intersect-intersect/slopet + intersectF = -intersectT * slope_t + pfreq = time * slope_t + intersectF + # print('intersect F=' + str(intersectF)) + + # intersectf=intersect#-intersect/slope + slopeF = 1 / slope_t + pfreq_forgamma = f * slopeF + intersectT + + # rint(pfreq.shape) + tt, line = np.meshgrid(time, pfreq_forgamma) + # print(line) + # print(tt) + func_t = gamma_time_normlized_amp_shifted( + tt, gammapar=tgammapar, loc=line, scale=tscale + ) + # func_t_temp= (tamp*np.exp(- (time-t_center )**4 / tsigma )) + # print(func_t.shape) """ Define X(f_max and U) here """ + def X(f_max, U10): - return 3.5**3 * g**2 /(U10 *f_max**3) + return 3.5**3 * g**2 / (U10 * f_max**3) - func_freq_temp= JONSWAP_default(f, X=X(f_max,U10), U=U10, gamma=gamma_peak) + func_freq_temp = JONSWAP_default(f, X=X(f_max, U10), U=U10, gamma=gamma_peak) - #func_freq_temp= (famp*np.exp(- (f-fcenter)**2 / fsigma )) - tt, func_freq= np.meshgrid( time, func_freq_temp) - tt3, ff3= np.meshgrid( time, func_freq_temp) + # func_freq_temp= (famp*np.exp(- (f-fcenter)**2 / fsigma )) + tt, func_freq = np.meshgrid(time, func_freq_temp) + tt3, ff3 = np.meshgrid(time, func_freq_temp) if plot: - plt.subplot(3, 1, 1) - plt.contourf( func_t) - #plt.plot(time, pfreq) - #plt.ylim(f.min(), f.max()) - + plt.contourf(func_t) + # plt.plot(time, pfreq) + # plt.ylim(f.min(), f.max()) plt.subplot(3, 1, 2) - #plt.plot(f, func_freq.mean(1)) - #plt.ylim(f.min(), f.max()) + # plt.plot(f, func_freq.mean(1)) + # plt.ylim(f.min(), f.max()) plt.contourf(func_freq) plt.subplot(3, 1, 3) - #plt.plot(time, pfreq) + # plt.plot(time, pfreq) plt.plot(time, pfreq) - plt.contourf((func_t * func_freq).reshape(tt.shape) ) - #plt.ylim(0.04,0.08) - #plt.grid() + plt.contourf((func_t * func_freq).reshape(tt.shape)) + # plt.ylim(0.04,0.08) + # plt.grid() return (func_t * func_freq).T - -def gamma_time_JONSWAP_nondim(time, f, - slope_t, intersectT, - tgammapar, tscale, - f_max, - U10, - gamma_peak, - plot=False): - +def gamma_time_JONSWAP_nondim( + time, f, slope_t, intersectT, tgammapar, tscale, f_max, U10, gamma_peak, plot=False +): """ This method calculated a 2D shaped function given the parameters: the JONSWAP function is non-dimensionalized to work with normalized data @@ -323,55 +337,60 @@ def gamma_time_JONSWAP_nondim(time, f, 2d nondim function with the shape of [time,freq] """ - #intersectf=intersect-intersect/slopet - intersectF=-intersectT*slope_t - pfreq=time*slope_t+intersectF - #print('intersect F=' + str(intersectF)) - - #intersectf=intersect#-intersect/slope - slopeF=1/slope_t - pfreq_forgamma=f*slopeF+intersectT - - #rint(pfreq.shape) - tt, line=np.meshgrid(time, pfreq_forgamma) - #print(line) - #print(tt) - func_t= gamma_time_normlized_amp_shifted(tt, gammapar=tgammapar, loc=line, scale=tscale) - #func_t_temp= (tamp*np.exp(- (time-t_center )**4 / tsigma )) - #print(func_t.shape) - g=9.81 + # intersectf=intersect-intersect/slopet + intersectF = -intersectT * slope_t + pfreq = time * slope_t + intersectF + # print('intersect F=' + str(intersectF)) + + # intersectf=intersect#-intersect/slope + slopeF = 1 / slope_t + pfreq_forgamma = f * slopeF + intersectT + + # rint(pfreq.shape) + tt, line = np.meshgrid(time, pfreq_forgamma) + # print(line) + # print(tt) + func_t = gamma_time_normlized_amp_shifted( + tt, gammapar=tgammapar, loc=line, scale=tscale + ) + # func_t_temp= (tamp*np.exp(- (time-t_center )**4 / tsigma )) + # print(func_t.shape) + g = 9.81 + def X(f_max, U10): - return 3.5**3.0 * g**2.0 /(U10 *f_max**3.0) + return 3.5**3.0 * g**2.0 / (U10 * f_max**3.0) - func_freq_temp= JONSWAP_default(f, X=X(f_max,U10), U=U10, gamma=gamma_peak) * (g**2.0 / U10**4.0) + func_freq_temp = JONSWAP_default(f, X=X(f_max, U10), U=U10, gamma=gamma_peak) * ( + g**2.0 / U10**4.0 + ) - #func_freq_temp= (famp*np.exp(- (f-fcenter)**2 / fsigma )) - tt, func_freq= np.meshgrid( time, func_freq_temp) - tt3, ff3= np.meshgrid( time, func_freq_temp) + # func_freq_temp= (famp*np.exp(- (f-fcenter)**2 / fsigma )) + tt, func_freq = np.meshgrid(time, func_freq_temp) + tt3, ff3 = np.meshgrid(time, func_freq_temp) if plot: - plt.subplot(3, 1, 1) - plt.contourf( func_t) - #plt.plot(time, pfreq) - #plt.ylim(f.min(), f.max()) - + plt.contourf(func_t) + # plt.plot(time, pfreq) + # plt.ylim(f.min(), f.max()) plt.subplot(3, 1, 2) - #plt.plot(f, func_freq.mean(1)) - #plt.ylim(f.min(), f.max()) + # plt.plot(f, func_freq.mean(1)) + # plt.ylim(f.min(), f.max()) plt.contourf(func_freq) plt.subplot(3, 1, 3) - #plt.plot(time, pfreq) + # plt.plot(time, pfreq) plt.plot(time, pfreq) - plt.contourf((func_t * func_freq).reshape(tt.shape) ) - #plt.ylim(0.04,0.08) - #plt.grid() + plt.contourf((func_t * func_freq).reshape(tt.shape)) + # plt.ylim(0.04,0.08) + # plt.grid() return (func_t * func_freq).T + + # %% # if __name__ == '__main__': # slope0= 1.2 @@ -384,8 +403,11 @@ def X(f_max, U10): # # + # %% build residual -def residual_JONSWAP_default_gamma(value_dict, time, f, data=None, weight=None, eps=None, plot_flag=False): +def residual_JONSWAP_default_gamma( + value_dict, time, f, data=None, weight=None, eps=None, plot_flag=False +): """ derived the residual between model and data given params_local. @@ -402,59 +424,74 @@ def residual_JONSWAP_default_gamma(value_dict, time, f, data=None, weight=None, from collections import OrderedDict if type(value_dict) is Parameters: - vd=value_dict.valuesdict() + vd = value_dict.valuesdict() elif (type(value_dict) is dict) | (type(value_dict) is OrderedDict): - vd=value_dict + vd = value_dict else: - raise ValueError('value_dict is eiher a dicitionary or a Params instance') - - r0 = vd['DT'] * 9.81 / (4* np.pi * vd['slope']) - nue = vd['attenuation_nue'] - attenuation = np.exp(-r0 * nue) - model= attenuation * gamma_time_JONSWAP_default(time, f, - vd['slope'], vd['intersect'], - vd['tgammapar'], vd['tscale'], - vd['f_max'],vd['U10'],vd['gamma_peak'], - plot=plot_flag ) - - #tt, tt= np.meshgrid(time, ff) - model1d=model.reshape(model.shape[0]*model.shape[1]) - model1d[np.isnan(model1d)]= 0 - #print(model1d) + raise ValueError("value_dict is eiher a dicitionary or a Params instance") + + r0 = vd["DT"] * 9.81 / (4 * np.pi * vd["slope"]) + nue = vd["attenuation_nue"] + attenuation = np.exp(-r0 * nue) + model = attenuation * gamma_time_JONSWAP_default( + time, + f, + vd["slope"], + vd["intersect"], + vd["tgammapar"], + vd["tscale"], + vd["f_max"], + vd["U10"], + vd["gamma_peak"], + plot=plot_flag, + ) + + # tt, tt= np.meshgrid(time, ff) + model1d = model.reshape(model.shape[0] * model.shape[1]) + model1d[np.isnan(model1d)] = 0 + # print(model1d) if data is not None: if np.size(data.shape) != 1: if model.shape == data.shape: - data1d=data.reshape(data.shape[0]*data.shape[1]) - nan_track=np.isnan(data1d) + data1d = data.reshape(data.shape[0] * data.shape[1]) + nan_track = np.isnan(data1d) elif model.shape == data.T.shape: - data1d=data.T.reshape(data.T.shape[0]*data.T.shape[1]) - nan_track=np.isnan(data1d) + data1d = data.T.reshape(data.T.shape[0] * data.T.shape[1]) + nan_track = np.isnan(data1d) else: raise TypeError("data shape does not match") if weight is not None: if (len(weight.shape) == 1) & (model1d.size == weight.size): - weight1d= weight + weight1d = weight elif (len(weight.shape) == 2) & (model1d.size == weight.size): - weight1d= weight.reshape(weight.shape[0]*weight.shape[1]).T + weight1d = weight.reshape(weight.shape[0] * weight.shape[1]).T else: - raise ValueError('weight has not the same dimensions as model. \n' + 'data ' +str(model.shape) + '\n weight '+str(weight.shape) ) - + raise ValueError( + "weight has not the same dimensions as model. \n" + + "data " + + str(model.shape) + + "\n weight " + + str(weight.shape) + ) if data is None: return model1d - if (weight is not None): - #print('use weight') - d=(model1d - data1d)*weight1d - d[nan_track]=np.nan + if weight is not None: + # print('use weight') + d = (model1d - data1d) * weight1d + d[nan_track] = np.nan return d if (weight is None) and (data is not None): - d= model1d - data1d - d[nan_track]=np.nan + d = model1d - data1d + d[nan_track] = np.nan return d -def residual_JONSWAP_nondim_gamma(value_dict, time, f, data=None, weight=None, eps=None, plot_flag=False): + +def residual_JONSWAP_nondim_gamma( + value_dict, time, f, data=None, weight=None, eps=None, plot_flag=False +): """ derived the residual between model and data given params_local. @@ -471,135 +508,151 @@ def residual_JONSWAP_nondim_gamma(value_dict, time, f, data=None, weight=None, e from collections import OrderedDict if type(value_dict) is Parameters: - vd=value_dict.valuesdict() + vd = value_dict.valuesdict() elif (type(value_dict) is dict) | (type(value_dict) is OrderedDict): - vd=value_dict + vd = value_dict else: - raise ValueError('value_dict is eiher a dicitionary or a Params instance') - - r0 = vd['DT'] * 9.81 / (4* np.pi * vd['slope']) - nue = vd['attenuation_nue'] - attenuation = np.exp(-r0 * nue) - model= attenuation * gamma_time_JONSWAP_nondim(time, f, - vd['slope'], vd['intersect'], - vd['tgammapar'], vd['tscale'], - vd['f_max'],vd['U10'],vd['gamma_peak'], - plot=plot_flag ) - - #tt, tt= np.meshgrid(time, ff) - model1d=model.reshape(model.shape[0]*model.shape[1]) - + raise ValueError("value_dict is eiher a dicitionary or a Params instance") + + r0 = vd["DT"] * 9.81 / (4 * np.pi * vd["slope"]) + nue = vd["attenuation_nue"] + attenuation = np.exp(-r0 * nue) + model = attenuation * gamma_time_JONSWAP_nondim( + time, + f, + vd["slope"], + vd["intersect"], + vd["tgammapar"], + vd["tscale"], + vd["f_max"], + vd["U10"], + vd["gamma_peak"], + plot=plot_flag, + ) + + # tt, tt= np.meshgrid(time, ff) + model1d = model.reshape(model.shape[0] * model.shape[1]) if data is not None: if np.size(data.shape) != 1: if model.shape == data.shape: - data1d=data.reshape(data.shape[0]*data.shape[1]) - nan_track=np.isnan(data1d) + data1d = data.reshape(data.shape[0] * data.shape[1]) + nan_track = np.isnan(data1d) elif model.shape == data.T.shape: - data1d=data.T.reshape(data.T.shape[0]*data.T.shape[1]) - nan_track=np.isnan(data1d) + data1d = data.T.reshape(data.T.shape[0] * data.T.shape[1]) + nan_track = np.isnan(data1d) else: raise TypeError("data shape does not match") if weight is not None: if (len(weight.shape) == 1) & (model1d.size == weight.size): - weight1d= weight + weight1d = weight elif (len(weight.shape) == 2) & (model1d.size == weight.size): - weight1d= weight.reshape(weight.shape[0]*weight.shape[1]).T + weight1d = weight.reshape(weight.shape[0] * weight.shape[1]).T else: - raise ValueError('weight has not the same dimensions as model. \n' + 'data ' +str(model.shape) + '\n weight '+str(weight.shape) ) - + raise ValueError( + "weight has not the same dimensions as model. \n" + + "data " + + str(model.shape) + + "\n weight " + + str(weight.shape) + ) if data is None: return model1d - if (weight is not None): - #print('use weight') - d=(model1d - data1d)*weight1d - d[nan_track]=np.nan + if weight is not None: + # print('use weight') + d = (model1d - data1d) * weight1d + d[nan_track] = np.nan return d if (weight is None) and (data is not None): - d= model1d - data1d - d[nan_track]=np.nan + d = model1d - data1d + d[nan_track] = np.nan return d -def gamma_from_params(value_dict, time, plot_flag=False): +def gamma_from_params(value_dict, time, plot_flag=False): from lmfit import Parameters from collections import OrderedDict if type(value_dict) is Parameters: - vd=value_dict.valuesdict() + vd = value_dict.valuesdict() elif (type(value_dict) is dict) | (type(value_dict) is OrderedDict): - vd=value_dict + vd = value_dict else: - raise ValueError('value_dict is eiher a dicitionary or a Params instance') + raise ValueError("value_dict is eiher a dicitionary or a Params instance") - spec = JONSWAP_default(f, X(vd['f_max'],vd['U10']) , vd['U10'], vd['gamma_peak']) - func_t= gamma_time_normlized_amp_shifted(time, gammapar=vd['tgammapar'], loc=0.5, scale=vd['tgammapar']) + spec = JONSWAP_default(f, X(vd["f_max"], vd["U10"]), vd["U10"], vd["gamma_peak"]) + func_t = gamma_time_normlized_amp_shifted( + time, gammapar=vd["tgammapar"], loc=0.5, scale=vd["tgammapar"] + ) if plot_flag: - plt.plot(f, spec, label='JONSWAP default') + plt.plot(f, spec, label="JONSWAP default") return spec -def JONSWAP_default_from_params(value_dict, f, plot_flag=False): +def JONSWAP_default_from_params(value_dict, f, plot_flag=False): from lmfit import Parameters from collections import OrderedDict if type(value_dict) is Parameters: - vd=value_dict.valuesdict() + vd = value_dict.valuesdict() elif (type(value_dict) is dict) | (type(value_dict) is OrderedDict): - vd=value_dict + vd = value_dict else: - raise ValueError('value_dict is eiher a dicitionary or a Params instance') + raise ValueError("value_dict is eiher a dicitionary or a Params instance") - spec = JONSWAP_default(f, X(vd['f_max'],vd['U10']) , vd['U10'], vd['gamma_peak']) + spec = JONSWAP_default(f, X(vd["f_max"], vd["U10"]), vd["U10"], vd["gamma_peak"]) if plot_flag: - plt.plot(f, spec, label='JONSWAP default') + plt.plot(f, spec, label="JONSWAP default") return spec -def JONSWAP_nondim_from_params(value_dict, f, plot_flag=False): +def JONSWAP_nondim_from_params(value_dict, f, plot_flag=False): from lmfit import Parameters from collections import OrderedDict if type(value_dict) is Parameters: - vd=value_dict.valuesdict() + vd = value_dict.valuesdict() elif (type(value_dict) is dict) | (type(value_dict) is OrderedDict): - vd=value_dict + vd = value_dict else: - raise ValueError('value_dict is eiher a dicitionary or a Params instance') + raise ValueError("value_dict is eiher a dicitionary or a Params instance") - g=9.81 - spec = JONSWAP_default(f, X(vd['f_max'],vd['U10']) , vd['U10'], vd['gamma_peak']) * (g**2 / vd['U10']**4) + g = 9.81 + spec = JONSWAP_default( + f, X(vd["f_max"], vd["U10"]), vd["U10"], vd["gamma_peak"] + ) * (g**2 / vd["U10"] ** 4) if plot_flag: - plt.plot(f, spec, label='JONSWAP default non-dim') + plt.plot(f, spec, label="JONSWAP default non-dim") return spec -if __name__ == '__main__': - #http://cars9.uchicago.edu/software/python/lmfit/fitting.html - params=Parameters() - params.add('slope', value= slope0, min=slope0*.1 , max=slope0*10) - params.add('intersect', value= intersect0, min=-0.5, max=.5) +if __name__ == "__main__": + # http://cars9.uchicago.edu/software/python/lmfit/fitting.html + params = Parameters() - #params.add('tamp', value= tamp0, min=0., max=1) - params.add('tgammapar', value= tgammapar0, min=0.0001, max=4) - params.add('tscale', value= tscale0, min=0, max=.1) + params.add("slope", value=slope0, min=slope0 * 0.1, max=slope0 * 10) + params.add("intersect", value=intersect0, min=-0.5, max=0.5) - params.add('f_max', value= 0.01, min=0., max=.1) - params.add('U10', value= 10, min=2, max=25) - params.add('gamma_peak', value= 3.3, min=0.1, max=4, vary=False) + # params.add('tamp', value= tamp0, min=0., max=1) + params.add("tgammapar", value=tgammapar0, min=0.0001, max=4) + params.add("tscale", value=tscale0, min=0, max=0.1) - params.add('amp', value= 1, min=1e-4, max=1e2) + params.add("f_max", value=0.01, min=0.0, max=0.1) + params.add("U10", value=10, min=2, max=25) + params.add("gamma_peak", value=3.3, min=0.1, max=4, vary=False) -if __name__ == '__main__': + params.add("amp", value=1, min=1e-4, max=1e2) + +if __name__ == "__main__": # %% should return model: - #JONSWAP_default_from_params(params, f, plot_flag=True) + # JONSWAP_default_from_params(params, f, plot_flag=True) model1d = residual_JONSWAP_default_gamma(params, time, f) M.figure_axis_xy(3, 6) @@ -618,11 +671,11 @@ def JONSWAP_nondim_from_params(value_dict, f, plot_flag=False): plt.plot(resid1d) plt.subplot(2, 1, 2) - plt.contour(time, f, fake_data, colors='k') + plt.contour(time, f, fake_data, colors="k") plt.contourf(time, f, resid1d.reshape(time.size, f.size).T) # %% should return the residual weighted residual - weightdummy=(fake_data*0+2) + weightdummy = fake_data * 0 + 2 len(weightdummy.shape) weightdummy.size len(fake_data.shape) @@ -630,8 +683,12 @@ def JONSWAP_nondim_from_params(value_dict, f, plot_flag=False): weightdummy.shape # %% - weightdummy =100.0 * np.random.random(fake_data.shape)#.reshape(fake_data.shape[0]*fake_data.shape[1] ) - resid1d_weight = residual_JONSWAP_default_gamma(params, time, f, data=fake_data, weight=weightdummy ) + weightdummy = 100.0 * np.random.random( + fake_data.shape + ) # .reshape(fake_data.shape[0]*fake_data.shape[1] ) + resid1d_weight = residual_JONSWAP_default_gamma( + params, time, f, data=fake_data, weight=weightdummy + ) M.figure_axis_xy(3, 6) @@ -640,13 +697,13 @@ def JONSWAP_nondim_from_params(value_dict, f, plot_flag=False): plt.plot(resid1d_weight, alpha=0.4) plt.subplot(2, 1, 2) - plt.contour(time, f, fake_data, colors='k') + plt.contour(time, f, fake_data, colors="k") plt.contourf(time, f, resid1d_weight.reshape(time.size, f.size).T) - # %% + def Jm_regulizer(value_dict, prior): """ returns a Model cost function as list. each item is the cost for each prior given the parameter value in value_dict @@ -659,27 +716,27 @@ def Jm_regulizer(value_dict, prior): from lmfit import Parameters from collections import OrderedDict - if type(value_dict) is Parameters: - vd=value_dict.valuesdict() + vd = value_dict.valuesdict() elif (type(value_dict) is dict) | (type(value_dict) is OrderedDict): - vd=value_dict + vd = value_dict else: - raise ValueError('value_dict is eiher a dicitionary or a Params instance') + raise ValueError("value_dict is eiher a dicitionary or a Params instance") - Jm=list() - for k,I in prior.items(): - #print(I) - if type(I['m_err']) is float: - Jm.append( (I['m0']- vd[k] ) / I['m_err'] ) + Jm = list() + for k, I in prior.items(): + # print(I) + if type(I["m_err"]) is float: + Jm.append((I["m0"] - vd[k]) / I["m_err"]) else: - if value_dict[k] >= I['m0']: - Jm.append( (I['m0']- vd[k] ) / I['m_err'][1] ) + if value_dict[k] >= I["m0"]: + Jm.append((I["m0"] - vd[k]) / I["m_err"][1]) else: - Jm.append( (I['m0']- vd[k] ) / I['m_err'][0] ) + Jm.append((I["m0"] - vd[k]) / I["m_err"][0]) return Jm -def Jm_dimensional_regulizer(value_dict, prior , prior_percentage=True): + +def Jm_dimensional_regulizer(value_dict, prior, prior_percentage=True): """ returns a Model cost function as list. each item is the cost for each prior given the parameter value in value_dict @@ -698,56 +755,61 @@ def Jm_dimensional_regulizer(value_dict, prior , prior_percentage=True): from lmfit import Parameters from collections import OrderedDict - if type(value_dict) is Parameters: - vd=value_dict.valuesdict() + vd = value_dict.valuesdict() elif (type(value_dict) is dict) | (type(value_dict) is OrderedDict): - vd=value_dict + vd = value_dict else: - raise ValueError('value_dict is eiher a dicitionary or a Params instance') + raise ValueError("value_dict is eiher a dicitionary or a Params instance") - Jm=list() + Jm = list() if prior_percentage: + def reg_func(p0, pi, p_err): - return ( (pi / p0 - 1) / p_err) + return (pi / p0 - 1) / p_err + else: + def reg_func(p0, pi, p_err_unit): - ppercent= p_err_unit/ p0 - return ( (pi / p0 - 1) / ppercent) - - for k,I in prior.items(): - #print(I) - if type(I['m_err']) is float: - Jm.append( reg_func(I['m0'], vd[k], I['m_err']) ) - #Jm.append( (I['m0']- vd[k] ) / I['m_err'] ) + ppercent = p_err_unit / p0 + return (pi / p0 - 1) / ppercent + + for k, I in prior.items(): + # print(I) + if type(I["m_err"]) is float: + Jm.append(reg_func(I["m0"], vd[k], I["m_err"])) + # Jm.append( (I['m0']- vd[k] ) / I['m_err'] ) else: - if value_dict[k] >= I['m0']: - Jm.append( reg_func(I['m0'], vd[k], I['m_err']) ) + if value_dict[k] >= I["m0"]: + Jm.append(reg_func(I["m0"], vd[k], I["m_err"])) else: - Jm.append( reg_func(I['m0'], vd[k], I['m_err']) ) + Jm.append(reg_func(I["m0"], vd[k], I["m_err"])) return Jm -if __name__ == '__main__': - #test Jm_regulizer with priors +if __name__ == "__main__": + # test Jm_regulizer with priors # create face priors - prior_errors={'slope': 1.0, 'intersect':.2, - 'tgammapar':.4, 'tscale':0.2, - 'f_max':0.01 , - 'amp': 10.0 - } - - priors=dict() - for k,I in prior_errors.items(): - priors[k]={'m_err':I, 'm0':params[k].value} + prior_errors = { + "slope": 1.0, + "intersect": 0.2, + "tgammapar": 0.4, + "tscale": 0.2, + "f_max": 0.01, + "amp": 10.0, + } + + priors = dict() + for k, I in prior_errors.items(): + priors[k] = {"m_err": I, "m0": params[k].value} # fake parameters - vd=copy.copy(params.valuesdict()) - for k,I in vd.items(): - vd[k]= I *np.random.rand() + vd = copy.copy(params.valuesdict()) + for k, I in vd.items(): + vd[k] = I * np.random.rand() - Jm =Jm_regulizer(vd , priors) + Jm = Jm_regulizer(vd, priors) print(Jm) @@ -762,7 +824,7 @@ def cost(value_dict, time, f, data=None, weight=None, prior=False, eps=None): """ from lmfit import Parameters - #Jd = residual_JONSWAP_nondim_gamma(value_dict, time, f, data=data, weight=weight) + # Jd = residual_JONSWAP_nondim_gamma(value_dict, time, f, data=data, weight=weight) Jd = residual_JONSWAP_default_gamma(value_dict, time, f, data=data, weight=weight) if prior is not False: @@ -774,16 +836,16 @@ def cost(value_dict, time, f, data=None, weight=None, prior=False, eps=None): # else: # raise ValueError('value_dict is eiher a dicitionary or a Params instance') - Jm = Jm_regulizer( value_dict, prior) - return np.concatenate(( Jd, Jm )) + Jm = Jm_regulizer(value_dict, prior) + return np.concatenate((Jd, Jm)) else: return Jd # %% test cost -if __name__ == '__main__': +if __name__ == "__main__": - cost1d_weight = cost(params, time, f, data=fake_data, weight=None, prior=False ) + cost1d_weight = cost(params, time, f, data=fake_data, weight=None, prior=False) M.figure_axis_xy(3, 6) @@ -792,21 +854,11 @@ def cost(value_dict, time, f, data=None, weight=None, prior=False, eps=None): plt.plot(resid1d_weight, alpha=0.4) plt.subplot(2, 1, 2) - plt.contour(time, f, fake_data, colors='k') + plt.contour(time, f, fake_data, colors="k") plt.contourf(time, f, cost1d_weight.reshape(time.size, f.size).T) # %% - cost1d_weight = cost(vd, time, f, data=fake_data, weight=None, prior= priors) - - - - - - - - - - + cost1d_weight = cost(vd, time, f, data=fake_data, weight=None, prior=priors) # %% build residual diff --git a/src/icesat2_tracks/local_modules/__init__.py b/src/icesat2_tracks/local_modules/__init__.py index 360a572a..0f73c056 100644 --- a/src/icesat2_tracks/local_modules/__init__.py +++ b/src/icesat2_tracks/local_modules/__init__.py @@ -1 +1 @@ -"""Utility functions to manage parameters to plot results""" \ No newline at end of file +"""Utility functions to manage parameters to plot results""" diff --git a/src/icesat2_tracks/local_modules/m_colormanager_ph3.py b/src/icesat2_tracks/local_modules/m_colormanager_ph3.py index 17163538..30d293a6 100644 --- a/src/icesat2_tracks/local_modules/m_colormanager_ph3.py +++ b/src/icesat2_tracks/local_modules/m_colormanager_ph3.py @@ -9,28 +9,30 @@ import os -#path='/Projects/2016_AA/others/color_def_mhell1.ase' +# path='/Projects/2016_AA/others/color_def_mhell1.ase' def ase_to_json(path): os.getcwd() import swatch - A=swatch.parse(path) - for i in A[0]['swatches']: - print(i['name'] + ' ' + str(i['data']['values'])) + A = swatch.parse(path) + + for i in A[0]["swatches"]: + print(i["name"] + " " + str(i["data"]["values"])) return A def json_save(name, path, data, verbose=False, return_name=False): import json + if not os.path.exists(path): os.makedirs(path) - full_name_root=os.path.join(path,name) - full_name= (os.path.join(full_name_root+ '.json')) - with open(full_name, 'w') as outfile: + full_name_root = os.path.join(path, name) + full_name = os.path.join(full_name_root + ".json") + with open(full_name, "w") as outfile: json.dump(data, outfile) if verbose: - print('save at: ',full_name) + print("save at: ", full_name) if return_name: return full_name_root else: @@ -39,252 +41,546 @@ def json_save(name, path, data, verbose=False, return_name=False): def json_load(name, path, verbose=False): import json - full_name= (os.path.join(path,name+ '.json')) - with open(full_name, 'r') as ifile: - data=json.load(ifile) + full_name = os.path.join(path, name + ".json") + + with open(full_name, "r") as ifile: + data = json.load(ifile) if verbose: - print('loaded from: ',full_name) + print("loaded from: ", full_name) return data - - class color: - def __init__(self, path=None, name=None): - self.white=(1,1,1) - if (path is not None) & (name is not None): - print('color theme: '+name) - try: - theme=json_load(name, path, verbose=True) - for k, v in theme.items(): - setattr(self, k, v) - #for k, v in theme['colors'].items(): - # print(k, v) - # setattr(self.colors, k, v) - except: - print('fail load theme, fall back to default theme') - print(path+'mhell_colortheme17') - self.default_colors() - - else: + def __init__(self, path=None, name=None): + self.white = (1, 1, 1) + if (path is not None) & (name is not None): + print("color theme: " + name) + try: + theme = json_load(name, path, verbose=True) + for k, v in theme.items(): + setattr(self, k, v) + # for k, v in theme['colors'].items(): + # print(k, v) + # setattr(self.colors, k, v) + except: + print("fail load theme, fall back to default theme") + print(path + "mhell_colortheme17") self.default_colors() - def alpha(self, color, a): - return ( color[0], color[1], color[2], a) - - - def colormaps2(self,n, gamma=None): - gamma=1 if gamma is None else gamma - - rels = self.rels - self.divergingwhite=LinearSegmentedColormap.from_list("my_colormap", (rels['plus'],rels['white'], rels['minus']), N=n, gamma=gamma) - self.divergingwhitelarge=LinearSegmentedColormap.from_list("my_colormap", - (rels['darkblue'], rels['blue'],rels['white'], rels['orange'], rels['red']), N=n, gamma=gamma) - - self.whiteblue=LinearSegmentedColormap.from_list("my_colormap", - (rels['white'],rels['lightblue'], rels['blue'],rels['darkblue']), N=n, gamma=gamma) - self.greyblue=LinearSegmentedColormap.from_list("my_colormap", - (rels['gridcolor'],rels['lightblue'], rels['blue'],rels['darkblue']), N=n, gamma=gamma) - self.greyblue_r=LinearSegmentedColormap.from_list("my_colormap", - (rels['darkblue'] , rels['blue'], rels['lightblue'], rels['gridcolor'] ), N=n, gamma=gamma) - - self.greyred=LinearSegmentedColormap.from_list("my_colormap", - (rels['gridcolor'], rels['rascade1'],rels['rascade2']), N=n, gamma=gamma) - self.greyred_r=LinearSegmentedColormap.from_list("my_colormap", - (rels['rascade3'] , rels['rascade2'], rels['rascade1'], rels['gridcolor'] ), N=n, gamma=gamma) - - self.greyredorange=LinearSegmentedColormap.from_list("my_colormap", - (rels['gridcolor'],rels['cascade3'], rels['red'],rels['orange']), N=n, gamma=gamma) - - self.white_base_blue=LinearSegmentedColormap.from_list("my_colormap", - (rels['white'], rels['darkblue'], rels['blue'],rels['lightblue']), N=n, gamma=gamma) - self.white_base_blue_r=LinearSegmentedColormap.from_list("my_colormap", - (rels['white'], rels['lightblue'], rels['blue'],rels['darkblue']), N=n, gamma=gamma) - - self.white_base_bluegreen=LinearSegmentedColormap.from_list("my_colormap", - (rels['white'], rels['darkblue'], rels['blue'],rels['green']), N=n, gamma=gamma) - self.white_base_bluegreen_r=LinearSegmentedColormap.from_list("my_colormap", - (rels['white'], rels['lightblue'], rels['blue'],rels['darkblue'],rels['group2']), N=n, gamma=gamma) - self.white_base_blgror=LinearSegmentedColormap.from_list("my_colormap", #rels['blue'], - (rels['white'], rels['gridcolor'] ,rels['cascade2'], rels['cascade1'],rels['group1'] ), N=n, gamma=gamma) - - - self.rainbow=LinearSegmentedColormap.from_list("my_colormap", - (rels['darkblue'], rels['blue'], rels['lightblue'], rels['green'],rels['orange'], rels['red'] ), N=n, gamma=gamma) - self.cascade=LinearSegmentedColormap.from_list("my_colormap", - (rels['cascade1'], rels['cascade2'], rels['cascade3'], rels['cascade4'] ), N=n, gamma=gamma) - self.cascade_r=LinearSegmentedColormap.from_list("my_colormap", - (rels['white'], rels['cascade4'], rels['cascade3'], rels['cascade2'], rels['cascade1'], rels['green'] ), N=n, gamma=gamma) - self.cascade_r_nowhite=LinearSegmentedColormap.from_list("my_colormap", - (rels['cascade4'], rels['cascade3'], rels['cascade2'], rels['cascade1'], rels['green'] ), N=n, gamma=gamma) - - self.cascade_highcontrast=LinearSegmentedColormap.from_list("my_colormap", - (rels['cascade1'], rels['cascade3'], rels['aug1'], rels['grey'], rels['aug2'] ), N=n, gamma=gamma) - - - self.circle_small=LinearSegmentedColormap.from_list("my_colormap", - (rels['minus'], rels['plus'],rels['plus'], rels['minus']), N=n, gamma=gamma) - self.circle_small_r=LinearSegmentedColormap.from_list("my_colormap", - (rels['plus'],rels['minus'], rels['minus'], rels['plus']), N=n, gamma=gamma) - - self.circle_medium=LinearSegmentedColormap.from_list("my_colormap", - (rels['red'],rels['green'], rels['darkblue'],rels['green'], rels['red']), N=n, gamma=gamma) - self.circle_medium_r=LinearSegmentedColormap.from_list("my_colormap", - (rels['darkblue'],rels['green'], rels['red'], rels['green'], rels['darkblue']), N=n, gamma=gamma) - - - self.circle_big=LinearSegmentedColormap.from_list("my_colormap", - (rels['darkblue'], rels['blue'], rels['orange'], rels['red'], rels['orange'], rels['blue']), N=n, gamma=gamma) - self.circle_big_r=LinearSegmentedColormap.from_list("my_colormap", - (rels['red'], rels['orange'], rels['blue'], rels['darkblue'], rels['blue'], rels['orange']), N=n, gamma=gamma) - - self.circle_medium_triple=LinearSegmentedColormap.from_list("my_colormap", - (rels['darkblue'], rels['orange'], rels['orange'] , rels['red'], rels['red'], rels['darkblue']), N=n, gamma=gamma) - self.circle_medium_triple_r=LinearSegmentedColormap.from_list("my_colormap", - (rels['red'], rels['darkblue'], rels['darkblue'] , rels['orange'], rels['orange'],rels['red']), N=n, gamma=gamma) - - def colormaps(self,n, gamma=None): - gamma=1 if gamma is None else gamma - self.divergingwhite=LinearSegmentedColormap.from_list("my_colormap", (self.plus,self.white, self.minus), N=n, gamma=gamma) - self.divergingwhitelarge=LinearSegmentedColormap.from_list("my_colormap", - (self.colors['darkblue'], self.colors['blue'],self.white, self.colors['orange'], self.colors['red']), N=n, gamma=gamma) - - self.whiteblue=LinearSegmentedColormap.from_list("my_colormap", - (self.white,self.colors['lightblue'], self.colors['blue'],self.colors['darkblue']), N=n, gamma=gamma) - self.greyblue=LinearSegmentedColormap.from_list("my_colormap", - (self.gridcolor,self.colors['lightblue'], self.colors['blue'],self.colors['darkblue']), N=n, gamma=gamma) - self.greyblue_r=LinearSegmentedColormap.from_list("my_colormap", - (self.colors['darkblue'] , self.colors['blue'], self.colors['lightblue'], self.gridcolor ), N=n, gamma=gamma) - self.greyredorange=LinearSegmentedColormap.from_list("my_colormap", - (self.gridcolor,self.cascade3, self.colors['red'],self.colors['orange']), N=n, gamma=gamma) - - self.white_base_blue=LinearSegmentedColormap.from_list("my_colormap", - (self.white, self.colors['darkblue'], self.colors['blue'],self.colors['lightblue']), N=n, gamma=gamma) - self.white_base_bluegreen=LinearSegmentedColormap.from_list("my_colormap", - (self.white, self.colors['darkblue'], self.colors['blue'],self.colors['green']), N=n, gamma=gamma) - self.rainbow=LinearSegmentedColormap.from_list("my_colormap", - (self.colors['darkblue'], self.colors['blue'], self.colors['lightblue'], self.colors['green'],self.colors['orange'], self.colors['red'] ), N=n, gamma=gamma) - self.cascade=LinearSegmentedColormap.from_list("my_colormap", - (self.cascade1, self.cascade2, self.cascade3, self.cascade4 ), N=n, gamma=gamma) - self.cascade_r=LinearSegmentedColormap.from_list("my_colormap", - (self.white, self.cascade4, self.cascade3, self.cascade2, self.cascade1, self.colors['green'] ), N=n, gamma=gamma) - self.cascade_r_nowhite=LinearSegmentedColormap.from_list("my_colormap", - (self.cascade4, self.cascade3, self.cascade2, self.cascade1, self.colors['green'] ), N=n, gamma=gamma) - - self.cascade_highcontrast=LinearSegmentedColormap.from_list("my_colormap", - (self.cascade1, self.cascade3, self.aug1, self.grey, self.aug2 ), N=n, gamma=gamma) - - def show(self): - for key in self.__dict__.keys(): - print(key) - - print(' rels dict:') - for key in self.rels.keys(): - print(' '+key) - - #print(self.__dict__) - def plot(self): - dd=self.__dict__.copy() - dd_colors=dd['rels'] - del dd['rels'] - - Y=2.0 - coldd=dd - ncolor=len(coldd) - fig = plt.figure(figsize=[2, Y]) - ax = fig.add_axes([0, 0, .5, Y]) - dy=1/ncolor - y=np.arange(0, 1, dy) - - j=0 - for k,I in coldd.items(): - try: - r1 = mpatch.Rectangle((0, y[j]), 1, dy, color=I) - txt = ax.text(1.1, y[j]+dy*.5, ' '+k, va='center', fontsize=10, - weight='regular') - - ax.add_patch(r1) - j+=1 - except: - pass - - coldd=dd_colors - ncolor=len(coldd) - #fig = plt.figure(figsize=[1.2, Y]) - ax1 = fig.add_axes([1.5, 0, .5, Y]) - dy=1/ncolor - y=np.arange(0, 1, dy) - - j=0 - for k,I in coldd.items(): + + else: + self.default_colors() + + def alpha(self, color, a): + return (color[0], color[1], color[2], a) + + def colormaps2(self, n, gamma=None): + gamma = 1 if gamma is None else gamma + + rels = self.rels + self.divergingwhite = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["plus"], rels["white"], rels["minus"]), + N=n, + gamma=gamma, + ) + self.divergingwhitelarge = LinearSegmentedColormap.from_list( + "my_colormap", + ( + rels["darkblue"], + rels["blue"], + rels["white"], + rels["orange"], + rels["red"], + ), + N=n, + gamma=gamma, + ) + + self.whiteblue = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["white"], rels["lightblue"], rels["blue"], rels["darkblue"]), + N=n, + gamma=gamma, + ) + self.greyblue = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["gridcolor"], rels["lightblue"], rels["blue"], rels["darkblue"]), + N=n, + gamma=gamma, + ) + self.greyblue_r = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["darkblue"], rels["blue"], rels["lightblue"], rels["gridcolor"]), + N=n, + gamma=gamma, + ) + + self.greyred = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["gridcolor"], rels["rascade1"], rels["rascade2"]), + N=n, + gamma=gamma, + ) + self.greyred_r = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["rascade3"], rels["rascade2"], rels["rascade1"], rels["gridcolor"]), + N=n, + gamma=gamma, + ) + + self.greyredorange = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["gridcolor"], rels["cascade3"], rels["red"], rels["orange"]), + N=n, + gamma=gamma, + ) + + self.white_base_blue = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["white"], rels["darkblue"], rels["blue"], rels["lightblue"]), + N=n, + gamma=gamma, + ) + self.white_base_blue_r = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["white"], rels["lightblue"], rels["blue"], rels["darkblue"]), + N=n, + gamma=gamma, + ) + + self.white_base_bluegreen = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["white"], rels["darkblue"], rels["blue"], rels["green"]), + N=n, + gamma=gamma, + ) + self.white_base_bluegreen_r = LinearSegmentedColormap.from_list( + "my_colormap", + ( + rels["white"], + rels["lightblue"], + rels["blue"], + rels["darkblue"], + rels["group2"], + ), + N=n, + gamma=gamma, + ) + self.white_base_blgror = LinearSegmentedColormap.from_list( + "my_colormap", # rels['blue'], + ( + rels["white"], + rels["gridcolor"], + rels["cascade2"], + rels["cascade1"], + rels["group1"], + ), + N=n, + gamma=gamma, + ) + + self.rainbow = LinearSegmentedColormap.from_list( + "my_colormap", + ( + rels["darkblue"], + rels["blue"], + rels["lightblue"], + rels["green"], + rels["orange"], + rels["red"], + ), + N=n, + gamma=gamma, + ) + self.cascade = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["cascade1"], rels["cascade2"], rels["cascade3"], rels["cascade4"]), + N=n, + gamma=gamma, + ) + self.cascade_r = LinearSegmentedColormap.from_list( + "my_colormap", + ( + rels["white"], + rels["cascade4"], + rels["cascade3"], + rels["cascade2"], + rels["cascade1"], + rels["green"], + ), + N=n, + gamma=gamma, + ) + self.cascade_r_nowhite = LinearSegmentedColormap.from_list( + "my_colormap", + ( + rels["cascade4"], + rels["cascade3"], + rels["cascade2"], + rels["cascade1"], + rels["green"], + ), + N=n, + gamma=gamma, + ) + + self.cascade_highcontrast = LinearSegmentedColormap.from_list( + "my_colormap", + ( + rels["cascade1"], + rels["cascade3"], + rels["aug1"], + rels["grey"], + rels["aug2"], + ), + N=n, + gamma=gamma, + ) + + self.circle_small = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["minus"], rels["plus"], rels["plus"], rels["minus"]), + N=n, + gamma=gamma, + ) + self.circle_small_r = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["plus"], rels["minus"], rels["minus"], rels["plus"]), + N=n, + gamma=gamma, + ) + + self.circle_medium = LinearSegmentedColormap.from_list( + "my_colormap", + (rels["red"], rels["green"], rels["darkblue"], rels["green"], rels["red"]), + N=n, + gamma=gamma, + ) + self.circle_medium_r = LinearSegmentedColormap.from_list( + "my_colormap", + ( + rels["darkblue"], + rels["green"], + rels["red"], + rels["green"], + rels["darkblue"], + ), + N=n, + gamma=gamma, + ) + + self.circle_big = LinearSegmentedColormap.from_list( + "my_colormap", + ( + rels["darkblue"], + rels["blue"], + rels["orange"], + rels["red"], + rels["orange"], + rels["blue"], + ), + N=n, + gamma=gamma, + ) + self.circle_big_r = LinearSegmentedColormap.from_list( + "my_colormap", + ( + rels["red"], + rels["orange"], + rels["blue"], + rels["darkblue"], + rels["blue"], + rels["orange"], + ), + N=n, + gamma=gamma, + ) + + self.circle_medium_triple = LinearSegmentedColormap.from_list( + "my_colormap", + ( + rels["darkblue"], + rels["orange"], + rels["orange"], + rels["red"], + rels["red"], + rels["darkblue"], + ), + N=n, + gamma=gamma, + ) + self.circle_medium_triple_r = LinearSegmentedColormap.from_list( + "my_colormap", + ( + rels["red"], + rels["darkblue"], + rels["darkblue"], + rels["orange"], + rels["orange"], + rels["red"], + ), + N=n, + gamma=gamma, + ) + + def colormaps(self, n, gamma=None): + gamma = 1 if gamma is None else gamma + self.divergingwhite = LinearSegmentedColormap.from_list( + "my_colormap", (self.plus, self.white, self.minus), N=n, gamma=gamma + ) + self.divergingwhitelarge = LinearSegmentedColormap.from_list( + "my_colormap", + ( + self.colors["darkblue"], + self.colors["blue"], + self.white, + self.colors["orange"], + self.colors["red"], + ), + N=n, + gamma=gamma, + ) + + self.whiteblue = LinearSegmentedColormap.from_list( + "my_colormap", + ( + self.white, + self.colors["lightblue"], + self.colors["blue"], + self.colors["darkblue"], + ), + N=n, + gamma=gamma, + ) + self.greyblue = LinearSegmentedColormap.from_list( + "my_colormap", + ( + self.gridcolor, + self.colors["lightblue"], + self.colors["blue"], + self.colors["darkblue"], + ), + N=n, + gamma=gamma, + ) + self.greyblue_r = LinearSegmentedColormap.from_list( + "my_colormap", + ( + self.colors["darkblue"], + self.colors["blue"], + self.colors["lightblue"], + self.gridcolor, + ), + N=n, + gamma=gamma, + ) + self.greyredorange = LinearSegmentedColormap.from_list( + "my_colormap", + (self.gridcolor, self.cascade3, self.colors["red"], self.colors["orange"]), + N=n, + gamma=gamma, + ) + + self.white_base_blue = LinearSegmentedColormap.from_list( + "my_colormap", + ( + self.white, + self.colors["darkblue"], + self.colors["blue"], + self.colors["lightblue"], + ), + N=n, + gamma=gamma, + ) + self.white_base_bluegreen = LinearSegmentedColormap.from_list( + "my_colormap", + ( + self.white, + self.colors["darkblue"], + self.colors["blue"], + self.colors["green"], + ), + N=n, + gamma=gamma, + ) + self.rainbow = LinearSegmentedColormap.from_list( + "my_colormap", + ( + self.colors["darkblue"], + self.colors["blue"], + self.colors["lightblue"], + self.colors["green"], + self.colors["orange"], + self.colors["red"], + ), + N=n, + gamma=gamma, + ) + self.cascade = LinearSegmentedColormap.from_list( + "my_colormap", + (self.cascade1, self.cascade2, self.cascade3, self.cascade4), + N=n, + gamma=gamma, + ) + self.cascade_r = LinearSegmentedColormap.from_list( + "my_colormap", + ( + self.white, + self.cascade4, + self.cascade3, + self.cascade2, + self.cascade1, + self.colors["green"], + ), + N=n, + gamma=gamma, + ) + self.cascade_r_nowhite = LinearSegmentedColormap.from_list( + "my_colormap", + ( + self.cascade4, + self.cascade3, + self.cascade2, + self.cascade1, + self.colors["green"], + ), + N=n, + gamma=gamma, + ) + + self.cascade_highcontrast = LinearSegmentedColormap.from_list( + "my_colormap", + (self.cascade1, self.cascade3, self.aug1, self.grey, self.aug2), + N=n, + gamma=gamma, + ) + + def show(self): + for key in self.__dict__.keys(): + print(key) + + print(" rels dict:") + for key in self.rels.keys(): + print(" " + key) + + # print(self.__dict__) + + def plot(self): + dd = self.__dict__.copy() + dd_colors = dd["rels"] + del dd["rels"] + + Y = 2.0 + coldd = dd + ncolor = len(coldd) + fig = plt.figure(figsize=[2, Y]) + ax = fig.add_axes([0, 0, 0.5, Y]) + dy = 1 / ncolor + y = np.arange(0, 1, dy) + + j = 0 + for k, I in coldd.items(): + try: r1 = mpatch.Rectangle((0, y[j]), 1, dy, color=I) - txt = ax1.text(1.1, y[j]+dy*.5, ' '+k, va='center', fontsize=10, - weight='regular') - - ax1.add_patch(r1) - j+=1 - #return fig - plt.title('rels') - - def add_standard_colors(self): - self.colors=dict() - self.gridcolor=(0,0,0) - - self.colors['red']=(203/255, 32/255, 39/255) - self.colors['green']=(15/255, 150/255, 72/255) - self.colors['orange']=(247/255, 191/255, 88/255) - - self.colors['grey1']=(167/255, 180/255, 183/255) - self.colors['grey']=self.colors['grey1'] - self.colors['lightgrey']=self.colors['grey1'] - self.colors['grey2']=(123/255, 121/255, 125/255) - self.colors['grey3']=(72/255, 70/255, 77/255) - - self.colors['darkblue']=(18/255, 78/255, 153/255) - self.colors['blue']=(85/255, 133/255, 196/255) - self.colors['lightblue']=(129/255, 140/255, 192/255) - - self.colors['blue4']=(7/255, 137/255, 198/255) - self.colors['black']=(0 , 0 ,0 ) - self.colors['white']=(1,1,1) - - - def default_colors(self): - self.colors=dict() - self.gridcolor=(0,0,0) - - self.colors['red']=(203/255, 32/255, 39/255) - self.colors['green']=(15/255, 150/255, 72/255) - self.colors['orange']=(247/255, 191/255, 88/255) - - self.colors['grey1']=(167/255, 180/255, 183/255) - self.colors['grey']=self.colors['grey1'] - self.colors['lightgrey']=self.colors['grey1'] - self.colors['grey2']=(123/255, 121/255, 125/255) - self.colors['grey3']=(72/255, 70/255, 77/255) - - self.colors['darkblue']=(18/255, 78/255, 153/255) - self.colors['blue']=(85/255, 133/255, 196/255) - self.colors['lightblue']=(129/255, 140/255, 192/255) - self.colors['blue4']=(7/255, 137/255, 198/255) - self.colors['black']=(0 , 0 ,0 ) - self.colors['white']=(1,1,1) - - converter={'lead1':'blue', - 'lead2': 'red', - 'lead3': 'green', - 'plus':'red', - 'minus': 'blue', - 'pair2_a':'darkblue', - 'pair2_b':'green', - 'aug1':'green', - 'aug2':'orange', - 'axiscolor':'black', - 'gridcolor':'grey2', - 'cascade1':'darkblue', - 'cascade2':'blue', - 'cascade3':'blue4', - 'cascade4':'lightblue'} - - for k,I in converter.items(): - setattr(self, k, self.colors[I]) + txt = ax.text( + 1.1, + y[j] + dy * 0.5, + " " + k, + va="center", + fontsize=10, + weight="regular", + ) + + ax.add_patch(r1) + j += 1 + except: + pass + + coldd = dd_colors + ncolor = len(coldd) + # fig = plt.figure(figsize=[1.2, Y]) + ax1 = fig.add_axes([1.5, 0, 0.5, Y]) + dy = 1 / ncolor + y = np.arange(0, 1, dy) + + j = 0 + for k, I in coldd.items(): + r1 = mpatch.Rectangle((0, y[j]), 1, dy, color=I) + txt = ax1.text( + 1.1, + y[j] + dy * 0.5, + " " + k, + va="center", + fontsize=10, + weight="regular", + ) + + ax1.add_patch(r1) + j += 1 + # return fig + plt.title("rels") + + def add_standard_colors(self): + self.colors = dict() + self.gridcolor = (0, 0, 0) + + self.colors["red"] = (203 / 255, 32 / 255, 39 / 255) + self.colors["green"] = (15 / 255, 150 / 255, 72 / 255) + self.colors["orange"] = (247 / 255, 191 / 255, 88 / 255) + + self.colors["grey1"] = (167 / 255, 180 / 255, 183 / 255) + self.colors["grey"] = self.colors["grey1"] + self.colors["lightgrey"] = self.colors["grey1"] + self.colors["grey2"] = (123 / 255, 121 / 255, 125 / 255) + self.colors["grey3"] = (72 / 255, 70 / 255, 77 / 255) + + self.colors["darkblue"] = (18 / 255, 78 / 255, 153 / 255) + self.colors["blue"] = (85 / 255, 133 / 255, 196 / 255) + self.colors["lightblue"] = (129 / 255, 140 / 255, 192 / 255) + + self.colors["blue4"] = (7 / 255, 137 / 255, 198 / 255) + self.colors["black"] = (0, 0, 0) + self.colors["white"] = (1, 1, 1) + + def default_colors(self): + self.colors = dict() + self.gridcolor = (0, 0, 0) + + self.colors["red"] = (203 / 255, 32 / 255, 39 / 255) + self.colors["green"] = (15 / 255, 150 / 255, 72 / 255) + self.colors["orange"] = (247 / 255, 191 / 255, 88 / 255) + + self.colors["grey1"] = (167 / 255, 180 / 255, 183 / 255) + self.colors["grey"] = self.colors["grey1"] + self.colors["lightgrey"] = self.colors["grey1"] + self.colors["grey2"] = (123 / 255, 121 / 255, 125 / 255) + self.colors["grey3"] = (72 / 255, 70 / 255, 77 / 255) + + self.colors["darkblue"] = (18 / 255, 78 / 255, 153 / 255) + self.colors["blue"] = (85 / 255, 133 / 255, 196 / 255) + self.colors["lightblue"] = (129 / 255, 140 / 255, 192 / 255) + self.colors["blue4"] = (7 / 255, 137 / 255, 198 / 255) + self.colors["black"] = (0, 0, 0) + self.colors["white"] = (1, 1, 1) + + converter = { + "lead1": "blue", + "lead2": "red", + "lead3": "green", + "plus": "red", + "minus": "blue", + "pair2_a": "darkblue", + "pair2_b": "green", + "aug1": "green", + "aug2": "orange", + "axiscolor": "black", + "gridcolor": "grey2", + "cascade1": "darkblue", + "cascade2": "blue", + "cascade3": "blue4", + "cascade4": "lightblue", + } + + for k, I in converter.items(): + setattr(self, k, self.colors[I]) diff --git a/src/icesat2_tracks/local_modules/m_general_ph3.py b/src/icesat2_tracks/local_modules/m_general_ph3.py index 6e24a8d9..e7434152 100644 --- a/src/icesat2_tracks/local_modules/m_general_ph3.py +++ b/src/icesat2_tracks/local_modules/m_general_ph3.py @@ -1,818 +1,949 @@ import os -#execfile(os.environ['PYTHONSTARTUP']) -#execfile(STARTUP_RIS_SeaIceThickness2016) +# execfile(os.environ['PYTHONSTARTUP']) +# execfile(STARTUP_RIS_SeaIceThickness2016) -#from __future__ import unicode_literals -#from __future__ import print_function -#from __future__ import division +# from __future__ import unicode_literals +# from __future__ import print_function +# from __future__ import division -#import matplotlib -#matplotlib.use('Agg') -#import matplotlib.pyplot as plt +# import matplotlib +# matplotlib.use('Agg') +# import matplotlib.pyplot as plt -#import numpy as np -#from scipy.io import netcdf +# import numpy as np +# from scipy.io import netcdf from scipy import signal -#import os +# import os from matplotlib.dates import DateFormatter, MinuteLocator from matplotlib.colors import LinearSegmentedColormap from matplotlib import dates import datetime as DT -#from cdo import * # python version + +# from cdo import * # python version from . import m_tools_ph3 as MT import numpy as np import matplotlib.pyplot as plt class color: - def __init__(self): - - self.red=(203/255, 32/255, 39/255) - self.green=(15/255, 150/255, 72/255) - self.orange=(247/255, 191/255, 88/255) - self.grey1=(167/255, 180/255, 183/255) - self.grey=self.grey1 - self.grey2=(123/255, 121/255, 125/255) - self.grey3=(72/255, 70/255, 77/255) - - self.blue1=(18/255, 78/255, 153/255) - self.blue=self.blue1 - self.blue2=(85/255, 133/255, 196/255) - self.blue3=(129/255, 140/255, 192/255) - self.blue4=(7/255, 137/255, 198/255) - self.black=(0 , 0 ,0 ) - self.white=(1,1,1) - - def alpha(self, color, a): - return ( color[0], color[1], color[2], a) - def colormaps(self,n, gamma=None): - gamma=1 if gamma is None else gamma - self.whitebluered=LinearSegmentedColormap.from_list("my_colormap", (self.white, self.blue2,self.blue1, self.red), N=n, gamma=gamma) - self.whiteblue=LinearSegmentedColormap.from_list("my_colormap", (self.white, self.blue2,self.blue1), N=n, gamma=gamma) - - def show(self): - for key in self.__dict__.keys(): - print(key) - #print(self.__dict__) + def __init__(self): + + self.red = (203 / 255, 32 / 255, 39 / 255) + self.green = (15 / 255, 150 / 255, 72 / 255) + self.orange = (247 / 255, 191 / 255, 88 / 255) + self.grey1 = (167 / 255, 180 / 255, 183 / 255) + self.grey = self.grey1 + self.grey2 = (123 / 255, 121 / 255, 125 / 255) + self.grey3 = (72 / 255, 70 / 255, 77 / 255) + + self.blue1 = (18 / 255, 78 / 255, 153 / 255) + self.blue = self.blue1 + self.blue2 = (85 / 255, 133 / 255, 196 / 255) + self.blue3 = (129 / 255, 140 / 255, 192 / 255) + self.blue4 = (7 / 255, 137 / 255, 198 / 255) + self.black = (0, 0, 0) + self.white = (1, 1, 1) + + def alpha(self, color, a): + return (color[0], color[1], color[2], a) + + def colormaps(self, n, gamma=None): + gamma = 1 if gamma is None else gamma + self.whitebluered = LinearSegmentedColormap.from_list( + "my_colormap", + (self.white, self.blue2, self.blue1, self.red), + N=n, + gamma=gamma, + ) + self.whiteblue = LinearSegmentedColormap.from_list( + "my_colormap", (self.white, self.blue2, self.blue1), N=n, gamma=gamma + ) + + def show(self): + for key in self.__dict__.keys(): + print(key) + # print(self.__dict__) # funny massage + class figure_axis_xy: - """define standart XY Plot with reduced grafics""" + """define standart XY Plot with reduced grafics""" + + def __init__( + self, + x_size=None, + y_size=None, + view_scale=None, + size_tuple=None, + fig_scale=None, + container=False, + dpi=180, + ): + import matplotlib.pyplot as plt + import string - def __init__(self,x_size=None,y_size=None,view_scale=None, size_tuple=None , fig_scale=None, container=False, dpi=180): - import matplotlib.pyplot as plt - import string + if size_tuple is not None: + xsize, ysize = size_tuple[0], size_tuple[1] + else: + xsize = x_size if x_size is not None else 8 + ysize = y_size if y_size is not None else 5 - if size_tuple is not None: - xsize, ysize =size_tuple[0], size_tuple[1] - else: - xsize=x_size if x_size is not None else 8 - ysize=y_size if y_size is not None else 5 + viewscale = view_scale if view_scale is not None else 0.5 + fig_scale = fig_scale if fig_scale is not None else 1 - viewscale=view_scale if view_scale is not None else 0.5 - fig_scale=fig_scale if fig_scale is not None else 1 + self.label_letters = iter([i + ") " for i in list(string.ascii_lowercase)]) - self.label_letters=iter([i+') ' for i in list(string.ascii_lowercase)]) + if container: + self.fig = plt.figure( + edgecolor="None", + dpi=dpi * viewscale, + figsize=(xsize * fig_scale, ysize * fig_scale), + facecolor="w", + ) + else: + self.fig, self.ax = plt.subplots( + num=None, + figsize=(xsize * fig_scale, ysize * fig_scale), + dpi=dpi * viewscale, + facecolor="w", + edgecolor="None", + ) + + def make_clear_weak(self): + # turn off axis spine to the right + # self.fig.tight_layout() + self.ax.spines["right"].set_color("none") + self.ax.yaxis.tick_left() # only ticks on the left side + self.ax.spines["top"].set_color("none") + self.ax.xaxis.tick_bottom() # only ticks on the left side + + def make_clear(self): + self.make_clear_weak() + + def make_clear_strong(self): + # turn off axis spine to the right + # self.fig.tight_layout() + self.ax.spines["right"].set_color("none") + self.ax.spines["left"].set_color("none") + self.ax.yaxis.tick_left() # only ticks on the left side + self.ax.spines["top"].set_color("none") + self.ax.spines["bottom"].set_color("none") + self.ax.xaxis.tick_bottom() # only ticks on the left side + + def tight(self): + # turn off axis spine to the right + self.fig.tight_layout() + + def label(self, x="x", y="y", t=None): + + self.ax.set_xlabel(x) + self.ax.set_ylabel(y) + self.ax.set_title(t, y=1.04) + + def save(self, name=None, path=None, verbose=True): + import datetime + + savepath = ( + path + if path is not None + else os.path.join(os.path.dirname(os.path.realpath("__file__")), "plot/") + ) + if not os.path.exists(savepath): + os.makedirs(savepath) + # os.makedirs(savepath, exist_ok=True) + name = ( + name + if name is not None + else datetime.date.today().strftime("%Y%m%d_%I%M%p") + ) + # print(savepath) + # print(name) + extension = ".pdf" + full_name = (os.path.join(savepath, name)) + extension + # print(full_name) + self.fig.savefig(full_name, bbox_inches="tight", format="pdf", dpi=180) + if verbose: + print("save at: " + name) + + def save_pup(self, name=None, path=None, verbose=True): + import datetime + import re + + name = re.sub("\.", "_", name) + + savepath = ( + path + if path is not None + else os.path.join(os.path.dirname(os.path.realpath("__file__")), "plot/") + ) + if not os.path.exists(savepath): + os.makedirs(savepath) + # os.makedirs(savepath, exist_ok=True) + name = ( + name + if name is not None + else datetime.date.today().strftime("%Y%m%d_%I%M%p") + ) + # print(savepath) + # print(name) + extension = ".pdf" + full_name = (os.path.join(savepath, name)) + extension + # print(full_name) + self.fig.savefig(full_name, bbox_inches="tight", format="pdf", dpi=300) + if verbose: + print("save at: ", full_name) + + def save_light(self, name=None, path=None, verbose=True): + import datetime + + savepath = ( + path + if path is not None + else os.path.join(os.path.dirname(os.path.realpath("__file__")), "plot/") + ) + if not os.path.exists(savepath): + os.makedirs(savepath) + # os.makedirs(savepath, exist_ok=True) + name = ( + name + if name is not None + else datetime.date.today().strftime("%Y%m%d_%I%M%p") + ) + # print(savepath) + # print(name) + extension = ".png" + full_name = (os.path.join(savepath, name)) + extension + # print(full_name) + self.fig.savefig(full_name, bbox_inches="tight", format="png", dpi=180) + if verbose: + print("save with: ", name) - if container: - self.fig=plt.figure(edgecolor='None',dpi=dpi*viewscale,figsize=(xsize*fig_scale, ysize*fig_scale),facecolor='w') - else: - self.fig, self.ax=plt.subplots(num=None, figsize=(xsize*fig_scale, ysize*fig_scale), dpi=dpi*viewscale, facecolor='w', edgecolor='None') - - def make_clear_weak(self): - #turn off axis spine to the right - #self.fig.tight_layout() - self.ax.spines['right'].set_color("none") - self.ax.yaxis.tick_left() # only ticks on the left side - self.ax.spines['top'].set_color("none") - self.ax.xaxis.tick_bottom() # only ticks on the left side - def make_clear(self): - self.make_clear_weak() - - def make_clear_strong(self): - #turn off axis spine to the right - #self.fig.tight_layout() - self.ax.spines['right'].set_color("none") - self.ax.spines['left'].set_color("none") - self.ax.yaxis.tick_left() # only ticks on the left side - self.ax.spines['top'].set_color("none") - self.ax.spines['bottom'].set_color("none") - self.ax.xaxis.tick_bottom() # only ticks on the left side - - def tight(self): - #turn off axis spine to the right - self.fig.tight_layout() - - def label(self, x='x',y='y',t=None): - - self.ax.set_xlabel(x) - self.ax.set_ylabel(y) - self.ax.set_title(t, y=1.04) - - def save(self,name=None,path=None, verbose=True): - import datetime - - savepath=path if path is not None else os.path.join(os.path.dirname(os.path.realpath('__file__')),'plot/') - if not os.path.exists(savepath): - os.makedirs(savepath) - #os.makedirs(savepath, exist_ok=True) - name=name if name is not None else datetime.date.today().strftime("%Y%m%d_%I%M%p") - #print(savepath) - #print(name) - extension='.pdf' - full_name= (os.path.join(savepath,name)) + extension - #print(full_name) - self.fig.savefig(full_name, bbox_inches='tight', format='pdf', dpi=180) - if verbose: - print('save at: '+name) - - def save_pup(self,name=None,path=None, verbose=True): - import datetime - import re - name=re.sub("\.", '_', name) - - savepath=path if path is not None else os.path.join(os.path.dirname(os.path.realpath('__file__')),'plot/') - if not os.path.exists(savepath): - os.makedirs(savepath) - #os.makedirs(savepath, exist_ok=True) - name=name if name is not None else datetime.date.today().strftime("%Y%m%d_%I%M%p") - #print(savepath) - #print(name) - extension='.pdf' - full_name= (os.path.join(savepath,name)) + extension - #print(full_name) - self.fig.savefig(full_name, bbox_inches='tight', format='pdf', dpi=300) - if verbose: - print('save at: ',full_name) - - def save_light(self,name=None,path=None, verbose=True): - import datetime - - savepath=path if path is not None else os.path.join(os.path.dirname(os.path.realpath('__file__')),'plot/') - if not os.path.exists(savepath): - os.makedirs(savepath) - #os.makedirs(savepath, exist_ok=True) - name=name if name is not None else datetime.date.today().strftime("%Y%m%d_%I%M%p") - #print(savepath) - #print(name) - extension='.png' - full_name= (os.path.join(savepath,name)) + extension - #print(full_name) - self.fig.savefig(full_name, bbox_inches='tight', format='png', dpi=180) - if verbose: - print('save with: ',name) class subplot_routines(figure_axis_xy): - def __init__(self, ax): - self.ax=ax - -class plot_sprecta: - def __init__(self,fs, Xdata,sample_unit=None,data_unit=None): - - self.fs=fs - self.Xdata=Xdata - self.sample_unit=sample_unit if sample_unit is not None else 'df' - self.data_unit=data_unit if data_unit is not None else 'X' - - - def linear(self, Color='b', fig_scale=2, fax='f'): - self.F=figure_axis_xy(fig_scale=fig_scale) - if fax=='f': - xax=self.fs - xlabelstr=('f (' + self.sample_unit+ ')') - elif fax=='w': - xax=2*np.pi*self.fs - xlabelstr=('w (rad ' + self.sample_unit+ ')') - - self.line=plt.plot(xax[1:],(self.Xdata[1:]), Color=Color) - - plt.ylabel(('|X|^2/f (' + self.data_unit + '^2/' + self.sample_unit+ ')')) - plt.xlabel(xlabelstr) - plt.xlim(xax[1] ,xax[-1]) - - self.F.make_clear() - plt.grid() - return self.F - - def power_linear(self, Color='b', fax='f'): - self.F=figure_axis_xy(fig_scale=2) - if fax=='f': - xax=self.fs - xlabelstr=('f (' + self.sample_unit+ ')') - elif fax=='w': - xax=2*np.pi*self.fs - xlabelstr=('w (rad ' + self.sample_unit+ ')') - - self.line=plt.plot(xax[1:],10*np.log10(self.Xdata[1:]), Color=Color) - - plt.ylabel(('Power db(' + self.data_unit + '^2/' + self.sample_unit+ ')')) - plt.xlabel(xlabelstr) - #plt.set_xlabel(fax) - #plt.xlim(np.log(fax[1]) ,np.log(fax[-1])) - plt.xlim(xax[1] ,xax[-1]) - #self.F.ax.set_xscale('log') - #self.F.ax.semilogx() - #ax1.set_xticks([20, 300, 500]) - self.F.make_clear() - plt.grid() - - def loglog(self, Color='b', fax='f'): - self.F=figure_axis_xy(fig_scale=2) - if fax=='f': - xax=self.fs - xlabelstr=('f (' + self.sample_unit+ ')') - elif fax=='w': - xax=2*np.pi*self.fs - xlabelstr=('w (rad ' + self.sample_unit+ ')') - - self.line=plt.loglog(xax[1:],(self.Xdata[1:]), Color=Color) - - plt.ylabel(('|X|^2/f (' + self.data_unit + '^2/' + self.sample_unit+ ')')) - plt.xlabel(xlabelstr) - plt.xlim(xax[1] ,xax[-1]) - - self.F.make_clear() - plt.grid() - - def power(self, Color='b', fig_scale=2, fax='f'): - self.F=figure_axis_xy(fig_scale=fig_scale) - if fax=='f': - xax=self.fs - xlabelstr=('f (' + self.sample_unit+ ')') - elif fax=='w': - xax=2*np.pi*self.fs - xlabelstr=('w (rad ' + self.sample_unit+ ')') - - self.line=plt.semilogx(xax[1:],10*np.log10(self.Xdata[1:]), Color=Color) - - plt.ylabel(('Power db(' + self.data_unit + '^2/' + self.sample_unit+ ')')) - plt.xlabel(xlabelstr) - #plt.set_xlabel(xax) - #plt.xlim(np.log(xax[1]) ,np.log(xax[-1])) - plt.xlim(xax[1] ,xax[-1]) - #self.F.ax.set_xscale('log') - #self.F.ax.semilogx() - #ax1.set_xticks([20, 300, 500]) - self.F.make_clear() - plt.grid() - -class plot_periodogram: - def __init__(self,time,fs, data,clevs=None, sample_unit=None, data_unit=None, - ylim=None, time_unit=None, cmap=None): - self.fs=fs[1:] - self.time=time - self.data=data[:,1:] - self.clevs=clevs - self.sample_unit=sample_unit if sample_unit is not None else 'df' - - self.data_unit=data_unit if data_unit is not None else 'X' - self.time_unit=time_unit if time_unit is not None else 'dt' - - self.cmap=cmap if cmap is not None else plt.cm.ocean_r - self.ylim=ylim if ylim is not None else [fs[0],fs[-1]] - - def loglog(self): - self.F=figure_axis_xy(fig_scale=2) - - plt.loglog(self.fs[1:],(self.Xdata[1:])) - - plt.ylabel(('|X|^2/f (' + self.data_unit + '^2/' + self.sample_unit+ ')')) - plt.xlabel(('f (' + self.sample_unit+ ')')) - plt.xlim(self.fs[1] ,self.fs[-1]) - - self.F.make_clear() - plt.grid() - - def linear(self): - self.F=figure_axis_xy(10,4,fig_scale=2) - dd=10*np.log10(self.data[:-2,:]).T - - self.clevs=self.clevs if self.clevs is not None else clevels(dd) - self.F.ax.set_yscale("log", nonposy='clip') - tt = self.time.astype(DT.datetime) - - self.cs=plt.contourf(tt[:-2], self.fs[:],dd, self.clevs,cmap=self.cmap) - #self.cs=plt.pcolormesh(self.time[:-2], self.fs[:],dd,cmap=self.cmap,shading='gouraud') - print(self.clevs) - plt.ylabel(('Power db(' + self.data_unit + '^2/' + self.sample_unit+ ')')) - plt.xlabel(('f (' + self.sample_unit+ ')')) - self.cbar= plt.colorbar(self.cs,pad=0.01)#, Location='right')# - self.cbar.ax.aspect=100 - self.cbar.outline.set_linewidth(0) - self.cbar.set_label('('+self.data_unit+')') - - - ax = plt.gca() - #Set y-lim - ax.set_ylim(self.ylim[0], self.ylim[1]) - - #format X-Axis - ax.xaxis_date() - Month = dates.MonthLocator() - Day = dates.DayLocator(interval=5)#bymonthday=range(1,32) - dfmt = dates.DateFormatter('%y-%b') - - ax.xaxis.set_major_locator(Month) - ax.xaxis.set_major_formatter(dfmt) - ax.xaxis.set_minor_locator(Day) - - # Set both ticks to be outside - ax.tick_params(which = 'both', direction = 'out') - ax.tick_params('both', length=6, width=1, which='major') - ax.tick_params('both', length=3, width=1, which='minor') - - # Make grid white - ax.grid() - gridlines = ax.get_xgridlines() + ax.get_ygridlines() - - for line in gridlines: - line.set_color('white') - #line.set_linestyle('-') - - def power(self, anomalie=False): - self.F=figure_axis_xy(10,4,fig_scale=2) - dd=10*np.log10(self.data[:-1,:]) - - if anomalie is True: - dd_tmp=dd.mean(axis=0).repeat(self.time.size-1) - dd=dd- dd_tmp.reshape(self.fs.size,self.time.size-1).T - dd=dd - - self.clevs=self.clevs if self.clevs is not None else clevels(dd) - self.F.ax.set_yscale("log", nonposy='clip') - tt = self.time.astype(DT.datetime) - - print(tt[:-1].shape, self.fs[:].shape,dd.T.shape) - self.cs=plt.contourf(tt[:-1], self.fs[:],dd.T, self.clevs,cmap=self.cmap) - self.x=np.arange(0,tt[:-1].size) - #self.cs=plt.pcolormesh(self.time[:-2], self.fs[:],dd,cmap=self.cmap,shading='gouraud') - print(self.clevs) - plt.xlabel('Time') - plt.ylabel(('f (' + self.sample_unit+ ')')) - self.cbar= plt.colorbar(self.cs,pad=0.01)#, Location='right')# - self.cbar.ax.aspect=100 - self.cbar.outline.set_linewidth(0) - self.cbar.set_label('Power db(' + self.data_unit + '^2/f ') - - - ax = plt.gca() - #Set y-lim - ax.set_ylim(self.ylim[0], self.ylim[1]) - - #format X-Axis - ax.xaxis_date() - Month = dates.MonthLocator() - Day = dates.DayLocator(interval=5)#bymonthday=range(1,32) - dfmt = dates.DateFormatter('%y-%b') - - ax.xaxis.set_major_locator(Month) - ax.xaxis.set_major_formatter(dfmt) - ax.xaxis.set_minor_locator(Day) - - # Set both ticks to be outside - ax.tick_params(which = 'both', direction = 'out') - ax.tick_params('both', length=6, width=1, which='major') - ax.tick_params('both', length=3, width=1, which='minor') - - # Make grid white - ax.grid() - gridlines = ax.get_xgridlines() + ax.get_ygridlines() - - for line in gridlines: - line.set_color('white') - line.set_linestyle('--') - def imshow(self, shading=None, downscale_fac=None, anomalie=False, downscale_type=None, fig_size=None, ax=None,cbar=True): - nopower=True - self.power_imshow(shading, downscale_fac , anomalie, downscale_type, fig_size , nopower, ax=ax, cbar=cbar) - if cbar: - self.cbar.set_label('Power (' + self.data_unit + '^2/f )') - def power_imshow(self, shading=None, downscale_fac=None, anomalie=False, - downscale_type=None, fig_size=None , nopower=False, ax=None, cbar=True): - import matplotlib.colors as colors - from matplotlib import dates - import time - import scipy.signal as signal - import matplotlib.ticker as ticker - - shading='gouraud' if shading is True else 'flat' - fig_size=[10,4] if fig_size is None else fig_size - if ax: - assert type(ax) is tuple, "put ax as tuple ax=(ax,F)" - self.F=ax[1] - ax_local=ax[0] - else: - self.F=figure_axis_xy(fig_size[0], fig_size[1], fig_scale=2) - ax_local=self.F.ax + def __init__(self, ax): + self.ax = ax - if nopower is True: - dd=self.data - else: - dd=10*np.log10(self.data[:-1,:]) - if anomalie is True: - dd_tmp=dd.mean(axis=0).repeat(self.time.size-1) - dd=dd- dd_tmp.reshape(self.fs.size,self.time.size-1).T - - self.clevs=self.clevs if self.clevs is not None else clevels(dd) - - norm = colors.BoundaryNorm(boundaries=self.clevs, ncolors=256) - - tt = self.time - - #tvec=np.arange(0,tt.size,1) - ax_local.set_yscale("log", nonposy='clip') - - if downscale_fac is not None: - if downscale_type =='inter': - fn=[] - for yr in np.arange(0,self.fs.size,downscale_fac): - fn.append(np.mean(self.fs[yr:yr+downscale_fac])) - else: - - ddn=np.empty((self.time.size-1)) - fsn_p=gen_log_space(self.fs.size,int(np.round(self.fs.size/downscale_fac))) - fsn_p_run=np.append(fsn_p,fsn_p[-1]) - dd=dd.T - #print(ddn.shape, fsn_p.shape) - for fr in np.arange(0,fsn_p.size,1): - #print(np.mean(dd[fsn_p[fr]:fsn_p[fr+1], :],axis=0).shape) - - ddn=np.vstack((ddn, np.mean(dd[fsn_p_run[fr]:fsn_p_run[fr+1], :],axis=0))) - ddn=np.delete(ddn, 0,0) - #print(ddn.shape) - dd2=ddn - fn=self.fs[fsn_p] +class plot_sprecta: + def __init__(self, fs, Xdata, sample_unit=None, data_unit=None): - if nopower is True: - tt=tt - else: - tt=tt[:-1] - #print(dd2.shape, fn.shape, tt.shape) + self.fs = fs + self.Xdata = Xdata + self.sample_unit = sample_unit if sample_unit is not None else "df" + self.data_unit = data_unit if data_unit is not None else "X" - else: - if nopower is True: - tt=tt - else: - tt=tt[:-1] - dd2=dd.T - fn=self.fs - - if isinstance(tt[0], np.datetime64): - print('time axis is numpy.datetime64, converted to number for plotting') - ttt=dates.date2num(tt.astype(DT.datetime)) - #print(ttt) - elif isinstance(tt[0], np.timedelta64): - print('time axis is numpy.timedelta64, converted to number for plotting') - #print(tt) - ttt=tt - else: - #print(type(tt[0])) - #print(tt) - - print('time axis is not converted') - ttt=tt - - MT.stats_format(dd2) - self.cs=plt.pcolormesh(ttt,fn ,dd2,cmap=self.cmap , norm=norm, - shading=shading)#, origin='lower',aspect='auto', - #interpolation='none', - #extent=[tvec.min(),tvec.max(),self.fs.min(),self.fs.max()]) - - #self.F.ax.set_yscale("log", nonposy='clip') - #self.cs=plt.pcolormesh(self.time[:-2], self.fs[:],dd,cmap=self.cmap,shading='gouraud') - #print(self.clevs) - plt.ylabel(('f (' + self.sample_unit+ ')')) - if cbar is True: - self.cbar= plt.colorbar(self.cs,pad=0.01)#, Location='right')# - self.cbar.ax.aspect=100 - self.cbar.outline.set_linewidth(0) - self.cbar.set_label('Power db(' + self.data_unit + '^2/f ') - - ax =ax_local#plt.gca() - if isinstance(tt[0], np.datetime64): - plt.xlabel('Time') - #Set y-lim - ax.set_ylim(self.ylim[0], self.ylim[1]) - ax.set_xlim(ttt[0], ttt[-1]) - - #format X-Axis - #ax.xaxis_date() - Month = dates.MonthLocator() - Day = dates.DayLocator(interval=5)#bymonthday=range(1,32) - dfmt = dates.DateFormatter('%b/%y') - - ax.xaxis.set_major_locator(Month) - ax.xaxis.set_major_formatter(dfmt) - ax.xaxis.set_minor_locator(Day) - elif isinstance(tt[0], np.float64): - plt.xlabel('Time') - #Set y-lim - ax.set_ylim(self.ylim[0], self.ylim[1]) - ax.set_xlim(ttt[0], ttt[-1]) - - #format X-Axis - #ax.xaxis_date() - Month = dates.MonthLocator() - Day = dates.DayLocator(interval=5)#bymonthday=range(1,32) - dfmt = dates.DateFormatter('%b/%y') - - ax.xaxis.set_major_locator(Month) - ax.xaxis.set_major_formatter(dfmt) - ax.xaxis.set_minor_locator(Day) - else: - plt.xlabel('Time (' + self.time_unit+ ')') - ax.set_ylim(self.ylim[0], self.ylim[1]) - ax.xaxis.set_major_locator(ticker.MultipleLocator(5)) - ax.xaxis.set_minor_locator(ticker.MultipleLocator(1)) - - #ax.xaxis.set_major_formatter(ticker.FormatStrFormatter('%0.1f')) - - # Set both ticks to be outside - ax.tick_params(which = 'both', direction = 'out') - ax.tick_params('both', length=6, width=1, which='major') - ax.tick_params('both', length=3, width=1, which='minor') - - # Make grid white - ax.grid() - self.ax=ax - gridlines = ax.get_xgridlines() + ax.get_ygridlines() - - for line in gridlines: - line.set_color('white') - #line.set_linestyle('-') - - self.x=ttt - - - def linear_imshow(self, shading=None, downscale_fac=None, anomalie=False, downscale_type=None, fig_size=None , nopower=False, ax=None): - import matplotlib.colors as colors - from matplotlib import dates - import time - import scipy.signal as signal - import matplotlib.ticker as ticker - - shading='gouraud' if shading is True else 'flat' - fig_size=[10,4] if fig_size is None else fig_size - if ax: - assert type(ax) is tuple, "put ax as tuple ax=(ax,F)" - self.F=ax[1] - ax_local=ax[0] - else: - self.F=figure_axis_xy(fig_size[0], fig_size[1], fig_scale=2) - ax_local=self.F.ax + def linear(self, Color="b", fig_scale=2, fax="f"): + self.F = figure_axis_xy(fig_scale=fig_scale) + if fax == "f": + xax = self.fs + xlabelstr = "f (" + self.sample_unit + ")" + elif fax == "w": + xax = 2 * np.pi * self.fs + xlabelstr = "w (rad " + self.sample_unit + ")" - if nopower is True: - dd=self.data - else: - dd=10*np.log10(self.data[:-1,:]) + self.line = plt.plot(xax[1:], (self.Xdata[1:]), Color=Color) - if anomalie is True: - dd_tmp=dd.mean(axis=0).repeat(self.time.size-1) - dd=dd- dd_tmp.reshape(self.fs.size,self.time.size-1).T + plt.ylabel(("|X|^2/f (" + self.data_unit + "^2/" + self.sample_unit + ")")) + plt.xlabel(xlabelstr) + plt.xlim(xax[1], xax[-1]) - self.clevs=self.clevs if self.clevs is not None else clevels(dd) + self.F.make_clear() + plt.grid() + return self.F + + def power_linear(self, Color="b", fax="f"): + self.F = figure_axis_xy(fig_scale=2) + if fax == "f": + xax = self.fs + xlabelstr = "f (" + self.sample_unit + ")" + elif fax == "w": + xax = 2 * np.pi * self.fs + xlabelstr = "w (rad " + self.sample_unit + ")" + + self.line = plt.plot(xax[1:], 10 * np.log10(self.Xdata[1:]), Color=Color) + + plt.ylabel(("Power db(" + self.data_unit + "^2/" + self.sample_unit + ")")) + plt.xlabel(xlabelstr) + # plt.set_xlabel(fax) + # plt.xlim(np.log(fax[1]) ,np.log(fax[-1])) + plt.xlim(xax[1], xax[-1]) + # self.F.ax.set_xscale('log') + # self.F.ax.semilogx() + # ax1.set_xticks([20, 300, 500]) + self.F.make_clear() + plt.grid() - norm = colors.BoundaryNorm(boundaries=self.clevs, ncolors=256) + def loglog(self, Color="b", fax="f"): + self.F = figure_axis_xy(fig_scale=2) + if fax == "f": + xax = self.fs + xlabelstr = "f (" + self.sample_unit + ")" + elif fax == "w": + xax = 2 * np.pi * self.fs + xlabelstr = "w (rad " + self.sample_unit + ")" - tt = self.time + self.line = plt.loglog(xax[1:], (self.Xdata[1:]), Color=Color) - #tvec=np.arange(0,tt.size,1) - self.F.ax.set_yscale("log", nonposy='clip') + plt.ylabel(("|X|^2/f (" + self.data_unit + "^2/" + self.sample_unit + ")")) + plt.xlabel(xlabelstr) + plt.xlim(xax[1], xax[-1]) - if downscale_fac is not None: - if downscale_type =='inter': - fn=[] - for yr in np.arange(0,self.fs.size,downscale_fac): - fn.append(np.mean(self.fs[yr:yr+downscale_fac])) - else: + self.F.make_clear() + plt.grid() - ddn=np.empty((self.time.size-1)) - fsn_p=gen_log_space(self.fs.size,int(np.round(self.fs.size/downscale_fac))) - fsn_p_run=np.append(fsn_p,fsn_p[-1]) - dd=dd.T - #print(ddn.shape, fsn_p.shape) - for fr in np.arange(0,fsn_p.size,1): - #print(np.mean(dd[fsn_p[fr]:fsn_p[fr+1], :],axis=0).shape) + def power(self, Color="b", fig_scale=2, fax="f"): + self.F = figure_axis_xy(fig_scale=fig_scale) + if fax == "f": + xax = self.fs + xlabelstr = "f (" + self.sample_unit + ")" + elif fax == "w": + xax = 2 * np.pi * self.fs + xlabelstr = "w (rad " + self.sample_unit + ")" + + self.line = plt.semilogx(xax[1:], 10 * np.log10(self.Xdata[1:]), Color=Color) + + plt.ylabel(("Power db(" + self.data_unit + "^2/" + self.sample_unit + ")")) + plt.xlabel(xlabelstr) + # plt.set_xlabel(xax) + # plt.xlim(np.log(xax[1]) ,np.log(xax[-1])) + plt.xlim(xax[1], xax[-1]) + # self.F.ax.set_xscale('log') + # self.F.ax.semilogx() + # ax1.set_xticks([20, 300, 500]) + self.F.make_clear() + plt.grid() - ddn=np.vstack((ddn, np.mean(dd[fsn_p_run[fr]:fsn_p_run[fr+1], :],axis=0))) - ddn=np.delete(ddn, 0,0) - #print(ddn.shape) - dd2=ddn - fn=self.fs[fsn_p] - if nopower is True: - tt=tt - else: - tt=tt[:-1] - #print(dd2.shape, fn.shape, tt.shape) +class plot_periodogram: + def __init__( + self, + time, + fs, + data, + clevs=None, + sample_unit=None, + data_unit=None, + ylim=None, + time_unit=None, + cmap=None, + ): + self.fs = fs[1:] + self.time = time + self.data = data[:, 1:] + self.clevs = clevs + self.sample_unit = sample_unit if sample_unit is not None else "df" + + self.data_unit = data_unit if data_unit is not None else "X" + self.time_unit = time_unit if time_unit is not None else "dt" + + self.cmap = cmap if cmap is not None else plt.cm.ocean_r + self.ylim = ylim if ylim is not None else [fs[0], fs[-1]] + + def loglog(self): + self.F = figure_axis_xy(fig_scale=2) + + plt.loglog(self.fs[1:], (self.Xdata[1:])) + + plt.ylabel(("|X|^2/f (" + self.data_unit + "^2/" + self.sample_unit + ")")) + plt.xlabel(("f (" + self.sample_unit + ")")) + plt.xlim(self.fs[1], self.fs[-1]) + + self.F.make_clear() + plt.grid() - else: - if nopower is True: - tt=tt - else: - tt=tt[:-1] - dd2=dd.T - fn=self.fs - - if isinstance(tt[0], np.datetime64): - print('numpy.datetime64') - ttt=dates.date2num(tt.astype(DT.datetime)) - #print(ttt) - elif isinstance(tt[0], np.timedelta64): - print('numpy.timedelta64') - #print(tt) - ttt=tt - else: - #print(type(tt[0])) - #print(tt) + def linear(self): + self.F = figure_axis_xy(10, 4, fig_scale=2) + dd = 10 * np.log10(self.data[:-2, :]).T + + self.clevs = self.clevs if self.clevs is not None else clevels(dd) + self.F.ax.set_yscale("log", nonposy="clip") + tt = self.time.astype(DT.datetime) + + self.cs = plt.contourf(tt[:-2], self.fs[:], dd, self.clevs, cmap=self.cmap) + # self.cs=plt.pcolormesh(self.time[:-2], self.fs[:],dd,cmap=self.cmap,shading='gouraud') + print(self.clevs) + plt.ylabel(("Power db(" + self.data_unit + "^2/" + self.sample_unit + ")")) + plt.xlabel(("f (" + self.sample_unit + ")")) + self.cbar = plt.colorbar(self.cs, pad=0.01) # , Location='right')# + self.cbar.ax.aspect = 100 + self.cbar.outline.set_linewidth(0) + self.cbar.set_label("(" + self.data_unit + ")") + + ax = plt.gca() + # Set y-lim + ax.set_ylim(self.ylim[0], self.ylim[1]) + + # format X-Axis + ax.xaxis_date() + Month = dates.MonthLocator() + Day = dates.DayLocator(interval=5) # bymonthday=range(1,32) + dfmt = dates.DateFormatter("%y-%b") + + ax.xaxis.set_major_locator(Month) + ax.xaxis.set_major_formatter(dfmt) + ax.xaxis.set_minor_locator(Day) + + # Set both ticks to be outside + ax.tick_params(which="both", direction="out") + ax.tick_params("both", length=6, width=1, which="major") + ax.tick_params("both", length=3, width=1, which="minor") + + # Make grid white + ax.grid() + gridlines = ax.get_xgridlines() + ax.get_ygridlines() + + for line in gridlines: + line.set_color("white") + # line.set_linestyle('-') + + def power(self, anomalie=False): + self.F = figure_axis_xy(10, 4, fig_scale=2) + dd = 10 * np.log10(self.data[:-1, :]) + + if anomalie is True: + dd_tmp = dd.mean(axis=0).repeat(self.time.size - 1) + dd = dd - dd_tmp.reshape(self.fs.size, self.time.size - 1).T + dd = dd + + self.clevs = self.clevs if self.clevs is not None else clevels(dd) + self.F.ax.set_yscale("log", nonposy="clip") + tt = self.time.astype(DT.datetime) + + print(tt[:-1].shape, self.fs[:].shape, dd.T.shape) + self.cs = plt.contourf(tt[:-1], self.fs[:], dd.T, self.clevs, cmap=self.cmap) + self.x = np.arange(0, tt[:-1].size) + # self.cs=plt.pcolormesh(self.time[:-2], self.fs[:],dd,cmap=self.cmap,shading='gouraud') + print(self.clevs) + plt.xlabel("Time") + plt.ylabel(("f (" + self.sample_unit + ")")) + self.cbar = plt.colorbar(self.cs, pad=0.01) # , Location='right')# + self.cbar.ax.aspect = 100 + self.cbar.outline.set_linewidth(0) + self.cbar.set_label("Power db(" + self.data_unit + "^2/f ") + + ax = plt.gca() + # Set y-lim + ax.set_ylim(self.ylim[0], self.ylim[1]) + + # format X-Axis + ax.xaxis_date() + Month = dates.MonthLocator() + Day = dates.DayLocator(interval=5) # bymonthday=range(1,32) + dfmt = dates.DateFormatter("%y-%b") + + ax.xaxis.set_major_locator(Month) + ax.xaxis.set_major_formatter(dfmt) + ax.xaxis.set_minor_locator(Day) + + # Set both ticks to be outside + ax.tick_params(which="both", direction="out") + ax.tick_params("both", length=6, width=1, which="major") + ax.tick_params("both", length=3, width=1, which="minor") + + # Make grid white + ax.grid() + gridlines = ax.get_xgridlines() + ax.get_ygridlines() + + for line in gridlines: + line.set_color("white") + line.set_linestyle("--") + + def imshow( + self, + shading=None, + downscale_fac=None, + anomalie=False, + downscale_type=None, + fig_size=None, + ax=None, + cbar=True, + ): + nopower = True + self.power_imshow( + shading, + downscale_fac, + anomalie, + downscale_type, + fig_size, + nopower, + ax=ax, + cbar=cbar, + ) + if cbar: + self.cbar.set_label("Power (" + self.data_unit + "^2/f )") + + def power_imshow( + self, + shading=None, + downscale_fac=None, + anomalie=False, + downscale_type=None, + fig_size=None, + nopower=False, + ax=None, + cbar=True, + ): + import matplotlib.colors as colors + from matplotlib import dates + import time + import scipy.signal as signal + import matplotlib.ticker as ticker + + shading = "gouraud" if shading is True else "flat" + fig_size = [10, 4] if fig_size is None else fig_size + if ax: + assert type(ax) is tuple, "put ax as tuple ax=(ax,F)" + self.F = ax[1] + ax_local = ax[0] + else: + self.F = figure_axis_xy(fig_size[0], fig_size[1], fig_scale=2) + ax_local = self.F.ax - print('something else') - ttt=tt + if nopower is True: + dd = self.data + else: + dd = 10 * np.log10(self.data[:-1, :]) + if anomalie is True: + dd_tmp = dd.mean(axis=0).repeat(self.time.size - 1) + dd = dd - dd_tmp.reshape(self.fs.size, self.time.size - 1).T - self.cs=plt.pcolormesh(ttt,fn ,dd2,cmap=self.cmap , norm=norm, - shading=shading)#, origin='lower',aspect='auto', - #interpolation='none', - #extent=[tvec.min(),tvec.max(),self.fs.min(),self.fs.max()]) + self.clevs = self.clevs if self.clevs is not None else clevels(dd) - #self.F.ax.set_yscale("log", nonposy='clip') - #self.cs=plt.pcolormesh(self.time[:-2], self.fs[:],dd,cmap=self.cmap,shading='gouraud') - #print(self.clevs) + norm = colors.BoundaryNorm(boundaries=self.clevs, ncolors=256) + tt = self.time - plt.ylabel(('f (' + self.sample_unit+ ')')) - self.cbar= plt.colorbar(self.cs,pad=0.01)#, Location='right')# - self.cbar.ax.aspect=100 - self.cbar.outline.set_linewidth(0) - self.cbar.set_label('Power db(' + self.data_unit + '^2/f ') + # tvec=np.arange(0,tt.size,1) + ax_local.set_yscale("log", nonposy="clip") - ax = plt.gca() - if isinstance(tt[0], np.datetime64): - plt.xlabel('Time') - #Set y-lim - ax.set_ylim(self.ylim[0], self.ylim[1]) - ax.set_xlim(ttt[0], ttt[-1]) + if downscale_fac is not None: + if downscale_type == "inter": + fn = [] + for yr in np.arange(0, self.fs.size, downscale_fac): + fn.append(np.mean(self.fs[yr : yr + downscale_fac])) + else: - #format X-Axis - #ax.xaxis_date() - Month = dates.MonthLocator() - Day = dates.DayLocator(interval=5)#bymonthday=range(1,32) - dfmt = dates.DateFormatter('%b/%y') + ddn = np.empty((self.time.size - 1)) + fsn_p = gen_log_space( + self.fs.size, int(np.round(self.fs.size / downscale_fac)) + ) + fsn_p_run = np.append(fsn_p, fsn_p[-1]) + dd = dd.T + # print(ddn.shape, fsn_p.shape) + for fr in np.arange(0, fsn_p.size, 1): + # print(np.mean(dd[fsn_p[fr]:fsn_p[fr+1], :],axis=0).shape) + + ddn = np.vstack( + (ddn, np.mean(dd[fsn_p_run[fr] : fsn_p_run[fr + 1], :], axis=0)) + ) + ddn = np.delete(ddn, 0, 0) + # print(ddn.shape) + dd2 = ddn + fn = self.fs[fsn_p] - ax.xaxis.set_major_locator(Month) - ax.xaxis.set_major_formatter(dfmt) - ax.xaxis.set_minor_locator(Day) + if nopower is True: + tt = tt else: - plt.xlabel('Time (' + self.time_unit+ ')') - ax.set_ylim(self.ylim[0], self.ylim[1]) - ax.xaxis.set_major_locator(ticker.MultipleLocator(5)) - ax.xaxis.set_minor_locator(ticker.MultipleLocator(1)) - - #ax.xaxis.set_major_formatter(ticker.FormatStrFormatter('%0.1f')) - - # Set both ticks to be outside - ax.tick_params(which = 'both', direction = 'out') - ax.tick_params('both', length=6, width=1, which='major') - ax.tick_params('both', length=3, width=1, which='minor') - - # Make grid white - ax.grid() - self.ax=ax - gridlines = ax.get_xgridlines() + ax.get_ygridlines() - - for line in gridlines: - line.set_color('white') - #line.set_linestyle('-') - - self.x=np.arange(0,ttt.size+1) - - - def set_xaxis_to_days(self, **kwargs): - set_timeaxis_days(self.ax, **kwargs) + tt = tt[:-1] + # print(dd2.shape, fn.shape, tt.shape) + else: + if nopower is True: + tt = tt + else: + tt = tt[:-1] + dd2 = dd.T + fn = self.fs + + if isinstance(tt[0], np.datetime64): + print("time axis is numpy.datetime64, converted to number for plotting") + ttt = dates.date2num(tt.astype(DT.datetime)) + # print(ttt) + elif isinstance(tt[0], np.timedelta64): + print("time axis is numpy.timedelta64, converted to number for plotting") + # print(tt) + ttt = tt + else: + # print(type(tt[0])) + # print(tt) + + print("time axis is not converted") + ttt = tt + + MT.stats_format(dd2) + self.cs = plt.pcolormesh( + ttt, fn, dd2, cmap=self.cmap, norm=norm, shading=shading + ) # , origin='lower',aspect='auto', + # interpolation='none', + # extent=[tvec.min(),tvec.max(),self.fs.min(),self.fs.max()]) + + # self.F.ax.set_yscale("log", nonposy='clip') + # self.cs=plt.pcolormesh(self.time[:-2], self.fs[:],dd,cmap=self.cmap,shading='gouraud') + # print(self.clevs) + plt.ylabel(("f (" + self.sample_unit + ")")) + if cbar is True: + self.cbar = plt.colorbar(self.cs, pad=0.01) # , Location='right')# + self.cbar.ax.aspect = 100 + self.cbar.outline.set_linewidth(0) + self.cbar.set_label("Power db(" + self.data_unit + "^2/f ") + + ax = ax_local # plt.gca() + if isinstance(tt[0], np.datetime64): + plt.xlabel("Time") + # Set y-lim + ax.set_ylim(self.ylim[0], self.ylim[1]) + ax.set_xlim(ttt[0], ttt[-1]) + + # format X-Axis + # ax.xaxis_date() + Month = dates.MonthLocator() + Day = dates.DayLocator(interval=5) # bymonthday=range(1,32) + dfmt = dates.DateFormatter("%b/%y") + + ax.xaxis.set_major_locator(Month) + ax.xaxis.set_major_formatter(dfmt) + ax.xaxis.set_minor_locator(Day) + elif isinstance(tt[0], np.float64): + plt.xlabel("Time") + # Set y-lim + ax.set_ylim(self.ylim[0], self.ylim[1]) + ax.set_xlim(ttt[0], ttt[-1]) + + # format X-Axis + # ax.xaxis_date() + Month = dates.MonthLocator() + Day = dates.DayLocator(interval=5) # bymonthday=range(1,32) + dfmt = dates.DateFormatter("%b/%y") + + ax.xaxis.set_major_locator(Month) + ax.xaxis.set_major_formatter(dfmt) + ax.xaxis.set_minor_locator(Day) + else: + plt.xlabel("Time (" + self.time_unit + ")") + ax.set_ylim(self.ylim[0], self.ylim[1]) + ax.xaxis.set_major_locator(ticker.MultipleLocator(5)) + ax.xaxis.set_minor_locator(ticker.MultipleLocator(1)) + + # ax.xaxis.set_major_formatter(ticker.FormatStrFormatter('%0.1f')) + + # Set both ticks to be outside + ax.tick_params(which="both", direction="out") + ax.tick_params("both", length=6, width=1, which="major") + ax.tick_params("both", length=3, width=1, which="minor") + + # Make grid white + ax.grid() + self.ax = ax + gridlines = ax.get_xgridlines() + ax.get_ygridlines() + + for line in gridlines: + line.set_color("white") + # line.set_linestyle('-') + + self.x = ttt + + def linear_imshow( + self, + shading=None, + downscale_fac=None, + anomalie=False, + downscale_type=None, + fig_size=None, + nopower=False, + ax=None, + ): + import matplotlib.colors as colors + from matplotlib import dates + import time + import scipy.signal as signal + import matplotlib.ticker as ticker + + shading = "gouraud" if shading is True else "flat" + fig_size = [10, 4] if fig_size is None else fig_size + if ax: + assert type(ax) is tuple, "put ax as tuple ax=(ax,F)" + self.F = ax[1] + ax_local = ax[0] + else: + self.F = figure_axis_xy(fig_size[0], fig_size[1], fig_scale=2) + ax_local = self.F.ax -class plot_polarspectra: - def __init__(self,f, thetas, data,unit=None, data_type='fraction' ,lims=None, verbose=False): + if nopower is True: + dd = self.data + else: + dd = 10 * np.log10(self.data[:-1, :]) - self.f=f - self.data=data - self.thetas=thetas + if anomalie is True: + dd_tmp = dd.mean(axis=0).repeat(self.time.size - 1) + dd = dd - dd_tmp.reshape(self.fs.size, self.time.size - 1).T - #self.sample_unit=sample_unit if sample_unit is not None else 'df' - self.unit=unit if unit is not None else 'X' + self.clevs = self.clevs if self.clevs is not None else clevels(dd) - # decided on freq limit - lims=[self.f.min(),self.f.max()] if lims is None else lims - self.lims= 1.0 /lims[1], 1.0/ lims[0] + norm = colors.BoundaryNorm(boundaries=self.clevs, ncolors=256) - freq_sel_bool=cut_nparray(self.f, lims[0], lims[1] ) + tt = self.time - self.min=np.nanmin(data[freq_sel_bool,:])#*0.5e-17 - self.max=np.nanmax(data[freq_sel_bool,:]) - if verbose: - print(str(self.min), str(self.max) ) + # tvec=np.arange(0,tt.size,1) + self.F.ax.set_yscale("log", nonposy="clip") - self.ylabels=np.arange(10, 100, 20) - self.data_type=data_type - if data_type == 'fraction': - self.clevs=np.linspace(0.01, self.max*.5, 21) - elif data_type == 'energy': - self.ctrs_min=self.min+self.min*.05 - #self.clevs=np.linspace(self.min, self.max, 21) - self.clevs=np.linspace(self.min+self.min*.05, self.max*.60, 21) + if downscale_fac is not None: + if downscale_type == "inter": + fn = [] + for yr in np.arange(0, self.fs.size, downscale_fac): + fn.append(np.mean(self.fs[yr : yr + downscale_fac])) + else: + ddn = np.empty((self.time.size - 1)) + fsn_p = gen_log_space( + self.fs.size, int(np.round(self.fs.size / downscale_fac)) + ) + fsn_p_run = np.append(fsn_p, fsn_p[-1]) + dd = dd.T + # print(ddn.shape, fsn_p.shape) + for fr in np.arange(0, fsn_p.size, 1): + # print(np.mean(dd[fsn_p[fr]:fsn_p[fr+1], :],axis=0).shape) + + ddn = np.vstack( + (ddn, np.mean(dd[fsn_p_run[fr] : fsn_p_run[fr + 1], :], axis=0)) + ) + ddn = np.delete(ddn, 0, 0) + # print(ddn.shape) + dd2 = ddn + fn = self.fs[fsn_p] - def linear(self, radial_axis='period', circles =None, ax=None, fontsize= 10 ): + if nopower is True: + tt = tt + else: + tt = tt[:-1] + # print(dd2.shape, fn.shape, tt.shape) - if ax is None: - ax = plt.subplot(111, polar=True) - self.title=plt.suptitle(' Polar Spectrum', y=0.95, x=0.5 , horizontalalignment='center') + else: + if nopower is True: + tt = tt else: - ax=ax - ax.set_theta_direction(-1) #left turned postive - ax.set_theta_zero_location("N") - - #cm=plt.cm.get_cmap('Reds') - #=plt.cm.get_cmap('PuBu') + tt = tt[:-1] + dd2 = dd.T + fn = self.fs + + if isinstance(tt[0], np.datetime64): + print("numpy.datetime64") + ttt = dates.date2num(tt.astype(DT.datetime)) + # print(ttt) + elif isinstance(tt[0], np.timedelta64): + print("numpy.timedelta64") + # print(tt) + ttt = tt + else: + # print(type(tt[0])) + # print(tt) + + print("something else") + ttt = tt + + self.cs = plt.pcolormesh( + ttt, fn, dd2, cmap=self.cmap, norm=norm, shading=shading + ) # , origin='lower',aspect='auto', + # interpolation='none', + # extent=[tvec.min(),tvec.max(),self.fs.min(),self.fs.max()]) + + # self.F.ax.set_yscale("log", nonposy='clip') + # self.cs=plt.pcolormesh(self.time[:-2], self.fs[:],dd,cmap=self.cmap,shading='gouraud') + # print(self.clevs) + + plt.ylabel(("f (" + self.sample_unit + ")")) + self.cbar = plt.colorbar(self.cs, pad=0.01) # , Location='right')# + self.cbar.ax.aspect = 100 + self.cbar.outline.set_linewidth(0) + self.cbar.set_label("Power db(" + self.data_unit + "^2/f ") + + ax = plt.gca() + if isinstance(tt[0], np.datetime64): + plt.xlabel("Time") + # Set y-lim + ax.set_ylim(self.ylim[0], self.ylim[1]) + ax.set_xlim(ttt[0], ttt[-1]) + + # format X-Axis + # ax.xaxis_date() + Month = dates.MonthLocator() + Day = dates.DayLocator(interval=5) # bymonthday=range(1,32) + dfmt = dates.DateFormatter("%b/%y") + + ax.xaxis.set_major_locator(Month) + ax.xaxis.set_major_formatter(dfmt) + ax.xaxis.set_minor_locator(Day) + else: + plt.xlabel("Time (" + self.time_unit + ")") + ax.set_ylim(self.ylim[0], self.ylim[1]) + ax.xaxis.set_major_locator(ticker.MultipleLocator(5)) + ax.xaxis.set_minor_locator(ticker.MultipleLocator(1)) - #ylabels=np.arange(10, 100, 20) - #ylabels = ([ 10, '', 20,'', 30,'', 40]) - ax.set_yticks(self.ylabels) - ax.set_yticklabels(' '+str(y)+ ' s' for y in self.ylabels) + # ax.xaxis.set_major_formatter(ticker.FormatStrFormatter('%0.1f')) - ## Set titles and colorbar - #plt.title(STID+' | '+p + ' | '+start_date+' | '+end_date, y=1.11, horizontalalignment='left') + # Set both ticks to be outside + ax.tick_params(which="both", direction="out") + ax.tick_params("both", length=6, width=1, which="major") + ax.tick_params("both", length=3, width=1, which="minor") - grid=ax.grid(color='k', alpha=.5, linestyle='--', linewidth=.5) + # Make grid white + ax.grid() + self.ax = ax + gridlines = ax.get_xgridlines() + ax.get_ygridlines() - if self.data_type == 'fraction': - cm=plt.cm.RdYlBu_r #brewer2mpl.get_map( 'RdYlBu','Diverging', 4, reverse=True).mpl_colormap - colorax = ax.contourf(self.thetas, 1/self.f, self.data, self.clevs, cmap=cm, zorder=1)# ,cmap=cm)#, vmin=self.ctrs_min) - elif self.data_type == 'energy': - cm=plt.cm.Paired#brewer2mpl.get_map( 'Paired','Qualitative', 8).mpl_colormap + for line in gridlines: + line.set_color("white") + # line.set_linestyle('-') - cm.set_under='w' - cm.set_bad='w' - colorax = ax.contourf(self.thetas, 1/self.f, self.data, self.clevs,cmap=cm, zorder=1)#, vmin=self.ctrs_min) - #divider = make_axes_locatable(ax) - #cax = divider.append_axes("right", size="5%", pad=0.05) + self.x = np.arange(0, ttt.size + 1) - if circles is not None: - theta = np.linspace(0, 2 * np.pi, 360) - r1 =theta*0+circles[0] - r2 =theta*0+circles[1] - plt.plot(theta, r1, c='red', alpha=0.6,linewidth=1, zorder=10) - plt.plot(theta, r2, c='red', alpha=0.6,linewidth=1, zorder=10) + def set_xaxis_to_days(self, **kwargs): + set_timeaxis_days(self.ax, **kwargs) - cbar = plt.colorbar(colorax, fraction=0.046, pad=0.06, orientation="horizontal") - if self.data_type == 'fraction': - cbar.set_label('Fraction of Energy', rotation=0)#, fontsize=fontsize) - elif self.data_type == 'energy': - cbar.set_label('Energy Density ('+self.unit+')', rotation=0)#, fontsize=fontsize) - cbar.ax.get_yaxis().labelpad = 30 - cbar.outline.set_visible(False) - #cbar.ticks. - #cbar.outline.clipbox - degrange = range(0,360,30) - lines, labels = plt.thetagrids(degrange, labels=None)#, frac = 1.07) +class plot_polarspectra: + def __init__( + self, f, thetas, data, unit=None, data_type="fraction", lims=None, verbose=False + ): - for line in lines: - #L=line.get_xgridlines - line.set_linewidth(5) - #line.set_linestyle(':') + self.f = f + self.data = data + self.thetas = thetas - ax.set_ylim(self.lims) - ax.spines['polar'].set_color("none") - ax.set_rlabel_position(87) - self.ax=ax + # self.sample_unit=sample_unit if sample_unit is not None else 'df' + self.unit = unit if unit is not None else "X" + # decided on freq limit + lims = [self.f.min(), self.f.max()] if lims is None else lims + self.lims = 1.0 / lims[1], 1.0 / lims[0] + freq_sel_bool = cut_nparray(self.f, lims[0], lims[1]) -def plot_scatter_2d(intersect_chain, slope_chain, xname='intersect', yname='slope'): + self.min = np.nanmin(data[freq_sel_bool, :]) # *0.5e-17 + self.max = np.nanmax(data[freq_sel_bool, :]) + if verbose: + print(str(self.min), str(self.max)) + + self.ylabels = np.arange(10, 100, 20) + self.data_type = data_type + if data_type == "fraction": + self.clevs = np.linspace(0.01, self.max * 0.5, 21) + elif data_type == "energy": + self.ctrs_min = self.min + self.min * 0.05 + # self.clevs=np.linspace(self.min, self.max, 21) + self.clevs = np.linspace(self.min + self.min * 0.05, self.max * 0.60, 21) + + def linear(self, radial_axis="period", circles=None, ax=None, fontsize=10): + + if ax is None: + ax = plt.subplot(111, polar=True) + self.title = plt.suptitle( + " Polar Spectrum", y=0.95, x=0.5, horizontalalignment="center" + ) + else: + ax = ax + ax.set_theta_direction(-1) # left turned postive + ax.set_theta_zero_location("N") + + # cm=plt.cm.get_cmap('Reds') + # =plt.cm.get_cmap('PuBu') + + # ylabels=np.arange(10, 100, 20) + # ylabels = ([ 10, '', 20,'', 30,'', 40]) + ax.set_yticks(self.ylabels) + ax.set_yticklabels(" " + str(y) + " s" for y in self.ylabels) + + ## Set titles and colorbar + # plt.title(STID+' | '+p + ' | '+start_date+' | '+end_date, y=1.11, horizontalalignment='left') + + grid = ax.grid(color="k", alpha=0.5, linestyle="--", linewidth=0.5) + + if self.data_type == "fraction": + cm = ( + plt.cm.RdYlBu_r + ) # brewer2mpl.get_map( 'RdYlBu','Diverging', 4, reverse=True).mpl_colormap + colorax = ax.contourf( + self.thetas, 1 / self.f, self.data, self.clevs, cmap=cm, zorder=1 + ) # ,cmap=cm)#, vmin=self.ctrs_min) + elif self.data_type == "energy": + cm = ( + plt.cm.Paired + ) # brewer2mpl.get_map( 'Paired','Qualitative', 8).mpl_colormap + + cm.set_under = "w" + cm.set_bad = "w" + colorax = ax.contourf( + self.thetas, 1 / self.f, self.data, self.clevs, cmap=cm, zorder=1 + ) # , vmin=self.ctrs_min) + # divider = make_axes_locatable(ax) + # cax = divider.append_axes("right", size="5%", pad=0.05) + + if circles is not None: + theta = np.linspace(0, 2 * np.pi, 360) + r1 = theta * 0 + circles[0] + r2 = theta * 0 + circles[1] + plt.plot(theta, r1, c="red", alpha=0.6, linewidth=1, zorder=10) + plt.plot(theta, r2, c="red", alpha=0.6, linewidth=1, zorder=10) + + cbar = plt.colorbar(colorax, fraction=0.046, pad=0.06, orientation="horizontal") + if self.data_type == "fraction": + cbar.set_label("Fraction of Energy", rotation=0) # , fontsize=fontsize) + elif self.data_type == "energy": + cbar.set_label( + "Energy Density (" + self.unit + ")", rotation=0 + ) # , fontsize=fontsize) + cbar.ax.get_yaxis().labelpad = 30 + cbar.outline.set_visible(False) + # cbar.ticks. + # cbar.outline.clipbox + degrange = range(0, 360, 30) + + lines, labels = plt.thetagrids(degrange, labels=None) # , frac = 1.07) + + for line in lines: + # L=line.get_xgridlines + line.set_linewidth(5) + # line.set_linestyle(':') + + ax.set_ylim(self.lims) + ax.spines["polar"].set_color("none") + ax.set_rlabel_position(87) + self.ax = ax + + +def plot_scatter_2d(intersect_chain, slope_chain, xname="intersect", yname="slope"): from matplotlib import colors + col.colormaps(20) - xbins=np.linspace(intersect_chain.quantile(.025),intersect_chain.quantile(.975), 30) - xlim=(xbins[0], xbins[-1]) - ybins=np.linspace(slope_chain.quantile(.025), slope_chain.quantile(.975), 30) - ylim=(ybins[0], ybins[-1]) + xbins = np.linspace( + intersect_chain.quantile(0.025), intersect_chain.quantile(0.975), 30 + ) + xlim = (xbins[0], xbins[-1]) + ybins = np.linspace(slope_chain.quantile(0.025), slope_chain.quantile(0.975), 30) + ylim = (ybins[0], ybins[-1]) - S1 = plt.subplot2grid((3,3), (0, 0),facecolor='white', colspan=1, rowspan=2) + S1 = plt.subplot2grid((3, 3), (0, 0), facecolor="white", colspan=1, rowspan=2) - H=plt.hist(slope_chain, ybins, orientation='horizontal') + H = plt.hist(slope_chain, ybins, orientation="horizontal") plt.gca().invert_xaxis() plt.ylabel(yname) - S2 = plt.subplot2grid((3,3), (0, 1),facecolor='white', colspan=2, rowspan=2) - H=plt.hist2d(list(intersect_chain), list(slope_chain) , bins=(xbins, ybins), norm=colors.LogNorm(),cmap = col.white_base_bluegreen) - - S2 = plt.subplot2grid((3,3), (2, 1),facecolor='white', colspan=2, rowspan=1) - H=plt.hist(intersect_chain, xbins) + S2 = plt.subplot2grid((3, 3), (0, 1), facecolor="white", colspan=2, rowspan=2) + H = plt.hist2d( + list(intersect_chain), + list(slope_chain), + bins=(xbins, ybins), + norm=colors.LogNorm(), + cmap=col.white_base_bluegreen, + ) + + S2 = plt.subplot2grid((3, 3), (2, 1), facecolor="white", colspan=2, rowspan=1) + H = plt.hist(intersect_chain, xbins) plt.xlabel(xname) @@ -820,292 +951,338 @@ def set_timeaxis_days(ax, int1=1, int2=2, bymonthday=None): # int1 interval of the major (labeld) days # int2 intercal of the minar (only ticks) days - bymonthday=bymonthday if bymonthday is not None else range(1,32) + bymonthday = bymonthday if bymonthday is not None else range(1, 32) Month = dates.MonthLocator() - Month_dfmt = dates.DateFormatter('%b/%y') - Day = dates.DayLocator(interval=int2, bymonthday=bymonthday)#bymonthday=range(1,32) - Day_dfmt = dates.DateFormatter('%d') - Day2 = dates.DayLocator(interval=int1, bymonthday=bymonthday)#bymonthday=range(1,32) - Day2_dfmt = dates.DateFormatter('') + Month_dfmt = dates.DateFormatter("%b/%y") + Day = dates.DayLocator( + interval=int2, bymonthday=bymonthday + ) # bymonthday=range(1,32) + Day_dfmt = dates.DateFormatter("%d") + Day2 = dates.DayLocator( + interval=int1, bymonthday=bymonthday + ) # bymonthday=range(1,32) + Day2_dfmt = dates.DateFormatter("") ax.xaxis.set_major_locator(Day) ax.xaxis.set_major_formatter(Day_dfmt) ax.xaxis.set_minor_locator(Day2) ax.xaxis.set_minor_formatter(Day2_dfmt) + def log_power(data): - return 10*np.log10(data) + return 10 * np.log10(data) + def echo_dt(a, as_string=False): - string=str(a.astype('timedelta64[s]'))+'/'+str(a.astype('timedelta64[m]'))+'/'+str(a.astype('timedelta64[h]'))+'/'+str(a.astype('timedelta64[D]')) - #print(string) + string = ( + str(a.astype("timedelta64[s]")) + + "/" + + str(a.astype("timedelta64[m]")) + + "/" + + str(a.astype("timedelta64[h]")) + + "/" + + str(a.astype("timedelta64[D]")) + ) + # print(string) if as_string: return string else: print(string) + def easy_dtstr(a): - if a.astype('timedelta64[s]') < np.timedelta64(60,'s'): - return str(a.astype('timedelta64[s]')) - elif a.astype('timedelta64[m]') < np.timedelta64(60,'m'): - return str(a.astype('timedelta64[m]')) - elif a.astype('timedelta64[h]') < np.timedelta64(24,'h'): - return str(a.astype('timedelta64[h]')) - elif a.astype('timedelta64[D]') < np.timedelta64(365,'D'): - return str(a.astype('timedelta64[D]')) - elif a.astype('timedelta64[M]') < np.timedelta64(24,'M'): - return str(a.astype('timedelta64[M]')) + if a.astype("timedelta64[s]") < np.timedelta64(60, "s"): + return str(a.astype("timedelta64[s]")) + elif a.astype("timedelta64[m]") < np.timedelta64(60, "m"): + return str(a.astype("timedelta64[m]")) + elif a.astype("timedelta64[h]") < np.timedelta64(24, "h"): + return str(a.astype("timedelta64[h]")) + elif a.astype("timedelta64[D]") < np.timedelta64(365, "D"): + return str(a.astype("timedelta64[D]")) + elif a.astype("timedelta64[M]") < np.timedelta64(24, "M"): + return str(a.astype("timedelta64[M]")) else: - return str(a.astype('timedelta64[Y]')) + return str(a.astype("timedelta64[Y]")) -def clevels(data, dstep=None, symmetric=False ): - dstep=dstep if dstep is not None else 21 - mmax=np.ceil(np.nanmax(data)) - mmin=np.floor(np.nanmin(data)) +def clevels(data, dstep=None, symmetric=False): + dstep = dstep if dstep is not None else 21 + + mmax = np.ceil(np.nanmax(data)) + mmin = np.floor(np.nanmin(data)) if symmetric: - tmax=np.max([mmax, -mmin]) - clim=np.linspace(-tmax,tmax,dstep) + tmax = np.max([mmax, -mmin]) + clim = np.linspace(-tmax, tmax, dstep) else: - clim=np.linspace(mmin,mmax,dstep) + clim = np.linspace(mmin, mmax, dstep) return clim -def save_anyfig(fig,name=None,path=None): - import datetime - - savepath=path if path is not None else os.path.join(os.path.dirname(os.path.realpath('__file__')),'plot/') - if not os.path.exists(savepath): - os.makedirs(savepath) - name=name if name is not None else datetime.date.today().strftime("%Y%m%d_%I%M%p") - #print(savepath) - #print(name) - extension='.png' - full_name= (os.path.join(savepath,name)) + extension - #print(full_name) - fig.savefig(full_name, bbox_inches='tight', format='png', dpi=180) - print('save at: ',full_name) -#def downscale_2d(data,x,dx): +def save_anyfig(fig, name=None, path=None): + import datetime + + savepath = ( + path + if path is not None + else os.path.join(os.path.dirname(os.path.realpath("__file__")), "plot/") + ) + if not os.path.exists(savepath): + os.makedirs(savepath) + name = name if name is not None else datetime.date.today().strftime("%Y%m%d_%I%M%p") + # print(savepath) + # print(name) + extension = ".png" + full_name = (os.path.join(savepath, name)) + extension + # print(full_name) + fig.savefig(full_name, bbox_inches="tight", format="png", dpi=180) + print("save at: ", full_name) + + +# def downscale_2d(data,x,dx): # s1,s2 =data.shape # if s1 == x.size # data[] def read_cdo(file): cdo = Cdo() - G=cdo.readCdf(file).variables + G = cdo.readCdf(file).variables print(G.keys()) return G -def build_timestamp(time, unit, start, verbose=True): #G.time,'h','1900-01-01' ): - timestamp=np.datetime64(start)+time[:].astype('m8['+unit+']') - #print('timespan:', timestamp[0], '-', timestamp[-1]) + +def build_timestamp(time, unit, start, verbose=True): # G.time,'h','1900-01-01' ): + timestamp = np.datetime64(start) + time[:].astype("m8[" + unit + "]") + # print('timespan:', timestamp[0], '-', timestamp[-1]) if verbose is True: print(timestamp) return timestamp -def build_timestamp_v2(time, unit, start, verbose=True): #G.time,'h','1900-01-01' ): - timestamp=np.datetime64(start)+time[:].astype('datetime64[s]') - #print('timespan:', timestamp[0], '-', timestamp[-1]) + +def build_timestamp_v2(time, unit, start, verbose=True): # G.time,'h','1900-01-01' ): + timestamp = np.datetime64(start) + time[:].astype("datetime64[s]") + # print('timespan:', timestamp[0], '-', timestamp[-1]) if verbose is True: print(timestamp) return timestamp + def cut_nparray(var, low, high, verbose=False): if low < high: if low < var[0]: if verbose: - print('out of lower limit!') + print("out of lower limit!") if high > var[-1]: if verbose: - print('out of upper limit!') - print(high ,'>', var[-1]) - return (var >=low) & (var <=high) + print("out of upper limit!") + print(high, ">", var[-1]) + return (var >= low) & (var <= high) elif high < low: if high < var[0]: - print('limits flipped, out of lower limit!') + print("limits flipped, out of lower limit!") if low > var[-1]: - print('limits flipped, out of lower limit!') + print("limits flipped, out of lower limit!") - return (var >=high) & (var <=low) + return (var >= high) & (var <= low) elif high == low: if verbose: - print('find nearest') - a=var-low - return np.unravel_index(np.abs(a).argmin(),np.transpose(a.shape)) + print("find nearest") + a = var - low + return np.unravel_index(np.abs(a).argmin(), np.transpose(a.shape)) else: - print('error') + print("error") return -def boxmean(data, lon , lat, xlim, ylim):#, data_index, G.lon[:], G.lat[:], [160, -140+360], [-50 , -70]): - xs=lon.shape - ys=lat.shape +def boxmean( + data, lon, lat, xlim, ylim +): # , data_index, G.lon[:], G.lat[:], [160, -140+360], [-50 , -70]): + + xs = lon.shape + ys = lat.shape - xp=data.shape.index(xs[0]) - yp=data.shape.index(ys[0]) + xp = data.shape.index(xs[0]) + yp = data.shape.index(ys[0]) if xlim[0] <= xlim[1]: - xbool=(lon >=xlim[0]) & (lon <=xlim[1]) + xbool = (lon >= xlim[0]) & (lon <= xlim[1]) else: - xbool=(lon >=xlim[1]) & (lon <=xlim[0]) + xbool = (lon >= xlim[1]) & (lon <= xlim[0]) if ylim[0] <= ylim[1]: - ybool=(lat >=ylim[0]) & (lat <=ylim[1]) + ybool = (lat >= ylim[0]) & (lat <= ylim[1]) else: - ybool=(lat >=ylim[1]) & (lat <=ylim[0]) - - if (xp == 0 and yp ==1): - #print('0,1') - datan=data[xbool,:,:][:,ybool,:].mean() - - elif (xp == 0 and yp ==2): - #print('0,2') - datan=data[xbool,:,:][:,:,ybool] - elif (xp == 1 and yp ==0): - #print('1,0') - datan=data[:,xbool,:][ybool,:,:] - - elif (xp == 1 and yp ==2): - #print('1,2') - datan=data[:,xbool,:][:,:,ybool] - - elif (xp == 2 and yp ==0): - #print('2,0') - datan=data[:,:, xbool][ybool,:,:] - elif (xp == 2 and yp ==1): - #print('2,1') - datan=data[:,:, xbool][:,ybool,:] + ybool = (lat >= ylim[1]) & (lat <= ylim[0]) + + if xp == 0 and yp == 1: + # print('0,1') + datan = data[xbool, :, :][:, ybool, :].mean() + + elif xp == 0 and yp == 2: + # print('0,2') + datan = data[xbool, :, :][:, :, ybool] + elif xp == 1 and yp == 0: + # print('1,0') + datan = data[:, xbool, :][ybool, :, :] + + elif xp == 1 and yp == 2: + # print('1,2') + datan = data[:, xbool, :][:, :, ybool] + + elif xp == 2 and yp == 0: + # print('2,0') + datan = data[:, :, xbool][ybool, :, :] + elif xp == 2 and yp == 1: + # print('2,1') + datan = data[:, :, xbool][:, ybool, :] else: - print('arrays have not the same shape') + print("arrays have not the same shape") - print('new shape', datan.shape) + print("new shape", datan.shape) + + return np.nanmean(np.nanmean(datan, axis=xp), axis=yp).squeeze() # , xbool , ybool - return np.nanmean(np.nanmean(datan, axis=xp), axis=yp).squeeze() #, xbool , ybool def detrend(data, od=None, x=None, plot=False, verbose=False): # data data that should be detrended - #od order of polynomial - #x optional xaxis, otherwise equal distance is assument - #plot True for plot - od=0 if od is None else od + # od order of polynomial + # x optional xaxis, otherwise equal distance is assument + # plot True for plot + od = 0 if od is None else od if od == 0: - d_detrend=data-np.nanmean(data) - d_org=[] - dline=[] - + d_detrend = data - np.nanmean(data) + d_org = [] + dline = [] - elif od > 0 : - if verbose: print('assume data is equal dist. You can define option x= if not.') + elif od > 0: + if verbose: + print("assume data is equal dist. You can define option x= if not.") - d_org=data-np.nanmean(data) - x=np.arange(0,d_org.size,1) if x is None else x + d_org = data - np.nanmean(data) + x = np.arange(0, d_org.size, 1) if x is None else x - #print(np.isnan(x).sum(), np.isnan(d_org).sum()) + # print(np.isnan(x).sum(), np.isnan(d_org).sum()) idx = np.isfinite(x) & np.isfinite(d_org) - px=np.polyfit(x[idx], d_org[idx], od) - dline=np.polyval( px, x) - d_detrend = d_org -dline + px = np.polyfit(x[idx], d_org[idx], od) + dline = np.polyval(px, x) + d_detrend = d_org - dline if plot == True: import matplotlib.pyplot as plt - F=figure_axis_xy(15, 5) + + F = figure_axis_xy(15, 5) if od > 0: - plt.plot(d_org, Color='black') - plt.plot(dline, Color='black') - plt.plot(d_detrend, Color='r') + plt.plot(d_org, Color="black") + plt.plot(dline, Color="black") + plt.plot(d_detrend, Color="r") F.make_clear() plt.grid() - plt.legend(['org', 'line', 'normalized']) + plt.legend(["org", "line", "normalized"]) - - stats=dict() - stats['org']=d_org - stats['std']=np.nanstd(d_detrend) + stats = dict() + stats["org"] = d_org + stats["std"] = np.nanstd(d_detrend) if od > 0: - stats['line']=dline - stats['polynom order']=od - stats['polyvals']=px - if verbose: print(stats) - return d_detrend/np.nanstd(d_detrend) , stats + stats["line"] = dline + stats["polynom order"] = od + stats["polyvals"] = px + if verbose: + print(stats) + return d_detrend / np.nanstd(d_detrend), stats + def normalize(data): return detrend(data)[0] + + def nannormalize(data): - return ( data-np.nanmean(data) ) /np.nanstd(data) + return (data - np.nanmean(data)) / np.nanstd(data) + def runningvar(var, m, tailcopy=False): - m=int(m) - s =var.shape - if s[0] <= 2*m: - print('0 Dimension is smaller then averaging length') + m = int(m) + s = var.shape + if s[0] <= 2 * m: + print("0 Dimension is smaller then averaging length") return - rr=np.asarray(var)*np.nan - var_range=np.arange(m,int(s[0])-m-1,1) -# print(var_range) -# print(np.isfinite(var)) -# print(var_range[np.isfinite(var[m:int(s[0])-m-1])]) - for i in var_range[np.isfinite(var[m:int(s[0])-m-1])]: - #rm.append(var[i-m:i+m].mean()) - rr[int(i)]=np.nanvar(var[i-m:i+m]) + rr = np.asarray(var) * np.nan + var_range = np.arange(m, int(s[0]) - m - 1, 1) + # print(var_range) + # print(np.isfinite(var)) + # print(var_range[np.isfinite(var[m:int(s[0])-m-1])]) + for i in var_range[np.isfinite(var[m : int(s[0]) - m - 1])]: + # rm.append(var[i-m:i+m].mean()) + rr[int(i)] = np.nanvar(var[i - m : i + m]) if tailcopy: -# print('tailcopy') - rr[0:m]=rr[m+1] - rr[-m-1:-1]=rr[-m-2] + # print('tailcopy') + rr[0:m] = rr[m + 1] + rr[-m - 1 : -1] = rr[-m - 2] return rr + def runningstd(var, m, tailcopy=False): return np.sqrt(runningvar(var, m, tailcopy=tailcopy)) + def runningmean_wrap_around(var, m): - m=int(m) - s =var.shape - if s[0] <= 2*m: - print('0 Dimension is smaller then averaging length') + m = int(m) + s = var.shape + if s[0] <= 2 * m: + print("0 Dimension is smaller then averaging length") return - rr=np.asarray(var)*np.nan - var_range=np.arange(var.size) - #print(var_range[0], var_range[-1]) + rr = np.asarray(var) * np.nan + var_range = np.arange(var.size) + # print(var_range[0], var_range[-1]) for i in var_range: - #print(i, i-m, i+m) - #print(i, var.take(range(i-m, i+m ), mode='wrap')) - #print(i, var[i-m:i+m]) - #rm.append(var[i-m:i+m].mean()) - rr[int(i)]=np.nanmean(var.take(range(i-m, i+m ), mode='wrap')) + # print(i, i-m, i+m) + # print(i, var.take(range(i-m, i+m ), mode='wrap')) + # print(i, var[i-m:i+m]) + # rm.append(var[i-m:i+m].mean()) + rr[int(i)] = np.nanmean(var.take(range(i - m, i + m), mode="wrap")) return rr + def runningmean(var, m, tailcopy=False): - m=int(m) - s =var.shape - if s[0] <= 2*m: - print('0 Dimension is smaller then averaging length') + m = int(m) + s = var.shape + if s[0] <= 2 * m: + print("0 Dimension is smaller then averaging length") return - rr=np.asarray(var)*np.nan - #print(type(rr)) - var_range=np.arange(m,int(s[0])-m-1,1) -# print(var_range) -# print(np.isfinite(var)) -# print(var_range[np.isfinite(var[m:int(s[0])-m-1])]) - for i in var_range[np.isfinite(var[m:int(s[0])-m-1])]: - #rm.append(var[i-m:i+m].mean()) - rr[int(i)]=np.nanmean(var[i-m:i+m]) + rr = np.asarray(var) * np.nan + # print(type(rr)) + var_range = np.arange(m, int(s[0]) - m - 1, 1) + # print(var_range) + # print(np.isfinite(var)) + # print(var_range[np.isfinite(var[m:int(s[0])-m-1])]) + for i in var_range[np.isfinite(var[m : int(s[0]) - m - 1])]: + # rm.append(var[i-m:i+m].mean()) + rr[int(i)] = np.nanmean(var[i - m : i + m]) if tailcopy: -# print('tailcopy') - rr[0:m]=rr[m+1] - rr[-m-1:-1]=rr[-m-2] + # print('tailcopy') + rr[0:m] = rr[m + 1] + rr[-m - 1 : -1] = rr[-m - 2] return rr -def find_max_ts(data_org, threshold=None, jump=None, smooth=True, spreed=None, plot=False, nocopy=False, verbose=True): +def find_max_ts( + data_org, + threshold=None, + jump=None, + smooth=True, + spreed=None, + plot=False, + nocopy=False, + verbose=True, +): """ This function finds local minima in a 1-dimensional array by asking where the gradient of the data changes sign @@ -1136,699 +1313,757 @@ def find_max_ts(data_org, threshold=None, jump=None, smooth=True, spreed=None, p """ if nocopy: - data=data_org + data = data_org else: - data=np.copy(data_org) - spreed=2 if spreed is None else spreed + data = np.copy(data_org) + spreed = 2 if spreed is None else spreed if smooth is True: - data=runningmean(data,spreed) - #print(threshold is not None and threshold > np.nanmin(data)) - #if threshold is not None and numpy.ndarray + data = runningmean(data, spreed) + # print(threshold is not None and threshold > np.nanmin(data)) + # if threshold is not None and numpy.ndarray if threshold is not None and threshold > np.nanmin(data): - data[np.isnan(data)]=0 - data[data= index.size-2: - nc=1 + if i >= index.size - 2: + nc = 1 elif sum(c[i:] >= jump) == 0: - nc=c[i:].size + nc = c[i:].size else: -# print(np.nonzero(c[i:] >= jump)) - nc=np.nonzero(c[i:] >= jump)[0][0] - b=np.append(b, np.round(np.mean(index[i:i+nc+1]))).astype(int) - # print(nc, index[i:i+nc+1], ' new', np.round(np.mean(index[i:i+nc+1]))) - i=i+nc+1 + # print(np.nonzero(c[i:] >= jump)) + nc = np.nonzero(c[i:] >= jump)[0][0] + b = np.append(b, np.round(np.mean(index[i : i + nc + 1]))).astype(int) + # print(nc, index[i:i+nc+1], ' new', np.round(np.mean(index[i:i+nc+1]))) + i = i + nc + 1 else: - b=np.append(b, index[i]).astype(int) - # print(' ', index[i], ' new', index[i]) - i=i+1 + b = np.append(b, index[i]).astype(int) + # print(' ', index[i], ' new', index[i]) + i = i + 1 if verbose: - print('index, edited ts, edit ts (index), org_index') + print("index, edited ts, edit ts (index), org_index") return b, data, data[b], index + def spickes_to_nan(ts, nloop=None, spreed=1): - nloop=0 if nloop is None else nloop - i=0 + nloop = 0 if nloop is None else nloop + i = 0 while i <= nloop: ts.max() - #print(np.where(ts == ts.max())) - pa=np.where(ts == np.nanmax(ts))[0][0] - ts[pa-spreed:pa+spreed]=np.NaN - i=i+1 + # print(np.where(ts == ts.max())) + pa = np.where(ts == np.nanmax(ts))[0][0] + ts[pa - spreed : pa + spreed] = np.NaN + i = i + 1 return ts + def spickes_to_mean(ts, nloop=None, spreed=1, gaussian=True): - nloop=0 if nloop is None else nloop - i=0 - tsmean=ts.mean() - b=2*spreed - gaus=signal.gaussian(b, std=b/10) + nloop = 0 if nloop is None else nloop + i = 0 + tsmean = ts.mean() + b = 2 * spreed + gaus = signal.gaussian(b, std=b / 10) while i <= nloop: - #print(i) - #ts.max() - #print(np.where(ts == ts.max())) - tsabs=np.abs(ts) - tmax=np.nanmax(tsabs) - #print(tsabs, tmax) - pa=np.where(tsabs == tmax)[0][0] + # print(i) + # ts.max() + # print(np.where(ts == ts.max())) + tsabs = np.abs(ts) + tmax = np.nanmax(tsabs) + # print(tsabs, tmax) + pa = np.where(tsabs == tmax)[0][0] if gaussian: - tsm=np.mean([ts[pa-spreed],ts[pa+spreed]]) #ts[pa-spreed:pa+spreed] - #print(ts[pa-spreed:pa+spreed].shape) - #print((gaus*(tmax-tsm)).shape) - #print(np.shape(gaus*(tmax-tsm))) - le=int(pa-spreed) - ue=int(pa+spreed) - - ts[le:ue]=ts[le:ue]-gaus*(tmax-tsm) + tsm = np.mean([ts[pa - spreed], ts[pa + spreed]]) # ts[pa-spreed:pa+spreed] + # print(ts[pa-spreed:pa+spreed].shape) + # print((gaus*(tmax-tsm)).shape) + # print(np.shape(gaus*(tmax-tsm))) + le = int(pa - spreed) + ue = int(pa + spreed) + + ts[le:ue] = ts[le:ue] - gaus * (tmax - tsm) else: - #tsm=np.mean([ts[pa-spreed],ts[pa+spreed]]) #ts[pa-spreed:pa+spreed] - #print(ts[pa-spreed:pa+spreed].shape) - #print((gaus*(tmax-tsm)).shape) - #print(np.shape(gaus*(tmax-tsm))) - #print(len(ts)) - #print(pa+spreed) - if pa+spreed > len(ts): - le= int(pa-spreed) - ts[le:-1]=np.linspace(ts[le],ts[-1],len(ts[le:-1])) + # tsm=np.mean([ts[pa-spreed],ts[pa+spreed]]) #ts[pa-spreed:pa+spreed] + # print(ts[pa-spreed:pa+spreed].shape) + # print((gaus*(tmax-tsm)).shape) + # print(np.shape(gaus*(tmax-tsm))) + # print(len(ts)) + # print(pa+spreed) + if pa + spreed > len(ts): + le = int(pa - spreed) + ts[le:-1] = np.linspace(ts[le], ts[-1], len(ts[le:-1])) else: - le=int(pa-spreed) - ue=int(pa+spreed) - #print(le, ue) - ts[ le : ue ]=np.linspace(ts[le],ts[ue],len(ts[le:ue])) + le = int(pa - spreed) + ue = int(pa + spreed) + # print(le, ue) + ts[le:ue] = np.linspace(ts[le], ts[ue], len(ts[le:ue])) - i=i+1 + i = i + 1 return ts + ## Composites + class composite_data: - def __init__(self,var, index_weight=None): - #print(var.shape) - self.composites=var - self.comp_mean=np.nanmean(var, axis=0) - self.comp_std=np.nanstd(var, axis=0) - if var.shape[0] != 1: - self.comp_norm=self.comp_mean/self.comp_std - self.comp_sum=np.nansum(var, axis=0) - if index_weight is not None: - self.weight(index_weight) - - def weight(self,index_weight): - length=self.comp_mean.size - #print('composites ', self.composites.shape) - #print('index respeat',index_weight.shape,self.comp_mean.shape) - weight_mat=np.repeat(index_weight,length).reshape((self.composites.shape))/index_weight.mean() - #print(weight_mat.shape) - #print(self.composites.shape) - self.comp_weighted_mean=np.nanmean(self.composites*weight_mat, axis=0) - self.comp_weighted_norm=self.comp_weighted_mean/np.nanstd(self.comp_weighted_mean, axis=0) - - def bootstrap(self, ci=[2.5, 50, 97.5], reps=1000,): - # bootstrap MC integration - reps = 1000 - xb = np.random.choice(x, (n, reps), replace=True) - yb = 1/np.arange(1, n+1)[:, None] * np.cumsum(xb, axis=0) - upper, lower = np.percentile(yb, [2.5, 97.5], axis=1) + def __init__(self, var, index_weight=None): + # print(var.shape) + self.composites = var + self.comp_mean = np.nanmean(var, axis=0) + self.comp_std = np.nanstd(var, axis=0) + if var.shape[0] != 1: + self.comp_norm = self.comp_mean / self.comp_std + self.comp_sum = np.nansum(var, axis=0) + if index_weight is not None: + self.weight(index_weight) + + def weight(self, index_weight): + length = self.comp_mean.size + # print('composites ', self.composites.shape) + # print('index respeat',index_weight.shape,self.comp_mean.shape) + weight_mat = ( + np.repeat(index_weight, length).reshape((self.composites.shape)) + / index_weight.mean() + ) + # print(weight_mat.shape) + # print(self.composites.shape) + self.comp_weighted_mean = np.nanmean(self.composites * weight_mat, axis=0) + self.comp_weighted_norm = self.comp_weighted_mean / np.nanstd( + self.comp_weighted_mean, axis=0 + ) + + def bootstrap( + self, + ci=[2.5, 50, 97.5], + reps=1000, + ): + # bootstrap MC integration + reps = 1000 + xb = np.random.choice(x, (n, reps), replace=True) + yb = 1 / np.arange(1, n + 1)[:, None] * np.cumsum(xb, axis=0) + upper, lower = np.percentile(yb, [2.5, 97.5], axis=1) + class comp_iter: - def __init__(self, span, dt=None, unit=None): - self.span=list(span) - for s in self.span: - assert type(s) is int, "span is not an integrer!" - - self.length=-self.span[0]+self.span[1] - self.loop_iter=np.arange(0,self.length,1) - self.index_iter=np.arange(self.span[0],self.span[1],1) - if dt is not None: - self.dt=dt - self.time_iter=self.index_iter*dt - time_str=[] - time_iter_axis=[] - self.unit='h' if unit is None else unit - for p in self.loop_iter: - time_str=np.append(time_str,str(self.time_iter[p].astype('timedelta64['+unit+']'))) - #time_iter_axis=np.append(time_iter_axis,self.time_iter[p].astype('timedelta64['+dt_unit+']')) - - self.time_iter_string=time_str + def __init__(self, span, dt=None, unit=None): + self.span = list(span) + for s in self.span: + assert type(s) is int, "span is not an integrer!" + + self.length = -self.span[0] + self.span[1] + self.loop_iter = np.arange(0, self.length, 1) + self.index_iter = np.arange(self.span[0], self.span[1], 1) + if dt is not None: + self.dt = dt + self.time_iter = self.index_iter * dt + time_str = [] + time_iter_axis = [] + self.unit = "h" if unit is None else unit + for p in self.loop_iter: + time_str = np.append( + time_str, str(self.time_iter[p].astype("timedelta64[" + unit + "]")) + ) + # time_iter_axis=np.append(time_iter_axis,self.time_iter[p].astype('timedelta64['+dt_unit+']')) + + self.time_iter_string = time_str + class composite: - def __init__(self,index, time=None, weigthing=False, span=None): - """ Initial Class for bulding composite based on: - index position in the time vector 'time' - time np.array of datetime64 times - optional: - weighting amplitude of the indexes for weighted composties (default=false) - span lead lag composite in units of timesteps in time (default=None) - """ - self.index=index - self.iter_operate=None - span=0 if span is None else span - self.span=[0, span] if type(span) == int else span - - self.weigthing=weigthing if weigthing is not False else None - self.comp=dict() - if time is None: - print('timeaxis is not defined. Make sure that both timeseries have the same timestamp') - self.time_index=None - else: - self.time_index=time - self.dtstamp=MT.dt_form_timestamp(self.time_index) - - def build_iter(self, dt=None, dt_unit=None): - self.iter=comp_iter(self.span, dt=dt, unit=dt_unit) - self.iter_operate=self.iter if self.iter_operate is None else self.iter_operate - - def corse_iter(self, dt, unit=None): - '''build intereration aaxis and timestamps for corser resoltions''' - #np.array(C.span)*dt_old - #print(type(self.dt), type(dt) - - assert unit == self.iter.unit, "span units do not match! old unit=" + self.iter.unit +", new unit="+ unit - - span=[] - # convert iter.dt in right unit - dt_format=np.timedelta64(self.iter.dt,self.iter.unit ).astype('timedelta64['+unit+']').astype('float') - span_new=np.array(self.span)*dt_format/dt - print('old span=',self.span ) - print('new span=',span_new ) - - for s in span_new: - span.append(int(np.floor(s))) - - print(span) - self.iter2=comp_iter(span, dt , unit=unit) - - def iter_info(self): - self.iter_operate.__dict__ - - print('available iters') - if self.iter is not None: - print('self.iter') - self.iter.__dict__ - if self.iter2 is not None: - print('self.iter2') - self.iter2.__dict__ - - - def info(self): - print('index', self.index) - print('span', self.span) - print('weight', self.weigthing) - print('comp', self.comp.keys()) - - def transform_index_time(self, time_index, time_composite): - """ find nearest time index of compostite time compared to index times""" - index_composite=[] - for i in self.index: - #print('old:',i, ' ',time_index[i]) - t_diff=time_index[i]-time_composite - nindex=np.unravel_index(np.abs(t_diff).argmin(),t_diff.shape) - #print('new:',nindex,time_composite[nindex])#, t_diff[nindex[0]-5:nindex[0]+5]) - index_composite=np.append(index_composite,nindex) - return index_composite.astype(int) - - - def compose_ts(self,ts,name, time=None): - if time is not None: - if self.time_index is None: - print('timeaxis of index TS is not defined!') - return - else: - iindex=self.transform_index_time(self.time_index, time) + def __init__(self, index, time=None, weigthing=False, span=None): + """Initial Class for bulding composite based on: + index position in the time vector 'time' + time np.array of datetime64 times + optional: + weighting amplitude of the indexes for weighted composties (default=false) + span lead lag composite in units of timesteps in time (default=None) + """ + self.index = index + self.iter_operate = None + span = 0 if span is None else span + self.span = [0, span] if type(span) == int else span + + self.weigthing = weigthing if weigthing is not False else None + self.comp = dict() + if time is None: + print( + "timeaxis is not defined. Make sure that both timeseries have the same timestamp" + ) + self.time_index = None + else: + self.time_index = time + self.dtstamp = MT.dt_form_timestamp(self.time_index) + + def build_iter(self, dt=None, dt_unit=None): + self.iter = comp_iter(self.span, dt=dt, unit=dt_unit) + self.iter_operate = ( + self.iter if self.iter_operate is None else self.iter_operate + ) + + def corse_iter(self, dt, unit=None): + """build intereration aaxis and timestamps for corser resoltions""" + # np.array(C.span)*dt_old + # print(type(self.dt), type(dt) + + assert unit == self.iter.unit, ( + "span units do not match! old unit=" + self.iter.unit + ", new unit=" + unit + ) + + span = [] + # convert iter.dt in right unit + dt_format = ( + np.timedelta64(self.iter.dt, self.iter.unit) + .astype("timedelta64[" + unit + "]") + .astype("float") + ) + span_new = np.array(self.span) * dt_format / dt + print("old span=", self.span) + print("new span=", span_new) + + for s in span_new: + span.append(int(np.floor(s))) + + print(span) + self.iter2 = comp_iter(span, dt, unit=unit) + + def iter_info(self): + self.iter_operate.__dict__ + + print("available iters") + if self.iter is not None: + print("self.iter") + self.iter.__dict__ + if self.iter2 is not None: + print("self.iter2") + self.iter2.__dict__ + + def info(self): + print("index", self.index) + print("span", self.span) + print("weight", self.weigthing) + print("comp", self.comp.keys()) + + def transform_index_time(self, time_index, time_composite): + """find nearest time index of compostite time compared to index times""" + index_composite = [] + for i in self.index: + # print('old:',i, ' ',time_index[i]) + t_diff = time_index[i] - time_composite + nindex = np.unravel_index(np.abs(t_diff).argmin(), t_diff.shape) + # print('new:',nindex,time_composite[nindex])#, t_diff[nindex[0]-5:nindex[0]+5]) + index_composite = np.append(index_composite, nindex) + return index_composite.astype(int) + + def compose_ts(self, ts, name, time=None): + if time is not None: + if self.time_index is None: + print("timeaxis of index TS is not defined!") + return else: - iindex=self.index - - span=self.iter_operate.span - print(iindex) - if self.span != [0, 0]: - comp=np.empty((-span[0]+span[1]))#*np.NaN - self.length=comp.size - - for i in iindex: - if i+span[0] < 0: - print('i', i, 'span:', span[0], span[1] ) - print('left postion:', i+span[0]) - raise ValueError("composite span exceeds ts limits") - #print('right postion:',i+self.span[1]) - return -1 - elif i+span[1] > ts.size: - return -1 - print(i, span[0], span[1] ) - print('i', i, 'span:', span[0], span[1] ) - #print('left postion:', i+self.span[0]) - print('right postion:',i+span[1]) - raise ValueError("composite span exceeds ts limits") - #print('right postion:',i+self.span[1]) - return -1 - - #self.comp[i]=ts[i] - print('comp', comp.shape) - print('ts', ts[i+span[0]:i+span[1]].shape) - #print('ltjergij') - print(i, span[0], span[1] ) - comp=np.vstack((comp,ts[i+span[0]:i+span[1]])) - #comp[:,i]=((comp,ts[i+self.span[0]:i+self.span[1]])) - #print(ts[i]) - #print(comp) - #self.comp.append(ts[i]) - - comp=np.delete(comp,0,0) - comp1=composite_data(comp, self.weigthing) - self.comp[name]=comp1 + iindex = self.transform_index_time(self.time_index, time) + else: + iindex = self.index + + span = self.iter_operate.span + print(iindex) + if self.span != [0, 0]: + comp = np.empty((-span[0] + span[1])) # *np.NaN + self.length = comp.size + + for i in iindex: + if i + span[0] < 0: + print("i", i, "span:", span[0], span[1]) + print("left postion:", i + span[0]) + raise ValueError("composite span exceeds ts limits") + # print('right postion:',i+self.span[1]) + return -1 + elif i + span[1] > ts.size: + return -1 + print(i, span[0], span[1]) + print("i", i, "span:", span[0], span[1]) + # print('left postion:', i+self.span[0]) + print("right postion:", i + span[1]) + raise ValueError("composite span exceeds ts limits") + # print('right postion:',i+self.span[1]) + return -1 + + # self.comp[i]=ts[i] + print("comp", comp.shape) + print("ts", ts[i + span[0] : i + span[1]].shape) + # print('ltjergij') + print(i, span[0], span[1]) + comp = np.vstack((comp, ts[i + span[0] : i + span[1]])) + # comp[:,i]=((comp,ts[i+self.span[0]:i+self.span[1]])) + # print(ts[i]) + # print(comp) + # self.comp.append(ts[i]) + + comp = np.delete(comp, 0, 0) + comp1 = composite_data(comp, self.weigthing) + self.comp[name] = comp1 + else: + comp1 = composite_data(ts[iindex], self.weigthing) + self.comp[name] = comp1 + + def compose_2d(self, field, name, time=None): + if time is not None: + if self.time_index is None: + print("timeaxis of index TS is not defined!") + return else: - comp1=composite_data(ts[iindex], self.weigthing) - self.comp[name]=comp1 - - - def compose_2d(self,field, name, time=None): - if time is not None: - if self.time_index is None: - print('timeaxis of index TS is not defined!') - return + iindex = self.transform_index_time(self.time_index, time) + else: + iindex = self.index + span = self.iter_operate.span + + if span != [0, 0]: + print(-span[0] + span[1], field.shape[1]) + comp = np.empty((-span[0] + span[1], field.shape[1])) * np.NaN + self.length = -span[0] + span[1] + # print('comp shape',comp.shape) + for i in iindex: + if i + span[1] > field.shape[0]: + ff = field[i + span[0] : field.shape[0], :] + cc = np.empty((-span[0] + span[1], field.shape[1])) * np.NaN + cc[0 : ff.shape[0], :] = ff + comp = np.vstack((comp, cc)) + elif i + span[0] < 0: + ff = field[0 : i + span[1], :] + cc = np.empty((-span[0] + span[1], field.shape[1])) * np.NaN + cc[-ff.shape[0] - 1 : -1, :] = ff + comp = np.vstack((comp, cc)) else: - iindex=self.transform_index_time(self.time_index, time) - else: - iindex=self.index - span=self.iter_operate.span - - if span != [0, 0]: - print(-span[0]+span[1],field.shape[1]) - comp=np.empty((-span[0]+span[1],field.shape[1]))*np.NaN - self.length=-span[0]+span[1] - # print('comp shape',comp.shape) - for i in iindex: - if i+span[1] > field.shape[0]: - ff=field[i+span[0]:field.shape[0],:] - cc=np.empty((-span[0]+span[1],field.shape[1]))*np.NaN - cc[0:ff.shape[0],:]=ff - comp=np.vstack((comp,cc)) - elif i+span[0] < 0: - ff=field[0:i+span[1],:] - cc=np.empty((-span[0]+span[1],field.shape[1]))*np.NaN - cc[-ff.shape[0]-1:-1,:]=ff - comp=np.vstack((comp,cc)) - else: - comp=np.vstack((comp,field[i+span[0]:i+span[1],:])) - #print(comp.shape) - - #print('comp shape',comp.shape) - #print(iindex.size*self.length ,iindex.size, self.length,field.shape[1]) - comp=comp.reshape( iindex.size+1,self.length,field.shape[1]) - #comp.shape - comp=np.delete(comp,0,0) - - comp1=composite_data(comp, self.weigthing) - #b2_mean=b2.mean(axis=0) - self.comp[name]=comp1 - else: - print('no span defined') - comp=field[iindex,:] - - comp1=composite_data(comp, self.weigthing) - self.comp[name]=comp1 + comp = np.vstack((comp, field[i + span[0] : i + span[1], :])) + # print(comp.shape) + + # print('comp shape',comp.shape) + # print(iindex.size*self.length ,iindex.size, self.length,field.shape[1]) + comp = comp.reshape(iindex.size + 1, self.length, field.shape[1]) + # comp.shape + comp = np.delete(comp, 0, 0) + + comp1 = composite_data(comp, self.weigthing) + # b2_mean=b2.mean(axis=0) + self.comp[name] = comp1 + else: + print("no span defined") + comp = field[iindex, :] + comp1 = composite_data(comp, self.weigthing) + self.comp[name] = comp1 - def compose_field(self,field, name, time=None): - if time is not None: - if self.time_index is None: - print('timeaxis of index TS is not defined!') - return - else: - iindex=self.transform_index_time(self.time_index, time) + def compose_field(self, field, name, time=None): + if time is not None: + if self.time_index is None: + print("timeaxis of index TS is not defined!") + return else: - iindex=self.index - - span=self.iter_operate.span - #print(field.shape) - #print('iindex', iindex) - if span != [0, 0]: - comp=np.empty((-span[0]+span[1],field.shape[1],field.shape[2]))*np.NaN - self.length=-span[0]+span[1] - #print(comp.shape) - for i in iindex: - - #print(i+span[0],i+span[1]) - comp=np.vstack((comp,field[i+span[0]:i+span[1],:,:])) - #print(comp.shape) - #print(iindex) - comp=comp.reshape( iindex.size+1,self.length,field.shape[1], field.shape[2]) - #comp.shape - comp=np.delete(comp,0,0) - - comp1=composite_data(comp, self.weigthing) - #b2_mean=b2.mean(axis=0) - self.comp[name]=comp1 - else: - print('no span defined') - comp=field[iindex,:,:] + iindex = self.transform_index_time(self.time_index, time) + else: + iindex = self.index + + span = self.iter_operate.span + # print(field.shape) + # print('iindex', iindex) + if span != [0, 0]: + comp = ( + np.empty((-span[0] + span[1], field.shape[1], field.shape[2])) * np.NaN + ) + self.length = -span[0] + span[1] + # print(comp.shape) + for i in iindex: + + # print(i+span[0],i+span[1]) + comp = np.vstack((comp, field[i + span[0] : i + span[1], :, :])) + # print(comp.shape) + # print(iindex) + comp = comp.reshape( + iindex.size + 1, self.length, field.shape[1], field.shape[2] + ) + # comp.shape + comp = np.delete(comp, 0, 0) + + comp1 = composite_data(comp, self.weigthing) + # b2_mean=b2.mean(axis=0) + self.comp[name] = comp1 + else: + print("no span defined") + comp = field[iindex, :, :] - comp1=composite_data(comp, self.weigthing) - self.comp[name]=comp1 + comp1 = composite_data(comp, self.weigthing) + self.comp[name] = comp1 def time_overlap(time1, time2): - if (time1[0]-time2[0]).astype(int) > 0: - #print('time1 is first') - start=time1[0] + if (time1[0] - time2[0]).astype(int) > 0: + # print('time1 is first') + start = time1[0] else: - start=time2[0] + start = time2[0] - if (time1[-1]-time2[-1]).astype(int) > 0: - end=time2[-1] + if (time1[-1] - time2[-1]).astype(int) > 0: + end = time2[-1] else: - end=time1[-1] - - timearray=time1 - dt_data=time1[1]-time1[0] - timecut1=[timearray >= start] - timecut2=[timearray <= (end)] - timecutA=[timecut1 & timecut2 for timecut1, timecut2 in zip(timecut1, timecut2)] + end = time1[-1] + timearray = time1 + dt_data = time1[1] - time1[0] + timecut1 = [timearray >= start] + timecut2 = [timearray <= (end)] + timecutA = [timecut1 & timecut2 for timecut1, timecut2 in zip(timecut1, timecut2)] - timearray=time2 - dt_data=time2[1]-time2[0] - timecut1=[timearray >= start] - timecut2=[timearray <= (end)] - timecutB=[timecut1 & timecut2 for timecut1, timecut2 in zip(timecut1, timecut2)] - + timearray = time2 + dt_data = time2[1] - time2[0] + timecut1 = [timearray >= start] + timecut2 = [timearray <= (end)] + timecutB = [timecut1 & timecut2 for timecut1, timecut2 in zip(timecut1, timecut2)] return timecutA[0], timecutB[0] + def gen_log_space(limit, n): result = [1] - if n>1: # just a check to avoid ZeroDivisionError - ratio = (float(limit)/result[-1]) ** (1.0/(n-len(result))) - while len(result) 1: # just a check to avoid ZeroDivisionError + ratio = (float(limit) / result[-1]) ** (1.0 / (n - len(result))) + while len(result) < n: + next_value = result[-1] * ratio if next_value - result[-1] >= 1: # safe zone. next_value will be a different integer result.append(next_value) else: # problem! same integer. we need to find next_value by artificially incrementing previous value - result.append(result[-1]+1) + result.append(result[-1] + 1) # recalculate the ratio so that the remaining values will scale correctly - ratio = (float(limit)/result[-1]) ** (1.0/(n-len(result))) + ratio = (float(limit) / result[-1]) ** (1.0 / (n - len(result))) # round, re-adjust to 0 indexing (i.e. minus 1) and return np.uint64 array - return np.asarray(list(map(lambda x: round(x)-1, result)), dtype=np.uint64) + return np.asarray(list(map(lambda x: round(x) - 1, result)), dtype=np.uint64) -def linefit2Points(time_lin, f, data, f1, f2, f_delta=None, plot=False): + +def linefit2Points(time_lin, f, data, f1, f2, f_delta=None, plot=False): if isinstance(time_lin[0], np.datetime64): - print('type is numpy.datetime64', time_lin.shape) - time_lin=time_lin.astype('m8[s]').astype(int) - #print(ttt) + print("type is numpy.datetime64", time_lin.shape) + time_lin = time_lin.astype("m8[s]").astype(int) + # print(ttt) - #if isinstance(time_delta, np.timedelta64): + # if isinstance(time_delta, np.timedelta64): # time_delta=time_delta.astype('m8[s]').astype(int) # #print(time_delta) - - if f.shape [0] != data.shape [0]: - print('ERROR: shapes are not correct') - print(f.shape, time_lin.shape, data.shape ) + if f.shape[0] != data.shape[0]: + print("ERROR: shapes are not correct") + print(f.shape, time_lin.shape, data.shape) return # find neerest discrete frequency - a=f-f1 - f1_approx=np.unravel_index(np.abs(a).argmin(),np.transpose(a.shape)) - a=f-f2 - f2_approx=np.unravel_index(np.abs(a).argmin(),np.transpose(a.shape)) - #print(f2_approx) + a = f - f1 + f1_approx = np.unravel_index(np.abs(a).argmin(), np.transpose(a.shape)) + a = f - f2 + f2_approx = np.unravel_index(np.abs(a).argmin(), np.transpose(a.shape)) + # print(f2_approx) # find postion of maximum at freqeuency band - dd=np.squeeze(data[f1_approx,:]) - fin1=np.where(dd==dd.max())[0][0] - dd=np.squeeze(data[f2_approx,:]) - fin2=np.where(dd==dd.max())[0][0] - #print(fin1,time_lin, time_lin.size) + dd = np.squeeze(data[f1_approx, :]) + fin1 = np.where(dd == dd.max())[0][0] + dd = np.squeeze(data[f2_approx, :]) + fin2 = np.where(dd == dd.max())[0][0] + # print(fin1,time_lin, time_lin.size) # define as point in time - x1=time_lin[fin1] - x2=time_lin[fin2] + x1 = time_lin[fin1] + x2 = time_lin[fin2] # and as point in Frequency space - y1=f[f1_approx] - y2=f[f2_approx] + y1 = f[f1_approx] + y2 = f[f2_approx] - if plot==True: - plt.plot(time_lin,data[f1_approx,:].T) - plt.scatter(x1, data[f1_approx,fin1], s=60, c='red', alpha=1 ) - - plt.plot(time_lin,data[f2_approx,:].T) - plt.scatter(x2, data[f2_approx,fin2], s=60, c='red', alpha=1 ) + if plot == True: + plt.plot(time_lin, data[f1_approx, :].T) + plt.scatter(x1, data[f1_approx, fin1], s=60, c="red", alpha=1) + plt.plot(time_lin, data[f2_approx, :].T) + plt.scatter(x2, data[f2_approx, fin2], s=60, c="red", alpha=1) - #fit simple line. - p1, p2=np.polyfit([y1 , y2], [x1, x2], 1) - limit_mid=np.polyval([p1, p2], f) - #print(p2, time_delta) - limit_line1=np.polyval([p1, p2], f-f_delta) - limit_line2=np.polyval([p1, p2], f+f_delta) + # fit simple line. + p1, p2 = np.polyfit([y1, y2], [x1, x2], 1) + limit_mid = np.polyval([p1, p2], f) + # print(p2, time_delta) + limit_line1 = np.polyval([p1, p2], f - f_delta) + limit_line2 = np.polyval([p1, p2], f + f_delta) - if plot==True: + if plot == True: plt.figure() - plt.contourf(time_lin, f, data,cmap='Greys') - plt.plot(limit_mid, f ) + plt.contourf(time_lin, f, data, cmap="Greys") + plt.plot(limit_mid, f) plt.plot(limit_line1, f) plt.plot(limit_line2, f) - plt.plot(time_lin, time_lin*0+y1, Color='black', alpha=1, LineWidth=2) - plt.plot(time_lin, time_lin*0+y2, Color='black', alpha=1, LineWidth=2) - plt.xlabel('time') - plt.ylabel('f') + plt.plot(time_lin, time_lin * 0 + y1, Color="black", alpha=1, LineWidth=2) + plt.plot(time_lin, time_lin * 0 + y2, Color="black", alpha=1, LineWidth=2) + plt.xlabel("time") + plt.ylabel("f") plt.xlim(time_lin[0], time_lin[-1]) - plt.ylim(y1*0.9, y2*1.2) + plt.ylim(y1 * 0.9, y2 * 1.2) return f, limit_line1, limit_line2, limit_mid, fin1 -def find_max_along_line(time_lin, f, data, f1, f2, f_delta=.05, spreed=10, plot=False, mode='free_limits'): - flin, line_left, line_right, line_mid, index=linefit2Points(time_lin, f, data, f1, f2, f_delta=f_delta, plot=False) - timestamp=time_lin +def find_max_along_line( + time_lin, f, data, f1, f2, f_delta=0.05, spreed=10, plot=False, mode="free_limits" +): + flin, line_left, line_right, line_mid, index = linefit2Points( + time_lin, f, data, f1, f2, f_delta=f_delta, plot=False + ) + + timestamp = time_lin if isinstance(time_lin[0], np.datetime64): - print('time is numpy.datetime64') - time_lin=time_lin.astype('m8[s]').astype(int) + print("time is numpy.datetime64") + time_lin = time_lin.astype("m8[s]").astype(int) if mode is None: - mode='free_limits' - if mode is 'free_limits' or mode is 'upper_limit': + mode = "free_limits" + if mode is "free_limits" or mode is "upper_limit": if line_left[0] > time_lin[0]: - f_start=0 - print(' left line > time0') - print(line_left[0] , time_lin[0]) + f_start = 0 + print(" left line > time0") + print(line_left[0], time_lin[0]) else: - print(' left line < time') - print(line_left[0] , time_lin[0]) - a=line_left-time_lin[0] - f_start=np.unravel_index(np.abs(a).argmin(),np.transpose(a.shape))[0]+1 + print(" left line < time") + print(line_left[0], time_lin[0]) + a = line_left - time_lin[0] + f_start = np.unravel_index(np.abs(a).argmin(), np.transpose(a.shape))[0] + 1 else: # find neerest discrete frequency - a=f-f1 - f_start=np.unravel_index(np.abs(a).argmin(),np.transpose(a.shape))[0] + a = f - f1 + f_start = np.unravel_index(np.abs(a).argmin(), np.transpose(a.shape))[0] - #print(f_start) - if mode == 'free_limits' or mode == 'lower_limit': + # print(f_start) + if mode == "free_limits" or mode == "lower_limit": if line_right[-1] > time_lin[-1]: - print(' right line > time window') - print( line_right[-1] ,time_lin[-1]) - a=line_right-time_lin[-1] - f_end=np.unravel_index(np.abs(a).argmin(),np.transpose(a.shape))[0]-1 + print(" right line > time window") + print(line_right[-1], time_lin[-1]) + a = line_right - time_lin[-1] + f_end = np.unravel_index(np.abs(a).argmin(), np.transpose(a.shape))[0] - 1 else: - print(' right line < time window') - print( line_right[-1] ,time_lin[-1]) - f_end=time_lin.size-2 + print(" right line < time window") + print(line_right[-1], time_lin[-1]) + f_end = time_lin.size - 2 else: - a=f-f2 - f_end=np.unravel_index(np.abs(a).argmin(),np.transpose(a.shape))[0] + a = f - f2 + f_end = np.unravel_index(np.abs(a).argmin(), np.transpose(a.shape))[0] - #print(f_end) + # print(f_end) if plot: plt.figure() plt.pcolor(time_lin, f, data) - plt.plot(f*0+int(time_lin[index]), f, linewidth=3, Color='black') - plt.plot(line_mid, f ) - plt.plot(line_left, f, Color='black') - plt.plot(line_right, f, Color='black') - plt.plot(time_lin, time_lin*0+flin[f_start], Color='red') - plt.plot(time_lin, time_lin*0+flin[f_end], Color='red') - - plt.plot(time_lin, time_lin*0+f1, Color='grey', linewidth=3) - plt.plot(time_lin, time_lin*0+f2, Color='grey') - - STR=dict() - STR['t_pos']=[] - STR['index']=index - #STR['cut_pos']=[] - STR['amp']=[] - STR['freq']=[] - STR['out']=[] - STR['amp_shape']=list() - - STR['left_limit']=line_left - STR['right_limit']=line_right - #print(flin[f_start:f_end].shape) + plt.plot(f * 0 + int(time_lin[index]), f, linewidth=3, Color="black") + plt.plot(line_mid, f) + plt.plot(line_left, f, Color="black") + plt.plot(line_right, f, Color="black") + plt.plot(time_lin, time_lin * 0 + flin[f_start], Color="red") + plt.plot(time_lin, time_lin * 0 + flin[f_end], Color="red") + + plt.plot(time_lin, time_lin * 0 + f1, Color="grey", linewidth=3) + plt.plot(time_lin, time_lin * 0 + f2, Color="grey") + + STR = dict() + STR["t_pos"] = [] + STR["index"] = index + # STR['cut_pos']=[] + STR["amp"] = [] + STR["freq"] = [] + STR["out"] = [] + STR["amp_shape"] = list() + + STR["left_limit"] = line_left + STR["right_limit"] = line_right + # print(flin[f_start:f_end].shape) for i in enumerate(flin[f_start:f_end]): - ii=f_start+i[0] - #print(i[0]) - #print('ii',ii , type(i[0]), type(i) ) - #print(round(line_left[ii]), round(line_right[ii])) - cut=cut_nparray(time_lin, line_left[ii], line_right[ii], verbose=False) - #print( cut[:data.shape[1]].shape, data.shape) - #plt.plot(data[i, cut], Color='grey', alpha=0.5) - #print(type(data[ii, cut]), type(cut[0]), type(ii)) - #print(data[ii, cut[:data.shape[1]]]) - out=find_max_ts(data[ii, cut[:data.shape[1]]], smooth=True, spreed=spreed, plot=plot, verbose=False) - #STR['cut_pos'].append(np.where(np.diff(cut) == True)[0][0]) - STR['out'].append(out[0][0]) - STR['t_pos'].append(out[0][0]+np.where(np.diff(cut) == True)[0][0]) - STR['amp'].append(out[2][0]) - STR['amp_shape'].append(out[1]) - STR['freq'].append(i[1]) - #plt.plot(out[1], Color='red', alpha=1) + ii = f_start + i[0] + # print(i[0]) + # print('ii',ii , type(i[0]), type(i) ) + # print(round(line_left[ii]), round(line_right[ii])) + cut = cut_nparray(time_lin, line_left[ii], line_right[ii], verbose=False) + # print( cut[:data.shape[1]].shape, data.shape) + # plt.plot(data[i, cut], Color='grey', alpha=0.5) + # print(type(data[ii, cut]), type(cut[0]), type(ii)) + # print(data[ii, cut[:data.shape[1]]]) + out = find_max_ts( + data[ii, cut[: data.shape[1]]], + smooth=True, + spreed=spreed, + plot=plot, + verbose=False, + ) + # STR['cut_pos'].append(np.where(np.diff(cut) == True)[0][0]) + STR["out"].append(out[0][0]) + STR["t_pos"].append(out[0][0] + np.where(np.diff(cut) == True)[0][0]) + STR["amp"].append(out[2][0]) + STR["amp_shape"].append(out[1]) + STR["freq"].append(i[1]) + # plt.plot(out[1], Color='red', alpha=1) if plot: plt.figure() - plt.pcolor(time_lin, f, data,cmap='Greys') - #plt.plot(limit_mid, f ) - plt.plot(line_left, f, Color='blue') - plt.plot(line_right, f, Color='blue') - #plt.plot(time_lin, time_lin*0+y1, Color='black', alpha=1, LineWidth=2) - #plt.plot(time_lin, time_lin*0+y2, Color='black', alpha=1, LineWidth=2) - plt.xlabel('time') - plt.ylabel('f') - plt.scatter(time_lin[STR['t_pos']], STR['freq'], s=20, c='blue', alpha=1) - #plt.scatter(time_lin[STR['cut_pos']], STR['freq'], s=20, c='orange', alpha=1) + plt.pcolor(time_lin, f, data, cmap="Greys") + # plt.plot(limit_mid, f ) + plt.plot(line_left, f, Color="blue") + plt.plot(line_right, f, Color="blue") + # plt.plot(time_lin, time_lin*0+y1, Color='black', alpha=1, LineWidth=2) + # plt.plot(time_lin, time_lin*0+y2, Color='black', alpha=1, LineWidth=2) + plt.xlabel("time") + plt.ylabel("f") + plt.scatter(time_lin[STR["t_pos"]], STR["freq"], s=20, c="blue", alpha=1) + # plt.scatter(time_lin[STR['cut_pos']], STR['freq'], s=20, c='orange', alpha=1) plt.xlim(time_lin[0], time_lin[-1]) - plt.ylim(flin[f_start] , flin[f_end]) + plt.ylim(flin[f_start], flin[f_end]) plt.figure() - for s in STR['amp_shape']: - plt.plot(s, Color='grey', alpha=0.6) - plt.scatter(STR['out'],STR['amp'], s=20) - #plt.xlim([40 , 70]) - #plt.ylim([20, 50]) + for s in STR["amp_shape"]: + plt.plot(s, Color="grey", alpha=0.6) + plt.scatter(STR["out"], STR["amp"], s=20) + # plt.xlim([40 , 70]) + # plt.ylim([20, 50]) return STR + def robust_regression(time, freq, plot=True): from sklearn import linear_model - time2=np.empty((time.size,1)) - time2[:,0]=time + + time2 = np.empty((time.size, 1)) + time2[:, 0] = time # Robustly fit linear model with RANSAC algorithm model_ransac = linear_model.RANSACRegressor(linear_model.LinearRegression()) model_ransac.fit(time2, freq) - predicted_line=model_ransac.predict(time[:, np.newaxis]) + predicted_line = model_ransac.predict(time[:, np.newaxis]) - slope=model_ransac.estimator_.coef_[0] - intercept=model_ransac.estimator_.intercept_ - #inlier_mask = model_ransac.inlier_mask_ - #outlier_mask = np.logical_not(inlier_mask) + slope = model_ransac.estimator_.coef_[0] + intercept = model_ransac.estimator_.intercept_ + # inlier_mask = model_ransac.inlier_mask_ + # outlier_mask = np.logical_not(inlier_mask) if plot: plt.figure() plt.scatter(time, freq) - plt.plot(time, predicted_line, color='red', linestyle='-', - linewidth=2, label='robust regression estimate') + plt.plot( + time, + predicted_line, + color="red", + linestyle="-", + linewidth=2, + label="robust regression estimate", + ) return slope, intercept, time, predicted_line def simple_RAMSAC_regression_estimator(x, Y): from sklearn import linear_model - #return stats.linregress(x[i], y[i])[2] - model_ransac = linear_model.RANSACRegressor(linear_model.LinearRegression())#,min_samples=1, - #residual_threshold=1) - #stop_n_inliers=.2) - x2=np.empty((x.size,1)) - x2[:,0]=x + + # return stats.linregress(x[i], y[i])[2] + model_ransac = linear_model.RANSACRegressor( + linear_model.LinearRegression() + ) # ,min_samples=1, + # residual_threshold=1) + # stop_n_inliers=.2) + x2 = np.empty((x.size, 1)) + x2[:, 0] = x model_ransac.fit(x2, Y) - slope=model_ransac.estimator_.coef_[0] - intercept=model_ransac.estimator_.intercept_ + slope = model_ransac.estimator_.coef_[0] + intercept = model_ransac.estimator_.intercept_ return slope, intercept -def RAMSAC_regression_bootstrap(time, freq, time_lin_arg=None, plot=False, **kwargs): - ''' bootstraps linear regression model using the RAMSAC algorythm +def RAMSAC_regression_bootstrap(time, freq, time_lin_arg=None, plot=False, **kwargs): + """bootstraps linear regression model using the RAMSAC algorythm outout: slope low_ci high_ci intercept low_ci high_ci x Y - ''' - import sklearn.bootstrap as boot # might not work in python 3 - + """ + import sklearn.bootstrap as boot # might not work in python 3 - #statfunction= - RAMS_slope, RAMS_intercept =simple_RAMSAC_regression_estimator(time, freq) - #self.clevs=self.clevs if self.clevs is not None else clevels(dd) + # statfunction= + RAMS_slope, RAMS_intercept = simple_RAMSAC_regression_estimator(time, freq) + # self.clevs=self.clevs if self.clevs is not None else clevels(dd) if time_lin_arg is not None: - time_lin=time_lin_arg - print('time lin is set') + time_lin = time_lin_arg + print("time lin is set") else: - print('create linear time axis') - time_lin=np.linspace(time.min(), time.max(), freq.size) - - RAMS_predicted_line=time_lin*RAMS_slope+ RAMS_intercept + print("create linear time axis") + time_lin = np.linspace(time.min(), time.max(), freq.size) + RAMS_predicted_line = time_lin * RAMS_slope + RAMS_intercept print(RAMS_slope, RAMS_intercept) - RAMS_out=boot.ci((time, freq), simple_RAMSAC_regression_estimator, method='bca' , **kwargs) + RAMS_out = boot.ci( + (time, freq), simple_RAMSAC_regression_estimator, method="bca", **kwargs + ) # Robustly fit linear model with RANSAC algorithm - #model_ransac = linear_model.RANSACRegressor(linear_model.LinearRegression()) - #model_ransac.fit(time2, freq) - #predicted_line=model_ransac.predict(time[:, np.newaxis]) - - #slope=model_ransac.estimator_.coef_[0] - #intercept=model_ransac.estimator_.intercept_ - #inlier_mask = model_ransac.inlier_mask_ - #outlier_mask = np.logical_not(inlier_mask) - - slope=np.append(RAMS_slope, RAMS_out[:,0]) - intercept=np.append(RAMS_intercept, RAMS_out[:,1]) - predicted_line=np.vstack((RAMS_predicted_line, (time_lin*slope[1]+intercept[2], time_lin*slope[2]+intercept[1]))) + # model_ransac = linear_model.RANSACRegressor(linear_model.LinearRegression()) + # model_ransac.fit(time2, freq) + # predicted_line=model_ransac.predict(time[:, np.newaxis]) + + # slope=model_ransac.estimator_.coef_[0] + # intercept=model_ransac.estimator_.intercept_ + # inlier_mask = model_ransac.inlier_mask_ + # outlier_mask = np.logical_not(inlier_mask) + + slope = np.append(RAMS_slope, RAMS_out[:, 0]) + intercept = np.append(RAMS_intercept, RAMS_out[:, 1]) + predicted_line = np.vstack( + ( + RAMS_predicted_line, + (time_lin * slope[1] + intercept[2], time_lin * slope[2] + intercept[1]), + ) + ) if plot: plt.figure() - plt.scatter(time, freq, color='black') - plt.plot(time_lin, RAMS_predicted_line.T, color='red', linestyle='-', - linewidth=2, label='robust regression estimate') - plt.plot(time_lin, time_lin*slope[1]+intercept[2], color='blue') - plt.plot(time_lin, time_lin*slope[2]+intercept[1], color='blue') + plt.scatter(time, freq, color="black") + plt.plot( + time_lin, + RAMS_predicted_line.T, + color="red", + linestyle="-", + linewidth=2, + label="robust regression estimate", + ) + plt.plot(time_lin, time_lin * slope[1] + intercept[2], color="blue") + plt.plot(time_lin, time_lin * slope[2] + intercept[1], color="blue") return slope, intercept, time_lin, predicted_line diff --git a/src/icesat2_tracks/local_modules/m_spectrum_ph3.py b/src/icesat2_tracks/local_modules/m_spectrum_ph3.py index f904b63c..e66564c4 100644 --- a/src/icesat2_tracks/local_modules/m_spectrum_ph3.py +++ b/src/icesat2_tracks/local_modules/m_spectrum_ph3.py @@ -1,4 +1,3 @@ - import numpy as np from numpy import pi from scipy.special import gammainc @@ -8,51 +7,58 @@ try: import mkl + np.use_fastnumpy = True except ImportError: pass + def calc_freq(self): - """ calculate array of spectral variable (frequency or - wavenumber) in cycles per unit of L """ + """calculate array of spectral variable (frequency or + wavenumber) in cycles per unit of L""" - self.df = 1./((self.n-1)*self.dt) + self.df = 1.0 / ((self.n - 1) * self.dt) if self.neven: - self.f = self.df*np.arange(self.n/2+1) + self.f = self.df * np.arange(self.n / 2 + 1) else: - self.f = self.df*np.arange( (self.n-1)/2. + 1 ) + self.f = self.df * np.arange((self.n - 1) / 2.0 + 1) + def calc_spectrum(self): - """ compute the 1d spectrum of a field phi """ + """compute the 1d spectrum of a field phi""" self.phih = np.fft.rfft(self.phi) # the factor of 2 comes from the symmetry of the Fourier coeffs - self.spec = 2.*(self.phih*self.phih.conj()).real / self.df /self.n**2 + self.spec = 2.0 * (self.phih * self.phih.conj()).real / self.df / self.n**2 # the zeroth frequency should be counted only once - self.spec[0] = self.spec[0]/2. + self.spec[0] = self.spec[0] / 2.0 if self.neven: - self.spec[-1] = self.spec[-1]/2. + self.spec[-1] = self.spec[-1] / 2.0 + def calc_var(self): - """ Compute total variance from spectrum """ - self.var = self.df*self.spec[1:].sum() # do not consider zeroth frequency + """Compute total variance from spectrum""" + self.var = self.df * self.spec[1:].sum() # do not consider zeroth frequency + def create_timeaxis_collection(time_as_datetime64): import datetime as DT from matplotlib import dates - T=dict() - T['sec']= time_as_datetime64.astype('M8[s]').astype('float') - T['day']= time_as_datetime64.astype('M8[D]').astype('float') - T['datetime']= time_as_datetime64.astype(DT.datetime) - T['float_plot']=dates.date2num(T['datetime']) - T['dt64']= time_as_datetime64 + T = dict() + T["sec"] = time_as_datetime64.astype("M8[s]").astype("float") + T["day"] = time_as_datetime64.astype("M8[D]").astype("float") + T["datetime"] = time_as_datetime64.astype(DT.datetime) + + T["float_plot"] = dates.date2num(T["datetime"]) + T["dt64"] = time_as_datetime64 return T -def spicke_remover(data, nstd=20.0, spreed=500.0, max_loops=10.0 , verbose=False): + +def spicke_remover(data, nstd=20.0, spreed=500.0, max_loops=10.0, verbose=False): """ This function removes spickes from timeseries based on its std and maximum values inputs: @@ -61,67 +67,72 @@ def spicke_remover(data, nstd=20.0, spreed=500.0, max_loops=10.0 , verbose=False spreed width of data between wich will be linear interpolated around the spike. width is in units of data points (dt) max_loops number of maximum possible loobs until the spike removal is stopped, even it the first creteria is not true """ - datastd=np.nanstd(data) - data2=np.copy(data) + datastd = np.nanstd(data) + data2 = np.copy(data) - peak_remove=True - looper_count=0 - act_flag=False + peak_remove = True + looper_count = 0 + act_flag = False while peak_remove is True: - if nstd* data.std() < np.max(np.abs(data2)): - act_flag=True + if nstd * data.std() < np.max(np.abs(data2)): + act_flag = True if verbose: - print('true: '+ str(nstd* datastd) +' < '+str( np.max(np.abs(data)) ) ) - data2=M.spickes_to_mean(data2, nloop=0, spreed=spreed, gaussian=False) - looper_count+=1 + print( + "true: " + str(nstd * datastd) + " < " + str(np.max(np.abs(data))) + ) + data2 = M.spickes_to_mean(data2, nloop=0, spreed=spreed, gaussian=False) + looper_count += 1 else: if verbose: - print('False: '+ str(nstd* datastd) +' > '+str( np.max(np.abs(data)) ) ) - peak_remove=False + print( + "False: " + str(nstd * datastd) + " > " + str(np.max(np.abs(data))) + ) + peak_remove = False if looper_count > max_loops: - peak_remove=False + peak_remove = False if verbose: - print('stoped by max#') - + print("stoped by max#") if verbose: - plt.plot(data, 'r') - plt.plot(data2, 'b') + plt.plot(data, "r") + plt.plot(data2, "b") + + return data2, act_flag - return data2 , act_flag class Spectrum: - """ A class that represents a single realization of - the one-dimensional spectrum of a given field phi """ + """A class that represents a single realization of + the one-dimensional spectrum of a given field phi""" - def __init__(self,data,dt, win_flag=1,pwelch=False, verbose=False): + def __init__(self, data, dt, win_flag=1, pwelch=False, verbose=False): import m_tools_ph3 as MT - self.data = data # field to be analyzed - self.dt = dt # sampling interval + + self.data = data # field to be analyzed + self.dt = dt # sampling interval self.n = data.size - self.hist='Spectorgram' - win = np.hanning(self.n) - factor=np.sqrt(self.n/(win**2).sum()) + self.hist = "Spectorgram" + win = np.hanning(self.n) + factor = np.sqrt(self.n / (win**2).sum()) if win_flag: if verbose: - print('window') - MT.write_log(self.hist, 'window') - self.win_flag=win_flag - self.phi=np.copy(self.data[:]) - self.phi*=win*np.sqrt(factor) + print("window") + MT.write_log(self.hist, "window") + self.win_flag = win_flag + self.phi = np.copy(self.data[:]) + self.phi *= win * np.sqrt(factor) else: if verbose: - print('no window') - MT.write_log(self.hist, 'no window') - self.win_flag=0 - self.phi =np.copy(self.data[:]) + print("no window") + MT.write_log(self.hist, "no window") + self.win_flag = 0 + self.phi = np.copy(self.data[:]) # test if n is even - if (self.n%2): + if self.n % 2: self.neven = False else: self.neven = True @@ -135,22 +146,34 @@ def __init__(self,data,dt, win_flag=1,pwelch=False, verbose=False): # calculate total var calc_var(self) -# def calc_var(self): -# """ Compute total variance from spectrum """ -# self.var = self.df*self.spec[1:].sum() # do not consider zeroth frequency + # def calc_var(self): + # """ Compute total variance from spectrum """ + # self.var = self.df*self.spec[1:].sum() # do not consider zeroth frequency def parceval(self): - print('Parcevals Theorem:') - print('variance of unweighted timeseries: ',self.data.var()) - print('variance of weighted timeseries: ',self.phi.var() if self.win_flag is 1 else 'data not windowed') - print('variance of weighted timeseries: ',self.phi.var() ) - print('variance of the Spectrum: ',self.var) - + print("Parcevals Theorem:") + print("variance of unweighted timeseries: ", self.data.var()) + print( + "variance of weighted timeseries: ", + self.phi.var() if self.win_flag is 1 else "data not windowed", + ) + print("variance of weighted timeseries: ", self.phi.var()) + print("variance of the Spectrum: ", self.var) class moments: - def __init__(self,data_org,dt, L=None, ov=None,window=None, save_chunks=False, plot_chunks=False, prewhite=None): + def __init__( + self, + data_org, + dt, + L=None, + ov=None, + window=None, + save_chunks=False, + plot_chunks=False, + prewhite=None, + ): """ This function calculates the spectral moments from a station (buoy, GPS, or seismic station) that measures displacement in 3 direction in a right hand coordinate system. The directions should have the same untis. @@ -163,198 +186,198 @@ def __init__(self,data_org,dt, L=None, ov=None,window=None, save_chunks=False, p """ self.data = np.array(data_org) if prewhite is None: - data = np.array(data_org) # field to be analyzed - elif prewhite ==1: - print('prewhite =1') - data=np.gradient(np.array(data_org) , axis=1) - elif prewhite ==2: - data=np.gradient(np.gradient(np.array(data_org), axis=1), axis=1) + data = np.array(data_org) # field to be analyzed + elif prewhite == 1: + print("prewhite =1") + data = np.gradient(np.array(data_org), axis=1) + elif prewhite == 2: + data = np.gradient(np.gradient(np.array(data_org), axis=1), axis=1) - #self.data = data # field to be analyzed - self.dt = dt # sampling interval - self.save_chunks=save_chunks + # self.data = data # field to be analyzed + self.dt = dt # sampling interval + self.save_chunks = save_chunks - data_size=np.shape(data)[1] - data_dim=np.shape(data)[0] + data_size = np.shape(data)[1] + data_dim = np.shape(data)[0] - L=int(np.round(data_size/10)) if L is None else L + L = int(np.round(data_size / 10)) if L is None else L if type(L) != int: M.echo_dt(L) - L=L.item().total_seconds() - #else: - #print('unknown L type') + L = L.item().total_seconds() + # else: + # print('unknown L type') - ov=int(np.round(L/2)) if ov is None else ov + ov = int(np.round(L / 2)) if ov is None else ov self.n = L if window is None: - win=np.hanning(self.n) + win = np.hanning(self.n) else: - win=window + win = window - #weigthing factor for preserved spectra - factor=np.sqrt(self.n/(win**2).sum()) - win*=factor + # weigthing factor for preserved spectra + factor = np.sqrt(self.n / (win**2).sum()) + win *= factor # test if n is even - if (self.n%2): + if self.n % 2: self.neven = False else: self.neven = True - #calculate freq + # calculate freq calc_freq(self) # exclude 0 freq. - self.f=self.f[1:] - #print('fsize', self.f.size) - #print(data.size, L, ov, int(L-ov) ) - nbin=int(np.floor(data_size/(L-ov)))-1 - self.nbin=nbin + self.f = self.f[1:] + # print('fsize', self.f.size) + # print(data.size, L, ov, int(L-ov) ) + nbin = int(np.floor(data_size / (L - ov))) - 1 + self.nbin = nbin if save_chunks: - chunks=np.empty([data_dim, int(nbin),int(L)]) + chunks = np.empty([data_dim, int(nbin), int(L)]) - #container for spectra - specs=np.empty([data_dim, int(nbin),self.f.size]) - self.mom_list=list() - k=0 - #print('iter range', np.arange(0,data.size,int(L-ov))) - for i in np.arange(0,data_size-int(L-ov)+1,int(L-ov)): + # container for spectra + specs = np.empty([data_dim, int(nbin), self.f.size]) + self.mom_list = list() + k = 0 + # print('iter range', np.arange(0,data.size,int(L-ov))) + for i in np.arange(0, data_size - int(L - ov) + 1, int(L - ov)): - if (plot_chunks) and (i >= data_size-6*int(L-ov)): + if (plot_chunks) and (i >= data_size - 6 * int(L - ov)): M.figure_axis_xy() - self.phi=data[:,int(i):int(i+L)] + self.phi = data[:, int(i) : int(i + L)] - #self.ii=np.append(self.ii,[i,i+L]) - #print(self.phi.max()) + # self.ii=np.append(self.ii,[i,i+L]) + # print(self.phi.max()) - #print(self.phi.mean()) - #print(self.phi.shape) - #print('i',int(i), int(i+L)) - #print(chunk.size, l) - if int(i+L) <= data_size-1: + # print(self.phi.mean()) + # print(self.phi.shape) + # print('i',int(i), int(i+L)) + # print(chunk.size, l) + if int(i + L) <= data_size - 1: if save_chunks: - chunks[:, k,:]=self.phi + chunks[:, k, :] = self.phi - #normalizing - #print('normalizing') - #self.phi=signal.detrend(self.phi,axis=1)*win - #self.phi=(self.phi.T/np.std(self.phi, axis=1)).T + # normalizing + # print('normalizing') + # self.phi=signal.detrend(self.phi,axis=1)*win + # self.phi=(self.phi.T/np.std(self.phi, axis=1)).T - #non normalizing - #de trending and prewhitening - self.phi=signal.detrend(self.phi,axis=1)*win + # non normalizing + # de trending and prewhitening + self.phi = signal.detrend(self.phi, axis=1) * win if plot_chunks: - #MT.stats_format(self.phi, 'chunk '+str(i)) + # MT.stats_format(self.phi, 'chunk '+str(i)) plt.plot(self.phi) - X=self.calc_spectral_density() - #print(np.shape(X)) - self.mom=self.calc_moments(X) + X = self.calc_spectral_density() + # print(np.shape(X)) + self.mom = self.calc_moments(X) self.mom_list.append(self.mom) - last_k=k - last_used_TS=int(i+L) - #if plot_chunks: + last_k = k + last_used_TS = int(i + L) + # if plot_chunks: # MT.stats_format(self.spec, 'spec '+str(i)) else: if plot_chunks: - print('end of TS is reached') - print('last spec No: '+str(last_k)) - print('spec container: '+str(specs.shape)) - print('last used Timestep: '+str(last_used_TS)) - print('length of TS '+ str(data.size)) + print("end of TS is reached") + print("last spec No: " + str(last_k)) + print("spec container: " + str(specs.shape)) + print("last used Timestep: " + str(last_used_TS)) + print("length of TS " + str(data.size)) - k+=1 + k += 1 - #print('last k', last_k) - #print('len mom_list', len(self.mom_list) ) - #print('nbin', nbin) + # print('last k', last_k) + # print('len mom_list', len(self.mom_list) ) + # print('nbin', nbin) if save_chunks: - self.chunks=chunks - #del(chunks) + self.chunks = chunks + # del(chunks) - self.moments_stack=dict() - self.moments_est=dict() + self.moments_stack = dict() + self.moments_est = dict() for k in self.mom.keys(): - #print(k) - stack=np.empty([nbin,self.f.size]) + # print(k) + stack = np.empty([nbin, self.f.size]) for i in range(len(self.mom_list)): - #stack=np.vstack((stack,I[k])) #stack them - #print(i, k) - stack[i,:]=self.mom_list[i][k] #stack them - #print(stack.shape) - #mom_est[k]=stack.mean(axis=0) - #mean them and decide prewhitheing or nots + # stack=np.vstack((stack,I[k])) #stack them + # print(i, k) + stack[i, :] = self.mom_list[i][k] # stack them + # print(stack.shape) + # mom_est[k]=stack.mean(axis=0) + # mean them and decide prewhitheing or nots if prewhite is None: - self.moments_stack[k]=stack - self.moments_est[k]=np.nanmean(stack,axis=0) - elif prewhite ==1: - self.moments_stack[k]=stack*(2*np.pi*self.f) - self.moments_est[k]=np.nanmean(stack, axis=0)*(2*np.pi*self.f) - elif prewhite ==2: - self.moments_stack[k]=stack*(2*np.pi*self.f)**2 - self.moments_est[k]=np.nanmean(stack, axis=0)*(2*np.pi*self.f)**2 - - - self.moments_unit='[data]^2' - self.n_spec=len(self.mom_list) - #self.calc_var() - #self.phi=self.data - #self.phi*=win*np.sqrt(factor) + self.moments_stack[k] = stack + self.moments_est[k] = np.nanmean(stack, axis=0) + elif prewhite == 1: + self.moments_stack[k] = stack * (2 * np.pi * self.f) + self.moments_est[k] = np.nanmean(stack, axis=0) * (2 * np.pi * self.f) + elif prewhite == 2: + self.moments_stack[k] = stack * (2 * np.pi * self.f) ** 2 + self.moments_est[k] = ( + np.nanmean(stack, axis=0) * (2 * np.pi * self.f) ** 2 + ) + + self.moments_unit = "[data]^2" + self.n_spec = len(self.mom_list) + # self.calc_var() + # self.phi=self.data + # self.phi*=win*np.sqrt(factor) def calc_spectral_density(self): - """ compute the 1d spectraln density of a field phi """ + """compute the 1d spectraln density of a field phi""" # the factor of 2 comes from the symmetry of the Fourier coeffs (rfft does only calculate the f>=0 part. # to preserve variance, we multiply by 2) - phih = 2 * np.fft.rfft(self.phi)/self.n**2 + phih = 2 * np.fft.rfft(self.phi) / self.n**2 # the 1/n**2 is debatalbe. Sarahs notes say, 1/n should be enough. # the zeroth frequency should be counted only once - #phih[0] = phih[0]/2. + # phih[0] = phih[0]/2. if self.neven: - phih[-1] = phih[-1]/2. + phih[-1] = phih[-1] / 2.0 return phih def calc_moments(self, X): - mom=dict() + mom = dict() # 1: for killing 0 freq. - mom['P11']=(X[0,1:]*X[0,1:].conj()).real - mom['P22']=(X[1,1:]*X[1,1:].conj()).real - mom['P33']=(X[2,1:]*X[2,1:].conj()).real + mom["P11"] = (X[0, 1:] * X[0, 1:].conj()).real + mom["P22"] = (X[1, 1:] * X[1, 1:].conj()).real + mom["P33"] = (X[2, 1:] * X[2, 1:].conj()).real - mom['Q12']=(X[0,1:]*X[1,1:].conj()).imag - mom['Q13']=(X[0,1:]*X[2,1:].conj()).imag - mom['Q23']=(X[1,1:]*X[2,1:].conj()).imag + mom["Q12"] = (X[0, 1:] * X[1, 1:].conj()).imag + mom["Q13"] = (X[0, 1:] * X[2, 1:].conj()).imag + mom["Q23"] = (X[1, 1:] * X[2, 1:].conj()).imag - mom['P13']=(X[0,1:]*X[2,1:].conj()).real - mom['P12']=(X[0,1:]*X[1,1:].conj()).real - mom['P23']=(X[1,1:]*X[2,1:].conj()).real + mom["P13"] = (X[0, 1:] * X[2, 1:].conj()).real + mom["P12"] = (X[0, 1:] * X[1, 1:].conj()).real + mom["P23"] = (X[1, 1:] * X[2, 1:].conj()).real return mom def plot_moments(self): - num=1 - F=plt.figure(figsize=(9,10) ) - - for k,I in self.moments_est.iteritems(): - plt.subplot(len(self.moments_est),1,num ) - plt.plot(self.f, I/self.f, c='k') - num+=1 - plt.xlim(0.001,.1 ) + num = 1 + F = plt.figure(figsize=(9, 10)) + + for k, I in self.moments_est.iteritems(): + plt.subplot(len(self.moments_est), 1, num) + plt.plot(self.f, I / self.f, c="k") + num += 1 + plt.xlim(0.001, 0.1) plt.ylabel(self.moments_unit) plt.title(k) - - def cal_MEM(self, theta=None, flim=(0.01, .5)): - """ wrapping method that calculated Maximum entrophy method for self.moments_est + def cal_MEM(self, theta=None, flim=(0.01, 0.5)): + """wrapping method that calculated Maximum entrophy method for self.moments_est RETURNS self.MEM (dict) D(\theta, f) normalized directional specturm @@ -366,38 +389,50 @@ def cal_MEM(self, theta=None, flim=(0.01, .5)): unit_dir unit of E """ - #theta=np.arange(1,361) if theta is None else theta + # theta=np.arange(1,361) if theta is None else theta - MEM=dict() - MEM['D'], MEM['S'], MEM['E'], MEM['freq'], MEM['theta'] = MEM_cal(self.moments_est,self.f,theta=theta ,flim=flim) - MEM['fmin']=flim[0] - MEM['fmax']=flim[1] - MEM['unit_power']=self.moments_unit + '/Hz' - MEM['unit_dir']=self.moments_unit + '/deg/Hz' + MEM = dict() + MEM["D"], MEM["S"], MEM["E"], MEM["freq"], MEM["theta"] = MEM_cal( + self.moments_est, self.f, theta=theta, flim=flim + ) + MEM["fmin"] = flim[0] + MEM["fmax"] = flim[1] + MEM["unit_power"] = self.moments_unit + "/Hz" + MEM["unit_dir"] = self.moments_unit + "/deg/Hz" - self.MEM =MEM + self.MEM = MEM - #def error(self, ci=0.95): + # def error(self, ci=0.95): # self.El, self.Eu =spec_error(self.spec_est,self.n_spec,ci=ci) - #def parceval(self): - ## method not checked jet + # def parceval(self): + ## method not checked jet # print('Parcevals Theorem:') # print('variance of unweighted timeseries: ',self.data.var()) # print('mean variance of timeseries chunks: ',self.chunks.var(axis=1).mean() if self.save_chunks is True else 'data not saved') - #print('variance of weighted timeseries: ',self.phi.var() ) - #self.calc_var(self) + # print('variance of weighted timeseries: ',self.phi.var() ) + # self.calc_var(self) # print('variance of the pwelch Spectrum: ',self.var) - #def calc_var(self): + # def calc_var(self): # """ Compute total variance from spectral density """ # self.var = self.specs[1:].mean(axis=0).sum() # do not consider zeroth frequency - class pwelch: - def __init__(self,data,dt,L=None, ov=None,window=None, save_chunks=False, plot_chunks=False, periodogram=False, prewhite=None): + def __init__( + self, + data, + dt, + L=None, + ov=None, + window=None, + save_chunks=False, + plot_chunks=False, + periodogram=False, + prewhite=None, + ): """ prewhite None(default) 1 = 1 timederivative @@ -407,413 +442,469 @@ def __init__(self,data,dt,L=None, ov=None,window=None, save_chunks=False, plot_c """ from m_general_ph3 import echo_dt + if prewhite is None: - self.data = data # field to be analyzed - elif prewhite ==1: - print('prewhite =1') - self.data=np.gradient(data) - elif prewhite ==2: - self.data=np.gradient(np.gradient(data)) - - self.data = data # field to be analyzed - self.dt = dt # sampling interval - self.save_chunks= save_chunks - dsize = dt * data.size - - L=int(np.round(data.size/10)) if L is None else L + self.data = data # field to be analyzed + elif prewhite == 1: + print("prewhite =1") + self.data = np.gradient(data) + elif prewhite == 2: + self.data = np.gradient(np.gradient(data)) + + self.data = data # field to be analyzed + self.dt = dt # sampling interval + self.save_chunks = save_chunks + dsize = dt * data.size + + L = int(np.round(data.size / 10)) if L is None else L if type(L) != int: echo_dt(L) - L=L.item().total_seconds() - #else: - #print('unknown L type') + L = L.item().total_seconds() + # else: + # print('unknown L type') - ov=int(np.round(L/2)) if ov is None else ov + ov = int(np.round(L / 2)) if ov is None else ov self.n = L if window is None: - win=np.hanning(self.n) + win = np.hanning(self.n) else: - win=window + win = window - factor=np.sqrt(self.n/(win**2).sum()) - win*=factor + factor = np.sqrt(self.n / (win**2).sum()) + win *= factor # test if n is even - if (self.n%2): + if self.n % 2: self.neven = False else: self.neven = True - #calculate freq + # calculate freq calc_freq(self) - #print(data.size, L, ov, int(L-ov) ) - nbin=int(np.floor(dsize/(L-ov))) - #print(nbin) + # print(data.size, L, ov, int(L-ov) ) + nbin = int(np.floor(dsize / (L - ov))) + # print(nbin) if periodogram: - self.nbin=nbin - self.dt_periodogram=L-ov + self.nbin = nbin + self.dt_periodogram = L - ov if save_chunks: - chunks=np.empty([int(nbin),int(L)]) - - specs=np.empty([int(nbin),self.f.size]) - #print(chunks.shape) - #result_array = np.empty((0, 100)) - #if plot_chunks: - #M.figure_axis_xy() - last_k=0 - k=0 - #print('iter range', np.arange(0,data.size,int(L-ov))) - for i in np.arange(0,dsize-int(L-ov)+1,int(L-ov)): - - if (plot_chunks) and (i >= dsize-6*int(L-ov)): + chunks = np.empty([int(nbin), int(L)]) + + specs = np.empty([int(nbin), self.f.size]) + # print(chunks.shape) + # result_array = np.empty((0, 100)) + # if plot_chunks: + # M.figure_axis_xy() + last_k = 0 + k = 0 + # print('iter range', np.arange(0,data.size,int(L-ov))) + for i in np.arange(0, dsize - int(L - ov) + 1, int(L - ov)): + + if (plot_chunks) and (i >= dsize - 6 * int(L - ov)): M.figure_axis_xy() - self.phi=data[int(i):int(i+L)] + self.phi = data[int(i) : int(i + L)] - #self.ii=np.append(self.ii,[i,i+L]) - #print(self.phi.max()) + # self.ii=np.append(self.ii,[i,i+L]) + # print(self.phi.max()) - #print(self.phi.mean()) - #print(self.phi.shape) - #print('i',int(i), int(i+L)) - #print(chunk.size, l) - if int(i+L) <= data.size-1: + # print(self.phi.mean()) + # print(self.phi.shape) + # print('i',int(i), int(i+L)) + # print(chunk.size, l) + if int(i + L) <= data.size - 1: if save_chunks: - chunks[k,:]=self.phi + chunks[k, :] = self.phi - - self.phi=signal.detrend(self.phi)*win + self.phi = signal.detrend(self.phi) * win if plot_chunks: - #MT.stats_format(self.phi, 'chunk '+str(i)) + # MT.stats_format(self.phi, 'chunk '+str(i)) plt.plot(self.phi) calc_spectrum(self) - specs[k,:]=self.spec - last_k=k - last_used_TS=int(i+L) - #if plot_chunks: + specs[k, :] = self.spec + last_k = k + last_used_TS = int(i + L) + # if plot_chunks: # MT.stats_format(self.spec, 'spec '+str(i)) - del(self.spec) + del self.spec else: if plot_chunks: - print('end of TS is reached') - print('last spec No: '+str(last_k)) - print('spec container: '+str(specs.shape)) - print('last used Timestep: '+str(last_used_TS)) - print('length of TS '+ str(dsize) +'ms') - - k+=1 + print("end of TS is reached") + print("last spec No: " + str(last_k)) + print("spec container: " + str(specs.shape)) + print("last used Timestep: " + str(last_used_TS)) + print("length of TS " + str(dsize) + "ms") + k += 1 if save_chunks: - self.chunks=chunks - #del(chunks) + self.chunks = chunks + # del(chunks) if prewhite is None: - self.specs=specs[:last_k,:] - self.spec_est=self.specs.mean(axis=0) - elif prewhite ==1: - self.specs=specs[:last_k,:]*(2*np.pi*self.f) - self.spec_est=self.specs.mean(axis=0) - elif prewhite ==2: - self.specs=specs[:last_k,:]*(2*np.pi*self.f)**2 - self.spec_est=self.specs.mean(axis=0) - - - self.n_spec,_=self.specs.shape + self.specs = specs[:last_k, :] + self.spec_est = self.specs.mean(axis=0) + elif prewhite == 1: + self.specs = specs[:last_k, :] * (2 * np.pi * self.f) + self.spec_est = self.specs.mean(axis=0) + elif prewhite == 2: + self.specs = specs[:last_k, :] * (2 * np.pi * self.f) ** 2 + self.spec_est = self.specs.mean(axis=0) + + self.n_spec, _ = self.specs.shape self.calc_var() - #self.phi=self.data - #self.phi*=win*np.sqrt(factor) + # self.phi=self.data + # self.phi*=win*np.sqrt(factor) def error(self, ci=0.95): - self.El, self.Eu =spec_error(self.spec_est,self.n_spec,ci=ci) + self.El, self.Eu = spec_error(self.spec_est, self.n_spec, ci=ci) def parceval(self): - print('Parcevals Theorem:') - print('variance of unweighted timeseries: ',self.data.var()) - print('mean variance of timeseries chunks: ',self.chunks.var(axis=1).mean() if self.save_chunks is True else 'data not saved') - #print('variance of weighted timeseries: ',self.phi.var() ) - #self.calc_var(self) - print('variance of the pwelch Spectrum: ',self.var) + print("Parcevals Theorem:") + print("variance of unweighted timeseries: ", self.data.var()) + print( + "mean variance of timeseries chunks: ", + ( + self.chunks.var(axis=1).mean() + if self.save_chunks is True + else "data not saved" + ), + ) + # print('variance of weighted timeseries: ',self.phi.var() ) + # self.calc_var(self) + print("variance of the pwelch Spectrum: ", self.var) def calc_var(self): - """ Compute total variance from spectrum """ - self.var = self.df*self.specs[1:].mean(axis=0).sum() # do not consider zeroth frequency + """Compute total variance from spectrum""" + self.var = ( + self.df * self.specs[1:].mean(axis=0).sum() + ) # do not consider zeroth frequency + class Spectogram_subsample(pwelch): - def __init__(self, data, dt, dt_unit=None, timestamp=None, L=None, ov=None, subL=None, window=None, save_chunks=False, plot_chunks=False, ci=0.95 ): + def __init__( + self, + data, + dt, + dt_unit=None, + timestamp=None, + L=None, + ov=None, + subL=None, + window=None, + save_chunks=False, + plot_chunks=False, + ci=0.95, + ): import m_general_ph3 as M - self.hist='Subsampled Spectogram' - L=int(np.round(data.size/10)) if L is None else L + + self.hist = "Subsampled Spectogram" + L = int(np.round(data.size / 10)) if L is None else L if type(L) != int: - #print('Length = ') + # print('Length = ') M.echo_dt(L) - self.write_log('Length = '+ M.echo_dt(L, as_string=True)) - L=int(L.item().total_seconds()) + self.write_log("Length = " + M.echo_dt(L, as_string=True)) + L = int(L.item().total_seconds()) else: - print('unknown L type') - self.write_log('Length = '+ 'unknown L type') + print("unknown L type") + self.write_log("Length = " + "unknown L type") - subL=int(np.round(L/10)) if subL is None else subL + subL = int(np.round(L / 10)) if subL is None else subL if type(subL) != int: - #print('Subsample Length= ') - #M.echo_dt(subL) - self.write_log('Subsample Length= '+ M.echo_dt(subL, as_string=True)) - subL=int(subL.item().total_seconds()) + # print('Subsample Length= ') + # M.echo_dt(subL) + self.write_log("Subsample Length= " + M.echo_dt(subL, as_string=True)) + subL = int(subL.item().total_seconds()) else: - #print('unknown subL type') - self.write_log('Length = '+ 'unknown subL type') + # print('unknown subL type') + self.write_log("Length = " + "unknown subL type") - ov=int(np.round(L/2.0)) if ov is None else ov + ov = int(np.round(L / 2.0)) if ov is None else ov if type(ov) != int: - #print('overlab Length= ') - #M.echo_dt(ov) - ov=int(ov.item().total_seconds()) + # print('overlab Length= ') + # M.echo_dt(ov) + ov = int(ov.item().total_seconds()) else: pass - #print('ov type:', type(ov)) + # print('ov type:', type(ov)) self.n = subL if window is None: - win=np.hanning(self.n) + win = np.hanning(self.n) else: - win=window - + win = window self.dt = float(dt) # test if n is even - if (self.n%2): + if self.n % 2: self.neven = False else: self.neven = True calc_freq(self) - #dt_inms = np.timedelta64(dt, dt_unit).astype('m8[ms]').astype(float) - data_size_adjust = dt * data.size # datasize in seconds - - nbin=np.floor(data_size_adjust /(L-ov)) - self.nbin=nbin - self.dt_periodogram=L-ov - specs=np.empty([int(nbin),self.f.size]) - error_El=[]#np.empty([int(nbin),self.f.size]) - error_Ey=[]#np.empty([int(nbin),self.f.size]) - dbin = .005 - yN = np.arange(0,2.+dbin,dbin) - - n_specs=[] - k=0 - print('subL', subL) - print('L', L) + # dt_inms = np.timedelta64(dt, dt_unit).astype('m8[ms]').astype(float) + data_size_adjust = dt * data.size # datasize in seconds + + nbin = np.floor(data_size_adjust / (L - ov)) + self.nbin = nbin + self.dt_periodogram = L - ov + specs = np.empty([int(nbin), self.f.size]) + error_El = [] # np.empty([int(nbin),self.f.size]) + error_Ey = [] # np.empty([int(nbin),self.f.size]) + dbin = 0.005 + yN = np.arange(0, 2.0 + dbin, dbin) + + n_specs = [] + k = 0 + print("subL", subL) + print("L", L) print(data_size_adjust) - #print(specs.shape, np.arange(0,data.size,int(L-ov)), np.arange(0,data.size-L,int(L-ov)).shape, ) - #print(specs.shape,np.arange(0,data_size_adjust-int(L-ov)+1,int(L-ov)) ) - for i in np.arange(0,data_size_adjust-int(L-ov)+1,int(L-ov)): + # print(specs.shape, np.arange(0,data.size,int(L-ov)), np.arange(0,data.size-L,int(L-ov)).shape, ) + # print(specs.shape,np.arange(0,data_size_adjust-int(L-ov)+1,int(L-ov)) ) + for i in np.arange(0, data_size_adjust - int(L - ov) + 1, int(L - ov)): - phi=data[int(i):int(i+L)] - #self.ii=np.append(self.ii,[i,i+L]) - #print(self.phi.max()) + phi = data[int(i) : int(i + L)] + # self.ii=np.append(self.ii,[i,i+L]) + # print(self.phi.max()) - #print(self.phi.mean()) - #print(phi.shape) - #print('i',i, i+L, data.size, i+L <= data.size) - #print(chunk.size, l) - if i+L <= data.size: + # print(self.phi.mean()) + # print(phi.shape) + # print('i',i, i+L, data.size, i+L <= data.size) + # print(chunk.size, l) + if i + L <= data.size: if plot_chunks: plt.plot(phi) if save_chunks: - chunks[k,:]=phi - - phi=signal.detrend(phi)#*win - - pwelch.__init__(self,phi,dt,L=subL,window=window, save_chunks=False, plot_chunks=False, periodogram=False) - - specs[k,:]=self.spec_est - sn,_=self.specs.shape - n_specs=np.append(n_specs, sn) - El,Ey=yNlu(sn,yN,ci=.95) - error_El=np.append(error_El, El) - error_Ey=np.append(error_Ey, Ey) - #error_El[k,:]=El - #error_Ey[k,:]=Ey - - del(self.spec_est) - elif i+L > data.size: - phi=data[-L:] - #print('phi shape',phi.shape ) + chunks[k, :] = phi + + phi = signal.detrend(phi) # *win + + pwelch.__init__( + self, + phi, + dt, + L=subL, + window=window, + save_chunks=False, + plot_chunks=False, + periodogram=False, + ) + + specs[k, :] = self.spec_est + sn, _ = self.specs.shape + n_specs = np.append(n_specs, sn) + El, Ey = yNlu(sn, yN, ci=0.95) + error_El = np.append(error_El, El) + error_Ey = np.append(error_Ey, Ey) + # error_El[k,:]=El + # error_Ey[k,:]=Ey + + del self.spec_est + elif i + L > data.size: + phi = data[-L:] + # print('phi shape',phi.shape ) if plot_chunks: plt.plot(phi) if save_chunks: - chunks[k,:]=phi - - phi=signal.detrend(phi)#*win - - pwelch.__init__(self,phi,dt,L=subL,window=window, save_chunks=False, plot_chunks=False, periodogram=False) - - specs[k,:]=self.spec_est - sn,_=self.specs.shape - n_specs=np.append(n_specs, sn) - El,Ey=yNlu(sn,yN,ci=.95) - error_El=np.append(error_El, El) - error_Ey=np.append(error_Ey, Ey) - #error_El[k,:]=El - #error_Ey[k,:]=Ey - - del(self.spec_est) - - #print(k) - #print('------') - k+=1 - - - self.n_specs=n_specs + chunks[k, :] = phi + + phi = signal.detrend(phi) # *win + + pwelch.__init__( + self, + phi, + dt, + L=subL, + window=window, + save_chunks=False, + plot_chunks=False, + periodogram=False, + ) + + specs[k, :] = self.spec_est + sn, _ = self.specs.shape + n_specs = np.append(n_specs, sn) + El, Ey = yNlu(sn, yN, ci=0.95) + error_El = np.append(error_El, El) + error_Ey = np.append(error_Ey, Ey) + # error_El[k,:]=El + # error_Ey[k,:]=Ey + + del self.spec_est + + # print(k) + # print('------') + k += 1 + + self.n_specs = n_specs # assign output values - self.data=specs # main data matrix - self.specs =specs - self.error_El=error_El - self.error_Ey=error_Ey - self.n_spec=n_specs + self.data = specs # main data matrix + self.specs = specs + self.error_El = error_El + self.error_Ey = error_Ey + self.n_spec = n_specs self.calc_var() - - if dt_unit is not None and type(dt_unit) == str: - dt_timedelta=np.timedelta64(int(dt),dt_unit) + dt_timedelta = np.timedelta64(int(dt), dt_unit) elif dt_unit is None: - dt_unit='s' - dt_timedelta=np.timedelta64(dt,dt_unit) + dt_unit = "s" + dt_timedelta = np.timedelta64(dt, dt_unit) - #print('sample resolution:') - #M.echo_dt(dt_timedelta) - self.write_log('basic sample resolution:'+ M.echo_dt(dt_timedelta, as_string=True)) + # print('sample resolution:') + # M.echo_dt(dt_timedelta) + self.write_log( + "basic sample resolution:" + M.echo_dt(dt_timedelta, as_string=True) + ) - timeres=np.timedelta64(int(self.dt_periodogram), 's') - #print('time resolution:') - #M.echo_dt(timeres) - self.write_log('Spectogram time res:'+ M.echo_dt(timeres, as_string=True)) + timeres = np.timedelta64(int(self.dt_periodogram), "s") + # print('time resolution:') + # M.echo_dt(timeres) + self.write_log("Spectogram time res:" + M.echo_dt(timeres, as_string=True)) if timestamp is None: - start_time=np.datetime64(0,dt_unit) - end_time=start_time+timeres*self.n_spec + start_time = np.datetime64(0, dt_unit) + end_time = start_time + timeres * self.n_spec else: - start_time=timestamp[0] - end_time=timestamp[-1] - - #print('Periodogram starttime and endtime:') - #print(start_time) - #print(end_time) - self.write_log('Spectogram starttme:'+ str(start_time)) - self.write_log(' endtime:'+ str(end_time)) - - #print(type(start_time), type(L), type(end_time), type(dt_timedelta), print(timeres) ) - time = np.arange(start_time+int(L/2), end_time+dt_timedelta, timeres) - #print(time) + start_time = timestamp[0] + end_time = timestamp[-1] + + # print('Periodogram starttime and endtime:') + # print(start_time) + # print(end_time) + self.write_log("Spectogram starttme:" + str(start_time)) + self.write_log(" endtime:" + str(end_time)) + + # print(type(start_time), type(L), type(end_time), type(dt_timedelta), print(timeres) ) + time = np.arange(start_time + int(L / 2), end_time + dt_timedelta, timeres) + # print(time) if time.shape[0] != self.data.shape[0]: - time=time[0:self.data.shape[0]] - self.time=time - self.timeres=timeres - self.dt_unit=dt_unit - #print(self.time) - self.time_dict=create_timeaxis_collection(self.time) - self.write_log('created timestamp collection') + time = time[0 : self.data.shape[0]] + self.time = time + self.timeres = timeres + self.dt_unit = dt_unit + # print(self.time) + self.time_dict = create_timeaxis_collection(self.time) + self.write_log("created timestamp collection") self.log() def write_log(self, s, verbose=False): from m_tools_ph3 import write_log - self.hist=write_log(self.hist, s, verbose=verbose) + + self.hist = write_log(self.hist, s, verbose=verbose) + def log(self): - print('.hist variable') + print(".hist variable") print(self.hist) + def power_anomalie(self, clim=None): - dd=10*np.log10(self.data[:,:]) - #print(dd) - #if anomalie is True: + dd = 10 * np.log10(self.data[:, :]) + # print(dd) + # if anomalie is True: # dd_tmp=dd.mean(axis=0).repeat(self.time.size-1) # dd=dd- dd_tmp.reshape(self.fs.size,self.time.size-1).T - self.data_power_mean=np.nanmedian(dd, axis=0) if clim is None else 10*np.log10(clim) - dd_tmp=self.data_power_mean.repeat(self.time.size) + self.data_power_mean = ( + np.nanmedian(dd, axis=0) if clim is None else 10 * np.log10(clim) + ) + dd_tmp = self.data_power_mean.repeat(self.time.size) print(self.data_power_mean.shape) - print(self.f.size,self.time.size) - self.data_power_ano=dd- dd_tmp.reshape(self.f.size,self.time.size).T + print(self.f.size, self.time.size) + self.data_power_ano = dd - dd_tmp.reshape(self.f.size, self.time.size).T def anomalie(self, clim=None): - #dd= - #print(dd) - #if anomalie is True: + # dd= + # print(dd) + # if anomalie is True: # dd_tmp=dd.mean(axis=0).repeat(self.time.size-1) # dd=dd- dd_tmp.reshape(self.fs.size,self.time.size-1).T - self.data_mean=np.nanmedian(self.data, axis=0) if clim is None else clim - dd_tmp=self.data_mean.repeat(self.time.size) - self.data_ano=self.data- dd_tmp.reshape(self.f.size,self.time.size).T + self.data_mean = np.nanmedian(self.data, axis=0) if clim is None else clim + dd_tmp = self.data_mean.repeat(self.time.size) + self.data_ano = self.data - dd_tmp.reshape(self.f.size, self.time.size).T class Periodogram(pwelch): - def __init__(self,data,dt,dt_unit=None, timestamp=None,L=None, ov=None,window=None, save_chunks=False, plot_chunks=False): - - pwelch.__init__(self,data,dt,L, ov,window, save_chunks, plot_chunks, periodogram=True) - self.data=self.specs + def __init__( + self, + data, + dt, + dt_unit=None, + timestamp=None, + L=None, + ov=None, + window=None, + save_chunks=False, + plot_chunks=False, + ): + + pwelch.__init__( + self, data, dt, L, ov, window, save_chunks, plot_chunks, periodogram=True + ) + self.data = self.specs if dt_unit is not None and type(dt_unit) == str: - dt_timedelta=np.timedelta64(int(dt),dt_unit) + dt_timedelta = np.timedelta64(int(dt), dt_unit) elif dt_unit is None: - dt_unit='s' - dt_timedelta=np.timedelta64(dt,dt_unit) + dt_unit = "s" + dt_timedelta = np.timedelta64(dt, dt_unit) - print('sample resolution:') + print("sample resolution:") M.echo_dt(dt_timedelta) - timeres=np.timedelta64(int(self.dt_periodogram), dt_unit) - print('time resolution:') + timeres = np.timedelta64(int(self.dt_periodogram), dt_unit) + print("time resolution:") M.echo_dt(timeres) if timestamp is None: - start_time=np.datetime64(0,dt_unit) - end_time=start_time+timeres*self.n_spec + start_time = np.datetime64(0, dt_unit) + end_time = start_time + timeres * self.n_spec else: - start_time=timestamp[0] - end_time=timestamp[-1] + start_time = timestamp[0] + end_time = timestamp[-1] - print('Periodogram starttime and endtime:') + print("Periodogram starttime and endtime:") print(start_time) print(end_time) - - time = np.arange(start_time+L/2, end_time+dt_timedelta, timeres) - #print(time) + time = np.arange(start_time + L / 2, end_time + dt_timedelta, timeres) + # print(time) if time.shape[0] != self.data.shape[0]: - time=time[0:self.data.shape[0]] - self.time=time - self.timeres=timeres - self.dt_unit=dt_unit - #G.dt_data=dt_data + time = time[0 : self.data.shape[0]] + self.time = time + self.timeres = timeres + self.dt_unit = dt_unit + # G.dt_data=dt_data + + # self.time=np.arange(,nin*) - #self.time=np.arange(,nin*) def save_data(self, path=None, S=None): - P=save_data_periodogram(self , S=S) + P = save_data_periodogram(self, S=S) print(P.meta) - print('constructed class for saving') + print("constructed class for saving") save_file(P, path) class save_data_periodogram: - def __init__(self,P, S=None): - self.meta=S.meta if S is not None else '' - self.data_unit=S.unit if S is not None else '' - self.time=P.time - self.f=P.f - self.data=P.data - self.dt_unit=P.dt_unit - self.time=P.time - #self.y_lim=y_lim - self.timeres=P.timeres - - -def MEM_cal(moments_est,freq,theta=None ,flim=None): + def __init__(self, P, S=None): + self.meta = S.meta if S is not None else "" + self.data_unit = S.unit if S is not None else "" + self.time = P.time + self.f = P.f + self.data = P.data + self.dt_unit = P.dt_unit + self.time = P.time + # self.y_lim=y_lim + self.timeres = P.timeres + + +def MEM_cal(moments_est, freq, theta=None, flim=None): import getmem as getmem """ @@ -844,33 +935,32 @@ def MEM_cal(moments_est,freq,theta=None ,flim=None): """ - theta=np.arange(1,361) if theta is None else theta - flim=(freq[0], freq[-1]) if flim is None else flim + theta = np.arange(1, 361) if theta is None else theta + flim = (freq[0], freq[-1]) if flim is None else flim - freq_sel_bool=M.cut_nparray(freq,flim[0], flim[1]) - freq_sel=freq[freq_sel_bool] + freq_sel_bool = M.cut_nparray(freq, flim[0], flim[1]) + freq_sel = freq[freq_sel_bool] + N_sel = dict() + for k, I in moments_est.iteritems(): + N_sel[k] = moments_est[k][freq_sel_bool] - N_sel=dict() - for k,I in moments_est.iteritems(): - N_sel[k]=moments_est[k][freq_sel_bool] - - d1=N_sel['Q12'] / np.sqrt(N_sel['P11'] *(N_sel['P22'] + N_sel['P33'] )) + d1 = N_sel["Q12"] / np.sqrt(N_sel["P11"] * (N_sel["P22"] + N_sel["P33"])) # Lygre and Krongstad 1986 have here sqrt(N_sel['P11'] *(N_sel['P22'] + N_sel['P33']). I guess its a typo. - d2=N_sel['Q13'] / np.sqrt(N_sel['P11'] *(N_sel['P22'] + N_sel['P33']) ) + d2 = N_sel["Q13"] / np.sqrt(N_sel["P11"] * (N_sel["P22"] + N_sel["P33"])) - d3=(N_sel['P22'] - N_sel['P33']) / (N_sel['P22'] + N_sel['P33']) - d4=2*N_sel['P23'] / (N_sel['P22'] + N_sel['P33']) + d3 = (N_sel["P22"] - N_sel["P33"]) / (N_sel["P22"] + N_sel["P33"]) + d4 = 2 * N_sel["P23"] / (N_sel["P22"] + N_sel["P33"]) - S=N_sel['P11']/freq_sel # define power spectrum + S = N_sel["P11"] / freq_sel # define power spectrum - D= GetMem(d1, d2, d3, d3, theta ) - E=np.tile(S.T, (theta.size,1)).T*D + D = GetMem(d1, d2, d3, d3, theta) + E = np.tile(S.T, (theta.size, 1)).T * D return D, S, E, freq_sel, theta -def GetMem(a1,b1,a2,b2, theta): +def GetMem(a1, b1, a2, b2, theta): import numpy as np from numpy import pi, sin, cos @@ -900,39 +990,39 @@ def GetMem(a1,b1,a2,b2, theta): where N is the number of frequency bands """ - d1 = np.reshape(a1,[len(a1),1]) - d2 = np.reshape(b1,[len(b1),1]) - d3 = np.reshape(a2,[len(a2),1]) - d4 = np.reshape(b2,[len(b2),1]) + d1 = np.reshape(a1, [len(a1), 1]) + d2 = np.reshape(b1, [len(b1), 1]) + d3 = np.reshape(a2, [len(a2), 1]) + d4 = np.reshape(b2, [len(b2), 1]) - c1 = (1.+ 0j)*d1 + (0 + 1.j)*d2 - c2 = (1.+ 0j)*d3 + (0 + 1.j)*d4 + c1 = (1.0 + 0j) * d1 + (0 + 1.0j) * d2 + c2 = (1.0 + 0j) * d3 + (0 + 1.0j) * d4 - p1 = (c1 - c2*c1.conjugate())/(1-abs(c1)**2) - p2 = c2 - c1*p1 - x = 1.-p1*c1.conjugate() - p2*c2.conjugate() + p1 = (c1 - c2 * c1.conjugate()) / (1 - abs(c1) ** 2) + p2 = c2 - c1 * p1 + x = 1.0 - p1 * c1.conjugate() - p2 * c2.conjugate() # calculate MEM using 1 deg resolution a = theta.T - #a = np.arange(1,361).T - a = a*pi/180. - a = np.reshape(a,[1,len(a)]) + # a = np.arange(1,361).T + a = a * pi / 180.0 + a = np.reshape(a, [1, len(a)]) - e1 = (1. + 0j)*cos(a) - (0 + 1.j)*sin(a) - e2 = (1. + 0j)*cos(2*a) - (0 + 1.j)*sin(2*a) - y = abs((1.+ 0j) - np.dot(p1,e1) - np.dot(p2,e2))**2 + e1 = (1.0 + 0j) * cos(a) - (0 + 1.0j) * sin(a) + e2 = (1.0 + 0j) * cos(2 * a) - (0 + 1.0j) * sin(2 * a) + y = abs((1.0 + 0j) - np.dot(p1, e1) - np.dot(p2, e2)) ** 2 - mem = abs((x*np.ones([1,a.size]))/y) + mem = abs((x * np.ones([1, a.size])) / y) summing = np.sum(mem, axis=1) - sum_mem = 1./np.reshape(summing, [len(summing),1]) - norm_mem = np.dot(sum_mem, np.ones([1,a.size]))*mem + sum_mem = 1.0 / np.reshape(summing, [len(summing), 1]) + norm_mem = np.dot(sum_mem, np.ones([1, a.size])) * mem return norm_mem -def peak_angle(data, freq,fpos, max_jump=5, smooth_l=5): +def peak_angle(data, freq, fpos, max_jump=5, smooth_l=5): """ This method computes the frequency of D mean between the given frequency limits fpos (fpos[0], fpos[-1]) it identifies the maxima and save them as degree from true north @@ -942,65 +1032,76 @@ def peak_angle(data, freq,fpos, max_jump=5, smooth_l=5): Return: peak angles index orded buy the peak height """ - #max_jump=5 - #smooth_l=5 + # max_jump=5 + # smooth_l=5 if data.shape[0] != freq.size: - raise Warning('shape of data in dim=0 must be equal to freq') - - f_bool=M.cut_nparray(freq, fpos[0], fpos[-1]) - dregree_series=data[f_bool,:].mean(0) - #plt.plot(dregree_series) - #print(dregree_series) - - a=M.find_max_ts(dregree_series,smooth=True, spreed=smooth_l, plot=False, jump=max_jump,verbose=False) - if len(a[0]) ==0: - a=M.find_max_ts(dregree_series,smooth=False, spreed=1, plot=False, jump=None,verbose=False) - - angles=np.flipud(np.sort(a[0])) + raise Warning("shape of data in dim=0 must be equal to freq") + + f_bool = M.cut_nparray(freq, fpos[0], fpos[-1]) + dregree_series = data[f_bool, :].mean(0) + # plt.plot(dregree_series) + # print(dregree_series) + + a = M.find_max_ts( + dregree_series, + smooth=True, + spreed=smooth_l, + plot=False, + jump=max_jump, + verbose=False, + ) + if len(a[0]) == 0: + a = M.find_max_ts( + dregree_series, smooth=False, spreed=1, plot=False, jump=None, verbose=False + ) + + angles = np.flipud(np.sort(a[0])) return angles -def save_file( data, path): + +def save_file(data, path): import pickle - outfile=path - f = open(outfile, 'wb') - pickle.dump(data,f, pickle.HIGHEST_PROTOCOL) - print('saved to:',outfile) + + outfile = path + f = open(outfile, "wb") + pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) + print("saved to:", outfile) f.close() + ## ceasars funcitons -def yNlu(sn,yN,ci): - """ compute yN[l] yN[u], that is, the lower and - upper limit of yN """ +def yNlu(sn, yN, ci): + """compute yN[l] yN[u], that is, the lower and + upper limit of yN""" # cdf of chi^2 dist. with 2*sn DOF - cdf = gammainc(sn,sn*yN) + cdf = gammainc(sn, sn * yN) # indices that delimit the wedge of the conf. interval fl = np.abs(cdf - ci).argmin() - fu = np.abs(cdf - 1. + ci).argmin() - - return yN[fl],yN[fu] + fu = np.abs(cdf - 1.0 + ci).argmin() + return yN[fl], yN[fu] -def spec_error(E,sn,ci=.95): - """ Computes confidence interval for one-dimensional spectral - estimate E. +def spec_error(E, sn, ci=0.95): + """Computes confidence interval for one-dimensional spectral + estimate E. - Parameters - =========== - - sn is the number of spectral realizations; - it can be either an scalar or an array of size(E) - - ci = .95 for 95 % confidence interval + Parameters + =========== + - sn is the number of spectral realizations; + it can be either an scalar or an array of size(E) + - ci = .95 for 95 % confidence interval - Output - ========== - lower (El) and upper (Eu) bounds on E """ + Output + ========== + lower (El) and upper (Eu) bounds on E""" - dbin = .005 - yN = np.arange(0,2.+dbin,dbin) + dbin = 0.005 + yN = np.arange(0, 2.0 + dbin, dbin) El, Eu = np.empty_like(E), np.empty_like(E) @@ -1014,70 +1115,70 @@ def spec_error(E,sn,ci=.95): assert n == E.size, " *** sn has different size than E " for i in range(n): - yNl,yNu = yNlu(sn[i],yN=yN,ci=ci) - El[i] = E[i]/yNl - Eu[i] = E[i]/yNu + yNl, yNu = yNlu(sn[i], yN=yN, ci=ci) + El[i] = E[i] / yNl + Eu[i] = E[i] / yNu else: - yNl,yNu = yNlu(sn,yN=yN,ci=ci) - El = E/yNl - Eu = E/yNu + yNl, yNu = yNlu(sn, yN=yN, ci=ci) + El = E / yNl + Eu = E / yNu return El, Eu + # for llc output only; this is temporary -def spec_est(A,d,axis=2,window=True,detrend=True, prewhiten=False): +def spec_est(A, d, axis=2, window=True, detrend=True, prewhiten=False): - l1,l2,l3,l4 = A.shape + l1, l2, l3, l4 = A.shape - if axis==2: + if axis == 2: l = l3 if prewhiten: - if l3>1: - _,A,_ = np.gradient(A,d,d,1.) + if l3 > 1: + _, A, _ = np.gradient(A, d, d, 1.0) else: - _,A = np.gradient(A.squeeze(),d,d) - A = A[...,np.newaxis] + _, A = np.gradient(A.squeeze(), d, d) + A = A[..., np.newaxis] if detrend: - A = signal.detrend(A,axis=axis,type='linear') + A = signal.detrend(A, axis=axis, type="linear") if window: win = np.hanning(l) - win = (l/(win**2).sum())*win - win = win[np.newaxis,np.newaxis,:,np.newaxis] + win = (l / (win**2).sum()) * win + win = win[np.newaxis, np.newaxis, :, np.newaxis] else: - win = np.ones(l)[np.newaxis,np.newaxis,:,np.newaxis] - elif axis==1: + win = np.ones(l)[np.newaxis, np.newaxis, :, np.newaxis] + elif axis == 1: l = l2 if prewhiten: - if l3 >1: - A,_,_ = np.gradient(A,d,d,1.) + if l3 > 1: + A, _, _ = np.gradient(A, d, d, 1.0) else: - A,_ = np.gradient(A.squeeze(),d,d) - A = A[...,np.newaxis] + A, _ = np.gradient(A.squeeze(), d, d) + A = A[..., np.newaxis] if detrend: - A = signal.detrend(A,axis=1,type='linear') + A = signal.detrend(A, axis=1, type="linear") if window: win = np.hanning(l) - win = (l/(win**2).sum())*win - win = win[np.newaxis,...,np.newaxis,np.newaxis] + win = (l / (win**2).sum()) * win + win = win[np.newaxis, ..., np.newaxis, np.newaxis] else: - win = np.ones(l)[np.newaxis,...,np.newaxis,np.newaxis] - + win = np.ones(l)[np.newaxis, ..., np.newaxis, np.newaxis] - df = 1./(d*l) - f = np.arange(0,l/2+1)*df + df = 1.0 / (d * l) + f = np.arange(0, l / 2 + 1) * df - Ahat = np.fft.rfft(win*A,axis=axis) - Aabs = 2 * (Ahat*Ahat.conjugate()) / l + Ahat = np.fft.rfft(win * A, axis=axis) + Aabs = 2 * (Ahat * Ahat.conjugate()) / l if prewhiten: - if axis==1: - fd = 2*np.pi*f[np.newaxis,:, np.newaxis] + if axis == 1: + fd = 2 * np.pi * f[np.newaxis, :, np.newaxis] else: - fd = 2*np.pi*f[...,np.newaxis,np.newaxis] + fd = 2 * np.pi * f[..., np.newaxis, np.newaxis] - Aabs = Aabs/(fd**2) - Aabs[0,0] = 0. + Aabs = Aabs / (fd**2) + Aabs[0, 0] = 0.0 - return Aabs,f + return Aabs, f diff --git a/src/icesat2_tracks/local_modules/m_tools_ph3.py b/src/icesat2_tracks/local_modules/m_tools_ph3.py index 85122738..9fee033e 100644 --- a/src/icesat2_tracks/local_modules/m_tools_ph3.py +++ b/src/icesat2_tracks/local_modules/m_tools_ph3.py @@ -1,150 +1,176 @@ import matplotlib -#matplotlib.use('Agg') + +# matplotlib.use('Agg') import matplotlib.pyplot as plt import numpy as np -#from scipy.io import netcdf -#from mpl_toolkits.basemap import Basemap,cm + +# from scipy.io import netcdf +# from mpl_toolkits.basemap import Basemap,cm import os import pickle -#from matplotlib.dates import DateFormatter, MinuteLocator -#from matplotlib import dates -#import datetime as DT + +# from matplotlib.dates import DateFormatter, MinuteLocator +# from matplotlib import dates +# import datetime as DT def dt_form_timestamp(timestamp, unit=None): - unit='h' if unit is None else unit - return (timestamp[1]-timestamp[0]).astype('m8['+unit+']') + unit = "h" if unit is None else unit + return (timestamp[1] - timestamp[0]).astype("m8[" + unit + "]") def tick_formatter(a, interval=2, rounder=2, expt_flag=True, shift=0): - O=int(np.log10(a.max())) - fact=10**(O-1) - b=np.round(a/fact, rounder+1)*fact - ticklabels=[' ' for i in range(len(b))] - N=int(np.ceil(len(b)/interval)) + O = int(np.log10(a.max())) + fact = 10 ** (O - 1) + b = np.round(a / fact, rounder + 1) * fact + ticklabels = [" " for i in range(len(b))] + N = int(np.ceil(len(b) / interval)) - tt=np.arange(shift,len(b),interval) + tt = np.arange(shift, len(b), interval) for t in tt: if expt_flag: - ticklabels[int(t)]='{:.2e}'.format(b[t]) + ticklabels[int(t)] = "{:.2e}".format(b[t]) else: - ticklabels[int(t)]=format(b[t], '.2f').rstrip('0').rstrip('.')#'{:.{2}f}'.format(b[t]) + ticklabels[int(t)] = ( + format(b[t], ".2f").rstrip("0").rstrip(".") + ) #'{:.{2}f}'.format(b[t]) - #ticks=a + # ticks=a return ticklabels, b def freq_lim_string(low, high): - a='%2.1e' % low - b='%2.1e' % high + a = "%2.1e" % low + b = "%2.1e" % high + + return a[0:3] + "-" + b + " Hz" - return a[0:3] +'-'+ b +' Hz' -def float_to_str(flt, r= 1): +def float_to_str(flt, r=1): return str(np.round(flt, r)) + def num_to_str(flt): return str(int(np.round(flt, 0))) def mkdirs_r(path): import os + if not os.path.exists(path): - os.makedirs(path) + os.makedirs(path) + def check_year(inputstr, yearstring): - a=np.datetime64(inputstr).astype(object).year - ref=np.datetime64(yearstring).astype(object).year + a = np.datetime64(inputstr).astype(object).year + ref = np.datetime64(yearstring).astype(object).year if a == ref: return True else: return False + def datetime64_to_sec(d): - return d.astype('M8[s]').astype('float') + return d.astype("M8[s]").astype("float") + + def datetime64_to_day(d): - return d.astype('M8[D]').astype('float') + return d.astype("M8[D]").astype("float") + def float_plot_time_to_sec(pp): - return np.datetime64(dates.num2date(pp)).astype('M8[s]').astype('float') + return np.datetime64(dates.num2date(pp)).astype("M8[s]").astype("float") + + def float_plot_time_to_dt64(pp): - return np.datetime64(dates.num2date(pp)).astype('M8[s]') + return np.datetime64(dates.num2date(pp)).astype("M8[s]") + def sec_to_dt64(pp): - return pp.astype('M8[s]') + return pp.astype("M8[s]") + def sec_to_float_plot(pp): from matplotlib import dates import datetime as DT - return dates.date2num(pp.astype('M8[s]').astype(DT.datetime)) + + return dates.date2num(pp.astype("M8[s]").astype(DT.datetime)) + + def sec_to_float_plot_single(pp): from matplotlib import dates import datetime as DT - return dates.date2num(np.datetime64(int(pp), 's').astype('M8[s]').astype(DT.datetime)) + + return dates.date2num( + np.datetime64(int(pp), "s").astype("M8[s]").astype(DT.datetime) + ) + + def fake_2d_data(verbose=True, timeaxis=False): - x=np.arange(0,100,1) - y=np.arange(0,40,1) - XX, YY= np.meshgrid(x,y) - - mu=x.size/2 - sigma=x.size/5 - z2= 1/(sigma * np.sqrt(2 * np.pi)) * np.exp( - (XX - mu)**2 / (2 * sigma**2) ) - z2=z2/z2.max() - - mu=y.size/2 - sigma=y.size/5 - z3= 1/(sigma * np.sqrt(2 * np.pi)) * np.exp( - (YY - mu)**2 / (2 * sigma**2) ) - z3=z3/z3.max() + x = np.arange(0, 100, 1) + y = np.arange(0, 40, 1) + XX, YY = np.meshgrid(x, y) + + mu = x.size / 2 + sigma = x.size / 5 + z2 = 1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(-((XX - mu) ** 2) / (2 * sigma**2)) + z2 = z2 / z2.max() + + mu = y.size / 2 + sigma = y.size / 5 + z3 = 1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(-((YY - mu) ** 2) / (2 * sigma**2)) + z3 = z3 / z3.max() if verbose: - print('x' , x.shape) - print('y' , y.shape) - print('z' , z3.shape) + print("x", x.shape) + print("y", y.shape) + print("z", z3.shape) - plt.contourf(x, y,z2/2+z3/2) + plt.contourf(x, y, z2 / 2 + z3 / 2) plt.colorbar() - plt.axis('scaled') + plt.axis("scaled") plt.show() - return x, y, z3 + def pickle_save(name, path, data, verbose=True): if not os.path.exists(path): os.makedirs(path) - full_name= (os.path.join(path,name+ '.npy')) - + full_name = os.path.join(path, name + ".npy") - with open(full_name, 'wb') as f2: + with open(full_name, "wb") as f2: pickle.dump(data, f2) if verbose: - print('save at: ',full_name) + print("save at: ", full_name) + def pickle_load(name, path, verbose=True): - #if not os.path.exists(path): + # if not os.path.exists(path): # os.makedirs(path) - full_name= (os.path.join(path,name+ '.npy')) + full_name = os.path.join(path, name + ".npy") - with open(full_name, 'r') as f: - data=pickle.load(f) + with open(full_name, "r") as f: + data = pickle.load(f) if verbose: - print('load from: ',full_name) + print("load from: ", full_name) return data def json_save(name, path, data, verbose=False, return_name=False): import json + if not os.path.exists(path): os.makedirs(path) - full_name_root=os.path.join(path,name) - full_name= (os.path.join(full_name_root+ '.json')) - with open(full_name, 'w') as outfile: + full_name_root = os.path.join(path, name) + full_name = os.path.join(full_name_root + ".json") + with open(full_name, "w") as outfile: json.dump(data, outfile, indent=2) if verbose: - print('save at: ',full_name) + print("save at: ", full_name) if return_name: return full_name_root else: @@ -153,11 +179,10 @@ def json_save(name, path, data, verbose=False, return_name=False): def json_save2(name, path, data, verbose=False, return_name=False): import json + class CustomJSONizer(json.JSONEncoder): def default(self, obj): - return bool(obj) \ - if isinstance(obj, np.bool_) \ - else super().default(obj) + return bool(obj) if isinstance(obj, np.bool_) else super().default(obj) # return super().encode(bool(obj)) \ # if isinstance(obj, np.bool_) \ @@ -165,12 +190,12 @@ def default(self, obj): if not os.path.exists(path): os.makedirs(path) - full_name_root=os.path.join(path,name) - full_name= (os.path.join(full_name_root+ '.json')) - with open(full_name, 'w') as outfile: + full_name_root = os.path.join(path, name) + full_name = os.path.join(full_name_root + ".json") + with open(full_name, "w") as outfile: json.dump(data, outfile, cls=CustomJSONizer, indent=2) if verbose: - print('save at: ',full_name) + print("save at: ", full_name) if return_name: return full_name_root else: @@ -179,97 +204,99 @@ def default(self, obj): def json_load(name, path, verbose=False): import json - full_name= (os.path.join(path,name+ '.json')) - with open(full_name, 'r') as ifile: - data=json.load(ifile) + full_name = os.path.join(path, name + ".json") + + with open(full_name, "r") as ifile: + data = json.load(ifile) if verbose: - print('loaded from: ',full_name) + print("loaded from: ", full_name) return data + def h5_load(name, path, verbose=False): import pandas as pd - full_name= (os.path.join(path,name+ '.h5')) - data=pd.read_hdf(full_name) - #with pd.HDFStore(full_name) as data: + + full_name = os.path.join(path, name + ".h5") + data = pd.read_hdf(full_name) + # with pd.HDFStore(full_name) as data: # data = pd.HDFStore(path+self.ID+'.h5') # data.close() return data + def h5_load_v2(name, path, verbose=False): import h5py - h5f = h5py.File(path + name + '.h5','r') + h5f = h5py.File(path + name + ".h5", "r") if verbose: print(h5f.keys()) - data_dict=dict() + data_dict = dict() for k, I in h5f.iteritems(): - data_dict[k] =I[:] + data_dict[k] = I[:] h5f.close() return data_dict -def h5_save(name, path, data_dict, verbose=False, mode='w'): + +def h5_save(name, path, data_dict, verbose=False, mode="w"): import h5py - mode = 'w' if mode is None else mode + mode = "w" if mode is None else mode if not os.path.exists(path): os.makedirs(path) - full_name= (os.path.join(path,name+ '.h5')) + full_name = os.path.join(path, name + ".h5") store = h5py.File(full_name, mode) - - for k, I in list(data_dict.items()): - store[k]=I + store[k] = I store.close() if verbose: - print('saved at: ' +full_name) + print("saved at: " + full_name) - -def h5_save(name, path, data_dict, verbose=False, mode='w'): +def h5_save(name, path, data_dict, verbose=False, mode="w"): import h5py - mode = 'w' if mode is None else mode + mode = "w" if mode is None else mode if not os.path.exists(path): os.makedirs(path) - full_name= (os.path.join(path,name+ '.h5')) + full_name = os.path.join(path, name + ".h5") store = h5py.File(full_name, mode) - - for k, I in list(data_dict.items()): - store[k]=I + store[k] = I store.close() if verbose: - print('saved at: ' +full_name) + print("saved at: " + full_name) -def load_pandas_table_dict(name , save_path): +def load_pandas_table_dict(name, save_path): import warnings from pandas import HDFStore from pandas.io.pytables import PerformanceWarning - warnings.filterwarnings('ignore',category=PerformanceWarning) - return_dict=dict() - with HDFStore(save_path+'/'+name+'.h5') as store: - #print(store) - #print(store.keys()) + warnings.filterwarnings("ignore", category=PerformanceWarning) + + return_dict = dict() + with HDFStore(save_path + "/" + name + ".h5") as store: + # print(store) + # print(store.keys()) for k in store.keys(): - return_dict[k[1:]]=store.get(k) + return_dict[k[1:]] = store.get(k) return return_dict -def save_pandas_table(table_dict, ID , save_path): + +def save_pandas_table(table_dict, ID, save_path): if not os.path.exists(save_path): os.makedirs(save_path) @@ -277,102 +304,125 @@ def save_pandas_table(table_dict, ID , save_path): import warnings from pandas import HDFStore from pandas.io.pytables import PerformanceWarning - warnings.filterwarnings('ignore',category=PerformanceWarning) - with HDFStore(save_path+'/'+ID+'.h5') as store: - for name,table in table_dict.items(): - store[name]=table + warnings.filterwarnings("ignore", category=PerformanceWarning) + + with HDFStore(save_path + "/" + ID + ".h5") as store: + for name, table in table_dict.items(): + store[name] = table -def write_log(hist, string, verbose=False, short=True , date=True): +def write_log(hist, string, verbose=False, short=True, date=True): import datetime as datetime + if short: now = datetime.datetime.now().strftime("%Y%m%d") else: now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") if date: - message='\n'+now+' '+string + message = "\n" + now + " " + string else: - message='\n '.ljust(2)+' '+string + message = "\n ".ljust(2) + " " + string - if verbose== True: + if verbose == True: print(message) - elif verbose == 'all': - print(hist+message) - return hist+message + elif verbose == "all": + print(hist + message) + return hist + message + def add_line_var(ss, name, var): - return ss + '\n '+name.ljust(5) + str(var) + return ss + "\n " + name.ljust(5) + str(var) + def write_variables_log(hist, var_list, locals, verbose=False, date=False): import datetime as datetime now = datetime.datetime.now().strftime("%Y%m%d") - var_dict=dict( (name,locals[name]) for name in var_list ) - stringg='' - for name,I in var_dict.items(): - stringg=stringg+ '\n '+name.ljust(5) + str(I) + var_dict = dict((name, locals[name]) for name in var_list) + stringg = "" + for name, I in var_dict.items(): + stringg = stringg + "\n " + name.ljust(5) + str(I) if date: - message='\n'+now+' '+stringg + message = "\n" + now + " " + stringg else: - message='\n '.ljust(5)+' '+stringg + message = "\n ".ljust(5) + " " + stringg - if verbose== True: + if verbose == True: print(message) - elif verbose == 'all': - print(hist+message) - return hist+message + elif verbose == "all": + print(hist + message) + return hist + message -def save_log_txt(name, path, hist,verbose=False): +def save_log_txt(name, path, hist, verbose=False): if not os.path.exists(path): os.makedirs(path) - full_name= (os.path.join(path,name+ '.hist.txt')) - with open(full_name, 'w') as ifile: + full_name = os.path.join(path, name + ".hist.txt") + with open(full_name, "w") as ifile: ifile.write(str(hist)) if verbose: - print('saved at: ',full_name) + print("saved at: ", full_name) + def load_log_txt(name, path): import glob - hist_file=name#'DR01.LHN.stormdetect.A02_geometry_cut_storm.hist.txt' #ID.string+'.A02**.txt' - f=[] - for h in glob.glob(os.path.join(path,hist_file)): - f.append(open(h, 'r').read()) - return '\n'.join(f) + + hist_file = name #'DR01.LHN.stormdetect.A02_geometry_cut_storm.hist.txt' #ID.string+'.A02**.txt' + f = [] + for h in glob.glob(os.path.join(path, hist_file)): + f.append(open(h, "r").read()) + return "\n".join(f) + def shape(a): - for i in a: - print(i.shape) + for i in a: + print(i.shape) -def find_O(a, case='round'): - if case=='round': - for k in np.logspace(0,24,25): - if np.ceil(a/k) == 1: + +def find_O(a, case="round"): + if case == "round": + for k in np.logspace(0, 24, 25): + if np.ceil(a / k) == 1: return k break - elif case=='floor': - for k in np.logspace(0,24,25): - if np.ceil(a/k) == 1: + elif case == "floor": + for k in np.logspace(0, 24, 25): + if np.ceil(a / k) == 1: return k break - elif case=='ceil': - for k in np.logspace(0,24,25): - if np.ceil(a/k) == 1: + elif case == "ceil": + for k in np.logspace(0, 24, 25): + if np.ceil(a / k) == 1: return k break else: - raise Warning('no propper case') + raise Warning("no propper case") + def stats(a): - print('shape' , a.shape) - print('Nans',np.sum(np.isnan(a))) - print('max' , np.nanmax(a)) - print('min' ,np.nanmin(a)) - print('mean' ,np.nanmean(a)) + print("shape", a.shape) + print("Nans", np.sum(np.isnan(a))) + print("max", np.nanmax(a)) + print("min", np.nanmin(a)) + print("mean", np.nanmean(a)) + def stats_format(a, name=None): - print('Name:', str(name),' Shape:' , a.shape ,' NaNs:',np.sum(np.isnan(a)),' max:', np.nanmax(a),' min', np.nanmin(a),' mean:', np.nanmean(a)) + print( + "Name:", + str(name), + " Shape:", + a.shape, + " NaNs:", + np.sum(np.isnan(a)), + " max:", + np.nanmax(a), + " min", + np.nanmin(a), + " mean:", + np.nanmean(a), + )