diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..2df66795 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.spec +*.pyc +*.nc diff --git a/.project b/.project new file mode 100644 index 00000000..0c810dbf --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + PyConform + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 00000000..b372d745 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,9 @@ + + + +python 2.7 +Homebrew Python 2.7 + +/${PROJECT_DIR_NAME}/source + + diff --git a/CHANGES.rst b/CHANGES.rst index 98373e44..f63be817 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,18 @@ See the LICENSE.rst file for details VERSION 0.0.1 ------------- +06 Jan 2016: + - Add date check/mapping ability. + +19 Nov 2015: + - Added initial CESM to CMIP6 JSON tables to the examples directory. + - Added the code within examples/CESM/CMIP6/src used to generate these tables. + +18 Nov 2015: + - Added initial CESM to CMIP5 JSON tables to examples directory + +17 Nov 2015: + - Add initial version of mip_table_parser.py. 03 Nov 2015: - Add initial versions of climIO.py and its unit test. diff --git a/README.rst b/README.rst index 61d8d2c6..b07db02b 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ A package for transforming a NetCDF dataset into a defined format suitable for publication according to a defined publication standard. :AUTHORS: Sheri Mickelson, Kevin Paul -:COPYRIGHT: 2015, University Corporation for Atmospheric Research +:COPYRIGHT: 2016, University Corporation for Atmospheric Research :LICENSE: See the LICENSE.rst file for details Send questions and comments to Kevin Paul (kpaul@ucar.edu) or diff --git a/examples/CESM/CMIP5/CMIP5_3hr.json b/examples/CESM/CMIP5/CMIP5_3hr.json new file mode 100644 index 00000000..11d299d7 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_3hr.json @@ -0,0 +1,23 @@ +"clt=CLDTOT unit conversion" +"hfls=LHFLX no change" +"hfss=SHFLX no change" +"huss=QREFHT no change" +"mrro=UNAVAILABLE" +"mrsos=UNAVAILABLE" +"pr=PRECC + PRECL and unit conversion" +"prc=PRECC unit conversion" +"prsn=PRECSC + PRECSL and unit conversion" +"ps=PS no change" +"rlds=FLDS no change" +"rldscs=FLDSC no change" +"rlus=FLDS + FLNS" +"rsds=FSDS no change" +"rsdsdiff=UNAVAILABLE" +"rsdscs=FSDSC no change" +"rsus=FSDS - FSNS" +"rsuscs=Would be FSDSC - FSNSC" +"tslsi=UNAVAILABLE" +"tso=UNAVAILABLE" +"tas=TREFHT no change" +"uas=UNAVAILABLE" +"vas=UNAVAILABLE" diff --git a/examples/CESM/CMIP5/CMIP5_6hrLev.json b/examples/CESM/CMIP5/CMIP5_6hrLev.json new file mode 100644 index 00000000..c83fc366 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_6hrLev.json @@ -0,0 +1,5 @@ +"hus=Q no change" +"ps=PS no change" +"ta=T no change" +"ua=U no change" +"va=V no change" diff --git a/examples/CESM/CMIP5/CMIP5_6hrPlev.json b/examples/CESM/CMIP5/CMIP5_6hrPlev.json new file mode 100644 index 00000000..1dafb269 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_6hrPlev.json @@ -0,0 +1,4 @@ +"psl=PSL no change" +"ta=T interpolated to standard plev3" +"ua=U interpolated to standard plev3" +"va=V interpolated to standard plev3" diff --git a/examples/CESM/CMIP5/CMIP5_Amon.json b/examples/CESM/CMIP5/CMIP5_Amon.json new file mode 100644 index 00000000..9047d915 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_Amon.json @@ -0,0 +1,77 @@ +"ccb=UNAVAILABLE" +"cct=UNAVAILABLE" +"cfc113global=UNKNOWN" +"cfc11global=f11vmr convert to mass" +"cfc12global=f12vmr convert to mass" +"ch4=CH4 interpolated to standard plevs" +"ch4Clim=UNKNOWN" +"ch4global=ch4vmr convert to mass" +"ch4globalClim=UNKNOWN" +"ci=FREQZM unit conversion" (*** cannot find the conversion in code) +"cl=CLOUD on model levels, no change" +"cli=CLDICE on model levels, no change" +"clivi=TGCLDIWP no change" +"clt=CLDTOT unit conversion" (*** cannot find the conversion in code) +"clw=CLDLIQ on model levels, no change" +"clwvi=TGCLDLWP no change" +"co2=CO2 interpolated to standard plevs, scaled by 28.966/44" +"co2Clim=UNKNOWN" +"co2mass=co2vmr convert to mass" +"co2massClim=UNKNOWN" +"evspsbl=QFLX no change" +"fco2antt=UNKNOWN" +"fco2fos=UNKNOWN" +"fco2nat=UNKNOWN" +"hcfc22global=UNKNOWN" +"hfls=LHFLX no change" +"hfss=SHFLX no change" +"hur=RELHUM interpolated to standard plevs" +"hurs=RHREFHT no change" +"hus=Q interpolated to standard plevs" +"huss=QREFHT no change" +"mc=CMFMC + CMFMCDZM on model levels, no change" +"n2o=N2O interpolated to standard plevs" +"n2oClim=UNKNOWN" +"n2oglobal=n2ovmr convert to mass" +"n2oglobalClim=UNKNOWN" +"prAdjust=UNKNOWN" +"pr=PRECC + PRECL and unit conversion from m s-1 to kg m-2 s-1" +"prc=PRECC unit conversion from m s-1 to kg m-2 s-1" +"prsn=PRECSC + PRECSL and unit conversion from m s-1 to kg m-2 s-1" +"prw=TMQ no change" +"ps=PS no change" +"pslAdjust=UNKNOWN" +"psl=PSL no change" +"rlds=FLDS no change" +"rldscs=FLDSC no change" +"rlus=FLDS + FLNS" +"rlut=FSNTOA-FSNT+FLNT" +"rlutcs=FLUTC no change" +"rsds=FSDS no change" +"rsdscs=FSDSC no change" +"rsdt=SOLIN no change" +"rsus=FSDS - FSNS" +"rsuscs=FSDSC - FSNSC" +"rsut=SOLIN - FSNTOA" +"rsutcs=SOLIN - FSNTOAC" +"rtmt=FSNT - FLNT" +"sbl=UNAVAILABLE" +"sci=FREQSH no change" +"sfcWind=U10 no change" +"ta=T interpolated to standard plevs" +"tasAdjust=UNKNOWN" +"tas=TREFHT no change" +"tasmax=TREFMXAV no change" +"tasmin=TREFMNAV no change" +"tauu=TAUX no change" +"tauv=TAUY no change" +"tro3=O3 interpolated to standard plevs" +"tro3Clim=UNKNOWN" +"tsAdjust=UNKNOWN" +"ts=TS no change" +"ua=U interpolated to standard plevs" +"uas=UNAVAILABLE" +"va=V interpolated to standard plevs" +"vas=UNAVAILABLE" +"wap=OMEGA interpolated to standard plevs" +"zg=Z3 interpolated to standard plevs" diff --git a/examples/CESM/CMIP5/CMIP5_LImon.json b/examples/CESM/CMIP5/CMIP5_LImon.json new file mode 100644 index 00000000..f59eb2f8 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_LImon.json @@ -0,0 +1,12 @@ +"agesno=UNAVAILABLE" +"hfdsn=FGR" +"lwsnl=SNOWLIQ no change" +"pflw=UNAVAILABLE" +"sbl=would be QFLX_SUB_SNOW" +"snc=FSNO, mulitply by 100 to get percentage" +"snd=SNOWDP unchanged" +"snm=QMELT, unit change from mm s-1 to kg m-2 s-1, same numerical value"(where (indat2a /= var_info(var_found(1,1))%missing_value), else 0) +"snw=H2OSNO, unit change from mm to kg m-2, same numerical value" +"sootsn=SNOBCMSL + SNODSTMSL + SNOOCMSL" +"tpf=UNAVAILABLE" +"tsn=would be SNOTTOPL" diff --git a/examples/CESM/CMIP5/CMIP5_Lmon.json b/examples/CESM/CMIP5/CMIP5_Lmon.json new file mode 100644 index 00000000..ddaf37a8 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_Lmon.json @@ -0,0 +1,59 @@ +"baresoilFrac=UNAVAILABLE" +"burntArea=ANN_FAREA_BURNED, unit change 'proportion' to percentage *100." +"c3PftFrac=UNAVAILABLE" +"c4PftFrac=UNAVAILABLE" +"cCwd=CWDC, Unit change - grams to kg" +"cLeaf=LEAFC, Unit change - grams to kg" +"cLitterAbove=UNAVAILABLE" +"cLitterBelow=UNAVAILABLE" +"cLitter=TOTLITC, Unit change - grams to kg" +"cMisc=STORVEGC, Unit change - grams to kg" +"cProduct=TOTPRODC, Unit change - grams to kg" +"cRoot=FROOTC+LIVE_ROOTC+DEAD_ROOTC, Unit change - grams to kg" +"cropFrac=UNAVAILABLE" +"cSoilFast=SOIL1C+SOIL2C, Unit change - grams to kg" +"cSoil=TOTSOMC, Unit change - grams to kg" +"cSoilMedium=SOIL3C, Unit change - grams to kg" +"cSoilSlow=SOIL4C, Unit change - grams to kg" +"cVeg=TOTVEGC, Unit change - grams to kg" +"cWood=WOODC, Unit change - grams to kg" +"evspsblsoi=QSOIL" +"evspsblveg=QVEGE" +"fFire=COL_FIRE_CLOSS, Unit change - grams to kg" +"fGrazing=UNAVAILABLE" +"fHarvest=UNAVAILABLE" +"fLitterSoil=LITR1C_TO_SOIL1C+LITR2C_TO_SOIL2C+LITR3C_TO_SOIL3C, Unit change - grams to kg" +"fLuc=DWT_CLOSS+PRODUCT_CLOSS, Unit change - grams to kg" +"fVegLitter=LITFALL, Unit change - grams to kg" +"fVegSoil=UNAVAILABLE" +"gpp=GPP, Unit change - grams to kg" +"grassFrac=UNAVAILABLE" +"lai=TLAI" +"landCoverFrac=UNAVAILABLE" +"mrfso=SOILICE integrated over all layers" +"mrlsl=SOILLIQ + SOILICE at each soil depth" +"mrro=QRUNOFF" +"mrros=QOVER" +"mrso=SOILICE+SOILLIQ, mask over ice-covered regions at 5000 kg m-2" +"mrsos=SOILWATER_10CM" +"nbp=NBP, Unit change - grams to kg" +"nep=UNKNOWN" +"npp=NPP, Unit change - grams to kg" +"nppLeaf=LEAFC_ALLOC, Unit change - grams to kg" +"nppRoot=FROOTC_ALLOC, Unit change - grams to kg" +"nppWood=WOODC_ALLOC, Unit change - grams to kg" +"pastureFrac=UNAVAILABLE" +"prveg=QINTR" +"ra=AR, Unit change - grams to kg" +"residualFrac=UNAVAILABLE" +"rGrowth=GR, Unit change - grams to kg" +"rh=HR, Unit change - grams to kg" +"rMaint=MR, Unit change - grams to kg" +"shrubFrac=UNAVAILABLE" +"tran=QSOIL + QVEGT" +"treeFrac=UNAVAILABLE" +"treeFracPrimDec=UNAVAILABLE" +"treeFracPrimEver=UNAVAILABLE" +"treeFracSecDec=UNAVAILABLE" +"treeFracSecEver=UNAVAILABLE" +"tsl=TSOI" diff --git a/examples/CESM/CMIP5/CMIP5_OImon.json b/examples/CESM/CMIP5/CMIP5_OImon.json new file mode 100644 index 00000000..0b766868 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_OImon.json @@ -0,0 +1,40 @@ +"ageice=UNAVAILABLE" +"bmelt=meltb, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"divice=divu" ????? +"eshrice=UNAVAILABLE" +"evap=evap_ai, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"grCongel=congel, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"grFrazil=frazil, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"grLateral=UNAVAILABLE" +"hcice=UNAVAILABLE" +"hflssi=flat_ai" +"hfssi=fsens_ai" +"ialb=UNAVAILABLE" +"nshrice=UNAVAILABLE" +"pr=rain_ai, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"prsn=snow_ai, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"ridgice=UNAVAILABLE" +"rldssi=flwdn" +"rlussi=flwup" +"rsdssi=fswdn" +"rsussi=UNAVAILABLE" +"sblsi=UNAVAILABLE" +"sic=aice" ????? +"sim=UNAVAILABLE" +"sit=hi unchanged" ????? +"snc=UNAVAILABLE" +"snd=hs" ????? +"snomelt=melts, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"snoToIce=snoice, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"ssi=UNAVAILABLE" +"strairx=strairx" +"strairy=strairy" +"streng=strength" ????? +"strocnx=strocnx" ????? +"strocny=strocny" ????? +"tmelt=meltt, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"transifs=UNAVAILABLE" +"transix=UNAVAILABLE" +"transiy=UNAVAILABLE" +"tsice=Tsfc" ????? +"tsnint=UNAVAILABLE" diff --git a/examples/CESM/CMIP5/CMIP5_Oclim.json b/examples/CESM/CMIP5/CMIP5_Oclim.json new file mode 100644 index 00000000..bea852da --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_Oclim.json @@ -0,0 +1,32 @@ +"difmxybo2d=UNKNOWN" +"difmxybo=UNKNOWN" +"difmxylo2d=UNKNOWN" +"difmxylo=UNKNOWN" +"diftrbbo2d=UNKNOWN" +"diftrbbo=UNKNOWN" +"diftrblo2d=UNKNOWN" +"diftrblo=UNKNOWN" +"diftrebo2d=UNKNOWN" +"diftrebo=UNKNOWN" +"diftrelo2d=UNKNOWN" +"diftrelo=UNKNOWN" +"diftrxybo2d=UNKNOWN" +"diftrxybo=UNKNOWN" +"diftrxylo2d=UNKNOWN" +"diftrxylo=UNKNOWN" +"difvho=UNKNOWN" +"difvmbo=UNKNOWN" +"difvmfdo=UNKNOWN" +"difvmo=UNKNOWN" +"difvmto=UNKNOWN" +"difvso=UNKNOWN" +"difvtrbo=UNKNOWN" +"difvtrto=UNKNOWN" +"dispkevfo=UNKNOWN" +"dispkexyfo2d=UNKNOWN" +"dispkexyfo=UNKNOWN" +"tnkebto2d=UNKNOWN" +"tnkebto=UNKNOWN" +"tnpeo=UNKNOWN" +"tnpeotb=UNKNOWN" +"tnpeot=UNKNOWN" diff --git a/examples/CESM/CMIP5/CMIP5_Omon.json b/examples/CESM/CMIP5/CMIP5_Omon.json new file mode 100644 index 00000000..81020a4a --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_Omon.json @@ -0,0 +1,187 @@ +"arag=UNAVAILABLE" +"bacc=UNAVAILABLE" +"bfe=UNAVAILABLE" +"bsi=UNAVAILABLE" +"calc=UNAVAILABLE" +"chlcalc=spChl*(spCaCO3/spC)" +"chldiat=diatChl" +"chldiaz=diazChl" +"chl=diatChl+spChl+diazChl, only topmost layer in sum" +"chlmisc=UNAVAILABLE" +"chlpico=spChl" +"co3=CO3" +"co3satarag=co3_sat_arag" +"co3satcalc=co3_sat_calc" +"detoc=UNAVAILABLE" +"dfe=Fe" +"dissic=DIC" +"dissoc=DOC" +"dms=UNAVAILABLE" +"dpco2=DpCO2, Convert ppmv to Pa via * 0.101325" +"dpo2=UNAVAILABLE" +"eparag100=UNAVAILABLE" +"epc100=POC_FLUX_IN, only "depth100m" meters; z_t at index 11" +"epcalc100=CaCO3_FLUX_IN, only "depth100m" meters; z_t at index 11" +"epfe100=P_iron_FLUX_IN, only "depth100m" meters; z_t at index 11" +"epsi100=SiO2_FLUX_IN, only "depth100m" meters; z_t at index 11" +"fbddtalk=Jint_100m_ALK, Convert meq/m3 cm/s to mol m-2 s-1 via * 1.e-5" +"fbddtdic=Jint_100m_DIC" +"fbddtdife=Jint_100m_Fe" +"fbddtdin=Jint_100m_NO3+Jint_100m_NH4" +"fbddtdip=Jint_100m_PO4" +"fbddtdisi=Jint_100m_SiO3" +"fddtalk=tend_zint_100m_ALK, Convert meq/m3 cm/s to mol m-2 s-1 via * 1.e-5" +"fddtdic=tend_zint_100m_DIC" +"fddtdife=tend_zint_100m_Fe" +"fddtdin=tend_zint_100m_NO3+tend_zint_100m_NH4" +"fddtdip=tend_zint_100m_PO4" +"fddtdisi=tend_zint_100m_SiO3" +"fgco2=FG_CO2, Convert mmol/m3 cm/s to kg m-2 s-1 via * 12.0e-8" +"fgdms=UNAVAILABLE" +"fgo2=STF_O2" +"frc=UNAVAILABLE" +"frfe=UNAVAILABLE" +"frn=Integrate -DENITRIF over z_t" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((-1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"fsc=UNAVAILABLE" +"fsfe=UNAVAILABLE" +"fsn=NOx_FLUX+NHy_FLUX+(integrate diaz_Nfix over z_t)" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((indat3a(i,j,k)*ocn_t_dz(k))*1.e-5)) +"intdic=Integrate DIC over z_t" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((indat3a(i,j,k)*12.0e-8)*ocn_t_dz(k))) +"intparag=UNAVAILABLE" +"intpbfe=UNAVAILABLE" +"intpbsi=Integrate (-bSi_form) over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((-1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"intpcalcite=Integrate -1*CaCO3_form over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((-1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"intpcalc=Integrate CaCO3_form over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"intpdiat=Integrate photoC_diat over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"intpdiaz=Integrate photoC_diaz over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"intpmisc=UNAVAILABLE" +"intpn2=Integrate diaz_Nfix over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"intpnitrate=photoC_NO3_diat_zint+photoC_NO3_sp_zint+photoC_NO3_diaz_zint" +"intppico=Integrate photoC_sp over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"intpp=Integrate (photoC_diat+photoC_sp+photoC_diaz) over z_t_150m" (cmordat2d(i,j) = cmordat2d(i,j) + ((((indat3a(i,j,k)*ocn_t_dz(k))+ (indat3b(i,j,k)*ocn_t_dz(k))+ (indat3c(i,j,k)*ocn_t_dz(k))))*1.e-5)) +"nh4=NH4" +"no3=NO3" +"o2min=O2_ZMIN" +"o2=O2" +"ph=pH_3D at k=1" +"phycalc=spC" +"phyc=spC+diatC+diazC, only topmost layer in sum" +"phydiat=diatC" +"phydiaz=diazC" +"phyfe=diatFe+spFe+diazFe, only topmost layer in sum" +"phymisc=UNAVAILABLE" +"phyn=(diatC+spC+diazC)*.137" (if (kmt(i,j) .ge. 1) then cmordat2d(i,j) = ((indat3a(i,j,1)+indat3b(i,j,1)+indat3c(i,j,1))*0.137)) +"phypico=spC" +"phyp=(diatC+spC)*.00855+diazC*.002735" (if (kmt(i,j) .ge. 1) then cmordat2d(i,j) = ((indat3a(i,j,1)+indat3b(i,j,1))*0.00855) + (indat3c(i,j,1)*0.002735)) +"physi=diatSi" +"po4=PO4" +"pon=UNAVAILABLE" +"pop=UNAVAILABLE" +"si=SiO3" +"spco2=pCO2SURF, Convert ppmv to Pa via * 0.101325" +"talk=ALK at k=1 (depth0m) and converted from meq/m3 to mol m-3 via * 1.e-3" +"zmeso=UNAVAILABLE" +"zmicro=UNAVAILABLE" +"zo2min=O2_ZMIN_DEPTH" +"zoocmisc=UNAVAILABLE" +"zooc=zooC" +"zsatarag=zsatarag" +"zsatcalc=zsatcalc" +"agessc=IAGE" +"cfc11=Divide CFC11 by 1.e12 to convert to approximate mol kg-1" +"evs=UNKNOWN" +"ficeberg2d=Not applicable" +"ficeberg=Not applicable" +"friver=UNKNOWN" +"fsitherm=UNKNOWN" +"hfbasinba=UNKNOWN" +"hfbasindiff=UNKNOWN" +"hfbasin=UNKNOWN" +"hfcorr=UNKNOWN" +"hfds=UNKNOWN" +"hfevapds=UNKNOWN" +"hfgeou=Not applicable" +"hfibthermds2d=Not applicable" +"hfibthermds=Not applicable" +"hfls=UNKNOWN" +"hfnorthba=UNKNOWN" +"hfnorthdiff=UNKNOWN" +"hfnorth=UNKNOWN" +"hfrainds=UNKNOWN" +"hfrunoffds2d=UNKNOWN" +"hfrunoffds=UNKNOWN" +"hfsifrazil2d=UNKNOWN" +"hfsifrazil=UNKNOWN" +"hfsithermds2d=UNKNOWN" +"hfsithermds=UNKNOWN" +"hfsnthermds2d=UNKNOWN" +"hfsnthermds=UNKNOWN" +"hfss=SENH_F" +"hfxba=UNKNOWN" +"hfxdiff=UNKNOWN" +"hfx=UNKNOWN" +"hfyba=UNKNOWN" +"hfydiff=UNKNOWN" +"hfy=UNKNOWN" +"htovgyre=UNKNOWN" +"htovovrt=UNKNOWN" +"masscello=UNKNOWN" +"masso=Integrate PD over ocean volume" (total mass (kg) if (indat3a(i,j,k).lt.1.e30) indat1a(it) = indat1a(it) + (indat3a(i,j,k)*volume(i,j,k))) +"mfo=UNKNOWN" +"mlotst=UNKNOWN" +"mlotstsq=UNKNOWN" +"msftbarot=BSF, convert with * (1.e6 * 1000.) ! 10^6 m3 s-1 to kg s-1" +"msftmrhozba=UNAVAILABLE" +"msftmrhoz=UNAVAILABLE" +"msftmyzba=MOC converted from Sv to kg s-1" (other code and *1.e6*1.e3) +"msftmyz=MOC converted from Sv to kg s-1" (other code and *1.e6*1.e3) +"msftyrhozba=UNAVAILABLE" +"msftyrhoz=UNAVAILABLE" +"msftyyzba=UNKNOWN" +"msftyyz=UNKNOWN" +"omldamax=UNKNOWN" +"omlmax=XMXL" +"pbo=UNKNOWN" +"pr=PREC_F" +"prsn=SNOW_F" +"pso=UNKNOWN" +"rhopoto=PD" +"rlds=LWDN_F + LWUP_F where IFRAC = 0" +"rsds=UNKNOWN" +"rsntds=QSW_HTP" +"sfdsi=UNKNOWN" +"sfriver=UNKNOWN" +"sltovgyre=UNKNOWN" +"sltovovrt=UNKNOWN" +"soga=SALT weighted global average" (if (indat3a(i,j,k).lt.1.e30) indat1a(it) = indat1a(it) + (indat3a(i,j,k)*volume(i,j,k))) +"so=SALT*1000 to handle CMOR change to psu" +"sos=SALT at topmost level * 1000" +"tauucorr=UNKNOWN" +"tauuo=TAUX" +"tauvcorr=UNKNOWN" +"tauvo=TAUY" +"thetaoga=TEMP weighted global average, units C to K" if (indat3a(i,j,k).lt.1.e30) indat1a(it) = indat1a(it) + (indat3a(i,j,k)*volume(i,j,k)) +"thetao=TEMP no change, units from C to K" +"thkcello=Inapplicable as cell thickness is fixed (dz)" +"tos=TEMP at topmost level, units from C to K" +"tossq=TEMP2" +"umo=(0.5*(UVEL(i,j)+UVEL(i,j-1))*HTE*dz*rho_0)/1000. ! g s-1 to kg s-1" +"uo=UVEL units from cm s-1 to m s-1" +"vmo=(0.5*(VVEL(i,j)+VVEL(i-1,j))*HTN*dz*rho_0)/1000. ! g s-1 to kg s-1" +"volo=PD" +"vo=VVEL units from cm s-1 to m s-1" +"vsfcorr=UNKNOWN" +"vsfevap=UNKNOWN" +"vsf=UNKNOWN" +"vsfpr=UNKNOWN" +"vsfriver=UNKNOWN" +"vsfsit=UNKNOWN" +"wfcorr=UNKNOWN" +"wfonocorr=UNKNOWN" +"wfo=UNKNOWN" +"wmo=(0.5*(WVEL(i,j,k)+WVEL(i,j,k+1))*TAREA*rho_0)/1000. ! g s-1 to kg s-1" +"wmosq=(0.5*(WVEL2(i,j,k)+WVEL2(i,j,k+1))*TAREA*rho_0")/1000. ! g s-1 to kg s-1 +"zosga=ave_slc from Aixue's SLC code; derived from SSH PD TEMP SALT" +"zos=SSH units from cm to m" +"zossga=ave_delta_steric from Aixue's SLC code; derived from SSH PD TEMP SALT" +"zossq=SSH2 units from cm2 to m2" +"zostoga=ave_slc_t from Aixue's SLC code; derived from SSH PD TEMP SALT" diff --git a/examples/CESM/CMIP5/CMIP5_Oyr.json b/examples/CESM/CMIP5/CMIP5_Oyr.json new file mode 100644 index 00000000..92cd5697 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_Oyr.json @@ -0,0 +1,71 @@ +"arag=UNKNOWN" +"bacc=UNKNOWN" +"bddtalk=UNKNOWN" +"bddtdic=UNKNOWN" +"bddtdife=UNKNOWN" +"bddtdin=UNKNOWN" +"bddtdip=UNKNOWN" +"bddtdisi=UNKNOWN" +"bfe=UNKNOWN" +"bsi=UNKNOWN" +"calc=UNKNOWN" +"chlcalc=spChl*(spCaCO3/spC)" ????? +"chldiat=diatChl" +"chldiaz=diazChl" +"chl=diatChl+spChl+diazChl" ????? +"chlmisc=UNKNOWN" +"chlpico=spChl" +"co3=CO3" +"co3satarag=co3_sat_arag" +"co3satcalc=co3_sat_calc" +"darag=UNKNOWN" +"dcalc=UNKNOWN" +"detoc=UNKNOWN" +"dfe=Fe" +"dissic=DIC" +"dissoc=DOC" +"dms=UNKNOWN" +"dpocdtcalc=UNKNOWN" +"dpocdtdiaz=UNKNOWN" +"dpocdtpico=UNKNOWN" +"exparag=UNKNOWN" +"expcalc=CaCO3_FLUX_IN" +"expcfe=P_iron_FLUX_IN" +"expc=POC_FLUX_IN" +"expn=UNKNOWN" +"expp=UNKNOWN" +"expsi=SiO2_FLUX_IN" +"fediss=UNKNOWN" +"fescav=UNKNOWN" +"graz=UNKNOWN" +"nh4=NH4" +"no3=NO3" +"o2=O2" +"parag=UNKNOWN" +"pbfe=UNKNOWN" +"pbsi=UNKNOWN" +"pcalc=UNKNOWN" +"pdi=UNKNOWN" +"ph=pH_3D" +"phycalc=spC" +"phyc=spC+diatC+diazC" ????? +"phydiat=diatC" +"phydiaz=diazC" +"phyfe=spFe" +"phymisc=diatFe+spFe+diazFe" ????? +"phyn=(diatC+spC+diazC)*.137" ????? +"phypico=spC" +"phypmisc=UNKNOWN" +"phyp=(diatC+spC)*.00855+diazC*.002735" ????? +"physi=diatSi" +"pnitrate=UNKNOWN" +"po4=PO4" +"pon=UNKNOWN" +"pop=UNKNOWN" +"pp=UNKNOWN" +"si=SiO3" +"talk=ALK" +"zmeso=UNKNOWN" +"zmicro=UNKNOWN" +"zoocmisc=UNKNOWN" +"zooc=zooC" diff --git a/examples/CESM/CMIP5/CMIP5_aero.json b/examples/CESM/CMIP5/CMIP5_aero.json new file mode 100644 index 00000000..266a891f --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_aero.json @@ -0,0 +1,82 @@ +"abs550aer=AODABS" +"cdnc=CDNUMC on model levels" +"chepsoa=UNKNOWN" +"cldnci=UNKNOWN" +"cldncl=UNKNOWN" +"cldnvi=UNKNOWN" +"concaerh2o=wat_a1+wat_a2+wat_a3 on model levels" +"concbb=UNKNOWN" +"concbc=bc_a1 on model levels" +"conccmcn=num_a3 on model levels" +"conccn=num_a1+num_a2+num_a3 on model levels" +"concdms=DMS on model levels" +"concdust=dst_a1+dst_a3 on model levels" +"concnh4=UNKNOWN" +"concnmcn=UNKNOWN" +"concno3=UNKNOWN" +"concoa=soa_a1+soa_a2+pom_a1 on model levels" +"concpoa=UNKNOWN" +"concso2=SO2 on model levels" +"concso4=so4_a1+so4_a2+so4_a3 on model levels" +"concsoa=soa_a1+soa_a2 on model levels" +"concss=ncl_a1+ncl_a2+ncl_a3 on model levels" +"drybc=bc_a1DDF" +"drydms=UNAVAILABLE" +"drydust=DSTSFDRY" +"drynh3=UNAVAILABLE" +"drynh4=UNAVAILABLE" +"dryoa=pomff_a1DDF+soa_a1DDF+soa_a2DDF" +"drypoa=pom_a1DDF" +"dryso2=DF_SO2" +"dryso4=so4_a1DDF+so4_a2DDF+so4_a3DDF" +"drysoa=UNAVAILABLE" +"dryss=ncl_a1DDF+ncl_a2DDF+ncl_a3DDF" +"ec550aer=UNKNOWN" +"emibb=UNKNOWN" +"emibc=SFbc_a1+bc_a1_CLXF" +"emidms=SFDMS" +"emidust=SFdst_a1+SFdst_a3" +"eminh3=UNKNOWN" +"emioa=UNKNOWN" +"emipoa=UNKNOWN" +"emiso2=SFSO2+SO2_CLXF" +"emiso4=so4_a1_CLXF convert molec/cm2/s to kg m-2 s-1" +"emiss=SFncl_a1+SFncl_a2+SFncl_a3" +"inc=UNKNOWN" +"loadbc=bc_a1" +"loaddust=dst_a1+dst_a3" +"loadnh4=UNKNOWN" +"loadno3=UNKNOWN" +"loadoa=UNKNOWN" +"loadpoa=pom_a1" +"loadso4=so4_a1+so4_a2+so4_a3" +"loadsoa=soa_a1+soa_a2" +"loadss=ncl_a1+ncl_a2+ncl_a3" +"od550aer=AODVIS" +"od550lt1aer=UNKNOWN" +"od870aer=UNKNOWN" +"ps=PS" +"reffclwc=UNKNOWN" +"reffclws=UNKNOWN" +"reffclwtop=AREL" +"rsdscsdiff=UNKNOWN" +"rsdsdiff=UNKNOWN" +"sconcbc=UNKNOWN" +"sconcdust=UNKNOWN" +"sconcnh4=UNKNOWN" +"sconcno3=UNKNOWN" +"sconcoa=UNKNOWN" +"sconcpoa=UNKNOWN" +"sconcso4=UNKNOWN" +"sconcsoa=UNKNOWN" +"sconcss=UNKNOWN" +"wetbc=bc_a1SFWET+bc_c1SFWET" +"wetdms=UNKNOWN" +"wetdust=dst_a1SFWET+dst_a3SFWET+dst_c1SFWET+dst_c3SFWET" +"wetnh4=UNKNOWN" +"wetoa=pom_a1SFWET+pom_c1SFWET+soa_a1SFWET+soa_a2SFWET+soa_c1SFWET+soa_c2SFWET" +"wetpoa=UNKNOWN" +"wetso2=WD_SO2" +"wetso4=so4_a1SFWET+so4_a2SFWET+so4_a3SFWET+so4_c1SFWET+so4_c2SFWET+so4_c3SFWET" +"wetsoa=UNKNOWN" +"wetss=ncl_a1SFWET+ncl_a2SFWET+ncl_a2SFWET+ncl_c1SFWET+ncl_c2SFWET+ncl_c3SFWET" diff --git a/examples/CESM/CMIP5/CMIP5_cf3hr.json b/examples/CESM/CMIP5/CMIP5_cf3hr.json new file mode 100644 index 00000000..13a4f586 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_cf3hr.json @@ -0,0 +1,78 @@ +"ccb=PCLDBOT" +"cct=PCLDTOP" +"cfadDbze94=CFAD_DBZE94_CS no change" +"cfadLidarsr532=CFAD_SR532_CAL no change" +"ci=FREQZM unit conversion" +"clcalipso2=CLD_CAL_NOTCS no change" +"clcalipso=CLD_CAL no change" +"clc=UNKNOWN" +"clhcalipso=CLDHGH_CAL no change" +"clic=CLDICECON on model levels" +"clis=CLDICESTR on model levels" +"clivi=TGCLDIWP no change" +"cllcalipso=CLDLOW_CAL no change" +"clmcalipso=CLDMED_CAL no change" +"cls=UNKNOWN" +"cltcalipso=CLDTOT_CAL no change" +"cltc=CONCLD" +"clt=CLDTOT unit conversion" +"clwc=CLDLIQCON on model levels" +"clws=CLDLIQSTR on model levels" +"clwvi=TGCLDLWP no change" +"demc=EMIS on model levels" +"dems=EMIS on model levels" +"dtauc=TOT_ICLD_VISTAU on model levels" +"dtaus=TOT_ICLD_VISTAU on model levels" +"evspsbl=QFLX no change" +"fco2antt=UNKNOWN" +"fco2fos=UNKNOWN" +"fco2nat=UNKNOWN" +"grpllsprof=UNKNOWN" +"h2o=Q+CLDICE+CLDLIQ on model levels" +"hfls=LHFLX no change" +"hfss=SHFLX no change" +"hurs=RHREFHT no change" +"huss=QREFHT no change" +"parasolRefl=RFL_PARASOL no change" +"prc=PRECC unit conversion" +"prcprof=ZMFLXPRC-ZMFLXSNW + RKFLXPRC-RKFLXSNW on model levels" (*** the Xwalk and code show this in commnets, but xwalk lists these vars, HKFLXPRC-HKFLXSNW, as 2nd args) +"prlsns=LS_FLXSNW on model levels" +"prlsprof=LS_FLXPRC-LS_FLXSNW on model levels" +"pr=PRECT unit conversion" +"prsnc=ZMFLXSNW+HKFLXSNW on model levels" +"prsn=PRECSC + PRECSL and unit conversion" +"prw=TMQ no change" +"psl=PSL no change" +"ps=PS no change" +"reffclic=UNKNOWN" +"reffclis=REI on model levels" +"reffclwc=UNKNOWN" +"reffclws=REL on model levels" +"reffgrpls=UNKNOWN" +"reffrainc=UNKNOWN" +"reffrains=UNKNOWN" +"reffsnowc=UNKNOWN" +"reffsnows=UNKNOWN" +"rldscs=FLDSC no change" +"rlds=FLDS no change" +"rlus=FLDS + FLNS" ????? +"rlutcs=FLUTC no change" +"rlut=FSNTOA-FSNT+FLNT" ????? +"rsdscs=FSDSC no change" +"rsds=FSDS no change" +"rsdt=SOLIN no change" +"rsuscs=FSDSC - FSNSC" ????? +"rsus=FSDS - FSNS" ????? +"rsutcs=SOLIN - FSNTOAC" ????? +"rsut=SOLIN - FSNTOA" ????? +"rtmt=FSNT - FLNT" ????? +"sbl=UNAVAILABLE" +"sci=FREQSH no change" +"sfcWind=UNAVAILABLE" +"ta=T on model levels" +"tas=TREFHT no change" +"tauu=TAUX no change" +"tauv=TAUY no change" +"ts=TS no change" +"uas=UNAVAILABLE" +"vas=UNAVAILABLE" diff --git a/examples/CESM/CMIP5/CMIP5_cfDay.json b/examples/CESM/CMIP5/CMIP5_cfDay.json new file mode 100644 index 00000000..da92ae00 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_cfDay.json @@ -0,0 +1,44 @@ +"albisccp=MEANCLDALB_ISCCP" +"ccb=PCLDBOT" +"cct=PCLDTOP" +"clcalipso=CLD_CAL no change" +"cl=CLOUD on model levels, no change" +"clhcalipso=CLDHGH_CAL no change" +"cli=CLDICE on model levels, no change" +"clisccp=FISCCP1_COSP" +"clivi=TGCLDIWP no change" +"cllcalipso=CLDLOW_CAL no change" +"clmcalipso=CLDMED_CAL no change" +"cltcalipso=CLDTOT_CAL no change" +"clt=CLDTOT unit conversion" +"cltisccp=CLDTOT_ISCCP" +"clw=CLDLIQ on model levels, no change" +"clwvi=TGCLDLWP no change" +"hfls=LHFLX no change" +"hfss=SHFLX no change" +"hur=RHCFMIP on model levels" +"hus=Q on model levels" +"mc=CMFMC + CMFMCDZM on model levels, no change" ???? +"parasolRefl=RFL_PARASOL no change" +"pctisccp=MEANPTOP_ISCCP" +"prc=PRECC unit conversion from m s-1 to kg m-2 s-1" +"ps=PS no change" +"rldscs=FLDSC no change" +"rlds=FLDS no change" +"rlus=FLDS + FLNS" +"rlutcs=FLUTC no change" +"rlut=FSNTOA-FSNT+FLNT" ???? +"rsdscs=FSDSC no change" +"rsds=FSDS no change" +"rsdt=SOLIN no change" +"rsuscs=FSDSC - FSNSC" +"rsus=FSDS - FSNS" +"rsutcs=SOLIN - FSNTOAC" +"rsut=FSUTOA no change" +"ta700=T700" +"ta=T on model levels" +"ua=U on model levels" +"va=V on model levels" +"wap500=OMEGA500" +"wap=OMEGA on model levels" +"zg=Z3 on model levels" diff --git a/examples/CESM/CMIP5/CMIP5_cfMon.json b/examples/CESM/CMIP5/CMIP5_cfMon.json new file mode 100644 index 00000000..50403f27 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_cfMon.json @@ -0,0 +1,98 @@ +"albisccp=MEANCLDALB_ISCCP" +"clcalipso=CLD_CAL no change" +"clc=UNKNOWN" +"clhcalipso=CLDHGH_CAL no change" +"clic=UNKNOWN" +"clisccp=FISCCP1_COSP" +"clis=UNKNOWN" +"cllcalipso=CLDLOW_CAL no change" +"clmcalipso=CLDMED_CAL no change" +"cls=UNKNOWN" +"cltcalipso=CLDTOT_CAL no change" +"cltisccp=CLDTOT_ISCCP" +"clwc=UNKNOWN" +"clws=UNKNOWN" +"dmc=UNKNOWN" +"evisct=UNKNOWN" +"eviscu=UNKNOWN" +"hur=RELHUM on model levels" +"hus=Q on model levels" +"mcd=UNKNOWN" +"mcu=UNKNOWN" +"parasolRefl=RFL_PARASOL no change" +"pctisccp=MEANPTOP_ISCCP" +"ps=PS no change" +"rld4co2=UNKNOWN" +"rldcs4co2=UNKNOWN" +"rldcs=UNKNOWN" +"rld=UNKNOWN" +"rlu4co2=UNKNOWN" +"rlucs4co2=UNKNOWN" +"rlucs=UNKNOWN" +"rlut4co2=UNKNOWN" +"rlutcs4co2=UNKNOWN" +"rlu=UNKNOWN" +"rsd4co2=UNKNOWN" +"rsdcs4co2=UNKNOWN" +"rsdcs=UNKNOWN" +"rsd=UNKNOWN" +"rsu4co2=UNKNOWN" +"rsucs4co2=UNKNOWN" +"rsucs=UNKNOWN" +"rsut4co2=UNKNOWN" +"rsutcs4co2=UNKNOWN" +"rsu=UNKNOWN" +"smc=UNKNOWN" +"ta=T on model levels, no change" +"tnhusa=TAQ on model levels, no change" +"tnhusc=ZMDQ+EVAPQZM+CMFDQ" +"tnhusd=VD01 on model levels" +"tnhusmp=PTEQ on model levels" +"tnhusscpbl=EVAPPREC+DQSED-CME" +"tnhus=TAQ+PTEQ" +"tnsccwacr=UNKNOWN" +"tnsccwacs=UNKNOWN" +"tnsccwa=UNKNOWN" +"tnsccwbl=UNKNOWN" +"tnsccwce=UNKNOWN" +"tnsccwcm=UNKNOWN" +"tnsccwif=UNKNOWN" +"tnsccw=UNKNOWN" +"tnscliag=UNKNOWN" +"tnsclias=UNKNOWN" +"tnsclia=UNKNOWN" +"tnsclibfpcl=UNKNOWN" +"tnsclibl=UNKNOWN" +"tnsclicd=UNKNOWN" +"tnsclicm=UNKNOWN" +"tnsclids=UNKNOWN" +"tnscliemi=UNKNOWN" +"tnsclihencl=UNKNOWN" +"tnsclihenv=UNKNOWN" +"tnsclihon=UNKNOWN" +"tnscliif=UNKNOWN" +"tnsclimcl=UNKNOWN" +"tnsclimr=UNKNOWN" +"tnscliricl=UNKNOWN" +"tnsclirir=UNKNOWN" +"tnscli=UNKNOWN" +"tnsclwac=UNKNOWN" +"tnsclwar=UNKNOWN" +"tnsclwas=UNKNOWN" +"tnsclwa=UNKNOWN" +"tnsclwbfpcli=UNKNOWN" +"tnsclwbl=UNKNOWN" +"tnsclwcd=UNKNOWN" +"tnsclwce=UNKNOWN" +"tnsclwcm=UNKNOWN" +"tnsclwhen=UNKNOWN" +"tnsclwhon=UNKNOWN" +"tnsclwmi=UNKNOWN" +"tnsclwri=UNKNOWN" +"tnsclw=UNKNOWN" +"tnta=UNKNOWN" +"tntc=ZMDT+EVAPTZM+ZMMTT+CMFDT" +"tntmp=PTTEND on model levels, no change" +"tntr=QRL+QRS" +"tntscpbl=HPROGCLD/CPAIR+HSED/CPAIR (CPAIR = 1004.64)" +"tnt=TTEND_TOT on model levels, no change" diff --git a/examples/CESM/CMIP5/CMIP5_cfOff.json b/examples/CESM/CMIP5/CMIP5_cfOff.json new file mode 100644 index 00000000..1df9ea25 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_cfOff.json @@ -0,0 +1,9 @@ +"cfadDbze94=CFAD_DBZE94_CS no change" +"cfadLidarsr532=CFAD_SR532_CAL no change" +"clcalipso2=no change" +"clcalipso=CLD_CAL no change" +"clhcalipso=CLDHGH_CAL no change" +"cllcalipso=UNKNOWN" +"clmcalipso=CLDMED_CAL no change" +"cltcalipso=CLDTOT_CAL no change" +"parasolRefl=RFL_PARASOL no change" diff --git a/examples/CESM/CMIP5/CMIP5_cfSites.json b/examples/CESM/CMIP5/CMIP5_cfSites.json new file mode 100644 index 00000000..e1e1ccf2 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_cfSites.json @@ -0,0 +1,75 @@ +"ccb=PCLDBOT" +"cct=PCLDTOP" +"ci=FREQZM unit conversion" +"cl=CLOUD on model levels, no change" +"cli=CLDICE on model levels, no change" +"clivi=TGCLDIWP no change" +"clt=CLDTOT unit conversion" +"clw=CLDLIQ on model levels, no change" +"clwvi=TGCLDLWP no change" +"edt=UNKNOWN" +"evspsbl=QFLX no change" +"evu=UNKNOWN" +"fco2antt=UNKNOWN" +"fco2fos=UNKNOWN" +"fco2nat=UNKNOWN" +"hfls=LHFLX no change" +"hfss=SHFLX no change" +"hur=RELHUM on model levels" +"hurs=RHREFHT no change" +"hus=Q on model levels" +"huss=QREFHT no change" +"mc=CMFMC + CMFMCDZM on model levels, no change" ???? +"prc=PRECC unit conversion from m s-1 to kg m-2 s-1" +"pr=PRECT unit conversion from m s-1 to kg m-2 s-1" ???? Code has PRECC + PRECL, XWalk just lists PRECT +"prsn=PRECSC + PRECSL and unit conversion from m s-1 to kg m-2 s-1" +"prw=TMQ no change" +"psl=PSL no change" +"ps=PS no change" +"rldcs=UNKNOWN" +"rld=UNKNOWN" +"rldscs=FLDSC no change" +"rlds=FLDS no change" +"rlucs=UNKNOWN" +"rlus=FLDS + FLNS" +"rlutcs=FLUTC no change" +"rlut=FSNTOA-FSNT+FLNT" ???? +"rlu=UNKNOWN" +"rsdcs=UNKNOWN" +"rsd=UNKNOWN" +"rsdscs=FSDSC no change" +"rsds=FSDS no change" +"rsdt=SOLIN no change" +"rsucs=UNKNOWN" +"rsuscs=FSDSC - FSNSC" +"rsus=FSDS - FSNS" +"rsutcs=SOLIN - FSNTOAC" +"rsut=SOLIN - FSNTOA" +"rsu=UNKNOWN" +"rtmt=FSNT - FLNT" +"sbl=UNAVAILABLE" +"sci=FREQSH no change" +"sfcWind=UNAVAILABLE" +"ta=T on model levels, no change" +"tas=TREFHT no change" +"tauu=TAUX no change" +"tauv=TAUY no change" +"tnhusa=UNKNOWN" +"tnhusc=UNKNOWN" +"tnhusd=UNKNOWN" +"tnhusmp=UNKNOWN" +"tnhusscpbl=UNKNOWN" +"tnhus=UNKNOWN" +"tnta=UNKNOWN" +"tntc=UNKNOWN" +"tntmp=UNKNOWN" +"tntr=UNKNOWN" +"tntscpbl=UNKNOWN" +"tnt=TTEND_TOT on model levels, no change" +"ts=TS no change" +"ua=U on model levels" +"uas=UNAVAILABLE" +"va=V on model levels" +"vas=UNAVAILABLE" +"wap=OMEGA on model levels" +"zg=Z3 on model levels" diff --git a/examples/CESM/CMIP5/CMIP5_day.json b/examples/CESM/CMIP5/CMIP5_day.json new file mode 100644 index 00000000..55ea18c6 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_day.json @@ -0,0 +1,42 @@ +"clt=CLDTOT unit conversion" +"hfls=LHFLX no change" +"hfss=SHFLX no change" +"hur=RELHUM interpolated to standard plevs" +"hus=Q interpolated to standard plevs" +"huss=QREFHT no change" +"mrro=QRUNOFF" +"mrsos=SOILWATER_10CM" +"omldamax=UNKNOWN" +"pr=PRECC + PRECL and unit conversion from m s-1 to kg m-2 s-1" +"prc=PRECC unit conversion from m s-1 to kg m-2 s-1" +"prsn=PRECSC + PRECSL and unit conversion from m s-1 to kg m-2 s-1" +"psl=PSL no change" +"rhs=RHREFHT" +"rhsmax=UNAVAILABLE" +"rhsmin=UNAVAILABLE" +"rlds=FLDS no change" +"rlus=FLDS + FLNS" +"rlut=FSNTOA-FSNT+FLNT" ???? +"rsds=FSDS no change" +"rsus=FSDS - FSNS" +"sfcWind=UNAVAILABLE" +"sfcWindmax=UNAVAILABLE" +"sic=aice_d no change" ????? +"sit=hi_d no change" ????? +"snc=FSNO, mulitply by 100 to get percentage" +"snw=SNOWDP, convert from m to kg m-2 by multiplying by 1000" +"ta=T interpolated to standard plevs" +"tas=TREFHT no change" +"tasmax=TREFHTMX no change" +"tasmin=TREFHTMN no change" +"tos=SST, units from C to K" ????? +"tossq=SST2, units from C to K" ????? +"tslsi=UNKNOWN" +"ua=U interpolated to standard plevs" +"uas=UNAVAILABLE" +"usi=uvel_d" ????? +"va=V interpolated to standard plevs" +"vas=UNAVAILABLE" +"vsi=vvel_d" ????? +"wap=OMEGA interpolated to standard plevs" +"zg=Z3 interpolated to standard plevs" diff --git a/examples/CESM/CMIP5/CMIP5_fx.json b/examples/CESM/CMIP5/CMIP5_fx.json new file mode 100644 index 00000000..c3175404 --- /dev/null +++ b/examples/CESM/CMIP5/CMIP5_fx.json @@ -0,0 +1,13 @@ +"areacella=AREA" +"areacello=TAREA convert cm2 to m2" +"basin=REGION_MASK" +"deptho=HT convert cm to m" +"hfgeou=Always set to zero in POP namelist" +"mrsofc=WATSAT" +"orog=Divide PHIS by g where LANDFRAC ne zero" (where (landfrac .ne. 0)->phis/shr_const_g, shr_const_g = 9.80616) +"rootd=UNKNOWN" +"sftgif=PCT_GLACIER" +"sftlf=LANDFRAC no change" +"sftof=0 where KMT = 0, 100 elsewhere" +"thkcello=dz at each gridcell" +"volcello=TAREA times dz, convert cm3 to m3" diff --git a/examples/CESM/CMIP6/CESM_CMIP6.ga b/examples/CESM/CMIP6/CESM_CMIP6.ga new file mode 100644 index 00000000..b24d51ee --- /dev/null +++ b/examples/CESM/CMIP6/CESM_CMIP6.ga @@ -0,0 +1,34 @@ +institute_id = NCAR +institute = NCAR (National Center for Atmospheric Research) Boulder, CO, USA +model_id = CESM2 +model = Community Earth System Model version 2 +realization_index = 1 +initialization_index = 1 +physics_index = 1 +forcing_index = 1 +grid_index = 1 +parent_experiment_id = historical +parent_experiment_ripfg = r1i1p1f1g1 +sub_experiment_idindex = xxxxxx +Convensions = CF-1.4 CMIP6-1.0 UGRID Conv +start_date = 1979-01-01 00:00:00 +creation_date = 2016-07-31T14:46:40Z +branch_time_in_parent = 19. +branch_method = hybrid +branch_time_units_in_parent = days since 1850-01-01 00:00:00 +branch_time_in_child = days since 1979-01-01 00:00:00 +further_info_url = http://purl.org/cmip6FurtherInfo/NCAR/CESM2/ +tracking_id = hdl:21.14100/xxxxxx +comment = CESM home page cesm.ucar.edu +contact = cesm_data@ucar.edu +references = Gent P. R., et.al. 2011, The Community Climate System Model version 4. J. Climate, doi:10.1175/2011JCLI4083.1 +title = CESM2 model output prepared for CMIP6 +acknowledgements = The CESM project is supported by the National Science Foundation and the Office of Science (BER) of the U.S. Department of Energy. NCAR is sponsored by the National Science Foundation. Computing resources were provided by the Climate Simulation Laboratory at the NCAR Computational and Information Systems Laboratory (CISL), sponsored by the National Science Foundation and other agencies. +cesm_casename = f20.1979_amip.track1.1deg.001 +cesm_repotag = cesm2 +cesm_compset = FAMIPC5 +resolution = f09_f09 (0.9x1.25_0.9x1.25) +forcing = Sl GHG Vl SS Ds SA BC MD OC Oz AA +forcing_note = Additional information on the external forcings used in this experiment can be found at http://www.cesm.ucar.edu/CMIP5/forcing_information + + diff --git a/examples/CESM/CMIP6/CESM_MastList.def b/examples/CESM/CMIP6/CESM_MastList.def new file mode 100644 index 00000000..7103dfbd --- /dev/null +++ b/examples/CESM/CMIP6/CESM_MastList.def @@ -0,0 +1,321 @@ +abs550aer = AODABS +agessc = IAGE +albisccp = MEANCLDALB_ISCCP +areacella = AREA +areacello = TAREA +basin = REGION_MASK +bmelt = meltb, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640 +burntArea = ANN_FAREA_BURNED +cCwd = CWDC +cLeaf = LEAFC +cLitter = TOTLITC +cMisc = STORVEGC +cProduct = TOTPRODC +cRoot = FROOTC+LIVE_ROOTC+DEAD_ROOTC +cSoil = TOTSOMC +cSoilFast = SOIL1C+SOIL2C +cSoilMedium = SOIL3C +cSoilSlow = SOIL4C +cVeg = TOTVEGC +cWood = WOODC +ccb = PCLDBOT +cct = PCLDTOP +cdnc = CDNUMC +cfadDbze94 = CFAD_DBZE94_CS +cfadLidarsr532 = CFAD_SR532_CAL +cfc11 = CFC11 +cfc11global = f11vmr +cfc12global = f12vmr +ch4 = vinterp(CH4, plevs) +ch4global = ch4vmr +chl = diatChl+spChl+diazChl +chlcalc = spChl*(spCaCO3/spC) +chldiat = diatChl +chldiaz = diazChl +chlpico = spChl +ci = FREQZM +cl = CLOUD +clcalipso = CLD_CAL +clhcalipso = CLDHGH_CAL +cli = CLDICE +clic = CLDICECON +clis = CLDICESTR +clisccp = FISCCP1_COSP +clivi = TGCLDIWP +cllcalipso = CLDLOW_CAL +clmcalipso = CLDMED_CAL +clt = CLDTOT +cltc = CONCLD +cltcalipso = CLDTOT_CAL +cltisccp = CLDTOT_ISCCP +clw = CLDLIQ +clwc = CLDLIQCON +clws = CLDLIQSTR +clwvi = TGCLDLWP +co2 = vinterp(CO2,plevs) +co2mass = co2vmr +co3 = CO3 +co3satarag = co3_sat_arag +co3satcalc = co3_sat_calc +concaerh2o = wat_a1+wat_a2+wat_a3 +concbc = bc_a1 +conccmcn = num_a3 +conccn = num_a1+num_a2+num_a3 +concdms = DMS +concdust = dst_a1+dst_a3 +concoa = soa_a1+soa_a2+pom_a1 +concso2 = SO2 +concso4 = so4_a1+so4_a2+so4_a3 +concsoa = soa_a1+soa_a2 +concss = ncl_a1+ncl_a2+ncl_a3 +demc = EMIS +dems = EMIS +deptho = HT +dfe = Fe +dissic = DIC +dissoc = DOC +divice = divu +dpco2 = DpCO2 +drybc = bc_a1DDF +drydust = DSTSFDRY +dryoa = pomff_a1DDF+soa_a1DDF+soa_a2DDF +drypoa = pom_a1DDF +dryso2 = DF_SO2 +dryso4 = so4_a1DDF+so4_a2DDF+so4_a3DDF +dryss = ncl_a1DDF+ncl_a2DDF+ncl_a3DDF +dtauc = TOT_ICLD_VISTAU +dtaus = TOT_ICLD_VISTAU +emibc = SFbc_a1+bc_a1_CLXF +emidms = SFDMS +emidust = SFdst_a1+SFdst_a3 +emiso2 = SFSO2+SO2_CLXF +emiso4 = so4_a1_CLXF +emiss = SFncl_a1+SFncl_a2+SFncl_a3 +epc100 = POC_FLUX_IN[only depth100m meters] +epcalc100 = CaCO3_FLUX_IN[only depth100m meters] +epfe100 = P_iron_FLUX_IN[only depth100m meters] +epsi100 = SiO2_FLUX_IN[only depth100m meters] +evap = (evap_ai/86400) +evspsbl = QFLX +evspsblsoi = QSOIL +evspsblveg = QVEGE +expc = POC_FLUX_IN +expcalc = CaCO3_FLUX_IN +expcfe = P_iron_FLUX_IN +expsi = SiO2_FLUX_IN +fFire = COL_FIRE_CLOSS +fLitterSoil = LITR1C_TO_SOIL1C+LITR2C_TO_SOIL2C+LITR3C_TO_SOIL3C +fLuc = DWT_CLOSS+PRODUCT_CLOSS +fVegLitter = LITFALL +fbddtalk = Jint_100m_ALK +fbddtdic = Jint_100m_DIC +fbddtdife = Jint_100m_Fe +fbddtdin = Jint_100m_NO3+Jint_100m_NH4 +fbddtdip = Jint_100m_PO4 +fbddtdisi = Jint_100m_SiO3 +fddtalk = tend_zint_100m_ALK +fddtdic = tend_zint_100m_DIC +fddtdife = tend_zint_100m_Fe +fddtdin = tend_zint_100m_NO3+tend_zint_100m_NH4 +fddtdip = tend_zint_100m_PO4 +fddtdisi = tend_zint_100m_SiO3 +fgco2 = FG_CO2 +fgo2 = STF_O2 +frn = Integrate -DENITRIF over z_t (if (kmt(i,j).ge.k) then cmordat2d(i,j) =cmordat2d(i,j) + ((-1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +fsn = NOx_FLUX+NHy_FLUX+(integrate diaz_Nfix over z_t) (if (kmt(i,j).ge.k) then cmordat2d(i,j) =cmordat2d(i,j) + ((indat3a(i,j,k)*ocn_t_dz(k))*1.e-5)) +gpp = GPP +grCongel = congel/8640 +grFrazil = frazil +h2o = Q+CLDICE+CLDLIQ +hfdsn = FGR +hfls = LHFLX +hflssi = flat_ai +hfss = SENH_F +hfssi = fsens_ai +hur = vinterp(RELHUM,plevs) +hurs = RHREFHT +hus = vinterp(Q,plevs) +huss = QREFHT +intdic = Integrate DIC over z_t (if (kmt(i,j).ge.k) then cmordat2d(i,j) =cmordat2d(i,j) + ((indat3a(i,j,k)*12.0e-8)*ocn_t_dz(k))) +intpbsi = Integrate (-bSi_form) over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j) =cmordat2d(i,j) + ((-1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +intpcalc = Integrate CaCO3_form over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j) =cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +intpcalcite = Integrate -1*CaCO3_form over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j) =cmordat2d(i,j) + ((-1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +intpdiat = Integrate photoC_diat over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j) =cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +intpdiaz = Integrate photoC_diaz over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j) =cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +intpn2 = Integrate diaz_Nfix over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j) =cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +intpnitrate = photoC_NO3_diat_zint+photoC_NO3_sp_zint+photoC_NO3_diaz_zint +intpp = Integrate (photoC_diat+photoC_sp+photoC_diaz) over z_t_150m (cmordat2d(i,j) =cmordat2d(i,j) + ((((indat3a(i,j,k)*ocn_t_dz(k))+ (indat3b(i,j,k)*ocn_t_dz(k))+ (indat3c(i,j,k)*ocn_t_dz(k))))*1.e-5)) +intppico = Integrate photoC_sp over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j) =cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +lai = TLAI +loadbc = bc_a1 +loaddust = dst_a1+dst_a3 +loadpoa = pom_a1 +loadso4 = so4_a1+so4_a2+so4_a3 +loadsoa = soa_a1+soa_a2 +loadss = ncl_a1+ncl_a2+ncl_a3 +lwsnl = SNOWLIQ +masso = Integrate PD over ocean volume (total mass (kg) if (indat3a(i,j,k).lt.1.e30) indat1a(it) =indat1a(it) + (indat3a(i,j,k)*volume(i,j,k))) +mc = CMFMC + CMFMCDZM +mrfso = SOILICE +mrlsl = SOILLIQ + SOILICE +mrro = QRUNOFF +mrros = QOVER +mrso = mask over ice-covered regions at 5000 kg m-2(SOILICE+SOILLIQ) +mrsofc = WATSAT +mrsos = SOILWATER_10CM +msftbarot = BSF +msftmyz = MOC +msftmyzba = MOC +n2o = vinterp(N2O,plevs) +n2oglobal = n2ovmr +nbp = NBP +nh4 = NH4 +no3 = NO3 +npp = NPP +nppLeaf = LEAFC_ALLOC +nppRoot = FROOTC_ALLOC +nppWood = WOODC_ALLOC +o2 = O2 +o2min = O2_ZMIN +od550aer = AODVIS +omlmax = XMXL +orog = PHIS/g +parasolRefl = RFL_PARASOL +pctisccp = MEANPTOP_ISCCP +ph = pH_3D +phyc = spC+diatC+diazC +phycalc = spC +phydiat = diatC +phydiaz = diazC +phyfe = spFe +phymisc = diatFe+spFe+diazFe +phyn = (diatC+spC+diazC)*.137 +phyp = (diatC+spC)*.00855+diazC*.002735 +phypico = spC +physi = diatSi +po4 = PO4 +pr = PREC_F +prc = PRECC +prcprof = ZMFLXPRC-ZMFLXSNW + RKFLXPRC-RKFLXSNW +prlsns = LS_FLXSNW +prlsprof = LS_FLXPRC-LS_FLXSNW +prsn = SNOW_F +prsnc = ZMFLXSNW+HKFLXSNW +prveg = QINTR +prw = TMQ +ps = PS +psl = PSL +rGrowth = GR +rMaint = MR +ra = AR +reffclis = REI +reffclws = REL +reffclwtop = AREL +rh = HR +rhopoto = PD +rhs = RHREFHT +rlds = LWDN_F + LWUP_F +rldscs = FLDSC +rldssi = flwdn +rlus = FLDS + FLNS +rlussi = flwup +rlut = FSNTOA-FSNT+FLNT +rlutcs = FLUTC +rsds = FSDS +rsdscs = FSDSC +rsdssi = fswdn +rsdt = SOLIN +rsntds = QSW_HTP +rsus = FSDS - FSNS +rsuscs = FSDSC - FSNSC +rsut = SOLIN - FSNTOA +rsutcs = SOLIN - FSNTOAC +rtmt = FSNT - FLNT +sbl = QFLX_SUB_SNOW +sci = FREQSH +sfcWind = U10 +sftgif = PCT_GLACIER +sftlf = LANDFRAC +sftof = 0 where KMT =0, 100 elsewhere +si = SiO3 +sic = aice +sit = hi +snc = FSNO +snd = hs +snm = QMELT +snoToIce = snoice/8640 +snomelt = melts/8640 +snw = H2OSNO +so = SALT +soga = weighted_global_average(SALT) +sootsn = SNOBCMSL + SNODSTMSL + SNOOCMSL +sos = SALT[at topmost level * 1000] +spco2 = pCO2SURF +strairx = strairx +strairy = strairy +streng = strength +strocnx = strocnx +strocny = strocny +ta = vinterp(T,plevs) +ta700 = T700 +talk = ALK +tas = TREFHT +tasmax = TREFHTMX +tasmin = TREFHTMN +tauu = TAUX +tauuo = TAUX +tauv = TAUY +tauvo = TAUY +thetao = TEMP +thetaoga = weighted_global_average(TEMP) +thkcello = dz +tmelt = meltt/8640 +tnhus = TAQ+PTEQ +tnhusa = TAQ +tnhusc = ZMDQ+EVAPQZM+CMFDQ +tnhusd = VD01 +tnhusmp = PTEQ +tnhusscpbl = EVAPPREC+DQSED-CME +tnt = TTEND_TOT +tntc = ZMDT+EVAPTZM+ZMMTT+CMFDT +tntmp = PTTEND +tntr = QRL+QRS +tntscpbl = HPROGCLD/CPAIR+HSED/1004.64 +tos = TEMP[at topmost level] +tossq = TEMP2 +tran = QSOIL + QVEGT +tro3 = vinterp(O3,plevs) +ts = TS +tsice = Tsfc +tsl = TSOI +tsn = SNOTTOPL +ua = vinterp(U,plevs) +umo = (0.5*(UVEL(i,j)+UVEL(i,j-1))*HTE*dz*rho_0) +uo = UVEL +usi = uvel_d +va = vinterp(V,plevs) +vmo = (0.5*(VVEL(i,j)+VVEL(i-1,j))*HTN*dz*rho_0) +vo = VVEL +volcello = TAREA*dz +volo = PD +vsi = vvel_d +wap = vinterp(OMEGA,plevs) +wap500 = OMEGA500 +wetbc = bc_a1SFWET+bc_c1SFWET +wetdust = dst_a1SFWET+dst_a3SFWET+dst_c1SFWET+dst_c3SFWET +wetoa = pom_a1SFWET+pom_c1SFWET+soa_a1SFWET+soa_a2SFWET+soa_c1SFWET+soa_c2SFWET +wetso2 = WD_SO2 +wetso4 = so4_a1SFWET+so4_a2SFWET+so4_a3SFWET+so4_c1SFWET+so4_c2SFWET+so4_c3SFWET +wetss = ncl_a1SFWET+ncl_a2SFWET+ncl_a2SFWET+ncl_c1SFWET+ncl_c2SFWET+ncl_c3SFWET +wmo = (0.5*(WVEL(i,j,k)+WVEL(i,j,k+1))*TAREA*rho_0) +wmosq = (0.5*(WVEL2(i,j,k)+WVEL2(i,j,k+1))*TAREA*rho_0) +zg = vinterp(Z3,plevs) +zo2min = O2_ZMIN_DEPTH +zooc = zooC +zos = SSH +zosga = ave_slc from Aixue's SLC code; derived from SSH PD TEMP SALT +zossga = ave_delta_steric from Aixue's SLC code; derived from SSH PD TEMP SALT +zossq = SSH2 +zostoga = ave_slc_t from Aixue's SLC code; derived from SSH PD TEMP SALT +zsatarag = zsatarag +zsatcalc = zsatcalc diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_3hr.def b/examples/CESM/CMIP6/CESMtoCMIP6_3hr.def new file mode 100644 index 00000000..aad44872 --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_3hr.def @@ -0,0 +1,23 @@ +prc: PRECC unit conversion +tas: TREFHT no change +rsuscs: Would be FSDSC - FSNSC +rsdscs: FSDSC no change +huss: QREFHT no change +hfss: SHFLX no change +hfls: LHFLX no change +rldscs: FLDSC no change +tslsi: UNAVAILABLE +rsdsdiff: UNAVAILABLE +pr: PRECC + PRECL and unit conversion +ps: PS no change +rlus: FLDS + FLNS +rlds: FLDS no change +mrsos: UNAVAILABLE +clt: CLDTOT unit conversion +vas: UNAVAILABLE +uas: UNAVAILABLE +tso: UNAVAILABLE +rsds: FSDS no change +mrro: UNAVAILABLE +rsus: FSDS - FSNS +prsn: PRECSC + PRECSL and unit conversion diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_6hrLev.def b/examples/CESM/CMIP6/CESMtoCMIP6_6hrLev.def new file mode 100644 index 00000000..578b6c11 --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_6hrLev.def @@ -0,0 +1,6 @@ +ps: PS no change +va: V no change +orog: MISSING +hus: Q no change +ua: U no change +ta: T no change diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_6hrPlev.def b/examples/CESM/CMIP6/CESMtoCMIP6_6hrPlev.def new file mode 100644 index 00000000..892b18e6 --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_6hrPlev.def @@ -0,0 +1,4 @@ +va: V interpolated to standard plev3 +ua: U interpolated to standard plev3 +psl: PSL no change +ta: T interpolated to standard plev3 diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_Amon.def b/examples/CESM/CMIP6/CESMtoCMIP6_Amon.def new file mode 100644 index 00000000..5e374428 --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_Amon.def @@ -0,0 +1,59 @@ +va: V interpolated to standard plevs +ci: FREQZM unit conversion +n2o: N2O interpolated to standard plevs +rsds: FSDS no change +prc: PRECC unit conversion from m s-1 to kg m-2 s-1 +tas: TREFHT no change +rsuscs: FSDSC - FSNSC +rsdscs: FSDSC no change +huss: QREFHT no change +hfss: SHFLX no change +rtmt: FSNT - FLNT +prw: TMQ no change +zg: Z3 interpolated to standard plevs +hfls: LHFLX no change +rldscs: FLDSC no change +pr: PRECC + PRECL and unit conversion from m s-1 to kg m-2 s-1 +ps: PS no change +co2: CO2 interpolated to standard plevs, scaled by 28.966/44 +ch4: CH4 interpolated to standard plevs +cli: CLDICE on model levels, no change +cfc11global: f11vmr convert to mass +clt: CLDTOT unit conversion +rlds: FLDS no change +cl: CLOUD on model levels, no change +ts: TS no change +clw: CLDLIQ on model levels, no change +ta: T interpolated to standard plevs +orog: MISSING +tasmin: TREFMNAV no change +phalf: MISSING +rlut: FSNTOA-FSNT+FLNT +hurs: RHREFHT no change +wap: OMEGA interpolated to standard plevs +n2oglobal: n2ovmr convert to mass +pfull: MISSING +ch4global: ch4vmr convert to mass +hur: RELHUM interpolated to standard plevs +hus: Q interpolated to standard plevs +tro3: O3 interpolated to standard plevs +rsdt: SOLIN no change +clivi: TGCLDIWP no change +rsutcs: SOLIN - FSNTOAC +co2mass: co2vmr convert to mass +rlus: FLDS + FLNS +mc: CMFMC + CMFMCDZM on model levels, no change +rlutcs: FLUTC no change +psl: PSL no change +evspsbl: QFLX no change +sfcWind: U10 no change +tauv: TAUY no change +rsut: SOLIN - FSNTOA +clwvi: TGCLDLWP no change +tauu: TAUX no change +tasmax: TREFMXAV no change +rsus: FSDS - FSNS +ua: U interpolated to standard plevs +cfc12global: f12vmr convert to mass +sci: FREQSH no change +prsn: PRECSC + PRECSL and unit conversion from m s-1 to kg m-2 s-1 diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_LImon.def b/examples/CESM/CMIP6/CESMtoCMIP6_LImon.def new file mode 100644 index 00000000..eea3e41c --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_LImon.def @@ -0,0 +1,9 @@ +tsn: would be SNOTTOPL +snd: SNOWDP unchanged +snm: QMELT, unit change from mm s-1 to kg m-2 s-1, same numerical value(where (indat2a /:var_info(var_found(1,1))%missing_value), else 0) +snc: FSNO, mulitply by 100 to get percentage +hfdsn: FGR +snw: H2OSNO, unit change from mm to kg m-2, same numerical value +sootsn: SNOBCMSL + SNODSTMSL + SNOOCMSL +lwsnl: SNOWLIQ no change +sbl: would be QFLX_SUB_SNOW diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_Lmon.def b/examples/CESM/CMIP6/CESMtoCMIP6_Lmon.def new file mode 100644 index 00000000..56e02ce0 --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_Lmon.def @@ -0,0 +1,39 @@ +fLitterSoil: LITR1C_TO_SOIL1C+LITR2C_TO_SOIL2C+LITR3C_TO_SOIL3C, Unit change - grams to kg +nppWood: WOODC_ALLOC, Unit change - grams to kg +prveg: QINTR +mrlsl: SOILLIQ + SOILICE at each soil depth +cProduct: TOTPRODC, Unit change - grams to kg +cWood: WOODC, Unit change - grams to kg +cSoilFast: SOIL1C+SOIL2C, Unit change - grams to kg +cLeaf: LEAFC, Unit change - grams to kg +cSoilMedium: SOIL3C, Unit change - grams to kg +fFire: COL_FIRE_CLOSS, Unit change - grams to kg +cVeg: TOTVEGC, Unit change - grams to kg +cRoot: FROOTC+LIVE_ROOTC+DEAD_ROOTC, Unit change - grams to kg +mrso: SOILICE+SOILLIQ, mask over ice-covered regions at 5000 kg m-2 +ra: AR, Unit change - grams to kg +mrsos: SOILWATER_10CM +npp: NPP, Unit change - grams to kg +cCwd: CWDC, Unit change - grams to kg +gpp: GPP, Unit change - grams to kg +rMaint: MR, Unit change - grams to kg +evspsblveg: QVEGE +evspsblsoi: QSOIL +nbp: NBP, Unit change - grams to kg +lai: TLAI +burntArea: ANN_FAREA_BURNED, unit change 'proportion' to percentage *100. +cSoilSlow: SOIL4C, Unit change - grams to kg +tsl: TSOI +mrros: QOVER +cLitter: TOTLITC, Unit change - grams to kg +cMisc: STORVEGC, Unit change - grams to kg +fLuc: DWT_CLOSS+PRODUCT_CLOSS, Unit change - grams to kg +cSoil: TOTSOMC, Unit change - grams to kg +nppLeaf: LEAFC_ALLOC, Unit change - grams to kg +mrfso: SOILICE integrated over all layers +rGrowth: GR, Unit change - grams to kg +rh: HR, Unit change - grams to kg +tran: QSOIL + QVEGT +mrro: QRUNOFF +nppRoot: FROOTC_ALLOC, Unit change - grams to kg +fVegLitter: LITFALL, Unit change - grams to kg diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_OImon.def b/examples/CESM/CMIP6/CESMtoCMIP6_OImon.def new file mode 100644 index 00000000..ffd7442e --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_OImon.def @@ -0,0 +1,24 @@ +tsice: Tsfc ????? +rlussi: flwup +bmelt: meltb, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640 +prsn: snow_ai, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640 +strocnx: strocnx ????? +strocny: strocny ????? +strairx: strairx +strairy: strairy +pr: rain_ai, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640 +sit: hi unchanged ????? +sic: aice ????? +grFrazil: frazil, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640 +tmelt: meltt, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640 +snd: hs ????? +evap: evap_ai, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640 +divice: divu ????? +streng: strength ????? +snoToIce: snoice, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640 +rsdssi: fswdn +hflssi: flat_ai +rldssi: flwdn +hfssi: fsens_ai +grCongel: congel, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640 +snomelt: melts, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640 diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_Oclim.def b/examples/CESM/CMIP6/CESMtoCMIP6_Oclim.def new file mode 100644 index 00000000..556e1587 --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_Oclim.def @@ -0,0 +1,34 @@ +diftrbbo2d: UNKNOWN +difvho: UNKNOWN +diftrxybo: UNKNOWN +tnpeot: UNKNOWN +difvtrto: UNKNOWN +difmxylo2d: UNKNOWN +difvmbo: UNKNOWN +tnkebto2d: UNKNOWN +diftrblo2d: UNKNOWN +difvtrbo: UNKNOWN +tnpeo: UNKNOWN +tnkebto: UNKNOWN +diftrxybo2d: UNKNOWN +diftrxylo2d: UNKNOWN +diftrbbo: UNKNOWN +diftrelo2d: UNKNOWN +difvmfdo: UNKNOWN +diftrebo2d: UNKNOWN +difmxybo: UNKNOWN +diftrblo: UNKNOWN +difvso: UNKNOWN +difvmto: UNKNOWN +difmxybo2d: UNKNOWN +difmxylo: UNKNOWN +dispkexyfo2d: UNKNOWN +zfull: MISSING +dispkexyfo: UNKNOWN +diftrelo: UNKNOWN +diftrxylo: UNKNOWN +difvmo: UNKNOWN +diftrebo: UNKNOWN +dispkevfo: UNKNOWN +tnpeotb: UNKNOWN +zhalf: MISSING diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_Omon.def b/examples/CESM/CMIP6/CESMtoCMIP6_Omon.def new file mode 100644 index 00000000..aea93c77 --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_Omon.def @@ -0,0 +1,102 @@ +vmo: (0.5*(VVEL(i,j)+VVEL(i-1,j))*HTN*dz*rho_0)/1000. ! g s-1 to kg s-1 +zo2min: O2_ZMIN_DEPTH +intpnitrate: photoC_NO3_diat_zint+photoC_NO3_sp_zint+photoC_NO3_diaz_zint +zsatcalc: zsatcalc +no3: NO3 +cfc11: Divide CFC11 by 1.e12 to convert to approximate mol kg-1 +soga: SALT weighted global average (if (indat3a(i,j,k).lt.1.e30) indat1a(it):indat1a(it) + (indat3a(i,j,k)*volume(i,j,k))) +co3satarag: co3_sat_arag +fbddtdin: Jint_100m_NO3+Jint_100m_NH4 +intppico: Integrate photoC_sp over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j):cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +fbddtdip: Jint_100m_PO4 +wmo: (0.5*(WVEL(i,j,k)+WVEL(i,j,k+1))*TAREA*rho_0)/1000. ! g s-1 to kg s-1 +tauvo: TAUY +zosga: ave_slc from Aixue's SLC code; derived from SSH PD TEMP SALT +fgo2: STF_O2 +fddtdin: tend_zint_100m_NO3+tend_zint_100m_NH4 +intpcalcite: Integrate -1*CaCO3_form over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j):cmordat2d(i,j) + ((-1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +msftmyz: MOC converted from Sv to kg s-1 (other code and *1.e6*1.e3) +omlmax: XMXL +fddtdip: tend_zint_100m_PO4 +zooc: zooC +ficeberg2d: Not applicable +dpco2: DpCO2, Convert ppmv to Pa via * 0.101325 +rsntds: QSW_HTP +rhopoto: PD +zsatarag: zsatarag +fgco2: FG_CO2, Convert mmol/m3 cm/s to kg m-2 s-1 via * 12.0e-8 +fbddtalk: Jint_100m_ALK, Convert meq/m3 cm/s to mol m-2 s-1 via * 1.e-5 +zossq: SSH2 units from cm2 to m2 +o2min: O2_ZMIN +co3: CO3 +intpn2: Integrate diaz_Nfix over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j):cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +rlds: LWDN_F + LWUP_F where IFRAC:0 +tauuo: TAUX +intpcalc: Integrate CaCO3_form over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j):cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +umo: (0.5*(UVEL(i,j)+UVEL(i,j-1))*HTE*dz*rho_0)/1000. ! g s-1 to kg s-1 +o2: O2 +fbddtdic: Jint_100m_DIC +thetaoga: TEMP weighted global average, units C to K if (indat3a(i,j,k).lt.1.e30) indat1a(it):indat1a(it) + (indat3a(i,j,k)*volume(i,j,k)) +fbddtdisi: Jint_100m_SiO3 +intpdiat: Integrate photoC_diat over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j):cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +msftmyzba: MOC converted from Sv to kg s-1 (other code and *1.e6*1.e3) +intpdiaz: Integrate photoC_diaz over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j):cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +phyfe: diatFe+spFe+diazFe, only topmost layer in sum +phypico: spC +chldiat: diatChl +intpp: Integrate (photoC_diat+photoC_sp+photoC_diaz) over z_t_150m (cmordat2d(i,j):cmordat2d(i,j) + ((((indat3a(i,j,k)*ocn_t_dz(k))+ (indat3b(i,j,k)*ocn_t_dz(k))+ (indat3c(i,j,k)*ocn_t_dz(k))))*1.e-5)) +zostoga: ave_slc_t from Aixue's SLC code; derived from SSH PD TEMP SALT +volo: PD +spco2: pCO2SURF, Convert ppmv to Pa via * 0.101325 +sos: SALT at topmost level * 1000 +fddtdisi: tend_zint_100m_SiO3 +hfibthermds2d: Not applicable +pr: PREC_F +co3satcalc: co3_sat_calc +physi: diatSi +tos: TEMP at topmost level, units from C to K +chl: diatChl+spChl+diazChl, only topmost layer in sum +frn: Integrate -DENITRIF over z_t (if (kmt(i,j).ge.k) then cmordat2d(i,j):cmordat2d(i,j) + ((-1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +epfe100: P_iron_FLUX_IN, only depth100m meters; z_t at index 11 +fddtalk: tend_zint_100m_ALK, Convert meq/m3 cm/s to mol m-2 s-1 via * 1.e-5 +dfe: Fe +ph: pH_3D at k:1 +epcalc100: CaCO3_FLUX_IN, only depth100m meters; z_t at index 11 +po4: PO4 +zossga: ave_delta_steric from Aixue's SLC code; derived from SSH PD TEMP SALT +nh4: NH4 +chlcalc: spChl*(spCaCO3/spC) +thetao: TEMP no change, units from C to K +epsi100: SiO2_FLUX_IN, only depth100m meters; z_t at index 11 +intpbsi: Integrate (-bSi_form) over z_t_150m (if (kmt(i,j).ge.k) then cmordat2d(i,j):cmordat2d(i,j) + ((-1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +uo: UVEL units from cm s-1 to m s-1 +phycalc: spC +chlpico: spChl +hfss: SENH_F +vo: VVEL units from cm s-1 to m s-1 +phydiaz: diazC +tossq: TEMP2 +prsn: SNOW_F +intdic: Integrate DIC over z_t (if (kmt(i,j).ge.k) then cmordat2d(i,j):cmordat2d(i,j) + ((indat3a(i,j,k)*12.0e-8)*ocn_t_dz(k))) +phydiat: diatC +fddtdic: tend_zint_100m_DIC +fbddtdife: Jint_100m_Fe +zos: SSH units from cm to m +wmosq: (0.5*(WVEL2(i,j,k)+WVEL2(i,j,k+1))*TAREA*rho_0)/1000. ! g s-1 to kg s-1 +masso: Integrate PD over ocean volume (total mass (kg) if (indat3a(i,j,k).lt.1.e30) indat1a(it):indat1a(it) + (indat3a(i,j,k)*volume(i,j,k))) +fddtdife: tend_zint_100m_Fe +epc100: POC_FLUX_IN, only depth100m meters; z_t at index 11 +phyp: (diatC+spC)*.00855+diazC*.002735 (if (kmt(i,j) .ge. 1) then cmordat2d(i,j):((indat3a(i,j,1)+indat3b(i,j,1))*0.00855) + (indat3c(i,j,1)*0.002735)) +dissoc: DOC +chldiaz: diazChl +phyc: spC+diatC+diazC, only topmost layer in sum +msftbarot: BSF, convert with * (1.e6 * 1000.) ! 10^6 m3 s-1 to kg s-1 +hfibthermds: Not applicable +phyn: (diatC+spC+diazC)*.137 (if (kmt(i,j) .ge. 1) then cmordat2d(i,j):((indat3a(i,j,1)+indat3b(i,j,1)+indat3c(i,j,1))*0.137)) +thkcello: Inapplicable as cell thickness is fixed (dz) +agessc: IAGE +dissic: DIC +fsn: NOx_FLUX+NHy_FLUX+(integrate diaz_Nfix over z_t) (if (kmt(i,j).ge.k) then cmordat2d(i,j):cmordat2d(i,j) + ((indat3a(i,j,k)*ocn_t_dz(k))*1.e-5)) +si: SiO3 +so: SALT*1000 to handle CMOR change to psu +talk: ALK at k:1 (depth0m) and converted from meq/m3 to mol m-3 via * 1.e-3 diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_Oyr.def b/examples/CESM/CMIP6/CESMtoCMIP6_Oyr.def new file mode 100644 index 00000000..3a68898a --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_Oyr.def @@ -0,0 +1,33 @@ +chlpico: spChl +co3satcalc: co3_sat_calc +co3satarag: co3_sat_arag +chldiaz: diazChl +phydiaz: diazC +no3: NO3 +phydiat: diatC +expcalc: CaCO3_FLUX_IN +expsi: SiO2_FLUX_IN +physi: diatSi +dfe: Fe +ph: pH_3D +o2: O2 +chldiat: diatChl +phyp: (diatC+spC)*.00855+diazC*.002735 ????? +po4: PO4 +dissoc: DOC +co3: CO3 +dissic: DIC +zooc: zooC +phyc: spC+diatC+diazC ????? +phyn: (diatC+spC+diazC)*.137 ????? +phymisc: diatFe+spFe+diazFe ????? +phyfe: spFe +phypico: spC +chlcalc: spChl*(spCaCO3/spC) ????? +nh4: NH4 +si: SiO3 +chl: diatChl+spChl+diazChl ????? +expc: POC_FLUX_IN +phycalc: spC +talk: ALK +expcfe: P_iron_FLUX_IN diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_aero.def b/examples/CESM/CMIP6/CESMtoCMIP6_aero.def new file mode 100644 index 00000000..bd9121ea --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_aero.def @@ -0,0 +1,41 @@ +loaddust: dst_a1+dst_a3 +concsoa: soa_a1+soa_a2 on model levels +concdust: dst_a1+dst_a3 on model levels +drybc: bc_a1DDF +concss: ncl_a1+ncl_a2+ncl_a3 on model levels +emiss: SFncl_a1+SFncl_a2+SFncl_a3 +concbc: bc_a1 on model levels +dryso4: so4_a1DDF+so4_a2DDF+so4_a3DDF +wetso2: WD_SO2 +wetso4: so4_a1SFWET+so4_a2SFWET+so4_a3SFWET+so4_c1SFWET+so4_c2SFWET+so4_c3SFWET +dryso2: DF_SO2 +concaerh2o: wat_a1+wat_a2+wat_a3 on model levels +wetbc: bc_a1SFWET+bc_c1SFWET +loadbc: bc_a1 +abs550aer: AODABS +emidms: SFDMS +emidust: SFdst_a1+SFdst_a3 +cdnc: CDNUMC on model levels +od550aer: AODVIS +emiso2: SFSO2+SO2_CLXF +loadss: ncl_a1+ncl_a2+ncl_a3 +loadso4: so4_a1+so4_a2+so4_a3 +drypoa: pom_a1DDF +reffclwtop: AREL +emiso4: so4_a1_CLXF convert molec/cm2/s to kg m-2 s-1 +concoa: soa_a1+soa_a2+pom_a1 on model levels +dryss: ncl_a1DDF+ncl_a2DDF+ncl_a3DDF +concso2: SO2 on model levels +concso4: so4_a1+so4_a2+so4_a3 on model levels +conccn: num_a1+num_a2+num_a3 on model levels +conccmcn: num_a3 on model levels +concdms: DMS on model levels +wetdust: dst_a1SFWET+dst_a3SFWET+dst_c1SFWET+dst_c3SFWET +loadsoa: soa_a1+soa_a2 +emibc: SFbc_a1+bc_a1_CLXF +dryoa: pomff_a1DDF+soa_a1DDF+soa_a2DDF +loadpoa: pom_a1 +wetss: ncl_a1SFWET+ncl_a2SFWET+ncl_a2SFWET+ncl_c1SFWET+ncl_c2SFWET+ncl_c3SFWET +drydust: DSTSFDRY +wetoa: pom_a1SFWET+pom_c1SFWET+soa_a1SFWET+soa_a2SFWET+soa_c1SFWET+soa_c2SFWET +ps: PS diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_cf3hr.def b/examples/CESM/CMIP6/CESMtoCMIP6_cf3hr.def new file mode 100644 index 00000000..6eefe849 --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_cf3hr.def @@ -0,0 +1,61 @@ +sci: FREQSH +parasolRefl: RFL_PARASOL +rsutcs: SOLIN - FSNTOAC +rldscs: FLDSC +ts: TS +reffclws: REL on model levels +clt: CLDTOT unit conversion +ta: T on model levels +psl: PSL +cfadDbze94: CFAD_DBZE94_CS +cfadLidarsr532: CFAD_SR532_CAL +clwvi: TGCLDLWP +rsdscs: FSDSC +huss: QREFHT +rtmt: FSNT - FLNT +evspsbl: QFLX +clmcalipso: CLDMED_CAL +rlds: FLDS +rlut: FSNTOA-FSNT+FLNT +clwc: CLDLIQCON on model levels +rsdt: SOLIN +rsds: FSDS +clws: CLDLIQSTR on model levels +rsus: FSDS - FSNS +rsut: SOLIN - FSNTOA +ci: FREQZM unit conversion +cltcalipso: CLDTOT_CAL +clcalipso2: CLD_CAL_NOTCS +rlutcs: FLUTC +hfls: LHFLX +cltc: CONCLD +demc: EMIS on model levels +pr: PRECT unit conversion +ps: PS +rlus: FLDS + FLNS +hurs: RHREFHT +prsnc: ZMFLXSNW+HKFLXSNW on model levels +clic: CLDICECON on model levels +dems: EMIS on model levels +tauv: TAUY +tauu: TAUX +clis: CLDICESTR on model levels +prc: PRECC unit conversion +tas: TREFHT +rsuscs: FSDSC - FSNSC +hfss: SHFLX +prlsns: LS_FLXSNW on model levels +prlsprof: LS_FLXPRC-LS_FLXSNW on model levels +prw: TMQ +prsn: PRECSC + PRECSL and unit conversion +dtauc: TOT_ICLD_VISTAU on model levels +dtaus: TOT_ICLD_VISTAU on model levels +clivi: TGCLDIWP +prcprof: ZMFLXPRC-ZMFLXSNW + RKFLXPRC-RKFLXSNW on model levels (*** the Xwalk and code show this in commnets, but xwalk lists these vars, HKFLXPRC-HKFLXSNW, as 2nd args) +cct: PCLDTOP +reffclis: REI on model levels +clcalipso: CLD_CAL +h2o: Q+CLDICE+CLDLIQ on model levels +cllcalipso: CLDLOW_CAL +clhcalipso: CLDHGH_CAL +ccb: PCLDBOT diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_cfDay.def b/examples/CESM/CMIP6/CESMtoCMIP6_cfDay.def new file mode 100644 index 00000000..ada41e86 --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_cfDay.def @@ -0,0 +1,44 @@ +va: V on model levels +hur: RHCFMIP on model levels +prc: PRECC unit conversion from m s-1 to kg m-2 s-1 +cl: CLOUD on model levels, no change +rsuscs: FSDSC - FSNSC +rsdscs: FSDSC no change +zg: Z3 on model levels +hfss: SHFLX no change +parasolRefl: RFL_PARASOL no change +rlutcs: FLUTC no change +rsutcs: SOLIN - FSNTOAC +hfls: LHFLX no change +rldscs: FLDSC no change +clmcalipso: CLDMED_CAL no change +ps: PS no change +cli: CLDICE on model levels, no change +rlus: FLDS + FLNS +rlds: FLDS no change +wap500: OMEGA500 +pctisccp: MEANPTOP_ISCCP +clt: CLDTOT unit conversion +clw: CLDLIQ on model levels, no change +clisccp: FISCCP1_COSP +ta: T on model levels +cltisccp: CLDTOT_ISCCP +rlut: FSNTOA-FSNT+FLNT ???? +rsdt: SOLIN no change +albisccp: MEANCLDALB_ISCCP +rsds: FSDS no change +hus: Q on model levels +clivi: TGCLDIWP no change +ta700: T700 +mc: CMFMC + CMFMCDZM on model levels, no change ???? +cct: PCLDTOP +wap: OMEGA on model levels +clcalipso: CLD_CAL no change +cltcalipso: CLDTOT_CAL no change +cllcalipso: CLDLOW_CAL no change +clwvi: TGCLDLWP no change +rsus: FSDS - FSNS +clhcalipso: CLDHGH_CAL no change +ccb: PCLDBOT +ua: U on model levels +rsut: FSUTOA no change diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_cfMon.def b/examples/CESM/CMIP6/CESMtoCMIP6_cfMon.def new file mode 100644 index 00000000..b8129f61 --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_cfMon.def @@ -0,0 +1,26 @@ +parasolRefl: RFL_PARASOL no change +tnhus: TAQ+PTEQ +ta: T on model levels, no change +tntscpbl: HPROGCLD/CPAIR+HSED/CPAIR (CPAIR:1004.64) +tnhusmp: PTEQ on model levels +orog: MISSING +tntc: ZMDT+EVAPTZM+ZMMTT+CMFDT +tntr: QRL+QRS +albisccp: MEANCLDALB_ISCCP +cltcalipso: CLDTOT_CAL no change +ps: PS no change +clisccp: FISCCP1_COSP +tntmp: PTTEND on model levels, no change +hur: RELHUM on model levels +hus: Q on model levels +tnt: TTEND_TOT on model levels, no change +clmcalipso: CLDMED_CAL no change +pctisccp: MEANPTOP_ISCCP +tnhusd: VD01 on model levels +tnhusa: TAQ on model levels, no change +tnhusc: ZMDQ+EVAPQZM+CMFDQ +cltisccp: CLDTOT_ISCCP +clcalipso: CLD_CAL no change +tnhusscpbl: EVAPPREC+DQSED-CME +clhcalipso: CLDHGH_CAL no change +cllcalipso: CLDLOW_CAL no change diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_cfOff.def b/examples/CESM/CMIP6/CESMtoCMIP6_cfOff.def new file mode 100644 index 00000000..251d1991 --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_cfOff.def @@ -0,0 +1,8 @@ +cltcalipso: CLDTOT_CAL no change +clcalipso2: no change +cfadDbze94: CFAD_DBZE94_CS no change +parasolRefl: RFL_PARASOL no change +clcalipso: CLD_CAL no change +clmcalipso: CLDMED_CAL no change +clhcalipso: CLDHGH_CAL no change +cfadLidarsr532: CFAD_SR532_CAL no change diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_cfSites.def b/examples/CESM/CMIP6/CESMtoCMIP6_cfSites.def new file mode 100644 index 00000000..fd3af6cb --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_cfSites.def @@ -0,0 +1,47 @@ +va: V on model levels +ci: FREQZM unit conversion +sci: FREQSH no change +hur: RELHUM on model levels +prc: PRECC unit conversion from m s-1 to kg m-2 s-1 +tas: TREFHT no change +rsuscs: FSDSC - FSNSC +rsdscs: FSDSC no change +zg: Z3 on model levels +hfss: SHFLX no change +rlutcs: FLUTC no change +prsn: PRECSC + PRECSL and unit conversion from m s-1 to kg m-2 s-1 +rldscs: FLDSC no change +pr: PRECT unit conversion from m s-1 to kg m-2 s-1 ???? Code has PRECC + PRECL, XWalk just lists PRECT +rlut: FSNTOA-FSNT+FLNT ???? +cli: CLDICE on model levels, no change +rlus: FLDS + FLNS +rtmt: FSNT - FLNT +rlds: FLDS no change +cl: CLOUD on model levels, no change +ts: TS no change +clw: CLDLIQ on model levels, no change +ta: T on model levels, no change +clivi: TGCLDIWP no change +huss: QREFHT no change +ps: PS no change +hurs: RHREFHT no change +clt: CLDTOT unit conversion +rsds: FSDS no change +hus: Q on model levels +tnt: TTEND_TOT on model levels, no change +rsutcs: SOLIN - FSNTOAC +mc: CMFMC + CMFMCDZM on model levels, no change ???? +rsdt: SOLIN no change +psl: PSL no change +cct: PCLDTOP +wap: OMEGA on model levels +prw: TMQ no change +evspsbl: QFLX no change +tauv: TAUY no change +clwvi: TGCLDLWP no change +tauu: TAUX no change +rsus: FSDS - FSNS +ccb: PCLDBOT +ua: U on model levels +rsut: SOLIN - FSNTOA +hfls: LHFLX no change diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_day.def b/examples/CESM/CMIP6/CESMtoCMIP6_day.def new file mode 100644 index 00000000..4853a499 --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_day.def @@ -0,0 +1,34 @@ +va: V interpolated to standard plevs +hur: RELHUM interpolated to standard plevs +prc: PRECC unit conversion from m s-1 to kg m-2 s-1 +tas: TREFHT no change +zg: Z3 interpolated to standard plevs +hfss: SHFLX no change +tossq: SST2, units from C to K ????? +prsn: PRECSC + PRECSL and unit conversion from m s-1 to kg m-2 s-1 +vsi: vvel_d ????? +pr: PRECC + PRECL and unit conversion from m s-1 to kg m-2 s-1 +rlut: FSNTOA-FSNT+FLNT ???? +rlus: FLDS + FLNS +sit: hi_d no change ????? +tos: SST, units from C to K ????? +usi: uvel_d ????? +sic: aice_d no change ????? +mrsos: SOILWATER_10CM +clt: CLDTOT unit conversion +wap: OMEGA interpolated to standard plevs +huss: QREFHT no change +snc: FSNO, mulitply by 100 to get percentage +psl: PSL no change +rhs: RHREFHT +snw: SNOWDP, convert from m to kg m-2 by multiplying by 1000 +rsds: FSDS no change +hus: Q interpolated to standard plevs +mrro: QRUNOFF +tasmin: TREFHTMN no change +rlds: FLDS no change +tasmax: TREFHTMX no change +rsus: FSDS - FSNS +ta: T interpolated to standard plevs +ua: U interpolated to standard plevs +hfls: LHFLX no change diff --git a/examples/CESM/CMIP6/CESMtoCMIP6_fx.def b/examples/CESM/CMIP6/CESMtoCMIP6_fx.def new file mode 100644 index 00000000..db21b085 --- /dev/null +++ b/examples/CESM/CMIP6/CESMtoCMIP6_fx.def @@ -0,0 +1,12 @@ +thkcello: dz at each gridcell +sftlf: LANDFRAC no change +areacello: TAREA convert cm2 to m2 +deptho: HT convert cm to m +volcello: TAREA times dz, convert cm3 to m3 +mrsofc: WATSAT +orog: Divide PHIS by g where LANDFRAC ne zero (where (landfrac .ne. 0)->phis/shr_const_g, shr_const_g:9.80616) +sftgif: PCT_GLACIER +hfgeou: Always set to zero in POP namelist +areacella: AREA +basin: REGION_MASK +sftof: 0 where KMT:0, 100 elsewhere diff --git a/examples/CESM/CMIP6/CMIP6_3hr.json b/examples/CESM/CMIP6/CMIP6_3hr.json new file mode 100644 index 00000000..d511acbb --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_3hr.json @@ -0,0 +1,23 @@ +"prc=PRECC unit conversion" +"tas=TREFHT no change" +"rsuscs=Would be FSDSC - FSNSC" +"rsdscs=FSDSC no change" +"huss=QREFHT no change" +"hfss=SHFLX no change" +"hfls=LHFLX no change" +"rldscs=FLDSC no change" +"tslsi=UNAVAILABLE" +"rsdsdiff=UNAVAILABLE" +"pr=PRECC + PRECL and unit conversion" +"ps=PS no change" +"rlus=FLDS + FLNS" +"rlds=FLDS no change" +"mrsos=UNAVAILABLE" +"clt=CLDTOT unit conversion" +"vas=UNAVAILABLE" +"uas=UNAVAILABLE" +"tso=UNAVAILABLE" +"rsds=FSDS no change" +"mrro=UNAVAILABLE" +"rsus=FSDS - FSNS" +"prsn=PRECSC + PRECSL and unit conversion" diff --git a/examples/CESM/CMIP6/CMIP6_6hrLev.json b/examples/CESM/CMIP6/CMIP6_6hrLev.json new file mode 100644 index 00000000..e723254f --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_6hrLev.json @@ -0,0 +1,6 @@ +"ps=PS no change" +"va=V no change" +"orog= MISSING" +"hus=Q no change" +"ua=U no change" +"ta=T no change" diff --git a/examples/CESM/CMIP6/CMIP6_6hrPlev.json b/examples/CESM/CMIP6/CMIP6_6hrPlev.json new file mode 100644 index 00000000..bc530d98 --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_6hrPlev.json @@ -0,0 +1,4 @@ +"va=V interpolated to standard plev3" +"ua=U interpolated to standard plev3" +"psl=PSL no change" +"ta=T interpolated to standard plev3" diff --git a/examples/CESM/CMIP6/CMIP6_Amon.json b/examples/CESM/CMIP6/CMIP6_Amon.json new file mode 100644 index 00000000..21da5745 --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_Amon.json @@ -0,0 +1,80 @@ +"va=V interpolated to standard plevs" +"ci=FREQZM unit conversion" (*** cannot find the conversion in code) +"n2oClim=UNKNOWN" +"n2o=N2O interpolated to standard plevs" +"rsds=FSDS no change" +"prc=PRECC unit conversion from m s-1 to kg m-2 s-1" +"tas=TREFHT no change" +"n2oglobalClim=UNKNOWN" +"rsuscs=FSDSC - FSNSC" +"rsdscs=FSDSC no change" +"huss=QREFHT no change" +"hfss=SHFLX no change" +"rtmt=FSNT - FLNT" +"prw=TMQ no change" +"tro3Clim=UNKNOWN" +"zg=Z3 interpolated to standard plevs" +"hfls=LHFLX no change" +"rldscs=FLDSC no change" +"co2massClim=UNKNOWN" +"fco2antt=UNKNOWN" +"pr=PRECC + PRECL and unit conversion from m s-1 to kg m-2 s-1" +"ps=PS no change" +"co2=CO2 interpolated to standard plevs, scaled by 28.966/44" +"ch4=CH4 interpolated to standard plevs" +"cli=CLDICE on model levels, no change" +"cfc11global=f11vmr convert to mass" +"clt=CLDTOT unit conversion" (*** cannot find the conversion in code) +"rlds=FLDS no change" +"cl=CLOUD on model levels, no change" +"fco2nat=UNKNOWN" +"ts=TS no change" +"tasAdjust=UNKNOWN" +"co2Clim=UNKNOWN" +"ch4globalClim=UNKNOWN" +"vas=UNAVAILABLE" +"clw=CLDLIQ on model levels, no change" +"uas=UNAVAILABLE" +"ta=T interpolated to standard plevs" +"orog= MISSING" +"hcfc22global=UNKNOWN" +"tasmin=TREFMNAV no change" +"phalf= MISSING" +"rlut=FSNTOA-FSNT+FLNT" +"hurs=RHREFHT no change" +"tsAdjust=UNKNOWN" +"fco2fos=UNKNOWN" +"wap=OMEGA interpolated to standard plevs" +"pslAdjust=UNKNOWN" +"n2oglobal=n2ovmr convert to mass" +"pfull= MISSING" +"ch4global=ch4vmr convert to mass" +"hur=RELHUM interpolated to standard plevs" +"hus=Q interpolated to standard plevs" +"tro3=O3 interpolated to standard plevs" +"rsdt=SOLIN no change" +"sbl=UNAVAILABLE" +"clivi=TGCLDIWP no change" +"rsutcs=SOLIN - FSNTOAC" +"co2mass=co2vmr convert to mass" +"rlus=FLDS + FLNS" +"mc=CMFMC + CMFMCDZM on model levels, no change" +"rlutcs=FLUTC no change" +"psl=PSL no change" +"cct=UNAVAILABLE" +"cfc113global=UNKNOWN" +"prAdjust=UNKNOWN" +"evspsbl=QFLX no change" +"sfcWind=U10 no change" +"ch4Clim=UNKNOWN" +"tauv=TAUY no change" +"rsut=SOLIN - FSNTOA" +"clwvi=TGCLDLWP no change" +"tauu=TAUX no change" +"tasmax=TREFMXAV no change" +"rsus=FSDS - FSNS" +"ccb=UNAVAILABLE" +"ua=U interpolated to standard plevs" +"cfc12global=f12vmr convert to mass" +"sci=FREQSH no change" +"prsn=PRECSC + PRECSL and unit conversion from m s-1 to kg m-2 s-1" diff --git a/examples/CESM/CMIP6/CMIP6_LImon.json b/examples/CESM/CMIP6/CMIP6_LImon.json new file mode 100644 index 00000000..c42b094e --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_LImon.json @@ -0,0 +1,12 @@ +"tsn=would be SNOTTOPL" +"tpf=UNAVAILABLE" +"snd=SNOWDP unchanged" +"snm=QMELT, unit change from mm s-1 to kg m-2 s-1, same numerical value"(where (indat2a /= var_info(var_found(1,1))%missing_value), else 0) +"snc=FSNO, mulitply by 100 to get percentage" +"agesno=UNAVAILABLE" +"hfdsn=FGR" +"snw=H2OSNO, unit change from mm to kg m-2, same numerical value" +"sootsn=SNOBCMSL + SNODSTMSL + SNOOCMSL" +"lwsnl=SNOWLIQ no change" +"pflw=UNAVAILABLE" +"sbl=would be QFLX_SUB_SNOW" diff --git a/examples/CESM/CMIP6/CMIP6_Lmon.json b/examples/CESM/CMIP6/CMIP6_Lmon.json new file mode 100644 index 00000000..168eaff2 --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_Lmon.json @@ -0,0 +1,59 @@ +"fLitterSoil=LITR1C_TO_SOIL1C+LITR2C_TO_SOIL2C+LITR3C_TO_SOIL3C, Unit change - grams to kg" +"nppWood=WOODC_ALLOC, Unit change - grams to kg" +"shrubFrac=UNAVAILABLE" +"prveg=QINTR" +"mrlsl=SOILLIQ + SOILICE at each soil depth" +"cProduct=TOTPRODC, Unit change - grams to kg" +"cWood=WOODC, Unit change - grams to kg" +"treeFracSecDec=UNAVAILABLE" +"fGrazing=UNAVAILABLE" +"cSoilFast=SOIL1C+SOIL2C, Unit change - grams to kg" +"cLeaf=LEAFC, Unit change - grams to kg" +"residualFrac=UNAVAILABLE" +"cLitterAbove=UNAVAILABLE" +"cSoilMedium=SOIL3C, Unit change - grams to kg" +"cropFrac=UNAVAILABLE" +"fFire=COL_FIRE_CLOSS, Unit change - grams to kg" +"baresoilFrac=UNAVAILABLE" +"cVeg=TOTVEGC, Unit change - grams to kg" +"cRoot=FROOTC+LIVE_ROOTC+DEAD_ROOTC, Unit change - grams to kg" +"mrso=SOILICE+SOILLIQ, mask over ice-covered regions at 5000 kg m-2" +"grassFrac=UNAVAILABLE" +"ra=AR, Unit change - grams to kg" +"mrsos=SOILWATER_10CM" +"npp=NPP, Unit change - grams to kg" +"cCwd=CWDC, Unit change - grams to kg" +"gpp=GPP, Unit change - grams to kg" +"rMaint=MR, Unit change - grams to kg" +"evspsblveg=QVEGE" +"treeFrac=UNAVAILABLE" +"evspsblsoi=QSOIL" +"nbp=NBP, Unit change - grams to kg" +"lai=TLAI" +"fHarvest=UNAVAILABLE" +"burntArea=ANN_FAREA_BURNED, unit change 'proportion' to percentage *100." +"cSoilSlow=SOIL4C, Unit change - grams to kg" +"tsl=TSOI" +"mrros=QOVER" +"cLitter=TOTLITC, Unit change - grams to kg" +"cMisc=STORVEGC, Unit change - grams to kg" +"fLuc=DWT_CLOSS+PRODUCT_CLOSS, Unit change - grams to kg" +"cSoil=TOTSOMC, Unit change - grams to kg" +"nppLeaf=LEAFC_ALLOC, Unit change - grams to kg" +"treeFracPrimDec=UNAVAILABLE" +"treeFracPrimEver=UNAVAILABLE" +"mrfso=SOILICE integrated over all layers" +"fVegSoil=UNAVAILABLE" +"rGrowth=GR, Unit change - grams to kg" +"rh=HR, Unit change - grams to kg" +"cLitterBelow=UNAVAILABLE" +"tran=QSOIL + QVEGT" +"mrro=QRUNOFF" +"nppRoot=FROOTC_ALLOC, Unit change - grams to kg" +"treeFracSecEver=UNAVAILABLE" +"pastureFrac=UNAVAILABLE" +"nep=UNKNOWN" +"c3PftFrac=UNAVAILABLE" +"fVegLitter=LITFALL, Unit change - grams to kg" +"c4PftFrac=UNAVAILABLE" +"landCoverFrac=UNAVAILABLE" diff --git a/examples/CESM/CMIP6/CMIP6_OImon.json b/examples/CESM/CMIP6/CMIP6_OImon.json new file mode 100644 index 00000000..b7b7b776 --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_OImon.json @@ -0,0 +1,40 @@ +"sblsi=UNAVAILABLE" +"ialb=UNAVAILABLE" +"transix=UNAVAILABLE" +"transiy=UNAVAILABLE" +"tsice=Tsfc" ????? +"ridgice=UNAVAILABLE" +"rlussi=flwup" +"bmelt=meltb, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"prsn=snow_ai, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"strocnx=strocnx" ????? +"strocny=strocny" ????? +"grLateral=UNAVAILABLE" +"strairx=strairx" +"strairy=strairy" +"pr=rain_ai, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"nshrice=UNAVAILABLE" +"sit=hi unchanged" ????? +"sic=aice" ????? +"grFrazil=frazil, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"tmelt=meltt, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"sim=UNAVAILABLE" +"hcice=UNAVAILABLE" +"snd=hs" ????? +"snc=UNAVAILABLE" +"evap=evap_ai, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"divice=divu" ????? +"rsussi=UNAVAILABLE" +"ssi=UNAVAILABLE" +"ageice=UNAVAILABLE" +"streng=strength" ????? +"transifs=UNAVAILABLE" +"snoToIce=snoice, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"rsdssi=fswdn" +"hflssi=flat_ai" +"rldssi=flwdn" +"hfssi=fsens_ai" +"grCongel=congel, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" +"tsnint=UNAVAILABLE" +"eshrice=UNAVAILABLE" +"snomelt=melts, (divide by 86400 s day-1, divide by 100 cm m-1, times 1000 kg m-3)/8640" diff --git a/examples/CESM/CMIP6/CMIP6_Oclim.json b/examples/CESM/CMIP6/CMIP6_Oclim.json new file mode 100644 index 00000000..698a2df8 --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_Oclim.json @@ -0,0 +1,34 @@ +"diftrbbo2d=UNKNOWN" +"difvho=UNKNOWN" +"diftrxybo=UNKNOWN" +"tnpeot=UNKNOWN" +"difvtrto=UNKNOWN" +"difmxylo2d=UNKNOWN" +"difvmbo=UNKNOWN" +"tnkebto2d=UNKNOWN" +"diftrblo2d=UNKNOWN" +"difvtrbo=UNKNOWN" +"tnpeo=UNKNOWN" +"tnkebto=UNKNOWN" +"diftrxybo2d=UNKNOWN" +"diftrxylo2d=UNKNOWN" +"diftrbbo=UNKNOWN" +"diftrelo2d=UNKNOWN" +"difvmfdo=UNKNOWN" +"diftrebo2d=UNKNOWN" +"difmxybo=UNKNOWN" +"diftrblo=UNKNOWN" +"difvso=UNKNOWN" +"difvmto=UNKNOWN" +"difmxybo2d=UNKNOWN" +"difmxylo=UNKNOWN" +"dispkexyfo2d=UNKNOWN" +"zfull= MISSING" +"dispkexyfo=UNKNOWN" +"diftrelo=UNKNOWN" +"diftrxylo=UNKNOWN" +"difvmo=UNKNOWN" +"diftrebo=UNKNOWN" +"dispkevfo=UNKNOWN" +"tnpeotb=UNKNOWN" +"zhalf= MISSING" diff --git a/examples/CESM/CMIP6/CMIP6_Omon.json b/examples/CESM/CMIP6/CMIP6_Omon.json new file mode 100644 index 00000000..bb4c7e83 --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_Omon.json @@ -0,0 +1,189 @@ +"vmo=(0.5*(VVEL(i,j)+VVEL(i-1,j))*HTN*dz*rho_0)/1000. ! g s-1 to kg s-1" +"mlotstsq=UNKNOWN" +"dpo2=UNAVAILABLE" +"mfo=UNKNOWN" +"msftyyzba=UNKNOWN" +"zo2min=O2_ZMIN_DEPTH" +"intpnitrate=photoC_NO3_diat_zint+photoC_NO3_sp_zint+photoC_NO3_diaz_zint" +"zsatcalc=zsatcalc" +"bsi=UNAVAILABLE" +"intpbfe=UNAVAILABLE" +"tauucorr=UNKNOWN" +"no3=NO3" +"hfxdiff=UNKNOWN" +"cfc11=Divide CFC11 by 1.e12 to convert to approximate mol kg-1" +"bfe=UNAVAILABLE" +"hfevapds=UNKNOWN" +"soga=SALT weighted global average" (if (indat3a(i,j,k).lt.1.e30) indat1a(it) = indat1a(it) + (indat3a(i,j,k)*volume(i,j,k))) +"co3satarag=co3_sat_arag" +"fbddtdin=Jint_100m_NO3+Jint_100m_NH4" +"intppico=Integrate photoC_sp over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"fbddtdip=Jint_100m_PO4" +"wmo=(0.5*(WVEL(i,j,k)+WVEL(i,j,k+1))*TAREA*rho_0)/1000. ! g s-1 to kg s-1" +"hfsnthermds=UNKNOWN" +"tauvo=TAUY" +"vsfcorr=UNKNOWN" +"zosga=ave_slc from Aixue's SLC code; derived from SSH PD TEMP SALT" +"pso=UNKNOWN" +"fgo2=STF_O2" +"fddtdin=tend_zint_100m_NO3+tend_zint_100m_NH4" +"intpcalcite=Integrate -1*CaCO3_form over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((-1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"eparag100=UNAVAILABLE" +"msftmyz=MOC converted from Sv to kg s-1" (other code and *1.e6*1.e3) +"hfbasindiff=UNKNOWN" +"omlmax=XMXL" +"omldamax=UNKNOWN" +"arag=UNAVAILABLE" +"fddtdip=tend_zint_100m_PO4" +"hfrunoffds2d=UNKNOWN" +"fsitherm=UNKNOWN" +"zooc=zooC" +"ficeberg2d=Not applicable" +"sltovovrt=UNKNOWN" +"dpco2=DpCO2, Convert ppmv to Pa via * 0.101325" +"hfnorthba=UNKNOWN" +"hfsithermds2d=UNKNOWN" +"hfsnthermds2d=UNKNOWN" +"rsntds=QSW_HTP" +"sltovgyre=UNKNOWN" +"evs=UNKNOWN" +"rhopoto=PD" +"zsatarag=zsatarag" +"bacc=UNAVAILABLE" +"fgco2=FG_CO2, Convert mmol/m3 cm/s to kg m-2 s-1 via * 12.0e-8" +"fsfe=UNAVAILABLE" +"frfe=UNAVAILABLE" +"vsf=UNKNOWN" +"zmicro=UNAVAILABLE" +"fbddtalk=Jint_100m_ALK, Convert meq/m3 cm/s to mol m-2 s-1 via * 1.e-5" +"zossq=SSH2 units from cm2 to m2" +"o2min=O2_ZMIN" +"co3=CO3" +"intpn2=Integrate diaz_Nfix over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"hfnorthdiff=UNKNOWN" +"rlds=LWDN_F + LWUP_F where IFRAC = 0" +"masscello=UNKNOWN" +"friver=UNKNOWN" +"tauuo=TAUX" +"intpcalc=Integrate CaCO3_form over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"umo=(0.5*(UVEL(i,j)+UVEL(i,j-1))*HTE*dz*rho_0)/1000. ! g s-1 to kg s-1" +"o2=O2" +"fbddtdic=Jint_100m_DIC" +"fgdms=UNAVAILABLE" +"thetaoga=TEMP weighted global average, units C to K" if (indat3a(i,j,k).lt.1.e30) indat1a(it) = indat1a(it) + (indat3a(i,j,k)*volume(i,j,k)) +"hfsithermds=UNKNOWN" +"fbddtdisi=Jint_100m_SiO3" +"hfrunoffds=UNKNOWN" +"hfcorr=UNKNOWN" +"wfonocorr=UNKNOWN" +"intpdiat=Integrate photoC_diat over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"pbo=UNKNOWN" +"rsds=UNKNOWN" +"phymisc=UNAVAILABLE" +"msftmyzba=MOC converted from Sv to kg s-1" (other code and *1.e6*1.e3) +"intpdiaz=Integrate photoC_diaz over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"hfsifrazil2d=UNKNOWN" +"phyfe=diatFe+spFe+diazFe, only topmost layer in sum" +"phypico=spC" +"msftyyz=UNKNOWN" +"vsfevap=UNKNOWN" +"htovgyre=UNKNOWN" +"chldiat=diatChl" +"wfcorr=UNKNOWN" +"hfgeou=Not applicable" +"vsfsit=UNKNOWN" +"hfydiff=UNKNOWN" +"intpp=Integrate (photoC_diat+photoC_sp+photoC_diaz) over z_t_150m" (cmordat2d(i,j) = cmordat2d(i,j) + ((((indat3a(i,j,k)*ocn_t_dz(k))+ (indat3b(i,j,k)*ocn_t_dz(k))+ (indat3c(i,j,k)*ocn_t_dz(k))))*1.e-5)) +"zostoga=ave_slc_t from Aixue's SLC code; derived from SSH PD TEMP SALT" +"volo=PD" +"spco2=pCO2SURF, Convert ppmv to Pa via * 0.101325" +"pop=UNAVAILABLE" +"hfrainds=UNKNOWN" +"pon=UNAVAILABLE" +"sos=SALT at topmost level * 1000" +"hfds=UNKNOWN" +"hfls=UNKNOWN" +"fddtdisi=tend_zint_100m_SiO3" +"vsfpr=UNKNOWN" +"hfibthermds2d=Not applicable" +"chlmisc=UNAVAILABLE" +"hfyba=UNKNOWN" +"pr=PREC_F" +"co3satcalc=co3_sat_calc" +"physi=diatSi" +"tos=TEMP at topmost level, units from C to K" +"chl=diatChl+spChl+diazChl, only topmost layer in sum" +"frn=Integrate -DENITRIF over z_t" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((-1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"hfnorth=UNKNOWN" +"epfe100=P_iron_FLUX_IN, only "depth100m" meters; z_t at index 11" +"fddtalk=tend_zint_100m_ALK, Convert meq/m3 cm/s to mol m-2 s-1 via * 1.e-5" +"dfe=Fe" +"ph=pH_3D at k=1" +"calc=UNAVAILABLE" +"epcalc100=CaCO3_FLUX_IN, only "depth100m" meters; z_t at index 11" +"msftmrhoz=UNAVAILABLE" +"sfdsi=UNKNOWN" +"ficeberg=Not applicable" +"msftyrhozba=UNAVAILABLE" +"po4=PO4" +"zossga=ave_delta_steric from Aixue's SLC code; derived from SSH PD TEMP SALT" +"nh4=NH4" +"chlcalc=spChl*(spCaCO3/spC)" +"sfriver=UNKNOWN" +"thetao=TEMP no change, units from C to K" +"epsi100=SiO2_FLUX_IN, only "depth100m" meters; z_t at index 11" +"wfo=UNKNOWN" +"zfull= MISSING" +"hfsifrazil=UNKNOWN" +"intpbsi=Integrate (-bSi_form) over z_t_150m" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((-1.0e-5*indat3a(i,j,k))*ocn_t_dz(k))) +"uo=UVEL units from cm s-1 to m s-1" +"dms=UNAVAILABLE" +"mlotst=UNKNOWN" +"phycalc=spC" +"intpmisc=UNAVAILABLE" +"htovovrt=UNKNOWN" +"chlpico=spChl" +"tauvcorr=UNKNOWN" +"msftyrhoz=UNAVAILABLE" +"hfss=SENH_F" +"vo=VVEL units from cm s-1 to m s-1" +"phydiaz=diazC" +"tossq=TEMP2" +"prsn=SNOW_F" +"intdic=Integrate DIC over z_t" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((indat3a(i,j,k)*12.0e-8)*ocn_t_dz(k))) +"zoocmisc=UNAVAILABLE" +"phydiat=diatC" +"hfbasin=UNKNOWN" +"fddtdic=tend_zint_100m_DIC" +"hfxba=UNKNOWN" +"msftmrhozba=UNAVAILABLE" +"fbddtdife=Jint_100m_Fe" +"zos=SSH units from cm to m" +"wmosq=(0.5*(WVEL2(i,j,k)+WVEL2(i,j,k+1))*TAREA*rho_0")/1000. ! g s-1 to kg s-1 +"masso=Integrate PD over ocean volume" (total mass (kg) if (indat3a(i,j,k).lt.1.e30) indat1a(it) = indat1a(it) + (indat3a(i,j,k)*volume(i,j,k))) +"vsfriver=UNKNOWN" +"fddtdife=tend_zint_100m_Fe" +"detoc=UNAVAILABLE" +"epc100=POC_FLUX_IN, only "depth100m" meters; z_t at index 11" +"phyp=(diatC+spC)*.00855+diazC*.002735" (if (kmt(i,j) .ge. 1) then cmordat2d(i,j) = ((indat3a(i,j,1)+indat3b(i,j,1))*0.00855) + (indat3c(i,j,1)*0.002735)) +"dissoc=DOC" +"zmeso=UNAVAILABLE" +"chldiaz=diazChl" +"phyc=spC+diatC+diazC, only topmost layer in sum" +"msftbarot=BSF, convert with * (1.e6 * 1000.) ! 10^6 m3 s-1 to kg s-1" +"hfibthermds=Not applicable" +"phyn=(diatC+spC+diazC)*.137" (if (kmt(i,j) .ge. 1) then cmordat2d(i,j) = ((indat3a(i,j,1)+indat3b(i,j,1)+indat3c(i,j,1))*0.137)) +"thkcello=Inapplicable as cell thickness is fixed (dz)" +"hfy=UNKNOWN" +"hfx=UNKNOWN" +"fsc=UNAVAILABLE" +"agessc=IAGE" +"dissic=DIC" +"fsn=NOx_FLUX+NHy_FLUX+(integrate diaz_Nfix over z_t)" (if (kmt(i,j).ge.k) then cmordat2d(i,j) = cmordat2d(i,j) + ((indat3a(i,j,k)*ocn_t_dz(k))*1.e-5)) +"frc=UNAVAILABLE" +"intparag=UNAVAILABLE" +"si=SiO3" +"so=SALT*1000 to handle CMOR change to psu" +"hfbasinba=UNKNOWN" +"zhalf= MISSING" +"talk=ALK at k=1 (depth0m) and converted from meq/m3 to mol m-3 via * 1.e-3" diff --git a/examples/CESM/CMIP6/CMIP6_Oyr.json b/examples/CESM/CMIP6/CMIP6_Oyr.json new file mode 100644 index 00000000..c34cdbec --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_Oyr.json @@ -0,0 +1,73 @@ +"pbsi=UNKNOWN" +"chlpico=spChl" +"bacc=UNKNOWN" +"co3satcalc=co3_sat_calc" +"co3satarag=co3_sat_arag" +"darag=UNKNOWN" +"pop=UNKNOWN" +"chldiaz=diazChl" +"pon=UNKNOWN" +"phydiaz=diazC" +"no3=NO3" +"zmicro=UNKNOWN" +"zoocmisc=UNKNOWN" +"phydiat=diatC" +"expcalc=CaCO3_FLUX_IN" +"phypmisc=UNKNOWN" +"pp=UNKNOWN" +"dpocdtpico=UNKNOWN" +"parag=UNKNOWN" +"bfe=UNKNOWN" +"pnitrate=UNKNOWN" +"expsi=SiO2_FLUX_IN" +"chlmisc=UNKNOWN" +"pcalc=UNKNOWN" +"bddtdisi=UNKNOWN" +"physi=diatSi" +"arag=UNKNOWN" +"pbfe=UNKNOWN" +"fediss=UNKNOWN" +"bddtdic=UNKNOWN" +"dfe=Fe" +"ph=pH_3D" +"calc=UNKNOWN" +"o2=O2" +"expn=UNKNOWN" +"detoc=UNKNOWN" +"exparag=UNKNOWN" +"chldiat=diatChl" +"phyp=(diatC+spC)*.00855+diazC*.002735" ????? +"po4=PO4" +"dissoc=DOC" +"co3=CO3" +"zmeso=UNKNOWN" +"dissic=DIC" +"zooc=zooC" +"dpocdtdiaz=UNKNOWN" +"bsi=UNKNOWN" +"fescav=UNKNOWN" +"bddtalk=UNKNOWN" +"phyc=spC+diatC+diazC" ????? +"phyn=(diatC+spC+diazC)*.137" ????? +"phymisc=diatFe+spFe+diazFe" ????? +"bddtdife=UNKNOWN" +"phyfe=spFe" +"phypico=spC" +"graz=UNKNOWN" +"chlcalc=spChl*(spCaCO3/spC)" ????? +"nh4=NH4" +"zfull= MISSING" +"bddtdip=UNKNOWN" +"expp=UNKNOWN" +"si=SiO3" +"dms=UNKNOWN" +"bddtdin=UNKNOWN" +"dcalc=UNKNOWN" +"pdi=UNKNOWN" +"chl=diatChl+spChl+diazChl" ????? +"expc=POC_FLUX_IN" +"phycalc=spC" +"dpocdtcalc=UNKNOWN" +"zhalf= MISSING" +"talk=ALK" +"expcfe=P_iron_FLUX_IN" diff --git a/examples/CESM/CMIP6/CMIP6_aero.json b/examples/CESM/CMIP6/CMIP6_aero.json new file mode 100644 index 00000000..558d1891 --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_aero.json @@ -0,0 +1,83 @@ +"loaddust=dst_a1+dst_a3" +"concsoa=soa_a1+soa_a2 on model levels" +"drydms=UNAVAILABLE" +"concdust=dst_a1+dst_a3 on model levels" +"emibb=UNKNOWN" +"drybc=bc_a1DDF" +"wetnh4=UNKNOWN" +"sconcnh4=UNKNOWN" +"sconcsoa=UNKNOWN" +"concss=ncl_a1+ncl_a2+ncl_a3 on model levels" +"emiss=SFncl_a1+SFncl_a2+SFncl_a3" +"emioa=UNKNOWN" +"loadoa=UNKNOWN" +"concbc=bc_a1 on model levels" +"dryso4=so4_a1DDF+so4_a2DDF+so4_a3DDF" +"wetso2=WD_SO2" +"wetso4=so4_a1SFWET+so4_a2SFWET+so4_a3SFWET+so4_c1SFWET+so4_c2SFWET+so4_c3SFWET" +"orog= MISSING" +"dryso2=DF_SO2" +"sconcpoa=UNKNOWN" +"concaerh2o=wat_a1+wat_a2+wat_a3 on model levels" +"sconcoa=UNKNOWN" +"wetbc=bc_a1SFWET+bc_c1SFWET" +"cldnvi=UNKNOWN" +"inc=UNKNOWN" +"reffclwc=UNKNOWN" +"sconcdust=UNKNOWN" +"loadbc=bc_a1" +"abs550aer=AODABS" +"ec550aer=UNKNOWN" +"loadno3=UNKNOWN" +"reffclws=UNKNOWN" +"concnh4=UNKNOWN" +"concpoa=UNKNOWN" +"emidms=SFDMS" +"emidust=SFdst_a1+SFdst_a3" +"wetdms=UNKNOWN" +"rsdsdiff=UNKNOWN" +"cdnc=CDNUMC on model levels" +"wetpoa=UNKNOWN" +"od550aer=AODVIS" +"sconcno3=UNKNOWN" +"wetsoa=UNKNOWN" +"drynh3=UNAVAILABLE" +"emiso2=SFSO2+SO2_CLXF" +"drysoa=UNAVAILABLE" +"drynh4=UNAVAILABLE" +"chepsoa=UNKNOWN" +"loadss=ncl_a1+ncl_a2+ncl_a3" +"loadso4=so4_a1+so4_a2+so4_a3" +"drypoa=pom_a1DDF" +"reffclwtop=AREL" +"emiso4=so4_a1_CLXF convert molec/cm2/s to kg m-2 s-1" +"sconcbc=UNKNOWN" +"concoa=soa_a1+soa_a2+pom_a1 on model levels" +"dryss=ncl_a1DDF+ncl_a2DDF+ncl_a3DDF" +"concso2=SO2 on model levels" +"concnmcn=UNKNOWN" +"concso4=so4_a1+so4_a2+so4_a3 on model levels" +"od550lt1aer=UNKNOWN" +"loadnh4=UNKNOWN" +"conccn=num_a1+num_a2+num_a3 on model levels" +"conccmcn=num_a3 on model levels" +"concdms=DMS on model levels" +"wetdust=dst_a1SFWET+dst_a3SFWET+dst_c1SFWET+dst_c3SFWET" +"cldncl=UNKNOWN" +"loadsoa=soa_a1+soa_a2" +"rsdscsdiff=UNKNOWN" +"cldnci=UNKNOWN" +"concbb=UNKNOWN" +"sconcss=UNKNOWN" +"sconcso4=UNKNOWN" +"emibc=SFbc_a1+bc_a1_CLXF" +"emipoa=UNKNOWN" +"od870aer=UNKNOWN" +"eminh3=UNKNOWN" +"dryoa=pomff_a1DDF+soa_a1DDF+soa_a2DDF" +"loadpoa=pom_a1" +"wetss=ncl_a1SFWET+ncl_a2SFWET+ncl_a2SFWET+ncl_c1SFWET+ncl_c2SFWET+ncl_c3SFWET" +"drydust=DSTSFDRY" +"wetoa=pom_a1SFWET+pom_c1SFWET+soa_a1SFWET+soa_a2SFWET+soa_c1SFWET+soa_c2SFWET" +"concno3=UNKNOWN" +"ps=PS" diff --git a/examples/CESM/CMIP6/CMIP6_cf3hr.json b/examples/CESM/CMIP6/CMIP6_cf3hr.json new file mode 100644 index 00000000..19bc7966 --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_cf3hr.json @@ -0,0 +1,86 @@ +"sci=FREQSH no change" +"parasolRefl=RFL_PARASOL no change" +"rsutcs=SOLIN - FSNTOAC" ????? +"rldscs=FLDSC no change" +"reffclwc=UNKNOWN" +"ts=TS no change" +"clc=UNKNOWN" +"reffclws=REL on model levels" +"clt=CLDTOT unit conversion" +"ta=T on model levels" +"psl=PSL no change" +"cfadDbze94=CFAD_DBZE94_CS no change" +"cfadLidarsr532=CFAD_SR532_CAL no change" +"sfcWind=UNAVAILABLE" +"zfull= MISSING" +"clwvi=TGCLDLWP no change" +"toffset= MISSING" +"reffgrpls=UNKNOWN" +"rsdscs=FSDSC no change" +"huss=QREFHT no change" +"rtmt=FSNT - FLNT" ????? +"evspsbl=QFLX no change" +"clmcalipso=CLDMED_CAL no change" +"rlds=FLDS no change" +"grpllsprof=UNKNOWN" +"rlut=FSNTOA-FSNT+FLNT" ????? +"clwc=CLDLIQCON on model levels" +"rsdt=SOLIN no change" +"rsds=FSDS no change" +"sbl=UNAVAILABLE" +"clws=CLDLIQSTR on model levels" +"rsus=FSDS - FSNS" ????? +"rsut=SOLIN - FSNTOA" ????? +"ci=FREQZM unit conversion" +"cltcalipso=CLDTOT_CAL no change" +"clcalipso2=CLD_CAL_NOTCS no change" +"rlutcs=FLUTC no change" +"hfls=LHFLX no change" +"cltc=CONCLD" +"demc=EMIS on model levels" +"pr=PRECT unit conversion" +"ps=PS no change" +"rlus=FLDS + FLNS" ????? +"vas=UNAVAILABLE" +"hurs=RHREFHT no change" +"reffsnows=UNKNOWN" +"reffrains=UNKNOWN" +"prsnc=ZMFLXSNW+HKFLXSNW on model levels" +"latitude= MISSING" +"pfull= MISSING" +"clic=CLDICECON on model levels" +"reffsnowc=UNKNOWN" +"reffrainc=UNKNOWN" +"dems=EMIS on model levels" +"longitude= MISSING" +"fco2antt=UNKNOWN" +"tauv=TAUY no change" +"tauu=TAUX no change" +"cls=UNKNOWN" +"orog= MISSING" +"clis=CLDICESTR on model levels" +"prc=PRECC unit conversion" +"tas=TREFHT no change" +"rsuscs=FSDSC - FSNSC" ????? +"uas=UNAVAILABLE" +"hfss=SHFLX no change" +"prlsns=LS_FLXSNW on model levels" +"prlsprof=LS_FLXPRC-LS_FLXSNW on model levels" +"prw=TMQ no change" +"prsn=PRECSC + PRECSL and unit conversion" +"fco2nat=UNKNOWN" +"phalf= MISSING" +"dtauc=TOT_ICLD_VISTAU on model levels" +"fco2fos=UNKNOWN" +"dtaus=TOT_ICLD_VISTAU on model levels" +"clivi=TGCLDIWP no change" +"prcprof=ZMFLXPRC-ZMFLXSNW + RKFLXPRC-RKFLXSNW on model levels" (*** the Xwalk and code show this in commnets, but xwalk lists these vars, HKFLXPRC-HKFLXSNW, as 2nd args) +"cct=PCLDTOP" +"reffclis=REI on model levels" +"clcalipso=CLD_CAL no change" +"h2o=Q+CLDICE+CLDLIQ on model levels" +"cllcalipso=CLDLOW_CAL no change" +"clhcalipso=CLDHGH_CAL no change" +"zhalf= MISSING" +"reffclic=UNKNOWN" +"ccb=PCLDBOT" diff --git a/examples/CESM/CMIP6/CMIP6_cfDay.json b/examples/CESM/CMIP6/CMIP6_cfDay.json new file mode 100644 index 00000000..2222d07a --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_cfDay.json @@ -0,0 +1,47 @@ +"va=V on model levels" +"hur=RHCFMIP on model levels" +"prc=PRECC unit conversion from m s-1 to kg m-2 s-1" +"cl=CLOUD on model levels, no change" +"rsuscs=FSDSC - FSNSC" +"rsdscs=FSDSC no change" +"zg=Z3 on model levels" +"hfss=SHFLX no change" +"parasolRefl=RFL_PARASOL no change" +"rlutcs=FLUTC no change" +"rsutcs=SOLIN - FSNTOAC" +"hfls=LHFLX no change" +"rldscs=FLDSC no change" +"clmcalipso=CLDMED_CAL no change" +"ps=PS no change" +"cli=CLDICE on model levels, no change" +"rlus=FLDS + FLNS" +"rlds=FLDS no change" +"wap500=OMEGA500" +"pctisccp=MEANPTOP_ISCCP" +"clt=CLDTOT unit conversion" +"clw=CLDLIQ on model levels, no change" +"clisccp=FISCCP1_COSP" +"ta=T on model levels" +"phalf= MISSING" +"cltisccp=CLDTOT_ISCCP" +"rlut=FSNTOA-FSNT+FLNT" ???? +"rsdt=SOLIN no change" +"pfull= MISSING" +"albisccp=MEANCLDALB_ISCCP" +"rsds=FSDS no change" +"hus=Q on model levels" +"clivi=TGCLDIWP no change" +"ta700=T700" +"mc=CMFMC + CMFMCDZM on model levels, no change" ???? +"cct=PCLDTOP" +"wap=OMEGA on model levels" +"clcalipso=CLD_CAL no change" +"cltcalipso=CLDTOT_CAL no change" +"cllcalipso=CLDLOW_CAL no change" +"clwvi=TGCLDLWP no change" +"rsus=FSDS - FSNS" +"clhcalipso=CLDHGH_CAL no change" +"ccb=PCLDBOT" +"ua=U on model levels" +"rsut=FSUTOA no change" +"orog= MISSING" diff --git a/examples/CESM/CMIP6/CMIP6_cfMon.json b/examples/CESM/CMIP6/CMIP6_cfMon.json new file mode 100644 index 00000000..2ef88771 --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_cfMon.json @@ -0,0 +1,99 @@ +"rlutcs4co2=UNKNOWN" +"tnsclibfpcl=UNKNOWN" +"parasolRefl=RFL_PARASOL no change" +"rlucs4co2=UNKNOWN" +"tnsclihencl=UNKNOWN" +"tnhus=TAQ+PTEQ" +"tnsclids=UNKNOWN" +"clc=UNKNOWN" +"tnsclia=UNKNOWN" +"ta=T on model levels, no change" +"tnsclimr=UNKNOWN" +"tnsclibl=UNKNOWN" +"rlucs=UNKNOWN" +"tnsclwac=UNKNOWN" +"rsucs=UNKNOWN" +"tnsclwar=UNKNOWN" +"tnsclwas=UNKNOWN" +"tnsclwmi=UNKNOWN" +"tntscpbl=HPROGCLD/CPAIR+HSED/CPAIR (CPAIR = 1004.64)" +"rsu4co2=UNKNOWN" +"tnhusmp=PTEQ on model levels" +"orog= MISSING" +"tnsclwri=UNKNOWN" +"rsdcs4co2=UNKNOWN" +"tnsclirir=UNKNOWN" +"tnsclwhon=UNKNOWN" +"tnsccwbl=UNKNOWN" +"tnsclw=UNKNOWN" +"rsdcs=UNKNOWN" +"tnscli=UNKNOWN" +"smc=UNKNOWN" +"tnsclihon=UNKNOWN" +"tntc=ZMDT+EVAPTZM+ZMMTT+CMFDT" +"tnta=UNKNOWN" +"rsd4co2=UNKNOWN" +"rld=UNKNOWN" +"tnscliemi=UNKNOWN" +"clwc=UNKNOWN" +"tntr=QRL+QRS" +"albisccp=MEANCLDALB_ISCCP" +"rlu=UNKNOWN" +"clws=UNKNOWN" +"tnsccw=UNKNOWN" +"tnsclwcd=UNKNOWN" +"tnsclwce=UNKNOWN" +"tnsclwcm=UNKNOWN" +"tnsclicm=UNKNOWN" +"tnsclicd=UNKNOWN" +"rldcs=UNKNOWN" +"rsut4co2=UNKNOWN" +"cltcalipso=CLDTOT_CAL no change" +"tnsclihenv=UNKNOWN" +"tnscliricl=UNKNOWN" +"tnsccwacs=UNKNOWN" +"tnsccwacr=UNKNOWN" +"ps=PS no change" +"tnsclimcl=UNKNOWN" +"tnscliif=UNKNOWN" +"tnsclwhen=UNKNOWN" +"clisccp=FISCCP1_COSP" +"tnsccwa=UNKNOWN" +"rsd=UNKNOWN" +"mcd=UNKNOWN" +"rldcs4co2=UNKNOWN" +"tntmp=PTTEND on model levels, no change" +"hur=RELHUM on model levels" +"hus=Q on model levels" +"rsu=UNKNOWN" +"mcu=UNKNOWN" +"tnt=TTEND_TOT on model levels, no change" +"tnsccwce=UNKNOWN" +"dmc=UNKNOWN" +"tnsccwcm=UNKNOWN" +"tnsclias=UNKNOWN" +"tnscliag=UNKNOWN" +"cls=UNKNOWN" +"clmcalipso=CLDMED_CAL no change" +"rsucs4co2=UNKNOWN" +"clis=UNKNOWN" +"pctisccp=MEANPTOP_ISCCP" +"tnsclwbfpcli=UNKNOWN" +"rlu4co2=UNKNOWN" +"clic=UNKNOWN" +"rld4co2=UNKNOWN" +"rsutcs4co2=UNKNOWN" +"tnsclwa=UNKNOWN" +"tnhusd=VD01 on model levels" +"tnhusa=TAQ on model levels, no change" +"tnhusc=ZMDQ+EVAPQZM+CMFDQ" +"tnsccwif=UNKNOWN" +"cltisccp=CLDTOT_ISCCP" +"rlut4co2=UNKNOWN" +"evisct=UNKNOWN" +"eviscu=UNKNOWN" +"clcalipso=CLD_CAL no change" +"tnsclwbl=UNKNOWN" +"tnhusscpbl=EVAPPREC+DQSED-CME" +"clhcalipso=CLDHGH_CAL no change" +"cllcalipso=CLDLOW_CAL no change" diff --git a/examples/CESM/CMIP6/CMIP6_cfOff.json b/examples/CESM/CMIP6/CMIP6_cfOff.json new file mode 100644 index 00000000..168b5219 --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_cfOff.json @@ -0,0 +1,9 @@ +"cltcalipso=CLDTOT_CAL no change" +"clcalipso2=no change" +"cfadDbze94=CFAD_DBZE94_CS no change" +"parasolRefl=RFL_PARASOL no change" +"clcalipso=CLD_CAL no change" +"clmcalipso=CLDMED_CAL no change" +"clhcalipso=CLDHGH_CAL no change" +"cfadLidarsr532=CFAD_SR532_CAL no change" +"cllcalipso=UNKNOWN" diff --git a/examples/CESM/CMIP6/CMIP6_cfSites.json b/examples/CESM/CMIP6/CMIP6_cfSites.json new file mode 100644 index 00000000..d85b92e7 --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_cfSites.json @@ -0,0 +1,80 @@ +"va=V on model levels" +"ci=FREQZM unit conversion" +"sci=FREQSH no change" +"hur=RELHUM on model levels" +"prc=PRECC unit conversion from m s-1 to kg m-2 s-1" +"tas=TREFHT no change" +"rsuscs=FSDSC - FSNSC" +"rsdscs=FSDSC no change" +"zg=Z3 on model levels" +"hfss=SHFLX no change" +"tntscpbl=UNKNOWN" +"rlutcs=FLUTC no change" +"tnhusmp=UNKNOWN" +"tnta=UNKNOWN" +"prsn=PRECSC + PRECSL and unit conversion from m s-1 to kg m-2 s-1" +"edt=UNKNOWN" +"rldscs=FLDSC no change" +"tnhus=UNKNOWN" +"pr=PRECT unit conversion from m s-1 to kg m-2 s-1" ???? Code has PRECC + PRECL, XWalk just lists PRECT +"rlut=FSNTOA-FSNT+FLNT" ???? +"cli=CLDICE on model levels, no change" +"rlus=FLDS + FLNS" +"rtmt=FSNT - FLNT" +"rlds=FLDS no change" +"tntc=UNKNOWN" +"cl=CLOUD on model levels, no change" +"fco2nat=UNKNOWN" +"ts=TS no change" +"rsdcs=UNKNOWN" +"rsd=UNKNOWN" +"tnhusd=UNKNOWN" +"latitude= MISSING" +"vas=UNAVAILABLE" +"clw=CLDLIQ on model levels, no change" +"tnhusa=UNKNOWN" +"uas=UNAVAILABLE" +"ta=T on model levels, no change" +"orog= MISSING" +"clivi=TGCLDIWP no change" +"huss=QREFHT no change" +"tnhusc=UNKNOWN" +"phalf= MISSING" +"ps=PS no change" +"hurs=RHREFHT no change" +"rld=UNKNOWN" +"fco2fos=UNKNOWN" +"tntr=UNKNOWN" +"clt=CLDTOT unit conversion" +"pfull= MISSING" +"fco2antt=UNKNOWN" +"tntmp=UNKNOWN" +"rsds=FSDS no change" +"hus=Q on model levels" +"rlu=UNKNOWN" +"rsu=UNKNOWN" +"sbl=UNAVAILABLE" +"rlucs=UNKNOWN" +"tnt=TTEND_TOT on model levels, no change" +"rsutcs=SOLIN - FSNTOAC" +"mc=CMFMC + CMFMCDZM on model levels, no change" ???? +"rsdt=SOLIN no change" +"psl=PSL no change" +"cct=PCLDTOP" +"wap=OMEGA on model levels" +"rldcs=UNKNOWN" +"prw=TMQ no change" +"longitude= MISSING" +"sfcWind=UNAVAILABLE" +"rsucs=UNKNOWN" +"evspsbl=QFLX no change" +"tauv=TAUY no change" +"clwvi=TGCLDLWP no change" +"tauu=TAUX no change" +"tnhusscpbl=UNKNOWN" +"rsus=FSDS - FSNS" +"ccb=PCLDBOT" +"ua=U on model levels" +"evu=UNKNOWN" +"rsut=SOLIN - FSNTOA" +"hfls=LHFLX no change" diff --git a/examples/CESM/CMIP6/CMIP6_day.json b/examples/CESM/CMIP6/CMIP6_day.json new file mode 100644 index 00000000..21aaf349 --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_day.json @@ -0,0 +1,42 @@ +"va=V interpolated to standard plevs" +"hur=RELHUM interpolated to standard plevs" +"prc=PRECC unit conversion from m s-1 to kg m-2 s-1" +"tas=TREFHT no change" +"rhsmin=UNAVAILABLE" +"uas=UNAVAILABLE" +"zg=Z3 interpolated to standard plevs" +"hfss=SHFLX no change" +"tossq=SST2, units from C to K" ????? +"prsn=PRECSC + PRECSL and unit conversion from m s-1 to kg m-2 s-1" +"vsi=vvel_d" ????? +"tslsi=UNKNOWN" +"pr=PRECC + PRECL and unit conversion from m s-1 to kg m-2 s-1" +"rlut=FSNTOA-FSNT+FLNT" ???? +"rlus=FLDS + FLNS" +"sit=hi_d no change" ????? +"tos=SST, units from C to K" ????? +"usi=uvel_d" ????? +"sic=aice_d no change" ????? +"mrsos=SOILWATER_10CM" +"clt=CLDTOT unit conversion" +"vas=UNAVAILABLE" +"wap=OMEGA interpolated to standard plevs" +"huss=QREFHT no change" +"snc=FSNO, mulitply by 100 to get percentage" +"psl=PSL no change" +"rhs=RHREFHT" +"rhsmax=UNAVAILABLE" +"snw=SNOWDP, convert from m to kg m-2 by multiplying by 1000" +"rsds=FSDS no change" +"hus=Q interpolated to standard plevs" +"omldamax=UNKNOWN" +"sfcWindmax=UNAVAILABLE" +"mrro=QRUNOFF" +"sfcWind=UNAVAILABLE" +"tasmin=TREFHTMN no change" +"rlds=FLDS no change" +"tasmax=TREFHTMX no change" +"rsus=FSDS - FSNS" +"ta=T interpolated to standard plevs" +"ua=U interpolated to standard plevs" +"hfls=LHFLX no change" diff --git a/examples/CESM/CMIP6/CMIP6_fx.json b/examples/CESM/CMIP6/CMIP6_fx.json new file mode 100644 index 00000000..41294c34 --- /dev/null +++ b/examples/CESM/CMIP6/CMIP6_fx.json @@ -0,0 +1,13 @@ +"thkcello=dz at each gridcell" +"sftlf=LANDFRAC no change" +"rootd=UNKNOWN" +"areacello=TAREA convert cm2 to m2" +"deptho=HT convert cm to m" +"volcello=TAREA times dz, convert cm3 to m3" +"mrsofc=WATSAT" +"orog=Divide PHIS by g where LANDFRAC ne zero" (where (landfrac .ne. 0)->phis/shr_const_g, shr_const_g = 9.80616) +"sftgif=PCT_GLACIER" +"hfgeou=Always set to zero in POP namelist" +"areacella=AREA" +"basin=REGION_MASK" +"sftof=0 where KMT = 0, 100 elsewhere" diff --git a/examples/CESM/CMIP6/src/mip_table_parser.py b/examples/CESM/CMIP6/src/mip_table_parser.py new file mode 100644 index 00000000..29671577 --- /dev/null +++ b/examples/CESM/CMIP6/src/mip_table_parser.py @@ -0,0 +1,588 @@ +import sys + +#============================================= +# Parse and Standardize the MIP table. +# Return a dcitionary that contains the parsed +# information. +#============================================= +def mip_table_parser(table_name,type=None,user_vars={},user_axes={},user_tableInfo={}): + """ + Function will parse three different MIP table formats: tab deliminated, CMOR, and XML. + After the table has been parsed, the information is standardized into the three dictionaries + and combined into one final dictionary. The three dictionaries are keyed with 'variables', + 'axes', and 'table_info'. The standardization methods found in the three dictionaries + resemble the identifiers used in the CMOR code. Variables are indexed by their string + id/label/variable_entry name. + Usage examples - table_dict = mip_table_parser('6hrLev','xml') + table_dict = mip_table_parser('Tables/CMIP5_Amon','cmor') + table_dict = mip_table_parser('Tables/CMIP6_MIP_tables.txt','excel') + + Parameters: + table_name (str): Full path to a table. Or in the case of XML, the experiment name. + type (str): One of 3 keys that identify table type: 'excel', 'cmor', or 'xml'. + user_var (dict): Allows users to add another synonym for one of the standard variable field keys. + user_axes (dict): Allows users to add another synonym for one of the standard axes field keys. + user_tableInfo (dict): Allows users to add another synonym for one of the standard table field keys. + + Returns: + table_dict (dict): A dictionary that holds all of the parsed table information. Contains the keys: + 'variables', 'axes', and 'table_info'. Each of these are dictionaries, keyed with variable names and + each variable has a value of a dictionary keyed with the standard field names. + """ + # Standardized identifiers for variable fields + # Format: standardName: [synonym list] + table_var_fields = { + 'cell_measures': ['cell_measures'], + 'cell_methods': ['cell_methods'], + 'cf_standard_name': ['CF Standard Name','CF_Standard_Name','cf_standard_name','standard_name'], + 'comment': ['comment'], + 'deflate': ['deflate'], + 'deflate_level': ['deflate_level'], + 'description': ['description'], + 'dimensions': ['dimensions'], + 'ext_cell_measures': ['ext_cell_measures'], + 'flag_meanings': ['flag_meanings'], + 'flag_values': ['flag_values'], + 'frequency': ['frequency'], + 'id': ['id','label','variable_entry'], + 'long_name': ['long_name'], + 'modeling_realm': ['modeling_realm'], + 'ok_min_mean_abs': ['ok_min_mean_abs'], + 'ok_max_mean_abs': ['ok_max_mean_abs'], + 'out_name': ['out_name'], + 'positive': ['positive'], + 'prov': ['prov'], + 'provNote': ['provNote'], + 'required': ['required'], + 'shuffle': ['shuffle'], + 'title': ['title'], + 'type': ['type'], + 'units': ['units'], + 'valid_max': ['valid_max'], + 'valid_min': ['valid_min'] + } + + # Standardized identifiers for axes fields + # Format: standardName: [synonym list] + table_axes_fields = { + 'axis': ['axis'], + 'bounds_values': ['bounds_values'], + 'climatology': ['climatology'], + 'convert_to': ['convert_to'], + 'coords_attrib': ['coords_attrib'], + 'formula': ['formula'], + 'id': ['id'], + 'index_only': ['index_only'], + 'long_name': ['long_name'], + 'must_call_cmor_grid': ['must_call_cmor_grid'], + 'must_have_bounds': ['must_have_bounds'], + 'out_name': ['out_name'], + 'positive': ['positive'], + 'requested': ['requested'], + 'requested_bounds': ['bounds_requested','requested_bounds'], + 'required': ['required'], + 'standard_name': ['standard_name'], + 'stored_direction': ['stored_direction'], + 'tolerance': ['tolerance'], + 'tol_on_requests': ['tol_on_requests'], + 'type': ['type'], + 'units': ['units'], + 'valid_max': ['valid_max'], + 'valid_min': ['valid_min'], + 'value': ['value'], + 'z_bounds_factors': ['z_bounds_factors'], + 'z_factors': ['z_factors'] + } + + # Standardized identifiers for table information fields + # Format: standardName: [synonym list] + table_fields = { + 'approx_interval': ['approx_interval'], + 'approx_interval_error': ['approx_interval_error'], + 'approx_interval_warning': ['approx_interval_warning'], + 'baseURL': ['baseURL'], + 'cf_version': ['cf_version'], + 'cmor_version': ['cmor_version'], + 'expt_id_ok': ['expt_id_ok'], + 'forcings': ['forcings'], + 'frequency': ['frequency'], + 'generic_levels': ['generic_levels'], + 'magic_number': ['magic_number'], + 'missing_value': ['missing_value'], + 'modeling_realm': ['modeling_realm'], + 'product': ['product'], + 'project_id': ['project_id'], + 'required_global_attributes': ['required_global_attributes'], + 'table_date': ['table_date'], + 'table_id': ['table_id'], + 'tracking_prefix': ['tracking_prefix'] + } + + # Set the type of table and return a dictionary that contains the read information. + if type == 'excel': + p = ParseExcel() + elif type == 'cmor': + p = ParseCmorTable() + elif type == 'xml': + p = ParseXML() + return p.parse_table(table_name,table_var_fields,table_axes_fields,table_fields, + user_vars,user_axes,user_tableInfo) + +#============================================= +# Find the standardization key to use +#============================================= +def _get_key(key, table, user_table): + """ + Find the standardization key to use. + + Parameters: + key (str): Supplied field name to match to a standard key + table (dict): One of the three standardized field dictionaries. + user_table (dict): A user created dictionary. Keys should match a key already in + the table argument, values are the new field names used in the table. + + Returns: + k (str): The key to use that matches the standardization. Program will exit if no matching + key is found. + """ + for k,v in table.iteritems(): + if key in v: # Search for field name in standard dictionary. + return k + elif k in user_table.keys(): # Search for field name in user created dictionary. + if key in user_table[k]: + return k + # field name is not recognized. Exit the program with an error. + print('Error: ',key,' is not a valid field name at this time. Please define and resubmit.') + sys.exit(1) + + +#============================================= +# Parse Excel Spreadsheets that have been +# reformatted to tab deliminated fields. +#============================================= +class ParseExcel(object): + + def __init__(self): + super(ParseExcel, self).__init__() + + #============================================= + # Parse the tab deliminated table file + #============================================= + def parse_table(self, table_name,table_var_fields,table_axes_fields,table_fields, + user_vars,user_axes,user_tableInfo): + """ + Function will parse a tab deliminated file and return a dictionary containing + the parsed fields. + + Parameter: + table_name (str): The full path to the table to be parsed. + table_var_fields (dict): Dictionary containing standardized field var names + as keys and acceptable synonyms as values. + table_axes_fields (dict): Dictionary containing standardized field axes names + as keys and acceptable synonyms as values. + table_fields (dict): Dictionary containing standardized table field names + as keys and acceptable synonyms as values. + user_vars (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + user_axes (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + user_tableInfo (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + + Returns: + table_dict (dict): A dictionary that holds all of the parsed table information. Contains the keys: + 'variables', 'axes', and 'table_info'. Each of these are dictionaries, keyed with variable names and + each variable has a value of a dictionary keyed with the standard field names. + """ + import csv + + table_dict = {} + + output_var_keys = table_var_fields['id'] + variables = {} + key = None + longest = 0 + found = False + fieldnames = [] + + # First time reading the file: Read the file and determine the longest line (first is recorded). + # This line is assumed to contain the field names. + f = open(table_name, 'rU') + reader = (f.read().splitlines()) + for row in reader: + r = row.split('\t') + i = 0 + l = len(r) + for g in r: + if g == '': + l = l-1 + i = i+1 + if l > longest: + fieldnames = list(r) + longest = l + + # Second time reading the file: Parse the file into a dictionary that uses the field + # names found in the first read as the keys. + reader = csv.DictReader(open(table_name, 'rU'), delimiter='\t',fieldnames=fieldnames) + + # Standardize the field names and store in a final dictionary. + for row in reader: + fields = {} + for k in row.keys(): + if k in output_var_keys: + key = row[k] + for k,v in row.iteritems(): + if v != '': + var_key = _get_key(k, table_var_fields, user_vars) + fields[var_key] = v + + variables[key] = fields + + table_dict['variables'] = variables + + return table_dict + + +#============================================= +# Parses standard CMOR table files +#============================================= +class ParseCmorTable(object): + + def _init_(): + super(ParseCmorTable, self).__init__() + + #============================================= + # Reset and get ready for a new variable set + #============================================= + def _reset(self, status, part, whole, name): + + whole[name] = part + part = {} + status = False + return status, part, whole + + #============================================= + # Setup for the new entry group found + #============================================= + def _new_entry(self, status, part, whole, name, key, value): + + if len(part.keys()) > 0: + whole[name] = part + part = {} + name = value + part['id'] = value + status = True + return status, part, whole, name + + #============================================= + # Parse a CMOR text file + #============================================= + def parse_table(self, table_name,table_var_fields,table_axes_fields,table_fields, + user_vars,user_axes,user_tableInfo): + """ + Function will parse a CMOR table text file and return a dictionary containing + the parsed fields. + + Parameter: + table_name (str): The full path to the table to be parsed. + table_var_fields (dict): Dictionary containing standardized field var names + as keys and acceptable synonyms as values. + table_axes_fields (dict): Dictionary containing standardized field axes names + as keys and acceptable synonyms as values. + table_fields (dict): Dictionary containing standardized table field names + as keys and acceptable synonyms as values. + user_vars (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + user_axes (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + user_tableInfo (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + + Returns: + table_dict (dict): A dictionary that holds all of the parsed table information. Contains the keys: + 'variables', 'axes', and 'table_info'. Each of these are dictionaries, keyed with variable names and + each variable has a value of a dictionary keyed with the standard field names. + """ + + # Initialize needed dictionaries + table_dict = {} + table_info = {} + expt_id_ok = {} + axes = {} + variables = {} + subroutines = {} + mapping = {} + axis = {} + var = {} + sub = {} + map = {} + + # Status Variables + current_axis = None + current_var = None + current_sub = None + current_mapping = None + in_axis = False + in_var = False + in_sub = False + in_mapping = False + + # Open table file + mip_file = open(table_name) + + for l in mip_file: + # if comment - don't proceed + #print l + l_no_comment = l.split('!') + l = l_no_comment[0].strip() + if len(l) > 0 and ':' in l: + # remove anything after a comment character + # parse the key from the value + parts = l.split(':') + if len(parts) > 2: + parts[1] = ''.join(parts[1:]) + key = parts[0].strip() + value = parts[1].strip() + #print l,'->',parts + + # add to table_info dictionary + # Start an entry for 'expt_id_ok' + if 'expt_id_ok' in key: + equiv = value.split("\' \'") + if len(equiv) == 2: + expt_id_ok[equiv[0]] = equiv[1] + elif len(equiv) == 1: + expt_id_ok[equiv[0]] = None + # Start an entry for 'axis_entry' + elif 'axis_entry' in key: + if in_var == True: + in_var,var,variables = self.reset(in_var,var,variables,current_var) + if in_sub == True: + in_sub,sub,subroutines = self._reset(in_sub,sub,subroutines,current_sub) + if in_mapping == True: + in_mapping,map,maping = self._reset(in_mapping,map,mapping,current_mapping) + in_axis,axis,axes,current_axis = self._new_entry(in_axis,axis,axes,current_axis,key,value) + # Start an entry for 'variable_entry' + elif 'variable_entry' in key: + if in_axis == True: + in_axis,axis,axes = self._reset(in_axis,axis,axes,current_axis) + if in_sub == True: + in_sub,sub,subroutines = self._reset(in_sub,sub,subroutines,current_sub) + if in_mapping == True: + in_mapping,map,maping = self._reset(in_mapping,map,mapping,current_mapping) + in_var,var,variables,current_var = self._new_entry(in_var,var,variables,current_var,key,value) + # Start an entry for 'subroutine_entry' + elif 'subroutine_entry' in key: + if in_axis == True: + in_axis,axis,axes = self._reset(in_axis,axis,axes,current_axis) + if in_var == True: + in_var,var,variables = self._reset(in_var,var,variables,current_var) + if in_mapping == True: + in_mapping,map,maping = self._reset(in_mapping,map,mapping,current_mapping) + in_sub,sub,subroutines,current_sub = self._new_entry(in_sub,sub,subroutines,current_sub,key,value) + # Start an entry for 'mapping_entry' + elif 'mapping_entry' in key: + if in_axis == True: + in_axis,axis,axes = self._reset(in_axis,axis,axes,current_axis) + if in_var == True: + in_var,var,variables = self._reset(in_var,var,variables,current_var) + if in_sub == True: + in_sub,sub,subroutines = self._reset(in_sub,sub,subroutines,current_sub) + in_mapping,map,maping,current_mapping = self._new_entry(in_mapping,map,mapping,current_mapping,key,value) + # The new entry has been started. If this point has been reached, parse this line into the correct standardized + # field name under the current activated entry. + else: + if (in_axis): #field added to axes variable + axis_key = _get_key(key, table_axes_fields, user_axes) + axis[axis_key] = value + elif (in_var): #field added to variable + var_key = _get_key(key, table_var_fields, user_vars) + var[var_key] = value + elif (in_sub): #field added to subroutine + sub[key] = value + elif (in_mapping): #field added to mapping + map[key] = value + else: #field added to table information + mip_key = _get_key(key, table_fields, user_tableInfo) + table_info[mip_key] = value + + # Add final entry into its group dictionary + if in_var == True: + variables[current_var] = var + if in_axis == True: + axes[current_axis] = axis + if in_sub == True: + subroutines[current_sub] = sub + if in_mapping == True: + mapping[current_mapping] = map + + # Combine three separate dictionaries into the table summary dictionary + table_dict['variables'] = variables + table_dict['axes'] = axes + table_dict['subroutines'] = subroutines + table_dict['table_info'] = table_info + + return table_dict + +#============================================= +# Parse the XML format +#============================================= +class ParseXML(object): + + def _init_(): + super(ParseXML, self, table_name,table_var_fields,table_axes_fields,table_fields).__init__() + + #============================================= + # Parse the XML format using dreqPy + #============================================= + def parse_table(self, table_name,table_var_fields,table_axes_fields,table_fields, + user_vars,user_axes,user_tableInfo): + """ + Function will parse an XML file using dreqPy and return a dictionary containing + the parsed fields. + + Parameter: + table_name (str): The full path to the table to be parsed. + table_var_fields (dict): Dictionary containing standardized field var names + as keys and acceptable synonyms as values. + table_axes_fields (dict): Dictionary containing standardized field axes names + as keys and acceptable synonyms as values. + table_fields (dict): Dictionary containing standardized table field names + as keys and acceptable synonyms as values. + user_vars (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + user_axes (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + user_tableInfo (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + + Returns: + table_dict (dict): A dictionary that holds all of the parsed table information. Contains the keys: + 'variables', 'axes', and 'table_info'. Each of these are dictionaries, keyed with variable names and + each variable has a value of a dictionary keyed with the standard field names. + """ + import dreq + + table_dict = {} + + variables = {} + axes = {} + table_info = {} + + dq = dreq.loadDreq() + + # Get table id + g_id = dq.inx.requestVarGroup.label[table_name] + + # Get the id's of the variables in this table + g_vars = dq.inx.iref_by_sect[g_id[0]].a + + # Loop through the variables and set their values + for v in g_vars['requestVar']: + var = {} + v_id = dq.inx.uid[v].vid # Get the CMORvar id + c_var = dq.inx.uid[v_id] + + # Set what we can from the CMORvar section + #var['comment']= c_var.comment + var['deflate']= c_var.deflate + var['deflate_level']= c_var.deflate_level + var['description']= c_var.description + #var['flag_meanings']= c_var.flag_meanings + #var['flag_values']= c_var.flag_values + var['frequency']= c_var.frequency + var['id']= c_var.label + var['modeling_realm']= c_var.modeling_realm + var['ok_min_mean_abs']= c_var.ok_min_mean_abs + var['ok_max_mean_abs']= c_var.ok_max_mean_abs + var['out_name']= c_var.label #? + var['positive']= c_var.positive + var['prov']= c_var.prov + var['provNote']= c_var.provNote + var['shuffle']= c_var.shuffle + var['title']= c_var.title + var['type']= c_var.type + var['valid_max']= c_var.valid_max + var['valid_min']= c_var.valid_min + + # Set what we can from the standard section + s_var = dq.inx.uid[c_var.stid] + var['cell_measures']= s_var.cell_measures + var['cell_methods']= s_var.cell_methods + + # Set what we can from the sp section + sp_var = dq.inx.uid[s_var.spid] + var['dimensions']= sp_var.dimensions + + # Set what we can from the variable section + v_var = dq.inx.uid[c_var.vid] + var['cf_standard_name']= v_var.sn + var['long_name']= v_var.sn + var['units']= v_var.units + + #var['ext_cell_measures']= + #var['required']= + + # Add variable to variable dictionary + variables[c_var.label] = var + + table_dict['variables'] = variables + + return table_dict + +# Generate CESM to CMIP5 Mapping json files +def xwalk(xWalk_fn): + + mip_table = {} + + j_fn = os.path.basename(xWalk_fn) + j_fn = j_fn.replace('xwalk_','') + json_f = open(j_fn+'.json', 'wb') + + xWalk_file = open(xWalk_fn) + for e_line in xWalk_file: + # initialize entry (line in table), split the line up, store values into entry + e_line = e_line.strip() + line = e_line.split(':') + vn = line[1] + formula = line[-1] + + json_f.write("{}\n".format(json.dumps(vn+'='+formula))) + + +# Open CMIP5 json files and compare it to CMIP6 parsed variables. +# For all CMIP6 needed variables for that table, find definition +# within the CMIP5 mapping files. If not found (new variable within +# CMIP), mark as MISSING. +def json_parser(fn, mip_dict, t): + + new_dict = {} + # Open table file + f = open(fn) + out = open('CMIP6_'+t+'.json','wb') + + # Create a new dictionary with variable names as keys from the CMIP5 json file + for l in f: + p1 = l.split('=') + p2 = p1[0].replace('\"','') + if p2 in mip_dict['variables'].keys(): + new_dict[p2] = l.rstrip() + else: + print p2,' in CMIP5, but not in CMIP6' + # Loop through CMIP6 var to find definition on how to the var was derived in CMIP5 + # New vars are marked as 'MISSING' + for v in mip_dict['variables'].keys(): + if v in new_dict.keys(): + out.write(new_dict[v]+'\n') + else: + out.write('\"'+v+'= MISSING\"\n') + + +import glob,json,os + +# Look through all tables and create new CESM to CMIP6 mapping files +tables = ['3hr','6hrLev','6hrPlev','aero','Amon','cf3hr','cfDay','cfMon','cfOff','cfSites','day','fx','LImon','Lmon','Oclim','OImon','Omon','Oyr'] +for t in tables: + print t + xml_table_dict = mip_table_parser(t,'xml') + json_parser('../../CMIP5/CMIP5_'+t+'.json', xml_table_dict,t) diff --git a/scripts/conform b/scripts/conform new file mode 100755 index 00000000..a9a6e1c5 --- /dev/null +++ b/scripts/conform @@ -0,0 +1,82 @@ +#!/usr/bin/env python +""" +PyConform - Command-Line Interface + +This is the command-line interface to the PyConform tool. It takes input from +the command-line directly, and from an "output specification" file, which +defines the output dataset entirely. + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from os.path import exists +from glob import glob +from json import load as json_load +from collections import OrderedDict +from argparse import ArgumentParser + +from pyconform.datasets import InputDataset, OutputDataset +from pyconform.conform import setup, run + + +#============================================================================== +# Command-line Interface +#============================================================================== +def cli(argv=None): + desc = """This tool is designed to run the PyConform command-line tool, + taking input from the command-line and a predefined output + specification file (specfile).""" + + parser = ArgumentParser(description=desc) + parser.add_argument('-i', '--infile', dest='infiles', default=[], + metavar='INFILE', action='append', type=str, + help='Input file path or globstring specifying input ' + 'data for the PyConform operation. If no input ' + 'files are specified, then PyConform will validate ' + 'the output specification file only, and then ' + 'exit. [No default]') + parser.add_argument('-s', '--serial', default=False, + action='store_true', dest='serial', + help='Whether to run in serial (True) or parallel ' + '(False). [Default: False]') + parser.add_argument('-v', '--verbosity', default=1, type=int, + help='Verbosity level for level of output. A value ' + 'of 0 means no output, and a value greater than ' + '0 means more output detail. [Default: 1]') + parser.add_argument('specfile', default=None, metavar='SPECFILE', type=str, + help='JSON-formatted output specification file ' + '[REQUIRED]') + + return parser.parse_args(argv) + + +#============================================================================== +# Main Script Function +#============================================================================== +def main(argv=None): + args = cli(argv) + + infiles = [] + for infile in args.infiles: + infiles.extend(glob(infile)) + + if not exists(args.specfile): + raise OSError(('Output specification file {!r} not ' + 'found').format(args.specfile)) + dsdict = json_load(open(args.specfile, 'r'), object_pairs_hook=OrderedDict) + + inpds = InputDataset(filenames=infiles) + outds = OutputDataset(dsdict=dsdict) + + # Setup the PyConform problem + agraph = setup(inpds, outds) + + # Run PyConform + run(inpds, outds, agraph) + +#============================================================================== +# Command-line Operation +#============================================================================== +if __name__ == '__main__': + main() diff --git a/scripts/createOutputSpecs b/scripts/createOutputSpecs new file mode 100755 index 00000000..74dd2619 --- /dev/null +++ b/scripts/createOutputSpecs @@ -0,0 +1,236 @@ +#! /usr/bin/env python +""" +createOutputSpecs + +Creates the Json output specification file. + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +import argparse, os +import json +import mip_table_parser +from dateutil.parser import parse +import datetime + +data_types = {'char': 'char', 'byte': 'int8', 'short': 'int16', 'int': 'int32', + 'float': 'float32', 'real': 'float32', 'double': 'float64', 'integer':'int32'} + +def parseArgs(argv = None): + + desc = "This tool creates a specification file that is needed to run PyConform." + + parser = argparse.ArgumentParser(prog='createOutputSpecs', + description=desc) + parser.add_argument('-d', '--defFile', default=None, type=str, + help='A file listing the variable definitions.', required=True) + parser.add_argument('-g', '--globalAttrFile', default=None, type=str, + help='A file listing the global attributes that ' + 'are common to all files.') + parser.add_argument('-e', '--exp', default=None, type=str, + help='The name of the experiment.') + parser.add_argument('-m', '--mipTable', default=None, type=str, + help='The name of the MIP table.') + parser.add_argument('-tt', '--mipTableType', default=None, type=str, + help='MIP table file type. Can be xml, cmor, or excel.') + parser.add_argument('-u', '--userList', default=None, type=str, + help='A file containing cf-compliant names to derive.') + parser.add_argument('-o', '--outputpath', default=os.getcwd(), type=str, + help='Output pathname for the output specification file(s).') + + return parser.parse_args(argv) + +def load(defs): + + def_dict = {} + for line in defs: + split = line.split('=') + if (len(split) == 2): + def_dict[split[0].strip()] = split[1].strip() + else: + if line != '\n': + print 'Could not parse this line: ',line + return def_dict + +def create_output(table_dict, id, definitions, attributes, output_path, args): + + outSpec = {} + variables = {} + axes = {} + table_info = {} + + variables = table_dict['variables'] + axes = table_dict['axes'] + table_info = table_dict['table_info'] + attributes.update(table_info) + if 'generic_levels' in table_info.keys(): + g_levels = table_info['generic_levels'] + g_split = g_levels.split(' ') + for l in g_split: + axes[l] = {} + #print '\n\n',variables.keys(),'\n\n'#,axes,'\n',table_info + + # Get variables needed to piece together the filename + identifier = id + if ('model_id') in attributes.keys(): + model = attributes['model_id'] + else: + model = '' + experiment = args.exp + ripgf_list = ['realization_index','initialization_index','physics_index','forcing_index','grid_index'] + if all (ripgf in attributes for ripgf in ripgf_list): + ripfg = ("r{0}i{1}p{2}f{3}g{4}".format(attributes['realization_index'], + attributes['initialization_index'], + attributes['physics_index'], + attributes['forcing_index'], + attributes['grid_index'])) + else: + ripfg = '' + table = identifier + attributes['experiment_id'] = table + attributes['experiment'] = table + today = datetime.datetime.now() + version = 'v'+str(today.year).zfill(4)+str(today.month).zfill(2)+str(today.day).zfill(2) + + # Do we need to add anything to further_info_url? + if 'further_info_url' in attributes.keys(): + fiu_split = str(attributes['further_info_url']).split('/') + if table not in fiu_split: + attributes['further_info_url'] = str(attributes['further_info_url'])+'/'+table + if experiment not in fiu_split: + attributes['further_info_url'] = str(attributes['further_info_url'])+'/'+experiment + + # Create Output Spec + # First add global attributes that are common to all files + outSpec["attributes"] = attributes + + var_list = {} + + # For each variable in the definition file, create a file entry in the spec and define it + for v,d in variables.iteritems(): + if v in definitions.keys(): + tper = variables[v]['frequency'] + component = variables[v]['modeling_realm'] + if ' ' in component: + component = component.replace(' ','_') + f_name = ("{0}/{1}/{2}/{3}/{4}/{5}/{6}/{7}_{8}_{9}_{10}_{11}".format(experiment, + tper,component,table,ripfg,version,v, + v,table,model,experiment,ripfg)) + f_id = v + var = {} + for k1,v1 in variables[v].iteritems(): + if not isinstance(v1,(list,str,float)): + variables[v][k1] = "None" + var["attributes"] = variables[v] + var["definition"] = definitions[v] + var["filename"] = f_name + if 'type' in variables[v].keys() and variables[v]['type'] != 'None' and variables[v]['type'] != '': + var["datatype"] = data_types[variables[v]['type']] + else: + var["datatype"] = 'None' + if 'requested' in variables[v].keys(): + if variables[v]['requested'] != '': + var['data'] = variables[v]['requested'] + if 'dimensions' in variables[v].keys(): + var["dimensions"] = variables[v]['dimensions'].split('|') + var_list[f_id] = var + else: + print id, ':', v, ' is being requested by the experiment, but a definition has not been supplied.' + + # Add axes into the variable list + for v,d in axes.iteritems(): + f_id = v + var = {} + var["attributes"] = axes[v] + if 'type' in axes[v].keys(): + var["datatype"] = data_types[axes[v]['type']] + if 'requested' in axes[v].keys(): + if axes[v]['requested'] != '': + var['data'] = axes[v]['requested'] + var_list[f_id] = var + + # Go through a user supplied list if specified + if args.userList and os.path.isfile(args.userList): + with open(args.userList) as f: + for vr in f: + vr = vr.strip() + if vr != "": + var = {} + f_id = vr + if vr in definitions.keys(): + var["attributes"] = {} + var["definition"] = definitions[vr] + f_name = ("{0}{1}".format(vr,fn_suffix)) + var["filename"] = f_name + var["datatype"] = "None" + var["dimensions"] = "None" + var_list[f_id] = var + else: + print vr, ' is being requested by the experiment, but a definition has not been supplied.' + else: + if args.userList and not os.path.isfile(args.userList): + print 'The User Variable List file does not exist: ',args.userList + os.sys.exit(1) + + outSpec["variables"] = var_list + + # Write the JSON output spec file + f = output_path + '/' + experiment + '_' + id + '_spec.json' + if not os.path.exists(output_path): + os.makedirs(output_path) + with open(f, 'w') as outfile: + json.dump(outSpec, outfile, sort_keys=True, indent=4) + + + +def main(argv=None): + + args = parseArgs(argv) + + print "\n" + print "------------------------------------------" + print 'Running createOutputSpecs with these args:\n' + print 'Variable Definitions: ', args.defFile + print 'Global Attributes to be added to each file: ', args.globalAttrFile + print 'Experiment Name: ', args.exp + print 'MIP Table to be used: ', args.mipTable + print 'MIP Table Type: ',args.mipTableType + print 'User supplied variable list: ',args.userList + print 'Will create output spec files within this directory:', args.outputpath + print "------------------------------------------" + + # Open/Read the definition file + if os.path.isfile(args.defFile): + with open(args.defFile) as y_definitions: + definitions = load(y_definitions) + #print 'DEFINITIONS: ',definitions + else: + print 'Definition file does not exist: ',args.defFile + os.sys.exit(1) + + # Open/Read the global attributes file + attributes = {} + if args.globalAttrFile and os.path.isfile(args.globalAttrFile): + with open(args.globalAttrFile) as y_attributes: + attributes = load(y_attributes) + #print 'GLOBAL ATTRIBUTES: ',attributes + else: + if args.globalAttrFile and not os.path.isfile(args.globalAttrFile): + print 'Global Attributes file does not exist: ',args.globalAttrFile + os.sys.exit(1) + + # Open/Read the MIP table +# if args.mipTable != None and args.mipTableType != None and args.exp != None: + if args.mipTableType != None and args.exp != None: + exp_dict = mip_table_parser.mip_table_parser(args.exp, args.mipTable,type=args.mipTableType) + + # Write the spec files out to disk + for t in exp_dict.keys(): + create_output(exp_dict[t], t, definitions, attributes, args.outputpath, args) + + for t in sorted(exp_dict.keys()): + print t, len(exp_dict[t]['variables']) + +if __name__ == '__main__': + main() diff --git a/scripts/requiredVars b/scripts/requiredVars new file mode 100755 index 00000000..d45008e1 --- /dev/null +++ b/scripts/requiredVars @@ -0,0 +1,168 @@ +#! /usr/bin/env python + +import os, argparse +import xlsxwriter +import dreq + +def parseArgs(argv = None): + + desc = "This tool creates a list of required variables." + + parser = argparse.ArgumentParser(prog='createOutputSpecs', + description=desc) + parser.add_argument('-d', '--defFile', default=None, type=str, + help='A file listing the variable definitions.', required=True) + parser.add_argument('outputfile', default='out.json', type=str, + help='Filename for the output variable lists.') + + return parser.parse_args(argv) + + +def load(defs): + + # Read in the user supplied definitions file + + def_dict = {} + for line in defs: + split = line.split('=') + if (len(split) == 2): + def_dict[split[0].strip()] = split[1].strip() + else: + if line != '\n': + print 'Could not parse this line: ',line + return def_dict + + +def write_xls(fn, total_request, definitions): + + # Create an excel workbook and setup some formatting + workbook = xlsxwriter.Workbook(fn) + header = workbook.add_format({'bold': True, 'font_size': 24}) + bold = workbook.add_format({'bold': True}) + format = workbook.add_format() + format.set_text_wrap() + format.set_border(1) + + # For each mip table, create a new worksheet and set up header information + for mt in sorted(total_request): + worksheet = workbook.add_worksheet(mt) + worksheet.set_column(0, 0, 20) + worksheet.set_column(1, 1, 60) + worksheet.set_column(2, 2, 15) + worksheet.set_column(3, 4, 40) + worksheet.set_column(5, 5, 80) + worksheet.write('A1', mt, header) + worksheet.write('B1', ' # of Variables: '+str(len(total_request[mt])), bold) + worksheet.write('A2', 'Variable', bold) + worksheet.write('B2', 'Title', bold) + worksheet.write('C2', 'Units', bold) + worksheet.write('D2', 'Dimensions', bold) + worksheet.write('E2', 'Requested By', bold) + worksheet.write('F2', 'Model->CMIP Definition', bold) + cr = 3 + # For each variable requested, add it's name, official title, + # and supply a def if we have one already + for v in sorted(total_request[mt]): + worksheet.write('A'+str(cr), v, format) + worksheet.write('B'+str(cr), total_request[mt][v]['description'], format) + worksheet.write('C'+str(cr), total_request[mt][v]['units'], format) + worksheet.write('D'+str(cr), total_request[mt][v]['dimensions'], format) + request = ' ,'.join(total_request[mt][v]['req_by']) + worksheet.write('E'+str(cr), request, format) + if v in definitions.keys(): + worksheet.write('F'+str(cr), definitions[v], format) + else: # We don't have a definition, create a border around a blank cell + worksheet.write('F'+str(cr), '', format) + cr = cr + 1 + + workbook.close() + + +def main(argv=None): + + dq = dreq.loadDreq() + total_request = {} + mips = [] + not_recognized = [] + + # Parse args + args = parseArgs(argv) + + # Open/Read the definition file + if os.path.isfile(args.defFile): + with open(args.defFile) as y_definitions: + definitions = load(y_definitions) + + for v in dq.coll['CMORvar'].items: + + if '-copy' not in v.label: + tab = v.mipTable + + if tab not in total_request.keys(): + total_request[tab] = {} + + if v.label not in total_request[tab].keys(): + total_request[tab][v.label] = {'req_by':[]} + + if v.prov not in total_request[tab][v.label]['req_by']: + total_request[tab][v.label]['req_by'].append(v.prov) + + v_id = v.uid + c_var = dq.inx.uid[v_id] + + if hasattr(c_var,'vid'): + v_var = dq.inx.uid[c_var.vid] + else: + print 'Not able to recognize v_var for ', v.label + v_var = None + if hasattr(c_var,'stid'): + s_var = dq.inx.uid[c_var.stid] + if hasattr(s_var,'spid'): + sp_var = dq.inx.uid[s_var.spid] + else: + sp_var = None + td = None + if hasattr(s_var,'tmid'): + t_var = dq.inx.uid[s_var.tmid] + if hasattr(t_var,'dimensions'): + t = t_var.dimensions + if t != '' and t != 'None': + td = t+'|' + else: + print 'Not able to recognize s_var for ', v + s_var = None + sp_var = None + total_request[tab][v.label]['description'] = c_var.title + if 'None' not in c_var.description and c_var.description != '': + total_request[tab][v.label]['description'] = total_request[tab][v.label]['description'] + ': ' + c_var.description + if hasattr(v_var,'units'): + total_request[tab][v.label]['units'] = v_var.units + if 'ch4global' in v.label: + print v_var.units + else: + total_request[tab][v.label]['units'] = 'No units listed' + if hasattr(sp_var, 'dimensions'): + if td != None: + total_request[tab][v.label]['dimensions'] = td + sp_var.dimensions + else: + total_request[tab][v.label]['dimensions'] = sp_var.dimensions + else: + total_request[tab][v.label]['dimensions'] = 'No dimensions listed' + + + # Write out the xls spreadsheet + write_xls(args.outputfile, total_request, definitions) + +# for mt in total_request.keys(): +# print '#############################################' +# print '\ntable:', mt, 'len: ',len(total_request[mt]) +# for v in total_request[mt].keys(): +# if v in definitions.keys(): +# d = definitions[v] +# else: +# d = '' +# print v,'|',total_request[mt][v],'|',d + + +if __name__ == '__main__': + main() diff --git a/source/pyconform/__init__.py b/source/pyconform/__init__.py index dd29a7ab..fe336f35 100644 --- a/source/pyconform/__init__.py +++ b/source/pyconform/__init__.py @@ -4,7 +4,7 @@ A package for conforming a NetCDF dataset for publication :AUTHORS: Sheri Mickelson, Kevin Paul -:COPYRIGHT: 2015, University Corporation for Atmospheric Research +:COPYRIGHT: 2016, University Corporation for Atmospheric Research :LICENSE: See the LICENSE.rst file for details Send questions and comments to Kevin Paul (kpaul@ucar.edu) or diff --git a/source/pyconform/actiongraphs.py b/source/pyconform/actiongraphs.py new file mode 100644 index 00000000..598aa369 --- /dev/null +++ b/source/pyconform/actiongraphs.py @@ -0,0 +1,442 @@ +""" +ActionGraph Class + +This module contains the ActionGraph class to be used with the Action +classes and the DiGraph class to build an "graph of actions." This graph +walks through a DiGraph of connected Actions performing the Action functions +in sequence, and using the output of each Action as input into the next. + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from pyconform.graphs import DiGraph +from pyconform.parsing import (ParsedFunction, ParsedVariable, ParsedUniOp, + ParsedBinOp, parse_definition) +from pyconform.datasets import InputDataset, OutputDataset +from pyconform.actions import Action, Reader, Evaluator, Finalizer +from pyconform.functions import (find_function, find, + UnitsError, DimensionsError) +from itertools import cycle +from os import linesep +from numpy import array + + +#=============================================================================== +# ActionGraph +#=============================================================================== +class ActionGraph(DiGraph): + """ + Action Graph + + A directed graph defining a connected set of operations whose results + are used as input to adjacent operators. + """ + + def __init__(self): + """ + Initialize + """ + super(ActionGraph, self).__init__() + + self._dim_map = {} + + @property + def dimension_map(self): + return self._dim_map + + def __str__(self): + """ + Display an ActionGraph as a string + """ + output = [] + sorted_handles = sorted((str(h), h) for h in self.handles()) + for _, h in sorted_handles: + nodes = self._actions_by_depth_(h) + houtput = [] + for i, u in nodes: + indent = ' ' * (i - 1) + if i > 0: + houtput.append('{0}|'.format(indent)) + houtput.append('{0}+-- {1!s}'.format(indent, u)) + else: + houtput.append('{1!s}'.format(indent, u)) + for i in xrange(len(houtput)): + for j in [j for j, c in enumerate(houtput[i]) if c == '|']: + k = i - 1 + while k > 0: + if houtput[k][j] == ' ': + houtput[k] = houtput[k][:j] + '|' + houtput[k][j + 1:] + k = k - 1 + else: + k = 0 + output.extend(houtput) + output.append('') + return linesep.join(output) + + def _actions_by_depth_(self, v, depth=0, visited=None): + if visited is None: + visited = set() + visited.add(v) + nodes = [(depth, v)] + for n in self.neighbors_to(v): + if n not in visited: + nodes.extend(self._actions_by_depth_(n, depth=depth + 1, + visited=visited)) + return nodes + + def add(self, vertex): + """ + Add a vertex to the graph + + Parameters: + vertex (Action): An Action vertex to be added to the graph + """ + if not isinstance(vertex, Action): + raise TypeError('ActionGraph must consist only of Actions') + super(ActionGraph, self).add(vertex) + + def __call__(self, root): + """ + Perform the ActionGraph operations + + Parameters: + root (Action): The root of the operation, from which data is + requested from the operation graph + """ + if root not in self: + raise KeyError('Action {!r} not in ActionGraph'.format(root)) + return root(*map(self.__call__, self.neighbors_to(root))) + + def handles(self): + """ + Return a list of all output variable handles in the graph + """ + return [op for op in self.vertices if isinstance(op, Finalizer)] + + def presets(self): + """ + Return a list of preset output variable handles in the graph + """ + return [op for op in self.vertices + if isinstance(op, Finalizer) and op.is_preset()] + + +#=============================================================================== +# GraphFiller +#=============================================================================== +class GraphFiller(object): + """ + Object that fills an ActionGraph + """ + + def __init__(self, inp, cyc=True): + """ + Initializer + + Parameters: + inp (InputDataset): The input dataset to use as reference when + parsing variable definitions + cyc (bool): If True, cycles which file to read metadata variables + from (i.e., variables that exist in all input files). If False, + always reads metadata variables from the first input file. + """ + # Input dataset + if not isinstance(inp, InputDataset): + raise TypeError('Input dataset must be of InputDataset type') + self._inputds = inp + + # Cyclic iterator over available input filenames + fnames = [v.filename for v in self._inputds.variables.itervalues()] + if cyc: + self._infile_cycle = cycle(filter(None, fnames)) + else: + self._infile_cycle = cycle([filter(None, fnames)[0]]) + + def from_definitions(self, graph, outds): + """ + Fill an ActionGraph from definitions in an output dataset + """ + # Action Graph + if not isinstance(graph, ActionGraph): + raise TypeError('Graph must be an ActionGraph object') + + # Output dataset + if not isinstance(outds, OutputDataset): + raise TypeError('Output dataset must be of OutputDataset type') + + # Separate output variables into preset and defined + preset = {} + defined = {} + for vname, vinfo in outds.variables.iteritems(): + if vinfo.data is not None: + preset[vname] = vinfo + elif vinfo.definition is not None: + defined[vname] = vinfo + else: + raise ValueError(('Output variable {0} does not have preset ' + 'data nor a definition').format(vname)) + + # Parse output variables with preset data + for vname, vinfo in preset.iteritems(): + vmin = vinfo.attributes.get('valid_min', None) + vmax = vinfo.attributes.get('valid_max', None) + vmin_ma = vinfo.attributes.get('ok_min_mean_abs', None) + vmax_ma = vinfo.attributes.get('ok_max_mean_abs', None) + vdata = array(vinfo.data, dtype=vinfo.datatype) + handle = Finalizer(vname, data=vdata, minimum=vmin, maximum=vmax, + min_mean_abs=vmin_ma, max_mean_abs=vmax_ma) + handle.units = vinfo.cfunits() + handle.dimensions = vinfo.dimensions + graph.add(handle) + + # Parse the output variable with definitions + for vname, vinfo in defined.iteritems(): + vmin = vinfo.attributes.get('valid_min', None) + vmax = vinfo.attributes.get('valid_max', None) + handle = Finalizer(vname, minimum=vmin, maximum=vmax) + handle.units = vinfo.cfunits() + handle.dimensions = vinfo.dimensions + + obj = parse_definition(vinfo.definition) + vtx = self._add_to_graph_(graph, obj) + graph.connect(vtx, handle) + + # Check to make sure the graph is not cyclic + if graph.is_cyclic(): + raise ValueError('Graph is cyclic. Cannot continue.') + + def _add_to_graph_(self, graph, obj): + vtx = self._convert_obj_(graph, obj) + graph.add(vtx) + if isinstance(obj.args, tuple): + for arg in obj.args: + if not isinstance(arg, (int, float)): + graph.connect(self._add_to_graph_(graph, arg), vtx) + return vtx + + def _convert_obj_(self, graph, obj): + if isinstance(obj, ParsedVariable): + vname = obj.key + if vname in self._inputds.variables: + var = self._inputds.variables[vname] + if var.filename: + fname = var.filename + else: + fname = self._infile_cycle.next() + return Reader(fname, vname, slicetuple=obj.args) + + else: + preset_dict = dict((f.key, f) for f in graph.presets()) + if vname in preset_dict: + return preset_dict[vname] + else: + raise KeyError('Variable {0!r} not found'.format(vname)) + + elif isinstance(obj, (ParsedUniOp, ParsedBinOp, ParsedFunction)): + name = obj.key + nargs = len(obj.args) + func = find(name, numargs=nargs) + args = [o if isinstance(o, (int, float)) else None + for o in obj.args] + if all(isinstance(o, (int, float)) for o in args): + return func(*args) + return Evaluator(name, str(obj), func, signature=args) + + else: + return obj + + def match_units(self, graph): + """ + Match units of connected Actions in an ActionGraph + + This will add new Actions to the ActionGraph, as necessary, to convert + units to match the necessary units needed by each Action. + + Parameters: + graph (ActionGraph): The ActionGraph in which to match units + """ + for handle in graph.handles(): + GraphFiller._compute_units_(graph, handle) + + @staticmethod + def _compute_units_(graph, vtx): + nbrs = graph.neighbors_to(vtx) + to_units = [GraphFiller._compute_units_(graph, nbr) for nbr in nbrs] + + if isinstance(vtx, Evaluator): + arg_units = [to_units.pop(0) if arg is None else arg + for arg in vtx.signature] + func = find(vtx.key, numargs=len(arg_units)) + ret_unit, new_units = func.units(*arg_units) + for i, new_unit in enumerate(new_units): + if new_unit is not None: + if vtx.signature[i] is not None: + raise UnitsError(('Argument {0} in action {1} requires ' + 'units {2}').format(i, vtx, new_unit)) + else: + old_unit = arg_units[i] + cvtx = GraphFiller._new_converter_(old_unit, new_unit) + nbr = nbrs[sum(map(lambda k: 1 if k is None else 0, + vtx.signature)[:i])] + cvtx.dimensions = nbr.dimensions + graph.insert(nbr, cvtx, vtx) + vtx.units = ret_unit + + elif isinstance(vtx, Finalizer): + if len(nbrs) == 0: + pass + elif len(nbrs) == 1: + nbr = nbrs[0] + old_unit = to_units[0] + if vtx.units != old_unit: + if old_unit.is_convertible(vtx.units): + cvtx = GraphFiller._new_converter_(old_unit, vtx.units) + cvtx.dimensions = vtx.dimensions + graph.insert(nbr, cvtx, vtx) + else: + if old_unit.calendar != vtx.units.calendar: + raise UnitsError(('Cannot convert {0} units to {1} ' + 'units. Calendars must be ' + 'same.').format(nbr, vtx)) + else: + raise UnitsError(('Cannot convert {0} units ' + '{1!r} to {2} units ' + '{3!r}').format(nbr, old_unit, + vtx, vtx.units)) + else: + raise ValueError(('Graph malformed. Finalizer with more than ' + 'one input edge {0}').format(vtx)) + + else: + if len(to_units) == 1: + vtx.units = to_units[0] + + return vtx.units + + @staticmethod + def _new_converter_(old_units, new_units): + name = 'convert({0!r}->{1!r})'.format(str(old_units), str(new_units)) + func = find_function('convert', 3) + action = Evaluator('convert', name, func, + signature=(None, old_units, new_units)) + action.units = new_units + return action + + def match_dimensions(self, graph): + """ + Match dimensions of connected Actions in an ActionGraph + + This will add new Actions to the ActionGraph, as necessary, to transpose + dimensions to match the necessary dimensions needed by each Action. + + Parameters: + graph (ActionGraph): The ActionGraph in which to match dimensions + + Returns: + dict: A dictionary of output dimension names mapped to their + corresponding input dimension names + """ + # Fill the graph with dimensions up to the OutputSliceHandles and + # compute the map of input dataset dimensions to output dataset dims + dmap = {} + handles = sorted(graph.handles(), key=lambda h: len(h.dimensions)) + for handle in handles: + GraphFiller._map_dimensions_(graph, handle, dmap) + + for handle in handles: + nbrs = graph.neighbors_to(handle) + if len(nbrs) == 0: + continue + nbr = nbrs[0] + for d in nbr.dimensions: + if d in dmap: + if (d not in self._inputds.dimensions and + d in handle.dimensions): + dmap.pop(d) + else: + if (d in self._inputds.dimensions or + d not in handle.dimensions): + unmapped_dims = tuple(d for d in nbr.dimensions + if d not in dmap) + raise DimensionsError(('Could not determine complete ' + 'dimension map for input dims ' + '{0}').format(unmapped_dims)) + mapped_dims = tuple(dmap[d] if d in dmap else d + for d in nbr.dimensions) + hdims = handle.dimensions + if hdims != mapped_dims: + if set(hdims) == set(mapped_dims): + tvtx = GraphFiller._new_transpositor_(mapped_dims, hdims) + tvtx.units = nbr.units + graph.insert(nbr, tvtx, handle) + + graph._dim_map = dict((v, k) for (k, v) in dmap.iteritems()) + + @staticmethod + def _map_dimensions_(graph, vtx, dmap={}): + nbrs = graph.neighbors_to(vtx) + nbrs_dims = [GraphFiller._map_dimensions_(graph, nbr, dmap) + for nbr in nbrs] + if isinstance(vtx, Evaluator): + arg_dims = [nbrs_dims.pop(0) if arg is None else arg + for arg in vtx.signature] + func = find(vtx.key, numargs=len(arg_dims)) + ret_dims, new_dims = func.dimensions(*arg_dims) + for i, new_dim in enumerate(new_dims): + if new_dim is not None: + if vtx.signature[i] is not None: + raise DimensionsError(('Argument {0} in action {1} ' + 'requires dimensions ' + '{2}').format(i, vtx, new_dim)) + else: + old_dim = arg_dims[i] + tvtx = GraphFiller._new_transpositor_(old_dim, new_dim) + nbr = nbrs[sum(map(lambda k: 1 if k is None else 0, + vtx.signature)[:i])] + tvtx.units = nbr.units + graph.insert(nbr, tvtx, vtx) + vtx.dimensions = ret_dims + + if isinstance(vtx, Finalizer): + if len(nbrs) == 0: + pass + elif len(nbrs) == 1: + nbr = nbrs[0] + nbr_dims = nbrs_dims[0] + if len(nbr_dims) != len(vtx.dimensions): + raise DimensionsError(('Action {0} with dimensions {1} ' + 'inconsistent with required output ' + 'variable {2} dimensions ' + '{3}').format(nbr, nbr_dims, vtx, + vtx.dimensions)) + else: + unmapped_nbr_dims = [d for d in nbr_dims if d not in dmap] + if len(unmapped_nbr_dims) > 1: + raise DimensionsError(('Cannot map input dimensions ' + '{0} to output ' + 'dimension').format(unmapped_nbr_dims)) + elif len(unmapped_nbr_dims) == 1: + unmapped_nbr_dim = unmapped_nbr_dims[0] + unmapped_vtx_dims = [d for d in vtx.dimensions + if d not in dmap.values()] + unmapped_vtx_dim = unmapped_vtx_dims[0] + dmap[unmapped_nbr_dim] = unmapped_vtx_dim + else: + raise ValueError(('Graph malformed. Finalizer with more than ' + 'one input edge {0}').format(vtx)) + + else: + if len(nbrs_dims) == 1: + vtx.dimensions = nbrs_dims[0] + + return vtx.dimensions + + @staticmethod + def _new_transpositor_(old_dims, new_dims): + new_order = [new_dims.index(d) for d in old_dims] + name = 'transpose({0}->{1})'.format(old_dims, new_dims) + func = find_function('transpose', 2) + action = Evaluator('transpose', name, func, signature=(None, new_order)) + action.dimensions = new_dims + return action diff --git a/source/pyconform/actions.py b/source/pyconform/actions.py new file mode 100644 index 00000000..405942c1 --- /dev/null +++ b/source/pyconform/actions.py @@ -0,0 +1,518 @@ +""" +Fundamental actions for the Action Graphs + +This module contains the Action objects to be used in the DiGraph-based +ActionGraphs that implement the data transformation operations. Each vertex +in an ActionGraph must be an Action. + +Some functions of the Action objects are specifically designed to work +on the output from other Actions, from within an ActionGraph object. +This is precisely what the Actions are designed to do. Some data of +the Actions pertain to the instance of the Action itself, and this data +is stored with the instance. Some data is determined entirely by the input +into the Action at runtime, which occurs within the ActionGraph data +structure. + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from __future__ import print_function +from abc import ABCMeta, abstractmethod +from pyconform.slicetuple import SliceTuple +from netCDF4 import Dataset +from os.path import exists +from cf_units import Unit +from mpi4py import MPI +from sys import stderr + +import numpy + + +#=============================================================================== +# warning - Helper function +#=============================================================================== +def warning(*objs): + print("WARNING:", *objs, file=stderr) + + +#=============================================================================== +# Action +#=============================================================================== +class Action(object): + """ + The abstract base class for the Action objects used in OperationGraphs + """ + + __metaclass__ = ABCMeta + + @abstractmethod + def __init__(self, key, name): + """ + Initializer + + Parameters: + key (hashable): A hashable key to associate with this Action + name (str): A string name/identifier for the operator + """ + if not isinstance(name, (str, unicode)): + raise TypeError('Action names must be strings') + self._name = name + + # Key + self._key = key + + # Default units + self._units = None + + # Default dimensions + self._dimensions = None + + @property + def name(self): + """ + Return the internal name of the Action + """ + return self._name + + @property + def key(self): + """ + The hashable key associated with this Action + """ + return self._key + + def __str__(self): + """ + String representation of the operation (its name) + """ + return str(self._name) + + @property + def units(self): + """ + The units of the data returned by the operation + """ + return self._units + + @units.setter + def units(self, u): + """ + Set the units of the data to be returned by the operation + """ + if isinstance(u, Unit): + self._units = u + elif isinstance(u, tuple): + self._units = Unit(u[0], calendar=u[1]) + else: + self._units = Unit(u) + + @property + def dimensions(self): + """ + The dimensions tuple of the data returned by the operation + """ + return self._dimensions + + @dimensions.setter + def dimensions(self, d): + """ + Set the dimensions tuple of the data returned by the operation + """ + self._dimensions = tuple(d) + + @abstractmethod + def __call__(self): + """ + Perform the operation and return the resulting data + """ + pass + + +#=============================================================================== +# Reader +#=============================================================================== +class Reader(Action): + """ + Action that reads an input variable slice upon calling + """ + + def __init__(self, filepath, variable, slicetuple=None): + """ + Initializer + + Parameters: + filepath (str): A string filepath to a NetCDF file + variable (str): A string variable name to read + slicetuple (SliceTuple): A tuple of slice objects specifying the + range of data to read from the file (in file-local indices) + """ + # Parse File Path + if not isinstance(filepath, (str, unicode)): + raise TypeError('Unrecognized file path object of type ' + '{!r}: {!r}'.format(type(filepath), filepath)) + if not exists(filepath): + raise OSError('File path not found: {!r}'.format(filepath)) + self._filepath = filepath + + # Attempt to open the NetCDF file + try: + ncfile = Dataset(self._filepath, 'r') + except: + raise OSError('Cannot open as a NetCDF file: ' + '{!r}'.format(self._filepath)) + + # Parse variable name + if not isinstance(variable, (str, unicode)): + raise TypeError('Unrecognized variable name object of type ' + '{!r}: {!r}'.format(type(variable), variable)) + if variable not in ncfile.variables: + raise OSError('Variable {!r} not found in NetCDF file: ' + '{!r}'.format(variable, self._filepath)) + + # Parse slice tuple + self._slice = SliceTuple(slicetuple) + + # Call base class initializer - sets self._name + slcstr = str(self._slice).replace('(', '[').replace(')', ']') + name = '{0}{1}'.format(variable, slcstr) + super(Reader, self).__init__(variable, name) + + # Read/set the units + if 'units' in ncfile.variables[variable].ncattrs(): + units = ncfile.variables[variable].getncattr('units') + else: + units = Unit(1) + if 'calendar' in ncfile.variables[variable].ncattrs(): + calendar = ncfile.variables[variable].getncattr('calendar') + else: + calendar = None + try: + unit_obj = Unit(units, calendar=calendar) + except: + unit_obj = Unit(1) + self._units = unit_obj + + # Read/set the dimensions + self._dimensions = ncfile.variables[variable].dimensions + + # Close the NetCDF file + ncfile.close() + + @Action.units.setter + def units(self, u): + pass # Prevent from changing units! + + @Action.dimensions.setter + def dimensions(self, d): + pass # Prevent from changing dimensions! + + @property + def slicetuple(self): + return self._slice + + def __call__(self): + """ + Make callable like a function + """ + ncfile = Dataset(self._filepath, 'r') + data = ncfile.variables[self._key][self._slice.index] + ncfile.close() + return data + + +#=============================================================================== +# Evaluator +#=============================================================================== +class Evaluator(Action): + """ + Generic function operator that acts on two operands + """ + + def __init__(self, key, name, func, signature=[]): + """ + Initializer + + Parameters: + key (hashable): Hashable key associated with the function + name (str): A string name/identifier for the operator + func (Function): A function with arguments taken from other operators + signature (list): Arguments to the function, in order, where 'None' + indicates an argument passed in at runtime + """ + # Check function + if not callable(func): + raise TypeError('Function object not callable: {!r}'.format(func)) + self._function = func + + # Check arguments + if not isinstance(signature, (tuple, list)): + raise TypeError('Signature not contained in list') + self._signature = signature + + # Count the number of runtime arguments needed + self._nargs = sum(arg is None for arg in signature) + + # Call base class initializer + super(Evaluator, self).__init__(key, name) + + @property + def signature(self): + return self._signature + + def __call__(self, *args): + """ + Make callable like a function + + Parameters: + args: List of arguments passed to the function + """ + if len(args) < self._nargs: + raise RuntimeError('Received {} arguments, expected ' + '{}'.format(len(args), self._nargs)) + tmp_args = list(args) + rtargs = [arg if arg else tmp_args.pop(0) for arg in self._signature] + rtargs.extend(tmp_args) + return self._function(*rtargs) + + +#=============================================================================== +# Finalizer +#=============================================================================== +class Finalizer(Action): + """ + Action that acts as a "handle" for output data streams + """ + + def __init__(self, variable, data=None, slicetuple=None, + minimum=None, maximum=None, + min_mean_abs=None, max_mean_abs=None): + """ + Initializer + + Parameters: + variable (str): A string name/identifier for the variable + data (array): An array of pre-defined data + slicetuple (tuple): The slice of the output variable into which + the data is to be written + minimum: The minimum value the data should have, if valid + maximum: The maximum value the data should have, if valid + min_mean_abs: The minimum acceptable value of the mean of the + absolute value of the data + max_mean_abs: The maximum acceptable value of the mean of the + absolute value of the data + """ + # Parse slice tuple + self._slice = SliceTuple(slicetuple) + + # Call base class initializer + slcstr = str(self._slice).replace('(', '[').replace(')', ']') + name = '{0}{1}'.format(variable, slcstr) + super(Finalizer, self).__init__(variable, name) + + # Store min/max + self._min = minimum + self._max = maximum + + # Stoe mean_abs min/max + self._min_mean_abs = min_mean_abs + self._max_mean_abs = max_mean_abs + + # Store predefined data, if available + if data is not None and not isinstance(data, numpy.ndarray): + raise TypeError('Data set in Handler object must be an array') + self._data = data + + def is_preset(self): + return self._data is not None + + @property + def slicetuple(self): + return self._slice.index + + def __call__(self, data=None): + """ + Make callable like a function + + Parameters: + data: The data passed to the mapper + """ + if self._data is not None: + if data is not None: + raise ValueError('Handler with internally defined data ' + 'received input from another Action') + else: + data = self._data + + if self._min is not None: + dmin = numpy.min(data) + if dmin < self._min: + warning(('Data from operator {!r} has minimum value ' + '{} but requires data greater than or equal to ' + '{}').format(self.name, dmin, self._min)) + + if self._max is not None: + dmax = numpy.max(data) + if dmax > self._max: + warning(('Data from operator {!r} has maximum value ' + '{} but requires data less than or equal to ' + '{}').format(self.name, dmax, self._max)) + + if self._min_mean_abs is not None or self._max_mean_abs is not None: + mean_abs = numpy.mean(numpy.abs(data)) + + if self._min_mean_abs is not None: + if mean_abs < self._min_mean_abs: + warning(('Data from operator {!r} has minimum mean_abs value ' + '{} but requires data greater than or equal to ' + '{}').format(self.name, mean_abs, self._min_mean_abs)) + + if self._max_mean_abs is not None: + if mean_abs > self._max: + warning(('Data from operator {!r} has maximum mean_abs value ' + '{} but requires data less than or equal to ' + '{}').format(self.name, mean_abs, self._max_mean_abs)) + + return data + + +#=============================================================================== +# MPISender +#=============================================================================== +class MPISender(Action): + """ + Send data to a specified remote rank in COMM_WORLD + """ + + def __init__(self, dest): + """ + Initializer + + Parameters: + dest (int): The destination rank in COMM_WORLD to send the data + units (Unit): The units of the data returned by the function + dimensions (tuple): Dimensions of the returned data + """ + # Check if the function is callable + if not isinstance(dest, int): + raise TypeError('Destination rank must be an integer') + size = MPI.COMM_WORLD.Get_size() + if dest < 0 or dest >= size: + raise ValueError(('Destination rank must be between 0 and ' + '{}').format(size)) + + # Call base class initializer + source = MPI.COMM_WORLD.Get_rank() + opname = 'send(to={},from={})'.format(dest, source) + super(MPISender, self).__init__((source, dest), opname) + + # Store the destination rank + self._dest = dest + + @property + def destination(self): + return self._dest + + def __call__(self, data): + """ + Make callable like a function + + Parameters: + data: The data to send + """ + # Check data type + if not isinstance(data, numpy.ndarray): + raise TypeError('MPISender only works with NDArrays') + + # Create the handshake message - Assumes data is an NDArray + msg = {} + msg['shape'] = data.shape + msg['dtype'] = data.dtype + + # Send the handshake message to the MPIReceiver + tag = MPI.COMM_WORLD.Get_rank() + MPI.COMM_WORLD.send(msg, dest=self._dest, tag=tag) + + # Receive the acknowledgement from the MPIReceiver + tag += MPI.COMM_WORLD.Get_size() + ack = MPI.COMM_WORLD.recv(source=self._dest, tag=tag) + + # Check the acknowledgement, if not OK error + if not ack: + raise RuntimeError(('MPISender on rank {} failed to ' + 'receive acknowledgement from rank ' + '{}').format(MPI.COMM_WORLD.Get_rank(), + self._dest)) + + # If OK, send the data to the MPIReceiver + tag += MPI.COMM_WORLD.Get_size() + MPI.COMM_WORLD.Send(data, dest=self._dest, tag=tag) + return None + + +#=============================================================================== +# MPIReceiver +#=============================================================================== +class MPIReceiver(Action): + """ + Receive data from a specified remote rank in COMM_WORLD + """ + + def __init__(self, source): + """ + Initializer + + Parameters: + source (int): The source rank in COMM_WORLD to send the data + units (Unit): The units of the data returned by the function + dimensions (tuple): Dimensions of the returned data + """ + # Check if the function is callable + if not isinstance(source, int): + raise TypeError('Source rank must be an integer') + size = MPI.COMM_WORLD.Get_size() + if source < 0 or source >= size: + raise ValueError(('Source rank must be between 0 and ' + '{}').format(size)) + + # Call base class initializer + dest = MPI.COMM_WORLD.Get_rank() + opname = 'recv(to={},from={})'.format(dest, source) + super(MPIReceiver, self).__init__((source, dest), opname) + + # Store the source rank + self._source = source + + @property + def source(self): + return self._source + + def __call__(self): + """ + Make callable like a function + """ + + # Receive the handshake message from the MPISender + tag = self._source + msg = MPI.COMM_WORLD.recv(source=self._source, tag=tag) + + # Check the message content + ack = (type(msg) is dict and + all([key in msg for key in ['shape', 'dtype']])) + + # Send acknowledgement back to the MPISender + tag += MPI.COMM_WORLD.Get_size() + MPI.COMM_WORLD.send(ack, dest=self._source, tag=tag) + + # If acknowledgement is bad, don't receive + if not ack: + raise RuntimeError(('MPIReceiver on rank {} failed to ' + 'receive acknowledgement from rank ' + '{}').format(MPI.COMM_WORLD.Get_rank(), + self._dest)) + + # Receive the data from the MPISender + tag += MPI.COMM_WORLD.Get_size() + recvd = numpy.empty(msg['shape'], dtype=msg['dtype']) + MPI.COMM_WORLD.Recv(recvd, source=self._source, tag=tag) + return recvd diff --git a/source/pyconform/climIO.py b/source/pyconform/climIO.py index 6c8124b1..7929d194 100644 --- a/source/pyconform/climIO.py +++ b/source/pyconform/climIO.py @@ -1,10 +1,20 @@ +""" +Climate I/O Facade + +This defines the facade for netCDF I/O operations, either with netCDF4 or +PyNIO. + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + import os,sys try: - import Nio - import netCDF4 + import Nio + import netCDF4 except ImportError: - # A failure here is okay. It will check again later while importing io port - check = False + check = False + #============================================= # Initializes the correct I/O version to use @@ -82,9 +92,6 @@ class PyNioPort(object): def __init__(self): super(PyNioPort, self).__init__() - #============================================= - # PyNIO: Open an existing NetCDF file - #============================================= def open_file(self, file_name): """ Open a NetCDF file for reading. @@ -99,9 +106,6 @@ def open_file(self, file_name): open_file = Nio.open_file(file_name,"r") return open_file - #============================================= - # PyNIO: Reads and returns a chunk of data - #============================================= def read_slice(self, open_file, var, index, all_values=False): """ Retreives a chunk of data. @@ -174,7 +178,7 @@ def create_file(self, new_file_name,ncformat,hist_string=None): opt.Format = 'NetCDF4Classic' opt.PreFill = False if hist_string is None: - hist_string = 'clim-convert'+new_file_name + hist_string = 'clim-convert'+new_file_name # Open new output file new_file = Nio.open_file(new_file_name, "w", options=opt, history=hist_string) diff --git a/source/pyconform/conform.py b/source/pyconform/conform.py new file mode 100644 index 00000000..a0ec8a67 --- /dev/null +++ b/source/pyconform/conform.py @@ -0,0 +1,141 @@ +""" +Main Conform Module + +This file contains the classes and functions for the main PyConform operation. + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from pyconform.datasets import InputDataset, OutputDataset +from pyconform.actiongraphs import ActionGraph, GraphFiller +from netCDF4 import Dataset as NCDataset +from mpi4py import MPI + + +#=============================================================================== +# setup +#=============================================================================== +def setup(inp, out): + """ + Setup the conform operation + + Parameters: + inp (InputDataset): The input Dataset + out (OutputDataset): The output Dataset + """ + if not isinstance(inp, InputDataset): + raise TypeError('Input dataset must be of InputDataset type') + if not isinstance(out, OutputDataset): + raise TypeError('Output dataset must be of OutputDataset type') + + # Compute the operation graph + agraph = ActionGraph() + + # Create a Graph Filler object to fill the graph + filler = GraphFiller(inp) + + # Fill the Graph with initial definitions + filler.from_definitions(agraph, out) + + # Attempt to match dimensions in the graph + filler.match_dimensions(agraph) + + # Attempt to match units in the graph + filler.match_units(agraph) + + return agraph + + +#=============================================================================== +# run +#=============================================================================== +def run(inp, out, agraph): + """ + Run the conform operation + + Parameters: + inp (InputDataset): The input Dataset + out (OutputDataset): The output Dataset + agraph (ActionGraph): The action graph needed to get the data + """ + + # MPI rank and size + mpi_rank = MPI.COMM_WORLD.Get_rank() + mpi_size = MPI.COMM_WORLD.Get_size() + + # Get time-series names and metadata names + tser_vars = [vname for vname, vinfo in out.variables.iteritems() + if vinfo.filename is not None] + meta_vars = [vname for vname in out.variables if vname not in tser_vars] + + # Get the set of necessary dimensions for the meta_vars + req_dims = set(d for mvar in meta_vars + for d in out.variables[mvar].dimensions) + + # Get the local list of tseries variable names + loc_vars = tser_vars[mpi_rank::mpi_size] + + # Fill a map of variable name to graph handle + # CURRENT: handle.key maps to output variable name + # FUTURE: handle.name (Key+Slice) should map to output identifier + handles = {} + for handle in agraph.handles(): + if handle.key in handles: + raise KeyError(('Doubly-mapped handle key in ActionGraph: ' + '{0!r}').format(handle.key)) + else: + handles[handle.key] = handle + + # Write each local time-series file + for tsvar in loc_vars: + print ('[{0}/{1}] Starting to write output variable: ' + '{2}').format(mpi_rank, mpi_size, tsvar) + + # Get the time-series variable info object + tsinfo = out.variables[tsvar] + + # Create the output file + ncf = NCDataset(tsinfo.filename, 'w') + + # Write the file attributes + ncf.setncatts(out.attributes) + + # Create the required dimensions + tsdims = req_dims.union(set(tsinfo.dimensions)) + for odim in tsdims: + if odim in agraph.dimension_map: + idim = agraph.dimension_map[odim] + dinfo = inp.dimensions[idim] + elif odim in out.dimensions: + dinfo = out.dimensions[odim] + else: + raise RuntimeError('Cannot find dimension {0}'.format(odim)) + if dinfo.unlimited: + ncf.createDimension(odim) + else: + ncf.createDimension(odim, dinfo.size) + + # Create the variables and write their attributes + ncvars = {} + for mvar in meta_vars: + mvinfo = out.variables[mvar] + ncvar = ncf.createVariable(mvar, mvinfo.datatype, mvinfo.dimensions) + for aname, avalue in mvinfo.attributes.iteritems(): + ncvar.setncattr(aname, avalue) + ncvars[mvar] = ncvar + + ncvar = ncf.createVariable(tsvar, tsinfo.datatype, tsinfo.dimensions) + for aname, avalue in tsinfo.attributes.iteritems(): + ncvar.setncattr(aname, avalue) + ncvars[tsvar] = ncvar + + # Now perform the operation graphs and write data to variables + for vname, vobj in ncvars.iteritems(): + groot = handles[vname] + vobj[:] = agraph(groot) + + ncf.close() + + print ('[{0}/{1}] Finished writing output variable: ' + '{2}').format(mpi_rank, mpi_size, tsvar) diff --git a/source/pyconform/datasets.py b/source/pyconform/datasets.py new file mode 100644 index 00000000..865da187 --- /dev/null +++ b/source/pyconform/datasets.py @@ -0,0 +1,478 @@ +""" +Dataset Interface Class + +This file contains the interface classes to the input and output multi-file +datasets. + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from os import linesep +from collections import OrderedDict +from numpy import dtype, array, array_equal +from netCDF4 import Dataset as NC4Dataset +from cf_units import Unit + + +#=============================================================================== +# DimensionInfo +#=============================================================================== +class DimensionInfo(object): + + def __init__(self, name, size=None, unlimited=False): + """ + Initializer + + Parameters: + name (str): Dimension name + size (int): Dimension size + unlimited (bool): Whether the dimension is unlimited or not + """ + self._name = str(name) + self._size = int(size) if size else None + self._unlimited = bool(unlimited) + + @property + def name(self): + return self._name + + @property + def size(self): + return self._size + + @property + def unlimited(self): + return self._unlimited + + def __eq__(self, other): + if self.name != other.name: + return False + if self.size != other.size: + return False + if self.unlimited != other.unlimited: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __str__(self): + unlim_str = ', unlimited' if self.unlimited else '' + return '{!r} [{}{}]'.format(self.name, self.size, unlim_str) + + +#========================================================================= +# VariableInfo +#========================================================================= +class VariableInfo(object): + + def __init__(self, name, + datatype='float32', + dimensions=(), + attributes=OrderedDict(), + definition=None, + data=None, + filename=None): + """ + Initializer + + Parameters: + name (str): Name of the variable + datatype (str): Numpy datatype of the variable data + dimensions (tuple): Tuple of dimension names in variable + attributes (dict): Dictionary of variable attributes + definition (str): Optional string definition of variable + data (tuple): Tuple of data to initialize the variable + filename (str): Filename for read/write of variable + """ + self._name = str(name) + self._datatype = '{!s}'.format(dtype(datatype)) + self._dimensions = dimensions + self._attributes = attributes + self._definition = definition + self._data = data + self._filename = filename + + @property + def name(self): + return self._name + + @property + def datatype(self): + return self._datatype + + @property + def dimensions(self): + return self._dimensions + + @property + def attributes(self): + return self._attributes + + @property + def definition(self): + return self._definition + + @property + def data(self): + return self._data + + @property + def filename(self): + return self._filename + + def __eq__(self, other): + if self.name != other.name: + return False + if self.datatype != other.datatype: + return False + if self.dimensions != other.dimensions: + return False + for k,v in other.attributes.iteritems(): + if k not in self.attributes: + return False + elif not array_equal(v, self.attributes[k]): + return False + if self.definition != other.definition: + return False + if not array_equal(self.data, other.data): + return False + if self.filename != other.filename: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __str__(self): + strval = 'Variable: {!r}'.format(self.name) + linesep + strval += ' datatype: {!r}'.format(self.datatype) + linesep + strval += ' dimensions: {!s}'.format(self.dimensions) + linesep + if self.definition is not None: + strval += ' definition: {!r}'.format(self.definition) + linesep + if self.data is not None: + strval += ' data: {!r}'.format(self.data) + linesep + if self.filename is not None: + strval += ' filename: {!r}'.format(self.filename) + linesep + if len(self.attributes) > 0: + strval += ' attributes:' + linesep + for aname, avalue in self.attributes.iteritems(): + strval += ' {}: {!r}'.format(aname, avalue) + linesep + return strval + + def standard_name(self): + return self.attributes.get('standard_name') + + def long_name(self): + return self.attributes.get('long_name') + + def units(self): + return self.attributes.get('units') + + def calendar(self): + return self.attributes.get('calendar') + + def cfunits(self): + return Unit(self.units(), calendar=self.calendar()) + + +#========================================================================= +# Dataset +#========================================================================= +class Dataset(object): + + def __init__(self, name='', + dimensions=OrderedDict(), + variables=OrderedDict(), + gattribs=OrderedDict()): + """ + Initializer + + Parameters: + name (str): String name to optionally give to a dataset + dimensions (dict): Dictionary of dimension sizes + variables (dict): Dictionary of VariableInfo objects defining + the dataset + gattribs (dict): Dictionary of attributes common to all files + in the dataset + """ + self._name = str(name) + + if not isinstance(variables, dict): + err_msg = ('Dataset {!r} variables must be given in a ' + 'dict').format(self.name) + raise TypeError(err_msg) + for vinfo in variables.itervalues(): + if not isinstance(vinfo, VariableInfo): + err_msg = ('Dataset {!r} variables must be of VariableInfo ' + 'type').format(self.name) + raise TypeError(err_msg) +# if not vinfo.units(): +# err_msg = ('Variable {!r} has no units in Dataset ' +# '{!r}').format(vinfo.name, self.name) +# raise ValueError(err_msg) +# if not vinfo.standard_name() and not vinfo.long_name(): +# err_msg = ('Variable {!r} has no standard_name or long_name ' +# 'in Dataset {!r}').format(vinfo.name, self.name) +# raise ValueError(err_msg) + self._variables = variables + + if not isinstance(dimensions, dict): + err_msg = ('Dataset {!r} dimensions must be given in a ' + 'dict').format(self.name) + raise TypeError(err_msg) + for dinfo in dimensions.itervalues(): + if dinfo is not None and not isinstance(dinfo, DimensionInfo): + err_msg = ('Dataset {!r} dimensions must be DimensionInfo ' + 'type or None').format(self.name) + raise TypeError(err_msg) + self._dimensions = dimensions + + if not isinstance(gattribs, dict): + err_msg = ('Dataset {!r} global attributes must be given in a ' + 'dict').format(self.name) + raise TypeError(err_msg) + self._attributes = gattribs + + @property + def name(self): + return self._name + + @property + def variables(self): + return self._variables + + @property + def dimensions(self): + return self._dimensions + + @property + def attributes(self): + return self._attributes + + def get_dict(self): + """ + Return the dictionary form of the Dataset definition + + Returns: + dict: The ordered dictionary describing the dataset + """ + dsdict = OrderedDict() + dsdict['attributes'] = self.attributes + dsdict['variables'] = OrderedDict() + for vinfo in self.variables.itervalues(): + vdict = OrderedDict() + vdict['datatype'] = vinfo.datatype + vdict['dimensions'] = vinfo.dimensions + if vinfo.definition is not None: + vdict['definition'] = vinfo.definition + if vinfo.data is not None: + vdict['data'] = vinfo.data + if vinfo.filename is not None: + vdict['filename'] = vinfo.filename + if vinfo.attributes is not None: + vdict['attributes'] = vinfo.attributes + dsdict['variables'][vinfo.name] = vdict + return dsdict + + def get_shape(self, name): + """ + Get the shape of a variable in the dataset + + Parameters: + name (str): name of the variable + """ + if name not in self.variables: + err_msg = 'Variable {!r} not in Dataset {!r}'.format(name, self.name) + raise KeyError(err_msg) + if self.dimensions: + return tuple(self.dimensions[d].size + for d in self.variables[name].dimensions) + else: + return None + + def get_size(self, name): + """ + Get the size of a variable in the dataset + + This is based on dimensions, so a variable that has no dimensions + returns a size of 0. + + Parameters: + name (str): name of the variable + """ + return sum(self.get_shape(name)) + + def get_bytesize(self, name): + """ + Get the size in bytes of a variable in the dataset + + If the size of the variable returns 0, then it assumes it is a + single-value (non-array) variable. + + Parameters: + name (str): name of the variable + """ + size = self.get_size(name) + itembytes = dtype(self.variables[name].datatype).itemsize + if size == 0: + return itembytes + else: + return itembytes * size + + +#========================================================================= +# InputDataset +#========================================================================= +class InputDataset(Dataset): + + def __init__(self, name='input', filenames=[]): + """ + Initializer + + Parameters: + name (str): String name to optionally give to a dataset + filenames (list): List of filenames in the dataset + """ + variables = OrderedDict() + varfiles = {} + dimensions = OrderedDict() + attributes = OrderedDict() + for fname in filenames: + + try: + ncfile = NC4Dataset(fname) + except: + err_msg = 'Could not open or read input file {!r}'.format(fname) + raise RuntimeError(err_msg) + + # Check attributes + for aname in ncfile.ncattrs(): + if aname not in attributes: + attributes[aname] = ncfile.getncattr(aname) + + # Check dimensions + for dname, dobj in ncfile.dimensions.iteritems(): + dinfo = DimensionInfo(dobj.name, dobj.size, dobj.isunlimited()) + if dname in dimensions: + if dinfo != dimensions[dname]: + err_msg = ('Dimension {!r} in input file {!r} is ' + 'different from expected dimension ' + '{!r}').format(dinfo, fname, + dimensions[dname]) + raise ValueError(err_msg) + else: + dimensions[dname] = dinfo + + # Check variables + for vname, vobj in ncfile.variables.iteritems(): + vattrs = OrderedDict() + for vattr in vobj.ncattrs(): + vattrs[vattr] = vobj.getncattr(vattr) + vinfo = VariableInfo(name=vname, + datatype='{!s}'.format(vobj.dtype), + dimensions=vobj.dimensions, + attributes=vattrs) + + if vname in variables: + if vinfo != variables[vname]: + err_msg = ('{}Variable {!r} in file {!r}:' + '{}{}' + 'differs from same variable in other files:' + '{}{}').format(linesep, vname, fname, + linesep, vinfo, + linesep, variables[vname]) + raise ValueError(err_msg) + else: + varfiles[vname].append(fname) + else: + variables[vname] = vinfo + varfiles[vname] = [fname] + + ncfile.close() + + # Check variable file occurrences + for vname, vfiles in varfiles.iteritems(): + if len(vfiles) == 1: + variables[vname]._filename = vfiles[0] + elif len(vfiles) < len(filenames): + missing_files = set(filenames) - set(vfiles) + wrn_msg = ('Variable {!r} appears to be metadata but does ' + 'not appear in the files:{}' + '{}').format(vname, linesep, missing_files) + raise RuntimeWarning(wrn_msg) + + super(InputDataset, self).__init__(name, dimensions, + variables, attributes) + + +#========================================================================= +# OutputDataset +#========================================================================= +class OutputDataset(Dataset): + + def __init__(self, name='output', dsdict=OrderedDict()): + """ + Initializer + + Parameters: + name (str): String name to optionally give to a dataset + dsdict (dict): Dictionary describing the dataset variables + """ + attributes = dsdict.get('attributes', OrderedDict()) + variables = OrderedDict() + invars = dsdict.get('variables', OrderedDict()) + for vname, vdict in invars.iteritems(): + kwargs = {} + if 'attributes' in vdict: + kwargs['attributes'] = vdict['attributes'] + if 'dimensions' not in vdict: + err_msg = ('Dimensions are required for variable ' + '{!r}').format(vname) + raise ValueError(err_msg) + else: + kwargs['dimensions'] = vdict['dimensions'] + if 'datatype' in vdict: + kwargs['datatype'] = vdict['datatype'] + has_defn = 'definition' in vdict + has_data = 'data' in vdict + if not has_data and not has_defn: + err_msg = ('Definition or data is required for output variable ' + '{!r}').format(vname) + raise ValueError(err_msg) + elif has_data and has_defn: + err_msg = ('Both definition and data cannot be defined for ' + 'output variable {!r}').format(vname) + raise ValueError(err_msg) + elif has_defn: + kwargs['definition'] = str(vdict['definition']) + elif has_data: + kwargs['data'] = array(vdict['data']) + if 'filename' in vdict: + kwargs['filename'] = vdict['filename'] + variables[vname] = VariableInfo(vname, **kwargs) + + dimensions = OrderedDict() + for vname, vinfo in variables.iteritems(): + if vinfo.data is not None: + dshape = vinfo.data.shape + if len(dshape) == 0: + dshape = (1,) + for dname, dsize in zip(vinfo.dimensions, dshape): + if dname in dimensions and dimensions[dname] is not None: + if dsize != dimensions[dname].size: + raise ValueError(('Dimension {0!r} is inconsistently ' + 'defined in OutputDataset ' + '{1!r}').format(dname, name)) + else: + dimensions[dname] = DimensionInfo(dname, dsize) + else: + for dname in vinfo.dimensions: + if dname not in dimensions: + dimensions[dname] = None + + super(OutputDataset, self).__init__(name, variables=variables, + dimensions=dimensions, + gattribs=attributes) + \ No newline at end of file diff --git a/source/pyconform/functions.py b/source/pyconform/functions.py new file mode 100644 index 00000000..ea232c2b --- /dev/null +++ b/source/pyconform/functions.py @@ -0,0 +1,405 @@ +""" +Functions for FunctionEvaluator Actions + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from os import linesep +from abc import ABCMeta, abstractmethod +from cf_units import Unit +from numpy import sqrt, transpose + + +class UnitsError(ValueError): + pass + + +class DimensionsError(ValueError): + pass + + +#=============================================================================== +# List all available functions or operators +#=============================================================================== +def available(): + return available_operators().union(available_functions()) + + +#=============================================================================== +# Find a function or operator based on key and number of arguments +#=============================================================================== +def find(key, numargs=2): + if (key, numargs) in available_operators(): + return find_operator(key, numargs) + elif (key, numargs) in available_functions(): + return find_function(key, numargs) + else: + raise KeyError(('{0}-arg operator/function with key {1!r} not ' + 'found').format(numargs, key)) + + +#=============================================================================== +# FunctionalAbstract - base class for Function and Operator Classes +#=============================================================================== +class FunctionAbstract(object): + __metaclass__ = ABCMeta + key = 'function' + numargs = 2 + + @abstractmethod + def units(self, *arg_units): + uret = arg_units[0] if isinstance(arg_units[0], Unit) else Unit(1) + uarg = (None,) * len(self.numargs) + return uret, uarg + + @abstractmethod + def dimensions(self, *arg_dims): + dret = arg_dims[0] if isinstance(arg_dims[0], tuple) else tuple() + darg = (None,) * len(self.numargs) + return dret, darg + + def __call__(self, *args): + return 1 + + +################################################################################ +##### OPERATORS ################################################################ +################################################################################ + +#=============================================================================== +# Get a list of the available functions by function key and number of arguments +#=============================================================================== +def available_operators(): + return set(__OPERATORS__.keys()) + + +#=============================================================================== +# Get the function associated with the given key-symbol +#=============================================================================== +def find_operator(key, numargs=2): + if (key, numargs) not in __OPERATORS__: + raise KeyError(('{0}-arg operator with key {1!r} not ' + 'found').format(numargs, key)) + return __OPERATORS__[(key, numargs)] + + +#=============================================================================== +# Operator - From which all 'X op Y'-pattern operators derive +#=============================================================================== +class Operator(FunctionAbstract): + key = '?' + numargs = 2 + + +#=============================================================================== +# NegationOperator +#=============================================================================== +class NegationOperator(Operator): + key = '-' + numargs = 1 + + @staticmethod + def units(unit): + uret = unit if isinstance(unit, Unit) else Unit(1) + return uret, (None,) + + @staticmethod + def dimensions(dims): + dret = dims if isinstance(dims, tuple) else () + return dret, (None,) + + def __call__(self, arg): + return -arg + + +#=============================================================================== +# AdditionOperator +#=============================================================================== +class AdditionOperator(Operator): + key = '+' + numargs = 2 + + @staticmethod + def units(lunit, runit): + ul = lunit if isinstance(lunit, Unit) else Unit(1) + ur = runit if isinstance(runit, Unit) else Unit(1) + if ul == ur: + return ul, (None, None) + elif ur.is_convertible(ul): + return ul, (None, ul) + else: + raise UnitsError(('Data with units {0!s} and {1!s} cannot be ' + 'added or subtracted').format(ul, ur)) + + @staticmethod + def dimensions(ldims, rdims): + dl = ldims if isinstance(ldims, tuple) else () + dr = rdims if isinstance(rdims, tuple) else () + if dl == (): + return dr, (None, None) + elif dl == dr or dr == (): + return dl, (None, None) + elif set(dl) == set(dr): + return dl, (None, dl) + else: + raise DimensionsError(('Data with dimensions {0} and {1} cannot ' + 'be added or subtracted').format(dl, dr)) + + def __call__(self, left, right): + return left + right + +#=============================================================================== +# SubtractionOperator +#=============================================================================== +class SubtractionOperator(Operator): + key = '-' + numargs = 2 + + @staticmethod + def units(lunit, runit): + return AdditionOperator.units(lunit, runit) + + @staticmethod + def dimensions(ldims, rdims): + return AdditionOperator.dimensions(ldims, rdims) + + def __call__(self, left, right): + return left - right + +#=============================================================================== +# PowerOperator +#=============================================================================== +class PowerOperator(Operator): + key = '^' + numargs = 2 + + @staticmethod + def units(lunit, runit): + if not isinstance(runit, (int, float)): + raise UnitsError('Exponent in power function must be a ' + 'unitless scalar') + try: + upow = lunit ** runit + except: + raise UnitsError(('Cannot exponentiate units to power ' + '{0}').format(runit)) + uret = upow if isinstance(upow, Unit) else Unit(1) + return uret, (None, None) + + @staticmethod + def dimensions(ldims, rdims): + if not isinstance(rdims, (int, float)): + raise DimensionsError('Exponent in power function must be a ' + 'dimensionless scalar') + dret = ldims if isinstance(ldims, tuple) else () + return dret, (None, None) + + def __call__(self, left, right): + return left ** right + + +#=============================================================================== +# MultiplicationOperator +#=============================================================================== +class MultiplicationOperator(Operator): + key = '*' + numargs = 2 + + @staticmethod + def units(lunit, runit): + ul = lunit if isinstance(lunit, Unit) else Unit(1) + ur = runit if isinstance(runit, Unit) else Unit(1) + try: + uret = ul * ur + except: + raise UnitsError(('Cannot multiply or divide units {0} and ' + '{1}').format(ul, ur)) + return uret, (None, None) + + @staticmethod + def dimensions(ldims, rdims): + dl = ldims if isinstance(ldims, tuple) else () + dr = rdims if isinstance(rdims, tuple) else () + if dl == (): + return dr, (None, None) + elif dl == dr or dr == (): + return dl, (None, None) + elif set(dl) == set(dr): + return dl, (None, dl) + else: + raise DimensionsError(('Data with dimensions {0} and {1} cannot ' + 'be multiplied or divided').format(dl, dr)) + + def __call__(self, left, right): + return left * right + + +#=============================================================================== +# DivisionOperator +#=============================================================================== +class DivisionOperator(Operator): + key = '-' + numargs = 2 + + @staticmethod + def units(lunit, runit): + ul = lunit if isinstance(lunit, Unit) else Unit(1) + ur = runit if isinstance(runit, Unit) else Unit(1) + try: + uret = ul / ur + except: + raise UnitsError(('Cannot multiply or divide units {0} and ' + '{1}').format(ul, ur)) + return uret, (None, None) + + @staticmethod + def dimensions(ldims, rdims): + return MultiplicationOperator.dimensions(ldims, rdims) + + def __call__(self, left, right): + return left / right + + +#=============================================================================== +# Operator map - Fixed to prevent user-redefinition! +#=============================================================================== + +__OPERATORS__ = {('-', 1): NegationOperator(), + ('^', 2): PowerOperator(), + ('+', 2): AdditionOperator(), + ('-', 2): SubtractionOperator(), + ('*', 2): MultiplicationOperator(), + ('/', 2): DivisionOperator()} + +################################################################################ +##### FUNCTIONS ################################################################ +################################################################################ + +#=============================================================================== +# Recursively return all subclasses of a given class +#=============================================================================== +def _all_subclasses_(cls): + return cls.__subclasses__() + [c for s in cls.__subclasses__() + for c in _all_subclasses_(s)] + +#=============================================================================== +# Check that all functions patterns are unique - Needed for User-Defined Funcs +#=============================================================================== +def check_functions(): + func_dict = {} + func_map = [((c.key, c.numargs), c) + for c in _all_subclasses_(Function)] + non_unique_patterns = [] + for func_pattern, class_name in func_map: + if func_pattern in func_dict: + func_dict[func_pattern].append(class_name) + non_unique_patterns.append(func_pattern) + else: + func_dict[func_pattern] = [class_name] + if len(non_unique_patterns) > 0: + err_msg = ['Some function patterns are multiply defined:'] + for func_pattern in non_unique_patterns: + func_descr = '{0[1]}-arg {0[0]!r}'.format(func_pattern) + class_names = ', '.join(func_dict[func_pattern]) + err_msg += [' {0} maps to {1}'.format(func_descr, class_names)] + raise RuntimeError(linesep.join(err_msg)) + +#=============================================================================== +# Get a list of the available functions by function key and number of arguments +#=============================================================================== +def available_functions(): + return set((c.key, c.numargs) + for c in _all_subclasses_(Function)) + +#=============================================================================== +# Get the function associated with the given key-symbol +#=============================================================================== +def find_function(key, numargs=2): + func_map = dict(((c.key, c.numargs), c()) + for c in _all_subclasses_(Function)) + if (key, numargs) not in func_map: + raise KeyError(('{0}-arg function with key {1!r} not ' + 'found').format(numargs, key)) + return func_map[(key, numargs)] + +#=============================================================================== +# Function - From which all 'func(...)'-pattern functions derive +#=============================================================================== +class Function(FunctionAbstract): + key = 'f' + numargs = 1 + +#=============================================================================== +# SquareRoot +#=============================================================================== +class SquareRootFunction(Function): + key = 'sqrt' + numargs = 1 + + @staticmethod + def units(unit): + u = unit if isinstance(unit, Unit) else Unit(1) + try: + uret = u.root(2) + except: + raise UnitsError(('Cannot take square-root of units {0}').format(u)) + return uret, (None,) + + @staticmethod + def dimensions(dims): + dret = dims if isinstance(dims, tuple) else () + return dret, (None,) + + def __call__(self, data): + return sqrt(data) + +#=============================================================================== +# ConvertFunction +#=============================================================================== +class ConvertFunction(Function): + key = 'convert' + numargs = 3 + + @staticmethod + def units(data_units, from_units, to_units): + ud = data_units if isinstance(data_units, Unit) else Unit(1) + uf = from_units if isinstance(from_units, Unit) else Unit(1) + ut = to_units if isinstance(to_units, Unit) else Unit(1) + if ud != uf: + raise UnitsError(('Input units {0} different from expected ' + 'units {1}').format(ud, uf)) + if not uf.is_convertible(ut): + raise UnitsError(('Ill-formed convert function. Cannot convert ' + 'units {0} to {1}').format(uf, ut)) + uret = ut + return uret, (None, None, None) + + @staticmethod + def dimensions(data_dims, from_units, to_units): + dret = data_dims if isinstance(data_dims, tuple) else () + return dret, (None, None, None) + + def __call__(self, data, from_units, to_units): + return from_units.convert(data, to_units) + +#=============================================================================== +# TransposeFunction +#=============================================================================== +class TransposeFunction(Function): + key = 'transpose' + numargs = 2 + + @staticmethod + def units(data_units, order): + uret = data_units if isinstance(data_units, Unit) else Unit(1) + return uret, (None, None) + + @staticmethod + def dimensions(data_dims, order): + d = data_dims if isinstance(data_dims, tuple) else () + dret = tuple(d[i] for i in order) + return dret, (None, None) + + def __call__(self, data, order): + return transpose(data, order) diff --git a/source/pyconform/graphs.py b/source/pyconform/graphs.py new file mode 100644 index 00000000..5de36c8d --- /dev/null +++ b/source/pyconform/graphs.py @@ -0,0 +1,419 @@ +""" +Directed Graph Data Structures and Tools + +This module contains the DiGraph directional graph generic data structure. + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from copy import deepcopy +from os import linesep + + +#=============================================================================== +# DiGraph - A directed Graph data structure +#=============================================================================== +class DiGraph(object): + """ + A rudimentary directed graph data structure + """ + + def __init__(self): + """ + Initializer + """ + self._forward_map = {} + self._reverse_map = {} + + def __str__(self): + """ + String representation of the graph + """ + outstrs = ['DiGraph:'] + for vertex, neighbors in self._forward_map.iteritems(): + if len(neighbors) == 0: + outstrs.append(' {0!s}'.format(vertex)) + else: + for neighbor in neighbors: + outstrs.append(' {0!s} --> {1!s}'.format(vertex, neighbor)) + return linesep.join(outstrs) + + def __eq__(self, other): + """ + Check if an other graph is equal to this graph + + Equality for the DiGraph is defined as when all of the vertex + objects and edges in the other graph are in this graph. + + Parameters: + other (DiGraph): the graph to compare against + + Returns: + bool: True if equal, False if not equal + """ + if isinstance(other, DiGraph): + if (self._forward_map == other._forward_map and + self._reverse_map == other._reverse_map): + return True + return False + + def __ne__(self, other): + """ + Check if an other graph is not equal to this graph + + Inquality for the DiGraph is defined as when the other graph is + found to NOT equal this graph. + + Parameters: + other (DiGraph): the graph to compare against + + Returns: + bool: True if not equal, False if equal + """ + return not (self == other) + + def __contains__(self, vertex): + """ + Check if a vertex is in the graph + + Returns: + bool: True if vertex in graph, False otherwise + """ + return vertex in self._forward_map or vertex in self._reverse_map + + def __len__(self): + """ + Returns the number of vertices in the graph + + Returns: + int: The number of vertices in the graph + """ + return self.nvertices + + @property + def nvertices(self): + """ + Returns the number of edges in the graph + + Returns: + int: The numebr of edges in the graph + """ + return len(self.vertices) + + @property + def nedges(self): + """ + Returns the number of edges in the graph + + Returns: + int: The numebr of edges in the graph + """ + return len(self.edges) + + def sinks(self): + """ + Returns the set of vertices in the graph with only incoming edges + + Returns: + set: The set of vertices in the graph with only incoming edges + """ + zero_outgoing = set(v for v, n in self._forward_map.iteritems() + if len(n) == 0) + plus_incoming = set(v for v, n in self._reverse_map.iteritems() + if len(n) > 0) + return plus_incoming.intersection(zero_outgoing) + + def sources(self): + """ + Returns the set of vertices in the graph with only outgoing edges + + Returns: + set: The set of vertices in the graph with only outgoing edges + """ + zero_incoming = set(v for v, n in self._reverse_map.iteritems() + if len(n) == 0) + plus_outgoing = set(v for v, n in self._forward_map.iteritems() + if len(n) > 0) + return plus_outgoing.intersection(zero_incoming) + + def clear(self): + """ + Remove all vertices and edges from the graph + """ + self._reverse_map = {} + self._forward_map = {} + + @property + def vertices(self): + """ + Return the list of vertices in the graph + + Returns: + set: The list of vertices contained in this graph + """ + return self._forward_map.keys() + + @property + def edges(self): + """ + Return the list of edges (vertex 2-tuples) in the graph + + Returns: + list: The list of edges contained in this graph + """ + return [(u, v) for u, vs in self._forward_map.iteritems() for v in vs] + + def add(self, vertex): + """ + Add a vertex to the graph + + Parameters: + vertex: The vertex object to be added to the graph + """ + if vertex not in self._forward_map: + self._forward_map[vertex] = [] + self._reverse_map[vertex] = [] + + def remove(self, vertex): + """ + Remove a vertex from the graph + + Parameters: + vertex: The vertex object to remove from the graph + """ + if vertex in self._forward_map: + self._forward_map.pop(vertex) + self._reverse_map.pop(vertex) + for v in self._forward_map: + if vertex in self._forward_map[v]: + self._forward_map[v].remove(vertex) + if vertex in self._reverse_map[v]: + self._reverse_map[v].remove(vertex) + + def update(self, other): + """ + Update this graph with the union of itself and another + + Parameters: + other (DiGraph): the other graph to union with + """ + if not isinstance(other, DiGraph): + raise TypeError("Cannot form union between DiGraph and non-DiGraph") + for vertex, neighbors in other._forward_map.iteritems(): + self.add(vertex) + self._forward_map[vertex].extend([n for n in neighbors if n not in + self._forward_map[vertex]]) + for vertex, neighbors in other._reverse_map.iteritems(): + self._reverse_map[vertex].extend([n for n in neighbors if n not in + self._forward_map[vertex]]) + + def union(self, other): + """ + Form the union of this graph with another + + Parameters: + other (DiGraph): the other graph to union with + + Returns: + DiGraph: A graph containing the union of this graph with another + """ + if not isinstance(other, DiGraph): + raise TypeError("Cannot for union between DiGraph and non-DiGraph") + G = deepcopy(self) + G.update(other) + return G + + def connect(self, start, stop): + """ + Add an edge to the graph + + If the vertices specified are not in the graph, they will be added. + + Parameters: + start: The starting point of the edge to be added to the graph + stop: The ending point of the edge to be added to the graph + """ + self.add(start) + self.add(stop) + if stop not in self._forward_map[start]: + self._forward_map[start].append(stop) + self._reverse_map[stop].append(start) + + def disconnect(self, start, stop): + """ + Remove an edge from the graph + + Parameters: + start: The starting point of the edge to be removed from the graph + stop: The ending point of the edge to be removed from the graph + """ + if start not in self._forward_map: + return + if stop in self._forward_map[start]: + self._forward_map[start].remove(stop) + self._reverse_map[stop].remove(start) + + def insert(self, start, vertex, stop): + """ + Insert a vertex into the graph between two given vertices + + This procedure preserves the order of the vertex neighbors + + Parameters: + start: The starting point of the edge to be removed from the graph + vertex: The vertex to be inserted on the edge from start to stop + stop: The ending point of the edge to be removed from the graph + """ + self.add(vertex) + if start not in self._forward_map: + return + if stop in self._forward_map[start]: + fidx = self._forward_map[start].index(stop) + self._forward_map[start][fidx] = vertex + self._forward_map[vertex].append(stop) + ridx = self._reverse_map[stop].index(start) + self._reverse_map[stop][ridx] = vertex + self._reverse_map[vertex].append(start) + + def neighbors_from(self, vertex): + """ + Return the list of neighbors on edges pointing from the given vertex + + Parameters: + vertex: The vertex object to query + + Returns: + list: The list of vertices with incoming edges from vertex + """ + return list(self._forward_map[vertex]) + + def neighbors_to(self, vertex): + """ + Return the list of neighbors on edges pointing to the given vertex + + Parameters: + vertex: The vertex object to query + + Returns: + list: The list of vertices with outgoing edges to vertex + """ + return list(self._reverse_map[vertex]) + + def iter_bfs(self, start, reverse=False): + """ + Breadth-First Search generator from the root node + + Parameters: + start: The starting vertex of the search + reverse (bool): Whether to perform the search "backwards" + through the graph (True) or "forwards" (False) + + Yields: + The next vertex found in the breadth-first search from start + """ + if start not in self.vertices: + raise KeyError('Root vertex not in graph') + + visited = set() + queue = [start] + if reverse: + neighbors = self._reverse_map + else: + neighbors = self._forward_map + while queue: + vertex = queue.pop(0) + if vertex not in visited: + yield vertex + visited.add(vertex) + queue.extend(v for v in neighbors[vertex] if v not in visited) + + def iter_dfs(self, start, reverse=False): + """ + Depth-First Search generator from the root node + + Parameters: + start: The starting vertex of the search + reverse (bool): Whether to perform the search "backwards" + through the graph (True) or "forwards" (False) + """ + if start not in self.vertices: + raise KeyError('Root vertex not in graph') + + visited = set() + stack = [start] + if reverse: + neighbors = self._reverse_map + else: + neighbors = self._forward_map + while stack: + vertex = stack.pop() + if vertex not in visited: + yield vertex + visited.add(vertex) + stack.extend(v for v in neighbors[vertex] if v not in visited) + + def toposort(self): + """ + Return a topological ordering of the vertices using Kahn's algorithm + + Returns: + list: If topological ordering is possible, then return the list of + topologically ordered vertices + None: If topological ordering is not possible (i.e., if the DiGraph + is cyclic), then return None + """ + G = deepcopy(self) + sorted_list = [] + stack = [v for v, n in G._reverse_map.iteritems() if len(n) == 0] + while stack: + vertex = stack.pop() + sorted_list.append(vertex) + for neighbor in list(G._forward_map[vertex]): + G.disconnect(vertex, neighbor) + if len(G._reverse_map[neighbor]) == 0: + stack.append(neighbor) + if sum(len(n) for n in G._forward_map.itervalues()) > 0: + return None + else: + return sorted_list + + def is_cyclic(self): + """ + Returns whether the graph is cyclic or not + """ + return self.toposort() is None + + def components(self): + """ + Return the connected components of the graph + + Returns: + list: A list of connected DiGraphs + """ + unvisited = set(self.vertices) + components = set() + while unvisited: + start = unvisited.pop() + comp = type(self)() + stack = [start] + while stack: + vertex = stack.pop() + # Forward + neighbors = [v for v in self._forward_map[vertex] + if v not in comp] + for neighbor in neighbors: + comp.connect(vertex, neighbor) + stack.extend(neighbors) + # Backward + neighbors = [v for v in self._reverse_map[vertex] + if v not in comp] + for neighbor in neighbors: + comp.connect(neighbor, vertex) + stack.extend(neighbors) + # Mark vertex as visited + if vertex in unvisited: + unvisited.remove(vertex) + if len(comp.vertices) > 0: + components.add(comp) + return components diff --git a/source/pyconform/mapdates.py b/source/pyconform/mapdates.py new file mode 100644 index 00000000..f8639c92 --- /dev/null +++ b/source/pyconform/mapdates.py @@ -0,0 +1,266 @@ +""" +Date Evaluator + +This evaluates the time slices in the file(s) to define the time period +between steps and verifies consistancy and sequential steps between files. + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +import climIO +from dateutil import parser +import os +import cf_units + + +#============================================= +# Get time information from the time slices +#============================================= +def __get_time_info__(f, io): + """ + Evaluates the time slices in the input files to verify + that there are the correct number of slices in the file + and that the slices are contiguous. It also pulls off + other information, such as start/en times, time period/spacing, + slice count, and the average of the slices (for ordering purposes). + + Parameters: + f (climIO file): A pointer to an open netCDF file. + io (climIO): An object that contains a setof io commands to use. + + Returns: + date_info (dict): Contains file information, such as time period(t_per), time step(t_step), + first/last step(t0 and tn), slice counts(cnt). + average (int): The average of all time slices. + """ + date_info = {} + _tc, _dim, att = io.get_var_info(f, 'time') + stand_cal = cf_units.Unit('days since 1-1-1 0:0:0', calendar=att['calendar']) + cal_unit = cf_units.Unit(att['units'], calendar=att['calendar']) + if ('bounds' in att.keys()): + # print 'Using bounds' + tb = f.variables[att['bounds']] + l = len(tb) + d0 = tb[0, 0] + d1 = tb[1, 0] + d2 = tb[2, 0] + dn = tb[l - 1, 1] - 1 + time = tb[:, 0] + else: + # print 'Using time' + tb = f.variables['time'] + l = len(tb) + d0 = tb[0] + d1 = tb[1] + d2 = tb[2] + dn = tb[l - 1] + time = tb[:] + date_info['time'] = time + + # Get second and third time bounds to figure out the time period + t1 = (parser.parse( + str(cf_units.num2date(d1, att['units'], calendar=att['calendar']))).timetuple()) + t2 = (parser.parse( + str(cf_units.num2date(d2, att['units'], calendar=att['calendar']))).timetuple()) + # Get time difference between the steps + t_step = d2 - d1 + h = t2[3] - t1[3] + if (t1[3] != t2[3]): + t_per = str(h) + 'hour' + elif (t1[2] != t2[2]): + t_per = 'day' + elif (t1[1] != t2[1]): + t_per = 'mon' + elif (t1[0] != t2[0]): + t_per = 'year' + else: + t_per = 'UNKNOWN' + date_info['t_per'] = t_per + date_info['t_step'] = t_step + # Get first and last dates + t0 = (parser.parse(str(cf_units.num2date(d0, att['units'], + calendar=att['calendar']))).timetuple()) + tn = (parser.parse(str(cf_units.num2date(dn, att['units'], + calendar=att['calendar']))).timetuple()) + date_info['t0'] = cal_unit.convert(d0, stand_cal) + date_info['tn'] = cal_unit.convert(dn, stand_cal) + date_info['cnt'] = l + average = (date_info['t0'] + date_info['tn']) / 2 + date_info['units'] = att['units'] + date_info['calendar'] = att['calendar'] + + # Check to see if the number of slices matches how many should be in the + # date range + if t_per == 'year': + _ok = (tn[0] - t0[0] == l) + elif t_per == 'mon': + _ok = (((tn[0] - t0[0]) * 12) + (tn[1] - t0[1] + 1) == l) + elif t_per == 'day': + _ok = ((dn - d0 + 1) == l) + elif 'hour' in t_per: + cnt_per_day = 24.0 / h + _ok = (((dn - d0) * cnt_per_day + 1) == l) + + return average, date_info + + +#============================================= +# Verify that there are no gaps in time from +# one file to the next +#============================================= +def __check_date_alignment__(keys, date_info): + """ + Evaluates the dates between files to verify that + no gaps are found in the time dimension. + + Parameters: + keys (list): A list of time slice references that are in correct time order. + date_info (dict): Contains file information, such as time period(t_per), time step(t_step), + first/last step(t0 and tn), slice counts(cnt). + + """ + prev_last = date_info[keys[0]]['tn'] + t_step = date_info[keys[0]]['t_step'] + + if date_info[keys[0]]['t_per'] == 'mon': + date = (parser.parse(str(cf_units.num2date(date_info[keys[0]]['tn'], + date_info[keys[0]]['units'], calendar=date_info[keys[0]]['calendar']))).timetuple()) + if date[1] == 12: + next_val = 1 + else: + next_val = date[1] + 1 + + else: + next_val = prev_last + t_step + + for i in range(1, len(keys)): + if date_info[keys[i]]['t_per'] == 'mon': + new_date = (parser.parse(str(cf_units.num2date(date_info[keys[i]]['t0'], + date_info[keys[i]]['units'], calendar=date_info[keys[i]]['calendar']))).timetuple()) + if (next_val == new_date[1]): + date = (parser.parse(str(cf_units.num2date(date_info[keys[i]]['tn'], + date_info[keys[i]]['units'], calendar=date_info[keys[i]]['calendar']))).timetuple()) + if date[1] == 12: + next_val = 1 + else: + next_val = date[1] + 1 + else: + print "Disconnect? Expected: ", next_val, " Got: ", new_date[1] + return 1 + else: + if (next_val == date_info[keys[i]]['t0']): + # print "Looks + # okay",date_info[keys[i]]['t0'],'-',date_info[keys[i]]['tn'] + prev_last = date_info[keys[i]]['tn'] + next_val = prev_last + t_step + else: + print "Disconnect? Expected: ", next_val, " Got: ", date_info[keys[i]]['t0'] + return 1 + + return 0 + + +#============================================= +# Verify that there are no gaps in time from +# one time step to the next in the same file. +#============================================= +def __check_date_alignment_in_file__(date): + """ + Evaluates the dates between time steps to verify that + no gaps are found in the time dimension. + + Parameters: + date (dict): Contains file information, such as time period(t_per), time step(t_step), + first/last step(t0 and tn), slice counts(cnt). + + """ + t_step = date['t_step'] + t_per = date['t_per'] + t = date['time'] + prev_time = t[0] + + # If the time period in monthly, then the time difference between slices is not uniform and + # requires looking into the month length array to get correct day #'s + # between slices. + if t_per == 'mon': + if t_per == 'mon': + date1 = (parser.parse(str(cf_units.num2date(t[0], + date['units'], calendar=date['calendar']))).timetuple()) + if date1[1] == 12: + next_val = 1 + else: + next_val = date1[1] + 1 + for i in range(1, len(t)): + new_date = (parser.parse(str(cf_units.num2date(t[i], + date['units'], calendar=date['calendar']))).timetuple()) + if (next_val == new_date[1]): + date1 = (parser.parse(str(cf_units.num2date(t[i], + date['units'], calendar=date['calendar']))).timetuple()) + if date1[1] == 12: + next_val = 1 + else: + next_val = date1[1] + 1 + else: + print "Disconnect? Expected: ", next_val, " Got: ", new_date[1], ' around time step: ', i + return 1 + # All other time periods should have the same number of days between + # slices. + else: + for i in range(1, len(t)): + if (prev_time + t_step == t[i]): + # Time step looks okay + prev_time = t[i] + else: + print "Disconnect? Expected: ", str(prev_time + t_step), " Got: ", t[i], ' around time step: ', i + return 1 + return 0 + + +#============================================= +# Examine the file list and put them in sequential +# order. +#============================================= +def get_files_in_order(files, alignment=True): + """ + Examine the file list and put the files in + sequential order. + + Parameters: + files (list): A list of file names to put into sequential order. + + Returns: + file_list (list): A list that contains the file names in sequential + order. + counts (list): A list containing the time slice counts for each of the + files. The order should match the order in which the + file_list lists the files. + """ + ncdf = climIO.init_climIO(override='netCDF4') + + date_info = {} + counts = [] + file_list = [] + + for fl in files: + fn = os.path.basename(fl) + f = ncdf.open_file(fl) + ave, date = __get_time_info__(f, ncdf) + date['fn'] = fn + date_info[ave] = date + if alignment: + error = __check_date_alignment_in_file__(date) + if error != 0: + return file_list, counts, error + + for d in sorted(date_info.keys()): + file_list.append(date_info[d]['fn']) + counts.append(date_info[d]['cnt']) + + # If there are more than 1 files, make sure the dates are aligned correctly + if len(fl) > 1: + error = __check_date_alignment__(sorted(date_info.keys()), date_info) + if error != 0: + return file_list, counts, error + + return file_list, counts, error diff --git a/source/pyconform/mip_table_parser.py b/source/pyconform/mip_table_parser.py new file mode 100644 index 00000000..2134a35b --- /dev/null +++ b/source/pyconform/mip_table_parser.py @@ -0,0 +1,660 @@ +import sys + +#============================================= +# Parse and Standardize the MIP table. +# Return a dcitionary that contains the parsed +# information. +#============================================= +def mip_table_parser(exp, table_name,type=None,user_vars={},user_axes={},user_tableInfo={}): + """ + Function will parse three different MIP table formats: tab deliminated, CMOR, and XML. + After the table has been parsed, the information is standardized into the three dictionaries + and combined into one final dictionary. The three dictionaries are keyed with 'variables', + 'axes', and 'table_info'. The standardization methods found in the three dictionaries + resemble the identifiers used in the CMOR code. Variables are indexed by their string + id/label/variable_entry name. + Usage examples - table_dict = mip_table_parser('6hrLev','xml') + table_dict = mip_table_parser('Tables/CMIP5_Amon','cmor') + table_dict = mip_table_parser('Tables/CMIP6_MIP_tables.txt','excel') + + Parameters: + exp (str): Name of the experiment. + table_name (str): Full path to a table. Or in the case of XML, the experiment name. + type (str): One of 3 keys that identify table type: 'excel', 'cmor', or 'xml'. + user_var (dict): Allows users to add another synonym for one of the standard variable field keys. + user_axes (dict): Allows users to add another synonym for one of the standard axes field keys. + user_tableInfo (dict): Allows users to add another synonym for one of the standard table field keys. + + Returns: + table_dict (dict): A dictionary that holds all of the parsed table information. Contains the keys: + 'variables', 'axes', and 'table_info'. Each of these are dictionaries, keyed with variable names and + each variable has a value of a dictionary keyed with the standard field names. + """ + # Standardized identifiers for variable fields + # Format: standardName: [synonym list] + table_var_fields = { + 'cell_measures': ['cell_measures'], + 'cell_methods': ['cell_methods'], + 'cf_standard_name': ['CF Standard Name','CF_Standard_Name','cf_standard_name','standard_name'], + 'comment': ['comment'], + 'deflate': ['deflate'], + 'deflate_level': ['deflate_level'], + 'description': ['description'], + 'dimensions': ['dimensions'], + 'ext_cell_measures': ['ext_cell_measures'], + 'flag_meanings': ['flag_meanings'], + 'flag_values': ['flag_values'], + 'frequency': ['frequency'], + 'id': ['id','label','variable_entry'], + 'long_name': ['long_name'], + 'modeling_realm': ['modeling_realm'], + 'ok_min_mean_abs': ['ok_min_mean_abs'], + 'ok_max_mean_abs': ['ok_max_mean_abs'], + 'out_name': ['out_name'], + 'positive': ['positive'], + 'prov': ['prov'], + 'provNote': ['provNote'], + 'required': ['required'], + 'shuffle': ['shuffle'], + 'title': ['title'], + 'type': ['type'], + 'units': ['units'], + 'valid_max': ['valid_max'], + 'valid_min': ['valid_min'] + } + + # Standardized identifiers for axes fields + # Format: standardName: [synonym list] + table_axes_fields = { + 'axis': ['axis'], + 'bounds_values': ['bounds_values'], + 'climatology': ['climatology'], + 'convert_to': ['convert_to'], + 'coords_attrib': ['coords_attrib'], + 'formula': ['formula'], + 'id': ['id'], + 'index_only': ['index_only'], + 'long_name': ['long_name'], + 'must_call_cmor_grid': ['must_call_cmor_grid'], + 'must_have_bounds': ['must_have_bounds'], + 'out_name': ['out_name'], + 'positive': ['positive'], + 'requested': ['requested'], + 'requested_bounds': ['bounds_requested','requested_bounds'], + 'required': ['required'], + 'standard_name': ['standard_name'], + 'stored_direction': ['stored_direction'], + 'tolerance': ['tolerance'], + 'tol_on_requests': ['tol_on_requests'], + 'type': ['type'], + 'units': ['units'], + 'valid_max': ['valid_max'], + 'valid_min': ['valid_min'], + 'value': ['value'], + 'z_bounds_factors': ['z_bounds_factors'], + 'z_factors': ['z_factors'] + } + + # Standardized identifiers for table information fields + # Format: standardName: [synonym list] + table_fields = { + 'approx_interval': ['approx_interval'], + 'approx_interval_error': ['approx_interval_error'], + 'approx_interval_warning': ['approx_interval_warning'], + 'baseURL': ['baseURL'], + 'cf_version': ['cf_version'], + 'cmor_version': ['cmor_version'], + 'expt_id_ok': ['expt_id_ok'], + 'forcings': ['forcings'], + 'frequency': ['frequency'], + 'generic_levels': ['generic_levels'], + 'magic_number': ['magic_number'], + 'missing_value': ['missing_value'], + 'modeling_realm': ['modeling_realm'], + 'product': ['product'], + 'project_id': ['project_id'], + 'required_global_attributes': ['required_global_attributes'], + 'table_date': ['table_date'], + 'table_id': ['table_id'], + 'tracking_prefix': ['tracking_prefix'] + } + + # Set the type of table and return a dictionary that contains the read information. + if type == 'excel': + p = ParseExcel() + elif type == 'cmor': + p = ParseCmorTable() + elif type == 'xml': + p = ParseXML() + return p.parse_table(exp,table_name,table_var_fields,table_axes_fields,table_fields, + user_vars,user_axes,user_tableInfo) + +#============================================= +# Find the standardization key to use +#============================================= +def _get_key(key, table, user_table): + """ + Find the standardization key to use. + + Parameters: + key (str): Supplied field name to match to a standard key + table (dict): One of the three standardized field dictionaries. + user_table (dict): A user created dictionary. Keys should match a key already in + the table argument, values are the new field names used in the table. + + Returns: + k (str): The key to use that matches the standardization. Program will exit if no matching + key is found. + """ + for k,v in table.iteritems(): + if key in v: # Search for field name in standard dictionary. + return k + elif k in user_table.keys(): # Search for field name in user created dictionary. + if key in user_table[k]: + return k + # field name is not recognized. Exit the program with an error. + print('Error: ',key,' is not a valid field name at this time. Please define and resubmit.') + sys.exit(1) + + +#============================================= +# Parse Excel Spreadsheets that have been +# reformatted to tab deliminated fields. +#============================================= +class ParseExcel(object): + + def __init__(self): + super(ParseExcel, self).__init__() + + #============================================= + # Parse the tab deliminated table file + #============================================= + def parse_table(self,exp,table_name,table_var_fields,table_axes_fields,table_fields, + user_vars,user_axes,user_tableInfo): + """ + Function will parse a tab deliminated file and return a dictionary containing + the parsed fields. + + Parameter: + exp (str): The name of the experiment. + table_name (str): The full path to the table to be parsed. + table_var_fields (dict): Dictionary containing standardized field var names + as keys and acceptable synonyms as values. + table_axes_fields (dict): Dictionary containing standardized field axes names + as keys and acceptable synonyms as values. + table_fields (dict): Dictionary containing standardized table field names + as keys and acceptable synonyms as values. + user_vars (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + user_axes (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + user_tableInfo (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + + Returns: + table_dict (dict): A dictionary that holds all of the parsed table information. Contains the keys: + 'variables', 'axes', and 'table_info'. Each of these are dictionaries, keyed with variable names and + each variable has a value of a dictionary keyed with the standard field names. + """ + import csv + + table_dict = {} + + output_var_keys = table_var_fields['id'] + variables = {} + key = None + longest = 0 + found = False + fieldnames = [] + + # First time reading the file: Read the file and determine the longest line (first is recorded). + # This line is assumed to contain the field names. + f = open(table_name, 'rU') + reader = (f.read().splitlines()) + for row in reader: + r = row.split('\t') + i = 0 + l = len(r) + for g in r: + if g == '': + l = l-1 + i = i+1 + if l > longest: + fieldnames = list(r) + longest = l + + # Second time reading the file: Parse the file into a dictionary that uses the field + # names found in the first read as the keys. + reader = csv.DictReader(open(table_name, 'rU'), delimiter='\t',fieldnames=fieldnames) + + # Standardize the field names and store in a final dictionary. + for row in reader: + fields = {} + for k in row.keys(): + if k in output_var_keys: + key = row[k] + for k,v in row.iteritems(): + if v != '': + var_key = _get_key(k, table_var_fields, user_vars) + fields[var_key] = v + + variables[key] = fields + + table_dict['variables'] = variables + + return table_dict + + +#============================================= +# Parses standard CMOR table files +#============================================= +class ParseCmorTable(object): + + def _init_(): + super(ParseCmorTable, self).__init__() + + #============================================= + # Reset and get ready for a new variable set + #============================================= + def _reset(self, status, part, whole, name): + + whole[name] = part + part = {} + status = False + return status, part, whole + + #============================================= + # Setup for the new entry group found + #============================================= + def _new_entry(self, status, part, whole, name, key, value): + + if len(part.keys()) > 0: + whole[name] = part + part = {} + name = value + part['id'] = value + status = True + return status, part, whole, name + + #============================================= + # Parse a CMOR text file + #============================================= + def parse_table(self, exp,table_name,table_var_fields,table_axes_fields,table_fields, + user_vars,user_axes,user_tableInfo): + """ + Function will parse a CMOR table text file and return a dictionary containing + the parsed fields. + + Parameter: + exp (str): The name of the experiment. + table_name (str): The full path to the table to be parsed. + table_var_fields (dict): Dictionary containing standardized field var names + as keys and acceptable synonyms as values. + table_axes_fields (dict): Dictionary containing standardized field axes names + as keys and acceptable synonyms as values. + table_fields (dict): Dictionary containing standardized table field names + as keys and acceptable synonyms as values. + user_vars (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + user_axes (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + user_tableInfo (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + + Returns: + table_dict (dict): A dictionary that holds all of the parsed table information. Contains the keys: + 'variables', 'axes', and 'table_info'. Each of these are dictionaries, keyed with variable names and + each variable has a value of a dictionary keyed with the standard field names. + """ + + # Initialize needed dictionaries + table_dict = {} + table_info = {} + expt_id_ok = {} + axes = {} + variables = {} + subroutines = {} + mapping = {} + axis = {} + var = {} + sub = {} + map = {} + + # Status Variables + current_axis = None + current_var = None + current_sub = None + current_mapping = None + in_axis = False + in_var = False + in_sub = False + in_mapping = False + + # Open table file + mip_file = open(table_name) + + for l in mip_file: + # if comment - don't proceed + #print l + l_no_comment = l.split('!') + l = l_no_comment[0].strip() + if len(l) > 0 and ':' in l: + # remove anything after a comment character + # parse the key from the value + parts = l.split(':') + if len(parts) > 2: + parts[1] = ''.join(parts[1:]) + key = parts[0].strip() + value = parts[1].strip() + #print l,'->',parts + + # add to table_info dictionary + # Start an entry for 'expt_id_ok' + if 'expt_id_ok' in key: + equiv = value.split("\' \'") + if len(equiv) == 2: + expt_id_ok[equiv[0]] = equiv[1] + elif len(equiv) == 1: + expt_id_ok[equiv[0]] = None + # Start an entry for 'axis_entry' + elif 'axis_entry' in key: + if in_var == True: + in_var,var,variables = self.reset(in_var,var,variables,current_var) + if in_sub == True: + in_sub,sub,subroutines = self._reset(in_sub,sub,subroutines,current_sub) + if in_mapping == True: + in_mapping,map,maping = self._reset(in_mapping,map,mapping,current_mapping) + in_axis,axis,axes,current_axis = self._new_entry(in_axis,axis,axes,current_axis,key,value) + # Start an entry for 'variable_entry' + elif 'variable_entry' in key: + if in_axis == True: + in_axis,axis,axes = self._reset(in_axis,axis,axes,current_axis) + if in_sub == True: + in_sub,sub,subroutines = self._reset(in_sub,sub,subroutines,current_sub) + if in_mapping == True: + in_mapping,map,maping = self._reset(in_mapping,map,mapping,current_mapping) + in_var,var,variables,current_var = self._new_entry(in_var,var,variables,current_var,key,value) + # Start an entry for 'subroutine_entry' + elif 'subroutine_entry' in key: + if in_axis == True: + in_axis,axis,axes = self._reset(in_axis,axis,axes,current_axis) + if in_var == True: + in_var,var,variables = self._reset(in_var,var,variables,current_var) + if in_mapping == True: + in_mapping,map,maping = self._reset(in_mapping,map,mapping,current_mapping) + in_sub,sub,subroutines,current_sub = self._new_entry(in_sub,sub,subroutines,current_sub,key,value) + # Start an entry for 'mapping_entry' + elif 'mapping_entry' in key: + if in_axis == True: + in_axis,axis,axes = self._reset(in_axis,axis,axes,current_axis) + if in_var == True: + in_var,var,variables = self._reset(in_var,var,variables,current_var) + if in_sub == True: + in_sub,sub,subroutines = self._reset(in_sub,sub,subroutines,current_sub) + in_mapping,map,maping,current_mapping = self._new_entry(in_mapping,map,mapping,current_mapping,key,value) + # The new entry has been started. If this point has been reached, parse this line into the correct standardized + # field name under the current activated entry. + else: + if (in_axis): #field added to axes variable + axis_key = _get_key(key, table_axes_fields, user_axes) + axis[axis_key] = value + elif (in_var): #field added to variable + var_key = _get_key(key, table_var_fields, user_vars) + var[var_key] = value + elif (in_sub): #field added to subroutine + sub[key] = value + elif (in_mapping): #field added to mapping + map[key] = value + else: #field added to table information + mip_key = _get_key(key, table_fields, user_tableInfo) + table_info[mip_key] = value + + # Add final entry into its group dictionary + if in_var == True: + variables[current_var] = var + if in_axis == True: + axes[current_axis] = axis + if in_sub == True: + subroutines[current_sub] = sub + if in_mapping == True: + mapping[current_mapping] = map + + # Combine three separate dictionaries into the table summary dictionary + table_dict['variables'] = variables + table_dict['axes'] = axes + table_dict['subroutines'] = subroutines + table_dict['table_info'] = table_info + + return table_dict + +#============================================= +# Parse the XML format +#============================================= +class ParseXML(object): + + def _init_(): + super(ParseXML, self, miptable,table_var_fields,table_axes_fields,table_fields).__init__() + + #============================================= + # Parse the XML format using dreqPy + #============================================= + def parse_table(self,exp,miptable,table_var_fields,table_axes_fields,table_fields, + user_vars,user_axes,user_tableInfo): + """ + Function will parse an XML file using dreqPy and return a dictionary containing + the parsed fields. + + Parameter: + exp (str): The name of the experiment. + miptable (str): The name of the miptable. + table_var_fields (dict): Dictionary containing standardized field var names + as keys and acceptable synonyms as values. + table_axes_fields (dict): Dictionary containing standardized field axes names + as keys and acceptable synonyms as values. + table_fields (dict): Dictionary containing standardized table field names + as keys and acceptable synonyms as values. + user_vars (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + user_axes (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + user_tableInfo (dict): User defined dictionary. Keys should match standard name. + Values should be a list of acceptable field names. + + Returns: + table_dict (dict): A dictionary that holds all of the parsed table information. Contains the keys: + 'variables', 'axes', and 'table_info'. Each of these are dictionaries, keyed with variable names and + each variable has a value of a dictionary keyed with the standard field names. + """ + import dreq + + dq = dreq.loadDreq() + + # Get table id +# if len(g_id) == 0: +# print 'ERROR: Variable group/table ',table_name, ' not supported.' +# print 'Please select from the following: ' +# print dq.inx.requestVarGroup.label.keys() +# print '\nIf your selected table is listed, it may not be supported in this verison of dreqpy.' +# print '\nEXITING. \n' +# sys.exit(1) +# # Get the id's of the variables in this table +# g_vars = dq.inx.iref_by_sect[g_id[0]].a + + # Get a list of mips for the experiment + e_mip = [] + e_id = dq.inx.experiment.label[exp] + e_vars = dq.inx.iref_by_sect[e_id[0]].a + for v in e_vars['requestItem']: + e_mip.append(dq.inx.uid[v].mip) + + # Loop through the needed mips and for the selected tables, pull variable uids + total_request = {} + for i in dq.coll['requestLink'].items: + if i.mip in e_mip: + if 'requestItem' in dq.inx.iref_by_sect[i.uid].a: + ##print '\nRequest size: ', len(dq.inx.iref_by_sect[i.uid].a['requestItem']), i.mip, i.tab + for u in dq.inx.iref_by_sect[i.uid].a['requestItem']: + if '!' in dq.inx.uid[u].tab: + tab_split = dq.inx.uid[u].tab.split('!') + tab = tab_split[len(tab_split)-1] + else: + tab = dq.inx.uid[u].tab + print miptable, tab + if miptable == None or miptable == tab: + g_id = dq.inx.requestVarGroup.label[tab] + if len(g_id) > 0: + table_dict = {} + + variables = {} + axes = {} + table_info = {} + data = {} + + vars = dq.inx.iref_by_sect[g_id[0]].a + #print '# of vars: ',len(vars['requestVar']) + axes_list = [] + for v in vars['requestVar']: + var = {} + v_id = dq.inx.uid[v].vid # Get the CMORvar id + c_var = dq.inx.uid[v_id] + + # Set what we can from the CMORvar section + #var['comment']= c_var.comment + if hasattr(c_var,'deflate'): + var['deflate']= c_var.deflate + if hasattr(c_var,'deflate_level'): + var['deflate_level']= c_var.deflate_level + if hasattr(c_var,'description'): + var['description']= c_var.description + #var['flag_meanings']= c_var.flag_meanings + #var['flag_values']= c_var.flag_values + if hasattr(c_var,'frequency'): + var['frequency']= c_var.frequency + if hasattr(c_var,'label'): + var['id']= c_var.label + if hasattr(c_var,'modeling_realm'): + var['modeling_realm']= c_var.modeling_realm + if hasattr(c_var,'ok_min_mean_abs'): + var['ok_min_mean_abs']= c_var.ok_min_mean_abs + if hasattr(c_var,'ok_max_mean_abs'): + var['ok_max_mean_abs']= c_var.ok_max_mean_abs + if hasattr(c_var,'out_name'): + var['out_name']= c_var.label #? + if hasattr(c_var,'positive'): + var['positive']= c_var.positive + if hasattr(c_var,'prov'): + var['prov']= c_var.prov + if hasattr(c_var,'procNote'): + var['provcNote']= c_var.procNote + if hasattr(c_var,'shuffle'): + var['shuffle']= c_var.shuffle + if hasattr(c_var,'title'): + var['title']= c_var.title + if hasattr(c_var,'type'): + var['type']= c_var.type + if hasattr(c_var,'valid_max'): + if isinstance(c_var.valid_max, (int, long, float, complex)): + var['valid_max']= c_var.valid_max + if hasattr(c_var,'valid_min'): + if isinstance(c_var.valid_min, (int, long, float, complex)): + var['valid_min']= c_var.valid_min + + # Set what we can from the standard section + if hasattr(c_var,'stid'): + s_var = dq.inx.uid[c_var.stid] + if hasattr(s_var,'cell_measures'): + var['cell_measures']= s_var.cell_measures + if hasattr(s_var,'cell_methods'): + var['cell_methods']= s_var.cell_methods + + # Set what we can from the time section + if hasattr(s_var, 'tmid'): + t_var = dq.inx.uid[s_var.tmid] + if hasattr(t_var,'dimensions'): + t = t_var.dimensions + if t != '' and t != 'None': + var['time'] = t + var['dimensions'] = t+'|' + if hasattr(t_var,'label'): + var['time_label'] = t_var.label + if hasattr(t_var,'title'): + var['time_title'] = t_var.title + + # Set what we can from the spatial section + if hasattr(s_var, 'spid'): + sp_var = dq.inx.uid[s_var.spid] + if hasattr(sp_var,'dimensions'): + if 'dimensions' in var.keys(): + var['dimensions'] = var['dimensions']+sp_var.dimensions + else: + var['dimensions'] = sp_var.dimensions + #dims = sp_var.dimensions.split('|') + dims = var['dimensions'].split('|') + for d in dims: + if d not in axes_list and d != '' and d != 'None': + axes_list.append(d) + + # Set what we can from the variable section + v_var = dq.inx.uid[c_var.vid] + if hasattr(v_var,'cf_standard_name'): + var['cf_standard_name']= v_var.sn + if hasattr(v_var,'long_name'): + var['long_name']= v_var.sn + if hasattr(v_var,'units'): + if v_var.units == "": + var['units']= '1' + else: + var['units']= v_var.units + + #var['ext_cell_measures']= + #var['required']= + + # Add variable to variable dictionary + variables[c_var.label] = var + + for a in axes_list: + id = dq.inx.grids.label[a] + ax = {} + if len(id) > 0: + v = dq.inx.grids.uid[id[0]] + if hasattr(v,'units'): + if v.units == "": + ax['units'] = '1' + else: + ax['units'] = v.units + if hasattr(v,'axis'): + ax['axis'] = v.axis + if hasattr(v,'valid_max'): + if isinstance(v.valid_max, (int, long, float, complex)): + ax['valid_max'] = v.valid_max + if hasattr(v,'valid_min'): + if isinstance(v.valid_min, (int, long, float, complex)): + ax['valid_min'] = v.valid_min + if hasattr(v,'cf_standard_name'): + ax['cf_standard_name'] = v.standardName + if hasattr(v,'type'): + ax['type'] = v.type + if hasattr(v,'id'): + ax['id'] = v.label + if hasattr(v,'positive'): + ax['positive'] = v.positive + if hasattr(v,'title'): + ax['title'] = v.title + if hasattr(v,'bounds'): + ax['bounds'] = v.bounds + if hasattr(v,'requested'): + ax['requested'] = v.requested + if hasattr(v,'boundsValues'): + ax['boundsValues'] = v.boundsValues + if hasattr(v,'coords'): + ax['coords'] = v.coords + axes[a] = ax + + table_dict['variables'] = variables + table_dict['axes'] = axes + table_dict['table_info'] = table_info + + total_request[i.mip+'_'+tab] = table_dict + + return total_request + diff --git a/source/pyconform/parsing.py b/source/pyconform/parsing.py new file mode 100644 index 00000000..fcdd9a5b --- /dev/null +++ b/source/pyconform/parsing.py @@ -0,0 +1,170 @@ +""" +Parsing Module + +This module defines the necessary elements to parse a string variable definition +into the recognized elements that are used to construct an Operation Graph. + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from slicetuple import SliceTuple +from pyparsing import (nums, alphas, alphanums, oneOf, delimitedList, + operatorPrecedence, opAssoc, Word, Combine, Literal, + Forward, Suppress, Group, CaselessLiteral, Optional, + QuotedString) + +#=============================================================================== +# ParsedFunction +#=============================================================================== +class ParsedFunction(object): + """ + A parsed function string-type + """ + + def __init__(self, tokens): + token = tokens[0] + self.key = token[0] + self.args = tuple(token[1:]) + def __repr__(self): + return "<{0} {1}{2!r} ('{3}') at {4}>".format(self.__class__.__name__, + self.key, + self.args, + str(self), + hex(id(self))) + def __str__(self): + strargs = '({0})'.format(','.join(str(arg) for arg in self.args)) + return "{0}{1!s}".format(self.key, strargs) + def __eq__(self, other): + return ((type(self) == type(other)) and + (self.key == other.key) and + (self.args == other.args)) + + +#=============================================================================== +# ParsedUniOp - Unary Operator ParsedFunction +#=============================================================================== +class ParsedUniOp(ParsedFunction): + """ + A parsed unary-operator string-type + """ + def __str__(self): + return "({0}{1!s})".format(self.key, self.args[0]) + + +#=============================================================================== +# ParsedBinOp - Binary Operator ParsedFunction +#=============================================================================== +class ParsedBinOp(ParsedFunction): + """ + A parsed binary-operator string-type + """ + def __str__(self): + return "({0!s}{1}{2!s})".format(self.args[0], self.key, self.args[1]) + + +#=============================================================================== +# ParsedVariable - Variable ParsedFunction +#=============================================================================== +class ParsedVariable(ParsedFunction): + """ + A parsed variable string-type + """ + def __init__(self, tokens): + super(ParsedVariable, self).__init__(tokens) + self.args = SliceTuple(self.args) if len(self.args) > 0 else SliceTuple() + def __repr__(self): + return "<{0} {1}{2} ('{3}') at {4}>".format(self.__class__.__name__, + self.key, + self.args.index, + str(self), + hex(id(self))) + def __str__(self): + if str(self.args) == '(::)': + strargs = '' + else: + strargs = str(self.args).replace('(', '[').replace(')', ']') + return "{0}{1}".format(self.key, strargs) + + +#=============================================================================== +# DefinitionParser +#=============================================================================== + +# Negation operator +def _negop_(tokens): + op, val = tokens[0] + if op == '+': + return val + else: + return ParsedUniOp([[op, val]]) + +# Binary Operators +def _binop_(tokens): + left, op, right = tokens[0] + return ParsedBinOp([[op, left, right]]) + +# INTEGERS: Just any word consisting only of numbers +_INT_ = Word(nums) +_INT_.setParseAction(lambda t: int(t[0])) + +# FLOATS: More complicated... can be decimal format or exponential +# format or a combination of the two +_DEC_FLT_ = (Combine(Word(nums) + '.' + Word(nums)) | + Combine(Word(nums) + '.') | + Combine('.' + Word(nums))) +_EXP_FLT_ = (Combine(CaselessLiteral('e') + + Optional(oneOf('+ -')) + + Word(nums))) +_FLOAT_ = (Combine(Word(nums) + _EXP_FLT_) | + Combine(_DEC_FLT_ + Optional(_EXP_FLT_))) +_FLOAT_.setParseAction(lambda t: float(t[0])) + +# QUOTED STRINGS: Any words between quotations +_STR_ = QuotedString('"', escChar='\\') + +# String _NAME_s ...identifiers for function or variable _NAME_s +_NAME_ = Word(alphas + "_", alphanums + "_") + +# FUNCTIONS: Function arguments can be empty or any combination of +# ints, _FLOAT_, variables, and even other functions. Hence, +# we need a Forward place-holder to start... +_EXPR_PARSER_ = Forward() +_FUNC_ = Group(_NAME_ + (Suppress('(') + + Optional(delimitedList(_EXPR_PARSER_)) + + Suppress(')'))) +_FUNC_.setParseAction(ParsedFunction) + +# VARIABLE NAMES: Can be just string _NAME_s or _NAME_s with blocks +# of indices (e.g., [1,2,-4]) +_INDEX_ = Combine(Optional('-') + Word(nums)) +_INDEX_.setParseAction(lambda t: int(t[0])) +_ISLICE_ = _INDEX_ + Optional(Suppress(':') + _INDEX_ + + Optional(Suppress(':') + _INDEX_)) +_ISLICE_.setParseAction(lambda t: slice(*t) if len(t) > 1 else t[0]) +# variable = Group(_NAME_ + Optional(Suppress('[') + +# delimitedList(_ISLICE_ | +# _EXPR_PARSER_) + +# Suppress(']'))) +_VARIABLE_ = Group(_NAME_ + Optional(Suppress('[') + + delimitedList(_ISLICE_) + + Suppress(']'))) +_VARIABLE_.setParseAction(ParsedVariable) + +# Expression parser +_EXPR_PARSER_ << operatorPrecedence(_FLOAT_ | _INT_ | _STR_ | _FUNC_ | _VARIABLE_, + [(Literal('^'), 2, opAssoc.RIGHT, _binop_), + (oneOf('+ -'), 1, opAssoc.RIGHT, _negop_), + (oneOf('* /'), 2, opAssoc.RIGHT, _binop_), + (oneOf('+ -'), 2, opAssoc.RIGHT, _binop_)]) + +# Parse a string variable definition +def parse_definition(strexpr): + return _EXPR_PARSER_.parseString(strexpr)[0] + + +#=============================================================================== +# Command-Line Operation +#=============================================================================== +if __name__ == "__main__": + pass diff --git a/source/pyconform/slicetuple.py b/source/pyconform/slicetuple.py new file mode 100644 index 00000000..8d2a4bc3 --- /dev/null +++ b/source/pyconform/slicetuple.py @@ -0,0 +1,84 @@ +""" +SliceTuple Class + +The SliceTuple object represents an object that can act as an array index, as +array slice, or a tuple of array indices or slices. This provides one compact +object that can index or slice an array. + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from copy import copy + + +#=============================================================================== +# SliceTuple +#=============================================================================== +class SliceTuple(object): + """ + Container class that represents a tuple of array slices + """ + + def __init__(self, obj=None): + """ + Initializer + + Parameters: + obj (tuple, slice, int): Initializing object, can be a tuple of + slice or int objects, or a single slice or int object. If None, + defaults to slice(None) + """ + if obj is None: + self._idx = (slice(None),) + elif isinstance(obj, SliceTuple): + self._idx = copy(obj._idx) + elif isinstance(obj, (int, slice)): + self._idx = (obj,) + elif isinstance(obj, tuple): + if len(obj) == 0: + raise TypeError('Empty tuple cannot be a SliceTuple') + idx = [] + for i, o in enumerate(obj): + if isinstance(o, (int, slice)): + idx.append(o) + else: + raise TypeError(('Element {0} not an int or slice in tuple ' + '{1!r}').format(i, obj)) + self._idx = tuple(idx) + else: + raise TypeError(('Object not an int, slice, or tuple ' + '{0!r}').format(obj)) + + def __str__(self): + """ + Compact string representation of SliceTuple + """ + strreps = [] + for idx in self._idx: + if isinstance(idx, int): + strreps.append(str(idx)) + elif isinstance(idx, slice): + sss = idx.start, idx.stop, idx.step + strrep = ':'.join('' if s is None else str(s) for s in sss) + strreps.append(strrep) + return '({0})'.format(','.join(strreps)) + + def __eq__(self, other): + """ + Check if two SliceTuples are equal + """ + return self._idx == other._idx + + def __ne__(self, other): + """ + Check if two SliceTuples are not equal + """ + return not (self._idx == other._idx) + + @property + def index(self): + """ + Return the object that indexes into an array + """ + return self._idx diff --git a/source/pyconform/test/actiongraphsTests.py b/source/pyconform/test/actiongraphsTests.py new file mode 100644 index 00000000..8d5f7e91 --- /dev/null +++ b/source/pyconform/test/actiongraphsTests.py @@ -0,0 +1,440 @@ +""" +ActionGraph Unit Tests + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from os import linesep, remove +from os.path import exists +from pyconform import datasets, actiongraphs, actions +from netCDF4 import Dataset as NCDataset +from collections import OrderedDict +from numpy import testing as nptst + +import operator +import numpy +import unittest + + +#========================================================================= +# print_test_message - Helper function +#========================================================================= +def print_test_message(testname, indata=None, actual=None, expected=None): + """ + Pretty-print a test message + + Parameters: + testname: String name of the test + indata: Input data for testing (if any) + actual: Actual return value/result + expected: Expected return value/result + """ + indent = linesep + ' ' * 14 + print '{}:'.format(testname) + if indata is not None: + s_indata = str(indata).replace(linesep, indent) + print ' input: {}'.format(s_indata) + if actual is not None: + s_actual = str(actual).replace(linesep, indent) + print ' actual: {}'.format(s_actual) + if expected is not None: + s_expected = str(expected).replace(linesep, indent) + print ' expected: {}'.format(s_expected) + print + + +#========================================================================= +# ActionGraphTests - Tests for the actiongraphs.ActionGraph class +#========================================================================= +class ActionGraphTests(unittest.TestCase): + """ + Unit Tests for the actiongraphs.ActionGraph class + """ + + def setUp(self): + self.filenames = OrderedDict([('u1', 'u1.nc'), + ('u2', 'u2.nc'), + ('u3', 'u3.nc')]) + self._clear_input_() + + self.fattribs = OrderedDict([('a1', 'attribute 1'), + ('a2', 'attribute 2')]) + self.dims = OrderedDict([('time', 4), ('lat', 3), ('lon', 2)]) + self.vdims = OrderedDict([('u1', ('time', 'lat', 'lon')), + ('u2', ('time', 'lat', 'lon')), + ('u3', ('time', 'lat', 'lon'))]) + self.vattrs = OrderedDict([('lat', {'units': 'degrees_north', + 'standard_name': 'latitude'}), + ('lon', {'units': 'degrees_east', + 'standard_name': 'longitude'}), + ('time', {'units': 'days since 1979-01-01 0:0:0', + 'calendar': 'noleap', + 'standard_name': 'time'}), + ('u1', {'units': 'm', + 'standard_name': 'u variable 1'}), + ('u2', {'units': 'm', + 'standard_name': 'u variable 2'}), + ('u3', {'units': 'km', + 'standard_name': 'u variable 3'})]) + self.dtypes = {'lat': 'f', 'lon': 'f', 'time': 'f', + 'u1': 'd', 'u2': 'd', 'u3': 'd'} + ydat = numpy.linspace(-90, 90, num=self.dims['lat'], + endpoint=True, dtype=self.dtypes['lat']) + xdat = numpy.linspace(-180, 180, num=self.dims['lon'], + endpoint=False, dtype=self.dtypes['lon']) + tdat = numpy.linspace(0, self.dims['time'], num=self.dims['time'], + endpoint=False, dtype=self.dtypes['time']) + ulen = reduce(lambda x,y: x*y, self.dims.itervalues(), 1) + ushape = tuple(d for d in self.dims.itervalues()) + u1dat = numpy.linspace(0, ulen, num=ulen, endpoint=False, + dtype=self.dtypes['u1']).reshape(ushape) + u2dat = numpy.linspace(0, ulen, num=ulen, endpoint=False, + dtype=self.dtypes['u2']).reshape(ushape) + u3dat = numpy.linspace(0, ulen, num=ulen, endpoint=False, + dtype=self.dtypes['u3']).reshape(ushape) + self.vdat = {'lat': ydat, 'lon': xdat, 'time': tdat, + 'u1': u1dat, 'u2': u2dat, 'u3': u3dat} + + for vname, fname in self.filenames.iteritems(): + ncf = NCDataset(fname, 'w') + ncf.setncatts(self.fattribs) + ncvars = {} + for dname, dvalue in self.dims.iteritems(): + dsize = dvalue if dname!='time' else None + ncf.createDimension(dname, dsize) + ncvars[dname] = ncf.createVariable(dname, 'd', (dname,)) + ncvars[vname] = ncf.createVariable(vname, 'd', self.vdims[vname]) + for vnam, vobj in ncvars.iteritems(): + for aname, avalue in self.vattrs[vnam].iteritems(): + setattr(vobj, aname, avalue) + vobj[:] = self.vdat[vnam] + ncf.close() + + self.inpds = datasets.InputDataset('inpds', self.filenames.values()) + + def tearDown(self): + self._clear_input_() + + def _clear_input_(self): + for fname in self.filenames.itervalues(): + if exists(fname): + remove(fname) + + def test_init(self): + g = actiongraphs.ActionGraph() + actual = type(g) + expected = actiongraphs.ActionGraph + print_test_message('type(ActionGraph)', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'ActionGraph type not correct') + + def test_add_op(self): + g = actiongraphs.ActionGraph() + u1r = actions.Reader(self.filenames['u1'], 'u1') + g.add(u1r) + actual = g.vertices + expected = [u1r] + print_test_message('ActionGraph.add(Action)', + actual=actual, expected=expected) + self.assertListEqual(actual, expected, + 'ActionGraph did not add Action') + + def test_add_int(self): + g = actiongraphs.ActionGraph() + expected = TypeError + print_test_message('ActionGraph.add(int) TypeError', + expected=expected) + self.assertRaises(expected, g.add, 1) + + def test_call(self): + g = actiongraphs.ActionGraph() + u1read = actions.Reader(self.filenames['u1'], 'u1') + u2read = actions.Reader(self.filenames['u2'], 'u2') + u1plusu2 = actions.Evaluator('+', '(u1+u2)', operator.add, + signature=[None, None]) + g.connect(u1read, u1plusu2) + g.connect(u2read, u1plusu2) + actual = g(u1plusu2) + expected = self.vdat['u1'] + self.vdat['u2'] + print_test_message('ActionGraph.__call__()', + actual=actual, expected=expected) + nptst.assert_array_equal(actual, expected, + 'ActionGraph() failed') + + def test_print(self): + g = actiongraphs.ActionGraph() + u1read = actions.Reader(self.filenames['u1'], 'u1') + u2read = actions.Reader(self.filenames['u2'], 'u2') + u1plusu2 = actions.Evaluator('+', '(u1+u2)', operator.add, + signature=[None, None]) + vhandle = actions.Finalizer('V') + g.connect(u1read, u1plusu2) + g.connect(u2read, u1plusu2) + g.connect(u1plusu2, vhandle) + print g + + def test_handles(self): + g = actiongraphs.ActionGraph() + u1read = actions.Reader(self.filenames['u1'], 'u1') + u2read = actions.Reader(self.filenames['u2'], 'u2') + u1plusu2 = actions.Evaluator('+', '(u1+u2)', operator.add, + signature=[None, None]) + vhandle = actions.Finalizer('V') + g.connect(u1read, u1plusu2) + g.connect(u2read, u1plusu2) + g.connect(u1plusu2, vhandle) + actual = g.handles()[0] + expected = vhandle + print_test_message('ActionGraph.handles()', + actual=actual, expected=expected) + self.assertEqual(actual, expected, 'ActionGraph() failed') + + +#=============================================================================== +# GraphFillerTests +#=============================================================================== +class GraphFillerTests(unittest.TestCase): + """ + Unit Tests for the actiongraphs.GraphFiller class + """ + + def setUp(self): + self.filenames = OrderedDict([('u1', 'w1.nc'), + ('u2', 'w2.nc'), + ('u3', 'w3.nc')]) + self._clear_input_() + + self.fattribs = OrderedDict([('a1', 'attribute 1'), + ('a2', 'attribute 2')]) + self.dims = OrderedDict([('time', 4), ('lat', 3), ('lon', 2)]) + self.vdims = OrderedDict([('u1', ('time', 'lat', 'lon')), + ('u2', ('time', 'lat', 'lon')), + ('u3', ('time', 'lat', 'lon'))]) + self.vattrs = OrderedDict([('lat', {'units': 'degrees_north', + 'standard_name': 'latitude'}), + ('lon', {'units': 'degrees_east', + 'standard_name': 'longitude'}), + ('time', {'units': 'days since 1979-01-01 0:0:0', + 'calendar': 'noleap', + 'standard_name': 'time'}), + ('u1', {'units': 'm', + 'standard_name': 'u variable 1'}), + ('u2', {'units': 'ft', + 'standard_name': 'u variable 2'}), + ('u3', {'units': 'km', + 'standard_name': 'u variable 3'})]) + self.dtypes = {'lat': 'f', 'lon': 'f', 'time': 'f', + 'u1': 'd', 'u2': 'd', 'u3': 'd'} + ydat = numpy.linspace(-90, 90, num=self.dims['lat'], + endpoint=True, dtype=self.dtypes['lat']) + xdat = numpy.linspace(-180, 180, num=self.dims['lon'], + endpoint=False, dtype=self.dtypes['lon']) + tdat = numpy.linspace(0, self.dims['time'], num=self.dims['time'], + endpoint=False, dtype=self.dtypes['time']) + ulen = reduce(lambda x,y: x*y, self.dims.itervalues(), 1) + ushape = tuple(d for d in self.dims.itervalues()) + u1dat = numpy.linspace(0, ulen, num=ulen, endpoint=False, + dtype=self.dtypes['u1']).reshape(ushape) + u2dat = numpy.linspace(0, ulen, num=ulen, endpoint=False, + dtype=self.dtypes['u2']).reshape(ushape) + u3dat = numpy.linspace(0, ulen, num=ulen, endpoint=False, + dtype=self.dtypes['u3']).reshape(ushape) + self.vdat = {'lat': ydat, 'lon': xdat, 'time': tdat, + 'u1': u1dat, 'u2': u2dat, 'u3': u3dat} + + for vname, fname in self.filenames.iteritems(): + ncf = NCDataset(fname, 'w') + ncf.setncatts(self.fattribs) + ncvars = {} + for dname, dvalue in self.dims.iteritems(): + dsize = dvalue if dname!='time' else None + ncf.createDimension(dname, dsize) + ncvars[dname] = ncf.createVariable(dname, 'd', (dname,)) + ncvars[vname] = ncf.createVariable(vname, 'd', self.vdims[vname]) + for vnam, vobj in ncvars.iteritems(): + for aname, avalue in self.vattrs[vnam].iteritems(): + setattr(vobj, aname, avalue) + vobj[:] = self.vdat[vnam] + ncf.close() + + self.inpds = datasets.InputDataset('inpds', self.filenames.values()) + + self.dsdict = OrderedDict() + self.dsdict['attributes'] = self.fattribs + self.dsdict['variables'] = OrderedDict() + vdicts = self.dsdict['variables'] + + vdicts['X'] = OrderedDict() + vdicts['X']['datatype'] = 'float64' + vdicts['X']['dimensions'] = ('x',) + vdicts['X']['definition'] = 'lon' + vattribs = OrderedDict() + vattribs['standard_name'] = 'longitude' + vattribs['units'] = 'degrees_east' + vdicts['X']['attributes'] = vattribs + + vdicts['Y'] = OrderedDict() + vdicts['Y']['datatype'] = 'float64' + vdicts['Y']['dimensions'] = ('y',) + vdicts['Y']['definition'] = 'lat' + vattribs = OrderedDict() + vattribs['standard_name'] = 'latitude' + vattribs['units'] = 'degrees_north' + vdicts['Y']['attributes'] = vattribs + + vdicts['T'] = OrderedDict() + vdicts['T']['datatype'] = 'float64' + vdicts['T']['dimensions'] = ('t',) + vdicts['T']['definition'] = 'time' + vattribs = OrderedDict() + vattribs['standard_name'] = 'time' + vattribs['units'] = 'hours since 0001-01-01 00:00:00' + vattribs['calendar'] = 'noleap' + vdicts['T']['attributes'] = vattribs + + vdicts['V1'] = OrderedDict() + vdicts['V1']['datatype'] = 'float64' + vdicts['V1']['dimensions'] = ('t', 'x', 'y') + vdicts['V1']['definition'] = 'u1 + u2' + vdicts['V1']['filename'] = 'var1.nc' + vattribs = OrderedDict() + vattribs['standard_name'] = 'variable 1' + vattribs['units'] = 'm' + vdicts['V1']['attributes'] = vattribs + + vdicts['V2'] = OrderedDict() + vdicts['V2']['datatype'] = 'float64' + vdicts['V2']['dimensions'] = ('t', 'y', 'x') + vdicts['V2']['definition'] = 'u2 - u1' + vdicts['V2']['filename'] = 'var2.nc' + vattribs = OrderedDict() + vattribs['standard_name'] = 'variable 2' + vattribs['units'] = 'm' + vdicts['V2']['attributes'] = vattribs + + vdicts['V3'] = OrderedDict() + vdicts['V3']['datatype'] = 'float64' + vdicts['V3']['dimensions'] = ('t', 'y', 'x') + vdicts['V3']['definition'] = '(u2 + u1) / u3' + vdicts['V3']['filename'] = 'var3.nc' + vattribs = OrderedDict() + vattribs['standard_name'] = 'variable 2' + vattribs['units'] = '1' + vdicts['V3']['attributes'] = vattribs + + self.outds = datasets.OutputDataset('outds', self.dsdict) + + self.outfiles = dict((vname, vdict['filename']) + for vname, vdict in vdicts.iteritems() + if 'filename' in vdict) + self._clear_output_() + + def tearDown(self): + self._clear_input_() + self._clear_output_() + + def _clear_input_(self): + for fname in self.filenames.itervalues(): + if exists(fname): + remove(fname) + + def _clear_output_(self): + for fname in self.outfiles.itervalues(): + if exists(fname): + remove(fname) + + def test_init(self): + gfiller = actiongraphs.GraphFiller(self.inpds) + actual = type(gfiller) + expected = actiongraphs.GraphFiller + print_test_message('type(GraphFiller)', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'GraphFiller type not correct') + + def test_from_definitions(self): + print_test_message('GraphFiller.from_definitions()') + g = actiongraphs.ActionGraph() + gfiller = actiongraphs.GraphFiller(self.inpds) + gfiller.from_definitions(g, self.outds) + print g + + def test_from_definitions_components(self): + print_test_message('GraphFiller.from_definitions().components()') + g = actiongraphs.ActionGraph() + gfiller = actiongraphs.GraphFiller(self.inpds) + gfiller.from_definitions(g, self.outds) + glist = g.components() + for ig in glist: + print 'GRAPH:' + print ig + + def test_match_units(self): + print_test_message('GraphFiller.match_units()') + g = actiongraphs.ActionGraph() + gfiller = actiongraphs.GraphFiller(self.inpds) + gfiller.from_definitions(g, self.outds) + gfiller.match_units(g) + print g + + def test_match_units_reapply(self): + g = actiongraphs.ActionGraph() + gfiller = actiongraphs.GraphFiller(self.inpds) + gfiller.from_definitions(g, self.outds) + gfiller.match_units(g) + expected = str(g) + gfiller.match_units(g) + actual = str(g) + print_test_message('GraphFiller.match_units() Reapplied', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'GraphFiller.match_units() reapplied failed') + + def test_match_dimensions(self): + print_test_message('GraphFiller.match_dimensions()') + g = actiongraphs.ActionGraph() + gfiller = actiongraphs.GraphFiller(self.inpds) + gfiller.from_definitions(g, self.outds) + gfiller.match_dimensions(g) + print g + + def test_match_dimensions_reapply(self): + g = actiongraphs.ActionGraph() + gfiller = actiongraphs.GraphFiller(self.inpds) + gfiller.from_definitions(g, self.outds) + indata = str(g) + gfiller.match_dimensions(g) + expected = str(g) + gfiller.match_dimensions(g) + actual = str(g) + print_test_message('GraphFiller.match_dimensions() Reapplied', + indata=indata, actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'GraphFiller.match_dimensions() reapplied failed') + + def test_match_units_dimensions(self): + print_test_message('GraphFiller.match_dimensions()') + g = actiongraphs.ActionGraph() + gfiller = actiongraphs.GraphFiller(self.inpds) + gfiller.from_definitions(g, self.outds) + gfiller.match_units(g) + gfiller.match_dimensions(g) + print g + + def test_match_dimensions_units(self): + print_test_message('GraphFiller.match_dimensions()') + g = actiongraphs.ActionGraph() + gfiller = actiongraphs.GraphFiller(self.inpds) + gfiller.from_definitions(g, self.outds) + gfiller.match_dimensions(g) + gfiller.match_units(g) + print g + +#=============================================================================== +# Command-Line Execution +#=============================================================================== +if __name__ == "__main__": + #import sys;sys.argv = ['', 'Test.testName'] + unittest.main() diff --git a/source/pyconform/test/actionsParTests.py b/source/pyconform/test/actionsParTests.py new file mode 100644 index 00000000..06837b88 --- /dev/null +++ b/source/pyconform/test/actionsParTests.py @@ -0,0 +1,104 @@ +""" +Fundamental Actions for the Operation Graph Unit Tests - Parallel + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from pyconform import actions as acts +from os import linesep +from mpi4py import MPI + +import numpy as np +import unittest + + +#=============================================================================== +# General Functions +#=============================================================================== +def print_test_message(testname, actual, expected): + rank = MPI.COMM_WORLD.Get_rank() + size = MPI.COMM_WORLD.Get_size() + msg = '[{}/{}] {}:{}'.format(rank, size, testname, linesep) + msg += ' - actual = {}'.format(actual).replace(linesep, ' ') + linesep + msg += ' - expected = {}'.format(expected).replace(linesep, ' ') + linesep + print msg + + +#=============================================================================== +# MPISendRecvTests +#=============================================================================== +class MPISendRecvTests(unittest.TestCase): + """ + Unit tests for the operators.MPISender and operators.MPIReceiver classes + """ + + def setUp(self): + self.params = [np.arange(2*3, dtype=np.float64).reshape((2,3)), + np.arange(2*3, dtype=np.float64).reshape((2,3)) + 10.] + + def tearDown(self): + pass + + def test_sendop_init(self): + testname = 'MPISender.__init__(1)' + SO = acts.MPISender(1) + actual = type(SO) + expected = acts.MPISender + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + + def test_recvop_init(self): + testname = 'MPIReceiver.__init__(1)' + RO = acts.MPIReceiver(1) + actual = type(RO) + expected = acts.MPIReceiver + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + + def test_send_recv(self): + testname = 'MPISender() -> MPIReceiver()' + data = np.arange(0,10) + if MPI.COMM_WORLD.Get_rank() == 0: + op = acts.MPISender(dest=1) + actual = op(data) + expected = None + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + elif MPI.COMM_WORLD.Get_rank() == 1: + op = acts.MPIReceiver(source=0) + actual = op() + expected = data + print_test_message(testname, actual, expected) + np.testing.assert_equal(actual, expected, '{} failed'.format(testname)) + else: + actual = None + expected = None + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + + +#=============================================================================== +# Command-Line Operation - For parallel tests only! +#=============================================================================== +if __name__ == "__main__": + hline = '=' * 70 + if MPI.COMM_WORLD.Get_rank() == 0: + print hline + print 'STANDARD OUTPUT FROM ALL TESTS:' + print hline + MPI.COMM_WORLD.Barrier() + + from cStringIO import StringIO + mystream = StringIO() + tests = unittest.TestLoader().loadTestsFromTestCase(MPISendRecvTests) + unittest.TextTestRunner(stream=mystream).run(tests) + MPI.COMM_WORLD.Barrier() + + results = MPI.COMM_WORLD.gather(mystream.getvalue()) + if MPI.COMM_WORLD.Get_rank() == 0: + for rank, result in enumerate(results): + print hline + print 'TESTS RESULTS FOR RANK ' + str(rank) + ':' + print hline + print str(result) diff --git a/source/pyconform/test/actionsSerTests.py b/source/pyconform/test/actionsSerTests.py new file mode 100644 index 00000000..3f7cfcb2 --- /dev/null +++ b/source/pyconform/test/actionsSerTests.py @@ -0,0 +1,500 @@ +""" +Fundamental Actions for the Action Graph Unit Tests + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from os import remove +from os.path import exists +from pyconform import actions as acts +from os import linesep +from cf_units import Unit + +import operator +import numpy as np +import numpy.testing as npt +import netCDF4 as nc +import unittest + + +#=============================================================================== +# General Functions +#=============================================================================== +def print_test_message(testname, actual, expected): + print '{}:'.format(testname) + print ' - actual = {}'.format(actual).replace(linesep, ' ') + print ' - expected = {}'.format(expected).replace(linesep, ' ') + print + + +#=============================================================================== +# ActionTests +#=============================================================================== +class MockAction(acts.Action): + def __init__(self, key, name): + super(MockAction, self).__init__(key, name) + def __call__(self): + super(MockAction, self).__call__() + + +class ActionTests(unittest.TestCase): + """ + Unit tests for the operators.Action class + """ + def setUp(self): + acts.Action._id_ = 0 + + def test_abc(self): + opname = 'xop' + testname = 'Action.__init__()' + self.assertRaises(TypeError, acts.Action, opname) + print_test_message(testname, TypeError, TypeError) + + def test_init(self): + opname = 'xop' + testname = 'Mock Action.__init__()' + O = MockAction(0, opname) + actual = isinstance(O, acts.Action) + expected = True + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + 'Could not create mock Action object') + + def test_name(self): + opname = 'xop' + testname = 'Mock Action.__init__({!r})'.format(opname) + O = MockAction(0, opname) + actual = O.name + expected = opname + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + 'Action name incorrect') + + def test_str(self): + opname = 'xop' + testname = 'Mock Action.__str__()' + O = MockAction('0', opname) + actual = str(O) + expected = opname + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + 'Action string conversion incorrect') + + def test_units_default(self): + opname = 'xop' + testname = 'Mock Action.units' + O = MockAction(0, opname) + actual = O.units + expected = None + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + 'Action units incorrect') + + def test_units_from_Unit(self): + opname = 'xop' + testname = 'Mock Action.units = Unit(m)' + O = MockAction(0, opname) + O.units = Unit('m') + actual = O.units + expected = Unit('m') + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + 'Action units incorrect') + + def test_units_from_str(self): + opname = 'xop' + testname = 'Mock Action.units = m' + O = MockAction(0, opname) + O.units = 'm' + actual = O.units + expected = Unit('m') + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + 'Action units incorrect') + + def test_units_from_tuple(self): + opname = 'xop' + testname = 'Mock Action.units = (days, standard)' + O = MockAction(0, opname) + O.units = ('days from 0001-01-01 00:00:00', 'standard') + actual = O.units + expected = Unit('days from 0001-01-01 00:00:00', calendar='standard') + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + 'Action units incorrect') + + def test_units_bad_unit(self): + opname = 'xop' + testname = 'Mock Action.units = ncxedajbec' + O = MockAction(0, opname) + expected = ValueError + try: + O.units = 'ncxedajbec' + except ValueError: + actual = ValueError + else: + actual = None + self.assertTrue(False, 'Action units did not fail') + print_test_message(testname, actual, expected) + + def test_units_bad_calendar(self): + opname = 'xop' + testname = 'Mock Action.units = (days, ncxedajbec)' + O = MockAction(0, opname) + expected = ValueError + try: + O.units = ('days since 0001-01-01 00:00:00', 'ncxedajbec') + except ValueError: + actual = ValueError + else: + actual = None + self.assertTrue(False, 'Action units did not fail') + print_test_message(testname, actual, expected) + + def test_dimensions_default(self): + opname = 'xop' + testname = 'Mock Action.dimensions' + O = MockAction(0, opname) + actual = O.dimensions + expected = None + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + 'Action dimensions incorrect') + + def test_dimensions_from_tuple(self): + opname = 'xop' + testname = 'Mock Action.dimensions = Unit(m)'.format(opname) + indata = ('t', 'x') + O = MockAction(0, opname) + O.dimensions = indata + actual = O.dimensions + expected = indata + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + 'Action dimensions incorrect') + + def test_equal_same(self): + nm = 'xop' + testname = ('Mock Action({!r}) == Action(' + '{!r})').format(nm, nm) + O1 = MockAction(1, nm) + O2 = MockAction(1, nm) + actual = (O1 == O2) + expected = False + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + 'Action equality not correct') + + def test_equal_diff_names(self): + nm1 = 'xop' + nm2 = 'yop' + testname = ('Mock Action({!r}) == Action(' + '{!r})').format(nm1, nm2) + O1 = MockAction(1, nm1) + O2 = MockAction(1, nm2) + actual = (O1 == O2) + expected = False + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + 'Action equality not correct') + + def test_equal_diff_keys(self): + nm = 'xop' + testname = ('Mock Action(1, {!r}) == Action(2, ' + '{!r})').format(nm, nm) + O1 = MockAction(1, nm) + O2 = MockAction(2, nm) + actual = (O1 == O2) + expected = False + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + 'Action equality not correct') + + +#=============================================================================== +# ReaderTests +#=============================================================================== +class ReaderTests(unittest.TestCase): + """ + Unit tests for the operators.Reader class + """ + + def setUp(self): + self.ncfile = 'vslicetest.nc' + self.shape = (2,4) + self.size = reduce(lambda x,y: x*y, self.shape, 1) + dataset = nc.Dataset(self.ncfile, 'w') + dataset.createDimension('x', self.shape[0]) + dataset.createDimension('t') + dataset.createVariable('x', 'd', ('x',)) + dataset.variables['x'][:] = np.arange(self.shape[0]) + dataset.createVariable('t', 'd', ('t',)) + dataset.variables['t'][:] = np.arange(self.shape[1]) + self.var = 'v' + dataset.createVariable(self.var, 'd', ('x', 't')) + self.vardata = np.arange(self.size, dtype=np.float64).reshape(self.shape) + dataset.variables[self.var][:] = self.vardata + dataset.close() + self.slice = (slice(0, 1), slice(1, 3)) + + def tearDown(self): + if exists(self.ncfile): + remove(self.ncfile) + + def test_init(self): + testname = 'Reader.__init__()' + VSR = acts.Reader(self.ncfile, self.var) + actual = type(VSR) + expected = acts.Reader + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + + def test_init_filename_failure(self): + testname = 'Reader.__init__(bad filename)' + actual = OSError + expected = OSError + self.assertRaises(OSError, + acts.Reader, 'badname.nc', self.var) + print_test_message(testname, actual, expected) + + def test_init_varname_failure(self): + testname = 'Reader.__init__(bad variable name)' + actual = OSError + expected = OSError + self.assertRaises(OSError, + acts.Reader, self.ncfile, 'badvar') + print_test_message(testname, actual, expected) + + def test_init_with_slice(self): + testname = 'Reader.__init__(slice)' + VSR = acts.Reader(self.ncfile, self.var, self.slice) + actual = type(VSR) + expected = acts.Reader + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + + def test_call(self): + testname = 'Reader().__call__()' + VSR = acts.Reader(self.ncfile, self.var) + actual = VSR() + expected = self.vardata + print_test_message(testname, actual, expected) + npt.assert_array_equal(actual, expected, + '{} failed'.format(testname)) + + def test_call_slice(self): + testname = 'Reader(slice).__call__()' + VSR = acts.Reader(self.ncfile, self.var, self.slice) + actual = VSR() + expected = self.vardata[self.slice] + print_test_message(testname, actual, expected) + npt.assert_array_equal(actual, expected, + '{} failed'.format(testname)) + + def test_equal(self): + testname = 'Reader() == Reader()' + VSR1 = acts.Reader(self.ncfile, self.var, self.slice) + VSR2 = acts.Reader(self.ncfile, self.var, self.slice) + actual = VSR1 == VSR2 + expected = False + print_test_message(testname, actual, expected) + self.assertFalse(actual, '{} failed'.format(testname)) + + +#=============================================================================== +# EvaluatorTests +#=============================================================================== +class EvaluatorTests(unittest.TestCase): + """ + Unit tests for the operators.Evaluator class + """ + + def setUp(self): + self.params = [np.arange(2*3, dtype=np.float64).reshape((2,3)), + np.arange(2*3, dtype=np.float64).reshape((2,3)) + 10.] + + def tearDown(self): + pass + + def test_init(self): + opname = '1' + testname = 'Evaluator.__init__(function)' + FE = acts.Evaluator('1', opname, lambda: 1) + actual = type(FE) + expected = acts.Evaluator + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + + def test_init_fail(self): + opname = 'int(1)' + testname = 'Evaluator.__init__(non-function)' + self.assertRaises(TypeError, acts.Evaluator, 1, opname, 1) + actual = TypeError + expected = TypeError + print_test_message(testname, actual, expected) + + def test_unity(self): + opname = 'identity' + testname = 'Evaluator(lambda x: x).__call__(x)' + FE = acts.Evaluator('I', opname, lambda x: x) + actual = FE(self.params[0]) + expected = self.params[0] + print_test_message(testname, actual, expected) + npt.assert_array_equal(actual, expected, '{} failed'.format(testname)) + + def test_add(self): + opname = 'add(a,b)' + testname = 'Evaluator(add).__call__(a, b)' + FE = acts.Evaluator('+', opname, operator.add) + actual = FE(*self.params) + expected = operator.add(*self.params) + print_test_message(testname, actual, expected) + npt.assert_array_equal(actual, expected, '{} failed'.format(testname)) + + def test_add_constant_1st(self): + opname = 'add(1,a)' + testname = 'Evaluator(add, 1).__call__(a)' + FE = acts.Evaluator('+', opname, operator.add, signature=[1]) + actual = FE(self.params[0]) + expected = operator.add(1, self.params[0]) + print_test_message(testname, actual, expected) + npt.assert_array_equal(actual, expected, '{} failed'.format(testname)) + + def test_add_constant_2nd(self): + opname = 'add(a,2)' + testname = 'Evaluator(add, None, 2).__call__(a)' + FE = acts.Evaluator('+', opname, operator.add, signature=[None, 2]) + actual = FE(self.params[0]) + expected = operator.add(self.params[0], 2) + print_test_message(testname, actual, expected) + npt.assert_array_equal(actual, expected, '{} failed'.format(testname)) + + def test_sub(self): + opname = 'sub(a,b)' + testname = 'Evaluator(sub).__call__(a, b)' + FE = acts.Evaluator('-', opname, operator.sub) + actual = FE(*self.params) + expected = operator.sub(*self.params) + print_test_message(testname, actual, expected) + npt.assert_array_equal(actual, expected, '{} failed'.format(testname)) + + def test_equal(self): + opname = 'sub(a,b)' + testname = 'Evaluator() == Evaluator()' + FE1 = acts.Evaluator('-', opname, operator.sub) + FE2 = acts.Evaluator('-', opname, operator.sub) + actual = FE1 == FE2 + expected = False + print_test_message(testname, actual, expected) + npt.assert_array_equal(actual, expected, + '{} failed'.format(testname)) + + +#=============================================================================== +# FinalizerTests +#=============================================================================== +class FinalizerTests(unittest.TestCase): + """ + Unit tests for the operators.Finalizer class + """ + + def setUp(self): + self.ncfile = 'vslicetest.nc' + self.shape = (2,4) + self.size = reduce(lambda x,y: x*y, self.shape, 1) + dataset = nc.Dataset(self.ncfile, 'w') + dataset.createDimension('x', self.shape[0]) + dataset.createDimension('t') + dataset.createVariable('x', 'd', ('x',)) + dataset.variables['x'][:] = np.arange(self.shape[0]) + dataset.createVariable('t', 'd', ('t',)) + dataset.variables['t'][:] = np.arange(self.shape[1]) + self.var = 'v' + dataset.createVariable(self.var, 'd', ('x', 't')) + self.vardata = np.arange(self.size, dtype=np.float64).reshape(self.shape) + dataset.variables[self.var][:] = self.vardata + dataset.close() + self.slice = (slice(0, 1), slice(1, 3)) + self.data = np.array([1,2,3,4,5], dtype='float32') + + def tearDown(self): + if exists(self.ncfile): + remove(self.ncfile) + + def test_init(self): + testname = 'Finalizer.__init__()' + H = acts.Finalizer('x') + actual = type(H) + expected = acts.Finalizer + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + + def test_init_data(self): + testname = 'Finalizer.__init__(data)' + H = acts.Finalizer('x', data=self.data) + actual = type(H) + expected = acts.Finalizer + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + + def test_call_data(self): + testname = 'Finalizer.__call__(data)' + H = acts.Finalizer('x', data=self.data) + actual = H() + expected = self.data + print_test_message(testname, actual, expected) + npt.assert_equal(actual, expected, '{} failed'.format(testname)) + + def test_call_data_failure(self): + testname = 'Finalizer.__call__(data)' + H = acts.Finalizer('x', data=self.data) + indata = np.array([1,2,3,4,8,6,4], dtype=np.float64) + actual = '???' + expected = ValueError + print_test_message(testname, actual, expected) + self.assertRaises(expected, H, indata) + + def test_min_ok(self): + indata = 1.0 + testname = 'Finalizer({})'.format(indata) + H = acts.Finalizer('x', minimum=0.0) + actual = H(indata) + expected = indata + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + + def test_min_warn(self): + indata = -1.0 + testname = 'Finalizer({})'.format(indata) + H = acts.Finalizer('x', minimum=0.0) + actual = H(indata) + expected = indata + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + + def test_max_ok(self): + indata = 1.0 + testname = 'Finalizer({})'.format(indata) + H = acts.Finalizer('x', maximum=10.0) + actual = H(indata) + expected = indata + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + + def test_max_warn(self): + indata = 11.0 + testname = 'Finalizer({})'.format(indata) + H = acts.Finalizer('x', maximum=10.0) + actual = H(indata) + expected = indata + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + + +#=============================================================================== +# Command-Line Action +#=============================================================================== +if __name__ == "__main__": + #import sys;sys.argv = ['', 'Test.testName'] + unittest.main() diff --git a/source/pyconform/test/climIOTests.py b/source/pyconform/test/climIOTests.py index bcd642fc..d979adb3 100644 --- a/source/pyconform/test/climIOTests.py +++ b/source/pyconform/test/climIOTests.py @@ -7,36 +7,35 @@ import unittest -import sys from glob import glob from cStringIO import StringIO from os import linesep as eol from os import remove -from os.path import exists import Nio import netCDF4 import numpy as np -import climIO +from pyconform import climIO +#=============================================================================== +# climIOTests +#=============================================================================== class climIOTests(unittest.TestCase): def setUp(self): # Init the I/O ports - self.io_ports = {'Nio':climIO.init_climIO(override='Nio'),'netCDF4':climIO.init_climIO(override='netCDF4')} + self.io_ports = {'Nio':climIO.init_climIO(override='Nio'), + 'netCDF4':climIO.init_climIO(override='netCDF4')} # Test Data Generation self._clean_directory() - #self.nlat = 19 - #self.nlon = 36 - #self.ntime = 10 - self.nlat = 4 - self.nlon = 8 - self.ntime = 1 - self.slices = 'input.nc' + nlat = 4 + nlon = 8 + ntime = 1 + self.slice = 'input.nc' self.scalars = ['scalar{}'.format(i) for i in xrange(2)] self.timvars = ['tim{}'.format(i) for i in xrange(2)] self.tvmvars = ['tvm{}'.format(i) for i in xrange(2)] @@ -44,42 +43,40 @@ def setUp(self): self.fattrs = {'attr1': 'attribute one', 'attr2': 'attribute two'} - # Open the file for writing - fname = self.slices - self.fobj = Nio.open_file(fname, 'w') + fobj = Nio.open_file(self.slice, 'w') # Write attributes to file for name, value in self.fattrs.iteritems(): - setattr(self.fobj, name, value) + setattr(fobj, name, value) # Create the dimensions in the file - self.fobj.create_dimension('lat', self.nlat) - self.fobj.create_dimension('lon', self.nlon) - self.fobj.create_dimension('time', None) + fobj.create_dimension('lat', nlat) + fobj.create_dimension('lon', nlon) + fobj.create_dimension('time', None) # Create the coordinate variables & add attributes - lat = self.fobj.create_variable('lat', 'f', ('lat',)) - lon = self.fobj.create_variable('lon', 'f', ('lon',)) - time = self.fobj.create_variable('time', 'f', ('time',)) + lat = fobj.create_variable('lat', 'f', ('lat',)) + lon = fobj.create_variable('lon', 'f', ('lon',)) + time = fobj.create_variable('time', 'f', ('time',)) # Set the coordinate variable attributes setattr(lat, 'long_name', 'latitude') setattr(lon, 'long_name', 'longitude') setattr(time, 'long_name', 'time') - setattr(lat, 'units', 'degrees north') - setattr(lon, 'units', 'degrees east') + setattr(lat, 'units', 'degrees_north') + setattr(lon, 'units', 'degrees_east') setattr(time, 'units', 'days from 01-01-0001') # Set the values of the coordinate variables - lat[:] = np.linspace(-90, 90, self.nlat).astype(np.float32)#, dtype=np.float32) - lon[:] = np.linspace(-180, 180, self.nlon, endpoint=False).astype(np.float32)#, dtype=np.float32) - time[:] = np.arange(i * self.ntime, (i + 1) * self.ntime, dtype=np.float32) + lat[:] = np.linspace(-90, 90, nlat).astype(np.float32) + lon[:] = np.linspace(-180, 180, nlon, endpoint=False).astype(np.float32) + time[:] = np.arange(i * ntime, (i + 1) * ntime, dtype=np.float32) # Create the scalar variables for n in xrange(len(self.scalars)): vname = self.scalars[n] - v = self.fobj.create_variable(vname, 'd', ()) + v = fobj.create_variable(vname, 'd', ()) setattr(v, 'long_name', 'scalar{}'.format(n)) setattr(v, 'units', '[{}]'.format(vname)) v.assign_value(np.float64(n * 10)) @@ -87,26 +84,29 @@ def setUp(self): # Create the time-invariant metadata variables for n in xrange(len(self.timvars)): vname = self.timvars[n] - v = self.fobj.create_variable(vname, 'd', ('lat', 'lon')) + v = fobj.create_variable(vname, 'd', ('lat', 'lon')) setattr(v, 'long_name', 'time-invariant metadata {}'.format(n)) setattr(v, 'units', '[{}]'.format(vname)) - v[:] = np.ones((self.nlat, self.nlon), dtype=np.float64) * n + v[:] = np.ones((nlat, nlon), dtype=np.float64) * n # Create the time-variant metadata variables for n in xrange(len(self.tvmvars)): vname = self.tvmvars[n] - v = self.fobj.create_variable(vname, 'd', ('time', 'lat', 'lon')) + v = fobj.create_variable(vname, 'd', ('time', 'lat', 'lon')) setattr(v, 'long_name', 'time-variant metadata {}'.format(n)) setattr(v, 'units', '[{}]'.format(vname)) - v[:] = np.ones((self.ntime, self.nlat, self.nlon), dtype=np.float64) * n + v[:] = np.ones((ntime, nlat, nlon), dtype=np.float64) * n # Create the time-series variable vname = self.tsvar - self.var = self.fobj.create_variable(vname, 'd', ('time', 'lat', 'lon')) - setattr(self.var, 'long_name', 'time-series variable {}'.format(n)) - setattr(self.var, 'units', '[{}]'.format(vname)) - self.tsvalues = np.ones((self.ntime, self.nlat, self.nlon), dtype=np.float64) - self.var[:] = self.tsvalues + var = fobj.create_variable(vname, 'd', ('time', 'lat', 'lon')) + setattr(var, 'long_name', 'time-series variable {}'.format(n)) + setattr(var, 'units', '[{}]'.format(vname)) + self.tsvalues = np.ones((ntime, nlat, nlon), dtype=np.float64) + var[:] = self.tsvalues + + # Close the file + fobj.close() def tearDown(self): self._clean_directory() @@ -129,36 +129,45 @@ def _assertion(self, name, actual, expected, else: self.assertEqual(actual, expected, msg) - def test_file_operations(self): - # Loop over each IO port and test all function calls (all ports have the same functions/arguments) + def test_open_file(self): + # Loop over each IO port and test all function calls + # (all ports have the same functions/arguments) for n,p in self.io_ports.items(): # Test open_file - f = p.open_file(self.slices) + f = p.open_file(self.slice) if (n == 'Nio'): - #self.assertIsInstance(f, nio._NioFile) - self.assertNotEqual(f, None,"{0}: open_file".format(n)) + self.assertNotEqual(f, None, + "{0}: open_file".format(n)) elif (n == 'netCDF4'): - self.assertIsInstance(f, netCDF4.Dataset, "{0}: open_file".format(n)) + self.assertIsInstance(f, netCDF4.Dataset, + "{0}: open_file".format(n)) + # Test read_slice test_vals = p.read_slice(f, self.tsvar, index=0, all_values=False) - self.assertTrue((test_vals==self.tsvalues[0]).all(), "{0}: read_slice,index".format(n)) + self.assertTrue((test_vals==self.tsvalues[0]).all(), + "{0}: read_slice,index".format(n)) test_vals = p.read_slice(f, self.tsvar, index=-99, all_values=True) - self.assertTrue((test_vals==self.tsvalues[:]).all(), "{0}: read_slice,all".format(n)) + self.assertTrue((test_vals==self.tsvalues[:]).all(), + "{0}: read_slice,all".format(n)) # Test create_file new_f = p.create_file('new.nc', 'netcdf4c') - self.assertNotEqual(type(new_f), None, "{0}: create_file,netcdf4c".format(n)) + self.assertNotEqual(type(new_f), None, + "{0}: create_file,netcdf4c".format(n)) remove('new.nc') new_f = p.create_file('new.nc', 'netcdf4') - self.assertNotEqual(type(new_f), None, "{0}: create_file,netcdf4".format(n)) + self.assertNotEqual(type(new_f), None, + "{0}: create_file,netcdf4".format(n)) remove('new.nc') new_f = p.create_file('new.nc', 'netcdf') - self.assertNotEqual(type(new_f), None, "{0}: create_file,netcdf".format(n)) + self.assertNotEqual(type(new_f), None, + "{0}: create_file,netcdf".format(n)) remove('new.nc') new_f = p.create_file('new.nc', 'netcdfLarge') - self.assertNotEqual(type(new_f), None, "{0}: create_file,netcdfLarge".format(n)) + self.assertNotEqual(type(new_f), None, + "{0}: create_file,netcdfLarge".format(n)) # Test get_var_info typeCode,dimnames,attr = p.get_var_info(f, self.tsvar) @@ -166,41 +175,53 @@ def test_file_operations(self): att_k = attr.keys() else: att_k = attr - self.assertEqual(typeCode, 'd', "{0}: get_var_info,typecode".format(n)) - self.assertEqual(dimnames, ['time', 'lat', 'lon'], "{0}: get_var_info,dimensions".format(n)) + self.assertEqual(typeCode, 'd', + "{0}: get_var_info,typecode".format(n)) + self.assertEqual(dimnames, ['time', 'lat', 'lon'], + "{0}: get_var_info,dimensions".format(n)) for k in att_k: - self.assertIn(k, ['units', 'long_name'],"{0}: get_var_info,attributes".format(n)) + self.assertIn(k, ['units', 'long_name'], + "{0}: get_var_info,attributes".format(n)) # Test define_file - all_vars,new_f = p.define_file(new_f, self.tsvar, self.timvars, self.slices, self.tsvar) + all_vars,new_f = p.define_file(new_f, self.tsvar, self.timvars, + self.slice, self.tsvar) if (n == 'Nio'): - self.assertNotEqual(new_f, None, "{0}: define_file,file".format(n)) + self.assertNotEqual(new_f, None, + "{0}: define_file,file".format(n)) elif (n == 'netCDF4'): - self.assertIsInstance(new_f, netCDF4.Dataset, "{0}: define_file,file".format(n)) + self.assertIsInstance(new_f, netCDF4.Dataset, + "{0}: define_file,file".format(n)) for k in all_vars.keys(): - self.assertIn(k, ['tsvar','tim0','tim1'], "{0}: define_file,varibales,{1}".format(n,k)) + self.assertIn(k, ['tsvar','tim0','tim1'], + "{0}: define_file,varibales,{1}".format(n,k)) # Test create_var new_v = p.create_var(new_f, self.tsvar+'2', typeCode, dimnames, attr) if (n == 'Nio'): - #self.assertIsInstance(f, nio._NioVariable) self.assertNotEqual(new_v, None,"{0}: create_var".format(n)) elif (n == 'netCDF4'): - self.assertIsInstance(new_v, netCDF4.Variable, "{0}: create_var".format(n)) + self.assertIsInstance(new_v, netCDF4.Variable, + "{0}: create_var".format(n)) # Test write_var p.write_var(all_vars, self.tsvalues[:], self.tsvar) - self.assertTrue((all_vars[self.tsvar]==self.tsvalues[0]).all(), "{0}: write_var,index 0".format(n)) + self.assertTrue((all_vars[self.tsvar]==self.tsvalues[0]).all(), + "{0}: write_var,index 0".format(n)) p.write_var(all_vars, self.tsvalues[:], self.tsvar, index=1) - self.assertTrue((all_vars[self.tsvar][1]==self.tsvalues[0]).all(), "{0}: write_var,index 1".format(n)) - + self.assertTrue((all_vars[self.tsvar][1]==self.tsvalues[0]).all(), + "{0}: write_var,index 1".format(n)) def test_close(self): - for n,p in self.io_ports.items(): - f = p.open_file(self.slices) + for p in self.io_ports.itervalues(): + f = p.open_file(self.slice) f = p.close_file(f) + +#=============================================================================== +# Command-Line Operation +#=============================================================================== if __name__ == "__main__": hline = '=' * 70 print hline diff --git a/source/pyconform/test/conformTests.py b/source/pyconform/test/conformTests.py new file mode 100644 index 00000000..b40c9d31 --- /dev/null +++ b/source/pyconform/test/conformTests.py @@ -0,0 +1,248 @@ +""" +Dataset Unit Tests + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from os import linesep, remove +from os.path import exists +from pyconform import conform, datasets +from collections import OrderedDict +from netCDF4 import Dataset as NCDataset + +import json +import numpy +import unittest + +class NDArrayEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, numpy.ndarray) and obj.ndim == 1: + return obj.tolist() + elif isinstance(obj, numpy.generic): + return obj.item() + return json.JSONEncoder.default(self, obj) + +#========================================================================= +# print_test_message - Helper function +#========================================================================= +def print_test_message(testname, indata=None, actual=None, expected=None): + """ + Pretty-print a test message + + Parameters: + testname: String name of the test + indata: Input data for testing (if any) + actual: Actual return value/result + expected: Expected return value/result + """ + indent = linesep + ' ' * 14 + print '{}:'.format(testname) + if indata: + s_indata = str(indata).replace(linesep, indent) + print ' input: {}'.format(s_indata) + if actual: + s_actual = str(actual).replace(linesep, indent) + print ' actual: {}'.format(s_actual) + if expected: + s_expected = str(expected).replace(linesep, indent) + print ' expected: {}'.format(s_expected) + print + + +#========================================================================= +# ConformTests - Tests for the setup module +#========================================================================= +class ConformTests(unittest.TestCase): + """ + Unit Tests for the pyconform.dataset module + """ + + def setUp(self): + self.filenames = OrderedDict([('u1', 'u1.nc'), + ('u2', 'u2.nc')]) + self._clear_input_() + + self.fattribs = OrderedDict([('a1', 'attribute 1'), + ('a2', 'attribute 2')]) + self.dims = OrderedDict([('time', 8), ('lat', 7), ('lon', 6)]) + self.vdims = OrderedDict([('u1', ('time', 'lat', 'lon')), + ('u2', ('time', 'lat', 'lon'))]) + self.vattrs = OrderedDict([('lat', {'units': 'degrees_north', + 'standard_name': 'latitude'}), + ('lon', {'units': 'degrees_east', + 'standard_name': 'longitude'}), + ('time', {'units': 'days since 1979-01-01 0:0:0', + 'calendar': 'noleap', + 'standard_name': 'time'}), + ('u1', {'units': 'km', + 'standard_name': 'u variable 1'}), + ('u2', {'units': 'm', + 'standard_name': 'u variable 2'})]) + self.dtypes = {'lat': 'f', 'lon': 'f', 'time': 'f', 'u1': 'd', 'u2': 'd'} + ydat = numpy.linspace(-90, 90, num=self.dims['lat'], + endpoint=True, dtype=self.dtypes['lat']) + xdat = numpy.linspace(-180, 180, num=self.dims['lon'], + endpoint=False, dtype=self.dtypes['lon']) + xdat = -xdat[::-1] + tdat = numpy.linspace(0, self.dims['time'], num=self.dims['time'], + endpoint=False, dtype=self.dtypes['time']) + ulen = reduce(lambda x, y: x * y, self.dims.itervalues(), 1) + ushape = tuple(d for d in self.dims.itervalues()) + u1dat = numpy.linspace(0, ulen, num=ulen, endpoint=False, + dtype=self.dtypes['u1']).reshape(ushape) + u2dat = numpy.linspace(0, ulen, num=ulen, endpoint=False, + dtype=self.dtypes['u2']).reshape(ushape) + self.vdat = {'lat': ydat, 'lon': xdat, 'time': tdat, + 'u1': u1dat, 'u2': u2dat} + + for vname, fname in self.filenames.iteritems(): + ncf = NCDataset(fname, 'w') + ncf.setncatts(self.fattribs) + ncvars = {} + for dname, dvalue in self.dims.iteritems(): + dsize = dvalue if dname != 'time' else None + ncf.createDimension(dname, dsize) + ncvars[dname] = ncf.createVariable(dname, 'd', (dname,)) + ncvars[vname] = ncf.createVariable(vname, 'd', self.vdims[vname]) + for vnam, vobj in ncvars.iteritems(): + for aname, avalue in self.vattrs[vnam].iteritems(): + setattr(vobj, aname, avalue) + vobj[:] = self.vdat[vnam] + ncf.close() + + self.inpds = datasets.InputDataset('inpds', self.filenames.values()) + + self.dsdict = OrderedDict() + self.dsdict['attributes'] = self.fattribs + self.dsdict['variables'] = OrderedDict() + vdicts = self.dsdict['variables'] + + vdicts['L'] = OrderedDict() + vdicts['L']['datatype'] = 'float32' + vdicts['L']['dimensions'] = ('l',) + vdicts['L']['data'] = tuple(range(5)) + vattribs = OrderedDict() + vattribs['standard_name'] = 'level' + vattribs['units'] = '1' + vdicts['L']['attributes'] = vattribs + + vdicts['X'] = OrderedDict() + vdicts['X']['datatype'] = 'float64' + vdicts['X']['dimensions'] = ('x',) + vdicts['X']['definition'] = 'lon' + vattribs = OrderedDict() + vattribs['standard_name'] = 'longitude' + vattribs['units'] = 'degrees_east' + vdicts['X']['attributes'] = vattribs + + vdicts['Y'] = OrderedDict() + vdicts['Y']['datatype'] = 'float64' + vdicts['Y']['dimensions'] = ('y',) + vdicts['Y']['definition'] = 'lat' + vattribs = OrderedDict() + vattribs['standard_name'] = 'latitude' + vattribs['units'] = 'degrees_north' + vdicts['Y']['attributes'] = vattribs + + vdicts['T'] = OrderedDict() + vdicts['T']['datatype'] = 'float64' + vdicts['T']['dimensions'] = ('t',) + vdicts['T']['definition'] = 'time' + vattribs = OrderedDict() + vattribs['standard_name'] = 'time' + vattribs['units'] = 'days since 0001-01-01 00:00:00' + vattribs['calendar'] = 'noleap' + vdicts['T']['attributes'] = vattribs + + vdicts['V1'] = OrderedDict() + vdicts['V1']['datatype'] = 'float64' + vdicts['V1']['dimensions'] = ('t', 'y', 'x') + vdicts['V1']['definition'] = '0.5*(u1 + u2)' + vdicts['V1']['filename'] = 'var1.nc' + vattribs = OrderedDict() + vattribs['standard_name'] = 'variable 1' + vattribs['units'] = 'cm' + vdicts['V1']['attributes'] = vattribs + + vdicts['V2'] = OrderedDict() + vdicts['V2']['datatype'] = 'float64' + vdicts['V2']['dimensions'] = ('t', 'y', 'x') + vdicts['V2']['definition'] = 'u2 - u1' + vdicts['V2']['filename'] = 'var2.nc' + vattribs = OrderedDict() + vattribs['standard_name'] = 'variable 2' + vattribs['units'] = 'cm' + vdicts['V2']['attributes'] = vattribs + + vdicts['V3'] = OrderedDict() + vdicts['V3']['datatype'] = 'float64' + vdicts['V3']['dimensions'] = ('x', 'y', 't') + vdicts['V3']['definition'] = 'u2' + vdicts['V3']['filename'] = 'var3.nc' + vattribs = OrderedDict() + vattribs['standard_name'] = 'variable 3' + vattribs['units'] = 'cm' + vdicts['V3']['attributes'] = vattribs + + vdicts['V4'] = OrderedDict() + vdicts['V4']['datatype'] = 'float64' + vdicts['V4']['dimensions'] = ('t', 'x', 'y') + vdicts['V4']['definition'] = 'u1' + vdicts['V4']['filename'] = 'var4.nc' + vattribs = OrderedDict() + vattribs['standard_name'] = 'variable 4' + vattribs['units'] = 'km' + vattribs['valid_min'] = 1.0 + vattribs['valid_max'] = 100.0 + vdicts['V4']['attributes'] = vattribs + + self.outds = datasets.OutputDataset('outds', self.dsdict) + + self.outfiles = dict((vname, vdict['filename']) for vname, vdict + in vdicts.iteritems() if 'filename' in vdict) + + self._clear_output_() + + def tearDown(self): + self._clear_input_() + self._clear_output_() + + def _clear_input_(self): + for fname in self.filenames.itervalues(): + if exists(fname): + remove(fname) + + def _clear_output_(self): + for fname in self.outfiles.itervalues(): + if exists(fname): + remove(fname) + + def test_setup(self): + with open('conform.spec', 'w') as fp: + json.dump(self.outds.get_dict(), fp, indent=4, cls=NDArrayEncoder) + actual = conform.setup(self.inpds, self.outds) + expected = None + print_test_message('setup()', + actual=actual, expected=expected) + # self.assertEqual(actual, expected, + # 'setup() incorrect') + + def test_run(self): + with open('conform.spec', 'w') as fp: + json.dump(self.outds.get_dict(), fp, indent=4, cls=NDArrayEncoder) + agraph = conform.setup(self.inpds, self.outds) + actual = conform.run(self.inpds, self.outds, agraph) + expected = None + print_test_message('run()', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'run() incorrect') + + +#=============================================================================== +# Command-Line Execution +#=============================================================================== +if __name__ == "__main__": + # import sys;sys.argv = ['', 'Test.testName'] + unittest.main() diff --git a/source/pyconform/test/datasetsTests.py b/source/pyconform/test/datasetsTests.py new file mode 100644 index 00000000..5e3cbeb8 --- /dev/null +++ b/source/pyconform/test/datasetsTests.py @@ -0,0 +1,576 @@ +""" +Dataset Unit Tests + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from os import linesep, remove +from os.path import exists +from pyconform import datasets +from collections import OrderedDict +from netCDF4 import Dataset as NCDataset +from cf_units import Unit + +import numpy as np +import numpy.testing as npt +import unittest + + +#========================================================================= +# print_test_message - Helper function +#========================================================================= +def print_test_message(testname, indata=None, actual=None, expected=None): + """ + Pretty-print a test message + + Parameters: + testname: String name of the test + indata: Input data for testing (if any) + actual: Actual return value/result + expected: Expected return value/result + """ + indent = linesep + ' ' * 14 + print '{}:'.format(testname) + if indata: + s_indata = str(indata).replace(linesep, indent) + print ' input: {}'.format(s_indata) + if actual: + s_actual = str(actual).replace(linesep, indent) + print ' actual: {}'.format(s_actual) + if expected: + s_expected = str(expected).replace(linesep, indent) + print ' expected: {}'.format(s_expected) + print + + +#=============================================================================== +# InfoObjTests +#=============================================================================== +class InfoObjTests(unittest.TestCase): + """ + Unit tests for Info objects + """ + + def test_dinfo_type(self): + dinfo = datasets.DimensionInfo('x') + actual = type(dinfo) + expected = datasets.DimensionInfo + print_test_message('type(DimensionInfo)', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'DimensionInfo has wrong type') + + def test_dinfo_name(self): + indata = 'x' + dinfo = datasets.DimensionInfo(indata) + actual = dinfo.name + expected = indata + print_test_message('DimensionInfo.name', indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'DimensionInfo.name does not match') + + def test_dinfo_size_default(self): + dinfo = datasets.DimensionInfo('x') + actual = dinfo.size + expected = None + print_test_message('DimensionInfo.size == None', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Default DimensionInfo.size is not None') + + def test_dinfo_size(self): + indata = 1 + dinfo = datasets.DimensionInfo('x', size=indata) + actual = dinfo.size + expected = indata + print_test_message('DimensionInfo.size', indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'DimensionInfo.size is not set properly') + + def test_dinfo_limited_default(self): + dinfo = datasets.DimensionInfo('x', size=1) + actual = dinfo.unlimited + expected = False + print_test_message('DimensionInfo.unlimited', + actual=str(actual), expected=str(expected)) + self.assertEqual(actual, expected, + 'Default DimensionInfo.unlimited is False') + + def test_dinfo_limited(self): + dinfo = datasets.DimensionInfo('x', size=1, unlimited=True) + actual = dinfo.unlimited + expected = True + print_test_message('DimensionInfo.unlimited == True', + actual=str(actual), expected=str(expected)) + self.assertEqual(actual, expected, + 'DimensionInfo.unlimited is not True') + + def test_dinfo_equals_same(self): + dinfo1 = datasets.DimensionInfo('x', size=1, unlimited=True) + dinfo2 = datasets.DimensionInfo('x', size=1, unlimited=True) + actual = dinfo1 + expected = dinfo2 + print_test_message('DimensionInfo == DimensionInfo', + actual=str(actual), expected=str(expected)) + self.assertEqual(actual, expected, + 'Identical DimensionInfo objects not equal') + + def test_dinfo_equals_diff_name(self): + dinfo1 = datasets.DimensionInfo('a', size=1, unlimited=True) + dinfo2 = datasets.DimensionInfo('b', size=1, unlimited=True) + actual = dinfo1 + expected = dinfo2 + print_test_message('DimensionInfo(a) != DimensionInfo(b)', + actual=str(actual), expected=str(expected)) + self.assertNotEqual(actual, expected, + 'Differently named DimensionInfo objects equal') + + def test_dinfo_equals_diff_size(self): + dinfo1 = datasets.DimensionInfo('x', size=1, unlimited=True) + dinfo2 = datasets.DimensionInfo('x', size=2, unlimited=True) + actual = dinfo1 + expected = dinfo2 + print_test_message('DimensionInfo(1) != DimensionInfo(2)', + actual=str(actual), expected=str(expected)) + self.assertNotEqual(actual, expected, + 'Differently sized DimensionInfo objects equal') + + def test_dinfo_equals_diff_ulim(self): + dinfo1 = datasets.DimensionInfo('x', size=1, unlimited=False) + dinfo2 = datasets.DimensionInfo('x', size=1, unlimited=True) + actual = dinfo1 + expected = dinfo2 + print_test_message('DimensionInfo(1) != DimensionInfo(2)', + actual=str(actual), expected=str(expected)) + self.assertNotEqual(actual, expected, + 'Differently limited DimensionInfo objects equal') + + def test_vinfo_type(self): + vinfo = datasets.VariableInfo('x') + actual = type(vinfo) + expected = datasets.VariableInfo + print_test_message('type(VariableInfo)', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'VariableInfo has wrong type') + + def test_vinfo_name(self): + indata = 'x' + vinfo = datasets.VariableInfo(indata) + actual = vinfo.name + expected = indata + print_test_message('VariableInfo.name', indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'VariableInfo.name does not match') + + def test_vinfo_dtype_default(self): + vinfo = datasets.VariableInfo('x') + actual = vinfo.datatype + expected = 'float32' + print_test_message('VariableInfo.datatype == float32', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Default VariableInfo.datatype is not float32') + + def test_vinfo_dimensions_default(self): + vinfo = datasets.VariableInfo('x') + actual = vinfo.dimensions + expected = tuple() + print_test_message('VariableInfo.dimensions == ()', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Default VariableInfo.dimensions is not ()') + + def test_vinfo_attributes_default(self): + vinfo = datasets.VariableInfo('x') + actual = vinfo.attributes + expected = OrderedDict() + print_test_message('VariableInfo.attributes == ()', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Default VariableInfo.attributes is not OrderedDict()') + + def test_vinfo_definition_default(self): + vinfo = datasets.VariableInfo('x') + actual = vinfo.definition + expected = None + print_test_message('VariableInfo.definition == ()', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Default VariableInfo.definition is not None') + + def test_vinfo_filename_default(self): + vinfo = datasets.VariableInfo('x') + actual = vinfo.filename + expected = None + print_test_message('VariableInfo.filename == ()', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Default VariableInfo.filename is not None') + + def test_vinfo_dtype(self): + vinfo = datasets.VariableInfo('x', datatype='float64') + actual = vinfo.datatype + expected = 'float64' + print_test_message('VariableInfo.datatype == float64', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Default VariableInfo.datatype is not float64') + + def test_vinfo_dimensions(self): + indata = ('y', 'z') + vinfo = datasets.VariableInfo('x', dimensions=indata) + actual = vinfo.dimensions + expected = indata + print_test_message('VariableInfo.dimensions == ()', indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Default VariableInfo.dimensions is not {}'.format(indata)) + + def test_vinfo_attributes(self): + indata = OrderedDict([('a1', 'attrib1'), ('a2', 'attrib2')]) + vinfo = datasets.VariableInfo('x', attributes=indata) + actual = vinfo.attributes + expected = indata + print_test_message('VariableInfo.attributes', indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Default VariableInfo.attributes is not {}'.format(indata)) + + def test_vinfo_definition(self): + indata = 'y + z' + vinfo = datasets.VariableInfo('x', definition=indata) + actual = vinfo.definition + expected = indata + print_test_message('VariableInfo.definition', indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Default VariableInfo.definition is not {!r}'.format(indata)) + + def test_vinfo_data(self): + indata = (1,2,3,4,5,6) + vinfo = datasets.VariableInfo('x', data=indata) + actual = vinfo.data + expected = indata + print_test_message('VariableInfo.data', indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Default VariableInfo.data is not {!r}'.format(indata)) + + def test_vinfo_filename(self): + indata = 'nc1.nc' + vinfo = datasets.VariableInfo('x', filename=indata) + actual = vinfo.filename + expected = indata + print_test_message('VariableInfo.filename', indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Default VariableInfo.filename is not {!r}'.format(indata)) + + def test_vinfo_equals_same(self): + kwargs = {'datatype': 'd', 'dimensions': ('a', 'b'), + 'attributes': {'a1': 'at1', 'a2': 'at2'}, + 'definition': 'y + z', 'filename': 'out.nc'} + vinfo1 = datasets.VariableInfo('x', **kwargs) + vinfo2 = datasets.VariableInfo('x', **kwargs) + actual = vinfo1 + expected = vinfo2 + print_test_message('VariableInfo == VariableInfo', + actual=str(actual), expected=str(expected)) + self.assertEqual(actual, expected, + 'Identical VariableInfo objects not equal') + + def test_vinfo_equals_diff_name(self): + kwargs = {'datatype': 'd', 'dimensions': ('a', 'b'), + 'attributes': {'a1': 'at1', 'a2': 'at2'}, + 'definition': 'y + z', 'filename': 'out.nc'} + vinfo1 = datasets.VariableInfo('a', **kwargs) + vinfo2 = datasets.VariableInfo('b', **kwargs) + actual = vinfo1 + expected = vinfo2 + print_test_message('VariableInfo(a) != VariableInfo(b)', + actual=str(actual), expected=str(expected)) + self.assertNotEqual(actual, expected, + 'Differently named VariableInfo objects equal') + + def test_vinfo_equals_diff_dtype(self): + vinfo1 = datasets.VariableInfo('x', datatype='d') + vinfo2 = datasets.VariableInfo('x', datatype='f') + actual = vinfo1 + expected = vinfo2 + print_test_message('VariableInfo(d) != VariableInfo(f)', + actual=str(actual), expected=str(expected)) + self.assertNotEqual(actual, expected, + 'Differently typed VariableInfo objects equal') + + def test_vinfo_equals_diff_dims(self): + vinfo1 = datasets.VariableInfo('x', dimensions=('a', 'b')) + vinfo2 = datasets.VariableInfo('x', dimensions=('a', 'b', 'c')) + actual = vinfo1 + expected = vinfo2 + print_test_message('VariableInfo(dims1) != VariableInfo(dims2)', + actual=str(actual), expected=str(expected)) + self.assertNotEqual(actual, expected, + 'Differently dimensioned VariableInfo objects equal') + + def test_vinfo_units_default(self): + vinfo = datasets.VariableInfo('x') + actual = vinfo.units() + expected = None + print_test_message('VariableInfo.units() == None', + actual=str(actual), expected=str(expected)) + self.assertEqual(actual, expected, + 'Default VariableInfo.units() not None') + + def test_vinfo_units(self): + indata = 'm' + vinfo = datasets.VariableInfo('x', attributes={'units': indata}) + actual = vinfo.units() + expected = indata + print_test_message('VariableInfo.units()', indata=indata, + actual=str(actual), expected=str(expected)) + self.assertEqual(actual, expected, + 'Default VariableInfo.units() not {}'.format(indata)) + + def test_vinfo_calendar_default(self): + vinfo = datasets.VariableInfo('x') + actual = vinfo.calendar() + expected = None + print_test_message('VariableInfo.calendar()', + actual=str(actual), expected=str(expected)) + self.assertEqual(actual, expected, + 'Default VariableInfo.calendar() not None') + + def test_vinfo_calendar(self): + indata = 'noleap' + vinfo = datasets.VariableInfo('x', attributes={'units': 'days', + 'calendar': indata}) + actual = vinfo.calendar() + expected = indata + print_test_message('VariableInfo.calendar()', indata=indata, + actual=str(actual), expected=str(expected)) + self.assertEqual(actual, expected, + 'VariableInfo.calendar() not {}'.format(indata)) + + def test_vinfo_cfunits_default(self): + vinfo = datasets.VariableInfo('time') + actual = vinfo.cfunits() + expected = Unit(None) + print_test_message('VariableInfo.cfunits() == None', + actual=str(actual), expected=str(expected)) + self.assertEqual(actual, expected, + 'Default VariableInfo.cfunits() not None') + + def test_vinfo_cfunits(self): + units = 'days' + calendar = 'noleap' + vinfo = datasets.VariableInfo('x', attributes={'units': units, + 'calendar': calendar}) + actual = vinfo.cfunits() + expected = Unit(units, calendar=calendar) + print_test_message('VariableInfo.cfunits()', + actual=str(actual), expected=str(expected)) + self.assertEqual(actual, expected, + 'Default VariableInfo.cfunits() not {}'.format(expected)) + + def test_vinfo_standard_name(self): + indata = 'X var' + vinfo = datasets.VariableInfo('x', attributes={'standard_name': indata}) + actual = vinfo.standard_name() + expected = indata + print_test_message('VariableInfo.standard_name()', indata=indata, + actual=str(actual), expected=str(expected)) + self.assertEqual(actual, expected, + 'Default VariableInfo.standard_name() not {}'.format(indata)) + + def test_vinfo_standard_name_default(self): + vinfo = datasets.VariableInfo('x') + actual = vinfo.standard_name() + expected = None + print_test_message('VariableInfo.standard_name() == None', + actual=str(actual), expected=str(expected)) + self.assertEqual(actual, expected, + 'Default VariableInfo.standard_name() not None') + + +#========================================================================= +# DatasetTests - Tests for the datasets module +#========================================================================= +class DatasetTests(unittest.TestCase): + """ + Unit Tests for the pyconform.datasets module + """ + + def setUp(self): + self.filenames = OrderedDict([('u1', 'u1.nc'), ('u2', 'u2.nc')]) + self._clear_() + + self.fattribs = OrderedDict([('a1', 'attribute 1'), + ('a2', 'attribute 2')]) + self.dims = OrderedDict([('time', 4), ('lat', 3), ('lon', 2)]) + self.vdims = OrderedDict([('u1', ('time', 'lat', 'lon')), + ('u2', ('time', 'lat', 'lon'))]) + self.vattrs = OrderedDict([('lat', {'units': 'degrees_north', + 'standard_name': 'latitude'}), + ('lon', {'units': 'degrees_east', + 'standard_name': 'longitude'}), + ('time', {'units': 'days since 1979-01-01 0:0:0', + 'calendar': 'noleap', + 'standard_name': 'time'}), + ('u1', {'units': 'm', + 'standard_name': 'u variable 1'}), + ('u2', {'units': 'm', + 'standard_name': 'u variable 2'})]) + self.dtypes = {'lat': 'f', 'lon': 'f', 'time': 'f', 'u1': 'd', 'u2': 'd'} + ydat = np.linspace(-90, 90, num=self.dims['lat'], + endpoint=True, dtype=self.dtypes['lat']) + xdat = np.linspace(-180, 180, num=self.dims['lon'], + endpoint=False, dtype=self.dtypes['lon']) + tdat = np.linspace(0, self.dims['time'], num=self.dims['time'], + endpoint=False, dtype=self.dtypes['time']) + ulen = reduce(lambda x,y: x*y, self.dims.itervalues(), 1) + ushape = tuple(d for d in self.dims.itervalues()) + u1dat = np.linspace(0, ulen, num=ulen, endpoint=False, + dtype=self.dtypes['u1']).reshape(ushape) + u2dat = np.linspace(0, ulen, num=ulen, endpoint=False, + dtype=self.dtypes['u2']).reshape(ushape) + self.vdat = {'lat': ydat, 'lon': xdat, 'time': tdat, + 'u1': u1dat, 'u2': u2dat} + + for vname, fname in self.filenames.iteritems(): + ncf = NCDataset(fname, 'w') + ncf.setncatts(self.fattribs) + ncvars = {} + for dname, dvalue in self.dims.iteritems(): + dsize = dvalue if dname!='time' else None + ncf.createDimension(dname, dsize) + ncvars[dname] = ncf.createVariable(dname, 'd', (dname,)) + ncvars[vname] = ncf.createVariable(vname, 'd', self.vdims[vname]) + for vnam, vobj in ncvars.iteritems(): + for aname, avalue in self.vattrs[vnam].iteritems(): + setattr(vobj, aname, avalue) + vobj[:] = self.vdat[vnam] + ncf.close() + + self.dsdict = OrderedDict() + self.dsdict['attributes'] = self.fattribs + self.dsdict['variables'] = OrderedDict() + vdicts = self.dsdict['variables'] + + vdicts['W'] = OrderedDict() + vdicts['W']['datatype'] = 'float64' + vdicts['W']['dimensions'] = ('w',) + vdicts['W']['data'] = [1., 2., 3., 4., 5., 6., 7., 8.] + vattribs = OrderedDict() + vattribs['standard_name'] = 'something' + vattribs['units'] = '1' + vdicts['W']['attributes'] = vattribs + + vdicts['X'] = OrderedDict() + vdicts['X']['datatype'] = 'float64' + vdicts['X']['dimensions'] = ('x',) + vdicts['X']['definition'] = 'lon' + vattribs = OrderedDict() + vattribs['standard_name'] = 'longitude' + vattribs['units'] = 'degrees_east' + vdicts['X']['attributes'] = vattribs + + vdicts['Y'] = OrderedDict() + vdicts['Y']['datatype'] = 'float64' + vdicts['Y']['dimensions'] = ('y',) + vdicts['Y']['definition'] = 'lat' + vattribs = OrderedDict() + vattribs['standard_name'] = 'latitude' + vattribs['units'] = 'degrees_north' + vdicts['Y']['attributes'] = vattribs + + vdicts['T'] = OrderedDict() + vdicts['T']['datatype'] = 'float64' + vdicts['T']['dimensions'] = ('t',) + vdicts['T']['definition'] = 'time' + vattribs = OrderedDict() + vattribs['standard_name'] = 'time' + vattribs['units'] = 'days since 01-01-0001 00:00:00' + vattribs['calendar'] = 'noleap' + vdicts['T']['attributes'] = vattribs + + vdicts['V1'] = OrderedDict() + vdicts['V1']['datatype'] = 'float64' + vdicts['V1']['dimensions'] = ('t', 'y', 'x') + vdicts['V1']['definition'] = 'u1 + u2' + vdicts['V1']['filename'] = 'var1.nc' + vattribs = OrderedDict() + vattribs['standard_name'] = 'variable 1' + vattribs['units'] = 'm' + vdicts['V1']['attributes'] = vattribs + + vdicts['V2'] = OrderedDict() + vdicts['V2']['datatype'] = 'float64' + vdicts['V2']['dimensions'] = ('t', 'y', 'x') + vdicts['V2']['definition'] = 'u2 - u1' + vdicts['V2']['filename'] = 'var2.nc' + vattribs = OrderedDict() + vattribs['standard_name'] = 'variable 2' + vattribs['units'] = 'm' + vdicts['V2']['attributes'] = vattribs + + def tearDown(self): + self._clear_() + + def _clear_(self): + for fname in self.filenames.itervalues(): + if exists(fname): + remove(fname) + + def test_dataset_type(self): + ds = datasets.Dataset() + actual = type(ds) + expected = datasets.Dataset + print_test_message('type(Dataset)', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Dataset has wrong type') + + def test_input_dataset_type(self): + inds = datasets.InputDataset('myinds', self.filenames.values()) + actual = type(inds) + expected = datasets.InputDataset + print_test_message('type(InputDataset)', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'InputDataset has wrong type') + + def test_output_dataset_type(self): + outds = datasets.OutputDataset('myoutds', self.dsdict) + actual = type(outds) + expected = datasets.OutputDataset + print_test_message('type(OutputDataset)', + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'OutputDataset has wrong type') + + def test_dataset_get_dict_from_output(self): + outds = datasets.OutputDataset('myoutds', self.dsdict) + actual = outds.get_dict() + expected = self.dsdict + print_test_message('OutputDataset.get_dict()', + actual=actual, expected=expected) + npt.assert_equal(actual, expected, + 'OutputDataset.get_dict() returns wrong data') + + def test_dataset_get_dict_from_input(self): + inds = datasets.InputDataset('myinds', self.filenames.values()) + actual = inds.get_dict() + print_test_message('InputDataset.get_dict()', actual=actual) + + +#=============================================================================== +# Command-Line Execution +#=============================================================================== +if __name__ == "__main__": + #import sys;sys.argv = ['', 'Test.testName'] + unittest.main() diff --git a/source/pyconform/test/functionsTests.py b/source/pyconform/test/functionsTests.py new file mode 100644 index 00000000..04031e7b --- /dev/null +++ b/source/pyconform/test/functionsTests.py @@ -0,0 +1,361 @@ +""" +Functions Unit Tests + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from pyconform import functions +from cf_units import Unit +from os import linesep + +import unittest +import numpy as np +import operator as op + + +#=============================================================================== +# General Functions +#=============================================================================== +def print_test_message(testname, indata=None, actual=None, expected=None): + print '{}:'.format(testname) + print ' - indata = {}'.format(indata) + print ' - actual = {}'.format(actual).replace(linesep, ' ') + print ' - expected = {}'.format(expected).replace(linesep, ' ') + print + + +#=============================================================================== +# FunctionsTests +#=============================================================================== +class FunctionsTests(unittest.TestCase): + """ + Unit tests for the functions module + """ + + def setUp(self): + self.all_operators = set((('-', 1), ('^', 2), ('+', 2), + ('-', 2), ('*', 2), ('/', 2))) + self.all_functions = set((('transpose', 2), ('sqrt', 1), ('convert', 3))) + self.all = (self.all_operators).union(self.all_functions) + + def test_available_operators(self): + testname = 'available_operators()' + actual = functions.available_operators() + expected = self.all_operators + print_test_message(testname, actual=actual, expected=expected) + self.assertSetEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_find_operator_neg(self): + indata = ('-', 1) + testname = 'find_operator({!r}, {})'.format(*indata) + actual = functions.find_operator(*indata) + expected = functions.NegationOperator + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertIsInstance(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_find_operator_add(self): + indata = ('+', 2) + testname = 'find_operator({!r}, {})'.format(*indata) + actual = functions.find_operator(*indata) + expected = functions.AdditionOperator + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertIsInstance(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_find_operator_sub(self): + indata = ('-', 2) + testname = 'find_operator({!r}, {})'.format(*indata) + actual = functions.find_operator(*indata) + expected = functions.SubtractionOperator + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertIsInstance(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_find_operator_mul(self): + indata = ('*', 2) + testname = 'find_operator({!r}, {})'.format(*indata) + actual = functions.find_operator(*indata) + expected = functions.MultiplicationOperator + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertIsInstance(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_find_operator_div(self): + indata = ('/', 2) + testname = 'find_operator({!r}, {})'.format(*indata) + actual = functions.find_operator(*indata) + expected = functions.DivisionOperator + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertIsInstance(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_find_operator_pow(self): + indata = ('^', 2) + testname = 'find_operator({!r}, {})'.format(*indata) + actual = functions.find_operator(*indata) + expected = functions.PowerOperator + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertIsInstance(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_find_operator_key_failure(self): + indata = ('?', 2) + testname = 'find_operator({!r}, {})'.format(*indata) + expected = KeyError + print_test_message(testname, indata=indata, expected=expected) + self.assertRaises(KeyError, functions.find_operator, *indata) + + def test_find_operator_numargs_failure(self): + indata = ('*', 1) + testname = 'find_operator({!r}, {})'.format(*indata) + expected = KeyError + print_test_message(testname, indata=indata, expected=expected) + self.assertRaises(KeyError, functions.find_operator, *indata) + + def test_available_functions(self): + testname = 'available_functions()' + actual = functions.available_functions() + expected = self.all_functions + print_test_message(testname, actual=actual, expected=expected) + self.assertSetEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_find_function_sqrt(self): + indata = ('sqrt', 1) + testname = 'find_function({!r}, {})'.format(*indata) + actual = functions.find_function(*indata) + expected = functions.SquareRootFunction + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertIsInstance(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_find_function_key_failure(self): + indata = ('f', 1) + testname = 'find_function({!r}, {})'.format(*indata) + expected = KeyError + print_test_message(testname, indata=indata, expected=expected) + self.assertRaises(KeyError, functions.find_function, *indata) + + def test_find_function_numargs_failure(self): + indata = ('sqrt', 2) + testname = 'find_function({!r}, {})'.format(*indata) + expected = KeyError + print_test_message(testname, indata=indata, expected=expected) + self.assertRaises(KeyError, functions.find_function, *indata) + + def test_available(self): + testname = 'available()' + actual = functions.available() + expected = self.all + print_test_message(testname, actual=actual, expected=expected) + self.assertSetEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_find_sqrt(self): + indata = ('sqrt', 1) + testname = 'find({!r}, {})'.format(*indata) + actual = functions.find(*indata) + expected = functions.SquareRootFunction + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertIsInstance(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_find_mul(self): + indata = ('*', 2) + testname = 'find({!r}, {})'.format(*indata) + actual = functions.find(*indata) + expected = functions.MultiplicationOperator + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertIsInstance(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_find_failure(self): + indata = ('*', 3) + testname = 'find({!r}, {})'.format(*indata) + expected = KeyError + print_test_message(testname, indata=indata, expected=expected) + self.assertRaises(KeyError, functions.find, *indata) + + def test_function_mul(self): + indata = (2.4, 3.2) + testname = 'find({!r}, {}).function'.format('*', 2) + actual = functions.find('*', 2)(*indata) + expected = op.mul(*indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_function_attribute_sqrt(self): + indata = (4.0,) + testname = 'find({!r}, {}).function'.format('sqrt', 1) + actual = functions.find('sqrt', 1)(*indata) + expected = np.sqrt(*indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_user_defined_function(self): + class myfunc(functions.Function): + key = 'myfunc' + numargs = 3 + def units(self, *arg_units): + uret = arg_units[0] if isinstance(arg_units[0], Unit) else Unit(1) + return uret, (None, None, None) + def dimensions(self, *arg_dims): + dret = arg_dims[0] if isinstance(arg_dims[0], tuple) else () + return dret, (None, None, None) + def __call(self, x, y, z): + return x + + indata = ('myfunc', 3) + testname = 'find({!r}, {}).function'.format(*indata) + actual = functions.find(*indata) + expected = myfunc + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertIsInstance(actual, expected, + '{} returned unexpected result'.format(testname)) + + +#=============================================================================== +# UnitsTests +#=============================================================================== +class UnitsTests(unittest.TestCase): + """ + Unit tests for the units methods of the functions.FunctionAbstract classes + """ + + def test_units_neg_m(self): + indata = (Unit('m'),) + testname = 'NegationOperator.units({!r})'.format(*indata) + actual = functions.NegationOperator.units(*indata) + expected = Unit(indata[0]), (None,) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertTupleEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_units_neg_1(self): + indata = (1,) + testname = 'NegationOperator.units({!r})'.format(*indata) + actual = functions.NegationOperator.units(*indata) + expected = Unit(indata[0]), (None,) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertTupleEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_units_add_m_m(self): + indata = (Unit('m'), Unit('m')) + testname = 'AdditionOperator.units({!r})'.format(*indata) + actual = functions.AdditionOperator.units(*indata) + expected = Unit(indata[0]), (None, None) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertTupleEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_units_add_m_km(self): + indata = (Unit('m'), Unit('km')) + testname = 'AdditionOperator.units({!r})'.format(*indata) + actual = functions.AdditionOperator.units(*indata) + expected = Unit(indata[0]), (None, Unit('m')) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertTupleEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_units_add_m_g(self): + indata = (Unit('m'), Unit('g')) + testname = 'AdditionOperator.units({!r})'.format(*indata) + expected = functions.UnitsError + print_test_message(testname, indata=indata, expected=expected) + self.assertRaises(expected, functions.AdditionOperator.units, *indata) + + def test_units_add_1_u1(self): + indata = (1, Unit(1)) + testname = 'AdditionOperator.units({!r})'.format(*indata) + actual = functions.AdditionOperator.units(*indata) + expected = Unit(indata[0]), (None, None) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertTupleEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_units_add_m_2(self): + indata = (Unit('m'), 2) + testname = 'AdditionOperator.units({!r})'.format(*indata) + expected = functions.UnitsError + print_test_message(testname, indata=indata, expected=expected) + self.assertRaises(expected, functions.AdditionOperator.units, *indata) + + def test_units_sub_m_km(self): + indata = (Unit('m'), Unit('km')) + testname = 'SubtractionOperator.units({!r})'.format(*indata) + actual = functions.SubtractionOperator.units(*indata) + expected = Unit(indata[0]), (None, Unit('m')) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertTupleEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_units_mul_m_m(self): + indata = (Unit('m'), Unit('m')) + testname = 'MultiplicationOperator.units({!r})'.format(*indata) + actual = functions.MultiplicationOperator.units(*indata) + expected = Unit(op.mul(*indata)), (None, None) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertTupleEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_units_mul_1_m(self): + indata = (1, Unit('m')) + testname = 'MultiplicationOperator.units({!r})'.format(*indata) + actual = functions.MultiplicationOperator.units(*indata) + expected = Unit(op.mul(*indata)), (None, None) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertTupleEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_units_div_1_m(self): + indata = (1, Unit('m')) + testname = 'DivisionOperator.units({!r})'.format(*indata) + actual = functions.DivisionOperator.units(*indata) + expected = Unit(1)/Unit('m'), (None, None) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertTupleEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_units_sqrt_m2(self): + indata = (Unit('m')**2,) + testname = 'SquareRootFunction.units({!r})'.format(*indata) + actual = functions.SquareRootFunction.units(*indata) + expected = Unit('m'), (None,) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertTupleEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + +#=============================================================================== +# Command-Line Operation +#=============================================================================== +if __name__ == "__main__": + #import sys;sys.argv = ['', 'Test.testName'] + unittest.main() diff --git a/source/pyconform/test/graphsTests.py b/source/pyconform/test/graphsTests.py new file mode 100644 index 00000000..82aaaf34 --- /dev/null +++ b/source/pyconform/test/graphsTests.py @@ -0,0 +1,542 @@ +""" +DiGraph Unit Tests + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from copy import deepcopy +from pyconform import graphs as graphs +from os import linesep + +import unittest +import operator as op + + +#=============================================================================== +# General Functions +#=============================================================================== +def print_test_message(testname, actual, expected): + print '{}:'.format(testname) + print ' - actual = {}'.format(actual).replace(linesep, ' ') + print ' - expected = {}'.format(expected).replace(linesep, ' ') + print + + +#=============================================================================== +# GraphTests +#=============================================================================== +class GraphTests(unittest.TestCase): + """ + Unit tests for the graphs.DiGraph class + """ + + def test_init(self): + testname = 'DiGraph.__init__()' + G = graphs.DiGraph() + actual = type(G) + expected = graphs.DiGraph + print_test_message(testname, actual, expected) + self.assertIsInstance(G, expected, + '{} returned unexpected result'.format(testname)) + + def test_equal(self): + testname = 'DiGraph == DiGraph' + G = graphs.DiGraph() + G.connect(1, 2) + H = graphs.DiGraph() + H.connect(1, 2) + actual = G == H + expected = True + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_unequal(self): + testname = 'DiGraph != DiGraph' + G = graphs.DiGraph() + G.connect(1, 2) + H = graphs.DiGraph() + H.connect(2, 1) + actual = G != H + expected = True + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_in(self): + indata = 1 + testname = '{} in DiGraph'.format(indata) + G = graphs.DiGraph() + G.add(2) + G.add(3) + G.add(indata) + G.add(5) + G.connect(4, 5) + actual = indata in G + expected = True + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_not_in(self): + indata = 1 + testname = '{} not in DiGraph'.format(indata) + G = graphs.DiGraph() + G.add(2) + G.add(3) + G.connect(4, 5) + actual = indata not in G + expected = True + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_len(self): + testname = 'len(DiGraph)' + G = graphs.DiGraph() + G.add(2) + G.add(3) + G.add(5) + actual = len(G) + expected = 3 + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_str(self): + G = graphs.DiGraph() + G.add(1) + G.connect(2, 3) + G.connect(3, 4) + G.connect(2, 5) + print G + + def test_clear(self): + G = graphs.DiGraph() + G.connect(1, 2) + G.connect(2, 3) + G.connect(2, 4) + G.clear() + testname = 'DiGraph.clear() vertices' + actual = G.vertices + expected = [] + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + testname = 'DiGraph.clear() - edges' + actual = G.edges + expected = [] + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_vertices(self): + testname = 'DiGraph.vertices()' + G = graphs.DiGraph() + indata = 1 + G.add(indata) + actual = G.vertices + expected = set([indata]) + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_edges(self): + indata = (1, 'a') + testname = 'DiGraph.edges()' + G = graphs.DiGraph() + G.connect(*indata) + actual = G.edges + expected = [indata] + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_add_hashable(self): + indata = 1 + testname = 'DiGraph.add({})'.format(indata) + G = graphs.DiGraph() + G.add(indata) + actual = G.vertices + expected = {indata} + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_add_unhashable(self): + indata = {'a': 1, 'b': 2} + testname = 'DiGraph.add({})'.format(indata) + G = graphs.DiGraph() + print_test_message(testname, '???', 'TypeError') + self.assertRaises(TypeError, G.add, indata) + + def test_remove(self): + indata = 1 + testname = 'DiGraph.remove({})'.format(indata) + G = graphs.DiGraph() + G.add(2) + G.add(indata) + G.remove(indata) + actual = G.vertices + expected = {2} + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_update(self): + G = graphs.DiGraph() + G.connect(1, 2) + G.connect(1, 3) + H = graphs.DiGraph() + H.connect(1, 4) + H.connect(2, 5) + I = deepcopy(G) + I.update(H) + actual = I.vertices + expected = [v for v in G.vertices] + expected.extend(v for v in H.vertices if v not in G.vertices) + testname = 'DiGraph.update(DiGraph).vertices' + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + actual = I.edges + expected = set(G.edges) + expected.update(H.edges) + testname = 'DiGraph.update(DiGraph).edges' + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_union(self): + G = graphs.DiGraph() + G.connect(1, 2) + G.connect(1, 3) + H = graphs.DiGraph() + H.connect(1, 4) + H.connect(2, 5) + I = G.union(H) + actual = I.vertices + expected = [v for v in G.vertices] + expected.extend(v for v in H.vertices if v not in G.vertices) + testname = 'DiGraph.union(DiGraph).vertices' + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + actual = I.edges + expected = set(G.edges) + expected.update(H.edges) + testname = 'DiGraph.union(DiGraph).edges' + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_connect(self): + indata = (1, 'a') + testname = 'DiGraph.connect({}, {})'.format(*indata) + G = graphs.DiGraph() + G.add(indata[0]) + G.add(indata[1]) + G.connect(*indata) + actual = G.edges + expected = {indata} + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_disconnect(self): + indata = (1, 'a') + testname = 'DiGraph.disconnect({}, {})'.format(*indata) + G = graphs.DiGraph() + G.connect(*indata) + G.connect(3, 5) + G.disconnect(*indata) + actual = G.edges + expected = {(3, 5)} + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_insert(self): + testname = 'DiGraph.insert()' + G = graphs.DiGraph() + G.connect(1, 2) + G.connect(1, 3) + G.connect(1, 4) + G.connect(3, 5) + G.connect(3, 6) + G.connect(3, 7) + print G + print + G.insert(1, 8, 3) + print G + print + actual = G.neighbors_from(1) + expected = [2, 8, 4] + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + actual = G.neighbors_to(3) + expected = [8] + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_neighbors_from(self): + indata = 1 + testname = 'DiGraph.neighbors_from({})'.format(indata) + G = graphs.DiGraph() + G.connect(indata, 'a') + G.connect(indata, 2) + actual = G.neighbors_from(indata) + expected = ['a', 2] + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_neighbors_to(self): + indata = 1 + testname = 'DiGraph.neighbors_to({})'.format(indata) + G = graphs.DiGraph() + G.connect(indata, 'a') + G.connect(2, indata) + actual = G.neighbors_to(indata) + expected = [2] + print_test_message(testname, actual, expected) + self.assertItemsEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_sinks(self): + testname = 'DiGraph.sinks()' + G = graphs.DiGraph() + G.connect(1, 3) + G.connect(2, 3) + G.connect(3, 4) + G.connect(3, 5) + actual = G.sinks() + expected = set([4, 5]) + print_test_message(testname, actual, expected) + self.assertSetEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_sources(self): + testname = 'DiGraph.sources()' + G = graphs.DiGraph() + G.connect(1, 3) + G.connect(2, 3) + G.connect(3, 4) + G.connect(3, 5) + actual = G.sources() + expected = set([1, 2]) + print_test_message(testname, actual, expected) + self.assertSetEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_bfs(self): + indata = 1 + testname = 'DiGraph.iter_bfs({})'.format(indata) + G = graphs.DiGraph() + G.add(0) + G.connect(1, 2) + G.connect(2, 3) + G.connect(2, 4) + G.connect(3, 5) + G.connect(4, 5) + actual = [v for v in G.iter_bfs(indata)] + expected = ([1, 2, 3, 4, 5], [1, 2, 4, 3, 5]) + print_test_message(testname, actual, expected) + self.assertIn(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_bfs_reversed(self): + indata = 5 + testname = 'DiGraph.iter_bfs({}, reverse=True)'.format(indata) + G = graphs.DiGraph() + G.add(0) + G.connect(1, 2) + G.connect(2, 3) + G.connect(2, 4) + G.connect(3, 5) + G.connect(4, 5) + actual = [v for v in G.iter_bfs(indata, reverse=True)] + expected = ([5, 3, 4, 2, 1], [5, 4, 3, 2, 1]) + print_test_message(testname, actual, expected) + self.assertIn(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_dfs(self): + indata = 1 + testname = 'DiGraph.iter_dfs({})'.format(indata) + G = graphs.DiGraph() + G.add(0) + G.connect(1, 2) + G.connect(2, 3) + G.connect(2, 4) + G.connect(3, 5) + G.connect(4, 5) + actual = [v for v in G.iter_dfs(indata)] + expected = ([1, 2, 3, 5, 4], [1, 2, 4, 5, 3]) + print_test_message(testname, actual, expected) + self.assertIn(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_dfs_reversed(self): + indata = 5 + testname = 'DiGraph.iter_dfs({}, reverse=True)'.format(indata) + G = graphs.DiGraph() + G.add(0) + G.connect(1, 2) + G.connect(2, 3) + G.connect(2, 4) + G.connect(3, 5) + G.connect(4, 5) + actual = [v for v in G.iter_dfs(indata, reverse=True)] + expected = ([5, 3, 2, 1, 4], [5, 4, 2, 1, 3]) + print_test_message(testname, actual, expected) + self.assertIn(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_toposort_acyclic(self): + testname = 'DiGraph.toposort(acyclic)' + G = graphs.DiGraph() + G.add(0) + G.connect(1, 2) + G.connect(2, 3) + G.connect(2, 4) + G.connect(3, 5) + G.connect(4, 5) + actual = G.toposort() + expected = ([0, 1, 2, 3, 4, 5], [1, 2, 3, 4, 5, 0], + [0, 1, 2, 4, 3, 5], [1, 2, 4, 3, 5, 0]) + print_test_message(testname, actual, expected) + self.assertIn(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_toposort_cyclic(self): + testname = 'DiGraph.toposort(cyclic)' + G = graphs.DiGraph() + G.add(0) + G.connect(1, 2) + G.connect(2, 3) + G.connect(2, 4) + G.connect(3, 5) + G.connect(4, 5) + G.connect(5, 1) + actual = G.toposort() + expected = None + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_is_not_cyclic(self): + testname = 'DiGraph.is_cyclic(acyclic)' + G = graphs.DiGraph() + G.add(0) + G.connect(1, 2) + G.connect(2, 3) + G.connect(2, 4) + G.connect(3, 5) + G.connect(4, 5) + actual = G.is_cyclic() + expected = False + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_is_cyclic(self): + testname = 'DiGraph.is_cyclic(cyclic)' + G = graphs.DiGraph() + G.add(0) + G.connect(1, 2) + G.connect(2, 3) + G.connect(2, 4) + G.connect(3, 5) + G.connect(4, 5) + G.connect(5, 1) + actual = G.is_cyclic() + expected = True + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_components(self): + testname = 'DiGraph.components()' + G = graphs.DiGraph() + G.connect(1, 2) + G.connect(1, 3) + H = graphs.DiGraph() + H.connect(4, 5) + H.connect(5, 6) + I = G.union(H) + actual = I.components() + expected = [G, H] + print_test_message(testname, actual, expected) + for g in actual: + self.assertTrue(g == G or g == H, + '{} returned unexpected result'.format(testname)) + + def test_commutative_call_graph(self): + testname = 'DiGraph Commutative Call-Graph Evaluation' + G = graphs.DiGraph() + f1 = lambda: 1 + f2 = lambda: 2 + G.add(f1) + G.add(f2) + G.add(op.add) + G.connect(f1, op.add) + G.connect(f2, op.add) # 1 + 2 = 3 + G.add(op.mul) + G.connect(op.add, op.mul) + G.connect(f2, op.mul) # 3 * 2 = 6 + def evaluate_G(v): + return v(*map(evaluate_G, G.neighbors_to(v))) + actual = evaluate_G(op.mul) + expected = 6 + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_noncommutative_call_graph(self): + testname = 'DiGraph Non-Commutative Call-Graph Evaluation' + G = graphs.DiGraph() + f6 = lambda: 6 + f2 = lambda: 2 + G.add(f6) + G.add(f2) + G.add(op.div) + G.connect(f6, op.div) + G.connect(f2, op.div) # 6 / 2 = 3 + G.add(op.sub) + G.connect(op.div, op.sub) + G.connect(f2, op.sub) # 3 - 2 = 1 + def evaluate_G(v): + return v(*map(evaluate_G, G.neighbors_to(v))) + actual = evaluate_G(op.sub) + expected = 1 + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + def test_incremented_call_graph(self): + testname = 'DiGraph Incremented Call-Graph Evaluation' + G = graphs.DiGraph() + f1 = lambda: 1 + finc = lambda x: x + 1 + G.connect((0, f1), (1, finc)) # 1 + 1 = 2 + G.connect((1, finc), (2, finc)) # 2 + 1 = 3 + G.connect((2, finc), (3, finc)) # 3 + 1 = 4 + def evaluate_G(v): + return v[1](*map(evaluate_G, G.neighbors_to(v))) + actual = evaluate_G((3, finc)) + expected = 4 + print_test_message(testname, actual, expected) + self.assertEqual(actual, expected, + '{} returned unexpected result'.format(testname)) + + +#=============================================================================== +# Command-Line Operation +#=============================================================================== +if __name__ == "__main__": + # import sys;sys.argv = ['', 'Test.testName'] + unittest.main() diff --git a/source/pyconform/test/mapdatesTests.py b/source/pyconform/test/mapdatesTests.py new file mode 100644 index 00000000..a2f2c361 --- /dev/null +++ b/source/pyconform/test/mapdatesTests.py @@ -0,0 +1,207 @@ +import numpy as np +from cStringIO import StringIO +from mkTestData import DataMaker +from pyconform import mapdates +import unittest + +files=['test1.nc', 'test2.nc', 'test3.nc'] + +#=============================================================================== +# dateMapTests +#=============================================================================== +class dateMapTests(unittest.TestCase): + + def test_get_files_in_order(self): + + # Create some data that should be contiguous + t = {'time': [3,3,3], 'space': 2} + dm = DataMaker(dimensions=t, filenames=files) + dm.write() + + # Call get_files_in_order and evaluate the return values + ordered_files,counts,error = mapdates.get_files_in_order(files) + self.assertTrue(ordered_files == ['test1.nc', 'test2.nc', 'test3.nc'], + "get_files_in_order, ordered_list".format()) + self.assertTrue(counts == [3,3,3], + "get_files_in_order, counts".format()) + self.assertTrue(error == 0, "get_files_in_order, error".format()) + dm.clear() + + def test_get_files_out_of_order(self): + + # Create some data that is not contiguous + a = np.asarray([1,2,3]) + b = np.asarray([7,8,9]) + c = np.asarray([4,5,6]) + t = {'time': [3,3,3], 'space': 2} + dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + dm.write() + + # Call get_files_in_order and evaluate the return values + ordered_files,counts,error = mapdates.get_files_in_order(files) + self.assertTrue(ordered_files == ['test1.nc', 'test3.nc', 'test2.nc'], + "get_files_in_order, ordered_list".format()) + self.assertTrue(counts == [3,3,3], + "get_files_in_order, counts".format()) + self.assertTrue(error == 0, "get_files_in_order, error".format()) + dm.clear() + + def test_get_files_gap_between_files(self): + + # Create some data that is not contiguous + a = np.asarray([1,2,3]) + b = np.asarray([4,5,6]) + c = np.asarray([10,11,12]) + t = {'time': [3,3,3], 'space': 2} + dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + dm.write() + + # Call get_files_in_order and evaluate the return values, return value should fail + _ordered_files, _counts, error = mapdates.get_files_in_order(files) + self.assertTrue(error == 1, "get_files_in_order, error".format()) + dm.clear() + + def test_get_files_gap_in_a_file(self): + + # Create some data that is not contiguous + a = np.asarray([1,2,4]) + b = np.asarray([5,6,7]) + c = np.asarray([8,9,10]) + t = {'time': [3,3,3], 'space': 2} + dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + dm.write() + + # Call get_files_in_order and evaluate the return values, return value should fail + _ordered_files, _counts, error = mapdates.get_files_in_order(files) + self.assertTrue(error == 1, "get_files_in_order, error".format()) + dm.clear() + + def test_get_files_6hr(self): + + from collections import OrderedDict + + # Create some data that is not contiguous + a = np.asarray([1,1.25,1.50,1.75]) + b = np.asarray([2,2.25,2.50,2.75]) + c = np.asarray([3,3.25,3.50,3.75]) + t = OrderedDict([('time', [4,4,4]),('space', 2)]) + dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + dm.write() + + # Call get_files_in_order and evaluate the return values + ordered_files,counts,error = mapdates.get_files_in_order(files) + self.assertTrue(ordered_files == ['test1.nc', 'test2.nc', 'test3.nc'], + "get_files_in_order, ordered_list".format()) + self.assertTrue(counts == [4,4,4], + "get_files_in_order, counts".format()) + self.assertTrue(error == 0, "get_files_in_order, error".format()) + dm.clear() + + def test_get_files_monthly(self): + + from collections import OrderedDict + + # Create some data + a = np.asarray([31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365]) + b1=[] + c1=[] + for i in range(0,12): + b1.append(a[i]+365) + c1.append(a[i]+730) + b = np.asarray(b1) + c = np.asarray(c1) + t = OrderedDict([('time', [12,12,12]),('space', 2)]) + dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + dm.write() + + # Call get_files_in_order and evaluate the return values + ordered_files,counts,error = mapdates.get_files_in_order(files) + self.assertTrue(ordered_files == ['test1.nc', 'test2.nc', 'test3.nc'], + "get_files_in_order, ordered_list".format()) + self.assertTrue(counts == [12,12,12], + "get_files_in_order, counts".format()) + self.assertTrue(error == 0, "get_files_in_order, error".format()) + dm.clear() + + def test_get_files_monthly_gap_between_files(self): + + from collections import OrderedDict + + # Create some data that is not contiguous + a = np.asarray([31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334]) + b1=[] + c1=[] + for i in range(0,11): + b1.append(a[i]+365) + c1.append(a[i]+730) + b = np.asarray(b1) + c = np.asarray(c1) + t = OrderedDict([('time', [11,11,11]),('space', 2)]) + dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + dm.write() + + # Call get_files_in_order and evaluate the return values + _ordered_files, _counts, error = mapdates.get_files_in_order(files) + self.assertTrue(error == 1, "get_files_in_order, error".format()) + dm.clear() + + def test_get_files_monthly_gap_in_a_file(self): + + from collections import OrderedDict + + # Create some data that is not contiguous + a = np.asarray([31, 59, 90, 120, 151, 181, 212, 243, 273, 334, 365]) + b1=[] + c1=[] + for i in range(0,11): + b1.append(a[i]+365) + c1.append(a[i]+730) + b = np.asarray(b1) + c = np.asarray(c1) + t = OrderedDict([('time', [11,11,11]),('space', 2)]) + dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + dm.write() + + # Call get_files_in_order and evaluate the return values + _ordered_files, _counts, error = mapdates.get_files_in_order(files) + self.assertTrue(error == 1, "get_files_in_order, error".format()) + dm.clear() + + def test_get_files_yearly(self): + + # Create some data + a = np.asarray([365, 730, 1095]) + b = np.asarray([1460, 1825, 2190]) + c = np.asarray([2555, 2920, 3285]) + t = {'time': [3,3,3], 'space': 2} + dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + dm.write() + + # Call get_files_in_order and evaluate the return values + ordered_files,counts,error = mapdates.get_files_in_order(files) + self.assertTrue(ordered_files == ['test1.nc', 'test2.nc', 'test3.nc'], + "get_files_in_order, ordered_list".format()) + self.assertTrue(counts == [3,3,3], + "get_files_in_order, counts".format()) + self.assertTrue(error == 0, "get_files_in_order, error".format()) + dm.clear() + +#=============================================================================== +# Command-Line Operation +#=============================================================================== +if __name__ == "__main__": + hline = '=' * 70 + print hline + print 'STANDARD OUTPUT FROM ALL TESTS:' + print hline + + mystream = StringIO() + tests = unittest.TestLoader().loadTestsFromTestCase(dateMapTests) + unittest.TextTestRunner(stream=mystream).run(tests) + + print hline + print 'TESTS RESULTS:' + print hline + print str(mystream.getvalue()) + + diff --git a/source/pyconform/test/mkTestData.py b/source/pyconform/test/mkTestData.py new file mode 100644 index 00000000..c2e31ed6 --- /dev/null +++ b/source/pyconform/test/mkTestData.py @@ -0,0 +1,135 @@ +""" +Make Testing Data + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from collections import OrderedDict +from os import remove +from os.path import exists +import netCDF4 +import numpy as np + + +#=============================================================================== +# DataMaker +#=============================================================================== +class DataMaker(object): + """ + Simple tool to write a "fake" dataset for testing purposes + """ + + def __init__(self, + filenames=['test1.nc', 'test2.nc', 'test3.nc'], + fileattribs=[OrderedDict([('a1', 'file 1 attrib 1'), + ('a2', 'file 1 attrib 2')]), + OrderedDict([('a1', 'file 2 attrib 1'), + ('a2', 'file 2 attrib 2')]), + OrderedDict([('a1', 'file 3 attrib 1'), + ('a2', 'file 3 attrib 2')]),], + dimensions=OrderedDict([('time', [3,4,5]), + ('space', 2)]), + vardims=OrderedDict([('T1', ('time', 'space')), + ('T2', ('space', 'time'))]), + vartypes=OrderedDict(), + varattribs=OrderedDict([('space', {'units': 'm'}), + ('time', {'units': 'days since 1979-01-01 00:00:00', + 'calendar': 'noleap'}), + ('T1', {'units': 'K'}), + ('T2', {'units': 'C'})]), + vardata=OrderedDict()): + + self.filenames = filenames + self.clear() + + self.fileattribs = fileattribs + self.dimensions = dimensions + self.vardims = vardims + self.vartypes = vartypes + self.varattribs = varattribs + + self.vardata = {} + for filenum, filename in enumerate(self.filenames): + self.vardata[filename] = {} + filevars = self.vardata[filename] + for coordname, dimsize in self.dimensions.iteritems(): + if coordname in vardata: + vdat = vardata[coordname][filenum] + elif isinstance(dimsize, (list, tuple)): + start = filenum * dimsize[filenum] + end = (filenum + 1) * dimsize[filenum] + dt = vartypes.get(coordname, 'float64') + vdat = np.arange(start, end, dtype=dt) + else: + dt = vartypes.get(coordname, 'float64') + vdat = np.arange(dimsize, dtype=dt) + filevars[coordname] = vdat + + for varname, vardims in self.vardims.iteritems(): + if varname in vardata: + vdat = vardata[varname][filenum] + else: + vshape = [] + for dim in vardims: + if isinstance(self.dimensions[dim], (list, tuple)): + vshape.append(self.dimensions[dim][filenum]) + else: + vshape.append(self.dimensions[dim]) + dt = vartypes.get(varname, 'float64') + vdat = np.ones(tuple(vshape), dtype=dt) + filevars[varname] = vdat + + def write(self, ncformat='NETCDF4'): + + for filenum, filename in enumerate(self.filenames): + ncfile = netCDF4.Dataset(filename, 'w', format=ncformat) + + for attrname, attrval in self.fileattribs[filenum].iteritems(): + setattr(ncfile, attrname, attrval) + + for dimname, dimsize in self.dimensions.iteritems(): + if isinstance(dimsize, (list, tuple)): + ncfile.createDimension(dimname) + else: + ncfile.createDimension(dimname, dimsize) + + for coordname in self.dimensions.iterkeys(): + if coordname in self.vartypes: + dtype = self.vartypes[coordname] + else: + dtype = 'd' + ncfile.createVariable(coordname, dtype, (coordname,)) + + for varname, vardims in self.vardims.iteritems(): + if varname in self.vartypes: + dtype = self.vartypes[varname] + else: + dtype = 'd' + ncfile.createVariable(varname, dtype, vardims) + + for coordname, dimsize in self.dimensions.iteritems(): + coordvar = ncfile.variables[coordname] + for attrname, attrval in self.varattribs[coordname].iteritems(): + setattr(coordvar, attrname, attrval) + coordvar[:] = self.vardata[filename][coordname] + + for varname, vardims in self.vardims.iteritems(): + var = ncfile.variables[varname] + for attrname, attrval in self.varattribs[varname].iteritems(): + setattr(var, attrname, attrval) + var[:] = self.vardata[filename][varname] + + ncfile.close() + + def clear(self): + for filename in self.filenames: + if exists(filename): + remove(filename) + + +#=============================================================================== +# Command-Line Operation +#=============================================================================== +if __name__ == '__main__': + pass diff --git a/source/pyconform/test/parsingTests.py b/source/pyconform/test/parsingTests.py new file mode 100644 index 00000000..d4c8174e --- /dev/null +++ b/source/pyconform/test/parsingTests.py @@ -0,0 +1,751 @@ +""" +Parsing Unit Tests + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from pyconform import parsing +from os import linesep + +import unittest + + +#=============================================================================== +# General Functions +#=============================================================================== +def print_test_message(testname, indata=None, actual=None, expected=None): + print '{0}:'.format(testname) + print ' - input = {0!r}'.format(indata).replace(linesep, ' ') + print ' - actual = {0!r}'.format(actual).replace(linesep, ' ') + print ' - expected = {0!r}'.format(expected).replace(linesep, ' ') + print + + +#=============================================================================== +# ParsedStringTypeTests +#=============================================================================== +class ParsedStringTypeTests(unittest.TestCase): + + def test_pst_init(self): + indata = (['x'], {}) + pst = parsing.ParsedFunction(indata) + actual = type(pst) + expected = parsing.ParsedFunction + testname = 'ParsedFunction.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Types do not match') + + def test_varpst_init(self): + indata = (['x'], {}) + pst = parsing.ParsedVariable(indata) + actual = type(pst) + expected = parsing.ParsedVariable + testname = 'ParsedVariable.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Types do not match') + + def test_funcpst_init(self): + indata = (['x'], {}) + pst = parsing.ParsedFunction(indata) + actual = type(pst) + expected = parsing.ParsedFunction + testname = 'ParsedFunction.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Types do not match') + + def test_operpst_init(self): + indata = (['x'], {}) + pst = parsing.ParsedBinOp(indata) + actual = type(pst) + expected = parsing.ParsedBinOp + testname = 'ParsedBinOp.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Types do not match') + + def test_pst_init_args(self): + indata = (['x', 1, -3.2], {}) + pst = parsing.ParsedFunction(indata) + actual = type(pst) + expected = parsing.ParsedFunction + testname = 'ParsedFunction.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Types do not match') + + def test_pst_obj(self): + indata = (['x', 1, -3.2], {}) + pst = parsing.ParsedFunction(indata) + actual = pst.key + expected = indata[0][0] + testname = 'ParsedFunction.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Names do not match') + + def test_pst_args(self): + indata = (['x', 1, -3.2], {}) + pst = parsing.ParsedFunction(indata) + actual = pst.args + expected = tuple(indata[0][1:]) + testname = 'ParsedFunction.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Names do not match') + + +#=============================================================================== +# DefinitionParserTests +#=============================================================================== +class DefinitionParserTests(unittest.TestCase): + +#===== QUOTED STRINGS ========================================================== + + def test_parse_quote(self): + indata = '"1"' + actual = parsing.parse_definition(indata) + expected = '1' + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'String parsing failed') + + def test_parse_quote_escaped(self): + indata = '"\\"1\\""' + actual = parsing.parse_definition(indata) + expected = '"1"' + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'String parsing failed') + +#===== INTEGERS ================================================================ + + def test_parse_integer(self): + indata = '1' + actual = parsing.parse_definition(indata) + expected = int(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Integer parsing failed') + + def test_parse_integer_large(self): + indata = '98734786423867234' + actual = parsing.parse_definition(indata) + expected = int(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Integer parsing failed') + +#===== FLOATS ================================================================== + + def test_parse_float_dec(self): + indata = '1.' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_dec_long(self): + indata = '1.8374755' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_dec_nofirst(self): + indata = '.35457' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_exp(self): + indata = '1e7' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_pos_exp(self): + indata = '1e+7' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_neg_exp(self): + indata = '1e-7' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_dec_exp(self): + indata = '1.e7' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_dec_pos_exp(self): + indata = '1.e+7' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_dec_neg_exp(self): + indata = '1.e-7' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_dec_long_exp(self): + indata = '1.324523e7' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_dec_long_pos_exp(self): + indata = '1.324523e+7' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_dec_long_neg_exp(self): + indata = '1.324523e-7' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_dec_nofirst_exp(self): + indata = '.324523e7' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_dec_nofirst_pos_exp(self): + indata = '.324523e+7' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + + def test_parse_float_dec_nofirst_neg_exp(self): + indata = '.324523e-7' + actual = parsing.parse_definition(indata) + expected = float(indata) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Float parsing failed') + +#===== FUNCTIONS =============================================================== + + def test_parse_func(self): + indata = 'f()' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedFunction(('f', {})) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Function parsing failed') + + def test_parse_func_arg(self): + indata = 'f(1)' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedFunction([['f', 1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Function parsing failed') + + def test_parse_func_nested(self): + g2 = parsing.ParsedFunction([['g', 2]]) + f1g = parsing.ParsedFunction([['f', 1, g2]]) + indata = 'f(1, g(2))' + actual = parsing.parse_definition(indata) + expected = f1g + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Function parsing failed') + +#===== VARIABLES =============================================================== + + def test_parse_var(self): + indata = 'x' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedVariable([['x']]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Variable parsing failed') + + def test_parse_var_index(self): + indata = 'x[1]' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedVariable([['x', 1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Variable parsing failed') + + def test_parse_var_slice(self): + indata = 'x[1:2:3]' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedVariable([['x', slice(1,2,3)]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Variable parsing failed') + +# def test_parse_var_index_nested(self): +# y0 = parsing.ParsedVariable([['y', 0]]) +# x1y = parsing.ParsedVariable([['x', 1, y0]]) +# indata = 'x[1, y[0]]' +# actual = parsing.parse_definition(indata) +# expected = x1y +# testname = 'parse_definition({0!r})'.format(indata) +# print_test_message(testname, indata=indata, +# actual=actual, expected=expected) +# self.assertEqual(actual, expected, +# 'Variable parsing failed') + +# def test_parse_var_slice_nested(self): +# y03 = parsing.ParsedVariable([['y', slice(0,3)]]) +# x14y = parsing.ParsedVariable([['x', slice(1,4), y03]]) +# indata = 'x[1:4, y[0:3]]' +# actual = parsing.parse_definition(indata) +# expected = x14y +# testname = 'parse_definition({0!r})'.format(indata) +# print_test_message(testname, indata=indata, +# actual=actual, expected=expected) +# self.assertEqual(actual, expected, +# 'Variable parsing failed') + +#===== NEGATION ================================================================ + + def test_parse_neg_integer(self): + indata = '-1' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedUniOp([['-', 1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Negation parsing failed') + + def test_parse_neg_float(self): + indata = '-1.4' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedUniOp([['-', 1.4]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Negation parsing failed') + + def test_parse_neg_var(self): + indata = '-x' + actual = parsing.parse_definition(indata) + x = parsing.ParsedVariable([['x']]) + expected = parsing.ParsedUniOp([['-', x]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Negation parsing failed') + + def test_parse_neg_func(self): + indata = '-f()' + actual = parsing.parse_definition(indata) + f = parsing.ParsedFunction([['f']]) + expected = parsing.ParsedUniOp([['-', f]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Negation parsing failed') + +#===== POSITIVE ================================================================ + + def test_parse_pos_integer(self): + indata = '+1' + actual = parsing.parse_definition(indata) + expected = 1 + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Positive operator parsing failed') + + def test_parse_pos_float(self): + indata = '+1e7' + actual = parsing.parse_definition(indata) + expected = 1e7 + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Positive operator parsing failed') + + def test_parse_pos_func(self): + indata = '+f()' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedFunction([['f']]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Positive operator parsing failed') + + def test_parse_pos_var(self): + indata = '+x[1]' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedVariable([['x', 1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Positive operator parsing failed') + +#===== POWER =================================================================== + + def test_parse_int_pow_int(self): + indata = '2^1' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedBinOp([['^', 2, 1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Power operator parsing failed') + + def test_parse_float_pow_float(self): + indata = '2.4 ^ 1e7' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedBinOp([['^', 2.4, 1e7]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Power operator parsing failed') + + def test_parse_func_pow_func(self): + indata = 'f() ^ g(1)' + actual = parsing.parse_definition(indata) + f = parsing.ParsedFunction([['f']]) + g1 = parsing.ParsedFunction([['g', 1]]) + expected = parsing.ParsedBinOp([['^', f, g1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Power operator parsing failed') + + def test_parse_var_pow_var(self): + indata = 'x[1] ^ y' + actual = parsing.parse_definition(indata) + x1 = parsing.ParsedVariable([['x', 1]]) + y = parsing.ParsedVariable([['y']]) + expected = parsing.ParsedBinOp([['^', x1, y]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Power operator parsing failed') + +#===== DIV ===================================================================== + + def test_parse_int_div_int(self): + indata = '2/1' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedBinOp([['/', 2, 1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Division operator parsing failed') + + def test_parse_float_div_float(self): + indata = '2.4/1e7' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedBinOp([['/', 2.4, 1e7]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Division operator parsing failed') + + def test_parse_func_div_func(self): + indata = 'f() / g(1)' + actual = parsing.parse_definition(indata) + f = parsing.ParsedFunction([['f']]) + g1 = parsing.ParsedFunction([['g', 1]]) + expected = parsing.ParsedBinOp([['/', f, g1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Division operator parsing failed') + + def test_parse_var_div_var(self): + indata = 'x[1] / y' + actual = parsing.parse_definition(indata) + x1 = parsing.ParsedVariable([['x', 1]]) + y = parsing.ParsedVariable([['y']]) + expected = parsing.ParsedBinOp([['/', x1, y]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Division operator parsing failed') + +#===== MUL ===================================================================== + + def test_parse_int_mul_int(self): + indata = '2*1' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedBinOp([['*', 2, 1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Multiplication operator parsing failed') + + def test_parse_float_mul_float(self): + indata = '2.4*1e7' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedBinOp([['*', 2.4, 1e7]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Multiplication operator parsing failed') + + def test_parse_func_mul_func(self): + indata = 'f() * g(1)' + actual = parsing.parse_definition(indata) + f = parsing.ParsedFunction([['f']]) + g1 = parsing.ParsedFunction([['g', 1]]) + expected = parsing.ParsedBinOp([['*', f, g1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Multiplication operator parsing failed') + + def test_parse_var_mul_var(self): + indata = 'x[1] * y' + actual = parsing.parse_definition(indata) + x1 = parsing.ParsedVariable([['x', 1]]) + y = parsing.ParsedVariable([['y']]) + expected = parsing.ParsedBinOp([['*', x1, y]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Multiplication operator parsing failed') + +#===== ADD ===================================================================== + + def test_parse_int_add_int(self): + indata = '2+1' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedBinOp([['+', 2, 1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Addition operator parsing failed') + + def test_parse_float_add_float(self): + indata = '2.4+1e7' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedBinOp([['+', 2.4, 1e7]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Addition operator parsing failed') + + def test_parse_func_add_func(self): + indata = 'f() + g(1)' + actual = parsing.parse_definition(indata) + f = parsing.ParsedFunction([['f']]) + g1 = parsing.ParsedFunction([['g', 1]]) + expected = parsing.ParsedBinOp([['+', f, g1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Addition operator parsing failed') + + def test_parse_var_add_var(self): + indata = 'x[1] + y' + actual = parsing.parse_definition(indata) + x1 = parsing.ParsedVariable([['x', 1]]) + y = parsing.ParsedVariable([['y']]) + expected = parsing.ParsedBinOp([['+', x1, y]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Addition operator parsing failed') + +#===== SUB ===================================================================== + + def test_parse_int_sub_int(self): + indata = '2-1' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedBinOp([['-', 2, 1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Subtraction operator parsing failed') + + def test_parse_float_sub_float(self): + indata = '2.4-1e7' + actual = parsing.parse_definition(indata) + expected = parsing.ParsedBinOp([['-', 2.4, 1e7]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Subtraction operator parsing failed') + + def test_parse_func_sub_func(self): + indata = 'f() - g(1)' + actual = parsing.parse_definition(indata) + f = parsing.ParsedFunction([['f']]) + g1 = parsing.ParsedFunction([['g', 1]]) + expected = parsing.ParsedBinOp([['-', f, g1]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Subtraction operator parsing failed') + + def test_parse_var_sub_var(self): + indata = 'x[1] - y' + actual = parsing.parse_definition(indata) + x1 = parsing.ParsedVariable([['x', 1]]) + y = parsing.ParsedVariable([['y']]) + expected = parsing.ParsedBinOp([['-', x1, y]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Subtraction operator parsing failed') + +#===== Integration ============================================================= + + def test_parse_integrated_1(self): + indata = '2-17.3*x^2' + actual = parsing.parse_definition(indata) + x = parsing.ParsedVariable([['x']]) + x2 = parsing.ParsedBinOp([['^', x, 2]]) + m17p3x2 = parsing.ParsedBinOp([['*', 17.3, x2]]) + expected = parsing.ParsedBinOp([['-', 2, m17p3x2]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Integrated operator parsing failed') + + def test_parse_integrated_2(self): + indata = '2-17.3*x / f(2.3, x[2:5])' + actual = parsing.parse_definition(indata) + x = parsing.ParsedVariable([['x']]) + x25 = parsing.ParsedVariable([['x', slice(2,5)]]) + f = parsing.ParsedFunction([['f', 2.3, x25]]) + dxf = parsing.ParsedBinOp([['/', x, f]]) + m17p3dxf = parsing.ParsedBinOp([['*', 17.3, dxf]]) + expected = parsing.ParsedBinOp([['-', 2, m17p3dxf]]) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Integrated operator parsing failed') + + +#=============================================================================== +# Command-Line Operation +#=============================================================================== +if __name__ == "__main__": + #import sys;sys.argv = ['', 'Test.testName'] + unittest.main() diff --git a/source/pyconform/test/slicetupleTests.py b/source/pyconform/test/slicetupleTests.py new file mode 100644 index 00000000..c61730e2 --- /dev/null +++ b/source/pyconform/test/slicetupleTests.py @@ -0,0 +1,249 @@ +""" +DiGraph Unit Tests + +COPYRIGHT: 2016, University Corporation for Atmospheric Research +LICENSE: See the LICENSE.rst file for details +""" + +from pyconform import slicetuple +from os import linesep + +import unittest + + +#=============================================================================== +# General Functions +#=============================================================================== +def print_test_message(testname, indata=None, actual=None, expected=None): + print '{}:'.format(testname) + print ' - indata = {0}'.format(indata).replace(linesep, ' ') + print ' - actual = {0}'.format(actual).replace(linesep, ' ') + print ' - expected = {0}'.format(expected).replace(linesep, ' ') + print + + +#=============================================================================== +# SliceTupleTests +#=============================================================================== +class SliceTupleTests(unittest.TestCase): + """ + Unit tests for the slicetuple.SliceTuple class + """ + + def test_init_default(self): + testname = 'SliceTuple.__init__()' + s = slicetuple.SliceTuple() + actual = type(s) + expected = slicetuple.SliceTuple + print_test_message(testname, + actual=actual, expected=expected) + self.assertIsInstance(s, expected, + '{0} returned unexpected result'.format(testname)) + + def test_init_int(self): + indata = 1 + testname = 'SliceTuple.__init__({0})'.format(indata) + s = slicetuple.SliceTuple(indata) + actual = type(s) + expected = slicetuple.SliceTuple + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertIsInstance(s, expected, + '{0} returned unexpected result'.format(testname)) + + def test_init_slice(self): + indata = slice(1,4) + testname = 'SliceTuple.__init__({0})'.format(indata) + s = slicetuple.SliceTuple(indata) + actual = type(s) + expected = slicetuple.SliceTuple + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertIsInstance(s, expected, + '{0} returned unexpected result'.format(testname)) + + def test_init_tuple(self): + indata = (1, slice(2,5)) + testname = 'SliceTuple.__init__({0})'.format(indata) + s = slicetuple.SliceTuple(indata) + actual = type(s) + expected = slicetuple.SliceTuple + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertIsInstance(s, expected, + '{0} returned unexpected result'.format(testname)) + + def test_str_default(self): + testname = 'str(SliceTuple())' + s = slicetuple.SliceTuple() + actual = str(s) + expected = '(::)' + print_test_message(testname, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + '{0} returned unexpected result'.format(testname)) + + def test_str_int(self): + indata = 1 + testname = 'str(SliceTuple({0}))'.format(indata) + s = slicetuple.SliceTuple(indata) + actual = str(s) + expected = '(1)' + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + '{0} returned unexpected result'.format(testname)) + + def test_str_slice_int(self): + indata = slice(1) + testname = 'str(SliceTuple({0}))'.format(indata) + s = slicetuple.SliceTuple(indata) + actual = str(s) + expected = '(:1:)' + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + '{0} returned unexpected result'.format(testname)) + + def test_str_slice_none_int(self): + indata = slice(None,4) + testname = 'str(SliceTuple({0}))'.format(indata) + s = slicetuple.SliceTuple(indata) + actual = str(s) + expected = '(:4:)' + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + '{0} returned unexpected result'.format(testname)) + + def test_str_slice_int_int(self): + indata = slice(1,4) + testname = 'str(SliceTuple({0}))'.format(indata) + s = slicetuple.SliceTuple(indata) + actual = str(s) + expected = '(1:4:)' + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + '{0} returned unexpected result'.format(testname)) + + def test_str_slice_int_int_int(self): + indata = slice(1,4,2) + testname = 'str(SliceTuple.({0}))'.format(indata) + s = slicetuple.SliceTuple(indata) + actual = str(s) + expected = '(1:4:2)' + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + '{0} returned unexpected result'.format(testname)) + + def test_str_tuple(self): + indata = (1, slice(2,5)) + testname = 'str(SliceTuple({0}))'.format(indata) + s = slicetuple.SliceTuple(indata) + actual = str(s) + expected = '(1,2:5:)' + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + '{0} returned unexpected result'.format(testname)) + + def test_index_default(self): + testname = 'SliceTuple().index' + s = slicetuple.SliceTuple() + actual = s.index + expected = (slice(None),) + print_test_message(testname, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + '{0} returned unexpected result'.format(testname)) + + def test_index_int(self): + indata = 1 + testname = 'SliceTuple({0}).index'.format(indata) + s = slicetuple.SliceTuple(indata) + actual = s.index + expected = (indata,) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + '{0} returned unexpected result'.format(testname)) + + def test_index_slice(self): + indata = slice(1,4,2) + testname = 'SliceTuple({0}).index'.format(indata) + s = slicetuple.SliceTuple(indata) + actual = s.index + expected = (indata,) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + '{0} returned unexpected result'.format(testname)) + + def test_index_tuple(self): + indata = (7, slice(1,4,2), 3) + testname = 'SliceTuple({0}).index'.format(indata) + s = slicetuple.SliceTuple(indata) + actual = s.index + expected = indata + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + '{0} returned unexpected result'.format(testname)) + + def test_equal_default(self): + testname = 'SliceTuple() == SliceTuple()' + s1 = slicetuple.SliceTuple() + s2 = slicetuple.SliceTuple() + actual = s1 == s2 + expected = True + print_test_message(testname, + actual=actual, expected=expected) + self.assertTrue(actual, + '{0} returned unexpected result'.format(testname)) + + def test_equal_int(self): + indata = 1 + testname = 'SliceTuple({0}) == SliceTuple({0})'.format(indata) + s1 = slicetuple.SliceTuple(indata) + s2 = slicetuple.SliceTuple(indata) + actual = s1 == s2 + expected = True + print_test_message(testname, + actual=actual, expected=expected) + self.assertTrue(actual, + '{0} returned unexpected result'.format(testname)) + + def test_equal_slice(self): + indata1 = slice(None, 4) + indata2 = slice(4) + testname = 'SliceTuple({0}) == SliceTuple({1})'.format(indata1, indata2) + s1 = slicetuple.SliceTuple(indata1) + s2 = slicetuple.SliceTuple(indata2) + actual = s1 == s2 + expected = True + print_test_message(testname, + actual=actual, expected=expected) + self.assertTrue(actual, + '{0} returned unexpected result'.format(testname)) + + def test_not_equal_slice(self): + indata1 = slice(None, 4, 2) + indata2 = slice(4) + testname = 'SliceTuple({0}) == SliceTuple({1})'.format(indata1, indata2) + s1 = slicetuple.SliceTuple(indata1) + s2 = slicetuple.SliceTuple(indata2) + actual = s1 != s2 + expected = True + print_test_message(testname, + actual=actual, expected=expected) + self.assertTrue(actual, + '{0} returned unexpected result'.format(testname)) + +#=============================================================================== +# Command-Line Operation +#=============================================================================== +if __name__ == "__main__": + #import sys;sys.argv = ['', 'Test.testName'] + unittest.main()