From 793dc96e923a78311fbd91139185672e89f07e45 Mon Sep 17 00:00:00 2001 From: Chris Piker Date: Tue, 3 Sep 2024 02:31:43 -0500 Subject: [PATCH] Stream encoding passes, SPICE and CSV improvments --- das2/buffer.h | 2 +- das2/codec.c | 611 +++++++++++++++++++++++++++++++++++++---- das2/codec.h | 150 +++++++++- das2/dataset.c | 91 ++++-- das2/dataset.h | 17 +- das2/dataset_hdr2.c | 14 +- das2/dataset_hdr3.c | 24 +- das2/encoding.c | 12 +- das2/encoding.h | 2 +- das2/io.c | 11 +- das2/iterator.c | 313 +++++++++++++++------ das2/iterator.h | 109 ++++++-- das2/plane.c | 42 +-- das2/stream.c | 12 +- das2/value.c | 503 ++++++++++++++++++++++++++++++++- das2/value.h | 137 +++++++-- das2/var_ary.c | 27 +- das2/var_base.c | 34 --- das2/var_bin.c | 2 +- das2/var_con.c | 2 +- das2/var_seq.c | 8 +- das2/variable.h | 17 +- test/TestIter.c | 4 +- utilities/das3_csv.c | 27 +- utilities/das3_spice.c | 41 ++- 25 files changed, 1818 insertions(+), 394 deletions(-) diff --git a/das2/buffer.h b/das2/buffer.h index 5dc8b54..4a6c605 100644 --- a/das2/buffer.h +++ b/das2/buffer.h @@ -88,7 +88,7 @@ DAS_API DasErrCode DasBuf_initReadWrite(DasBuf* pThis, char* sBuf, size_t uLen); DAS_API DasErrCode DasBuf_initReadOnly(DasBuf* pThis, const char* sBuf, size_t uLen); /** Re-initialize a buffer including read and write points - * This version can be a little quicker than init_DasBuffer() because it only + * This version can be a little quicker than DasBuffer_init() because it only * zero's out the bytes that were written, not the entire buffer. * * @memberof DasBuf diff --git a/das2/codec.c b/das2/codec.c index 17924d0..111a3f0 100644 --- a/das2/codec.c +++ b/das2/codec.c @@ -20,12 +20,15 @@ #include #include #include +#include + +#include "value.h" +#include "log.h" +#include "iterator.h" #define _das_codec_c_ #include "codec.h" #undef _das_codec_c_ -#include "value.h" -#include "log.h" /* Standard separators for ragged binary real value encoding @@ -63,14 +66,22 @@ const ubyte DAS_DOUBLE_SEP[DASIDX_MAX][8] = { }; - /* Operations flags */ -/* (hdr) DASENC_VALID 0x0001 / * If not 1, not a valid encoder */ -#define DASENC_SWAP 0x0002 /* If set bytes must be swapped prior to IO */ -#define DASENC_CAST 0x0004 /* If set bytes must be transformed prior to IO */ -#define DASENC_TEXT 0x0008 /* Input is text */ -#define DASENC_PARSE 0x0010 /* Input is text that should be parsed to a value */ -#define DASENC_VARSZ 0x0020 /* Input is varible size text items */ +/* (hdr) DASENC_VALID 0x0001 / * If not 1, not a valid encoder */ +#define DASENC_SWAP 0x0002 /* If set bytes must be swapped prior to IO */ + +#define DASENC_CAST_UP 0x0004 /* On read save in larger integral size */ + /* on write save in smaller integral size */ + +#define DASENC_TEXT 0x0008 /* Input is text */ +#define DASENC_PARSE 0x0010 /* Input is text that should be parsed to a value */ + +#define DASENC_VARSZ 0x0020 /* Input is varible size text items */ + +#define DASENC_CAST_DOWN 0x0040 /* On read, halt with error */ + /* on write save in larger integral size */ + +#define DASENC_READER 0x0080 /* If set I'm buffer -> array, if not I'm array -> buffer */ /* Used in the big switch, ignores the valid bit since that's assumed by then */ #define DASENC_MAJ_MASK 0x00FE /* Everyting concerned with the input buffer */ @@ -83,17 +94,52 @@ const ubyte DAS_DOUBLE_SEP[DASIDX_MAX][8] = { #define ENCODER_SETUP_ERROR "Logic error in encoder setup" +/* Change the external format info for a codec, very useful for das3_text! */ +DasErrCode DasCodec_update( + bool bRead, DasCodec* pThis, const char* sEncType, int16_t nSzEach, + ubyte cSep, das_units epoch, const char* sOutFmt +){ + /* Can't just point to existing items, memset is going to erase them! */ + DasAry* _pAry = pThis->pAry; /* okay, is external */ + const char* _sSemantic = pThis->sSemantic; /* Okay, is external */ + + char _sEncType[DASENC_TYPE_LEN] = {'\0'}; + strncpy(_sEncType, (sEncType != NULL) ? sEncType : pThis->sEncType, DASENC_TYPE_LEN - 1); + + int _nSzEach = (nSzEach != 0) ? nSzEach : pThis->nBufValSz; + char _cSep = (cSep != '\0') ? cSep : pThis->sSepSet[0]; + das_units _epoch = (epoch != NULL) ? epoch : pThis->timeUnits; + + char _sOutFmt[DASENC_FMT_LEN] = {'\0'}; + strncpy(_sOutFmt, (sOutFmt != NULL) ? sOutFmt : pThis->sOutFmt, DASENC_TYPE_LEN - 1); + + return DasCodec_init( + bRead, pThis, _pAry, _sSemantic, _sEncType, _nSzEach, _cSep, _epoch, _sOutFmt + ); +} + /* Perform various checks to see if this is even possible */ DasErrCode DasCodec_init( - DasCodec* pThis, DasAry* pAry, const char* sSemantic, const char* sEncType, - int16_t nSzEach, ubyte cSep, das_units epoch + bool bRead, DasCodec* pThis, DasAry* pAry, const char* sSemantic, + const char* sEncType, int16_t nSzEach, ubyte cSep, das_units epoch, + const char* sOutFmt ){ memset(pThis, 0, sizeof(DasCodec)); - pThis->cSep = cSep; + pThis->sSepSet[0] = cSep; pThis->nSep = 1; pThis->pAry = pAry; pThis->nBufValSz = nSzEach; assert(pAry != NULL); + if(bRead) pThis->uProc |= DASENC_READER; + + /* Just save it off. Equivalent information exists in the vtBuf value */ + strncpy(pThis->sEncType, sEncType, DASENC_TYPE_LEN-1); + + /* Save off the semantic & the output format */ + pThis->sSemantic = sSemantic; + if(sOutFmt != NULL) + strncpy(pThis->sOutFmt, sOutFmt, DASENC_FMT_LEN-1); + if(nSzEach == 0) return das_error(DASERR_ENC, "Invalid item size in buffer: 0"); @@ -121,7 +167,7 @@ DasErrCode DasCodec_init( case 4: pThis->vtBuf = vtInt; break; case 2: pThis->vtBuf = vtShort; break; case 1: pThis->vtBuf = vtByte; break; - default: goto UNSUPPORTED; + default: goto BAD_FORMAT; } #ifdef HOST_IS_LSB_FIRST pThis->uProc |= DASENC_SWAP; @@ -134,7 +180,7 @@ DasErrCode DasCodec_init( case 4: pThis->vtBuf = vtInt; break; case 2: pThis->vtBuf = vtShort; break; case 1: pThis->vtBuf = vtByte; break; - default: goto UNSUPPORTED; + default: goto BAD_FORMAT; } bIntegral = true; } @@ -144,7 +190,7 @@ DasErrCode DasCodec_init( case 4: pThis->vtBuf = vtUInt; break; case 2: pThis->vtBuf = vtUShort; break; case 1: pThis->vtBuf = vtUByte; break; - default: goto UNSUPPORTED; + default: goto BAD_FORMAT; } #ifdef HOST_IS_LSB_FIRST pThis->uProc |= DASENC_SWAP; @@ -157,7 +203,7 @@ DasErrCode DasCodec_init( case 4: pThis->vtBuf = vtUInt; break; case 2: pThis->vtBuf = vtUShort; break; case 1: pThis->vtBuf = vtUByte; break; - default: goto UNSUPPORTED; + default: goto BAD_FORMAT; } bIntegral = true; } @@ -165,7 +211,7 @@ DasErrCode DasCodec_init( switch(nSzEach){ case 8: pThis->vtBuf = vtDouble; break; case 4: pThis->vtBuf = vtFloat; break; - default: goto UNSUPPORTED; + default: goto BAD_FORMAT; } #ifdef HOST_IS_LSB_FIRST pThis->uProc |= DASENC_SWAP; @@ -176,36 +222,39 @@ DasErrCode DasCodec_init( switch(nSzEach){ case 8: pThis->vtBuf = vtDouble; break; case 4: pThis->vtBuf = vtFloat; break; - default: goto UNSUPPORTED; + default: goto BAD_FORMAT; } bIntegral = true; } else if(strcmp(sEncType, "byte") == 0){ - if(nSzEach != 1) goto UNSUPPORTED; + if(nSzEach != 1) goto BAD_FORMAT; pThis->vtBuf = vtByte; bIntegral = true; } else if(strcmp(sEncType, "ubyte") == 0){ - if(nSzEach != 1) goto UNSUPPORTED; + if(nSzEach != 1) goto BAD_FORMAT; pThis->vtBuf = vtUByte; bIntegral = true; } if(bIntegral){ - if(das_vt_size(pThis->vtBuf) > das_vt_size(vtAry)) - goto UNSUPPORTED; + if(das_vt_size(pThis->vtBuf) > das_vt_size(vtAry)){ + if(bRead) goto UNSUPPORTED_READ; + else pThis->uProc |= DASENC_CAST_DOWN; + } - /* If the array value type is floating point then it must - and the buffer type is integer, then it must be wider then - the integers */ + /* If the array value type is floating point and the buffer type is + integer, then it must be wider then the integers ??? */ if(das_vt_isint(pThis->vtBuf) && das_vt_isreal(vtAry)){ - if(das_vt_size(vtAry) == das_vt_size(pThis->vtBuf)) - goto UNSUPPORTED; + if(das_vt_size(vtAry) == das_vt_size(pThis->vtBuf)){ + if(bRead) goto UNSUPPORTED_READ; + else pThis->uProc |= DASENC_CAST_DOWN; + } } /* I need to cast values up to a larger size, flag that */ - if(das_vt_size(pThis->vtBuf) != das_vt_size(vtAry)) - pThis->uProc |= DASENC_CAST; + if(das_vt_size(pThis->vtBuf) < das_vt_size(vtAry)) + pThis->uProc |= DASENC_CAST_UP; /* Temporary: Remind myself to call DasAry_markEnd() when writing non-string variable length items */ @@ -217,7 +266,8 @@ DasErrCode DasCodec_init( } if(strcmp(sEncType, "utf8") != 0){ - goto UNSUPPORTED; + /* goto UNSUPPORTED; */ + goto UNSUPPORTED_READ; /* <-- could use generic unsupported instead */ } pThis->vtBuf = vtText; @@ -246,15 +296,15 @@ DasErrCode DasCodec_init( if(vtAry != vtTime){ if( (epoch == NULL) || (! Units_haveCalRep(epoch) ) ) - goto UNSUPPORTED; + goto UNSUPPORTED_READ; /* just UNSUPPORTED ? */ /* Check that the array element size is big enough for the units in question If the data are not stored as text */ if((epoch == UNIT_TT2000)&&(vtAry != vtLong)&&(vtAry != vtDouble)) - goto UNSUPPORTED; + goto UNSUPPORTED_READ; /* just UNSUPPORTED ? */ else if((vtAry != vtDouble)&&(vtAry != vtFloat)) - goto UNSUPPORTED; + goto UNSUPPORTED_READ; /* just UNSUPPORTED ? */ } pThis->timeUnits = epoch; /* In addition to parsing, we have to convert */ @@ -268,7 +318,8 @@ DasErrCode DasCodec_init( /* Expect uByte storage for strings not vtText as there is no external place to put the string data */ if((vtAry != vtUByte)&&(vtAry != vtByte)) - goto UNSUPPORTED; + /* goto UNSUPPORTED; */ + goto UNSUPPORTED_READ; if(DasAry_getUsage(pThis->pAry) & D2ARY_AS_STRING){ pThis->uProc |= DASENC_NULLTERM; @@ -276,14 +327,17 @@ DasErrCode DasCodec_init( /* If storing string data, we need to see if the last index of the array is big enough */ - if((nLastIdxSz != DASIDX_RAGGED)&&(nLastIdxSz < nSzEach)) - goto UNSUPPORTED; + if((nLastIdxSz != DASIDX_RAGGED)&&(nLastIdxSz < nSzEach)){ + /* goto UNSUPPORTED; <-- could probably use generic one here */ + goto UNSUPPORTED_READ; + } if((nLastIdxSz == DASIDX_RAGGED)&&(nRank > 1)) pThis->uProc |= DASENC_WRAP; /* Wrap last index for ragged strings */ } else{ - goto UNSUPPORTED; + /* goto UNSUPPORTED; <-- could probably use generic one here */ + goto UNSUPPORTED_READ; } @@ -291,7 +345,12 @@ DasErrCode DasCodec_init( pThis->uProc |= DASENC_VALID; /* Set the valid encoding bit */ return DAS_OKAY; - UNSUPPORTED: + BAD_FORMAT: + return das_error(DASERR_ENC, "For array %s: %d byte %s encoding is not understood.", + nSzEach, sEncType + ); + + UNSUPPORTED_READ: if(bDateTime) return das_error(DASERR_ENC, "For array %s: Can not encode/decode datetime data from buffers " "with encoding '%s' for items of %hd bytes each to/from an array of " @@ -305,6 +364,33 @@ DasErrCode DasCodec_init( ); } +DasErrCode DasCodec_setUtf8Fmt( + DasCodec* pThis, const char* sValFmt, int16_t nFmtWidth, ubyte nSep, + const char* sSepSet +){ + if((pThis->vtBuf != vtText)||(pThis->vtBuf != vtTime)) + return das_error(DASERR_ENC, "Output encoding is, %s, not UTF-8", + das_vt_serial_type(pThis->vtBuf) + ); + + if(nFmtWidth > 0){ + pThis->nBufValSz = nFmtWidth + 1; + } + else{ + pThis->nBufValSz = -1; /* Go to pure variable with output */ + } + + strncpy(pThis->sOutFmt, sValFmt, DASENC_FMT_LEN - 1); + if((nSep > 0) && (sSepSet != NULL)) + memcpy(pThis->sSepSet, sSepSet, (nSep < DASIDX_MAX ? nSep : DASIDX_MAX) ); + + return DAS_OKAY; +} + +bool DasCodec_isReader(const DasCodec* pThis){ + return (pThis->uProc & DASENC_READER); +} + /* ************************************************************************* */ void DasCodec_postBlit(DasCodec* pThis, DasAry* pAry){ @@ -369,6 +455,7 @@ static DasErrCode _swap_read(ubyte* pDest, const ubyte* pSrc, size_t uVals, int /* ************************************************************************* */ /* Read helper */ +/* TODO: Refactor via das_value_binXform */ static DasErrCode _cast_read( ubyte* pDest, const ubyte* pSrc, size_t uVals, das_val_type vtAry, das_val_type vtBuf ){ @@ -452,6 +539,7 @@ static DasErrCode _cast_read( /* ************************************************************************* */ /* Read helper */ +/* TODO: Refactor via das_value_binXform */ #define _SWP2(p) val[0] = *(p+1); val[1] = *(p) #define _SWP4(p) val[0] = *(p+3); val[1] = *(p+2); val[2] = *(p+1); val[3] = *(p) @@ -667,7 +755,7 @@ static int _var_text_read( bool bParse = ((pThis->uProc & DASENC_PARSE) != 0); int nRet; - char cSep = pThis->cSep; + char cSep = pThis->sSepSet[0]; const char* pRead = (const char*)pBuf; int nLeft = nBufLen; @@ -753,6 +841,11 @@ int DasCodec_decode( DasCodec* pThis, const ubyte* pBuf, int nBufLen, int nExpect, int* pValsRead ){ assert(pThis->uProc & DASENC_VALID); + + if((pThis->uProc & DASENC_READER) == 0) + return -1 * das_error(DASERR_ENC, + "Codec is set to encode mode, call DasEncode_update() to change" + ); if(nExpect == 0) return nBufLen; /* Successfully do nothing */ if(nBufLen == 0) return 0; @@ -800,7 +893,7 @@ int DasCodec_decode( switch(pThis->uProc & DASENC_MAJ_MASK){ /* Easy mode, external data and internal array have the same value type */ - case 0: + case DASENC_READER: assert(pThis->nBufValSz == pThis->nAryValSz); assert(pThis->nBufValSz > 0); assert(nValsToRead > 0); @@ -813,7 +906,7 @@ int DasCodec_decode( /* Almost easy, only need to swap to get into internal storage */ - case DASENC_SWAP: + case DASENC_READER|DASENC_SWAP: assert(pThis->nBufValSz == pThis->nAryValSz); assert(nValsToRead > 0); assert(pThis->nBufValSz > 0); @@ -829,8 +922,8 @@ int DasCodec_decode( break; - /* Need to cast values to a larger type for storage */ - case DASENC_CAST: + /* Need to cast values up to a larger type for storage */ + case DASENC_READER|DASENC_CAST_UP: assert(nValsToRead > 0); assert(pThis->nBufValSz > 0); @@ -844,9 +937,14 @@ int DasCodec_decode( nBytesRead = nValsToRead * (pThis->nBufValSz); break; + case DASENC_READER|DASENC_CAST_DOWN: + case DASENC_READER|DASENC_CAST_DOWN|DASENC_SWAP: + return das_error(DASERR_ENC, "Downcasting to smaller types not supported on read"); + break; + /* Bigest binary change, swap and cast to a larger type for storage */ - case DASENC_CAST|DASENC_SWAP: + case DASENC_READER|DASENC_CAST_UP|DASENC_SWAP: assert(nValsToRead > 0); assert(pThis->nBufValSz > 0); @@ -861,7 +959,7 @@ int DasCodec_decode( /* Easy, just run in the text, don't markEnd */ - case DASENC_TEXT: + case DASENC_READER|DASENC_TEXT: assert(nValsToRead > 0); assert(pThis->nBufValSz > 0); assert((pThis->uProc & DASENC_WRAP) == 0); @@ -881,7 +979,7 @@ int DasCodec_decode( /* Fixed length text to parse, then run in, also common in das2 */ - case DASENC_TEXT|DASENC_PARSE: + case DASENC_READER|DASENC_TEXT|DASENC_PARSE: assert(nValsToRead > 0); assert(pThis->nBufValSz > 0); @@ -895,8 +993,8 @@ int DasCodec_decode( /* Search for the end, run and data, markEnd if array is variable size */ /* or search of the end, parse, then run in the data */ - case DASENC_TEXT|DASENC_VARSZ: - case DASENC_TEXT|DASENC_PARSE|DASENC_VARSZ: + case DASENC_READER|DASENC_TEXT|DASENC_VARSZ: + case DASENC_READER|DASENC_TEXT|DASENC_PARSE|DASENC_VARSZ: { int nValsDidRead = 0; nBytesRead = _var_text_read(pThis, pBuf, nBufLen, nValsToRead, &nValsDidRead); @@ -922,10 +1020,417 @@ int DasCodec_decode( /* ************************************************************************* */ /* Main encoder */ -int DasCodec_encode(DasCodec* pThis, DasBuf* pBuf, int nWrite, bool bLast) -{ - DasBuf_printf(pBuf, "Adding %d values here\n", nWrite); - return nWrite; +/* Encode helper: Swap items before writing ******************************** */ + +/* TODO: Refactor via das_value_binXform */ + +static DasErrCode _swap_write(DasBuf* pBuf, const ubyte* pSrc, size_t uVals, int nSzEa){ + /* Now swap and write */ + ubyte uSwap[8]; + + switch(nSzEa){ + case 2: + for(size_t u = 0; u < (uVals*2); u += 2){ + uSwap[0] = pSrc[u+1]; + uSwap[1] = pSrc[u]; + DasBuf_write(pBuf, uSwap, 2); + } + case 4: + for(size_t u = 0; u < (uVals*4); u += 4){ + uSwap[0] = pSrc[u+3]; + uSwap[1] = pSrc[u+2]; + uSwap[2] = pSrc[u+1]; + uSwap[3] = pSrc[u]; + DasBuf_write(pBuf, uSwap, 4); + } + case 8: + for(size_t u = 0; u < (uVals*8); u += 8){ + uSwap[0] = pSrc[u+7]; + uSwap[1] = pSrc[u+6]; + uSwap[2] = pSrc[u+5]; + uSwap[3] = pSrc[u+4]; + uSwap[4] = pSrc[u+3]; + uSwap[5] = pSrc[u+2]; + uSwap[6] = pSrc[u+1]; + uSwap[7] = pSrc[u]; + DasBuf_write(pBuf, uSwap, 8); + } + default: + return das_error(DASERR_ENC, "Logic error"); + } + return DAS_OKAY; +} + +/* Encode helper: change widths with checks ******************************** */ + +static DasErrCode _cast_write( + DasBuf* pBuf, const ubyte* pSrc, size_t uVals, das_val_type vtAry, + const ubyte* pFillIn, das_val_type vtBuf, const ubyte* pFillOut +){ + + uint64_t outval; + uint64_t outswap; + + size_t outsize = das_vt_size(vtBuf); + assert(outsize <= 8); + + ubyte* pIn = NULL; + ubyte* pOut = NULL; + + int nRet; + for(size_t u = 0; u < (uVals); u += outsize){ + + nRet = das_value_binXform( + vtAry, pSrc + u, pFillIn, + vtBuf, (ubyte*)(&outval), pFillOut, + 0 + ); + if(nRet != DAS_OKAY) return nRet; + + pIn = (ubyte*)(&outval) + (outsize - 1); + pOut = (ubyte*)(&outswap); + for(size_t v = 0; v < outsize; ++v){ + *pOut = *pIn; + --pIn; + ++pOut; + } + + nRet = DasBuf_write(pBuf, (ubyte*)&outswap, outsize); + if(nRet != DAS_OKAY) return nRet; + } + + return DAS_OKAY; +} + +/* Encode helper: change widths with checks and swap *********************** */ + +static DasErrCode _cast_swap_write( + DasBuf* pBuf, const ubyte* pSrc, size_t uVals, das_val_type vtAry, + const ubyte* pFillIn, das_val_type vtBuf, const ubyte* pFillOut +){ + + uint64_t outval; + size_t outsize = das_vt_size(vtBuf); + + int nRet; + for(size_t u = 0; u < (uVals); u += outsize){ + + nRet = das_value_binXform( + vtAry, pSrc + u, pFillIn, + vtBuf, (ubyte*)(&outval), pFillOut, + 0 + ); + if(nRet != DAS_OKAY) return nRet; + + nRet = DasBuf_write(pBuf, (ubyte*)&outval, outsize); + if(nRet != DAS_OKAY) return nRet; + } + + return DAS_OKAY; +} + +/* Encode helper: Print as text ******************************************** */ -} \ No newline at end of file +/* This one has more presentation layer stuff the normal so that blocks + in headers look attractive */ + +DasErrCode _DasCodec_printItems( + DasCodec* pThis, DasBuf* pBuf, const ubyte* pItem0, int nToWrite, + uint32_t uFlags +){ + das_val_type vt = DasAry_valType(pThis->pAry); + size_t uSzEa = DasAry_valSize(pThis->pAry); + char cSep = pThis->sSepSet[0]; + if(cSep == '\0') cSep = ' '; /* They didn't set one, pick a default */ + + das_time dt; + + int nRet = 0; + if(pThis->sOutFmt[0] == '\0'){ + /* We add a separator after all ascii items, so use one less then + the allotted space for the format string */ + nRet = das_value_fmt( + pThis->sOutFmt, DASENC_FMT_LEN, vt, pThis->sSemantic, + (pThis->nBufValSz > 1) ? (pThis->nBufValSz -1) : - 1 + ); + if(nRet != DAS_OKAY) + return nRet; + } + + /* If the header flag is set wrap after 100 chars or so */ + int nRoughOutEa = 25; + int nTmp = pThis->nBufValSz; + switch(vt){ + case vtUByte: case vtByte: nRoughOutEa = (nTmp > 1 ) ? nTmp : 5; break; + case vtUShort: case vtShort: nRoughOutEa = (nTmp > 1 ) ? nTmp : 8; break; + case vtUInt: case vtInt: nRoughOutEa = (nTmp > 1 ) ? nTmp :12; break; + case vtULong: case vtLong: nRoughOutEa = (nTmp > 1 ) ? nTmp :20; break; + case vtFloat: nRoughOutEa = (nTmp > 1 ) ? nTmp :12; break; /* Assume that alot of these get trimmed */ + case vtDouble: nRoughOutEa = (nTmp > 1 ) ? nTmp :15; break; + case vtTime: nRoughOutEa = (nTmp > 1 ) ? nTmp :24; break; + default: nRoughOutEa = 25; break; + } + + int nRowLen = 0; + bool bInHdr = uFlags & DASENC_IN_HDR; + char sReal[64] = {'\0'}; + + for(int i = 0; i < nToWrite; ++i){ + if(i > 0){ + if(bInHdr&&(nRowLen > 100)&&(cSep == ' ')){ + DasBuf_write(pBuf, "\n ", 9); + nRowLen = 0; + } + else{ DasBuf_write(pBuf, &cSep, 1); } + } + else{ if(bInHdr) DasBuf_write(pBuf, " ", 8); } + + switch(vt){ + case vtUByte: DasBuf_printf(pBuf, pThis->sOutFmt, *(pItem0 + i) ); break; + case vtByte: DasBuf_printf(pBuf, pThis->sOutFmt, *((ubyte* )(pItem0 + i)) ); break; + case vtUShort: DasBuf_printf(pBuf, pThis->sOutFmt, *((uint16_t*)(pItem0 + i*uSzEa)) ); break; + case vtShort: DasBuf_printf(pBuf, pThis->sOutFmt, *((int16_t*)(pItem0 + i*uSzEa)) ); break; + case vtUInt: DasBuf_printf(pBuf, pThis->sOutFmt, *((uint32_t*)(pItem0 + i*uSzEa)) ); break; + case vtInt: DasBuf_printf(pBuf, pThis->sOutFmt, *((int32_t*)(pItem0 + i*uSzEa)) ); break; + case vtULong: DasBuf_printf(pBuf, pThis->sOutFmt, *((uint64_t*)(pItem0 + i*uSzEa)) ); break; + case vtLong: DasBuf_printf(pBuf, pThis->sOutFmt, *((int64_t*)(pItem0 + i*uSzEa)) ); break; + case vtFloat: + case vtDouble: + if(vt == vtFloat) + snprintf(sReal, 63, pThis->sOutFmt, (double) *((float* )(pItem0 + i*uSzEa))); + else + snprintf(sReal, 63, pThis->sOutFmt, *((double* )(pItem0 + i*uSzEa))); + if(bInHdr) das_value_trimReal(sReal); + DasBuf_write(pBuf, sReal, strlen(sReal)); + break; + + case vtTime: + dt = *((das_time*)(pItem0 + i*uSzEa)); + /* why this works... extra arguments are ignored if format string doesn't mention them :-) */ + DasBuf_printf(pBuf, pThis->sOutFmt, dt.year, dt.month, dt.mday, dt.hour, dt.minute, dt.second); + break; + default: + return das_error(DASERR_ENC, "Guess I forgot about '%s'", das_vt_toStr(vt)); + } + nRowLen += nRoughOutEa; + } + + /* Space to next thing */ + if(uFlags & DASENC_PKT_LAST) + DasBuf_write(pBuf, "\n", 1); + else + DasBuf_write(pBuf, &cSep, 1); + + return DAS_OKAY; +} + +/* Main Encoder ************************************************************ */ + +/* The goal of this function is to emitt all data from continuous range of + * indexes starting at a given point. Examples of setting the start location: + * + * nDim=0, pLoc=NULL as DIM0 => Emitt the entire array + * + * nDim=1, pLoc={I} as DIM1_AT(I) => Emit all data for one increment of the + * highest index + * + * nDim=2, pLoc={I,J} as DIM2_AT(I,J) => Emit all data for one increment of the + * the next highest index. + * + * To write all data for an array set: nDim = 0, pLoc = NULL (aka use DIM0 ) + * + * @returns negative error code, or the number of values written + */ + +int DasCodec_encode( + DasCodec* pThis, DasBuf* pBuf, int nDim, ptrdiff_t* pLoc, int nExpect, uint32_t uFlags +){ + + if((pThis->uProc & DASENC_READER) != 0) + return -1 * das_error(DASERR_ENC, + "Codec is set to decode mode, call DasEncode_update() to change" + ); + + DasErrCode nRet = DAS_OKAY; + DasAry* pAry = pThis->pAry; + das_val_type vtAry = DasAry_valType(pAry); + + size_t uAvailable; /* Total items, but for strings this is total bytes + nulls! */ + + const ubyte* pItem0 = DasAry_getIn(pThis->pAry, vtAry, nDim, pLoc, &uAvailable); + + if(uAvailable > 2147483648L) + return -1 * das_error(DASERR_ENC, "too many values at index"); + int nAvailable = (int)uAvailable; + if(nAvailable == 0) + return -1* das_error(DASERR_ENC, "No values were available to write from array %s", + DasAry_id(pAry) + ); + + int nSzEa = (int) DasAry_valSize(pAry); + + /* Make sure the available data = the write request if not var write */ + if((nExpect > 0)&&(nAvailable < nExpect)){ + if(nDim == 0){ + return -1 * das_error(DASERR_ENC, + "Expected to write %d values for %s, but only %d were available in " + "the array", nExpect, DasAry_id(pAry), nAvailable + ); + } + else{ + char sBuf[64] = {'\0'}; + return -1 * das_error(DASERR_ENC, + "Expected to write %d values for %s, but only %d were available under " + "index %td", nExpect, DasAry_id(pAry), nAvailable, + das_idx_prn(nDim, pLoc, 63, sBuf) + ); + } + } + + /* Big switch to avoid decision making in loops. Most of the items can + just be streamed to the output buffer without making decisions about + how to encode each item in a tight loop. */ + + switch(pThis->uProc & DASENC_MAJ_MASK){ + + /* Easy mode, interal and external data format match */ + case 0: + assert(pThis->nBufValSz == nSzEa); + assert(pThis->nBufValSz > 0); + + if( (nRet = DasBuf_write(pBuf, pItem0, nAvailable*nSzEa)) != DAS_OKAY) + return -1 * nRet; + return nAvailable; + + case DASENC_SWAP: + assert(pThis->nBufValSz == nSzEa); + assert(pThis->nBufValSz > 0); + + if((nRet = _swap_write(pBuf, pItem0, nAvailable, nSzEa)) != DAS_OKAY) + return -1 * nRet; + return nAvailable; + break; + + case DASENC_CAST_UP: /* Were read WIDE, write NARROW */ + case DASENC_CAST_DOWN: /* Want even wider output */ + assert(pThis->nBufValSz > 0); + + /* Name not a typo, the #defines are written from the encoding point of view */ + nRet = _cast_write( + pBuf, pItem0, nAvailable, vtAry, DasAry_getFill(pAry), pThis->vtBuf, + das_vt_fill(pThis->vtBuf) + ); + if(nRet != DAS_OKAY) + return -1 * nRet; + + return nAvailable; + break; + + case DASENC_CAST_UP|DASENC_SWAP: /* Were read WIDE, write NARROW, opposite endian */ + case DASENC_CAST_DOWN|DASENC_SWAP: /* Want wider swapped output */ + + assert(pThis->nBufValSz > 0); + + nRet = _cast_swap_write( + pBuf, pItem0, nAvailable, vtAry, DasAry_getFill(pAry), pThis->vtBuf, + das_vt_fill(pThis->vtBuf) + ); + if(nRet != DAS_OKAY) + return -1 * nRet; + + return nAvailable; + break; + + + /* Text stored, text output. Easy, just run out in fixed lengths */ + case DASENC_TEXT: + assert(pThis->nBufValSz > 0); + + /* No wrapping usually means output is fixed size */ + assert((pThis->uProc & DASENC_WRAP) == 0); + int nNulls = 0; + if(pThis->uProc & DASENC_NULLTERM){ + + /* The input must have been fixed length text strings so boundaries are + found by lengths in the header statement. So it looks like we can + just run strings out to the buffer value size, then skip the NULL and + do it again. */ + int iBeg = 0; + while(iBeg < nAvailable){ + nRet = DasBuf_write(pBuf, pItem0 + iBeg, pThis->nBufValSz); + ++nNulls; + iBeg += pThis->nBufValSz + 1; /* skip over the null */ + if(nRet != DAS_OKAY) + break; + } + } + else{ + nRet = DasBuf_write(pBuf, pItem0, nAvailable); + } + if(nRet != DAS_OKAY) return -1 * nRet; + + assert( ((nAvailable - nNulls) % pThis->nBufValSz) == 0); + return (nAvailable - nNulls) / pThis->nBufValSz; + break; + + + /* Binary stored, text output, common in das2 */ + case DASENC_TEXT|DASENC_PARSE: + case DASENC_TEXT|DASENC_PARSE|DASENC_VARSZ: + + /* Note, pThis->nBufValSz can be -1, triggers variable length output */ + + nRet = _DasCodec_printItems(pThis, pBuf, pItem0, nAvailable, uFlags); + if(nRet != DAS_OKAY) + return -1* nRet; + + return nAvailable; + break; + + /* variable width text output, can't avoid array geometry any longer, + use an iterator to stay sane */ + case DASENC_TEXT|DASENC_VARSZ: + { + char cSep = ' '; + if(pThis->sSepSet[0] != '\0') cSep = pThis->sSepSet[0]; + DasAryIter iter; + DasAryIter_init( + &iter, + pAry, + nDim, /* First index to iterate over */ + -2, /* Last index to iterate over (-2 means nAryRank - 2) */ + pLoc, /* Starting location */ + NULL /* No ending location, just exhaust the sub-space */ + ); + + size_t uStrLen = 0; + int nWrote = 0; + size_t uRowChars = 0; + const char* sStr = NULL; + int nAryRank = DasAry_rank(pAry); + for(; !iter.done; DasAryIter_next(&iter)){ + if(uRowChars > 0){ + if(uRowChars > 80) + DasBuf_write(pBuf, "\n", 1); + else + DasBuf_write(pBuf, &cSep, 1); + } + + sStr = DasAry_getCharsIn(pAry, nAryRank - 1, iter.index, &uStrLen); + DasBuf_write(pBuf, sStr, uStrLen); + + uRowChars += uStrLen; + ++nWrote; + } + return nWrote; + } + break; + + default: + break; + } + + /* I must have forgot one ... */ + return -1 * das_error(DASERR_ENC, ENCODER_SETUP_ERROR); +} diff --git a/das2/codec.h b/das2/codec.h index aa8d673..133fb37 100644 --- a/das2/codec.h +++ b/das2/codec.h @@ -26,12 +26,15 @@ #include #include +#include /* <-- only to get DASENC_FMT_LEN, DASENC_TYPE_LEN */ + /* otherwise independent */ + #ifdef __cplusplus extern "C" { #endif /* Not public, only here because used in a macro */ -#define DASENC_VALID 0x0001 +#define DASENC_VALID 0x0001 /** @addtogroup IO * @{ @@ -54,31 +57,43 @@ extern const ubyte DAS_DOUBLE_SEP[DASIDX_MAX][8]; /** Reading and writing array data to buffers */ typedef struct das_codec { - bool bResLossWarn; /* If true, the resolution loss warning has already been emitted */ + bool bResLossWarn; /* If true, the resolution loss warning has already been emitted */ uint32_t uProc; /* Internal processing flags setup on the call to _init */ - int nAryValSz; /* The size of each array value in internal buffer */ + int nAryValSz; /* The size of each array value in internal buffer */ + + char sEncType[DASENC_TYPE_LEN]; - int16_t nBufValSz; /* Width of a single value in the external buffer */ + int16_t nBufValSz; /* Width of a single value in the external buffer */ das_val_type vtBuf; /* The value type in the external buffer */ + const char* sSemantic; /* The intended meaning for the externa item */ + DasAry* pAry; /* The array for which values are encoded/decoded */ - ubyte cSep; /* Split strings on this byte value, in addition to null */ + ubyte nSep; + char sSepSet[DASIDX_MAX]; /* Split strings on these chars by rank */ + + uint32_t uMaxString; /* If we are storing fixed strings, this is set */ - uint32_t uMaxString; /* If we are storing fixed strings, this is set */ + das_units timeUnits; /* If ascii times are to be stored as an integral type + this is needed */ - das_units timeUnits; /* If ascii times are to be stored as an integral type - this is needed */ + /* For output, thte sprintf string (if UTF8) or the stream encode type */ + char sOutFmt[DASENC_FMT_LEN]; - char* pOverflow; /* If the size of a variable length value breaks */ - size_t uOverflow; /* the small vector assumption, extra space is here */ + char* pOverflow; /* If the size of a variable length value breaks the */ + size_t uOverflow; /* small vector assumption, extra space is here */ } DasCodec; +#define DASENC_VARSZOUT -1 +#define DASENC_READ true +#define DASENC_WRITE false + /** Has the memory for this encoder been initialized? * * @memberof DasCodec @@ -87,6 +102,9 @@ typedef struct das_codec { /** Initialize a serial buffer decoder/encoder * + * @param bRead if set to DASENC_READ, perform checks for value reading codecs + * if set to DASENC_WRITE, perform checks for value writing codecs + * * @param pThis A pointer to the memory area to initialize * * @param pAry A pointer to the array which either receive or supply values. @@ -118,9 +136,18 @@ typedef struct das_codec { * string data. By default any space character marks the end of * a string. Use 0 to ignore. * - * @param epoch If time data needs to be converted from UTC strings an epoch + * @param epoch If time data needs to be decoded from UTC strings an epoch * will be needed. Otherwise this field can be NULL * + * @param sOutFmt a printf style format string, may be NULL for input only + * codecs, or to have the initializer set a default output format + * string + * + * Typical strings for general + * data values would be: '%9.2e', '%+13.6e'. In general strings + * such as '%13.3f' should @b not be used as these aren't guarunteed + * to have a fix output width and your value strings may be truncated. + * * @returns DAS_OKAY if an decoder/encoder for can be created for the given * arguments, an error code otherwise. * @@ -133,8 +160,57 @@ typedef struct das_codec { * @memberof DasCodec */ DAS_API DasErrCode DasCodec_init( - DasCodec* pThis, DasAry* pAry, const char* sSemantic, const char* sEncType, - int16_t uSzEach, ubyte cSep, das_units epoch + bool bRead, DasCodec* pThis, DasAry* pAry, const char* sSemantic, + const char* sEncType, int16_t uSzEach, ubyte cSep, das_units epoch, + const char* sOutFmt +); + +/** Update external aspects of a serial buffer decoder/encoder + * + * Other then three manditory items, only propertise you want to change need + * to be non-null, or not-flag values. + * + * @param bRead if set to DASENC_READ, perform checks for value reading codecs + * if set to DASENC_WRITE, perform checks for value writing codecs + * + * @param pThis A pointer to the memory area to initialize + * + * @param sEncType The basic encoding type of data in the buffer, one of: + * - byte : 8-bit signed integer + * - ubyte : 8-bit un-signed integer + * - BEint : A signed integer 2+ bytes long, most significant byte first + * - BEuint : An un-signed integer 2+ bytes long MSB first + * - LEint : Little-endian version of BEint + * - LEuint : Little-endian version of BEuint + * - BEreal : An IEEE-754 floating point value, MSB first + * - LEreal : An IEEE-754 floating point value, LSB first + * - utf8 : A string of text bytes + * or use NULL to leave unchanged + * + * @param nSzEach the number of bytes in an item. Use 0 t leave unchanged. + * + * @param cSep A single byte used to mark the end of a byte sequence for + * string data. Use 0 to leave unchanged. + * + * @param epoch If time data needs to be decoded from UTC strings an epoch + * will be needed. Use NULL to leave unchanged + * + * @param sOutFmt a printf style format string, may be NULL to leave unchanged. + * + * @returns DAS_OKAY if an decoder/encoder for can be created for the given + * arguments, an error code otherwise. + * + * @note For 'string' semantic data where the last index in the array is + * ragged DasAry_markEnd() will be called after each string is read. + * Otherwise, no string larger then the last index will be written + * and zeros will be appended to fill out the last index when reading + * data. + * + * @memberof DasCodec + */ +DAS_API DasErrCode DasCodec_update( + bool bRead, DasCodec* pThis, const char* sEncType, int16_t uSzEach, + ubyte cSep, das_units epoch, const char* sOutFmt ); /** Fix array pointer after a DasCodec memory copy @@ -147,6 +223,12 @@ DAS_API DasErrCode DasCodec_init( */ DAS_API void DasCodec_postBlit(DasCodec* pThis, DasAry* pAry); +/** Is this codec setup and a reader from external buffers or a write to them ? + * + * @memberof DasCodec + */ +DAS_API bool DasCodec_isReader(const DasCodec* pThis); + /** Read values from a simple buffer into an array * * Unlike the old das2 version, this encoder doesn't have a built-in number @@ -181,16 +263,54 @@ DAS_API void DasCodec_postBlit(DasCodec* pThis, DasAry* pAry); * @memberof DasCodec */ DAS_API int DasCodec_decode( - DasCodec* pThis, const ubyte* pBuf, int nBufLen, int nExpect, int* pValsRead + DasCodec* pThis, const ubyte* pBuf, int nBufLen, int nExpect, int* pValsRead ); +#define DASENC_PKT_LAST 0x02 +#define DASENC_IN_HDR 0x04 + /** Write values from an array into a buffer, does not change the array * + * The goal of this function is to emitt all data from continuous range of + * indexes starting at a given point. Examples of setting the start location: + * + * nDim=0, pLoc=NULL as DIM0 => Emitt the entire array + * + * nDim=1, pLoc={I} as DIM1_AT(I) => Emit all data for one increment of the + * highest index + * + * nDim=2, pLoc={I,J} as DIM2_AT(I,J) => Emit all data for one increment of the + * the next highest index. + * + * To write all data for an array set: nDim = 0, pLoc = NULL (aka use DIM0 ) + * + * @param pThis The codec structure + * + * @param pBuf The output receiver + * + * @param nDim Part of the start location specification, see description above + * + * @param pLoc Part of the start location specification, see description above + * + * @param nExpect The number of items expected to be written. If -1 then + * the output is variable length. In the case of text items this is + * the number of strings, not the number of total characters + * + * @param uFlags that affect the output. The following are defined, mostly + * for text output: + * + * DASENC_PKTLAST - This is the last item output for for a packet + * Add a new line character after it if text. + * + * DASENC_INHDR - Encoding is being performed for a header, so + * don't emmit too many items in a single row. + * * @returns The number of values written or a negative ERR code if a data * conversion error occured. */ DAS_API int DasCodec_encode( - DasCodec* pThis, DasBuf* pBuf, int nWrite, bool bLast + DasCodec* pThis, DasBuf* pBuf, int nDim, ptrdiff_t* pLoc, int nExpect, + uint32_t uFlags ); /** Release the reference count on the array given to this encoder/decoder diff --git a/das2/dataset.c b/das2/dataset.c index 4584ce1..c9d367e 100644 --- a/das2/dataset.c +++ b/das2/dataset.c @@ -382,7 +382,7 @@ void _DasDs_codecsGoLarger(DasDs* pThis) DasCodec* DasDs_addFixedCodec( DasDs* pThis, const char* sAryId, const char* sSemantic, - const char* sEncType, int nItemBytes, int nNumItems + const char* sEncType, int nItemBytes, int nNumItems, bool bRead ){ /* Go dynamic? */ @@ -403,7 +403,7 @@ DasCodec* DasDs_addFixedCodec( DasCodec* pCodec = pThis->lCodecs + pThis->uCodecs; DasErrCode nRet = DasCodec_init( - pCodec, pAry, sSemantic, sEncType, nItemBytes, 0, pAry->units + bRead, pCodec, pAry, sSemantic, sEncType, nItemBytes, 0, pAry->units, NULL ); if(nRet != DAS_OKAY) @@ -416,7 +416,8 @@ DasCodec* DasDs_addFixedCodec( } DasCodec* DasDs_addFixedCodecFrom( - DasDs* pThis, const char* sAryId, const DasCodec* pOther, int nNumItems + DasDs* pThis, const char* sAryId, const DasCodec* pOther, int nNumItems, + bool bRead ){ /* Go dynamic? */ if(pThis->uCodecs == DASDS_LOC_ENC_SZ) @@ -444,6 +445,8 @@ DasCodec* DasDs_addFixedCodecFrom( functions only! */ memcpy(pDest, pOther, sizeof(DasCodec)); DasCodec_postBlit(pDest, pAry); + if(DasCodec_isReader(pDest) != bRead) + DasCodec_update(bRead, pDest, NULL, 0, '\0', NULL, NULL); pThis->lItems[pThis->uCodecs] = nNumItems; pThis->uCodecs += 1; @@ -453,7 +456,8 @@ DasCodec* DasDs_addFixedCodecFrom( DasCodec* DasDs_addRaggedCodec( DasDs* pThis, const char* sAryId, const char* sSemantic, const char* sEncType, - int nItemBytes, int nSeps, ubyte uSepLen, const ubyte* pSepByIdx + int nItemBytes, int nSeps, ubyte uSepLen, const ubyte* pSepByIdx, + bool bRead ){ das_error(DASERR_NOTIMP, "Ragged codec creation not yet implimented"); return NULL; @@ -648,14 +652,20 @@ void del_DasDs(DasDs* pThis){ size_t DasDs_clearRagged0(DasDs* pThis) { size_t uBytesCleared = 0; - - int nRank; - ptrdiff_t aShape[DASIDX_MAX] = DASIDX_INIT_UNUSED; - for(int i = 0; i < pThis->uArrays; ++i){ - nRank = DasAry_shape(pThis->lArrays[i], aShape); - - if((nRank >= 1)&&(aShape[0] == DASIDX_RAGGED)) - uBytesCleared += DasAry_clear(pThis->lArrays[i]); + DasDim* pDim = NULL; + DasVar* pVar = NULL; + DasAry* pAry = NULL; + + for(size_t d = 0; d < pThis->uDims; ++d){ + pDim = pThis->lDims[d]; + for(size_t v = 0; v < pDim->uVars; ++v){ + pVar = pDim->aVars[v]; + if(!DasVar_degenerate(pVar, 0)){ + if( (pAry = DasVar_getArray(pVar)) != NULL){ + uBytesCleared += DasAry_clear(pAry) * DasAry_valSize(pAry); + } + } + } } return uBytesCleared; @@ -729,7 +739,7 @@ DasErrCode DasDs_encode(DasDs* pThis, DasBuf* pBuf) ptrdiff_t aShape[DASIDX_MAX] = DASIDX_INIT_UNUSED; int nRank = DasDs_shape(pThis, aShape); - DasBuf_printf(pBuf, " 0) DasBuf_puts(pBuf, ";"); if((i==0)||(aShape[i] == DASIDX_RAGGED)) DasBuf_puts(pBuf, "*"); @@ -853,14 +863,59 @@ DasErrCode DasDs_decodeData(DasDs* pThis, DasBuf* pBuf) * for one increment of the major index of the dataset. * * @returns DAS_OKAY to indicate data was serialized for the given index. - * - * -1 to indicate that data was not sent because iIdx0 is outside - * the range of valid index values - * * A positive error code if there was a problem sending data. */ DasErrCode DasDs_encodeData(DasDs* pThis, DasBuf* pBuf, ptrdiff_t iIdx0) { - return das_error(DASERR_NOTIMP, "Encoding not yet drafted"); + + if(DasDs_numCodecs(pThis) == 0){ + return das_error(DASERR_SERIAL, + "No decoders are defined for dataset %02d in group %s", DasDs_id(pThis), DasDs_group(pThis) + ); + } + + int nSzEncs = (int)DasDs_numCodecs(pThis); + int nValsExpect = 0; + int nValsWrote = 0; + bool bLast = false; + for(int i = 0; i < nSzEncs; ++i){ + DasCodec* pCodec = DasDs_getCodec(pThis, i); + + /* Encoder returns the number of values it wrote */ + nValsWrote = 0; + nValsExpect = DasDs_pktItems(pThis, i); + + if((nValsExpect < 1)&&(i < (nSzEncs - 1))) + return das_error(DASERR_NOTIMP, + "To handle parsing ragged non-text arrays that's not at the end of " + "a packet, add searching for binary sentinals to DasCodec_decode" + ); + + + bLast = (i == ((nSzEncs) - 1)); /* Last encoder can be and uses \n for val sep */ + nValsWrote = DasCodec_encode(pCodec, pBuf, DIM1_AT(iIdx0), nValsExpect, bLast); + if(nValsWrote < 0){ + return -1*nValsWrote; /* negative indicates error condition */ + } + + if(nValsExpect > 0){ + if(nValsExpect != nValsWrote) + return das_error(DASERR_SERIAL, + "Expected to write %d values to a packet for array %s in dataset %s " + "but wrote %d instead.", nValsExpect, DasAry_id(pCodec->pAry), + DasDs_id(pThis), nValsWrote + ); + } + else{ + /* Even for variable number of items, we expect to write something */ + if(nValsWrote == 0){ + return das_error(DASERR_SERIAL, "No values written for array %s in dataset %s", + DasAry_id(pCodec->pAry), DasDs_id(pThis) + ); + } + } + } + + return DAS_OKAY; } \ No newline at end of file diff --git a/das2/dataset.h b/das2/dataset.h index 2a31b58..53228ec 100644 --- a/das2/dataset.h +++ b/das2/dataset.h @@ -614,6 +614,9 @@ DAS_API DasCodec* DasDs_getCodecFor( * * @param nNumItems The number of items to read/write at a time. * + * @param bRead If true initialize a decoder, if false initialize an encoder. + * For readability the macros DASENC_READ and DASENC_WRITE can be used + * * @returns NULL if the codec couldn't be defined, or a pointer to * the new codec otherwise * @@ -621,7 +624,7 @@ DAS_API DasCodec* DasDs_getCodecFor( */ DAS_API DasCodec* DasDs_addFixedCodec( DasDs* pThis, const char* sAryId, const char* sSemantic, - const char* sEncType, int nItemBytes, int nNumItems + const char* sEncType, int nItemBytes, int nNumItems, bool bRead ); /** Add a new codec that initialized via some other codec @@ -640,13 +643,17 @@ DAS_API DasCodec* DasDs_addFixedCodec( * * @param nNumItems The number of items to read/write at a time. * + * @param bRead If true initialize a decoder, if false initialize an encoder. + * For readability the macros DASENC_READ and DASENC_WRITE can be used + * * @returns NULL if the codec couldn't be defined, or a pointer to * the new codec otherwise * * @memberof DasDs */ DAS_API DasCodec* DasDs_addFixedCodecFrom( - DasDs* pThis, const char* sAryId, const DasCodec* pOther, int nNumItems + DasDs* pThis, const char* sAryId, const DasCodec* pOther, int nNumItems, + bool bRead ); @@ -678,6 +685,9 @@ DAS_API DasCodec* DasDs_addFixedCodecFrom( * @param pSepByIdx Pointer to an array of separator bytes. This must * be nSeps * uSepLen long. * + * @param bRead If true initialize a decoder, if false initialize an encoder. + * For readability the macros DASENC_READ and DASENC_WRITE can be used + * * @returns NULL if the codec couldn't be defined, or a pointer to * the new codec otherwise * @@ -685,7 +695,8 @@ DAS_API DasCodec* DasDs_addFixedCodecFrom( */ DAS_API DasCodec* DasDs_addRaggedCodec( DasDs* pThis, const char* sAryId, const char* sSemantic, - const char* sEncType, int nItemBytes, int nSeps, ubyte uSepLen, const ubyte* pSepByIdx + const char* sEncType, int nItemBytes, int nSeps, ubyte uSepLen, + const ubyte* pSepByIdx, bool bRead ); /** Get the number of bytes in each record of this dataset when serialized diff --git a/das2/dataset_hdr2.c b/das2/dataset_hdr2.c index 05347f9..f45c0a8 100644 --- a/das2/dataset_hdr2.c +++ b/das2/dataset_hdr2.c @@ -300,7 +300,7 @@ DasErrCode _serial_addCodec( } DasCodec* pCodec = DasDs_addFixedCodec( - pDs, sAryId, sSemantic, sEncType, nItemBytes, nItems + pDs, sAryId, sSemantic, sEncType, nItemBytes, nItems, DASENC_READ ); if(pCodec == NULL) return das_error(DASERR_BLDR, "Couldn't generate codec for array %s", sAryId); @@ -672,27 +672,27 @@ bool _serial_checkYTags(PktDesc* pPd) return true; } -double* _serial_yTagVals(PlaneDesc* pPlane) +float* _serial_yTagVals(PlaneDesc* pPlane) { if(pPlane->planeType != YScan){ das_error(DASERR_BLDR, "Program logic error"); return NULL; } - double* pTags = (double*)calloc(PlaneDesc_getNItems(pPlane), sizeof(double)); + float* pTags = (float*)calloc(PlaneDesc_getNItems(pPlane), sizeof(float)); size_t u, uItems = PlaneDesc_getNItems(pPlane); const double* pListTags = NULL; double rInterval, rMin, rMax; switch(pPlane->ytag_spec){ case ytags_list: pListTags = PlaneDesc_getYTags(pPlane); - for(u = 0; u < uItems; ++u) pTags[u] = pListTags[u]; + for(u = 0; u < uItems; ++u) pTags[u] = (float) pListTags[u]; break; case ytags_none: for(u = 0; u < uItems; ++u) pTags[u] = u; break; case ytags_series: PlaneDesc_getYTagSeries(pPlane, &rInterval, &rMin, &rMax); - for(u = 0; u < uItems; ++u) pTags[u] = rMin + (rInterval * u); + for(u = 0; u < uItems; ++u) pTags[u] = (float) rMin + (rInterval * u); break; } return pTags; @@ -773,7 +773,7 @@ DasDs* _serial_initYScan( char aDimSrc[LEGACY_MAX_DIMS*LEGACY_SRC_ARY_SZ] = {'\0'}; size_t uDims = 0; - double* pYTags = NULL; + float* pYTags = NULL; bool bAddedYTags = false; for(size_t u = 0; u < pPd->uPlanes; ++u){ pPlane = pPd->planes[u]; @@ -876,7 +876,7 @@ DasDs* _serial_initYScan( else pYTagDim = "ytags"; } } - pAry = new_DasAry(pYTagDim, vtDouble, 0, NULL, RANK_1(uItems), Yunits); + pAry = new_DasAry(pYTagDim, vtFloat, 0, NULL, RANK_1(uItems), Yunits); if(pAry == NULL) return NULL; if(DasDs_addAry(pDs, pAry) != DAS_OKAY) return NULL; pYTags = _serial_yTagVals(pPlane); diff --git a/das2/dataset_hdr3.c b/das2/dataset_hdr3.c index da847aa..489979f 100644 --- a/das2/dataset_hdr3.c +++ b/das2/dataset_hdr3.c @@ -337,6 +337,7 @@ static void _serial_onOpenDim( const char* sPhysDim = NULL; const char* sFrame = NULL; char sAxis[48] = {'\0'}; + char sAnnot[48] = {'\0'}; for(int i = 0; psAttr[i] != NULL; i+=2){ if(strcmp(psAttr[i],"physDim")==0) sPhysDim = psAttr[i+1]; @@ -344,6 +345,8 @@ static void _serial_onOpenDim( else if(strcmp(psAttr[i],"frame")==0) sFrame = psAttr[i+1]; else if((strcmp(psAttr[i],"axis")==0) &&(psAttr[i+1][0] != '\0')) strncpy(sAxis, psAttr[i+1], 47); + else if((strcmp(psAttr[i],"annotation")==0) &&(psAttr[i+1][0] != '\0')) + strncpy(sAnnot, psAttr[i+1], 47); else daslog_warn_v( "Unknown attribute %s in <%s> for dataset ID %02d", psAttr[i], sDimType, id @@ -363,9 +366,9 @@ static void _serial_onOpenDim( if(sPhysDim[0] == '\0') sPhysDim = "none"; - if((dt == DASDIM_COORD) && (sAxis[0] == '\0')){ + if((dt == DASDIM_COORD) && (sAxis[0] == '\0') && (sAnnot[0] == '\0')){ pCtx->nDasErr = das_error(DASERR_SERIAL, - "Attribute \"axis\" missing for physical dimension %s in dataset ID %d", + "Both \"axis\" and \"annotation\" missing for coordinate dimension %s in dataset ID %d", sPhysDim, id ); return; @@ -401,6 +404,11 @@ static void _serial_onOpenDim( } DasDim_primeCoord(pDim, true); } + else if(sAnnot[0] != '\0'){ + strncpy(pDim->axes[0], sAnnot, DASDIM_AXLEN-1); + DasDim_primeCoord(pDim, false); + } + if((sFrame != NULL)&&(sFrame[0] != '\0')) DasDim_setFrame(pDim, sFrame); @@ -724,11 +732,12 @@ static DasErrCode _serial_makeVarAry(context_t* pCtx, bool bHandleFill) right now */ aShape[0] = 0; - das_val_type vt; + das_val_type vt = vtUnknown; if(pCtx->valStorage[0] != '\0'){ vt = das_vt_fromStr(pCtx->valStorage); } - else{ + /* that didn't work, try using the val semantic + encoding */ + if(vt == vtUnknown){ vt = das_vt_store_type(pCtx->sValEncType, pCtx->nPktItemBytes, pCtx->valSemantic); if(vt == vtUnknown){ @@ -956,8 +965,8 @@ static void _serial_onOpenVals(context_t* pCtx, const char** psAttr) /* By default utf8 is whitespace separated, could provide a separator here... */ nRet = DasCodec_init( - &(pCtx->codecHdrVals), pCtx->pCurAry, pCtx->valSemantic, "utf8", - DASIDX_RAGGED, 0, pCtx->varUnits + DASENC_READ, &(pCtx->codecHdrVals), pCtx->pCurAry, pCtx->valSemantic, "utf8", + DASIDX_RAGGED, '\0', pCtx->varUnits, NULL ); if(nRet != DAS_OKAY) pCtx->nDasErr = nRet; @@ -1354,7 +1363,8 @@ static void _serial_onCloseVar(context_t* pCtx) else{ nRet = (DasDs_addFixedCodec( pCtx->pDs, DasAry_id(pCtx->pCurAry), pCtx->valSemantic, - pCtx->sValEncType, pCtx->nPktItemBytes, pCtx->nPktItems + pCtx->sValEncType, pCtx->nPktItemBytes, pCtx->nPktItems, + DASENC_READ ) != NULL) ? DAS_OKAY : DASERR_SERIAL ; } if(nRet != DAS_OKAY){ diff --git a/das2/encoding.c b/das2/encoding.c index 93b34f3..e36541f 100644 --- a/das2/encoding.c +++ b/das2/encoding.c @@ -30,12 +30,12 @@ /* ************************************************************************* */ /* FILL */ double getDas2Fill() { - return DAS_FILL_VALUE; + return DAS_FILL_VALUE; } int isDas2Fill( double value ) { - double fill= getDas2Fill(); - return fabs((fill-value)/fill)<0.00001; + double fill= getDas2Fill(); + return fabs((fill-value)/fill)<0.00001; } /* ************************************************************************* */ @@ -58,7 +58,7 @@ DasEncoding* new_DasEncoding(int nCat, int nWidth, const char* sFmt){ if(nWidth < 2 || nWidth > 127){ das_error(14, "Error in encoding type %s, valid field width range " - "is 2 to 127 characters", nWidth); + "is 2 to 127 characters", nWidth); return NULL; } @@ -145,7 +145,7 @@ DasEncoding* new_DasEncoding_str(const char* sType) } } - + int i = 0; int nOff = 0; if(strncmp(sType, "ascii", 5) == 0){ @@ -170,7 +170,7 @@ DasEncoding* new_DasEncoding_str(const char* sType) if(pThis->nWidth < 2 || pThis->nWidth > 127){ free(pThis); das_error(14, "Error in encoding type %s, valid field width range " - "is 2 to 127 characters", sType); + "is 2 to 127 characters", sType); } return pThis; diff --git a/das2/encoding.h b/das2/encoding.h index 2b2b0e9..629352c 100644 --- a/das2/encoding.h +++ b/das2/encoding.h @@ -77,7 +77,7 @@ DAS_API int isDas2Fill( double value ); #define DAS2DT_LE_UINT 0x0008 #define DASENC_FMT_LEN 64 -#define DASENC_TYPE_LEN 48 +#define DASENC_TYPE_LEN 32 /** @addtogroup IO diff --git a/das2/io.c b/das2/io.c index 8b3638b..91bb269 100644 --- a/das2/io.c +++ b/das2/io.c @@ -1703,25 +1703,26 @@ DasErrCode DasIO_writeData(DasIO* pThis, DasDesc* pDesc, int iPktId) DasIO_write(pThis, pBuf->pReadBeg, DasBuf_unread(pBuf)); } else if(type == DATASET){ - /* May print many packets */ + /* This may print many packets */ DasDs* pDs = (DasDs*)pDesc; if(! pDs->bSentHdr) return das_error(DASERR_IO, "Send packet header ID %02d first", iPktId); - ptrdiff_t iIdx0 = 0; - while(true){ + ptrdiff_t aZeros[DASIDX_MAX] = DASIDX_INIT_BEGIN; + ptrdiff_t nSz0 = DasDs_lengthIn(pDs, 0, aZeros); + for(ptrdiff_t iIdx0 = 0; iIdx0 < nSz0; ++iIdx0){ + + /* TODO: add "flush-on-full" to buffer to minimize writes */ DasBuf_reinit(pBuf); nRet = DasDs_encodeData(pDs, pBuf, iIdx0); - if(nRet == -1) break; if(nRet != DAS_OKAY) return nRet; DasIO_printf(pThis, "|Pd|%d|%d|", iPktId, DasBuf_unread(pBuf)); /* How to check if this failed? */ DasIO_write(pThis, pBuf->pReadBeg, DasBuf_unread(pBuf)); - ++iIdx0; } } else diff --git a/das2/iterator.c b/das2/iterator.c index 6dc4a00..ad9adaf 100644 --- a/das2/iterator.c +++ b/das2/iterator.c @@ -22,25 +22,164 @@ #include "iterator.h" /* ************************************************************************* */ -/* The iteration support */ +/* Array iteration support */ -void das_iter_init(das_iter* pIter, const DasDs* pDs){ +void DasAryIter_init( + DasAryIter* pThis, const DasAry* pAry, int iDimMin, int iDimMax, + ptrdiff_t* pLocBeg, ptrdiff_t* pLocEnd +) +{ + memset(pThis, 0, sizeof(DasAryIter)); + + pThis->pAry = pAry; + pThis->rank = DasAry_shape(pThis->pAry, pThis->shape); + if(pThis->shape[0] == 0){ /* Can't iterate an empty array */ + pThis->done = true; + return; + } + + pThis->ragged = false; + for(int i = 1; i < pThis->rank; ++i){ + if(pThis->shape[i] == DASIDX_RAGGED){ + pThis->ragged = true; + break; + } + } + + pThis->dim_min = iDimMin; + if(pThis->dim_min >= pThis->rank){ + pThis->done = true; + return; + } + pThis->dim_max = iDimMax; + if(pThis->dim_max < 0) pThis->dim_max = pThis->rank - pThis->dim_max; + if((pThis->dim_max < 0)||(pThis->dim_max < pThis->dim_min)){ + pThis->done = true; + return; + } + + /* .index is treated as the "start" value. We continually increment + the start value until it is >= the end value */ + + /* Start off at all zeros, or at the specified location */ + if(pLocBeg != NULL) + memcpy(pThis->index, pLocBeg, (pThis->dim_max+1)*sizeof(ptrdiff_t)); + + /* End at all zeros, or at the end of the array. end is an exclusive upper index */ + if(pLocEnd != NULL) + memcpy(pThis->index, pLocEnd, (pThis->dim_max+1)*sizeof(ptrdiff_t)); + else + pThis->bNaturalEnd = true; + + /* If I'm ragged, get the length in the last dimension I can iterate */ + if(pThis->ragged){ + pThis->nLenLast = DasAry_lengthIn(pAry, pThis->dim_max, pThis->index); + } +} + +/* Helper, is index before end? Depends on normalized indexes */ +/* The way to visualize this is text highlight select with rows and columns + + Consider the text, where ^ points out first and last values + + 0123456789012345678901234567890123456789012 + 0 asnt asnteoh tanehu asuh ansteohu asnehuhu + ^ + 1 asnotuna otah tohuh + ^ + Beg = 0,16 + End = 1,13 + + if I > 1, I'm done. + if I == 1 && J >= 13 I'm done. + +*/ +bool _DasAryIter_beforeEnd(DasAryIter* pThis) +{ + for(int iDim = pThis->dim_min; iDim <= pThis->dim_max; ++iDim){ + if(pThis->index[iDim] > pThis->end_idx[iDim]){ + pThis->done = true; + return false; + } + if(pThis->index[iDim] < pThis->end_idx[iDim]) + return true; + + /* not conclusive, check next lower (left most) index */ + } + + pThis->done = true; + return false; +} + +bool DasAryIter_next(DasAryIter* pThis) +{ + if(pThis->done) return false; + + if(! pThis->ragged){ + + /* Quicker function for CUBIC arrays */ + for(int iDim = pThis->dim_max; iDim >= pThis->dim_min; --iDim){ + if(pThis->index[iDim] < (pThis->shape[iDim] - 1)){ + pThis->index[iDim] += 1; + return (pThis->bNaturalEnd ? true : _DasAryIter_beforeEnd(pThis)); + } + else{ + pThis->index[iDim] = 0; /* and go again, incrementing next index up */ + } + } + + pThis->done = true; /* must have run out if dimensions */ + return false; + } + + /* Ahh, ragged dense arrays. Guess someone selected hard mode */ + ptrdiff_t nLenInIdx = 0; + for(int iDim = pThis->dim_max; iDim >= pThis->dim_min; --iDim){ + if(iDim == pThis->dim_min) + nLenInIdx = pThis->nLenLast; + else + nLenInIdx = DasAry_lengthIn(pThis->pAry, iDim, pThis->index); + + if(pThis->index[iDim] < (nLenInIdx - 1)){ + pThis->index[iDim] += 1; + + /* Look ahead. If bumping an index that's not the last, save off the + length of the last run that we care about */ + if(iDim < (pThis->dim_max - 1)) + pThis->nLenLast = DasAry_lengthIn(pThis->pAry, pThis->dim_max, pThis->index); + + return (pThis->bNaturalEnd ? true : _DasAryIter_beforeEnd(pThis)); + } + else{ + pThis->index[iDim] = 0; + } + } + + pThis->done = true; + return false; + +} + +/* ************************************************************************* */ +/* Dataset iteration support */ + +void DasDsIter_init(DasDsIter* pThis, const DasDs* pDs){ - memset(pIter, 0, sizeof(das_iter)); + memset(pThis, 0, sizeof(DasDsIter)); - pIter->rank = DasDs_shape(pDs, pIter->shape); - pIter->pDs = pDs; + pThis->rank = DasDs_shape(pDs, pThis->shape); + pThis->pDs = pDs; /* If this is an empty dataset, we're already done */ - if(pIter->shape[0] == 0){ - pIter->done = true; + if(pThis->shape[0] == 0){ + pThis->done = true; return; } - pIter->ragged = false; - for(int i = 1; i < pIter->rank; ++i){ /* Ignore ragged on first index */ - if(pIter->shape[i] == DASIDX_RAGGED){ - pIter->ragged = true; + pThis->ragged = false; + for(int i = 1; i < pThis->rank; ++i){ /* Ignore ragged on first index */ + if(pThis->shape[i] == DASIDX_RAGGED){ + pThis->ragged = true; break; } } @@ -49,29 +188,29 @@ void das_iter_init(das_iter* pIter, const DasDs* pDs){ /* If I'm ragged I'm going to need the size of the last index at the * lowest point of all previous indexes, get that. */ - if(pIter->ragged){ - pIter->nLenIn = DasDs_lengthIn(pDs, pIter->rank - 1, pIter->index); - if(pIter->nLenIn < 0) pIter->done = true; + if(pThis->ragged){ + pThis->nLenIn = DasDs_lengthIn(pDs, pThis->rank - 1, pThis->index); + if(pThis->nLenIn < 0) pThis->done = true; } } -bool das_iter_next(das_iter* pIter){ +bool DasDsIter_next(DasDsIter* pThis){ - if(pIter->done) return false; + if(pThis->done) return false; - if(! pIter->ragged){ + if(! pThis->ragged){ /* Quicker function for CUBIC datasets */ - for(int iDim = pIter->rank - 1; iDim >= 0; --iDim){ - if(pIter->index[iDim] < (pIter->shape[iDim] - 1)){ - pIter->index[iDim] += 1; + for(int iDim = pThis->rank - 1; iDim >= 0; --iDim){ + if(pThis->index[iDim] < (pThis->shape[iDim] - 1)){ + pThis->index[iDim] += 1; return true; } else{ - pIter->index[iDim] = 0; + pThis->index[iDim] = 0; } } - pIter->done = true; + pThis->done = true; return false; } @@ -79,45 +218,45 @@ bool das_iter_next(das_iter* pIter){ * at least save off the length of the last index at this point * and only change it when a roll occurs */ ptrdiff_t nLenInIdx = 0; - for(int iDim = pIter->rank - 1; iDim >= 0; --iDim){ + for(int iDim = pThis->rank - 1; iDim >= 0; --iDim){ - if(iDim == (pIter->rank - 1)) - nLenInIdx = pIter->nLenIn; + if(iDim == (pThis->rank - 1)) + nLenInIdx = pThis->nLenIn; else - nLenInIdx = DasDs_lengthIn(pIter->pDs, iDim, pIter->index); + nLenInIdx = DasDs_lengthIn(pThis->pDs, iDim, pThis->index); - if(pIter->index[iDim] < (nLenInIdx - 1)){ - pIter->index[iDim] += 1; + if(pThis->index[iDim] < (nLenInIdx - 1)){ + pThis->index[iDim] += 1; /* If bumping an index that's not the last, recompute the length * of the last run */ - if(iDim < (pIter->rank - 1)) - pIter->nLenIn = DasDs_lengthIn(pIter->pDs, pIter->rank - 1, pIter->index); + if(iDim < (pThis->rank - 1)) + pThis->nLenIn = DasDs_lengthIn(pThis->pDs, pThis->rank - 1, pThis->index); return true; } else{ - pIter->index[iDim] = 0; + pThis->index[iDim] = 0; } } - pIter->done = true; + pThis->done = true; return false; } /* ************************************************************************* */ -void das_uniq_iter_init( - das_uniq_iter* pIter, const DasDs* pDs, const DasVar* pVar +void DasDsUniqIter_init( + DasDsUniqIter* pThis, const DasDs* pDs, const DasVar* pVar ){ - memset(pIter, 0, sizeof(das_uniq_iter)); + memset(pThis, 0, sizeof(DasDsUniqIter)); - pIter->rank = DasDs_shape(pDs, pIter->shape); - pIter->pDs = pDs; + pThis->rank = DasDs_shape(pDs, pThis->shape); + pThis->pDs = pDs; /* If this is an empty dataset, we're already done */ - if(pIter->shape[0] == 0){ - pIter->done = true; + if(pThis->shape[0] == 0){ + pThis->done = true; return; } @@ -126,59 +265,59 @@ void das_uniq_iter_init( /* Lock the indexes that are ignored by this variable to 0, and determine * if I'm ragged in a used index */ - pIter->ragged = false; - pIter->first = -1; - pIter->last = -1; - for(int i = 0; i < pIter->rank; ++i){ + pThis->ragged = false; + pThis->first = -1; + pThis->last = -1; + for(int i = 0; i < pThis->rank; ++i){ if(aVarShape[i] == DASIDX_UNUSED){ - pIter->lock[i] = true; + pThis->lock[i] = true; } else{ - pIter->last = i; - if(pIter->first == -1) pIter->first = i; + pThis->last = i; + if(pThis->first == -1) pThis->first = i; } - if((!pIter->lock[i])&&(i > 0)&&(pIter->shape[i] == DASIDX_RAGGED)) - pIter->ragged = true; + if((!pThis->lock[i])&&(i > 0)&&(pThis->shape[i] == DASIDX_RAGGED)) + pThis->ragged = true; } /* In the odd case of a constant, just set done right now */ - if((pIter->first == -1)||(pIter->first == -1)) - pIter->done = true; + if((pThis->first == -1)||(pThis->first == -1)) + pThis->done = true; /* Start off index at all zeros, which memset insures above */ /* If I'm ragged I'm going to need the size of the last used index * at the lowest point of all previous indexes, get that. */ - if(pIter->ragged){ - pIter->nLenIn = DasDs_lengthIn(pDs, pIter->last, pIter->index); - if(pIter->nLenIn < 0) pIter->done = true; + if(pThis->ragged){ + pThis->nLenIn = DasDs_lengthIn(pDs, pThis->last, pThis->index); + if(pThis->nLenIn < 0) pThis->done = true; } } -bool das_uniq_iter_next(das_uniq_iter* pIter){ +bool DasDsUniqIter_next(DasDsUniqIter* pThis){ - if(pIter->done) return false; + if(pThis->done) return false; - if(! pIter->ragged){ + if(! pThis->ragged){ /* Quicker function for CUBIC datasets, as long as you're at the last index of an array dimension, keep setting zero and rolling previous */ - for(int iDim = pIter->last; iDim >= pIter->first; --iDim){ - if(pIter->lock[iDim]) continue; + for(int iDim = pThis->last; iDim >= pThis->first; --iDim){ + if(pThis->lock[iDim]) continue; - if(pIter->index[iDim] < (pIter->shape[iDim] - 1)){ - pIter->index[iDim] += 1; + if(pThis->index[iDim] < (pThis->shape[iDim] - 1)){ + pThis->index[iDim] += 1; return true; } else{ - pIter->index[iDim] = 0; + pThis->index[iDim] = 0; } } - pIter->done = true; + pThis->done = true; return false; } @@ -186,74 +325,74 @@ bool das_uniq_iter_next(das_uniq_iter* pIter){ * at least save off the length of the last index at this point * and only change it when a roll occurs */ ptrdiff_t nLenInIdx = 0; - for(int iDim = pIter->last; iDim >= pIter->first; --iDim){ + for(int iDim = pThis->last; iDim >= pThis->first; --iDim){ - if(iDim == pIter->last) - nLenInIdx = pIter->nLenIn; + if(iDim == pThis->last) + nLenInIdx = pThis->nLenIn; else - nLenInIdx = DasDs_lengthIn(pIter->pDs, iDim, pIter->index); + nLenInIdx = DasDs_lengthIn(pThis->pDs, iDim, pThis->index); - if(pIter->index[iDim] < (nLenInIdx - 1)){ - pIter->index[iDim] += 1; + if(pThis->index[iDim] < (nLenInIdx - 1)){ + pThis->index[iDim] += 1; /* If bumping an index that's not the last, recompute the length * of the last run that I care about */ - if(iDim < pIter->last) - pIter->nLenIn = DasDs_lengthIn(pIter->pDs, pIter->last, pIter->index); + if(iDim < pThis->last) + pThis->nLenIn = DasDs_lengthIn(pThis->pDs, pThis->last, pThis->index); return true; } else{ - pIter->index[iDim] = 0; + pThis->index[iDim] = 0; } } - pIter->done = true; + pThis->done = true; return false; } /* ************************************************************************* */ -void das_cube_iter_init( - das_cube_iter* pIter, int nRank, ptrdiff_t* pMin, ptrdiff_t* pMax +void DasDsCubeIter_init( + DasDsCubeIter* pThis, int nRank, ptrdiff_t* pMin, ptrdiff_t* pMax ){ - pIter->done = true; + pThis->done = true; if((nRank < 1)||(nRank > DASIDX_MAX)) das_error(DASERR_ITER, "Invalid array rank %d", nRank); - memcpy(pIter->idxmin, pMin, sizeof(ptrdiff_t)*nRank); - memcpy(pIter->index, pMin, sizeof(ptrdiff_t)*nRank); - memcpy(pIter->idxmax, pMax, sizeof(ptrdiff_t)*nRank); + memcpy(pThis->idxmin, pMin, sizeof(ptrdiff_t)*nRank); + memcpy(pThis->index, pMin, sizeof(ptrdiff_t)*nRank); + memcpy(pThis->idxmax, pMax, sizeof(ptrdiff_t)*nRank); - pIter->rank = nRank; + pThis->rank = nRank; /* Check to see if any max values are greater then the corresponding min */ for(int i = 0; i < nRank; ++i){ if(pMax[i] > pMin[i]){ - pIter->done = false; + pThis->done = false; return; } } } -bool das_cube_iter_next(das_cube_iter* pIter) +bool DasDsCubeIter_next(DasDsCubeIter* pThis) { - if(pIter->done) return false; + if(pThis->done) return false; - for(int i = (pIter->rank - 1); i > -1; --i){ + for(int i = (pThis->rank - 1); i > -1; --i){ /* Increment this index, if you can */ - if((pIter->index[i] + 1) < pIter->idxmax[i]){ - pIter->index[i] += 1; + if((pThis->index[i] + 1) < pThis->idxmax[i]){ + pThis->index[i] += 1; return true; } else{ /* Need to bump next lower index, if no lower index, we're done */ if(i != 0) - pIter->index[i] = pIter->idxmin[i]; + pThis->index[i] = pThis->idxmin[i]; } } - pIter->done = true; /* no next loop, no need for a break */ + pThis->done = true; /* no next loop, no need for a break */ return false; } diff --git a/das2/iterator.h b/das2/iterator.h index 016e655..27b9188 100644 --- a/das2/iterator.h +++ b/das2/iterator.h @@ -78,9 +78,9 @@ typedef struct dasds_iterator_t{ ptrdiff_t nLenIn; /* Used for ragged datasets */ bool ragged; const DasDs* pDs; -} das_iter; +} DasDsIter; -#define dasds_iterator das_iter +#define dasds_iterator DasDsIter /** Initialize a const dataset iterator * @@ -91,15 +91,15 @@ typedef struct dasds_iterator_t{ * * For usage see the example in ::das_iterator * - * @param pIter A pointer to an iterator, will be initialize to index 0 + * @param pThis A pointer to an iterator, will be initialize to index 0 * * @param pDs A pointer to a dataset. If the dataset changes while the * iterator is in use invalid memory access could occur * - * @memberof dasds_iterator + * @memberof DasDsIter */ -DAS_API void das_iter_init(dasds_iterator* pIter, const DasDs* pDs); -#define dasds_iter_init das_iter_init +DAS_API void DasDsIter_init(DasDsIter* pThis, const DasDs* pDs); +#define dasds_iter_init DasDsIter_init /** Increment the iterator's index by one position, rolling as needed at * data boundaries. @@ -109,16 +109,16 @@ DAS_API void das_iter_init(dasds_iterator* pIter, const DasDs* pDs); * * For usage see the example in ::das_iterator * - * @param pIter A pointer to an iterator. The index member of the iterator + * @param pThis A pointer to an iterator. The index member of the iterator * will be incremented. * * @return true if the new index is within range, false if the index could not * be incremented without producing an invalid location. * - * @memberof dasds_iterator + * @memberof DasDsIter */ -DAS_API bool das_iter_next(dasds_iterator* pIter); -#define dasds_iter_next das_iter_next +DAS_API bool DasDsIter_next(DasDsIter* pThis); +#define dasds_iter_next DasDsIter_next /* ************************************************************************* */ @@ -144,7 +144,7 @@ typedef struct das_uniq_iter_t{ ptrdiff_t nLenIn; /* Used for ragged datasets */ bool ragged; const DasDs* pDs; -} das_uniq_iter; +} DasDsUniqIter; /** Initialize a non-degenerate iterator for a variable * @@ -155,16 +155,16 @@ typedef struct das_uniq_iter_t{ * * For usage see the example in ::das_iterator * - * @param pIter A pointer to an iterator, will be initialize to index 0 + * @param pThis A pointer to an iterator, will be initialize to index 0 * * @param pDs A pointer to a dataset. If the dataset changes while the * iterator is in use invalid memory access could occur * - * @memberof das_uniq_iter + * @memberof DasDsUniqIter */ -DAS_API void das_uniq_iter_init( - das_uniq_iter* pIter, const DasDs* pDs, const DasVar* pVar +DAS_API void DasDsUniqIter_init( + DasDsUniqIter* pThis, const DasDs* pDs, const DasVar* pVar ); /** Increment the iterator's index by one position, rolling as needed at @@ -175,15 +175,15 @@ DAS_API void das_uniq_iter_init( * * For usage see the example in ::das_iterator * - * @param pIter A pointer to an iterator. The index member of the iterator + * @param pThis A pointer to an iterator. The index member of the iterator * will be incremented. * * @return true if the new index is within range, false if the index could not * be incremented without producing an invalid location. * - * @memberof das_uniq_iter + * @memberof DasDsUniqIter */ -DAS_API bool das_uniq_iter_next(das_uniq_iter* pIter); +DAS_API bool DasDsUniqIter_next(DasDsUniqIter* pThis); /* ************************************************************************* */ @@ -202,19 +202,22 @@ typedef struct das_cube_iter_t{ ptrdiff_t idxmin[DASIDX_MAX]; ptrdiff_t idxmax[DASIDX_MAX]; -} das_cube_iter; +} DasDsCubeIter; +#define das_cube_iter DasDsCubeIter +#define das_cube_iter_init DasDsCubeIter_init +#define das_cube_iter_next DasDsCubeIter_next /** Initialize an iterator to cubic section in index space * @memberof das_cube_iterator */ -DAS_API void das_cube_iter_init( - das_cube_iter* pIter, int nRank, ptrdiff_t* pMin, ptrdiff_t* pMax +DAS_API void DasDsCubeIter_init( + das_cube_iter* pThis, int nRank, ptrdiff_t* pMin, ptrdiff_t* pMax ); /** Increment a cubic iterator by one position, rolling as needed * - * @param pIter A pointer to an iterator. The index member of the iterator + * @param pThis A pointer to an iterator. The index member of the iterator * will be incremented. * * @return true if the new index is within range, false if the index could not @@ -222,7 +225,67 @@ DAS_API void das_cube_iter_init( * * @memberof das_cube_iterator */ -DAS_API bool das_cube_iter_next(das_cube_iter* pIter); +DAS_API bool DasDsCubeIter_next(das_cube_iter* pThis); + +/** Iterate over a sub-set of the index space of a DasAry */ +typedef struct das_array_iter_t{ + const DasAry* pAry; + bool done; + bool ragged; /* flag for no end_idx */ + bool bNaturalEnd; + int rank; + ptrdiff_t index[DASIDX_MAX]; /* current index */ + ptrdiff_t end_idx[DASIDX_MAX]; /* 1 after last valid index */ + int dim_min; + int dim_max; + + ptrdiff_t shape[DASIDX_MAX]; /* Used for CUBIC arrays */ + ptrdiff_t nLenLast; /* Used for ragged arrays */ +} DasAryIter; + + +/** Initialize an array iterator + * + * The iterator can be used to simple iterate over the whole array when + * nDimMin = 0, nDimMax = -1, pBeg = NULL, and pEnd = NULL, though it's + * primary purpose it to iterate over a subset of indices. + * + * @param pThis - The iterator to initialize + * + * @param pAry - The array over which it will iterate + * + * @param iDimMin - The minimum (left most) index to change. Use 0 to increment + * the higest level index as needed, 1 for the next highest and so on. + * + * @param iDimMax - The maximum (right most) index to change. So for a rank 3 + * array this would be 2. Use -1 to allow changes for the last index, + * -2 for the next to last and so on. + * + * @param pBeg - The starting point for iteration, use NULL to start at + * the beginning. + * + * @param pEnd - The ending point for iteration. This is an exclusive + * upper bound! Use NULL to end after the last valid index in the + * array. + * + * @memberof DasAryIter + */ +DAS_API void DasAryIter_init( + DasAryIter* pThis, const DasAry* pAry, int iDimMin, int iDimMax, ptrdiff_t* pBeg, + ptrdiff_t* pEnd +); + + +/** Increment an array iterated to the next expected index + * + * Updates the .index member to the next value in range. + * + * @param pThis - The iterator in increment + * + * @returns true if the new index is in range, false otherwise and the .done + * member is set to true. + */ +DAS_API bool DasAryIter_next(DasAryIter* pThis); #ifdef __cplusplus diff --git a/das2/plane.c b/das2/plane.c index 006dac6..e06caa8 100644 --- a/das2/plane.c +++ b/das2/plane.c @@ -215,46 +215,6 @@ PlaneDesc* new_PlaneDesc_yscan_series( /* Key/Value Pairs Constructor */ -/* Helper for trimming zeros after decimal */ -void _trimTrailingZeros(char* sVal){ - - if(strchr(sVal, '.') == NULL) return; - - /* technically could handle normalizing stuff like - 10000e6 as well, but that's rare so forget it for now */ - - int iDec = strchr(sVal, '.') - sVal; - int iExp = -1; - int v = strlen(sVal); - int i = -1, j = -1; - - if(strchr(sVal, 'e') || strchr(sVal, 'E')){ - if(strchr(sVal, 'e')) iExp = strchr(sVal, 'e') - sVal; - else iExp = strchr(sVal, 'E') - sVal; - - for(i = iExp-1; i > iDec; --i){ - /* Shift out Zeros after decimal but before exponent*/ - if(sVal[i] == '0'){ - for(j = i; j < v-1; ++j) - sVal[j] = sVal[j+1]; - --v; - sVal[v] = '\0'; - --iExp; - } - else{ - break; - } - } - } - else{ - while((v > 0) && (sVal[ v - 1] == '0')){ - /* NULL out Zeros after the decimal*/ - sVal[ v - 1] = '\0'; - v = strlen(sVal); - } - } -} - /* Helper for new_PlaneDesc_pairs, tries to determine a reasonable encoding for the ytags values, assumes nItems is already valid */ DasErrCode _PlaneDesc_decodeYTags(PlaneDesc* pThis, const char* sYTags) @@ -342,7 +302,7 @@ DasErrCode _PlaneDesc_decodeYTags(PlaneDesc* pThis, const char* sYTags) /* Trim Trailing zeros, if there is a decimal point in the number. make sure to move to before the exponent (if present) before doing this */ - _trimTrailingZeros(sVal); + das_value_trimReal(sVal); /* Convert it to a double */ if( sVal[0] != '\0'){ diff --git a/das2/stream.c b/das2/stream.c index 2f157bf..b54d6d4 100644 --- a/das2/stream.c +++ b/das2/stream.c @@ -1063,7 +1063,15 @@ DasDesc* DasDesc_decode( DasBuf_setReadOffset(pBuf, uPos); /* <-- the key call, back up the buffer */ if(strcmp(sName, "stream") == 0){ - return (DasDesc*) new_DasStream_str(pBuf, nModel); + DasStream* pSd = new_DasStream_str(pBuf, nModel); + + /* Have to up-convert the type designation on the stream as well, go + with the non-namespace streams by default */ + if(nModel == STREAM_MODEL_V3){ + strncpy(pSd->type, "das-basic-stream", STREAMDESC_TYPE_SZ - 1); + } + + return (DasDesc*) pSd; } if(strcmp(sName, "packet") == 0){ @@ -1089,3 +1097,5 @@ DasDesc* DasDesc_decode( das_error(DASERR_STREAM, "Unknown top-level descriptor object: %s", sName); return NULL; } + + diff --git a/das2/value.c b/das2/value.c index 9082542..d8469ea 100644 --- a/das2/value.c +++ b/das2/value.c @@ -19,8 +19,11 @@ #include #include +#include /* Get format strings for 64-bit items */ +#include #include #include +#include /* get FLT_MAX */ #ifdef _WIN32 #define strcasecmp _stricmp #else @@ -53,7 +56,7 @@ static const int64_t g_longFill = -9223372036854775807L; static const uint64_t g_ulongFill = 18446744073709551615UL; static const float g_floatFill = DAS_FILL_VALUE; static const double g_doubleFill = DAS_FILL_VALUE; -static const das_time g_timeFill = {0, 0, 0, 0, 0, 0, 0.0}; +static const das_time g_timeFill = {1, 1, 1, 1, 0, 0, 0.0}; static const das_geovec g_geovecFill = {{0,0,0}, 0, 0, 0, 0, 0, {0,0,0}}; const void* das_vt_fill(das_val_type et) @@ -234,14 +237,49 @@ const char* das_vt_serial_type(das_val_type et) case vtLong: return LE ? "LEint" : "BEint"; case vtFloat: return LE ? "LEreal" : "BEreal"; case vtDouble: return LE ? "LEreal" : "BEreal"; - case vtTime: return "utf8"; + case vtTime: return "utf8"; /* encodes as old ASCII(X) */ case vtGeoVec: return NULL; - case vtText: return "utf8"; + case vtText: return "utf8"; /* encodes as old TIME(X) */ case vtByteSeq: return "ubyte"; default: return NULL; } } +/* ************************************************************************* */ +/* semantics */ + +const char* DAS_SEM_BIN = "binary"; +const char* DAS_SEM_BOOL = "bool"; +const char* DAS_SEM_DATE = "datetime"; +const char* DAS_SEM_INT = "int"; +const char* DAS_SEM_REAL = "real"; +const char* DAS_SEM_TEXT = "string"; + +/** Given a value type, suggest a default semantic */ +const char* das_sem_default(das_val_type vt) +{ + switch(vt){ + case vtFloat: case vtDouble: return DAS_SEM_REAL; + case vtTime: return DAS_SEM_DATE; + case vtText: return DAS_SEM_TEXT; + default: return DAS_SEM_INT; + } + /* Non atomic types do not have defaults, this includes: + vtGeoVec, vtPixel, vtByteSeq, vtIndex etc. */ +} + +/* Given a semantic, suggest a default value type */ +das_val_type das_vt_default(const char* sSemantic) +{ + if(strcmp(sSemantic, "bool")) return vtByte; + if(strcmp(sSemantic, "datetime")) return vtTime; + if(strcmp(sSemantic, "int")) return vtInt; + if(strcmp(sSemantic, "real")) return vtDouble; + if(strcmp(sSemantic, "string")) return vtText; + return vtByteSeq; +} + + /* das_val_type das_vt_guess_store(const char* sInterp, const char* sValue) { @@ -430,7 +468,7 @@ das_val_type das_vt_merge(das_val_type left, int op, das_val_type right) #define HAS_FLT 0x2 #define HAS_BOTH 0x3 -int das_vt_cmpAny( +int das_value_cmpAny( const ubyte* pA, das_val_type vtA, const ubyte* pB, das_val_type vtB ){ int nCmp = 0; @@ -543,6 +581,257 @@ int das_vt_cmpAny( #undef HAS_FLT #undef HAS_BOTH +/* ************************************************************************** */ +/* Convert any one itegral value type into any other, with range and + * resolution checks + */ + +/* No range checks are needed to go big to same signed type, + or larger type with different sign +*/ +#define GO_BIG(TY_OUT, TY_IN) ( *((TY_OUT*)pO) = *((TY_IN*)pI) ) + +/* Go to bigger or same size, so long as the value is 0 or greater */ +#define GO_POSI(TY_OUT, TY_IN) \ + if(bRng &&( *((TY_IN*)pI) < 0) ){ goto ERR_RANGE;} *((TY_OUT*)pO) = *((TY_IN*)pI) + +/* Go to the bigger size so long as val is > 0 and < max */ +#define GO_ZMAX(TY_OUT, TY_IN, MAX_OK) \ + if(bRng &&( (*((TY_IN*)pI) < 0) || (*((TY_IN*)pI) > MAX_OK ) ) ){goto ERR_RANGE;} *((TY_OUT*)pO) = *((TY_IN*)pI) + +/* Go to other size if under a maximum limit */ +#define GO_MAX(TY_OUT, TY_IN, MAX_OK) \ + if(bRng && (*((TY_IN*)pI) > MAX_OK )){ goto ERR_RANGE;} *((TY_OUT*)pO) = *((TY_IN*)pI) + +/* Go to other size if within range */ +#define GO_RNG(TY_OUT, TY_IN, MIN_OK, MAX_OK) \ + if(bRng &&( (*((TY_IN*)pI) < MIN_OK) || (*((TY_IN*)pI) > MAX_OK ) ) ){ goto ERR_RANGE;} *((TY_OUT*)pO) = *((TY_IN*)pI) + +/* Go to the other size if max value doesn't incure resolution loss */ +#define GO_ZRES(TY_OUT, TY_IN, MAX_OK) \ + if(bRes && (*((TY_IN*)pI) > MAX_OK )){ goto ERR_RESLOSS;} *((TY_OUT*)pO) = *((TY_IN*)pI) + +/* Go to the other size if min or max value don't incure resolution loss */ +#define GO_RES(TY_OUT, TY_IN, MIN_OK, MAX_OK) \ + if(bRes &&( (*((TY_IN*)pI) < MIN_OK) || (*((TY_IN*)pI) > MAX_OK ) ) ){ goto ERR_RESLOSS;} *((TY_OUT*)pO) = *((TY_IN*)pI) + +/* Got to an integer if I'm in range, and lose too much resolution, + + Min and max are straightforward, but resolution loss is hard to quantify. + If the fractional part is far enough away from an integer, call it a conversion error. Thus: + + 1.9998 would convert to 2 without error but + 1.5 would not. +*/ +#define GO_TRUNC(TY_IN, TY_OUT, MIN_OK, MAX_OK, EPSILON) \ + rIn = *((TY_IN*)pI); \ + if(bRng && ((rIn < MIN_OK)||(rIn > MAX_OK))) goto ERR_RANGE; \ + if(bRes){ rRnd = round(rIn); if( fabs(rIn - rRnd) > EPSILON) goto ERR_RESLOSS; } \ + *((TY_OUT*)pO) = rIn + + +DasErrCode das_value_binXform( + das_val_type vtIn, const ubyte* pI, const ubyte* pFI, + das_val_type vtOut, ubyte* pO, const ubyte* pFO, + uint32_t uFlags +){ + DasErrCode nRet; + + /* Handle fill up-front */ + size_t uInSz = das_vt_size(vtIn); + size_t uOutSz = das_vt_size(vtOut); + if((pFI != NULL) && (memcmp(pI, pFI, uInSz) == 0)){ + memcpy(pO, pFO, uOutSz); + return DAS_OKAY; + } + + bool bRng = (uFlags & DAS_VAL_NOERR_RNG) ? false : true; /* on true, issue range errors */ + bool bRes = (uFlags & DAS_VAL_ERR_RESLOSS); + double rIn, rRnd; /* Used by the trunc macros */ + + /* Value conversions */ + switch(vtIn){ + case vtUByte : + switch(vtOut){ + case vtUByte : GO_BIG( uint8_t, uint8_t ); break; /* works for same */ + case vtByte : GO_MAX( int8_t, uint8_t, INT8_MAX); break; + case vtUShort: GO_BIG(uint16_t, uint8_t ); break; + case vtShort : GO_BIG( int16_t, uint8_t ); break; + case vtUInt : GO_BIG(uint32_t, uint8_t ); break; + case vtInt : GO_BIG( int32_t, uint8_t ); break; + case vtULong : GO_BIG(uint64_t, uint8_t ); break; + case vtLong : GO_BIG( int64_t, uint8_t ); break; + case vtFloat : GO_BIG( float, uint8_t ); break; + case vtDouble: GO_BIG( double, uint8_t ); break; + default: goto ERR_NO_XFORM; break; + } + break; + case vtByte : + switch(vtOut){ + case vtUByte : GO_POSI( uint8_t, int8_t); break; + case vtByte : GO_BIG ( int8_t, int8_t); break; /* works for same */ + case vtUShort: GO_POSI(uint16_t, int8_t); break; + case vtShort : GO_BIG ( int16_t, int8_t); break; + case vtUInt : GO_POSI(uint32_t, int8_t); break; + case vtInt : GO_BIG ( int32_t, int8_t); break; + case vtULong : GO_POSI(uint64_t, int8_t); break; + case vtLong : GO_BIG ( int64_t, int8_t); break; + case vtFloat : GO_BIG ( float, int8_t); break; + case vtDouble: GO_BIG ( double, int8_t); break; + default: goto ERR_NO_XFORM; break; + } + break; + case vtUShort: + switch(vtOut){ + case vtUByte : GO_MAX( uint8_t, uint16_t, UINT8_MAX); break; + case vtByte : GO_MAX( int8_t, uint16_t, INT8_MAX); break; + case vtUShort: GO_BIG(uint16_t, uint16_t); break; /* works for same */ + case vtShort : GO_MAX( int8_t, uint16_t, INT16_MAX); break; + case vtUInt : GO_BIG(uint32_t, uint16_t ); break; + case vtInt : GO_BIG( int32_t, uint16_t ); break; + case vtULong : GO_BIG(uint64_t, uint16_t ); break; + case vtLong : GO_BIG( int64_t, uint16_t ); break; + case vtFloat : GO_BIG( float, uint16_t ); break; + case vtDouble: GO_BIG( double, uint16_t ); break; + default: goto ERR_NO_XFORM; break; + } + break; + case vtShort : + switch(vtOut){ + case vtUByte : GO_ZMAX( uint8_t, int16_t, UINT8_MAX); break; + case vtByte : GO_RNG( int8_t, int16_t, INT8_MIN, INT8_MAX); break; + case vtUShort: GO_POSI(uint16_t, int16_t); break; + case vtShort : GO_BIG ( int16_t, int16_t); break; /* works for same */ + case vtUInt : GO_POSI(uint32_t, int16_t); break; + case vtInt : GO_BIG ( int32_t, int16_t); break; + case vtULong : GO_POSI(uint64_t, int16_t); break; + case vtLong : GO_BIG ( int64_t, int16_t); break; + case vtFloat : GO_BIG ( float, int16_t); break; + case vtDouble: GO_BIG ( double, int16_t); break; + default: goto ERR_NO_XFORM; break; + } + break; + case vtUInt : + switch(vtOut){ + case vtUByte : GO_MAX ( uint8_t, uint32_t, UINT8_MAX); break; + case vtByte : GO_MAX ( int8_t, uint32_t, INT8_MAX); break; + case vtUShort: GO_MAX (uint16_t, uint32_t, UINT16_MAX); break; + case vtShort : GO_MAX ( int16_t, uint32_t, INT16_MAX); break; + case vtUInt : GO_BIG (uint32_t, uint32_t); break; /* works for same */ + case vtInt : GO_MAX ( int32_t, uint32_t, UINT32_MAX); break; + case vtULong : GO_BIG (uint64_t, uint32_t); break; + case vtLong : GO_BIG ( int64_t, uint32_t); break; + case vtFloat : GO_ZRES( float, uint32_t, 8388608U); break; + case vtDouble: GO_BIG ( double, uint32_t); break; + default: goto ERR_NO_XFORM; break; + } + break; + case vtInt : + switch(vtOut){ + case vtUByte : GO_ZMAX( uint8_t, int32_t, UINT8_MAX); break; + case vtByte : GO_RNG( int8_t, int32_t, INT8_MIN, INT8_MAX); break; + case vtUShort: GO_ZMAX(uint16_t, int32_t, UINT16_MAX); break; + case vtShort : GO_RNG( int16_t, int32_t, INT16_MIN, INT16_MAX); break; + case vtUInt : GO_POSI(uint32_t, int32_t); break; + case vtInt : GO_BIG ( int32_t, int32_t); break; /* works for same */ + case vtULong : GO_POSI(uint64_t, int32_t); break; + case vtLong : GO_BIG ( int64_t, int32_t); break; + case vtFloat : GO_RES ( float, int32_t, -8388608, 8388608); break; + case vtDouble: GO_BIG ( double, int32_t); break; + default: goto ERR_NO_XFORM; break; + } + break; + case vtULong : + switch(vtOut){ + case vtUByte : GO_MAX( uint8_t, uint64_t, UINT8_MAX); break; + case vtByte : GO_MAX( int8_t, uint64_t, INT8_MAX); break; + case vtUShort: GO_MAX(uint16_t, uint64_t, UINT16_MAX); break; + case vtShort : GO_MAX( int16_t, uint64_t, INT16_MAX); break; + case vtUInt : GO_MAX(uint32_t, uint64_t, UINT32_MAX); break; + case vtInt : GO_MAX( int32_t, uint64_t, INT32_MAX); break; + case vtULong : GO_BIG(uint64_t, uint64_t); break; /* works for same */ + case vtLong : GO_MAX( int64_t, uint64_t, INT64_MAX); break; + case vtFloat : GO_ZRES( float, uint32_t, 999999); break; + case vtDouble: GO_ZRES( double, uint32_t, 9007199254740992ULL ); break; + default: goto ERR_NO_XFORM; break; + } + break; + case vtLong : + switch(vtOut){ + case vtUByte : GO_ZMAX( uint8_t, int64_t, UINT8_MAX); break; + case vtByte : GO_RNG( int8_t, int64_t, INT8_MIN, INT8_MAX); break; + case vtUShort: GO_ZMAX(uint16_t, int64_t, UINT16_MAX); break; + case vtShort : GO_RNG( int16_t, int64_t, INT16_MIN, INT16_MAX); break; + case vtUInt : GO_ZMAX(uint32_t, int64_t, UINT32_MAX); break; + case vtInt : GO_RNG( int32_t, int64_t, INT32_MIN, INT32_MAX); break; + case vtULong : GO_POSI(uint64_t, int64_t); break; + case vtLong : GO_BIG ( int64_t, int64_t); break; /* works for same */ + case vtFloat : GO_RES ( float, int32_t, -9007199254740992LL, 9007199254740992LL); break; + case vtDouble: break; + default: goto ERR_NO_XFORM; break; + } + break; + case vtFloat : + switch(vtOut){ + case vtUByte : GO_TRUNC( uint8_t, float, 0, UINT8_MAX, 0.02); break; + case vtByte : GO_TRUNC( int8_t, float, INT8_MIN, INT8_MAX, 0.02); break; + case vtUShort: GO_TRUNC(uint16_t, float, 0, UINT16_MAX, 0.02); break; + case vtShort : GO_TRUNC( int16_t, float, INT16_MIN, INT16_MAX, 0.02); break; + case vtUInt : GO_TRUNC(uint32_t, float, 0, UINT32_MAX, 0.02); break; + case vtInt : GO_TRUNC( int32_t, float, INT32_MIN, INT32_MAX, 0.02); break; + case vtULong : GO_TRUNC(uint64_t, float, 0, UINT64_MAX, 0.02); break; + case vtLong : GO_TRUNC( int64_t, float, INT64_MIN, INT64_MAX, 0.02); break; + case vtFloat : GO_BIG( float, float); break; /* works for same */ + case vtDouble: GO_BIG( double, float); break; + default: goto ERR_NO_XFORM; break; + } + break; + case vtDouble: + switch(vtOut){ + case vtUByte : GO_TRUNC( uint8_t, double, 0, UINT8_MAX, 0.02); break; + case vtByte : GO_TRUNC( int8_t, double, INT8_MIN, INT8_MAX, 0.02); break; + case vtUShort: GO_TRUNC(uint16_t, double, 0, UINT16_MAX, 0.02); break; + case vtShort : GO_TRUNC( int16_t, double, INT16_MIN, INT16_MAX, 0.02); break; + case vtUInt : GO_TRUNC(uint32_t, double, 0, UINT32_MAX, 0.02); break; + case vtInt : GO_TRUNC( int32_t, double, INT32_MIN, INT32_MAX, 0.02); break; + case vtULong : GO_TRUNC(uint64_t, double, 0, UINT64_MAX, 0.02); break; + case vtLong : GO_TRUNC( int64_t, double, INT64_MIN, INT64_MAX, 0.02); break; + case vtFloat : GO_RNG( float, double, (-1*FLT_MAX), FLT_MAX); break; + case vtDouble: GO_BIG( double, double); break; /* works for same */ + default: goto ERR_NO_XFORM; break; + } + break; + + default: goto ERR_NO_XFORM; break; + } + + return DAS_OKAY; + + char sFmt[32] = {'\0'}; + char sVal[32] = {'\0'}; + +ERR_RESLOSS: + nRet = das_value_fmt(sFmt, 31, vtIn, "", -1); + if(nRet != DAS_OKAY) return nRet; + snprintf(sVal, 31, sFmt, pI); /* <-- compilers hate variable fmt strings */ + return das_error(DASERR_VALUE, "Resolution loss converting %s (%s) to %s", + sVal, das_vt_toStr(vtIn), das_vt_toStr(vtOut) + ); + +ERR_RANGE: + nRet = das_value_fmt(sFmt, 31, vtIn, "", -1); + if(nRet != DAS_OKAY) return nRet; + snprintf(sVal, 31, sFmt, pI); /* <-- compilers hate variable fmt strings */ + return das_error(DASERR_VALUE, "Range violation converting %s (%s) to %s", + sVal, das_vt_toStr(vtIn), das_vt_toStr(vtOut) + ); + +ERR_NO_XFORM: + return das_error(DASERR_VALUE, "No conversion from %s to %s defined", + das_vt_toStr(vtIn), das_vt_toStr(vtOut) + ); +} + /* ************************************************************************** */ /* Parse any string into a value */ @@ -625,6 +914,212 @@ DasErrCode das_value_fromStr( } } +/* ************************************************************************* */ +/* Generate a printf string for any value type, if with supplied try to fit + * it in a certian width. It's often that case that values are stored in + * types that have far greater range then the actual data */ + +/* Don't fail for certian semantics, but do offer extra support where detected */ +DasErrCode das_value_fmt( + char* sBuf, int nBufLen, das_val_type vt, const char* sSemantic, int nFitTo +){ + bool bBin = (strcmp(sSemantic, DAS_SEM_BIN) == 0); + bool bText = (strcmp(sSemantic, DAS_SEM_TEXT) == 0); + + switch(vt){ + case vtUByte: + if(nFitTo < 1){ + if(bBin) strncpy(sBuf, "%0hhX", nBufLen); + else if(bText) strncpy(sBuf, "%s", nBufLen); + else strncpy(sBuf, "%hhu", nBufLen); + } + else{ + if(bBin) snprintf(sBuf, nBufLen, "%%0%dhhX", nFitTo); + else if(bText) snprintf(sBuf, nBufLen, "%% %ds", nFitTo); + else snprintf(sBuf, nBufLen, "%% %dhhu", nFitTo); + } + break; + case vtByte: + if(nFitTo < 1){ + if(bText) strncpy(sBuf, "%s", nBufLen); + else strncpy(sBuf, "%hhd", nBufLen); + } + else{ + if(bText) snprintf(sBuf, nBufLen, "%% %ds", nFitTo); + else snprintf(sBuf, nBufLen, "%% %dhhd", nFitTo); + } + break; + + + case vtUShort: + if(nFitTo < 1){ + if(bBin) strncpy(sBuf, "%0hX", nBufLen); + else strncpy(sBuf, "%hu", nBufLen); + } + else{ + if(bBin) snprintf(sBuf, nBufLen, "%%0%dhX", nFitTo); + else snprintf(sBuf, nBufLen, "%% %dhu", nFitTo); + } + break; + case vtShort: + if(nFitTo < 1) strncpy(sBuf, "%hd", nBufLen); + else snprintf(sBuf, nBufLen, "%% %dhd", nFitTo); + break; + + + case vtUInt: + if(nFitTo < 1){ + if(bBin) strncpy(sBuf, "%0X", nBufLen); + else strncpy(sBuf, "%u", nBufLen); + } + else{ + if(bBin) snprintf(sBuf, nBufLen, "%%0%dX", nFitTo); + else snprintf(sBuf, nBufLen, "%% %du", nFitTo); + } + break; + case vtInt: + if(nFitTo < 1) strncpy(sBuf, "%d", nBufLen); + else snprintf(sBuf, nBufLen, "%% %dd", nFitTo); + break; + + case vtULong: + if(nFitTo < 1){ + if(bBin) strncpy(sBuf, "%0" PRIX64, nBufLen); /* from inttypes.h */ + else strncpy(sBuf, "%" PRIu64, nBufLen); + } + else{ + if(bBin) snprintf(sBuf, nBufLen, "%%0%d" PRIX64, nFitTo); + else snprintf(sBuf, nBufLen, "%% %d" PRIu64, nFitTo); + } + break; + case vtLong: + if(nFitTo < 1) strncpy(sBuf, PRId64, nBufLen); + else snprintf(sBuf, nBufLen, "%% %d" PRId64, nFitTo); + break; + + case vtFloat: + case vtDouble: + if(nFitTo < 1){ + if(vt == vtFloat) strncpy(sBuf, "%.4e", nBufLen); + else strncpy(sBuf, "%.8e", nBufLen); + } + else{ + /* Get a precision as a function of field width if it's big enough + * to give 2 digits after the decimal */ + if(nFitTo >= 9) + snprintf(sBuf, nBufLen, "%% %d.%de", nFitTo, nFitTo - 6); + else + snprintf(sBuf, nBufLen, "%% %d.%de", nFitTo, 2); + } + break; + + case vtTime: + if(nFitTo < 1){ + /* No guidance, just pick milliseconds, usually pretty good in space physics */ + strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d:%06.3f", nBufLen); + } + else{ + /* Okay, they want resolution to fit a particular width, here goes... */ + switch(nFitTo){ + /* Year only */ + case 4: strncpy(sBuf, "%04d", nBufLen); break; + case 5: strncpy(sBuf, "%04d ", nBufLen); break; + case 6: strncpy(sBuf, "%04d ", nBufLen); break; + + /* Year and Month */ + case 7: strncpy(sBuf, "%04d-%02d",nBufLen); break; + case 8: strncpy(sBuf, "%04d-%02d ",nBufLen); break; + case 9: strncpy(sBuf, "%04d-%02d ",nBufLen); break; + + /* Year/Month/Day of Month */ + case 10: strncpy(sBuf, "%04d-%02d-%02d",nBufLen); break; + case 11: strncpy(sBuf, "%04d-%02d-%02d ",nBufLen); break; + case 12: strncpy(sBuf, "%04d-%02d-%02d ",nBufLen); break; + + /* Date + Hour */ + case 13: strncpy(sBuf, "%04d-%02d-%02dT%02d",nBufLen); break; + case 14: strncpy(sBuf, "%04d-%02d-%02dT%02d ",nBufLen); break; + case 15: strncpy(sBuf, "%04d-%02d-%02dT%02d ",nBufLen); break; + + /* Date + Hour:min */ + case 16: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d",nBufLen); break; + case 17: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d ",nBufLen); break; + case 18: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d ",nBufLen); break; + + /* Date + Hour:min:sec */ + case 19: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d:%02.0f",nBufLen); break; + case 20: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d:%02.0f ",nBufLen); break; + + /* Date + hour:min:sec + frac seconds */ + case 21: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d:%04.1f",nBufLen); break; + case 22: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d:%05.2f",nBufLen); break; + case 23: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d:%06.3f",nBufLen); break; + case 24: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d:%06.3f ",nBufLen); break; + case 25: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d:%06.3f ",nBufLen); break; + case 26: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d:%09.6f",nBufLen); break; + case 27: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d:%09.6f ",nBufLen); break; + case 28: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d:%09.6f ",nBufLen); break; + case 29: strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d:%012.9f",nBufLen); break; + + /* No more resolution available, just space pad */ + default: + strncpy(sBuf, "%04d-%02d-%02dT%02d:%02d:%012.9f",nBufLen); break; + for(int i = 32; i < nBufLen; ++i) sBuf[i] = ' '; + } + } + break; + + default: + return das_error(DASERR_VALUE, + "Default format string not available for type '%s'", das_vt_toStr(vt) + ); + } + + return DAS_OKAY; +} + +/* ************************************************************************* */ +/* Helper for trimming zeros after decimal */ + +void das_value_trimReal(char* sVal){ + + if(strchr(sVal, '.') == NULL) return; + + /* technically could handle normalizing stuff like + 10000e6 as well, but that's rare so forget it for now */ + + int iDec = strchr(sVal, '.') - sVal; + int iExp = -1; + int v = strlen(sVal); + int i = -1, j = -1; + + if(strchr(sVal, 'e') || strchr(sVal, 'E')){ + if(strchr(sVal, 'e')) iExp = strchr(sVal, 'e') - sVal; + else iExp = strchr(sVal, 'E') - sVal; + + for(i = iExp-1; i > iDec; --i){ + /* Shift out Zeros after decimal but before exponent*/ + if(sVal[i] == '0'){ + for(j = i; j < v-1; ++j) + sVal[j] = sVal[j+1]; + --v; + sVal[v] = '\0'; + --iExp; + } + else{ + break; + } + } + } + else{ + while((v > 0) && (sVal[ v - 1] == '0')){ + /* NULL out Zeros after the decimal*/ + sVal[ v - 1] = '\0'; + v = strlen(sVal); + } + } +} + /* ************************************************************************* */ /* String to Value utilities */ diff --git a/das2/value.h b/das2/value.h index 96cbcda..f65f125 100644 --- a/das2/value.h +++ b/das2/value.h @@ -63,52 +63,52 @@ typedef struct das_byteseq_t{ */ typedef enum das_val_type_e { - /** For generic storage, designates elements as unknown, you have to cast + /** For generic storage, designates elements as unknown, you have to cast * the array return values yourself.*/ vtUnknown = 0, - - /** The basic types */ + + /** The basic types */ - /* VT_MIN_SIMPLE = vtUByte */ + /* VT_MIN_SIMPLE = vtUByte */ /** Indicates array values are unsigned 8-bit integers (bytes) */ vtUByte = 1, - /** Indicates array values are signed 8-bit integers (signed bytes) */ - vtByte = 2, + /** Indicates array values are signed 8-bit integers (signed bytes) */ + vtByte = 2, - /** Indicates array values are unsigned 16-bit integers (shorts) */ + /** Indicates array values are unsigned 16-bit integers (shorts) */ vtUShort = 3, - /** Indicates array values are signed 16-bit integers (shorts)*/ + /** Indicates array values are signed 16-bit integers (shorts)*/ vtShort = 4, - /** Indicates array values are unsigned 32-bit integers (uints) */ - vtUInt = 5, + /** Indicates array values are unsigned 32-bit integers (uints) */ + vtUInt = 5, - /** Indicates array values are signed 32-bit integers (ints) */ + /** Indicates array values are signed 32-bit integers (ints) */ vtInt = 6, - /** Indicates array values are unsigned 64-bit unsigned integers (ulongs) */ + /** Indicates array values are unsigned 64-bit unsigned integers (ulongs) */ vtULong = 7, - /** Indicates array values are unsigned 64-bit integers (longs) */ - vtLong = 8, + /** Indicates array values are unsigned 64-bit integers (longs) */ + vtLong = 8, - /** Indicates array values are 32-bit floating point values (floats) */ + /** Indicates array values are 32-bit floating point values (floats) */ vtFloat = 9, - /** Indicates array values are 64-bit floating point values (doubles) */ + /** Indicates array values are 64-bit floating point values (doubles) */ vtDouble = 10, /** Indicates array values are das_time_t structures */ vtTime = 11, - /* VT_MAX_SIMPLE = vtTime */ + /* VT_MAX_SIMPLE = vtTime */ - /* The following type is not used by datums, but by array indexing elements - * that track the size and location of child dimensions */ - vtIndex = 12, + /* The following type is not used by datums, but by array indexing elements + * that track the size and location of child dimensions */ + vtIndex = 12, /* The following two types are only used by variables and datums * @@ -132,15 +132,32 @@ typedef enum das_val_type_e { /** Value are a vector struct as defined by vector.h */ vtGeoVec = 14, - /** Values are a picture element, posibly in multiple planes */ - /* Include later: vtPixel = 15, */ + /** Values are a picture element, posibly in multiple planes */ + /* Include later: vtPixel = 15, */ - /** Indicates values are size_t plus const ubyte* pairs, no more is - * known about the bytes */ - vtByteSeq = 15 - + /** Indicates values are size_t plus const ubyte* pairs, no more is + * known about the bytes */ + vtByteSeq = 15 + } das_val_type; + +/* Fixed values are used here so that we can pointer equality comparisons */ +#ifndef _das_value_c +extern const char* DAS_SEM_BIN; +extern const char* DAS_SEM_BOOL; +extern const char* DAS_SEM_DATE; +extern const char* DAS_SEM_INT; +extern const char* DAS_SEM_REAL; +extern const char* DAS_SEM_TEXT; +#endif + +/** Given a value type, suggest a default semantic */ +const char* das_sem_default(das_val_type vt); + +/** Given a semantic meaning, suggest a default value type */ +das_val_type das_vt_default(const char* sSemantic); + /** @} */ @@ -166,7 +183,7 @@ typedef enum das_val_type_e { * @memberof das_val_type */ DAS_API das_val_type das_vt_store_type( - const char* sEncType, int nItemBytes, const char* sInterp + const char* sEncType, int nItemBytes, const char* sInterp ); /** Get the serialization type given common packet encodings @@ -242,10 +259,13 @@ DAS_API das_valcmp_func das_vt_getcmp(das_val_type vt); * @returns -1 if A is less than B, 0 if equal, +1 if A is greater * than B or -2 if A is not comparable to B. */ -DAS_API int das_vt_cmpAny( +DAS_API int das_value_cmpAny( const ubyte* pA, das_val_type vtA, const ubyte* pB, das_val_type vtB ); + +#define das_vt_cmpAny das_value_cmpAny + /* In the future the token ID will come from the lexer, for now just make * something up*/ #define D2OP_PLUS 100 @@ -278,6 +298,65 @@ DAS_API int das_vt_cmpAny( */ DAS_API das_val_type das_vt_merge(das_val_type right, int op, das_val_type left); +#define DAS_VAL_NOERR_RNG 0x1 +#define DAS_VAL_ERR_RESLOSS 0x2 +/** Convert any integral type to any other with range checking and swapping + * + * @Note this function does not trigger on resolution loss unless requested. + * + * If one or both fill value pointers are NULL, all values are converted + * as if they represented valid items. + * + * @param vtIn the value type of the input + * @param pValIn pointer to the input value + * @param pFillIn the fill value for the input type, or NULL + * + * @param vtOut the value type of the output + * @param pValOut pointer to the output value storage + * @param pFillOut the fill value for the output type, or NULL + * + * @param uFlags A set of flags or'ed together. These are: + * - DAS_VAL_NOERR_RNG no error return on range violations + * - DAS_VAL_ERR_RESLOSS error return or resolution loss + * + * @returns DAS_OKAY if the conversion was sucessful, a positive error + * value if a range violation was triggered. + */ +DAS_API DasErrCode das_value_binXform( + das_val_type vtIn, const ubyte* pValIn, const ubyte* pFillIn, + das_val_type vtOut, ubyte* pValOut, const ubyte* pFillOut, + uint32_t uFlags +); + +/** Generate a printf style format code for a value type, usage and buffer size + * + * @note If nFitTo is too short you might get a format string that's too long + * without any warning. Checking the length of text output produced + * when writing a formated value is recommended. + * + * @param sBuf where to store the format string + * @param nBufLen space for the format string storage + * @param vt the value type in need of a format string + * @param sSemantic how the value is used. Format code changes for binary + * usage versus text or regular values. + * @param nFitTo If -1 the format string will produce variable length output + * if a positive number > 2 a fixed lenght format will be generated. + * + * @returns DAS_OKAY if a format string could be generated, a positive error + * value otherwise. + */ +DAS_API DasErrCode das_value_fmt( + char* sBuf, int nBufLen, das_val_type vt, const char* sSemantic, int nFitTo +); + +/** Small helper for printing reals in less space + * + * Since we don't know how precise a double precision value is, it's common + * to want to print these to 14 digits (or similar). Many times all those + * extra digits are just zeros. Use this function to shorten real values + * that after they have been written to a buffer. + */ +DAS_API void das_value_trimReal(char* sVal); /** Get a das value from a null terminated string * @@ -286,7 +365,7 @@ DAS_API das_val_type das_vt_merge(das_val_type right, int op, das_val_type left) * @returns DAS_OKAY if parsing was successful, an error return code otherwise. */ DAS_API DasErrCode das_value_fromStr( - ubyte* pBuf, int uBufLen, das_val_type vt, const char* sStr + ubyte* pBuf, int uBufLen, das_val_type vt, const char* sStr ); diff --git a/das2/var_ary.c b/das2/var_ary.c index 77a93f1..debfa87 100644 --- a/das2/var_ary.c +++ b/das2/var_ary.c @@ -114,11 +114,11 @@ bool DasVarAry_isNumeric(const DasVar* pBase) return false; } -DasAry* DasVarAry_getArray(DasVar* pThis) +DasAry* DasVar_getArray(DasVar* pBase) { - if( pThis->vartype != D2V_ARRAY) return NULL; - DasVarAry* pReallyThis = (DasVarAry*)pThis; - return pReallyThis->pAry; + if( pBase->vartype != D2V_ARRAY) return NULL; + DasVarAry* pThis = (DasVarAry*)pBase; + return pThis->pAry; } /* Public function, call from the top level @@ -234,7 +234,9 @@ ptrdiff_t DasVarAry_lengthIn(const DasVar* pBase, int nIdx, ptrdiff_t* pLoc) * size */ int i = 0; int nIndexes = 0; - for(i = 0; i < nIdx; ++i){ + /* nIdx is the number of indexes they want to "lock down", if they + don't want to lock down any then nIdx is 0 */ + for(i = 0; i <= nIdx; ++i){ if(pLoc[i] < 0){ das_error(DASERR_VAR, "Location index must not contain negative values"); @@ -945,7 +947,7 @@ DasErrCode init_DasVarAry( } pThis->base.vsize = das_vt_size(pThis->base.vt); - pThis->base.semantic = das_def_semantic(pThis->base.vt); + pThis->base.semantic = das_sem_default(pThis->base.vt); inc_DasAry(pAry); /* Increment the reference count for this array */ return DAS_OKAY; @@ -1157,7 +1159,7 @@ DasErrCode DasVarAry_encode(DasVar* pBase, const char* sRole, DasBuf* pBuf) /* 2. Get a Codec, either the one predefined for packet values, or make a new one for header values */ - DasAry* pAry = DasVarAry_getArray(pBase); + DasAry* pAry = DasVar_getArray(pBase); int nItemsPerWrite = 0; const DasCodec* pCodec = DasDs_getCodecFor(pDs, DasAry_id(pAry), &nItemsPerWrite); das_units units = pThis->base.units; @@ -1173,7 +1175,10 @@ DasErrCode DasVarAry_encode(DasVar* pBase, const char* sRole, DasBuf* pBuf) ); } - DasCodec_init(&codecHdr, pAry, pBase->semantic, "utf8", DASIDX_RAGGED, 0, units); + DasCodec_init( + DASENC_WRITE, &codecHdr, pAry, pBase->semantic, "utf8", DASIDX_RAGGED, 0, + units, NULL + ); pCodec = &codecHdr; } @@ -1183,7 +1188,7 @@ DasErrCode DasVarAry_encode(DasVar* pBase, const char* sRole, DasBuf* pBuf) /* 3. Define the variable in the output header, if vector add components */ - const char* sStorage = das_vt_toStr(vtExt); + const char* sStorage = vtAry == vtTime ? "struct" : das_vt_toStr(vtAry); const char* sType = (pThis->varsubtype == D2V_GEOVEC) ? "vector" : "scalar"; DasBuf_printf(pBuf, @@ -1214,7 +1219,7 @@ DasErrCode DasVarAry_encode(DasVar* pBase, const char* sRole, DasBuf* pBuf) if(aExtShape[0] == DASIDX_UNUSED){ DasBuf_puts(pBuf, " \n"); int nWrite = (int)DasAry_size(pAry); - int nVals = DasCodec_encode(&codecHdr, pBuf, nWrite, true); + int nVals = DasCodec_encode(&codecHdr, pBuf, DIM0, nWrite, DASENC_IN_HDR|DASENC_PKT_LAST); if(nVals < 0){ return das_error(DASERR_VAR, "Error encoding data for %s/%s/%s/%s", DasDs_id(pDs), DasDim_typeName(pDim), DasDim_id(pDim), sRole @@ -1230,7 +1235,7 @@ DasErrCode DasVarAry_encode(DasVar* pBase, const char* sRole, DasBuf* pBuf) das_datum_toStrValOnly(&dmFill, sFill, 63, 6); DasBuf_printf(pBuf, - " \n", + " \n", nItems, pCodec->nBufValSz, das_vt_serial_type(vtExt), sFill ); } diff --git a/das2/var_base.c b/das2/var_base.c index 8ec4d58..43451a7 100644 --- a/das2/var_base.c +++ b/das2/var_base.c @@ -24,40 +24,6 @@ /* This would be alot easier to implement in D using sumtype... oh well */ -/* ************************************************************************* */ -/* semantics */ - -const char* D2V_SEM_BOOL = "bool"; -const char* D2V_SEM_DATE = "datetime"; -const char* D2V_SEM_INT = "int"; -const char* D2V_SEM_REAL = "real"; -const char* D2V_SEM_TEXT = "string"; - -/** Given a value type, suggest a default semantic */ -const char* das_def_semantic(das_val_type vt) -{ - switch(vt){ - case vtFloat: case vtDouble: return D2V_SEM_REAL; - case vtTime: return D2V_SEM_DATE; - case vtText: return D2V_SEM_TEXT; - default: return D2V_SEM_INT; - } - /* Non atomic types do not have defaults, this includes: - vtGeoVec, vtPixel, vtByteSeq, vtIndex etc. */ -} - -/* Given a semantic, suggest a default value type */ -das_val_type das_def_valtype(const char* sSemantic) -{ - if(strcmp(sSemantic, "bool")) return vtByte; - if(strcmp(sSemantic, "datetime")) return vtTime; - if(strcmp(sSemantic, "int")) return vtInt; - if(strcmp(sSemantic, "real")) return vtDouble; - if(strcmp(sSemantic, "string")) return vtText; - return vtByteSeq; -} - - /* ************************************************************************* */ /* Set index printing direction... NOT thread safe */ bool g_bFastIdxLast = false; diff --git a/das2/var_bin.c b/das2/var_bin.c index 73ecd71..9b65dee 100644 --- a/das2/var_bin.c +++ b/das2/var_bin.c @@ -605,7 +605,7 @@ DasVar* new_DasVarBinary_tok( pThis->et = das_vt_merge( DasVar_elemType(pLeft), op, DasVar_elemType(pRight) ); - pThis->base.semantic = das_def_semantic(pThis->et); + pThis->base.semantic = das_sem_default(pThis->et); pThis->nOp = op; pThis->pLeft = pLeft; diff --git a/das2/var_con.c b/das2/var_con.c index 5049a46..b557fb7 100644 --- a/das2/var_con.c +++ b/das2/var_con.c @@ -235,7 +235,7 @@ DasVar* new_DasConstant(const char* sId, const das_datum* pDm) /* Vsize setting */ pThis->base.vsize = das_vt_size(pDm->vt); - pThis->base.semantic = das_def_semantic(pDm->vt); /* Def. semantic based on val type */ + pThis->base.semantic = das_sem_default(pDm->vt); /* Def. semantic based on val type */ strncpy(pThis->sId, sId, DAS_MAX_ID_BUFSZ - 1); diff --git a/das2/var_seq.c b/das2/var_seq.c index c2b000b..9193b06 100644 --- a/das2/var_seq.c +++ b/das2/var_seq.c @@ -563,7 +563,7 @@ DasVar* new_DasVarSeq( pThis->pM = pThis->M; double rScale; - pThis->base.semantic = D2V_SEM_INT; /* assume integer, till poven different */ + pThis->base.semantic = DAS_SEM_INT; /* assume integer, till poven different */ switch(vt){ case vtUByte: @@ -596,12 +596,12 @@ DasVar* new_DasVarSeq( case vtFloat: *((float*)(pThis->pB)) = *((float*)pMin); *((float*)(pThis->pM)) = *((float*)pInterval); - pThis->base.semantic = D2V_SEM_REAL; + pThis->base.semantic = DAS_SEM_REAL; break; case vtDouble: *((double*)(pThis->pB)) = *((double*)pMin); *((double*)(pThis->pM)) = *((double*)pInterval); - pThis->base.semantic = D2V_SEM_REAL; + pThis->base.semantic = DAS_SEM_REAL; break; case vtTime: /* Use the units to get the conversion factor for the slope to seconds */ @@ -611,7 +611,7 @@ DasVar* new_DasVarSeq( pThis->base.units = UNIT_UTC; *((das_time*)pThis->pB) = *((das_time*)pMin); - pThis->base.semantic = D2V_SEM_DATE; + pThis->base.semantic = DAS_SEM_DATE; break; default: das_error(DASERR_VAR, "Value type %d not yet supported for sequences", vt); diff --git a/das2/variable.h b/das2/variable.h index c2dfb47..db20e72 100644 --- a/das2/variable.h +++ b/das2/variable.h @@ -106,19 +106,6 @@ DAS_API ptrdiff_t das_varlength_merge(ptrdiff_t nLeft, ptrdiff_t nRight); #define D2V_EXP_INTR 0x10 #define D2V_EXP_TYPE 0x20 -/* Internal storage of the array may represent multiple things. For example - many types can represent booleans, or pixels on a screen */ -#ifndef _das_variable_c -extern const char* D2V_SEM_BOOL; -extern const char* D2V_SEM_DATE; -extern const char* D2V_SEM_INT; -extern const char* D2V_SEM_REAL; -extern const char* D2V_SEM_TEXT; -#endif - -/** Given a value type, suggest a default semantic */ -const char* das_def_semantic(das_val_type vt); - /** Given a semantic, suggest a default value type */ das_val_type das_def_valtype(const char* sSemantic); @@ -761,8 +748,8 @@ DAS_API das_units DasVar_units(const DasVar* pThis); * * @memberof DasVar */ -DAS_API DasAry* DasVarAry_getArray(DasVar* pThis); - +DAS_API DasAry* DasVar_getArray(DasVar* pThis); +#define DasVarAry_getArray DasVar_getArray /* Evaluate all sub-variable expressions and a single array variable */ diff --git a/test/TestIter.c b/test/TestIter.c index a0f69ff..c92ac05 100644 --- a/test/TestIter.c +++ b/test/TestIter.c @@ -63,11 +63,11 @@ int main(int argc, char** argv) DasVar* pCent = DasDim_getVar(pFreq, "center"); /* Print all unique frequencies, no matter the DS shape */ - das_uniq_iter iterU; + DasDsUniqIter iterU; das_datum dm; char sTest[20*100] = {'\0'}; char* pWrite = sTest; - for(das_uniq_iter_init(&iterU, pDs, pCent); !iterU.done; das_uniq_iter_next(&iterU)){ + for(DasDsUniqIter_init(&iterU, pDs, pCent); !iterU.done; DasDsUniqIter_next(&iterU)){ DasVar_get(pCent, iterU.index, &dm); das_datum_toStrValOnly(&dm, pWrite, 32, 4); pWrite += strlen(pWrite); diff --git a/utilities/das3_csv.c b/utilities/das3_csv.c index 9cf7683..51770dc 100644 --- a/utilities/das3_csv.c +++ b/utilities/das3_csv.c @@ -323,7 +323,7 @@ void _prnVecLblHdr(const DasDim* pDim, const DasVar* pVar) 3) I have no label, just print dirs */ const DasProp* pProp = DasDesc_getProp((DasDesc*)pDim, "label"); - const char* sVal = DasProp_value(pProp); + const char* sVal = (pProp == NULL) ? NULL : DasProp_value(pProp); ubyte uComp; const ubyte* pDir = DasVar_getDirs(pVar, &uComp); @@ -469,14 +469,14 @@ DasErrCode onData(StreamDesc* pSd, int iPktId, DasDs* pDs, void* pUser) dataset */ das_datum dm; - dasds_iterator iter; + DasDsUniqIter iter; char sBuf[128] = {'\0'}; /* for this dataset, get the list of variables that are worth printing Should actually do this once and save it! */ - const DasDim* pDim; - DasVar* pVar; - DasVar* aVars[128]; + const DasDim* pDim = NULL; + DasVar* pVar = NULL; + DasVar* aVars[128] = {0}; size_t uVars = 0; enum dim_type aDt[2] = {DASDIM_COORD, DASDIM_DATA}; @@ -484,7 +484,6 @@ DasErrCode onData(StreamDesc* pSd, int iPktId, DasDs* pDs, void* pUser) if(g_bIds) printf("%d%s", iPktId, g_sSep); if(g_bHeaders) printf("\"values\"%s", g_sSep); - bool bFirst = true; for(size_t c = 0; c < 2; ++c){ for(size_t u = 0; u < DasDs_numDims(pDs, aDt[c]); ++u){ pDim = DasDs_getDimByIdx(pDs, u, aDt[c]); @@ -498,8 +497,10 @@ DasErrCode onData(StreamDesc* pSd, int iPktId, DasDs* pDs, void* pUser) } } + bool bFirst = true; for(size_t v = 0; v < uVars; ++v){ - for(dasds_iter_init(&iter, pDs); !iter.done; dasds_iter_next(&iter)){ + DasDsUniqIter_init(&iter, pDs, aVars[v]); + for(; !iter.done; DasDsUniqIter_next(&iter)){ DasVar_get(aVars[v], iter.index, &dm); if(bFirst) bFirst = false; @@ -507,12 +508,12 @@ DasErrCode onData(StreamDesc* pSd, int iPktId, DasDs* pDs, void* pUser) fputs(g_sSep, stdout); fputs(das_datum_toStrValOnlySep(&dm, sBuf, 127, 6, g_sSep), stdout); } - - /* clean out the record varying stuff */ - DasAry* pAry = (DasAry*) DasVarAry_getArray(aVars[v]); - if(pAry) DasAry_clear(pAry); } fputs("\r\n", stdout); + + /* clean out the record varying stuff */ + size_t uCleared = DasDs_clearRagged0(pDs); + daslog_debug_v("Cleared %zu bytes of dataset memory", uCleared); return DAS_OKAY; } @@ -580,6 +581,10 @@ int main( int argc, char *argv[]) { continue; } if(strcmp(argv[i], "-l") == 0 || strcmp(argv[i], "--log") == 0){ + ++i; + if(i >= argc){ + return das_error(PERR, "Log level missing after -l"); + } sLevel = argv[i]; continue; } diff --git a/utilities/das3_spice.c b/utilities/das3_spice.c index 35978db..eb0b596 100644 --- a/utilities/das3_spice.c +++ b/utilities/das3_spice.c @@ -749,7 +749,8 @@ DasErrCode _addLocation(XCalc* pCalc, DasDs* pDsOut, const char* sAnnoteAxis) /* The new codec for output */ DasDs_addFixedCodec( - pDsOut, DasAry_id(pAryOut), "real", g_sFloatEnc, das_vt_size(vtFloat), 3 + pDsOut, DasAry_id(pAryOut), "real", g_sFloatEnc, das_vt_size(vtFloat), 3, + DASENC_WRITE ); /* The new variable to interface to the array */ @@ -832,7 +833,8 @@ DasErrCode _addRotation(XCalc* pCalc, const char* sAnonFrame, DasDs* pDsOut) /* Now add a codec for this array, assumes time is record varying */ if(nItems > 0){ DasDs_addFixedCodec( - pDsOut, DasAry_id(pAryOut), "real", g_sFloatEnc, das_vt_size(vtFloat), nItems + pDsOut, DasAry_id(pAryOut), "real", g_sFloatEnc, das_vt_size(vtFloat), nItems, + DASENC_WRITE ); } else{ @@ -840,7 +842,7 @@ DasErrCode _addRotation(XCalc* pCalc, const char* sAnonFrame, DasDs* pDsOut) point serialization. See codec.c */ DasDs_addRaggedCodec( pDsOut, DasAry_id(pAryOut), "real", g_sFloatEnc, das_vt_size(vtFloat), - nAryRank, das_vt_size(vtFloat), &(DAS_FLOAT_SEP[0][0]) + nAryRank, das_vt_size(vtFloat), &(DAS_FLOAT_SEP[0][0]), DASENC_WRITE ); } @@ -1024,7 +1026,9 @@ DasErrCode onDataSet(DasStream* pSdIn, int iPktId, DasDs* pDsIn, void* pUser) /* If there's no codec for the input array, we don't need to worry about it because these are header only values */ if(pCodec != NULL){ - DasCodec* pCodecOut = DasDs_addFixedCodecFrom(pDsOut, NULL, pCodec, nItems); + DasCodec* pCodecOut = DasDs_addFixedCodecFrom( + pDsOut, NULL, pCodec, nItems, DASENC_WRITE + ); if(!pCodecOut) return PERR; /* Tweek the output codec here. If the array vt is time, add two @@ -1146,10 +1150,10 @@ DasErrCode _writeLocation(DasDs* pDsIn, XCalc* pCalc) flatOut = (radOut - aTmp[0]) / radOut; } - das_uniq_iter iter; /* Produces unique indexes for given DS and Var */ - das_uniq_iter_init(&iter, pDsIn, pCalc->pVarOut); + DasDsUniqIter iter; /* Produces unique indexes for given DS and Var */ + DasDsUniqIter_init(&iter, pDsIn, pCalc->pVarOut); das_datum dm; - for(; !iter.done; das_uniq_iter_next(&iter)){ + for(; !iter.done; DasDsUniqIter_next(&iter)){ DasVar_get(pCalc->pTime, iter.index, &dm); rEt = _dm2et(&dm); @@ -1242,11 +1246,11 @@ DasErrCode _writeRotation(DasDs* pDsIn, XCalc* pCalc) DasVar* pVarOut = pCalc->pVarOut; uSysOut = pReq->uOutSystem; - das_uniq_iter iter; /* Produces unique indexes for given DS and Var */ - das_uniq_iter_init(&iter, pDsIn, pVarOut); + DasDsUniqIter iter; /* Produces unique indexes for given DS and Var */ + DasDsUniqIter_init(&iter, pDsIn, pVarOut); das_datum dm; - for(; !iter.done; das_uniq_iter_next(&iter)){ + for(; !iter.done; DasDsUniqIter_next(&iter)){ DasVar_get(pCalc->pTime, iter.index, &dm); @@ -1336,8 +1340,9 @@ DasErrCode writeAndClearDs(Context* pCtx, int iPktId, DasDs* pDsIn) if( (nRet = DasIO_writeData(pCtx->pOut, (DasDesc*)pDsOut, iPktId)) != DAS_OKAY) return nRet; - if( (nRet = DasDs_clearRagged0(pDsOut)) != DAS_OKAY) return nRet; - return DasDs_clearRagged0(pDsIn); + DasDs_clearRagged0(pDsOut); + DasDs_clearRagged0(pDsIn); + return DAS_OKAY; } DasErrCode onData(StreamDesc* pSd, int iPktId, DasDs* pDsIn, void* pUser) @@ -1370,11 +1375,19 @@ DasErrCode onClose(StreamDesc* pSdIn, void* pUser) /* Loop over all the datasets in the stream and make sure they are flushed */ int nPktId = 0; DasDesc* pDescIn = NULL; + DasDs* pDs = NULL; DasErrCode nRet; + ptrdiff_t aShape[DASIDX_MAX] = DASIDX_INIT_UNUSED; while((pDescIn = DasStream_nextDesc(pSdIn, &nPktId)) != NULL){ if(DasDesc_type(pDescIn) == DATASET){ - if((nRet = writeAndClearDs(pCtx, nPktId, (DasDs*)pDescIn)) != DAS_OKAY) - return nRet; + + /* If we have any data, then write it */ + pDs = (DasDs*)pDescIn; + DasDs_shape(pDs, aShape); + if(aShape[0] > 0){ + if((nRet = writeAndClearDs(pCtx, nPktId, (DasDs*)pDescIn)) != DAS_OKAY) + return nRet; + } } }