Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

3.0 #6

Merged
merged 41 commits into from
Feb 29, 2024
Merged
Changes from 1 commit
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
aea8340
Property storage re-work in progress
cpiker Jan 15, 2024
78ad919
Draft reworked properties descriptor
cpiker Jan 18, 2024
5dde71c
das2/3 descriptors compile
cpiker Jan 19, 2024
2272032
Using memory pool for property storage
cpiker Jan 21, 2024
784e880
Validating v2 test output with new properties
cpiker Jan 21, 2024
70ed802
New-style properties pass all unit tests
cpiker Jan 22, 2024
f550cfe
Draft das3 tag parsing
cpiker Jan 24, 2024
45c902f
das3 stream properties parsing works
cpiker Jan 24, 2024
98647fe
Var internal comp. defined, frame definitions added
cpiker Jan 28, 2024
fa6d574
Purpose of interal indexes well defined.
cpiker Jan 31, 2024
78f8389
Variables with internal structure pass unit tests
cpiker Jan 31, 2024
b0ec254
First draft code for das3 parser
cpiker Feb 14, 2024
8542616
Draft das3 parser builds
cpiker Feb 14, 2024
44ff737
Rank 3 header unittest passes
cpiker Feb 15, 2024
a173bf7
All data structure creation and printing tests pass
cpiker Feb 16, 2024
0fa6777
Draft data packet reader, debug in progress
cpiker Feb 16, 2024
2246a09
codec re-written to handle variable number of fixed length items per …
cpiker Feb 17, 2024
75b6df4
das3 parser passes first full stream unit test
cpiker Feb 17, 2024
1019c15
Started cdf writer, explicit stream model selection
cpiker Feb 17, 2024
9bac525
Allow das2 streams into das3 data models
cpiker Feb 17, 2024
c47e7bb
checkpoint: cdf writer devel
cpiker Feb 18, 2024
b125eae
Further work stream to CDF conversion
cpiker Feb 19, 2024
0e0d752
Can create headers and read data
cpiker Feb 19, 2024
4b29ddc
Draft CDF Writer variable output
cpiker Feb 22, 2024
2d9a65e
CDF creator can make vars, but dosen't fill them yet
cpiker Feb 22, 2024
c5dc423
CDF generator runs for rank 1 case
cpiker Feb 22, 2024
ef9c3e9
Complex CDF conversion, partial pass, needs work
cpiker Feb 23, 2024
4e509e2
Write rank-3 cdfs but dependences not assigned correctly
cpiker Feb 24, 2024
7378413
Dependencies fixed, time values are wrong
cpiker Feb 24, 2024
e4224a8
time struct->TT2k conversion, units improvements
cpiker Feb 24, 2024
72e2d80
Added CDF compression & CDF unittest
cpiker Feb 25, 2024
91dc116
Added make install target for das3_cdf
cpiker Feb 26, 2024
7f1c7b2
Finding libcdf via ENV vars
cpiker Feb 26, 2024
b9eaadb
Doxygen updates, still working on function groups
cpiker Feb 27, 2024
693d09d
First vector CDF test passes
cpiker Feb 28, 2024
1bbdbe8
First vector CDFs pass
cpiker Feb 29, 2024
4a9bba1
Better CDF label handling, release builds
cpiker Feb 29, 2024
0b2efc2
Handling null phydims and real props
cpiker Feb 29, 2024
61fb576
Updated cdf unittest, stray void removal
cpiker Feb 29, 2024
4940816
Added test cmd
cpiker Feb 29, 2024
46842f8
Merge remote-tracking branch 'origin' into 3.0
cpiker Feb 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
All data structure creation and printing tests pass
  • Loading branch information
cpiker committed Feb 16, 2024
commit a173bf771aa285fedc611d5b995b7db373fbbed4
33 changes: 9 additions & 24 deletions das2/dataset.c
Original file line number Diff line number Diff line change
@@ -371,8 +371,8 @@ char* DasDs_toStr(const DasDs* pThis, char* sBuf, int nLen)
{
char sDimBuf[1024] = {'\0'};
int nWritten = 0; /* Not necessarily the actual number of bytes written
* but good enough to know if we should exit due to
* running out of buffer space */
* but good enough to know if we should exit due to
* running out of buffer space */
char* pWrite = sBuf;
const char* pRead = NULL;

@@ -388,40 +388,25 @@ char* DasDs_toStr(const DasDs* pThis, char* sBuf, int nLen)
ptrdiff_t aShape[DASIDX_MAX];
DasDs_shape(pThis, aShape);

char* pSubWrite;
pSubWrite = das_shape_prnRng(aShape, pThis->nRank, pThis->nRank, pWrite, nLen);
nLen -= pSubWrite - pWrite;
char* pSubWrite = das_shape_prnRng(aShape, pThis->nRank, pThis->nRank, pWrite, nLen);
nLen -= (pSubWrite - pWrite);
pWrite = pSubWrite;
if(nLen < 20) return sBuf;

*pWrite = '\n'; ++pWrite; --nLen;
/*nWritten = snprintf(pWrite, nLen - 1, " | contains ...\n");
pWrite += nWritten; nLen -= nWritten;*/

/* Larry wanted the properties printed as well */
char sInfo[128] = {'\0'};
size_t u = 0;
const char* sName = NULL;
const char* sType = NULL;
const char* sVal = NULL;
const DasDesc* pBase = (const DasDesc*)pThis;
size_t uProps = DasDesc_length(pBase);
for(u = 0; u < uProps; ++u){
sName = DasDesc_getNameByIdx(pBase, u);
sType = DasDesc_getTypeByIdx(pBase, u);
sVal = DasDesc_getValByIdx(pBase, u);
strncpy(sInfo, sVal, 48); sInfo[48] = '\0';
nWritten = snprintf(pWrite, nLen - 1, " Property: %s | %s | %s\n",
sType, sName, sVal);
pWrite += nWritten; nLen -= nWritten;
if(nLen < 4) return sBuf;
}
pSubWrite = DasDesc_info((DasDesc*)pThis, pWrite, nLen, " ");
nLen -= (pSubWrite - pWrite);
pWrite = pSubWrite;

/* *pWrite = '\n'; ++pWrite; --nLen; */
nWritten = snprintf(pWrite, nLen-1, "\n ");
pWrite += nWritten; nLen -= nWritten;

/* Do data first... */
for(u = 0; u < pThis->uDims; ++u){
for(uint32_t u = 0; u < pThis->uDims; ++u){

if(pThis->lDims[u]->dtype != DASDIM_DATA) continue;

75 changes: 75 additions & 0 deletions das2/descriptor.c
Original file line number Diff line number Diff line change
@@ -88,6 +88,66 @@ void DasDesc_freeProps(DasDesc* pThis){
DasAry_deInit(&(pThis->properties));
}

/* ************************************************************************* */
/* Property summaries */

char* DasDesc_info(const DasDesc* pThis, char* sBuf, int nLen, char* sIndent)
{
char* pWrite = sBuf;

const DasAry* pProps = &(pThis->properties);
size_t uProps = DasAry_lengthIn(pProps, DIM0);

size_t uPropLen = 0;
for(size_t u = 0; u < uProps; ++u){
if(nLen < 80)
return pWrite;

const DasProp* pProp = (const DasProp*) DasAry_getBytesIn(
pProps, DIM1_AT(u), &uPropLen
);

if(!DasProp_isValid(pProp))
continue;

int nWritten = snprintf(pWrite, nLen - 1,
"%sProperty: %s | %s | ", sIndent, DasProp_name(pProp),
DasProp_typeStr3(pProp)
);
pWrite += nWritten; nLen -= nWritten;

if(nLen < 64)
return pWrite;

size_t uToWrite = strlen(DasProp_value(pProp));
bool bElipse = false;
if(uToWrite > 48){
uToWrite = 48;
bElipse = true;
}

strncpy(pWrite, DasProp_value(pProp), uToWrite);
pWrite += uToWrite; nLen -= uToWrite;
if(bElipse){
strcpy(pWrite, "...");
pWrite += 3; nLen -= 3;
}

if((pProp->units != NULL)&&(pProp->units != UNIT_DIMENSIONLESS)){
if(nLen < 32){
return pWrite;
}

nWritten = snprintf(pWrite, nLen - 1, " (%s)", Units_toStr(pProp->units));
pWrite += nWritten; nLen -= nWritten;
}

*pWrite = '\n'; ++pWrite; --nLen;
if(nLen < 40) return pWrite;
}
return pWrite;
}

/* ************************************************************************* */
/* Ownership */

@@ -291,6 +351,21 @@ const char* DasDesc_getTypeByIdx(const DasDesc* pThis, size_t uIdx)
return DasProp_typeStr2(pProp);
}

const char* DasDesc_getTypeByIdx3(const DasDesc* pThis, size_t uIdx)
{
const DasAry* pProps = &(pThis->properties);
size_t uProps = DasAry_lengthIn(pProps, DIM0);
if(uIdx >= uProps)
return NULL;

size_t uPropLen;
const DasProp* pProp = (const DasProp*) DasAry_getBytesIn(
pProps, DIM1_AT(uIdx), &uPropLen
);

return DasProp_typeStr3(pProp);
}

const char* DasDesc_getStr(const DasDesc* pThis, const char* sName)
{
return DasDesc_get(pThis, sName);
18 changes: 17 additions & 1 deletion das2/descriptor.h
Original file line number Diff line number Diff line change
@@ -126,8 +126,19 @@ typedef struct das_descriptor {
*/

/** @{ */
/** Initialize a memory location as a valid das descriptor
*
* @memberof DasDesc
*/
DAS_API void DasDesc_init(DasDesc* pThis, desc_type_t type);


/** Print 1-line versions of each property in a descriptor
*
* @memberof DasDesc
*/
DAS_API char* DasDesc_info(const DasDesc* pThis, char* sBuf, int nLen, char* sIndent);

/* Make an 'Unknown' type descriptor, incase you like using descriptor objects
* to store things in your code, not used by the library
* @memberof DasDesc
@@ -220,7 +231,12 @@ DAS_API const char* DasDesc_getValByIdx(const DasDesc* pThis, size_t uIdx);
*/
DAS_API const char* DasDesc_getTypeByIdx(const DasDesc* pThis, size_t uIdx);



/** Get a data type of a property by an index, das3 covention
* @memberof DasDesc
*/
DAS_API const char* DasDesc_getTypeByIdx3(const DasDesc* pThis, size_t uIdx);

/** Determine if a property is present in a Descriptor or it's ancestors.
*
* @param pThis the descriptor object to query
25 changes: 7 additions & 18 deletions das2/dimension.c
Original file line number Diff line number Diff line change
@@ -137,37 +137,26 @@ char* DasDim_toStr(const DasDim* pThis, char* sBuf, int nLen)
{
size_t u = 0;
char* pWrite = sBuf;
char sInfo[256] = {'\0'};

const char* sDimType = "Data";
if(pThis->dtype == DASDIM_COORD) sDimType = "Coordinate";
int nWritten = snprintf(sBuf, nLen - 1, "%s Dimension: %s\n",
sDimType, pThis->sDim);
pWrite += nWritten; nLen -= nWritten;
if(nLen < 1) return sBuf;
if(nLen < 40) return sBuf;

/* Larry wanted the properties printed as well */
const char* sName = NULL;
const char* sType = NULL;
const char* sVal = NULL;
const DasDesc* pBase = (const DasDesc*)pThis;
size_t uProps = DasDesc_length(pBase);
for(u = 0; u < uProps; ++u){
sName = DasDesc_getNameByIdx(pBase, u);
sType = DasDesc_getTypeByIdx(pBase, u);
sVal = DasDesc_getValByIdx(pBase, u);
strncpy(sInfo, sVal, 48); sInfo[48] = '\0';
nWritten = snprintf(pWrite, nLen - 1, " Property: %s | %s | %s\n",
sType, sName, sVal);
pWrite += nWritten; nLen -= nWritten;
if(nLen < 1) return sBuf;
}
char* pSubWrite = DasDesc_info((DasDesc*)pThis, pWrite, nLen, " ");
bool bReturn = (pSubWrite != pWrite);
nLen -= (pSubWrite - pWrite);
pWrite = pSubWrite;

if(nLen < 4) return sBuf;
if(uProps > 0){ *pWrite = '\n'; ++pWrite; --nLen; }
if(bReturn){ *pWrite = '\n'; ++pWrite; --nLen; }

/* Yea this is order N^2, but we never have that many variables in a dim */
/* Do the recognized variables first */
char sInfo[256] = {'\0'};
for(int nOrder = 0; nOrder < 16; ++nOrder){

for(u = 0; u < pThis->uVars; ++u){
63 changes: 63 additions & 0 deletions das2/frame.c
Original file line number Diff line number Diff line change
@@ -51,6 +51,69 @@ DasFrame* new_DasFrame(DasDesc* pParent, ubyte id, const char* sName, const char
return NULL;
}

char* DasFrame_info(const DasFrame* pThis, char* sBuf, int nLen)
{
if(nLen < 30)
return sBuf;

char* pWrite = sBuf;

int nWritten = snprintf(pWrite, nLen - 1, "\n Vector Frame %02hhu: %s |",
pThis->id, pThis->name
);

pWrite += nWritten;
nLen -= nWritten;

/* The directions */
for(uint32_t u = 0; u < pThis->ndirs; ++u){
if(nLen < 40) return pWrite;

if(u > 0){
*pWrite = ','; ++pWrite; *pWrite = ' '; ++pWrite;
nLen -= 2;
}
else{
*pWrite = ' '; ++pWrite; --nLen;
}
strncpy(pWrite, pThis->dirs[u], 40);
nWritten = strlen(pThis->dirs[u]);
pWrite += nWritten; nLen -= nWritten;
}

/* Type and inertial */
if(nLen < 40) return pWrite;
switch(pThis->flags & DASFRM_TYPE_MASK){
case DASFRM_CARTESIAN : nWritten = snprintf(pWrite, nLen - 1, " | cartesian"); break;
case DASFRM_POLAR : nWritten = snprintf(pWrite, nLen - 1, " | polar"); break;
case DASFRM_SPHERE_SURFACE: nWritten = snprintf(pWrite, nLen - 1, " | sphere_surface"); break;
case DASFRM_CYLINDRICAL : nWritten = snprintf(pWrite, nLen - 1, " | cylindrical"); break;
case DASFRM_SPHERICAL : nWritten = snprintf(pWrite, nLen - 1, " | spherical"); break;
default: nWritten = 0;
}

pWrite += nWritten; nLen -= nWritten;
if(nLen < 40) return pWrite;

if(pThis->flags & DASFRM_INERTIAL)
nWritten = snprintf(pWrite, nLen - 1, " (inertial)\n");
else
nWritten = snprintf(pWrite, nLen - 1, " (non-inertial)\n");

pWrite += nWritten; nLen -= nWritten;
if(nLen < 40) return pWrite;

char* pSubWrite = DasDesc_info((DasDesc*)pThis, pWrite, nLen, " ");
nLen -= (pSubWrite - pWrite);
pWrite = pSubWrite;

if(nLen > 4){
nWritten = snprintf(pWrite, nLen-1, "\n");
pWrite += nWritten; nLen -= nWritten;
}
return pWrite;
}

void DasFrame_inertial(DasFrame* pThis, bool bInertial)
{
if(bInertial)
6 changes: 6 additions & 0 deletions das2/frame.h
Original file line number Diff line number Diff line change
@@ -75,6 +75,12 @@ DAS_API DasFrame* new_DasFrame(
DasDesc* pParent, ubyte id, const char* sName, const char* sType
);

/** Print a 1-line summary of a frame and then it's properties
*
* @memberof DasFrame
*/
DAS_API char* DasFrame_info(const DasFrame* pThis, char* sBuf, int nLen);

/** Change the frame name
* @memberof DasFrame
*/
2 changes: 1 addition & 1 deletion das2/io.c
Original file line number Diff line number Diff line change
@@ -1162,7 +1162,7 @@ DasErrCode _DasIO_handleDesc(
StreamDesc_freeDesc(pSd, nPktId);
}

if((nRet = StreamDesc_addPktDesc(pSd, (PktDesc*)pDesc, nPktId)) != 0)
if((nRet = StreamDesc_addPktDesc(pSd, pDesc, nPktId)) != 0)
return nRet;
}
else{
2 changes: 1 addition & 1 deletion das2/processor.h
Original file line number Diff line number Diff line change
@@ -87,7 +87,7 @@ typedef DasErrCode (*DsDescHandler)(StreamDesc* sd, DasDs* dd, void* ud);
* new data were added
* @param ud A pointer to a user data structure, may be NULL
*/
typedef DasErrCode (*DsDataHandler)(StreamDesc* sd, DasDs* dd, ptrdiff_t* pi, void* ud);
typedef DasErrCode (*DsDataHandler)(StreamDesc* sd, DasDs* dd, void* ud);

/** Callback functions that are invoked on Stream Close
* callback function that is called at the end of the stream
5 changes: 3 additions & 2 deletions das2/property.c
Original file line number Diff line number Diff line change
@@ -360,7 +360,8 @@ const char* DasProp_typeStr2(const DasProp* pProp)

const char* DasProp_typeStr3(const DasProp* pProp)
{
switch(pProp->flags & DASPROP_MULTI_MASK){
uint64_t uInfo = pProp->flags & (DASPROP_TYPE_MASK |DASPROP_MULTI_MASK) ;
switch(uInfo){
case DASPROP_STRING |DASPROP_SINGLE: return "string";
case DASPROP_STRING |DASPROP_SET: return "stringArray";
case DASPROP_BOOL |DASPROP_SINGLE: return "bool";
@@ -374,7 +375,7 @@ const char* DasProp_typeStr3(const DasProp* pProp)
case DASPROP_DATETIME|DASPROP_SINGLE: return "datetime";
case DASPROP_DATETIME|DASPROP_RANGE: return "datetimeRange";
case DASPROP_DATETIME|DASPROP_SET: return "datetimeArray";
default: return NULL;
default: return "";
}
}

Loading