From a2954bc6e9f5eece7d3f3d45faacc1bdcccd104f Mon Sep 17 00:00:00 2001 From: Chris Piker Date: Tue, 31 Oct 2023 02:12:16 -0500 Subject: [PATCH] New catalog utility, http port change bug fix --- buildfiles/Linux.mak | 2 +- das2/http.c | 93 ++++++++++++++++++-- das2/http.h | 3 +- das2/log.c | 13 +++ das2/log.h | 17 +++- das2/node.c | 9 +- das2/node.h | 5 ++ utilities/das2_node.c | 197 ++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 326 insertions(+), 13 deletions(-) create mode 100644 utilities/das2_node.c diff --git a/buildfiles/Linux.mak b/buildfiles/Linux.mak index 548e948..3db7a18 100644 --- a/buildfiles/Linux.mak +++ b/buildfiles/Linux.mak @@ -27,7 +27,7 @@ endif UTIL_PROGS=das1_inctime das2_prtime das1_fxtime das2_ascii das2_bin_avg \ das2_bin_avgsec das2_bin_peakavgsec das2_from_das1 das2_from_tagged_das1 \ das1_ascii das1_bin_avg das2_bin_ratesec das2_psd das2_hapi das2_histo \ - das2_cache_rdr + das2_cache_rdr das2_node TEST_PROGS:=TestUnits TestArray TestVariable LoadStream TestBuilder \ TestAuth TestCatalog TestTT2000 ex_das_cli ex_das_ephem diff --git a/das2/http.c b/das2/http.c index 14fc8fe..60898db 100644 --- a/das2/http.c +++ b/das2/http.c @@ -174,6 +174,8 @@ bool das_http_setup_ssl(){ pthread_mutex_lock(&g_mtxHttp); + daslog_debug("Setting up SSL context"); + /* Now check a second time, ctx could have been setup while we were * waiting */ if(g_pSslCtx != NULL){ @@ -186,7 +188,7 @@ bool das_http_setup_ssl(){ ERR_load_crypto_strings(); SSL_load_error_strings(); - const SSL_METHOD* pMeth = SSLv23_client_method(); + const SSL_METHOD* pMeth = TLS_client_method(); g_pSslCtx = SSL_CTX_new(pMeth); if(g_pSslCtx == NULL){ /* have to use not thread locking error report here */ @@ -269,6 +271,10 @@ bool DasHttpResp_init(DasHttpResp* pRes, const char* sUrl) struct das_url* pUrl = &(pRes->url); memset(pUrl, 0, sizeof(struct das_url)); + pRes->nSockFd = -1; + pRes->nCode = -1; + pRes->url.sPort[0] = '8'; pRes->url.sPort[1] = '0'; + /* Get the scheme, this is a PITA but I don't want a large library * dependency and uriparser doesn't want to deal with utf-8 natively. * I'm sure curl would take care of it nicely, but does that exist on @@ -293,7 +299,7 @@ bool DasHttpResp_init(DasHttpResp* pRes, const char* sUrl) while(*pIn == '/') pIn++; /* Advance past a couple /'s or four */ pOut = pUrl->sHost; - while( (*pIn != '\0')&&(*pIn != ':')&&(*pIn != '/')&& + while( (*pIn != '\0')&&(*pIn != ':')&&(*pIn != '/')&&(*pIn != '?')&& ((pOut - pUrl->sHost) <= DASURL_SZ_HOST)){ *pOut = *pIn; ++pOut; ++pIn; } @@ -413,6 +419,10 @@ struct addrinfo* _das_http_getsrvaddr(DasHttpResp* pRes) struct addrinfo hints; struct addrinfo* pAddr = NULL; + char sHostAndPort[DASURL_SZ_HOST + DASURL_SZ_PORT + 2]; + snprintf(sHostAndPort, DASURL_SZ_HOST + DASURL_SZ_PORT + 2, "%s:%s", + pUrl->sHost, pUrl->sPort + ); /* First see if I already have the address info I need in the cache */ pthread_mutex_lock(&g_mtxAddrArys); @@ -422,7 +432,7 @@ struct addrinfo* _das_http_getsrvaddr(DasHttpResp* pRes) for(size_t u = 0; u < uHosts; ++u){ sName = DasAry_getCharsIn(g_pHostAry, DIM1_AT(u), &uStrLen); - if(strcmp(pUrl->sHost, sName) == 0){ + if(strcmp(sHostAndPort, sName) == 0){ pAddr = *((struct addrinfo**) DasAry_getAt(g_pAddrAry, vtUnknown, IDX0(u))); break; } @@ -526,7 +536,7 @@ struct addrinfo* _das_http_getsrvaddr(DasHttpResp* pRes) DasAry_append(g_pAddrAry, (const byte*) &pAddr, 1); - DasAry_append(g_pHostAry, (byte*)(pUrl->sHost), strlen(pUrl->sHost) + 1); + DasAry_append(g_pHostAry, (byte*)(sHostAndPort), strlen(sHostAndPort) + 1); DasAry_markEnd(g_pHostAry, DIM1); /* Roll first index, last idx is ragged */ pthread_mutex_unlock(&g_mtxAddrArys); @@ -548,7 +558,7 @@ bool _das_http_connect(DasHttpResp* pRes, struct timeval* pTimeOut) if(pAddr == NULL) return false; int nErr = 0; errno = 0; - int nFd = socket(pAddr->ai_family, pAddr->ai_socktype, pAddr->ai_protocol); + int nFd = socket(pAddr->ai_family, pAddr->ai_socktype, pAddr->ai_protocol); if(nFd == -1){ nErr = errno; pRes->sError = das_string( @@ -559,9 +569,9 @@ bool _das_http_connect(DasHttpResp* pRes, struct timeval* pTimeOut) return false; } else{ - daslog_debug_v("Connected to host %s, socket info follows\n" - "ai_family %d ai_socktype %d ai_protocol %d", pUrl->sHost, - pAddr->ai_family, pAddr->ai_socktype, pAddr->ai_protocol + daslog_debug_v("Connecting to host %s, socket info follows\n" + "ai_family %d ai_socktype %d ai_protocol %d sock_fd %d", pUrl->sHost, + pAddr->ai_family, pAddr->ai_socktype, pAddr->ai_protocol, nFd ); } @@ -659,6 +669,7 @@ bool _das_http_connect(DasHttpResp* pRes, struct timeval* pTimeOut) SSL* pSsl = NULL; pthread_mutex_lock(&g_mtxHttp); + daslog_debug_v("Creating new SSL session for fd %d", nFd); pSsl = SSL_new(g_pSslCtx); pthread_mutex_unlock(&g_mtxHttp); @@ -938,6 +949,22 @@ bool _das_http_readHdrs(DasHttpResp* pRes, DasBuf* pBuf) return true; /* They should be ready to recieve the msg body now */ } +/* ************************************************************************* */ +void _das_http_drain_socket(DasHttpResp* pRes){ + char sBuf[1024]; + ssize_t nTotal = 0; + ssize_t nRead = 0; + if(pRes->pSsl){ + while((nRead = SSL_read(pRes->pSsl, sBuf, 1024)) > 0) + nTotal += nRead; + } + else{ + while((nRead = recv(pRes->nSockFd, sBuf, 1024, 0)) > 0) + nTotal += nRead; + } + daslog_debug_v("Drained %d further bytes from %s", nTotal, pRes->url.sHost); +} + /* ************************************************************************* */ bool _das_http_redirect(DasHttpResp* pRes, DasBuf* pBuf) { @@ -948,6 +975,36 @@ bool _das_http_redirect(DasHttpResp* pRes, DasBuf* pBuf) return false; } + //size_t uEnd = 0; + //if(sNewUrl[0] != '\0'){ + // uEnd = strlen(sNewUrl) - 1; + // if(sNewUrl[uEnd] == '?') + // sNewUrl[uEnd] = '\0'; + //} + daslog_debug_v("Redirected to: %s", sNewUrl); + _das_http_drain_socket(pRes); // Read and toss any remaining data... + + // Tear down the existing SSL socket if needed + if(pRes->pSsl){ + daslog_debug("Old SSL socket teardown"); + pRes->nSockFd = SSL_get_fd((SSL*)pRes->pSsl); + if(SSL_shutdown((SSL*)pRes->pSsl) == 0){ + SSL_shutdown((SSL*)pRes->pSsl); + } + SSL_free((SSL*)pRes->pSsl); + pRes->pSsl = NULL; + } + + daslog_debug_v("Shutting down socket: %d", pRes->nSockFd); +#ifndef _WIN32 + shutdown(pRes->nSockFd, SHUT_RDWR); + close(pRes->nSockFd); +#else + shutdown(pRes->nSockFd, SD_BOTH); + closesocket(pRes->nSockFd); +#endif + pRes->nSockFd = -1; + /* Parse a new URL into the url structure */ if(!DasHttpResp_init(pRes, sNewUrl)) return false; return true; @@ -1026,9 +1083,12 @@ bool das_http_getBody( case HTTP_Found: case HTTP_TempRedir: case HTTP_PermRedir: + // uses the existing socket to pull down the redirect, then + // tears it down so the SSL context can be re-used. if(! _das_http_redirect(pRes, pBuf)) goto CLEANUP_ERROR; break; case HTTP_AuthReq: + _das_http_drain_socket(pRes); // Read and toss any remaining data... if(pRes->pSsl){ pRes->nSockFd = SSL_get_fd((SSL*)pRes->pSsl); if(SSL_shutdown((SSL*)pRes->pSsl) == 0){ @@ -1170,6 +1230,13 @@ DasAry* das_http_readUrl( nTotal += nRead; if(!DasAry_append(pAry, (const byte*) buf, nRead)) return false; /* Yay data! */ } + + pRes->nSockFd = SSL_get_fd((SSL*)pRes->pSsl); + if(SSL_shutdown((SSL*)pRes->pSsl) == 0) // If 0 needs 2-stage shutdown + SSL_shutdown((SSL*)pRes->pSsl); + + SSL_free((SSL*)pRes->pSsl); + pRes->pSsl = NULL; } else{ while((nLimit == -1)||(nTotal < nLimit)){ @@ -1195,5 +1262,15 @@ DasAry* das_http_readUrl( else daslog_debug_v("%ld bytes read from %s", DasAry_size(pAry), sUrl); + daslog_debug_v("Shutting down socket %d", pRes->nSockFd); +#ifndef _WIN32 + shutdown(pRes->nSockFd, SHUT_RDWR); + close(pRes->nSockFd); +#else + shutdown(pRes->nSockFd, SD_BOTH); + closesocket(pRes->nSockFd); +#endif + pRes->nSockFd = -1; + return pAry; } diff --git a/das2/http.h b/das2/http.h index 97591aa..3341570 100644 --- a/das2/http.h +++ b/das2/http.h @@ -43,6 +43,7 @@ extern "C" { #define DASURL_SZ_PATH 127 #define DASURL_SZ_QUERY 511 #define DASURL_SZ_DATASET 63 +#define DASURL_SZ_PORT 7 /* Called from das_init(), no need to call directly */ bool das_http_init(const char* sProgName); @@ -78,7 +79,7 @@ struct das_url { char sDataset[DASURL_SZ_DATASET+1]; /** The port number used to make the request, saved as a string */ - char sPort[8]; + char sPort[DASURL_SZ_PORT+1]; }; /** Encapsulates the status of a HTTP resource request */ diff --git a/das2/log.c b/das2/log.c index e5c5293..5d0de31 100644 --- a/das2/log.c +++ b/das2/log.c @@ -39,6 +39,19 @@ das_log_handler_t das_curMsgHandler = das_def_log_handler; #define LOCK() pthread_mutex_lock(&mtxDasLog) #define UNLOCK() pthread_mutex_unlock(&mtxDasLog) +int daslog_strlevel(const char* sLevel){ + if(sLevel != NULL){ + if(sLevel[0] == 'c'||sLevel[0] == 'C') return DASLOG_CRIT; + if(sLevel[0] == 'e'||sLevel[0] == 'E') return DASLOG_ERROR; + if(sLevel[0] == 'w'||sLevel[0] == 'W') return DASLOG_WARN; + if(sLevel[0] == 'i'||sLevel[0] == 'I') return DASLOG_INFO; + if(sLevel[0] == 'd'||sLevel[0] == 'D') return DASLOG_DEBUG; + if(sLevel[0] == 't'||sLevel[0] == 'T') return DASLOG_TRACE; + } + + return DASLOG_NOTHING; +} + bool daslog_set_showline(int nLevel){ int old; diff --git a/das2/log.h b/das2/log.h index 211f0e7..bc2c979 100644 --- a/das2/log.h +++ b/das2/log.h @@ -79,8 +79,8 @@ extern "C" { /** Get the log level. * - * @returns one of: DAS_LL_CRIT, DAS_LL_ERROR, DAS_LL_WARN, DAS_LL_INFO, - * DAS_LL_DEBUG, DAS_LL_TRACE + * @returns one of: DASLOG_CRIT, DASLOG_ERROR, DASLOG_WARN, DASLOG_INFO, + * DASLOG_DEBUG, DASLOG_TRACE */ DAS_API int daslog_level(void); @@ -99,6 +99,19 @@ DAS_API int daslog_level(void); */ DAS_API int daslog_setlevel(int nLevel); +/** Get a logging level integer from a string. + * This function may safely be called prior to das_init() + * + * @param sLevel One of "crit", "err", "warn", "info", + * "debug", "trace". Case is not signifiant, extra letters + * after the first are actually ignored. + * + * @returns One of DASLOG_CRIT, DASLOG_ERROR, DASLOG_WARN, + * DASLOG_INFO, DASLOG_DEBUG, DASLOG_TRACE, if the string is + * understood, DASLOG_NOTHING if not. + * */ +DAS_API int daslog_strlevel(const char* sLevel); + /** Output source file and line numbers for messages at or above this level */ DAS_API bool daslog_set_showline(int nLevel); diff --git a/das2/node.c b/das2/node.c index 09b42be..dcc6343 100644 --- a/das2/node.c +++ b/das2/node.c @@ -50,6 +50,13 @@ static const char* g_lDasDefRoots[2] = { static int g_nDasDefRoots = 2; +const char** das_root_urls(size_t* pLen){ + if(pLen != NULL) + *pLen = g_nDasDefRoots; + return g_lDasDefRoots; +} + + #define D2C_LOCAL_SUB_SZ 32 /* Catalog nodes have extra arrays to cache nodes */ @@ -182,7 +189,7 @@ DasNode* _DasNode_mkNode( sUrl, sAgent, pMgr, &httpRes, 1024*1024*20, rConSec ); if(pBytesAry == NULL){ - daslog_info(httpRes.sError); + daslog_warn(httpRes.sError); return NULL; } diff --git a/das2/node.h b/das2/node.h index 1ea33bd..f39b3df 100644 --- a/das2/node.h +++ b/das2/node.h @@ -167,6 +167,11 @@ typedef struct das_node { /** @} */ +/** Return the compiled in catalog root URLs + * @param pLen - A pointer to a size_t to received the number of builtin roots + * @return A pointer to an array of null terminated strings. + */ +DAS_API const char** das_root_urls(size_t* pLen); /** Create a new root catalog node via a path URI * diff --git a/utilities/das2_node.c b/utilities/das2_node.c new file mode 100644 index 0000000..78d96ea --- /dev/null +++ b/utilities/das2_node.c @@ -0,0 +1,197 @@ +/* This small utility is mostly meant to demonstrate an interface and thus + * has a liberal open source license... + * + * The MIT License + * + * Copyright 2023 Chris Piker + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include + +#include + +#define PROG_ERR 64 + +#define AGENT "das2C" + +/* ************************************************************************* */ +void prnHelp() +{ + fprintf(stderr, +"SYNOPSIS\n" +" das2_node - Read a node out of the federated catalog\n" +"\n" +"USAGE\n" +" das2_node [-h] [-R] [-a ALT_ROOT] [TAG_URI]\n" +"\n" +"DESCRIPTION\n" +" das2_node is a small utility which resolves a catalog URI to URL and then\n" +" writes the named catalog node to standard output. By default the\n" +" builtin root nodes are loaded first, then the catalog is walked to find the\n" +" requested sub-node. The walking algorithm automatically backs-up and tries\n" +" alternate branches when URL resolution fails. If there is only one URL for\n" +" a given branch, or if walking all branches still fails to load the node then\n" +" resolution fails.\n" +"\n" +" Any node of type Catalog may be a used as the root node. To do so provide\n" +" and absolute URL to the root in the optional second argument ALT_ROOT_URL\n" +"\n" +"OPTIONS\n" +"\n" +" -h,--help\n" +" Print this help text\n" +"\n" +" -R,--roots\n" +" Print the builtin root URLs and exit\n" +"\n" +" -a URL,--alt-root URL\n" +" Don't use the compiled in root URLs, look for the given object under\n" +" this alternate root catalog object. Useful for testing detached\n" +" catalogs.\n" +"\n" +" -l,--level\n" +" The logging level, one of 'none','crit', 'error', 'warn', 'info', \n" +" 'debug', or 'trace'.\n" +"\n" +"\n" +"EXAMPLES\n" +" Print the compiled in default federated catalog roots:\n" +" das2_node -R\n" +"\n" +" Get the U. Iowa Juno site data source catalog:\n" +" das2_node tag:das2.org,2012:site:/uiowa/juno\n" +"\n" +" Retrieve a HttpStreamSrc node for Juno Waves Survey data given an\n" +" explicit URL for the root node:\n" +" das2_node -a https://das2.org/catalog/das/site/uiowa.json juno/wav/survey/das2\n" +"\n" +"AUTHOR\n" +" chris-piker@uiowa.edu\n" +"\n"); +} + +/* ************************************************************************* */ + +int main(int argc, char** argv) +{ + /* Exit on errors, log info messages and above */ + das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_INFO, NULL); + + const char* sRootUrl = NULL; + const char* sNodeUri = NULL; + size_t uRoots = 0; + const char** psRoots; + + for(int i = 1; i < argc; i++){ + if(strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 ){ + prnHelp(); + return 0; + } + + if(strcmp(argv[i], "-R") == 0 || strcmp(argv[i], "--roots") == 0 ){ + psRoots = das_root_urls(&uRoots); + fprintf(stdout, "Compiled in das federated catalog URLs:\n"); + for(size_t u = 0; u < uRoots; ++u) + fprintf(stdout, " %s\n", psRoots[u]); + return 0; + } + + if(strcmp(argv[i], "-l") == 0 || strcmp(argv[i], "--level") == 0){ + if(argc <= i){ + daslog_error("Logging level argument missing, use -h for help"); + return 4; + } + ++i; + daslog_setlevel(daslog_strlevel(argv[i])); + continue; + } + + if(strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--alt-root") == 0){ + if(argc <= i){ + daslog_error( + "Alternate root URL missing after -a or --alt-root, use -h for help." + ); + return 5; + } + ++i; + sRootUrl = argv[i]; + continue; + } + + if(sNodeUri == NULL){ + sNodeUri = argv[i]; + continue; + } + + daslog_error_v("Unknown extra parameter: %s\n", argv[i]); + return 3; + } + + DasNode* pRoot; + if(sRootUrl) + pRoot = new_RootNode_url(sRootUrl, NULL, NULL, AGENT); + else + pRoot = new_RootNode(NULL, NULL, AGENT); + + if(pRoot == NULL){ + fprintf(stderr, "ERROR: Couldn't get the root node"); + return 4; + } + + DasNode* pNode; + if(sNodeUri) + pNode = DasNode_subNode(pRoot, sNodeUri, NULL, AGENT); + else + pNode = pRoot; + + if(pNode != NULL){ + fprintf(stdout, + "Loaded node: %s\n" + "From URL: %s\n", + DasNode_name(pNode), DasNode_srcUrl(pNode) + ); + } + else{ + daslog_error_v("Couldn't load %s starting from %s", sNodeUri, DasNode_srcUrl(pRoot)); + return 7; + } + + + if(DasNode_isJson(pNode)){ + const DasJdo* pJdo = DasNode_getJdo(pNode, NULL); + const char* pPretty = (char*) DasJdo_writePretty(pJdo, " ", "\n", NULL); + + fprintf(stdout, "\nIt has the following content:\n%s\n", pPretty); + } + else{ + fprintf(stdout, + "The object was type %d, there's no printer for it yet.\n", + pRoot->nType + ); + } + + del_RootNode(pRoot); + + return 0; +}