From 2f6d926cd8aa6954e756916bf3f2e476ad76971a Mon Sep 17 00:00:00 2001 From: Takashi Kojo Date: Tue, 18 Jan 2022 08:37:54 +0900 Subject: [PATCH 1/3] fixing select usage --- tls/server-tls-nonblocking.c | 59 ++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/tls/server-tls-nonblocking.c b/tls/server-tls-nonblocking.c index 6dc17121..3c372668 100644 --- a/tls/server-tls-nonblocking.c +++ b/tls/server-tls-nonblocking.c @@ -52,36 +52,34 @@ enum { static int tcp_select(SOCKET_T socketfd, int to_sec, int rx) { - fd_set fds, errfds; - fd_set* recvfds = NULL; - fd_set* sendfds = NULL; + fd_set errfds; + fd_set recvfds; + fd_set sendfds; SOCKET_T nfds = socketfd + 1; struct timeval timeout; int result; - FD_ZERO(&fds); - FD_SET(socketfd, &fds); + timeout.tv_sec = to_sec; + + FD_ZERO(&recvfds); + FD_SET(socketfd, &recvfds); + FD_ZERO(&sendfds); + FD_SET(socketfd, &sendfds); FD_ZERO(&errfds); FD_SET(socketfd, &errfds); - if (rx) - recvfds = &fds; - else - sendfds = &fds; - - result = select(nfds, recvfds, sendfds, &errfds, &timeout); - + result = select(nfds, &recvfds, &sendfds, &errfds, &timeout); + printf("fd = %d, select = %d\n", socketfd, result); + sleep(1); if (result == 0) return TEST_TIMEOUT; else if (result > 0) { - if (FD_ISSET(socketfd, &fds)) { - if (rx) - return TEST_RECV_READY; - else - return TEST_SEND_READY; - } - else if (FD_ISSET(socketfd, &errfds)) - return TEST_ERROR_READY; + if (FD_ISSET(socketfd, &recvfds)) + printf("Socket is ready for recv\n"); + if (FD_ISSET(socketfd, &sendfds)) + printf("Socket is ready for recv\n"); + if (FD_ISSET(socketfd, &errfds)) + printf("Socket is ready for error\n"); } return TEST_SELECT_FAIL; @@ -128,8 +126,6 @@ int main() goto exit; } - - /* Create and initialize WOLFSSL_CTX */ if ((ctx = wolfSSL_CTX_new(wolfTLSv1_2_server_method())) == NULL) { fprintf(stderr, "ERROR: failed to create WOLFSSL_CTX\n"); @@ -154,8 +150,6 @@ int main() goto exit; } - - /* Initialize the server address struct with zeros */ memset(&servAddr, 0, sizeof(servAddr)); @@ -200,6 +194,13 @@ int main() goto exit; } + /* Set the socket options to use nonblocking I/O */ + if (fcntl(connd, F_SETFL, O_NONBLOCK) == -1) { + fprintf(stderr, "ERROR: failed to set socket options\n"); + ret = -1; + goto exit; + } + /* Create a WOLFSSL object */ if ((ssl = wolfSSL_new(ctx)) == NULL) { fprintf(stderr, "ERROR: failed to create WOLFSSL object\n"); @@ -220,7 +221,7 @@ int main() ret = wolfSSL_accept(ssl); err = wolfSSL_get_error(ssl, ret); if (err == WOLFSSL_ERROR_WANT_READ) - tcp_select(sockfd, SELECT_WAIT_SEC, 1); + tcp_select(connd, SELECT_WAIT_SEC, 1); } while (err == WOLFSSL_ERROR_WANT_READ || err == WOLFSSL_ERROR_WANT_WRITE); if (ret != WOLFSSL_SUCCESS) { fprintf(stderr, "wolfSSL_accept error %d (%d)\n", err, ret); @@ -234,9 +235,8 @@ int main() do { ret = wolfSSL_read(ssl, buff, sizeof(buff)-1); err = wolfSSL_get_error(ssl, ret); - if (err == WOLFSSL_ERROR_WANT_READ) - tcp_select(sockfd, SELECT_WAIT_SEC, 1); + tcp_select(connd, SELECT_WAIT_SEC, 1); } while (err == WOLFSSL_ERROR_WANT_READ); if (ret < 0) { @@ -253,8 +253,6 @@ int main() shutdown = 1; } - - /* Write our reply into buff */ memset(buff, 0, sizeof(buff)); memcpy(buff, reply, strlen(reply)); @@ -264,7 +262,8 @@ int main() do { ret = wolfSSL_write(ssl, reply, len); err = wolfSSL_get_error(ssl, ret); - sleep(1); + if (err == WOLFSSL_ERROR_WANT_WRITE) + tcp_select(connd, SELECT_WAIT_SEC, 0); } while (err == WOLFSSL_ERROR_WANT_WRITE); if (ret < 0) { From 0842f8fe732aec7c37793ba70ea041baa9e41093 Mon Sep 17 00:00:00 2001 From: Takashi Kojo Date: Tue, 18 Jan 2022 19:26:44 +0900 Subject: [PATCH 2/3] poll version --- tls/server-tls-nonblocking.c | 69 ++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/tls/server-tls-nonblocking.c b/tls/server-tls-nonblocking.c index 3c372668..f5238eb0 100644 --- a/tls/server-tls-nonblocking.c +++ b/tls/server-tls-nonblocking.c @@ -30,6 +30,7 @@ #include #include #include +#include /* wolfSSL */ #include @@ -37,7 +38,7 @@ #include #define DEFAULT_PORT 11111 -#define SELECT_WAIT_SEC 1 +#define SELECT_WAIT_mSEC 1000 #define CERT_FILE "../certs/server-cert.pem" #define KEY_FILE "../certs/server-key.pem" @@ -50,39 +51,29 @@ enum { TEST_ERROR_READY }; -static int tcp_select(SOCKET_T socketfd, int to_sec, int rx) +static int tcp_poll(SOCKET_T socketfd, int msec, int rx) { - fd_set errfds; - fd_set recvfds; - fd_set sendfds; - SOCKET_T nfds = socketfd + 1; - struct timeval timeout; - int result; - - timeout.tv_sec = to_sec; - - FD_ZERO(&recvfds); - FD_SET(socketfd, &recvfds); - FD_ZERO(&sendfds); - FD_SET(socketfd, &sendfds); - FD_ZERO(&errfds); - FD_SET(socketfd, &errfds); - - result = select(nfds, &recvfds, &sendfds, &errfds, &timeout); - printf("fd = %d, select = %d\n", socketfd, result); - sleep(1); - if (result == 0) - return TEST_TIMEOUT; - else if (result > 0) { - if (FD_ISSET(socketfd, &recvfds)) - printf("Socket is ready for recv\n"); - if (FD_ISSET(socketfd, &sendfds)) - printf("Socket is ready for recv\n"); - if (FD_ISSET(socketfd, &errfds)) - printf("Socket is ready for error\n"); - } - - return TEST_SELECT_FAIL; + struct pollfd fds[1]; + int ret; + + fds[0].fd = socketfd; + if(rx) + fds[0].events = POLLIN; + else + fds[0].events = POLLOUT; + + if ((ret = poll(fds, 1, msec)) == 0) + ret = TEST_TIMEOUT; + else if (ret > 0) { + if (fds[0].revents & POLLIN) { + ret = TEST_RECV_READY; + } else if (fds[0].revents & POLLOUT) + ret = TEST_SEND_READY; + else + ret = TEST_SELECT_FAIL; + } else ret = TEST_SELECT_FAIL; + printf("Poll result = %d\n", ret); + return ret; } @@ -183,7 +174,7 @@ int main() == -1) { if (errno == EAGAIN || errno == EWOULDBLOCK) { /* non-blocking, wait for read activity on socket */ - tcp_select(sockfd, SELECT_WAIT_SEC, 1); + tcp_poll(sockfd, SELECT_WAIT_mSEC, 1); continue; } else if (errno == EINPROGRESS || errno == EALREADY) { @@ -193,7 +184,7 @@ int main() ret = -1; goto exit; } - + /* Set the socket options to use nonblocking I/O */ if (fcntl(connd, F_SETFL, O_NONBLOCK) == -1) { fprintf(stderr, "ERROR: failed to set socket options\n"); @@ -221,7 +212,7 @@ int main() ret = wolfSSL_accept(ssl); err = wolfSSL_get_error(ssl, ret); if (err == WOLFSSL_ERROR_WANT_READ) - tcp_select(connd, SELECT_WAIT_SEC, 1); + tcp_poll(connd, SELECT_WAIT_mSEC, 1); } while (err == WOLFSSL_ERROR_WANT_READ || err == WOLFSSL_ERROR_WANT_WRITE); if (ret != WOLFSSL_SUCCESS) { fprintf(stderr, "wolfSSL_accept error %d (%d)\n", err, ret); @@ -236,7 +227,7 @@ int main() ret = wolfSSL_read(ssl, buff, sizeof(buff)-1); err = wolfSSL_get_error(ssl, ret); if (err == WOLFSSL_ERROR_WANT_READ) - tcp_select(connd, SELECT_WAIT_SEC, 1); + tcp_poll(sockfd, SELECT_WAIT_mSEC, 1); } while (err == WOLFSSL_ERROR_WANT_READ); if (ret < 0) { @@ -263,7 +254,7 @@ int main() ret = wolfSSL_write(ssl, reply, len); err = wolfSSL_get_error(ssl, ret); if (err == WOLFSSL_ERROR_WANT_WRITE) - tcp_select(connd, SELECT_WAIT_SEC, 0); + tcp_poll(sockfd, SELECT_WAIT_mSEC, 0); } while (err == WOLFSSL_ERROR_WANT_WRITE); if (ret < 0) { @@ -276,7 +267,7 @@ int main() ret = wolfSSL_shutdown(ssl); err = wolfSSL_get_error(ssl, 0); if (err == WOLFSSL_ERROR_WANT_READ) - tcp_select(sockfd, SELECT_WAIT_SEC, 1); + tcp_poll(sockfd, SELECT_WAIT_mSEC, 1); } while (err == WOLFSSL_ERROR_WANT_READ || err == WOLFSSL_ERROR_WANT_WRITE); /* Cleanup after this connection */ From f2cbc37af492619558371698e874f0c7cddb0e80 Mon Sep 17 00:00:00 2001 From: Takashi Kojo Date: Sun, 23 Jan 2022 09:36:30 +0900 Subject: [PATCH 3/3] single polling point --- tls/server-tls-nonblocking.c | 320 ++++++++++++++++++++++++++--------- 1 file changed, 237 insertions(+), 83 deletions(-) diff --git a/tls/server-tls-nonblocking.c b/tls/server-tls-nonblocking.c index f5238eb0..4a27028b 100644 --- a/tls/server-tls-nonblocking.c +++ b/tls/server-tls-nonblocking.c @@ -43,45 +43,134 @@ #define CERT_FILE "../certs/server-cert.pem" #define KEY_FILE "../certs/server-key.pem" -enum { - TEST_SELECT_FAIL, - TEST_TIMEOUT, - TEST_RECV_READY, - TEST_SEND_READY, - TEST_ERROR_READY -}; - -static int tcp_poll(SOCKET_T socketfd, int msec, int rx) +typedef enum { + SV_IDLE, /* Free to use */ + SV_BEGIN, /* ready to begin */ + SV_ACCEPT, + SV_SSL_ACCEPT, + SV_SSL_READ, + SV_SSL_WRITE, + SV_SSL_SHUTDOWN +} SV_STATUS; + +typedef enum { + NB_CONTINUE, + NB_COMPLETE, + NB_FAITAL, +} NV_RETURN; + +typedef struct { + /* core context */ + SV_STATUS stat; + int sock; + + /* application context */ + int sockfd; + int connd; + WOLFSSL_CTX *ctx; + WOLFSSL *ssl; +} SV_CTX; + +#define MAX_CONTEXT 10 +static SV_CTX sv_tbl[MAX_CONTEXT]; + +static void sv_init() { - struct pollfd fds[1]; - int ret; - - fds[0].fd = socketfd; - if(rx) - fds[0].events = POLLIN; - else - fds[0].events = POLLOUT; - - if ((ret = poll(fds, 1, msec)) == 0) - ret = TEST_TIMEOUT; - else if (ret > 0) { - if (fds[0].revents & POLLIN) { - ret = TEST_RECV_READY; - } else if (fds[0].revents & POLLOUT) - ret = TEST_SEND_READY; - else - ret = TEST_SELECT_FAIL; - } else ret = TEST_SELECT_FAIL; - printf("Poll result = %d\n", ret); - return ret; + int i; + for(i = 0; i < MAX_CONTEXT; i++) { + sv_tbl[i].stat = SV_IDLE; + } +} + +static void sv_sock(SV_CTX *current, int sock) +{ + current->sock = sock; +} + +static SV_CTX * sv_fork(SV_CTX *current, int sock) +{ + int i; + for (i = 0; i < MAX_CONTEXT; i++) { + if (sv_tbl[i].stat == SV_IDLE) { + sv_tbl[i] = *current; + sv_tbl[i].sock = sock; + #ifdef SV_VERBOUSE + printf("sv_fork: %lx, sock = %d\n", (unsigned long)&sv_tbl[i], sock); + #endif + return &sv_tbl[i]; + } + } + #ifdef SV_VERBOUSE + printf("sv_fork: NONE\n"); + #endif + return NULL; +} + +static void sv_new() +{ + int i; + for (i = 0; i < MAX_CONTEXT; i++) { + if (sv_tbl[i].stat == SV_IDLE) { + sv_tbl[i].stat = SV_BEGIN; + sv_tbl[i].sock = -1; + #ifdef SV_VERBOUSE + printf("sv_new: %d, %lx\n", i, (unsigned long)&sv_tbl[i]); + #endif + return; + } + } + #ifdef SV_VERBOUSE + printf("sv_new: NONE\n"); + #endif + return; +} + +static void sv_free(SV_CTX *sv_ctx) +{ + #ifdef SV_VERBOUSE + printf("sv_free: %lx\n", (unsigned long)sv_ctx); + #endif + sv_ctx->stat = SV_IDLE; } +#define WAIT_mSEC 1000 + +static int scan = 0; + +static SV_CTX *sv_poll() { + int i; + int nfd = 0; + struct pollfd fds[MAX_CONTEXT]; + SV_CTX *next = NULL; -int main() + /* Scan context table */ + for (i = 0; i < MAX_CONTEXT; i++) { + if (sv_tbl[i].stat >= SV_BEGIN && sv_tbl[i].sock != SOCKET_INVALID) { + fds[nfd].events = POLLIN | POLLOUT; + fds[nfd].fd = sv_tbl[i].sock; + nfd++; + } + } + + poll(fds, nfd, WAIT_mSEC); + + for (i = 0; i < MAX_CONTEXT; i++) { + if(sv_tbl[scan].stat != SV_IDLE) { + next = &sv_tbl[scan]; + scan = (scan + 1) % MAX_CONTEXT; + break; + } + scan = (scan + 1) % MAX_CONTEXT; + } + return next; +} + + +int sv_main(SV_CTX *sv_ctx) { - int ret, err; - int sockfd = SOCKET_INVALID; - int connd = SOCKET_INVALID; + int ret = NB_FAITAL; + int err; + struct sockaddr_in servAddr; struct sockaddr_in clientAddr; socklen_t size = sizeof(clientAddr); @@ -90,17 +179,23 @@ int main() int shutdown = 0; const char* reply = "I hear ya fa shizzle!\n"; - /* declare wolfSSL objects */ - WOLFSSL_CTX* ctx = NULL; - WOLFSSL* ssl = NULL; + if(sv_ctx == NULL) + return NB_CONTINUE; +switch(sv_ctx->stat) { +case SV_BEGIN: + /* Initialize application context. Note they are with sv_ctx */ + #define sockfd sv_ctx->sockfd + #define connd sv_ctx->connd + #define ctx sv_ctx->ctx + #define ssl sv_ctx->ssl - /* Initialize wolfSSL */ - wolfSSL_Init(); - //wolfSSL_Debugging_ON(); - - + sockfd = SOCKET_INVALID; + connd = SOCKET_INVALID; + ctx = NULL; + ssl = NULL; + /* Create a socket that uses an internet IPv4 address, * Sets the socket to be stream based (TCP), * 0 means choose the default protocol. */ @@ -110,6 +205,8 @@ int main() goto exit; } + sv_sock(sv_ctx, sockfd); /* attach socket to the server context */ + /* Set the socket options to use nonblocking I/O */ if (fcntl(sockfd, F_SETFL, O_NONBLOCK) == -1) { fprintf(stderr, "ERROR: failed to set socket options\n"); @@ -117,6 +214,8 @@ int main() goto exit; } + + /* Create and initialize WOLFSSL_CTX */ if ((ctx = wolfSSL_CTX_new(wolfTLSv1_2_server_method())) == NULL) { fprintf(stderr, "ERROR: failed to create WOLFSSL_CTX\n"); @@ -141,6 +240,8 @@ int main() goto exit; } + + /* Initialize the server address struct with zeros */ memset(&servAddr, 0, sizeof(servAddr)); @@ -168,23 +269,28 @@ int main() /* Continue to accept clients until shutdown is issued */ while (!shutdown) { printf("Waiting for a connection...\n"); + sv_ctx->stat = SV_ACCEPT; + FALL_THROUGH; +case SV_ACCEPT: /* Accept client connections */ - while ((connd = accept(sockfd, (struct sockaddr*)&clientAddr, &size)) + if((connd = accept(sockfd, (struct sockaddr*)&clientAddr, &size)) == -1) { - if (errno == EAGAIN || errno == EWOULDBLOCK) { - /* non-blocking, wait for read activity on socket */ - tcp_poll(sockfd, SELECT_WAIT_mSEC, 1); - continue; - } - else if (errno == EINPROGRESS || errno == EALREADY) { - break; + if (errno == EAGAIN || errno == EWOULDBLOCK || + errno == EINPROGRESS || errno == EALREADY) { + return NB_CONTINUE; } fprintf(stderr, "ERROR: failed to accept the connection\n\n"); ret = -1; goto exit; } - + + if((sv_ctx = sv_fork(sv_ctx, connd)) == NULL) { + fprintf(stderr, "ERROR: failed to folk the context\n"); + ret = -1; + goto exit; + } + /* Set the socket options to use nonblocking I/O */ if (fcntl(connd, F_SETFL, O_NONBLOCK) == -1) { fprintf(stderr, "ERROR: failed to set socket options\n"); @@ -207,30 +313,35 @@ int main() /* Establish TLS connection */ printf("wolfSSL_accepting\n"); - - do { - ret = wolfSSL_accept(ssl); - err = wolfSSL_get_error(ssl, ret); - if (err == WOLFSSL_ERROR_WANT_READ) - tcp_poll(connd, SELECT_WAIT_mSEC, 1); - } while (err == WOLFSSL_ERROR_WANT_READ || err == WOLFSSL_ERROR_WANT_WRITE); - if (ret != WOLFSSL_SUCCESS) { + + sv_ctx->stat = SV_SSL_ACCEPT; + FALL_THROUGH; + +case SV_SSL_ACCEPT: + ret = wolfSSL_accept(ssl); + err = wolfSSL_get_error(ssl, ret); + if (ret != WOLFSSL_SUCCESS && + (err == WOLFSSL_ERROR_WANT_READ || err == WOLFSSL_ERROR_WANT_WRITE)) + return NB_CONTINUE; + else if (ret != WOLFSSL_SUCCESS) { fprintf(stderr, "wolfSSL_accept error %d (%d)\n", err, ret); goto exit; } printf("client connected successfully\n"); + sv_ctx->stat = SV_SSL_READ; + FALL_THROUGH; + +case SV_SSL_READ: /* read the client data into our buff array */ memset(buff, 0, sizeof(buff)); - do { - ret = wolfSSL_read(ssl, buff, sizeof(buff)-1); - err = wolfSSL_get_error(ssl, ret); - if (err == WOLFSSL_ERROR_WANT_READ) - tcp_poll(sockfd, SELECT_WAIT_mSEC, 1); - } - while (err == WOLFSSL_ERROR_WANT_READ); - if (ret < 0) { + + ret = wolfSSL_read(ssl, buff, sizeof(buff)-1); + err = wolfSSL_get_error(ssl, ret); + if (ret < 0 && err == WOLFSSL_ERROR_WANT_READ) + return NB_CONTINUE; + else if (ret < 0) { fprintf(stderr, "ERROR %d: failed to read\n", ret); goto exit; } @@ -243,42 +354,55 @@ int main() printf("Shutdown command issued!\n"); shutdown = 1; } + sv_ctx->stat = SV_SSL_WRITE; + FALL_THROUGH; + + +case SV_SSL_WRITE: /* Write our reply into buff */ memset(buff, 0, sizeof(buff)); memcpy(buff, reply, strlen(reply)); len = strnlen(buff, sizeof(buff)); /* Reply back to the client */ - do { - ret = wolfSSL_write(ssl, reply, len); - err = wolfSSL_get_error(ssl, ret); - if (err == WOLFSSL_ERROR_WANT_WRITE) - tcp_poll(sockfd, SELECT_WAIT_mSEC, 0); - } - while (err == WOLFSSL_ERROR_WANT_WRITE); - if (ret < 0) { + ret = wolfSSL_write(ssl, reply, len); + err = wolfSSL_get_error(ssl, ret); + if (ret < 0 && err == WOLFSSL_ERROR_WANT_WRITE) + return NB_CONTINUE; + else if (ret < 0) { fprintf(stderr, "ERROR %d: failed to write\n", ret); goto exit; } + sv_ctx->stat = SV_SSL_SHUTDOWN; + FALL_THROUGH; + +case SV_SSL_SHUTDOWN: /* send close notify */ - do { - ret = wolfSSL_shutdown(ssl); - err = wolfSSL_get_error(ssl, 0); - if (err == WOLFSSL_ERROR_WANT_READ) - tcp_poll(sockfd, SELECT_WAIT_mSEC, 1); - } while (err == WOLFSSL_ERROR_WANT_READ || err == WOLFSSL_ERROR_WANT_WRITE); + ret = wolfSSL_shutdown(ssl); + err = wolfSSL_get_error(ssl, 0); + if (ret != WOLFSSL_SUCCESS && err == WOLFSSL_ERROR_WANT_READ) + return NB_CONTINUE; /* Cleanup after this connection */ wolfSSL_free(ssl); /* Free the wolfSSL object */ close(connd); /* Close the connection to the client */ + /* Parent context keeps sockfd */ connd = SOCKET_INVALID; + + sv_ctx->stat = SV_IDLE; + FALL_THROUGH; + +case SV_IDLE: + return NB_CONTINUE; + } printf("Shutdown complete\n"); +} /* End of switch(sv_ctx->stat) */ - ret = 0; +ret = NB_COMPLETE; exit: /* Cleanup and return */ @@ -291,6 +415,36 @@ int main() if (ctx) wolfSSL_CTX_free(ctx); /* Free the wolfSSL context object */ wolfSSL_Cleanup(); /* Cleanup the wolfSSL environment */ - + return ret; } + + +int main(int argc, char ** argv) +{ + SV_CTX *sv_ctx; + + /* Initialize wolfSSL */ + wolfSSL_Init(); + // wolfSSL_Debugging_ON(); + + + sv_init(); + sv_new(); /* get a new context so that sv_poll can pick it up */ + + /* Super Loop */ + while(1) { + switch (sv_main(sv_ctx = sv_poll())) { + case NB_CONTINUE: + break; + case NB_COMPLETE: + sv_free(sv_ctx); + sv_new(); + break; + case NB_FAITAL: + sv_free(sv_ctx); + break; + } + } + return 0; +}