From dabbfa6fe0d5db7450c133a4271ae7dd23c6a6d0 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 26 Apr 2017 01:20:08 +0200 Subject: [PATCH 01/85] Reafactor sending methods - wip - bb test --- examples/tneat.c | 27 ++++++-------- neat.h | 2 +- neat_core.c | 93 +++++++++++++++++++++++------------------------- neat_internal.h | 5 +-- 4 files changed, 59 insertions(+), 68 deletions(-) diff --git a/examples/tneat.c b/examples/tneat.c index 4fcab935..5ede9f82 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -26,7 +26,7 @@ default values */ static uint32_t config_rcv_buffer_size = 1024; -static uint32_t config_snd_buffer_size = 1024; +static uint32_t config_snd_buffer_size = 102400; static uint32_t config_message_count = 10; static uint32_t config_runtime_max = 0; static uint16_t config_active = 0; @@ -76,7 +76,6 @@ struct tneat_flow_direction { }; struct tneat_flow { - uint8_t done; struct tneat_flow_direction rcv; struct tneat_flow_direction snd; }; @@ -110,8 +109,8 @@ print_usage() static neat_error_code on_error(struct neat_flow_operations *opCB) { - fprintf(stderr, "%s()\n", __func__); + exit(EXIT_FAILURE); return NEAT_OK; } @@ -142,12 +141,17 @@ on_all_written(struct neat_flow_operations *opCB) printf("\tduration\t: %.2fs\n", time_elapsed); printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->snd.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); - tnf->done = 1; + opCB->on_writable = NULL; + opCB->on_all_written = NULL; + neat_set_operations(opCB->ctx, opCB->flow, opCB); + + neat_shutdown(opCB->ctx, opCB->flow); + } else { + opCB->on_writable = on_writable; + opCB->on_all_written = NULL; + neat_set_operations(opCB->ctx, opCB->flow, opCB); } - opCB->on_writable = on_writable; - opCB->on_all_written = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); return NEAT_OK; } @@ -194,14 +198,6 @@ on_writable(struct neat_flow_operations *opCB) code = neat_write(opCB->ctx, opCB->flow, tnf->snd.buffer, config_snd_buffer_size, NULL, 0); - if (tnf->done) { - opCB->on_writable = NULL; - opCB->on_all_written = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); - neat_shutdown(opCB->ctx, opCB->flow); - return NEAT_OK; - } - if (code != NEAT_OK) { fprintf(stderr, "%s - neat_write error: code %d\n", __func__, (int)code); return on_error(opCB); @@ -315,7 +311,6 @@ on_connected(struct neat_flow_operations *opCB) } // reset stats - tnf->done = 0; tnf->snd.calls = 0; tnf->snd.bytes = 0; tnf->rcv.calls = 0; diff --git a/neat.h b/neat.h index d1270767..111e75c8 100644 --- a/neat.h +++ b/neat.h @@ -55,7 +55,7 @@ struct neat_flow_operations { void *userData; neat_error_code status; - int stream_id; + uint16_t stream_id; neat_flow_operations_fx on_connected; neat_flow_operations_fx on_error; neat_flow_operations_fx on_readable; diff --git a/neat_core.c b/neat_core.c index 195bb6ce..93d23d0e 100644 --- a/neat_core.c +++ b/neat_core.c @@ -88,6 +88,8 @@ static void neat_free_flow(struct neat_flow *flow); static neat_flow * do_accept(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *socket); neat_flow * neat_find_flow(neat_ctx *, struct sockaddr_storage *, struct sockaddr_storage *); +static void io_all_written(neat_ctx *ctx, neat_flow *flow, uint16_t stream_id); + #define TAG_STRING(tag) [tag] = #tag const char *neat_tag_name[NEAT_TAG_LAST] = { TAG_STRING(NEAT_TAG_STREAM_ID), @@ -933,23 +935,38 @@ static void io_connected(neat_ctx *ctx, neat_flow *flow, flow->operations->on_connected(flow->operations); } -static void io_writable(neat_ctx *ctx, neat_flow *flow, int stream_id, neat_error_code code) +static void +io_writable(neat_ctx *ctx, neat_flow *flow, neat_error_code code) { + // for callback struct + uint32_t stream_id = 0; + neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); + // we have buffered data, send to socket if (flow->isDraining) { - neat_write_flush(ctx, flow); + code = neat_write_flush(ctx, flow); + if (code != NEAT_OK && code != NEAT_ERROR_WOULD_BLOCK) { + neat_log(ctx, NEAT_LOG_ERROR, "error : %d", code); + neat_io_error(ctx, flow, code); + return; + } + // no buffered datat, notifiy application about writable flow + } else if (flow->operations && flow->operations->on_writable) { + READYCALLBACKSTRUCT; + flow->operations->on_writable(flow->operations); } - if (!flow->operations || !flow->operations->on_writable || flow->isDraining) { - return; + + if (!flow->isDraining) { + io_all_written(ctx, flow, 0); } - READYCALLBACKSTRUCT; - flow->operations->on_writable(flow->operations); + } // Translate SCTP cause codes (RFC4960 sect.3.3.10) // into NEAT error codes -static neat_error_code sctp_to_neat_code(uint16_t sctp_code) +static neat_error_code +sctp_to_neat_code(uint16_t sctp_code) { neat_error_code outcode; @@ -1591,7 +1608,7 @@ io_readable(neat_ctx *ctx, neat_flow *flow, } static void -io_all_written(neat_ctx *ctx, neat_flow *flow, int stream_id) +io_all_written(neat_ctx *ctx, neat_flow *flow, uint16_t stream_id) { neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); stream_id = NEAT_INVALID_STREAM; @@ -2134,30 +2151,18 @@ void uvpollable_cb(uv_poll_t *handle, int status, int events) } #endif - // newly created flow, trigger "on_connected" callback + // newly created flow if ((events & UV_WRITABLE) && flow->firstWritePending) { flow->firstWritePending = 0; io_connected(ctx, flow, NEAT_OK); } - // socket is writable and flow has outstanding data so send, drain buffer before calling "on_writable" - if ((events & UV_WRITABLE) && flow->isDraining) { - neat_error_code code = neat_write_flush(ctx, flow); - if (code != NEAT_OK && code != NEAT_ERROR_WOULD_BLOCK) { - neat_io_error(ctx, flow, code); - return; - } - if (!flow->isDraining) { - io_all_written(ctx, flow, 0); - } - } - - // socket is writable, trigger "on_writable" + // socket is writable if (events & UV_WRITABLE) { - io_writable(ctx, flow, 0, NEAT_OK); // TODO: Remove stream param + io_writable(ctx, flow, NEAT_OK); } - // socket is readable, trigger "on_readable" + // socket is readable if (events & UV_READABLE) { io_readable(ctx, flow, pollable_socket, NEAT_OK); } @@ -4339,6 +4344,7 @@ neat_write_flush(struct neat_ctx *ctx, struct neat_flow *flow) if (TAILQ_EMPTY(&flow->bufferedMessages)) { return NEAT_OK; } + TAILQ_FOREACH_SAFE(msg, &flow->bufferedMessages, message_next, next_msg) { do { iov.iov_base = msg->buffered + msg->bufferedOffset; @@ -4371,7 +4377,7 @@ neat_write_flush(struct neat_ctx *ctx, struct neat_flow *flow) if ((flow->socket->sctp_explicit_eor) && (len == msg->bufferedSize)) { sndinfo->snd_flags |= SCTP_EOR; } -#endif +#endif // defined(SCTP_EOR) #elif defined (SCTP_SNDRCV) msghdr.msg_control = cmsgbuf; msghdr.msg_controllen = CMSG_SPACE(sizeof(struct sctp_sndrcvinfo)); @@ -4386,11 +4392,11 @@ neat_write_flush(struct neat_ctx *ctx, struct neat_flow *flow) if ((flow->isSCTPExplicitEOR) && (len == msg->bufferedSize)) { sndrcvinfo->sinfo_flags |= SCTP_EOR; } -#endif -#else +#endif // defined(SCTP_EOR) +#else // defined(SCTP_SNDINFO) msghdr.msg_control = NULL; msghdr.msg_controllen = 0; -#endif +#endif // defined(SCTP_SNDINFO) } else { msghdr.msg_control = NULL; msghdr.msg_controllen = 0; @@ -4426,6 +4432,7 @@ neat_write_flush(struct neat_ctx *ctx, struct neat_flow *flow) msg->bufferedOffset += rv; msg->bufferedSize -= rv; } while (msg->bufferedSize > 0); + TAILQ_REMOVE(&flow->bufferedMessages, msg, message_next); free(msg->buffered); free(msg); @@ -4504,6 +4511,7 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, ssize_t rv = 0; size_t len; int atomic; + neat_error_code code = NEAT_OK; int stream_id = 0; int has_stream_id = 0; @@ -4547,15 +4555,15 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, HANDLE_OPTIONAL_ARGUMENTS_END(); if (has_stream_id && stream_id < 0) { - neat_log(ctx, NEAT_LOG_DEBUG, "Invalid stream id: Must be 0 or greater"); + neat_log(ctx, NEAT_LOG_DEBUG, "%s - invalid stream id: Must be 0 or greater", __func__); return NEAT_ERROR_BAD_ARGUMENT; } else if (has_stream_id && flow->socket->sctp_streams_available == 1 && stream_id != 0) { - neat_log(ctx, NEAT_LOG_DEBUG, "Tried to specify stream id when only a single stream is in use. Ignoring."); + neat_log(ctx, NEAT_LOG_WARNING, "%s - tried to specify stream id when only a single stream is in use - ignoring", __func__); stream_id = 0; } else if (has_stream_id && flow->socket->stack != NEAT_STACK_SCTP) { // For now, warn about this. Maybe we emulate multistreaming over TCP in // the future? - neat_log(ctx, NEAT_LOG_DEBUG, "Tried to specify stream id when using a protocol which does not support multistreaming. Ignoring."); + neat_log(ctx, NEAT_LOG_WARNING, "%s - tried to specify stream id when using a protocol which does not support multistreaming - ignoring", __func__); stream_id = 0; } @@ -4587,12 +4595,10 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, break; } if (atomic && flow->socket->write_size > 0 && amt > flow->socket->write_size) { + neat_log(ctx, NEAT_LOG_DEBUG, "%s - message size exceeds limit - aborting transmission", __func__); return NEAT_ERROR_MESSAGE_TOO_BIG; } - neat_error_code code = neat_write_flush(ctx, flow); - if (code != NEAT_OK && code != NEAT_ERROR_WOULD_BLOCK) { - return code; - } + if (TAILQ_EMPTY(&flow->bufferedMessages) && code == NEAT_OK && amt > 0) { iov.iov_base = (void *)buffer; if ((neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) && @@ -5919,7 +5925,7 @@ handle_connect(struct socket *sock, void *arg, int flags) io_connected(flow->ctx, flow, NEAT_OK); } if ((usrsctp_get_events(sock) & SCTP_EVENT_WRITE) && flow->operations->on_writable) { - io_writable(flow->ctx, flow, 0, NEAT_OK); + io_writable(flow->ctx, flow, NEAT_OK); } } else { neat_log(candidate->ctx, NEAT_LOG_DEBUG, "%s - NOT first connect", __func__); @@ -5970,21 +5976,10 @@ handle_upcall(struct socket *sock, void *arg, int flags) return; } - if (events & SCTP_EVENT_WRITE && flow->isDraining) { - neat_error_code code = neat_write_flush(ctx, flow); - if (code != NEAT_OK && code != NEAT_ERROR_WOULD_BLOCK) { - neat_io_error(ctx, flow, code); - return; - } - if (!flow->isDraining) { - io_all_written(ctx, flow, 0); - } - } - if (events & SCTP_EVENT_WRITE && flow->operations->on_writable) { if (flow->firstWritePending) flow->firstWritePending = 0; - io_writable(ctx, flow, 0, NEAT_OK); + io_writable(ctx, flow, NEAT_OK); } if (events & SCTP_EVENT_READ && flow->operations->on_readable) { @@ -5997,7 +5992,7 @@ handle_upcall(struct socket *sock, void *arg, int flags) } events = usrsctp_get_events(sock); if (events & SCTP_EVENT_WRITE && flow->operations->on_writable) - io_writable(ctx, flow, 0, NEAT_OK); + io_writable(ctx, flow, NEAT_OK); } } diff --git a/neat_internal.h b/neat_internal.h index 3e11f572..e850d4b2 100644 --- a/neat_internal.h +++ b/neat_internal.h @@ -34,7 +34,8 @@ struct neat_event_cbs* event_cbs; \ uint8_t src_addr_cnt -#define NEAT_MAX_NUM_PROTO 4 +#define NEAT_MAX_NUM_PROTO 4 +#define MAX_LOCAL_ADDR 64 struct neat_event_cb; struct neat_addr; @@ -187,7 +188,7 @@ struct neat_pollable_socket struct sockaddr_storage src_sockaddr; socklen_t src_len; - #define MAX_LOCAL_ADDR 64 + struct sockaddr_storage local_addr[MAX_LOCAL_ADDR]; unsigned int nr_local_addr; From 10b393ccdd5fd3d64a71a74c815aa1dc94be5472 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 26 Apr 2017 01:50:53 +0200 Subject: [PATCH 02/85] Wait until all data has been drained until calling shutdown on socket --- examples/tneat.c | 12 ++++++------ neat_core.c | 16 ++++++++++++++-- neat_internal.h | 3 ++- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/examples/tneat.c b/examples/tneat.c index 5ede9f82..25b9690e 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -173,11 +173,6 @@ on_writable(struct neat_flow_operations *opCB) gettimeofday(&(tnf->snd.tv_first), NULL); } - // set callbacks - opCB->on_writable = NULL; - opCB->on_all_written = on_all_written; - neat_set_operations(opCB->ctx, opCB->flow, opCB); - // increase stats tnf->snd.calls++; tnf->snd.bytes += config_snd_buffer_size; @@ -203,6 +198,11 @@ on_writable(struct neat_flow_operations *opCB) return on_error(opCB); } + // set callbacks + opCB->on_writable = NULL; + opCB->on_all_written = on_all_written; + neat_set_operations(opCB->ctx, opCB->flow, opCB); + return NEAT_OK; } @@ -275,7 +275,7 @@ on_readable(struct neat_flow_operations *opCB) } } - neat_shutdown(opCB->ctx, opCB->flow); + neat_close(opCB->ctx, opCB->flow); } return NEAT_OK; diff --git a/neat_core.c b/neat_core.c index 93d23d0e..104a5566 100644 --- a/neat_core.c +++ b/neat_core.c @@ -958,7 +958,12 @@ io_writable(neat_ctx *ctx, neat_flow *flow, neat_error_code code) } if (!flow->isDraining) { - io_all_written(ctx, flow, 0); + if (flow->isClosing) { + // neat_shutdown has been called while flow was draining, run shutdown procedure + neat_shutdown(ctx, flow); + } else { + io_all_written(ctx, flow, 0); + } } } @@ -6135,6 +6140,12 @@ neat_error_code neat_shutdown(struct neat_ctx *ctx, struct neat_flow *flow) { neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); + flow->isClosing = 1; + + if (flow->isDraining) { + return NEAT_OK; + } + #if defined(USRSCTP_SUPPORT) if (neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) { return neat_shutdown_via_usrsctp(ctx, flow); @@ -6315,7 +6326,8 @@ void neat_notify_aborted(neat_flow *flow) } // Notify application a connection has closed -void neat_notify_close(neat_flow *flow) +void +neat_notify_close(neat_flow *flow) { const int stream_id = NEAT_INVALID_STREAM; //READYCALLBACKSTRUCT expects this: diff --git a/neat_internal.h b/neat_internal.h index e850d4b2..693a6001 100644 --- a/neat_internal.h +++ b/neat_internal.h @@ -219,6 +219,7 @@ struct neat_pollable_socket struct neat_flow { // Main socket used for communication, not listening + uint8_t state; struct neat_pollable_socket *socket; TAILQ_HEAD(neat_listen_socket_head, neat_pollable_socket) listen_sockets; struct neat_flow_operations *operations; // see ownedByCore flag @@ -227,7 +228,6 @@ struct neat_flow uint16_t port; uint8_t qos; uint8_t ecn; - //uint16_t stream_count; struct neat_resolver_results *resolver_results; const struct sockaddr *sockAddr; // raw unowned pointer into resolver_results struct neat_ctx *ctx; // raw convenience pointer @@ -274,6 +274,7 @@ struct neat_flow unsigned int isServer : 1; // i.e. created via accept() unsigned int isSCTPMultihoming : 1; unsigned int isSCTPIdata : 1; + unsigned int isClosing : 1; unsigned int streams_requested; From 60e5e4d32278d253f9a4d8c27b96118f3c0108d1 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 26 Apr 2017 10:15:34 +0200 Subject: [PATCH 03/85] Cleanup --- neat_core.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/neat_core.c b/neat_core.c index 104a5566..02ee0eaf 100644 --- a/neat_core.c +++ b/neat_core.c @@ -938,9 +938,7 @@ static void io_connected(neat_ctx *ctx, neat_flow *flow, static void io_writable(neat_ctx *ctx, neat_flow *flow, neat_error_code code) { - // for callback struct - uint32_t stream_id = 0; - + const uint16_t stream_id = NEAT_INVALID_STREAM; neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); // we have buffered data, send to socket @@ -957,6 +955,7 @@ io_writable(neat_ctx *ctx, neat_flow *flow, neat_error_code code) flow->operations->on_writable(flow->operations); } + // flow is not draining (anymore) if (!flow->isDraining) { if (flow->isClosing) { // neat_shutdown has been called while flow was draining, run shutdown procedure @@ -5995,6 +5994,8 @@ handle_upcall(struct socket *sock, void *arg, int flags) if (code == READ_WITH_ZERO && flow->operations && flow->operations->on_readable) flow->operations->on_readable(flow->operations); } + + // xxx why two times? events = usrsctp_get_events(sock); if (events & SCTP_EVENT_WRITE && flow->operations->on_writable) io_writable(ctx, flow, NEAT_OK); From cc23222442037f9c785a39cabbf5619f4c9b3812 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 26 Apr 2017 14:19:24 +0200 Subject: [PATCH 04/85] refactoring and bugfixes for clinet_http_run_once --- examples/client_http_run_once.c | 59 +++++++++++++++++++++------------ examples/tneat.c | 40 +++++++++++----------- neat_core.c | 2 +- 3 files changed, 58 insertions(+), 43 deletions(-) diff --git a/examples/client_http_run_once.c b/examples/client_http_run_once.c index 6e3ad6a4..ecf2daeb 100644 --- a/examples/client_http_run_once.c +++ b/examples/client_http_run_once.c @@ -27,9 +27,10 @@ **********************************************************************/ +#define MAX_FLOWS 500 +#define MAX_CTXS 10 + static uint32_t config_rcv_buffer_size = 65536; -static uint32_t config_max_flows = 500; -static uint32_t config_max_ctxs = 50; static char request[512]; static const char *request_tail = "HTTP/1.0\r\nUser-agent: libneat\r\nConnection: close\r\n\r\n"; static char *config_property_sctp_tcp = "{\ @@ -70,10 +71,12 @@ static char *config_property_sctp = "{\ }"; static unsigned char *buffer = NULL; static int streams_going = 0; +static uint32_t ctx_flows[MAX_CTXS]; struct user_flow { struct neat_flow *flow; uint32_t id; + uint32_t ctx_id; }; static neat_error_code @@ -102,14 +105,20 @@ on_readable(struct neat_flow_operations *opCB) if (!bytes_read) { // eof struct user_flow *f = opCB->userData; + ctx_flows[f->ctx_id] = ctx_flows[f->ctx_id] - 1; streams_going--; /* one stream less */ - fprintf(stderr, "[Flow %u ended, %d to go]\n", f->id, streams_going); + fprintf(stderr, "[Flow %u ended, %d to go, %d for ctx %d]\n", f->id, streams_going, ctx_flows[f->ctx_id], f->ctx_id); fflush(stdout); opCB->on_readable = NULL; // do not read more neat_set_operations(opCB->ctx, opCB->flow, opCB); - neat_stop_event_loop(opCB->ctx); + + if (ctx_flows[f->ctx_id] == 0) { + fprintf(stderr, "%s - no flows left for ctx, stopping event loop\n", __func__); + neat_stop_event_loop(opCB->ctx); + } + } else if (bytes_read > 0) { - fwrite(buffer, sizeof(char), bytes_read, stdout); + //fwrite(buffer, sizeof(char), bytes_read, stdout); } return 0; } @@ -140,24 +149,26 @@ on_connected(struct neat_flow_operations *opCB) int main(int argc, char *argv[]) { - struct neat_ctx *ctx[config_max_ctxs]; - struct user_flow flows[config_max_flows]; - struct neat_flow_operations ops[config_max_flows]; - struct pollfd fds[config_max_ctxs]; + struct neat_ctx *ctx[MAX_CTXS]; + struct user_flow flows[MAX_FLOWS]; + struct neat_flow_operations ops[MAX_FLOWS]; + struct pollfd fds[MAX_CTXS]; int result = 0; int arg = 0; uint32_t num_flows = 3; uint32_t i = 0; uint32_t c = 0; - int backend_fds[config_max_ctxs]; - uint32_t num_ctxs = 1; + int backend_fds[MAX_CTXS]; + uint32_t num_ctxs = 10; int config_log_level = NEAT_LOG_WARNING; const char *config_property; result = EXIT_SUCCESS; - memset(&ops, 0, sizeof(ops)); - memset(flows, 0, sizeof(flows)); - memset(ctx, 0, sizeof(ctx)); + memset(&ops, 0, sizeof(ops)); + memset(ctx, 0, sizeof(ctx)); + memset(&flows, 0, sizeof(flows)); + memset(&ctx_flows, 0, sizeof(ctx_flows)); + config_property = config_property_sctp_tcp; snprintf(request, sizeof(request), "GET %s %s", "/", request_tail); @@ -169,15 +180,15 @@ main(int argc, char *argv[]) break; case 'n': num_flows = strtoul (optarg, NULL, 0); - if (num_flows > config_max_flows) { - num_flows = config_max_flows; + if (num_flows > MAX_FLOWS) { + num_flows = MAX_FLOWS; } fprintf(stderr, "%s - option - number of flows: %d\n", __func__, num_flows); break; case 'c': num_ctxs = strtoul (optarg, NULL, 0); - if (num_ctxs > config_max_ctxs) { - num_ctxs = config_max_ctxs; + if (num_ctxs > MAX_CTXS) { + num_ctxs = MAX_CTXS; } fprintf(stderr, "%s - option - number of contexts: %d\n", __func__, num_ctxs); break; @@ -241,10 +252,16 @@ main(int argc, char *argv[]) neat_set_property(ctx[c], flows[i].flow, config_property); + ctx_flows[c]++; + + ops[i].on_connected = on_connected; - ops[i].on_error = on_error; - flows[i].id = streams_going; + ops[i].on_error = on_error; + + flows[i].id = streams_going; + flows[i].ctx_id = c; ops[i].userData = &flows[i]; + streams_going++; neat_set_operations(ctx[c], flows[i].flow, &(ops[i])); @@ -253,7 +270,7 @@ main(int argc, char *argv[]) fprintf(stderr, "Could not open flow\n"); result = EXIT_FAILURE; } else { - fprintf(stderr, "Opened flow %d\n", i); + fprintf(stderr, "Opened flow %d for ctx %d\n", i, c); } } diff --git a/examples/tneat.c b/examples/tneat.c index 25b9690e..a4440f25 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -136,19 +136,18 @@ on_all_written(struct neat_flow_operations *opCB) // print statistics printf("neat_write finished - statistics\n"); - printf("\tbytes\t\t: %u\n", tnf->snd.bytes); - printf("\tsnd-calls\t: %u\n", tnf->snd.calls); + printf("\tbytes\t\t: %u\n", tnf->snd.bytes); + printf("\tsnd-calls\t: %u\n", tnf->snd.calls); printf("\tduration\t: %.2fs\n", time_elapsed); - printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->snd.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); + printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->snd.bytes / time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); - opCB->on_writable = NULL; - opCB->on_all_written = NULL; + opCB->on_writable = NULL; + opCB->on_all_written = NULL; neat_set_operations(opCB->ctx, opCB->flow, opCB); - neat_shutdown(opCB->ctx, opCB->flow); } else { - opCB->on_writable = on_writable; - opCB->on_all_written = NULL; + opCB->on_writable = on_writable; + opCB->on_all_written = NULL; neat_set_operations(opCB->ctx, opCB->flow, opCB); } @@ -199,8 +198,8 @@ on_writable(struct neat_flow_operations *opCB) } // set callbacks - opCB->on_writable = NULL; - opCB->on_all_written = on_all_written; + opCB->on_writable = NULL; + opCB->on_all_written = on_all_written; neat_set_operations(opCB->ctx, opCB->flow, opCB); return NEAT_OK; @@ -253,9 +252,9 @@ on_readable(struct neat_flow_operations *opCB) printf("connection closed\n"); } - opCB->on_readable = NULL; - opCB->on_writable = NULL; - opCB->on_all_written = NULL; + opCB->on_readable = NULL; + opCB->on_writable = NULL; + opCB->on_all_written = NULL; neat_set_operations(opCB->ctx, opCB->flow, opCB); if (!config_active) { @@ -317,9 +316,9 @@ on_connected(struct neat_flow_operations *opCB) tnf->rcv.bytes = 0; // set callbacks - opCB->on_readable = on_readable; + opCB->on_readable = on_readable; if (config_active) { - opCB->on_writable = on_writable; + opCB->on_writable = on_writable; } neat_set_operations(opCB->ctx, opCB->flow, opCB); @@ -332,10 +331,11 @@ on_close(struct neat_flow_operations *opCB) struct tneat_flow *tnf = opCB->userData; // cleanup - opCB->on_close = NULL; - opCB->on_readable = NULL; - opCB->on_writable = NULL; - opCB->on_error = NULL; + opCB->on_close = NULL; + opCB->on_readable = NULL; + opCB->on_writable = NULL; + opCB->on_error = NULL; + neat_set_operations(opCB->ctx, opCB->flow, opCB); if (tnf->snd.buffer) { free(tnf->snd.buffer); @@ -349,8 +349,6 @@ on_close(struct neat_flow_operations *opCB) free(tnf); } - neat_set_operations(opCB->ctx, opCB->flow, opCB); - fprintf(stderr, "%s - flow closed OK!\n", __func__); // stop event loop if we are active part diff --git a/neat_core.c b/neat_core.c index 02ee0eaf..a708ad4a 100644 --- a/neat_core.c +++ b/neat_core.c @@ -964,7 +964,6 @@ io_writable(neat_ctx *ctx, neat_flow *flow, neat_error_code code) io_all_written(ctx, flow, 0); } } - } // Translate SCTP cause codes (RFC4960 sect.3.3.10) @@ -5980,6 +5979,7 @@ handle_upcall(struct socket *sock, void *arg, int flags) return; } + // remove "on_writable" callback check if (events & SCTP_EVENT_WRITE && flow->operations->on_writable) { if (flow->firstWritePending) flow->firstWritePending = 0; From 0781e1bcf1666cb038c9a64bf51d65b32fde186e Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 26 Apr 2017 14:36:56 +0200 Subject: [PATCH 05/85] call io_all_written only if neat_write has been called prior and do this only once. --- neat_core.c | 8 +++++++- neat_internal.h | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/neat_core.c b/neat_core.c index a708ad4a..bbdae27b 100644 --- a/neat_core.c +++ b/neat_core.c @@ -961,6 +961,7 @@ io_writable(neat_ctx *ctx, neat_flow *flow, neat_error_code code) // neat_shutdown has been called while flow was draining, run shutdown procedure neat_shutdown(ctx, flow); } else { + // outgoing flow buffer is empty io_all_written(ctx, flow, 0); } } @@ -1616,9 +1617,12 @@ io_all_written(neat_ctx *ctx, neat_flow *flow, uint16_t stream_id) neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); stream_id = NEAT_INVALID_STREAM; - if (!flow->operations || !flow->operations->on_all_written) { + if (!flow->operations || !flow->operations->on_all_written || !flow->notifyDrainPending) { return; } + + flow->notifyDrainPending = 0; + neat_error_code code = NEAT_OK; READYCALLBACKSTRUCT; flow->operations->on_all_written(flow->operations); @@ -6085,6 +6089,8 @@ neat_write(struct neat_ctx *ctx, assert(flow->multistream_reset_out == false); #endif + flow->notifyDrainPending = 1; + for (struct neat_iofilter *filter = flow->iofilters; filter; filter = filter->next) { // find the first filter and call it if (!filter->writefx) { diff --git a/neat_internal.h b/neat_internal.h index 693a6001..d2004f08 100644 --- a/neat_internal.h +++ b/neat_internal.h @@ -275,6 +275,7 @@ struct neat_flow unsigned int isSCTPMultihoming : 1; unsigned int isSCTPIdata : 1; unsigned int isClosing : 1; + unsigned int notifyDrainPending : 1; unsigned int streams_requested; From b51b97b20358b0ca4cafeb2157d4fdfb277fd178 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 26 Apr 2017 22:38:20 +0200 Subject: [PATCH 06/85] docs --- docs/neat_close.md | 9 +++++++++ docs/neat_read.md | 10 +++++----- docs/neat_shutdown.md | 10 ++++++++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/docs/neat_close.md b/docs/neat_close.md index 98a6f0ce..eb879654 100644 --- a/docs/neat_close.md +++ b/docs/neat_close.md @@ -2,6 +2,15 @@ Close this flow and free all associated data. +This is the *last call*: +* all flow related data (e.g. `userData`) should be freed in advance +* no flow related *callbacks* are fired +* all data in the *receive buffer* gets discarded +* all data in the *send buffer* will be transmitted +* initiates a *graceful connection shutdown* + +**This function must always the be called to free ressources** + ### Syntax ```c diff --git a/docs/neat_read.md b/docs/neat_read.md index 7108cca4..2d489d96 100644 --- a/docs/neat_read.md +++ b/docs/neat_read.md @@ -1,6 +1,8 @@ # neat_read -Read data from a neat flow. Should only be called from within the `on_readable` +Read data from a neat flow. + +Should only be called from within the `on_readable` callback specified with `neat_set_operations`. ```c @@ -41,11 +43,9 @@ parameter. ### Remarks -This function should only be called from within the `on_readable` callback -specified with `neat_set_operations`, as this is the only way to guarantee -that the call will not block. NEAT does not permit a blocking read operation. +This function should only be called from within the `on_readable` callback specified with `neat_set_operations`, as this is the only way to guarantee that the call will not block. NEAT does not permit a blocking read operation. -The **actual_amount** value is set to 0 when this function returns error. +The **actual_amount** value is set to 0 when the remote side has closed the connection. ### Examples diff --git a/docs/neat_shutdown.md b/docs/neat_shutdown.md index e0a1b83a..6743dc0c 100644 --- a/docs/neat_shutdown.md +++ b/docs/neat_shutdown.md @@ -1,7 +1,13 @@ # neat_shutdown -Initiate a graceful shutdown of this flow. All previously written data will be -sent. Data can still be read from the flow. +Initiate a graceful shutdown of this flow. + +* the receive buffer can still be read and `on_readable` gets fired like in normal operation +* receiving **new** data from the peer **may** fail +* all data in the *send buffer* will be transmitted +* `neat_write` will fail and `on_writable` will not be called + +If the peer also has closed the connection, the `on_close` callback gets fired. ### Syntax From 1ca65c8a03b3021b80592b6d7da82ede527ce088 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 26 Apr 2017 23:31:46 +0200 Subject: [PATCH 07/85] client_http_get : free stat --- examples/client_http_get.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/client_http_get.c b/examples/client_http_get.c index 7ab46be0..2e547953 100644 --- a/examples/client_http_get.c +++ b/examples/client_http_get.c @@ -240,6 +240,8 @@ on_close(struct neat_flow_operations *opCB) opCB->on_error = NULL; neat_set_operations(opCB->ctx, opCB->flow, opCB); + free(opCB->userData); + // stop event loop if all flows are closed flows_active--; if (config_log_level >= 1) { From 511902f0e511b8eae4a3752d3a2232e2278d88bf Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Fri, 5 May 2017 13:43:56 +0200 Subject: [PATCH 08/85] Refactoring - sync wip --- examples/CMakeLists.txt | 2 +- examples/client_http_get.c | 32 ++++--- examples/client_https_get.c | 156 -------------------------------- examples/prop_tcp_security.json | 13 +++ neat_security.c | 1 + 5 files changed, 35 insertions(+), 169 deletions(-) delete mode 100644 examples/client_https_get.c create mode 100644 examples/prop_tcp_security.json diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index dd81e5a1..b19b44a3 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -5,7 +5,6 @@ LIST(APPEND neat_programs client_data.c client_http_get.c client_http_run_once.c - client_https_get.c server_chargen.c server_daytime.c server_discard.c @@ -23,6 +22,7 @@ LIST(APPEND neat_property_examples prop_sctp.json prop_sctp_multihomed.json prop_tcp.json + prop_tcp_security.json ) # BUILD EACH PROGRAM diff --git a/examples/client_http_get.c b/examples/client_http_get.c index 2e547953..72201c2d 100644 --- a/examples/client_http_get.c +++ b/examples/client_http_get.c @@ -34,14 +34,14 @@ } while (0) #endif -static int result = 0; -static uint32_t config_rcv_buffer_size = 32*1024*1024; // 32MB rcv buffer -static uint32_t config_max_flows = 2000; -static uint8_t config_log_level = 0; +static int result = 0; +static uint32_t config_rcv_buffer_size = 32*1024*1024; // 32MB rcv buffer +static uint32_t config_max_flows = 2000; +static uint8_t config_log_level = 0; +static uint16_t config_port = 80; static char request[512]; -static uint32_t flows_active = 0; -static const char *request_tail = "HTTP/1.0\r\nUser-agent: libneat\r\nConnection: close\r\n\r\n"; -static char *config_property = "\ +static uint32_t flows_active = 0; +static char *config_property = "\ {\ \"transport\": [\ {\ @@ -160,7 +160,7 @@ on_readable(struct neat_flow_operations *opCB) gettimeofday(&(stat->tv_last), NULL); if (config_log_level >= 1) { fprintf(stderr, "%s - received %d bytes\n", __func__, bytes_read); - // fwrite(buffer, sizeof(char), bytes_read, stdout); + fwrite(buffer, sizeof(char), bytes_read, stdout); } } return NEAT_OK; @@ -275,9 +275,10 @@ main(int argc, char *argv[]) memset(&ops, 0, sizeof(ops)); memset(flows, 0, sizeof(flows)); - snprintf(request, sizeof(request), "GET %s %s", "/", request_tail); + // request index directory by default + snprintf(request, sizeof(request), "GET %s HTTP/1.1\r\nHost: %s\r\nUser-agent: libneat\r\nConnection: close\r\n\r\n", "/", argv[argc - 1]); - while ((arg = getopt(argc, argv, "P:R:u:n:v:")) != -1) { + while ((arg = getopt(argc, argv, "P:p:R:u:n:v:")) != -1) { switch(arg) { case 'P': if (read_file(optarg, &arg_property) < 0) { @@ -289,11 +290,18 @@ main(int argc, char *argv[]) fprintf(stderr, "%s - option - properties: %s\n", __func__, arg_property); } break; + case 'p': + if (atoi(optarg) > 0 && atoi(optarg) < 65556) { + config_port = atoi(optarg); + } else { + fprintf(stderr, "%s - option - port - invalid port, using default : %u\n", __func__, config_port); + } + break; case 'R': config_rcv_buffer_size = atoi(optarg); break; case 'u': - snprintf(request, sizeof(request), "GET %s %s", optarg, request_tail); + snprintf(request, sizeof(request), "GET %s HTTP/1.1\r\nHost: %s\r\nUser-agent: libneat\r\nConnection: close\r\n\r\n", optarg, argv[argc - 1]); break; case 'n': num_flows = strtoul(optarg, NULL, 0); @@ -364,7 +372,7 @@ main(int argc, char *argv[]) neat_set_operations(ctx, flows[i], &(ops[i])); // wait for on_connected or on_error to be invoked - if (neat_open(ctx, flows[i], argv[argc - 1], 80, NULL, 0) != NEAT_OK) { + if (neat_open(ctx, flows[i], argv[argc - 1], config_port, NULL, 0) != NEAT_OK) { fprintf(stderr, "Could not open flow\n"); result = EXIT_FAILURE; } else { diff --git a/examples/client_https_get.c b/examples/client_https_get.c deleted file mode 100644 index a3caa0ff..00000000 --- a/examples/client_https_get.c +++ /dev/null @@ -1,156 +0,0 @@ -#include - -#include -#include -#include - -/********************************************************************** - - HTTPS-GET client in neat - - client_https_get HOST [URI] - - * connect to HOST and send GET request - * write response to stdout - -**********************************************************************/ - -static uint32_t config_rcv_buffer_size = 1024; -static char request[512]; -static const char *request_tail = "User-agent: libneat\r\nConnection: close\r\n\r\n"; -static char *config_property = "{\ - \"transport\": [\ - {\ - \"value\": \"SCTP\",\ - \"precedence\": 1\ - },\ - {\ - \"value\": \"SCTP/UDP\",\ - \"precedence\": 1\ - },\ - {\ - \"value\": \"TCP\",\ - \"precedence\": 1\ - }\ - ],\ - \"security\": {\ - \"value\": true,\ - \"precedence\": 2\ - }\ -}";\ - -static neat_error_code on_error(struct neat_flow_operations *opCB) -{ - fprintf(stderr, "%s\n", __func__); - exit(EXIT_FAILURE); -} - -static neat_error_code on_readable(struct neat_flow_operations *opCB) -{ - // data is available to read - unsigned char buffer[config_rcv_buffer_size]; - uint32_t bytes_read = 0; - neat_error_code code; - - code = neat_read(opCB->ctx, opCB->flow, buffer, config_rcv_buffer_size, &bytes_read, NULL, 0); - if (code == NEAT_ERROR_WOULD_BLOCK) { - return 0; - } else if (code != NEAT_OK) { - return on_error(opCB); - } - - if (!bytes_read) { // eof - fflush(stdout); - opCB->on_readable = NULL; // do not read more - neat_set_operations(opCB->ctx, opCB->flow, opCB); - neat_stop_event_loop(opCB->ctx); - } else if (bytes_read > 0) { - fwrite(buffer, sizeof(char), bytes_read, stdout); - } - return 0; -} - -static neat_error_code on_writable(struct neat_flow_operations *opCB) -{ - neat_error_code code; - code = neat_write(opCB->ctx, opCB->flow, (const unsigned char *)request, strlen(request), NULL, 0); - if (code != NEAT_OK) { - return on_error(opCB); - } - opCB->on_writable = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); - return 0; -} - -static neat_error_code on_connected(struct neat_flow_operations *opCB) -{ - // now we can start writing - opCB->on_readable = on_readable; - opCB->on_writable = on_writable; - neat_set_operations(opCB->ctx, opCB->flow, opCB); - return 0; -} - -int main(int argc, char *argv[]) -{ - struct neat_ctx *ctx = NULL; - struct neat_flow *flow = NULL; - struct neat_flow_operations ops; - int result; - - memset(&ops, 0, sizeof(ops)); - - result = EXIT_SUCCESS; - - if (argc < 2 || argc > 3) { - fprintf(stderr, "usage: neat_https_get HOST [URI]\n"); - result = EXIT_FAILURE; - goto cleanup; - } - - if (argc == 3) { - snprintf(request, sizeof(request), "GET %s HTTP/1.1\r\nHost: %s\r\n%s", - argv[2], argv[1], request_tail); - } else { - snprintf(request, sizeof(request), "GET %s HTTP/1.1\r\nHost: %s\r\n%s", - "/", argv[1], request_tail); - - } - printf("requesting: %s\n", request); - - if ((ctx = neat_init_ctx()) == NULL) { - fprintf(stderr, "could not initialize context\n"); - result = EXIT_FAILURE; - goto cleanup; - } - - if ((flow = neat_new_flow(ctx)) == NULL) { - fprintf(stderr, "could not initialize context\n"); - result = EXIT_FAILURE; - goto cleanup; - } - - if (neat_set_property(ctx, flow, config_property)) { - fprintf(stderr, "%s - error: neat_set_property\n", __func__); - result = EXIT_FAILURE; - goto cleanup; - } - - ops.on_connected = on_connected; - ops.on_error = on_error; - neat_set_operations(ctx, flow, &ops); - - // wait for on_connected or on_error to be invoked - if (neat_open(ctx, flow, argv[1], 443, NULL, 0) == NEAT_OK) - neat_start_event_loop(ctx, NEAT_RUN_DEFAULT); - else { - fprintf(stderr, "Could not open flow\n"); - result = EXIT_FAILURE; - } - -cleanup: - if (ctx != NULL) { - neat_free_ctx(ctx); - } - exit(result); -} diff --git a/examples/prop_tcp_security.json b/examples/prop_tcp_security.json new file mode 100644 index 00000000..54cfc382 --- /dev/null +++ b/examples/prop_tcp_security.json @@ -0,0 +1,13 @@ +{ + "transport": [ + { + "value": "TCP", + "precedence": 1 + } + ], + "security" : + { + "value": true, + "precedence": 2 + } +} diff --git a/neat_security.c b/neat_security.c index 462ef1e0..ddb54cdd 100644 --- a/neat_security.c +++ b/neat_security.c @@ -467,6 +467,7 @@ neat_security_close(neat_ctx *ctx) neat_error_code neat_security_install(neat_ctx *ctx, neat_flow *flow) { + neat_log(ctx, NEAT_LOG_ERROR, "Library compiled without security support"); return NEAT_ERROR_SECURITY; } From c47fb50c616f708129ebb6cfdd19008d97211671 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Sun, 7 May 2017 15:13:09 +0200 Subject: [PATCH 09/85] CMAKE from DTLS Branch --- CMakeLists.txt | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eda23694..7e228d65 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,7 @@ add_custom_target(dist COMMAND ${CMAKE_MAKE_PROGRAM} clean package_source) INCLUDE(CheckIncludeFile) INCLUDE(CheckStructHasMember) +INCLUDE(CheckTypeSize) SET(CMAKE_MACOSX_RPATH 1) # SOURCES + HEADERS @@ -80,13 +81,13 @@ LIST(APPEND neat_sources # OS DEPENDENT ################################################# IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") - MESSAGE(STATUS ${CMAKE_SYSTEM_NAME} " supported") + MESSAGE(${CMAKE_SYSTEM_NAME} " supported") ADD_DEFINITIONS("-D_DEFAULT_SOURCE") LIST(APPEND neat_headers neat_linux.h) LIST(APPEND neat_sources neat_linux.c) ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") - MESSAGE(STATUS ${CMAKE_SYSTEM_NAME} " supported") + MESSAGE(${CMAKE_SYSTEM_NAME} " supported") LIST(APPEND neat_headers neat_bsd.h) LIST(APPEND neat_sources neat_bsd.c) SET(CMAKE_REQUIRED_INCLUDES "/usr/local/include" "/usr/include") @@ -94,7 +95,7 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") INCLUDE_DIRECTORIES("/usr/local/include") ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - MESSAGE(STATUS ${CMAKE_SYSTEM_NAME} " supported") + MESSAGE(${CMAKE_SYSTEM_NAME} " supported") LIST(APPEND neat_headers neat_bsd.h) LIST(APPEND neat_sources neat_bsd.c) SET(CMAKE_REQUIRED_INCLUDES "/usr/local/include" "/usr/include" "/usr/local/opt/openssl/include") @@ -102,7 +103,7 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") INCLUDE_DIRECTORIES("/usr/local/include" "/usr/local/opt/openssl/include") ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "NetBSD") - MESSAGE(STATUS ${CMAKE_SYSTEM_NAME} " supported") + MESSAGE(${CMAKE_SYSTEM_NAME} " supported") LIST(APPEND neat_headers neat_bsd.h) LIST(APPEND neat_sources neat_bsd.c) SET(CMAKE_REQUIRED_INCLUDES "/usr/pkg/include" "/usr/include" "/usr/local/include") @@ -148,7 +149,7 @@ ENDIF() ################################################# # Debug build type as default IF (NOT CMAKE_BUILD_TYPE) - message(STATUS "No build type selected, using DEBUG") + message("No build type selected, using DEBUG") set(CMAKE_BUILD_TYPE "DEBUG") # _FORTIFY_SOURCE requires compiling with optimization SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O") @@ -165,7 +166,7 @@ IF (USRSCTP_SUPPORT) FIND_LIBRARY(USRSCTP_LIB usrsctp) LIST(APPEND neat_libs ${USRSCTP_LIB}) - MESSAGE(STATUS "USRSCTP found: " ${USRSCTP_LIB}) + MESSAGE("USRSCTP found: " ${USRSCTP_LIB}) ENDIF() OPTION(NEAT_LOG "enable NEAT log module" 1) @@ -187,29 +188,42 @@ ENDIF() ################################################# CHECK_INCLUDE_FILE(uv.h HAVE_SYS_UV_H) FIND_LIBRARY(UV_LIB uv) -MESSAGE(STATUS "UV found: " ${UV_LIB}) + IF (NOT HAVE_SYS_UV_H OR NOT UV_LIB) MESSAGE(FATAL_ERROR "uv.h not found - libuv installed?") +ELSE() + MESSAGE("UV found: " ${UV_LIB}) ENDIF() CHECK_INCLUDE_FILE(ldns/ldns.h HAVE_SYS_LDNS_H) FIND_LIBRARY(LDNS_LIB ldns) -MESSAGE(STATUS "LDNS found: " ${LDNS_LIB}) IF (NOT HAVE_SYS_LDNS_H OR NOT LDNS_LIB) MESSAGE(FATAL_ERROR "ldns/ldns.h not found - libdns installed?") +ELSE() + MESSAGE("LDNS found: " ${LDNS_LIB}) ENDIF() FIND_PACKAGE(OpenSSL) IF (NOT OPENSSL_FOUND) - MESSAGE(STATUS "openssl >= 1.0.2 required for TLS - none found") + MESSAGE(WARNING "openssl >= 1.0.2 required for TLS - none found") ELSE() - MESSAGE(STATUS "OPENSSL version found: " ${OPENSSL_VERSION}) + MESSAGE("OPENSSL version found: " ${OPENSSL_VERSION}) IF (OPENSSL_VERSION VERSION_LESS "1.0.2") - MESSAGE(STATUS "openssl >= 1.0.2 required for TLS") + MESSAGE(WARNING "openssl >= 1.0.2 required for TLS") ELSE() CHECK_INCLUDE_FILE(openssl/ssl.h HAVE_SYS_OPENSSL_H) MESSAGE(STATUS "OPENSSL Crypto found: " ${OPENSSL_LIBRARIES}) ADD_DEFINITIONS(-DNEAT_USETLS) + SET(CMAKE_EXTRA_INCLUDE_FILES "openssl/ssl.h") + + IF ((OPENSSL_VERSION VERSION_EQUAL "1.1.0") OR (OPENSSL_VERSION VERSION_GREATER "1.1.0")) + CHECK_TYPE_SIZE("struct bio_dgram_sctp_sndinfo" OPENSSL_DTLS) + IF (HAVE_OPENSSL_DTLS) + MESSAGE("DTLS for SCTP supported") + ADD_DEFINITIONS(-DNEAT_SCTP_DTLS) + ENDIF() + ENDIF() + ENDIF() ENDIF() @@ -218,19 +232,18 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") CHECK_INCLUDE_FILE(libmnl/libmnl.h HAVE_SYS_LIBMNL_H) IF(NOT HAVE_SYS_LIBMNL_H OR NOT MNL_LIB) MESSAGE(FATAL_ERROR "libmnl/libmnl.h not found - libmnl installed?") + ELSE() + MESSAGE("LIBMNL found: " ${MNL_LIB}) ENDIF() FIND_LIBRARY(SCTP_LIB sctp) - IF(EXISTS "/proc/sys/net/mptcp/mptcp_enabled") - MESSAGE(STATUS "MPTCP found") - ADD_DEFINITIONS(-DMPTCP_SUPPORT) - ENDIF() ENDIF() CHECK_INCLUDE_FILE(jansson.h HAVE_JANSSON_H) FIND_LIBRARY(JANSSON_LIB jansson) -MESSAGE(STATUS "jansson found: " ${JANSSON_LIB}) IF (NOT HAVE_JANSSON_H OR NOT JANSSON_LIB) MESSAGE(FATAL_ERROR "jansson.h not found - jansson installed?") +ELSE() + MESSAGE(STATUS "jansson found: " ${JANSSON_LIB}) ENDIF() CHECK_INCLUDE_FILE(netinet/sctp.h HAVE_NETINET_SCTP_H) @@ -306,7 +319,7 @@ TARGET_LINK_LIBRARIES (neat ${neat_libs}) # INSTALL ################################################# -MESSAGE(STATUS "Install directory: ${CMAKE_INSTALL_PREFIX}") +MESSAGE("Install directory: ${CMAKE_INSTALL_PREFIX}") INSTALL(TARGETS neat neat-static DESTINATION ${CMAKE_INSTALL_LIBDIR}) INSTALL(FILES ${neat_headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) From 022496c90f9b4089216f224869729771acfdcc27 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Sun, 7 May 2017 16:16:58 +0200 Subject: [PATCH 10/85] call on_close when peer has disconnected --- neat_core.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/neat_core.c b/neat_core.c index e493a670..7c5b13ac 100644 --- a/neat_core.c +++ b/neat_core.c @@ -1228,6 +1228,8 @@ io_readable(neat_ctx *ctx, neat_flow *flow, struct sockaddr_storage peerAddr; socklen_t peerAddrLen = sizeof(struct sockaddr_storage); int stream_id = -1; + int retval; + char buffer[1]; ssize_t n; struct msghdr msghdr; //Not used when notifications aren't available: @@ -1279,7 +1281,6 @@ io_readable(neat_ctx *ctx, neat_flow *flow, * anything else will. */ if (!flow->operations->on_readable && flow->acceptPending) { - if (socket->stack != NEAT_STACK_UDP && socket->stack != NEAT_STACK_UDPLITE) { neat_log(ctx, NEAT_LOG_WARNING, "%s - READ_WITH_ERROR 1", __func__); return READ_WITH_ERROR; @@ -1606,6 +1607,18 @@ io_readable(neat_ctx *ctx, neat_flow *flow, } } + if (socket->stack == NEAT_STACK_TCP) { + retval = recv(flow->socket->fd, buffer, 1, MSG_PEEK); + if (retval <= 0) { + neat_log(ctx, NEAT_LOG_INFO, "%s - TCP connection peek: %d - connection closed", __func__, retval); + if (flow->operations->on_close) { + READYCALLBACKSTRUCT; + flow->operations->on_close(flow->operations); + } + return READ_WITH_ZERO; + } + } + if (flow->operations->on_readable) { READYCALLBACKSTRUCT; flow->operations->on_readable(flow->operations); @@ -6412,7 +6425,8 @@ neat_notify_close(neat_flow *flow) // Notify application about network changes. // Code should identify what happened. -void neat_notify_network_status_changed(neat_flow *flow, neat_error_code code) +void +neat_notify_network_status_changed(neat_flow *flow, neat_error_code code) { const int stream_id = NEAT_INVALID_STREAM; //READYCALLBACKSTRUCT expects this: @@ -6457,7 +6471,8 @@ neat_close(struct neat_ctx *ctx, struct neat_flow *flow) } // ABORT, D1.2 sect. 3.2.4 -neat_error_code neat_abort(struct neat_ctx *ctx, struct neat_flow *flow) +neat_error_code +neat_abort(struct neat_ctx *ctx, struct neat_flow *flow) { struct linger ling; From fdf625511608876a783b1e0e1574aaed8c32988f Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Sun, 7 May 2017 16:17:55 +0200 Subject: [PATCH 11/85] http_client - print statistics when on_close has been called --- examples/client_http_get.c | 107 +++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 59 deletions(-) diff --git a/examples/client_http_get.c b/examples/client_http_get.c index 72201c2d..3f3e80aa 100644 --- a/examples/client_http_get.c +++ b/examples/client_http_get.c @@ -67,6 +67,7 @@ struct stat_flow { struct timeval tv_first; struct timeval tv_last; struct timeval tv_delta; + uint16_t protocol; uv_timer_t timer; struct neat_flow *flow; }; @@ -91,17 +92,10 @@ on_readable(struct neat_flow_operations *opCB) uint32_t bytes_read = 0; struct stat_flow *stat = opCB->userData; neat_error_code code; - struct timeval tv_duration; - double time_elapsed = 0.0; - char buffer_filesize_human[32]; - char buffer_bandwidth_human[32]; struct neat_tlv options[1]; - - //last_stream = (last_stream + 1) % opCB->flow->stream_count; options[0].tag = NEAT_TAG_TRANSPORT_STACK; options[0].type = NEAT_TYPE_INTEGER; - //fprintf(stderr, "%s - reading from flow\n", __func__); code = neat_read(opCB->ctx, opCB->flow, buffer, config_rcv_buffer_size, &bytes_read, options, 1); if (code == NEAT_ERROR_WOULD_BLOCK) { if (config_log_level >= 1) { @@ -112,49 +106,9 @@ on_readable(struct neat_flow_operations *opCB) return on_error(opCB); } - if (!bytes_read) { // eof - uv_timer_stop(&(stat->timer)); - - if (config_log_level >= 1) { - fprintf(stderr, "%s - neat_read() returned 0 bytes - connection closed\n", __func__); - } - - timersub(&(stat->tv_last), &(stat->tv_first), &tv_duration); - time_elapsed = tv_duration.tv_sec + (double)tv_duration.tv_usec / 1000000.0; - filesize_human(8 * (stat->rcv_bytes) / time_elapsed, buffer_bandwidth_human, sizeof(buffer_bandwidth_human)); - filesize_human(stat->rcv_bytes, buffer_filesize_human, sizeof(buffer_filesize_human)); - - printf("########################################################\n"); - printf("# %p - transfer finished\n", (void *)opCB->flow); - printf("########################################################\n"); - printf("# size:\t\t%s\n", buffer_filesize_human); - printf("# duration:\t%.2f s\n", time_elapsed); - printf("# bandwidth:\t%sit/s\n", buffer_bandwidth_human); - printf("# protocol:\t"); - - switch ((int)options[0].value.integer) { - case NEAT_STACK_TCP: - printf("TCP"); - break; - case NEAT_STACK_SCTP: - printf("SCTP"); - break; - case NEAT_STACK_SCTP_UDP: - printf("SCTP/UDP"); - break; - default: - printf("OTHER"); - break; - } - printf("\n"); - - printf("########################################################\n"); - - fflush(stdout); - on_close(opCB); - - } else if (bytes_read > 0) { + if (bytes_read > 0) { stat = opCB->userData; + stat->protocol = (int)options[0].value.integer; stat->rcv_bytes += bytes_read; stat->rcv_calls++; gettimeofday(&(stat->tv_last), NULL); @@ -233,12 +187,50 @@ on_connected(struct neat_flow_operations *opCB) static neat_error_code on_close(struct neat_flow_operations *opCB) { - // cleanup - opCB->on_close = NULL; - opCB->on_readable = NULL; - opCB->on_writable = NULL; - opCB->on_error = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); + struct stat_flow *stat = opCB->userData; + struct timeval tv_duration; + double time_elapsed = 0.0; + char buffer_filesize_human[32]; + char buffer_bandwidth_human[32]; + + uv_timer_stop(&(stat->timer)); + + if (config_log_level >= 1) { + fprintf(stderr, "%s - neat_read() returned 0 bytes - connection closed\n", __func__); + } + + timersub(&(stat->tv_last), &(stat->tv_first), &tv_duration); + time_elapsed = tv_duration.tv_sec + (double)tv_duration.tv_usec / 1000000.0; + filesize_human(8 * (stat->rcv_bytes) / time_elapsed, buffer_bandwidth_human, sizeof(buffer_bandwidth_human)); + filesize_human(stat->rcv_bytes, buffer_filesize_human, sizeof(buffer_filesize_human)); + + printf("########################################################\n"); + printf("# %p - transfer finished\n", (void *)opCB->flow); + printf("########################################################\n"); + printf("# size:\t\t%s\n", buffer_filesize_human); + printf("# duration:\t%.2f s\n", time_elapsed); + printf("# bandwidth:\t%sit/s\n", buffer_bandwidth_human); + printf("# protocol:\t"); + + switch (stat->protocol) { + case NEAT_STACK_TCP: + printf("TCP"); + break; + case NEAT_STACK_SCTP: + printf("SCTP"); + break; + case NEAT_STACK_SCTP_UDP: + printf("SCTP/UDP"); + break; + default: + printf("OTHER"); + break; + } + printf("\n"); + + printf("########################################################\n"); + + fflush(stdout); free(opCB->userData); @@ -252,7 +244,6 @@ on_close(struct neat_flow_operations *opCB) if (config_log_level >= 1) { fprintf(stderr, "%s - stopping event loop\n", __func__); } - neat_stop_event_loop(opCB->ctx); } @@ -384,9 +375,6 @@ main(int argc, char *argv[]) neat_start_event_loop(ctx, NEAT_RUN_DEFAULT); cleanup: - for (i = 0; i < num_flows; i++) { - //free((flows[i])->userData); - } if (ctx != NULL) { neat_free_ctx(ctx); @@ -395,6 +383,7 @@ main(int argc, char *argv[]) if (arg_property) { free(arg_property); } + if (buffer) { free(buffer); } From bdf109d6f036b067d0a4c442d87be52e8d1c28bb Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 8 May 2017 00:07:33 +0200 Subject: [PATCH 12/85] closing procedure improvements --- neat_core.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/neat_core.c b/neat_core.c index 7c5b13ac..b78657a1 100644 --- a/neat_core.c +++ b/neat_core.c @@ -1611,10 +1611,7 @@ io_readable(neat_ctx *ctx, neat_flow *flow, retval = recv(flow->socket->fd, buffer, 1, MSG_PEEK); if (retval <= 0) { neat_log(ctx, NEAT_LOG_INFO, "%s - TCP connection peek: %d - connection closed", __func__, retval); - if (flow->operations->on_close) { - READYCALLBACKSTRUCT; - flow->operations->on_close(flow->operations); - } + neat_notify_close(flow); return READ_WITH_ZERO; } } @@ -6414,13 +6411,21 @@ neat_notify_close(neat_flow *flow) neat_error_code code = NEAT_ERROR_OK; neat_ctx *ctx = flow->ctx; - neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); - if (!flow->operations || !flow->operations->on_close) { + if (flow->state == NEAT_FLOW_CLOSED) { + neat_log(ctx, NEAT_LOG_WARNING, "%s - flow already closed - skipping", __func__); return; } - READYCALLBACKSTRUCT; - flow->operations->on_close(flow->operations); + flow->state = NEAT_FLOW_CLOSED; + + neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); + if (flow->operations && flow->operations->on_close) { + READYCALLBACKSTRUCT; + flow->operations->on_close(flow->operations); + } + + // this was the last callback - free all ressources + neat_free_flow(flow); } // Notify application about network changes. @@ -6465,8 +6470,6 @@ neat_close(struct neat_ctx *ctx, struct neat_flow *flow) } #endif - neat_free_flow(flow); - return NEAT_OK; } From 30bd0a93884ceb134f2e8ac70907e4f1e803cce0 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 8 May 2017 05:51:09 +0200 Subject: [PATCH 13/85] neat closing refactoring --- examples/client_http_get.c | 4 ++-- neat_core.c | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/examples/client_http_get.c b/examples/client_http_get.c index 3f3e80aa..7cb2fe39 100644 --- a/examples/client_http_get.c +++ b/examples/client_http_get.c @@ -232,7 +232,7 @@ on_close(struct neat_flow_operations *opCB) fflush(stdout); - free(opCB->userData); + free(stat); // stop event loop if all flows are closed flows_active--; @@ -244,7 +244,7 @@ on_close(struct neat_flow_operations *opCB) if (config_log_level >= 1) { fprintf(stderr, "%s - stopping event loop\n", __func__); } - neat_stop_event_loop(opCB->ctx); + //neat_stop_event_loop(opCB->ctx); } return NEAT_OK; diff --git a/neat_core.c b/neat_core.c index b78657a1..248b1a2f 100644 --- a/neat_core.c +++ b/neat_core.c @@ -287,14 +287,16 @@ static void neat_core_cleanup(struct neat_ctx *nc) neat_close_loop(nc); neat_addr_free_src_list(nc); - if (nc->cleanup) + if (nc->cleanup) { nc->cleanup(nc); + } } //Free any resource used by the context. Loop must be stopped before this is //called //TODO: Consider adding callback, like for resolver -void neat_free_ctx(struct neat_ctx *nc) +void +neat_free_ctx(struct neat_ctx *nc) { struct neat_flow *flow, *prev_flow = NULL; neat_log(nc, NEAT_LOG_DEBUG, "%s", __func__); @@ -633,7 +635,8 @@ free_cb(uv_handle_t *handle) } -static int neat_close_socket(struct neat_ctx *ctx, struct neat_flow *flow) +static int +neat_close_socket(struct neat_ctx *ctx, struct neat_flow *flow) { struct neat_pollable_socket *s; struct neat_pollable_socket *stemp; @@ -675,8 +678,7 @@ neat_free_flow(neat_flow *flow) neat_free_candidates(ctx, flow->candidate_list); - if (flow->socket->handle != NULL - && flow->socket->handle->type != UV_UNKNOWN_HANDLE + if (flow->socket->handle != NULL && flow->socket->handle->type != UV_UNKNOWN_HANDLE #ifdef SCTP_MULTISTREAMING && (!flow->socket->multistream || flow->socket->sctp_streams_used == 0) #endif @@ -5382,7 +5384,7 @@ neat_close_via_kernel(struct neat_ctx *ctx, struct neat_flow *flow) // kernel and userspace.. same for connect read and write if (flow->socket->fd != 0) { - neat_log(ctx, NEAT_LOG_DEBUG, "%s: Close fd %d", __func__, flow->socket->fd); + neat_log(ctx, NEAT_LOG_DEBUG, "%s - close fd %d", __func__, flow->socket->fd); close(flow->socket->fd); } @@ -5390,8 +5392,10 @@ neat_close_via_kernel(struct neat_ctx *ctx, struct neat_flow *flow) // further status of the close op for TCP. // taps-transports-usage does not specify CLOSE-EVENT.TCP, // maybe it should be dropped from the implementation? - neat_notify_close(flow); } + + neat_notify_close(flow); + return 0; } @@ -5400,7 +5404,7 @@ neat_close_via_kernel_2(struct neat_ctx *ctx, int fd) { neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); if (fd != -1) { - neat_log(ctx, NEAT_LOG_DEBUG, "%s: Close fd %d", __func__, fd); + neat_log(ctx, NEAT_LOG_DEBUG, "%s - close fd %d", __func__, fd); close(fd); } return 0; From 57cef76187ff27002f1dee1d985c79f212a01f08 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 8 May 2017 14:17:21 +0200 Subject: [PATCH 14/85] sync upstream CMAKE file --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 041fa22e..ba6d8aaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -236,6 +236,10 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") MESSAGE("LIBMNL found: " ${MNL_LIB}) ENDIF() FIND_LIBRARY(SCTP_LIB sctp) + IF(EXISTS "/proc/sys/net/mptcp/mptcp_enabled") + MESSAGE(STATUS "MPTCP found") + ADD_DEFINITIONS(-DMPTCP_SUPPORT) + ENDIF() ENDIF() CHECK_INCLUDE_FILE(jansson.h HAVE_JANSSON_H) From cf1e05a6a9ad1810287952b5d3de52da663ee6e8 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 8 May 2017 19:05:32 +0200 Subject: [PATCH 15/85] sync wip --- examples/client_http_get.c | 4 ++-- neat_core.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/client_http_get.c b/examples/client_http_get.c index 7cb2fe39..50df0f81 100644 --- a/examples/client_http_get.c +++ b/examples/client_http_get.c @@ -232,7 +232,7 @@ on_close(struct neat_flow_operations *opCB) fflush(stdout); - free(stat); + //free(stat); // stop event loop if all flows are closed flows_active--; @@ -244,7 +244,7 @@ on_close(struct neat_flow_operations *opCB) if (config_log_level >= 1) { fprintf(stderr, "%s - stopping event loop\n", __func__); } - //neat_stop_event_loop(opCB->ctx); + neat_stop_event_loop(opCB->ctx); } return NEAT_OK; diff --git a/neat_core.c b/neat_core.c index e5737f28..926bf58d 100644 --- a/neat_core.c +++ b/neat_core.c @@ -691,7 +691,6 @@ neat_free_flow(neat_flow *flow) LIST_REMOVE(flow, next_flow); - #if defined(USRSCTP_SUPPORT) if (neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) { synchronous_free(flow); @@ -1758,6 +1757,7 @@ updatePollHandle(neat_ctx *ctx, neat_flow *flow, uv_poll_t *handle) assert(flow->socket->handle); if (handle->loop == NULL || uv_is_closing((uv_handle_t *)handle)) { + neat_log(ctx, NEAT_LOG_DEBUG, "%s - loop is NULL or handle is closing - skipping", __func__); return; } From a6d1f2037d04878e5ffda8fe449965c4bf499016 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 8 May 2017 20:07:28 +0200 Subject: [PATCH 16/85] sync - wip --- CMakeLists.txt | 40 +++++++++++++++++++++----------------- examples/client_http_get.c | 6 +++++- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ba6d8aaf..80d676ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -184,6 +184,8 @@ IF (SCTP_MULTISTREAMING) ADD_DEFINITIONS(-DSCTP_MULTISTREAMING) ENDIF() +OPTION(OPENSSL_SUPPORT "Include support for OpenSSL" 1) + # REQUIREMENTS ################################################# CHECK_INCLUDE_FILE(uv.h HAVE_SYS_UV_H) @@ -203,27 +205,29 @@ ELSE() MESSAGE("LDNS found: " ${LDNS_LIB}) ENDIF() -FIND_PACKAGE(OpenSSL) -IF (NOT OPENSSL_FOUND) - MESSAGE(WARNING "openssl >= 1.0.2 required for TLS - none found") -ELSE() - MESSAGE("OPENSSL version found: " ${OPENSSL_VERSION}) - IF (OPENSSL_VERSION VERSION_LESS "1.0.2") - MESSAGE(WARNING "openssl >= 1.0.2 required for TLS") +IF (OPENSSL_SUPPORT) + FIND_PACKAGE(OpenSSL) + IF (NOT OPENSSL_FOUND) + MESSAGE(WARNING "openssl >= 1.0.2 required for TLS - none found") ELSE() - CHECK_INCLUDE_FILE(openssl/ssl.h HAVE_SYS_OPENSSL_H) - MESSAGE(STATUS "OPENSSL Crypto found: " ${OPENSSL_LIBRARIES}) - ADD_DEFINITIONS(-DNEAT_USETLS) - SET(CMAKE_EXTRA_INCLUDE_FILES "openssl/ssl.h") - - IF ((OPENSSL_VERSION VERSION_EQUAL "1.1.0") OR (OPENSSL_VERSION VERSION_GREATER "1.1.0")) - CHECK_TYPE_SIZE("struct bio_dgram_sctp_sndinfo" OPENSSL_DTLS) - IF (HAVE_OPENSSL_DTLS) - MESSAGE("DTLS for SCTP supported") - ADD_DEFINITIONS(-DNEAT_SCTP_DTLS) + MESSAGE("OPENSSL version found: " ${OPENSSL_VERSION}) + IF (OPENSSL_VERSION VERSION_LESS "1.0.2") + MESSAGE(WARNING "openssl >= 1.0.2 required for TLS") + ELSE() + CHECK_INCLUDE_FILE(openssl/ssl.h HAVE_SYS_OPENSSL_H) + MESSAGE(STATUS "OPENSSL Crypto found: " ${OPENSSL_LIBRARIES}) + ADD_DEFINITIONS(-DNEAT_USETLS) + SET(CMAKE_EXTRA_INCLUDE_FILES "openssl/ssl.h") + + IF ((OPENSSL_VERSION VERSION_EQUAL "1.1.0") OR (OPENSSL_VERSION VERSION_GREATER "1.1.0")) + CHECK_TYPE_SIZE("struct bio_dgram_sctp_sndinfo" OPENSSL_DTLS) + IF (HAVE_OPENSSL_DTLS) + MESSAGE("DTLS for SCTP supported") + ADD_DEFINITIONS(-DNEAT_SCTP_DTLS) + ENDIF() ENDIF() - ENDIF() + ENDIF() ENDIF() ENDIF() diff --git a/examples/client_http_get.c b/examples/client_http_get.c index 50df0f81..d7890d6c 100644 --- a/examples/client_http_get.c +++ b/examples/client_http_get.c @@ -107,7 +107,6 @@ on_readable(struct neat_flow_operations *opCB) } if (bytes_read > 0) { - stat = opCB->userData; stat->protocol = (int)options[0].value.integer; stat->rcv_bytes += bytes_read; stat->rcv_calls++; @@ -194,6 +193,7 @@ on_close(struct neat_flow_operations *opCB) char buffer_bandwidth_human[32]; uv_timer_stop(&(stat->timer)); + uv_close((uv_handle_t *) &(stat->timer), NULL); if (config_log_level >= 1) { fprintf(stderr, "%s - neat_read() returned 0 bytes - connection closed\n", __func__); @@ -376,6 +376,10 @@ main(int argc, char *argv[]) cleanup: + for (i = 0; i < num_flows; i++) { + free(ops[i].userData); + } + if (ctx != NULL) { neat_free_ctx(ctx); } From 142062149bc6876e68aa8f3c481b38186218dce9 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 8 May 2017 20:45:37 +0200 Subject: [PATCH 17/85] improve closing mechanism --- examples/client_http_get.c | 2 +- neat_core.c | 14 ++++++++++---- neat_resolver.h | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/examples/client_http_get.c b/examples/client_http_get.c index d7890d6c..7f1b9ba9 100644 --- a/examples/client_http_get.c +++ b/examples/client_http_get.c @@ -193,7 +193,7 @@ on_close(struct neat_flow_operations *opCB) char buffer_bandwidth_human[32]; uv_timer_stop(&(stat->timer)); - uv_close((uv_handle_t *) &(stat->timer), NULL); + //uv_close((uv_handle_t *) &(stat->timer), NULL); if (config_log_level >= 1) { fprintf(stderr, "%s - neat_read() returned 0 bytes - connection closed\n", __func__); diff --git a/neat_core.c b/neat_core.c index 926bf58d..17a810a3 100644 --- a/neat_core.c +++ b/neat_core.c @@ -1762,7 +1762,7 @@ updatePollHandle(neat_ctx *ctx, neat_flow *flow, uv_poll_t *handle) } do { - neat_log(ctx, NEAT_LOG_DEBUG, "%s - iterating flows ...", __func__); + //neat_log(ctx, NEAT_LOG_DEBUG, "%s - iterating flows ...", __func__); assert(flow); flow->isPolling = 0; @@ -1814,8 +1814,10 @@ updatePollHandle(neat_ctx *ctx, neat_flow *flow, uv_poll_t *handle) } #ifdef SCTP_MULTISTREAMING - flow = LIST_NEXT(flow, multistream_next_flow); - neat_log(ctx, NEAT_LOG_DEBUG, "%s - next multistream flow : %p", __func__, flow); + if (pollable_socket->multistream == 1) { + flow = LIST_NEXT(flow, multistream_next_flow); + neat_log(ctx, NEAT_LOG_DEBUG, "%s - next multistream flow : %p", __func__, flow); + } #endif // iterate through all flows @@ -6808,15 +6810,19 @@ neat_close(struct neat_ctx *ctx, struct neat_flow *flow) } if (!flow->socket->multistream || flow->socket->sctp_streams_used == 0) { - neat_log(ctx, NEAT_LOG_DEBUG, "%s - not multistream socket or all streams closed", __func__); + //neat_log(ctx, NEAT_LOG_DEBUG, "%s - not multistream socket or all streams closed", __func__); #endif // SCTP_MULTISTREAMING if (flow->isPolling && uv_is_active((uv_handle_t*)flow->socket->handle)) { + neat_log(ctx, NEAT_LOG_DEBUG, "%s - stopping polling", __func__); uv_poll_stop(flow->socket->handle); } #ifdef SCTP_MULTISTREAMING } #endif + neat_notify_close(flow); + + return NEAT_OK; } diff --git a/neat_resolver.h b/neat_resolver.h index 62ae3027..01c18980 100644 --- a/neat_resolver.h +++ b/neat_resolver.h @@ -9,7 +9,7 @@ #include "neat_addr.h" //Timeout for complete DNS query -#define DNS_TIMEOUT 30000 +#define DNS_TIMEOUT 10000 //Timeout after first good reply #define DNS_RESOLVED_TIMEOUT 1000 #define DNS_LITERAL_TIMEOUT 1 From 195aa46fcd419fe9539a5deac79fd0525d83e35f Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 8 May 2017 21:02:02 +0200 Subject: [PATCH 18/85] remove stats timer --- examples/client_http_get.c | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/examples/client_http_get.c b/examples/client_http_get.c index 7f1b9ba9..905ca29d 100644 --- a/examples/client_http_get.c +++ b/examples/client_http_get.c @@ -68,7 +68,6 @@ struct stat_flow { struct timeval tv_last; struct timeval tv_delta; uint16_t protocol; - uv_timer_t timer; struct neat_flow *flow; }; @@ -138,31 +137,11 @@ on_writable(struct neat_flow_operations *opCB) return NEAT_OK; } -static void -print_timer_stats(uv_timer_t *handle) -{ - struct stat_flow *stat = handle->data; - struct timeval tv_now, tv_delta; - double time_elapsed = 0.0; - char buffer_filesize_human[32]; - - gettimeofday(&tv_now, NULL); - timersub(&tv_now, &(stat->tv_delta), &tv_delta); - time_elapsed = tv_delta.tv_sec + (double)tv_delta.tv_usec / 1000000.0; - filesize_human(8 * (stat->rcv_bytes - stat->rcv_bytes_last) / time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human)); - - fprintf(stderr, "%p - %d bytes in %.2fs = %sit/s\n", (void *) stat->flow, stat->rcv_bytes - stat->rcv_bytes_last, time_elapsed, buffer_filesize_human); - - stat->rcv_bytes_last = stat->rcv_bytes; - gettimeofday(&(stat->tv_delta), NULL); - uv_timer_again(&(stat->timer)); -} - static neat_error_code on_connected(struct neat_flow_operations *opCB) { struct stat_flow *stat = opCB->userData; - uv_loop_t *loop = neat_get_event_loop(opCB->ctx); + // now we can start writing if (config_log_level >= 1) { fprintf(stderr, "%s - connection established\n", __func__); @@ -172,10 +151,6 @@ on_connected(struct neat_flow_operations *opCB) gettimeofday(&(stat->tv_last), NULL); gettimeofday(&(stat->tv_delta), NULL); - uv_timer_init(loop, &(stat->timer)); - stat->timer.data = stat; - uv_timer_start(&(stat->timer), print_timer_stats, 1000, 1000); - opCB->on_readable = on_readable; opCB->on_writable = on_writable; neat_set_operations(opCB->ctx, opCB->flow, opCB); @@ -192,9 +167,6 @@ on_close(struct neat_flow_operations *opCB) char buffer_filesize_human[32]; char buffer_bandwidth_human[32]; - uv_timer_stop(&(stat->timer)); - //uv_close((uv_handle_t *) &(stat->timer), NULL); - if (config_log_level >= 1) { fprintf(stderr, "%s - neat_read() returned 0 bytes - connection closed\n", __func__); } From 94dcd9c4251a46155c77d2cb119635e48b42462c Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 10:31:07 +0200 Subject: [PATCH 19/85] tneat for new semantic --- examples/tneat.c | 68 ++++++++++++++++-------------------------------- 1 file changed, 22 insertions(+), 46 deletions(-) diff --git a/examples/tneat.c b/examples/tneat.c index cfd49fb5..9a0e7d12 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -163,7 +163,6 @@ on_writable(struct neat_flow_operations *opCB) { struct tneat_flow *tnf = opCB->userData; neat_error_code code; -printf("tneat: on_writable\n"); if (config_log_level >= 2) { fprintf(stderr, "%s()\n", __func__); } @@ -199,10 +198,7 @@ printf("tneat: on_writable\n"); code = neat_write(opCB->ctx, opCB->flow, tnf->snd.buffer, config_snd_buffer_size, NULL, 0); if (tnf->done) { - opCB->on_writable = NULL; - opCB->on_all_written = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); - neat_shutdown(opCB->ctx, opCB->flow); + neat_close(opCB->ctx, opCB->flow); return NEAT_OK; } @@ -219,11 +215,10 @@ on_readable(struct neat_flow_operations *opCB) { struct tneat_flow *tnf = opCB->userData; uint32_t buffer_filled; - struct timeval diff_time; neat_error_code code; - char buffer_filesize_human[32]; - double time_elapsed; -printf("tneat: on_readable\n"); + + + if (config_log_level >= 2) { fprintf(stderr, "%s()\n", __func__); } @@ -256,35 +251,6 @@ printf("tneat: on_readable\n"); } } // peer disconnected - } else if (buffer_filled == 0){ - if (config_log_level >= 1) { - printf("connection closed\n"); - } - - if (config_active) { - on_close(opCB); - } else { - // we are server - opCB->on_readable = NULL; - opCB->on_writable = NULL; - opCB->on_all_written = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); - - // print statistics - timersub(&(tnf->rcv.tv_last), &(tnf->rcv.tv_first), &diff_time); - time_elapsed = diff_time.tv_sec + (double)diff_time.tv_usec/1000000.0; - - printf("%u, %u, %.2f, %.2f, %s\n", tnf->rcv.bytes, tnf->rcv.calls, time_elapsed, tnf->rcv.bytes/time_elapsed, filesize_human(tnf->rcv.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); - - if (config_log_level >= 1) { - printf("client disconnected - statistics\n"); - printf("\tbytes\t\t: %u\n", tnf->rcv.bytes); - printf("\trcv-calls\t: %u\n", tnf->rcv.calls); - printf("\tduration\t: %.2fs\n", time_elapsed); - printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->rcv.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); - } - neat_shutdown(opCB->ctx, opCB->flow); - } } return NEAT_OK; @@ -297,7 +263,6 @@ static neat_error_code on_connected(struct neat_flow_operations *opCB) { struct tneat_flow *tnf = NULL; -printf("tneat: on_connected\n"); if (config_log_level >= 1) { fprintf(stderr, "%s() - connection established\n", __func__); } @@ -340,12 +305,25 @@ static neat_error_code on_close(struct neat_flow_operations *opCB) { struct tneat_flow *tnf = opCB->userData; + char buffer_filesize_human[32]; + double time_elapsed; + struct timeval diff_time; - // cleanup - opCB->on_close = NULL; - opCB->on_readable = NULL; - opCB->on_writable = NULL; - opCB->on_error = NULL; + if (!config_active) { + // print statistics + timersub(&(tnf->rcv.tv_last), &(tnf->rcv.tv_first), &diff_time); + time_elapsed = diff_time.tv_sec + (double)diff_time.tv_usec/1000000.0; + + printf("%u, %u, %.2f, %.2f, %s\n", tnf->rcv.bytes, tnf->rcv.calls, time_elapsed, tnf->rcv.bytes/time_elapsed, filesize_human(tnf->rcv.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); + + if (config_log_level >= 1) { + printf("client disconnected - statistics\n"); + printf("\tbytes\t\t: %u\n", tnf->rcv.bytes); + printf("\trcv-calls\t: %u\n", tnf->rcv.calls); + printf("\tduration\t: %.2fs\n", time_elapsed); + printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->rcv.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); + } + } if (tnf->snd.buffer) { free(tnf->snd.buffer); @@ -359,8 +337,6 @@ on_close(struct neat_flow_operations *opCB) free(tnf); } - neat_set_operations(opCB->ctx, opCB->flow, opCB); - fprintf(stderr, "%s - flow closed OK!\n", __func__); // stop event loop if we are active part From 8bd3fa833c89da298fcd9e4b97389e3ce716a025 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 10:36:27 +0200 Subject: [PATCH 20/85] tneat improvements --- examples/tneat.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/tneat.c b/examples/tneat.c index 9a0e7d12..1acafc80 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -39,7 +39,7 @@ static char *config_property = "\ {\ \"transport\": [\ {\ - \"value\": \"SCTP\",\ + \"value\": \"TCP\",\ \"precedence\": 1\ },\ {\ @@ -389,8 +389,7 @@ main(int argc, char *argv[]) break; case 'P': if (read_file(optarg, &arg_property) < 0) { - fprintf(stderr, "Unable to read properties from %s: %s", - optarg, strerror(errno)); + fprintf(stderr, "Unable to read properties from %s: %s", optarg, strerror(errno)); result = EXIT_FAILURE; goto cleanup; } @@ -553,5 +552,9 @@ main(int argc, char *argv[]) if (ctx != NULL) { neat_free_ctx(ctx); } + + if (arg_property != config_property && arg_property != NULL) { + free(arg_property); + } exit(result); } From 3110d239ab4ff5ab96561d53efa33041e4ad5abb Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 11:27:06 +0200 Subject: [PATCH 21/85] tneat improvements --- examples/tneat.c | 61 ++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/examples/tneat.c b/examples/tneat.c index 1acafc80..ad634ce7 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -35,6 +35,7 @@ static uint16_t config_port = 23232; static uint16_t config_log_level = 1; static uint16_t config_num_flows = 1; static uint16_t config_max_flows = 100; +static uint16_t close_after = 2; static char *config_property = "\ {\ \"transport\": [\ @@ -78,7 +79,6 @@ struct tneat_flow_direction { }; struct tneat_flow { - uint8_t done; struct tneat_flow_direction rcv; struct tneat_flow_direction snd; }; @@ -125,7 +125,7 @@ on_all_written(struct neat_flow_operations *opCB) struct tneat_flow *tnf = opCB->userData; struct timeval now, diff_time; double time_elapsed; - char buffer_filesize_human[32]; + if (config_log_level >= 2) { fprintf(stderr, "%s()\n", __func__); @@ -138,16 +138,10 @@ on_all_written(struct neat_flow_operations *opCB) // runtime- or message-limit reached if ((config_runtime_max > 0 && time_elapsed >= config_runtime_max) || (config_message_count > 0 && tnf->snd.calls >= config_message_count)) { + neat_close(opCB->ctx, opCB->flow); + } - // print statistics - printf("neat_write finished - statistics\n"); - printf("\tbytes\t\t: %u\n", tnf->snd.bytes); - printf("\tsnd-calls\t: %u\n", tnf->snd.calls); - printf("\tduration\t: %.2fs\n", time_elapsed); - printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->snd.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); - tnf->done = 1; - } opCB->on_writable = on_writable; opCB->on_all_written = NULL; @@ -197,11 +191,6 @@ on_writable(struct neat_flow_operations *opCB) code = neat_write(opCB->ctx, opCB->flow, tnf->snd.buffer, config_snd_buffer_size, NULL, 0); - if (tnf->done) { - neat_close(opCB->ctx, opCB->flow); - return NEAT_OK; - } - if (code != NEAT_OK) { fprintf(stderr, "%s - neat_write error: code %d\n", __func__, (int)code); return on_error(opCB); @@ -285,7 +274,6 @@ on_connected(struct neat_flow_operations *opCB) } // reset stats - tnf->done = 0; tnf->snd.calls = 0; tnf->snd.bytes = 0; tnf->rcv.calls = 0; @@ -309,20 +297,30 @@ on_close(struct neat_flow_operations *opCB) double time_elapsed; struct timeval diff_time; + fprintf(stderr, "%s\n", __func__); + if (!config_active) { // print statistics timersub(&(tnf->rcv.tv_last), &(tnf->rcv.tv_first), &diff_time); time_elapsed = diff_time.tv_sec + (double)diff_time.tv_usec/1000000.0; - printf("%u, %u, %.2f, %.2f, %s\n", tnf->rcv.bytes, tnf->rcv.calls, time_elapsed, tnf->rcv.bytes/time_elapsed, filesize_human(tnf->rcv.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); + //rintf("%u, %u, %.2f, %.2f, %s\n", tnf->rcv.bytes, tnf->rcv.calls, time_elapsed, tnf->rcv.bytes/time_elapsed, filesize_human(tnf->rcv.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); + printf("flow closed - statistics\n"); + printf("\tbytes\t\t: %u\n", tnf->rcv.bytes); + printf("\trcv-calls\t: %u\n", tnf->rcv.calls); + printf("\tduration\t: %.2fs\n", time_elapsed); + printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->rcv.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); + + } else { + // print statistics + timersub(&(tnf->snd.tv_last), &(tnf->snd.tv_first), &diff_time); + time_elapsed = diff_time.tv_sec + (double)diff_time.tv_usec/1000000.0; - if (config_log_level >= 1) { - printf("client disconnected - statistics\n"); - printf("\tbytes\t\t: %u\n", tnf->rcv.bytes); - printf("\trcv-calls\t: %u\n", tnf->rcv.calls); - printf("\tduration\t: %.2fs\n", time_elapsed); - printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->rcv.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); - } + printf("flow closed - statistics\n"); + printf("\tbytes\t\t: %u\n", tnf->snd.bytes); + printf("\tsnd-calls\t: %u\n", tnf->snd.calls); + printf("\tduration\t: %.2fs\n", time_elapsed); + printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->snd.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); } if (tnf->snd.buffer) { @@ -346,6 +344,12 @@ on_close(struct neat_flow_operations *opCB) fprintf(stderr, "%s - stopping event loop\n", __func__); neat_stop_event_loop(opCB->ctx); } + } else { + close_after--; + if (!close_after) { + fprintf(stderr, "%s - stopping event loop\n", __func__); + neat_stop_event_loop(opCB->ctx); + } } return NEAT_OK; @@ -436,14 +440,10 @@ main(int argc, char *argv[]) if (optind == argc) { config_active = 0; - if (config_log_level >= 1) { - printf("role: passive\n"); - } + printf("role: passive\n"); } else if (optind + 1 == argc) { config_active = 1; - if (config_log_level >= 1) { - printf("role: active\n"); - } + printf("role: active\n"); } else { fprintf(stderr, "%s - argument error\n", __func__); print_usage(); @@ -506,6 +506,7 @@ main(int argc, char *argv[]) ops[0].on_connected = on_connected; ops[0].on_error = on_error; + ops[0].on_close = on_close; if (neat_set_operations(ctx, flows[0], &(ops[0]))) { fprintf(stderr, "%s - neat_set_operations failed\n", __func__); From 3ee9b8df481dcb043e79987aaa27eb8edeb6a026 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 11:27:30 +0200 Subject: [PATCH 22/85] flow closing improvements --- neat_core.c | 46 ++++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/neat_core.c b/neat_core.c index 17a810a3..54962efa 100644 --- a/neat_core.c +++ b/neat_core.c @@ -904,8 +904,6 @@ static void io_connected(neat_ctx *ctx, neat_flow *flow, #endif // #if defined(IPPROTO_SCTP) && defined(SCTP_INTERLEAVING_SUPPORTED) && !defined(USRSCTP_SUPPORT) char proto[16]; - - switch (flow->socket->stack) { case NEAT_STACK_UDP: snprintf(proto, 16, "UDP"); @@ -955,6 +953,8 @@ static void io_connected(neat_ctx *ctx, neat_flow *flow, neat_log(ctx, NEAT_LOG_INFO, "Connected: %s/%s", proto, (flow->socket->family == AF_INET ? "IPv4" : "IPv6")); + flow->state = NEAT_FLOW_OPEN; + if (flow->operations && flow->operations->on_connected) { READYCALLBACKSTRUCT; flow->operations->on_connected(flow->operations); @@ -1611,6 +1611,7 @@ io_readable(neat_ctx *ctx, neat_flow *flow, multistream_flow->operations->on_connected = listen_flow->operations->on_connected; multistream_flow->operations->on_readable = listen_flow->operations->on_readable; multistream_flow->operations->on_writable = listen_flow->operations->on_writable; + multistream_flow->operations->on_close = listen_flow->operations->on_close; multistream_flow->operations->on_error = listen_flow->operations->on_error; multistream_flow->operations->ctx = ctx; multistream_flow->operations->flow = multistream_flow; @@ -2499,6 +2500,7 @@ do_accept(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *listen_so newFlow->operations->on_connected = flow->operations->on_connected; newFlow->operations->on_readable = flow->operations->on_readable; newFlow->operations->on_writable = flow->operations->on_writable; + newFlow->operations->on_close = flow->operations->on_close; newFlow->operations->on_error = flow->operations->on_error; newFlow->operations->ctx = ctx; newFlow->operations->flow = flow; @@ -4512,19 +4514,16 @@ neat_error_code neat_accept(struct neat_ctx *ctx, struct neat_flow *flow, uint16_t port, struct neat_tlv optional[], unsigned int opt_count) { - // const char *service_name = NULL; - const char *local_name = NULL; - json_t *val = NULL, *security = NULL; - int stream_count = 0; - neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); + const char *local_name = NULL; + json_t *val = NULL; + json_t *security = NULL; + int stream_count = 0; - //nr_of_stacks = neat_property_translate_protocols(flow->propertyMask, stacks); - - //if (nr_of_stacks == 0) - //return NEAT_ERROR_UNABLE; + neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); - if (flow->name) + if (flow->name) { return NEAT_ERROR_BAD_ARGUMENT; + } HANDLE_OPTIONAL_ARGUMENTS_START() OPTIONAL_STRING(NEAT_TAG_LOCAL_NAME, local_name) @@ -4537,16 +4536,17 @@ neat_accept(struct neat_ctx *ctx, struct neat_flow *flow, neat_log(ctx, NEAT_LOG_DEBUG, "%s - %d streams", __func__, flow->streams_requested); } - if (!local_name) + if (!local_name) { local_name = "0.0.0.0"; + } - flow->name = strdup(local_name); - if (flow->name == NULL) { + ; + if ((flow->name = strdup(local_name)) == NULL) { return NEAT_ERROR_OUT_OF_MEMORY; } - flow->port = port; - flow->ctx = ctx; + flow->port = port; + flow->ctx = ctx; if ((security = json_object_get(flow->properties, "security")) != NULL && (val = json_object_get(security, "value")) != NULL && @@ -4556,14 +4556,15 @@ neat_accept(struct neat_ctx *ctx, struct neat_flow *flow, flow->security_needed = 0; } - if (!ctx->resolver) + if (!ctx->resolver) { ctx->resolver = neat_resolver_init(ctx, "/etc/resolv.conf"); + } - if (!ctx->pvd) + if (!ctx->pvd) { ctx->pvd = neat_pvd_init(ctx); + } - neat_resolve(ctx->resolver, AF_INET, flow->name, flow->port, - accept_resolve_cb, flow); + neat_resolve(ctx->resolver, AF_INET, flow->name, flow->port, accept_resolve_cb, flow); return NEAT_OK; } @@ -6760,6 +6761,8 @@ neat_notify_close(neat_flow *flow) neat_error_code code = NEAT_ERROR_OK; neat_ctx *ctx = flow->ctx; + neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); + if (flow->state == NEAT_FLOW_CLOSED) { neat_log(ctx, NEAT_LOG_WARNING, "%s - flow already closed - skipping", __func__); return; @@ -6767,7 +6770,6 @@ neat_notify_close(neat_flow *flow) flow->state = NEAT_FLOW_CLOSED; - neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); if (flow->operations && flow->operations->on_close) { READYCALLBACKSTRUCT; flow->operations->on_close(flow->operations); From 0936dc7adcef58b6257111e6ecd396ad2e386675 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 11:30:42 +0200 Subject: [PATCH 23/85] check if handle->data exists --- neat_core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/neat_core.c b/neat_core.c index 54962efa..7885ff42 100644 --- a/neat_core.c +++ b/neat_core.c @@ -1744,6 +1744,7 @@ updatePollHandle(neat_ctx *ctx, neat_flow *flow, uv_poll_t *handle) neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); assert(handle); + assert(handle->data); pollable_socket = handle->data; #ifdef SCTP_MULTISTREAMING From 28828d59a4577c3af1a48b7aef2812457adeb5a7 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 11:35:47 +0200 Subject: [PATCH 24/85] revert assert check for pollable socket --- neat_core.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/neat_core.c b/neat_core.c index 7885ff42..36775e08 100644 --- a/neat_core.c +++ b/neat_core.c @@ -1744,7 +1744,6 @@ updatePollHandle(neat_ctx *ctx, neat_flow *flow, uv_poll_t *handle) neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); assert(handle); - assert(handle->data); pollable_socket = handle->data; #ifdef SCTP_MULTISTREAMING @@ -1816,7 +1815,7 @@ updatePollHandle(neat_ctx *ctx, neat_flow *flow, uv_poll_t *handle) } #ifdef SCTP_MULTISTREAMING - if (pollable_socket->multistream == 1) { + if (pollable_socket && pollable_socket->multistream == 1) { flow = LIST_NEXT(flow, multistream_next_flow); neat_log(ctx, NEAT_LOG_DEBUG, "%s - next multistream flow : %p", __func__, flow); } From a6d686555e332508c93a7c0308cf8d6fc9df36bf Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 13:29:06 +0200 Subject: [PATCH 25/85] new closing semantic for SCTP flows --- neat_core.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/neat_core.c b/neat_core.c index 36775e08..d08ae89d 100644 --- a/neat_core.c +++ b/neat_core.c @@ -1256,10 +1256,9 @@ io_readable(neat_ctx *ctx, neat_flow *flow, { struct sockaddr_storage peerAddr; socklen_t peerAddrLen = sizeof(struct sockaddr_storage); - int stream_id = -1; - int retval; + int stream_id = -1; + ssize_t n = 0; char buffer[1]; - ssize_t n; struct msghdr msghdr; //Not used when notifications aren't available: #ifdef MSG_NOTIFICATION @@ -1285,8 +1284,6 @@ io_readable(neat_ctx *ctx, neat_flow *flow, #if (defined(SCTP_RCVINFO) || defined (SCTP_SNDRCV)) struct cmsghdr *cmsg; #endif - - struct iovec iov; #else // !defined(USRSCTP_SUPPORT) @@ -1685,14 +1682,20 @@ io_readable(neat_ctx *ctx, neat_flow *flow, } if (socket->stack == NEAT_STACK_TCP) { - retval = recv(flow->socket->fd, buffer, 1, MSG_PEEK); - if (retval <= 0) { - neat_log(ctx, NEAT_LOG_INFO, "%s - TCP connection peek: %d - connection closed", __func__, retval); + n = recv(flow->socket->fd, buffer, 1, MSG_PEEK); + if (n <= 0) { + neat_log(ctx, NEAT_LOG_INFO, "%s - TCP connection peek: %d - connection closed", __func__, n); neat_notify_close(flow); return READ_WITH_ZERO; } } + if (socket->stack == NEAT_STACK_SCTP && n == 0 && flow->readBufferSize == 0) { + neat_log(ctx, NEAT_LOG_INFO, "%s - SCTP connection closed and no outstanding messages buffered", __func__); + neat_notify_close(flow); + return READ_WITH_ZERO; + } + if (flow->operations->on_readable) { READYCALLBACKSTRUCT; flow->operations->on_readable(flow->operations); From eede9cfd25faade09b472420de981c6ceeb57360 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 14:13:28 +0200 Subject: [PATCH 26/85] asserts for usrsctp bugs --- neat_core.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/neat_core.c b/neat_core.c index d08ae89d..7afd23e5 100644 --- a/neat_core.c +++ b/neat_core.c @@ -570,6 +570,7 @@ synchronous_free(neat_flow *flow) neat_log(flow->ctx, NEAT_LOG_DEBUG, "%s", __func__); assert(flow); + assert(flow->socket); if (!flow->socket->multistream #ifdef SCTP_MULTISTREAMING @@ -583,6 +584,7 @@ synchronous_free(neat_flow *flow) free((char *)flow->server_pem); free((char *)flow->cert_pem); free((char *)flow->key_pem); + if (flow->cc_algorithm) { free((char*)flow->cc_algorithm); } @@ -1227,6 +1229,9 @@ resize_read_buffer(neat_flow *flow) ssize_t spaceFree; ssize_t spaceNeeded, spaceThreshold; + assert(flow); + assert(flow->socket); + spaceFree = flow->readBufferAllocation - flow->readBufferSize; if (flow->socket->read_size > 0) { spaceThreshold = (flow->socket->read_size / 4 + 8191) & ~8191; From 549ebaf082676d80ce569aefb4108dddaef0556d Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 14:24:02 +0200 Subject: [PATCH 27/85] silence valgrind complains about openssl --- neat_security.c | 1 + 1 file changed, 1 insertion(+) diff --git a/neat_security.c b/neat_security.c index 8c4c2f04..2ca20fa1 100644 --- a/neat_security.c +++ b/neat_security.c @@ -692,6 +692,7 @@ neat_security_close(neat_ctx *ctx) #if (OPENSSL_VERSION_NUMBER < 0x10100000L) ERR_remove_state(0); #endif + SSL_COMP_free_compression_methods(); } #endif From 6c3418766e6ce6310afc64efacb4927f537e84f4 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 14:26:55 +0200 Subject: [PATCH 28/85] test for HTTPS added --- tests/run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/run.py b/tests/run.py index 54621031..b49d1b1f 100755 --- a/tests/run.py +++ b/tests/run.py @@ -21,6 +21,7 @@ tests_general.append([1, 0, workdir + 'client_http_get -u /cgi-bin/he -v 2 buildbot.nplab.de']) tests_general.append([0, 0, workdir + 'client_http_get -n 2 -u /files/4M bsd10.nplab.de']) if (platform.system() == "FreeBSD") or (platform.system() == "Linux"): + tests_general.append([1, 0, workdir + 'client_http_get -P ' + workdir = 'prop_tcp_security.json -p 443 -v 2 www.neat-project.org']) tests_general.append([0, 0, workdir + 'tneat -v 2 -P ' + workdir + 'prop_sctp_dtls.json interop.fh-muenster.de']) #tests_general.append([0, 0, 'python3.5 ../../policy/pmtests.py']) From 1616014b23a1ae2206e275fc532aa6ead4047746 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 14:36:44 +0200 Subject: [PATCH 29/85] testpath fix --- tests/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run.py b/tests/run.py index b49d1b1f..f0d88837 100755 --- a/tests/run.py +++ b/tests/run.py @@ -21,7 +21,7 @@ tests_general.append([1, 0, workdir + 'client_http_get -u /cgi-bin/he -v 2 buildbot.nplab.de']) tests_general.append([0, 0, workdir + 'client_http_get -n 2 -u /files/4M bsd10.nplab.de']) if (platform.system() == "FreeBSD") or (platform.system() == "Linux"): - tests_general.append([1, 0, workdir + 'client_http_get -P ' + workdir = 'prop_tcp_security.json -p 443 -v 2 www.neat-project.org']) + tests_general.append([1, 0, workdir + 'client_http_get -P ' + workdir + 'prop_tcp_security.json -p 443 -v 2 www.neat-project.org']) tests_general.append([0, 0, workdir + 'tneat -v 2 -P ' + workdir + 'prop_sctp_dtls.json interop.fh-muenster.de']) #tests_general.append([0, 0, 'python3.5 ../../policy/pmtests.py']) From 463f038f06572289fbf0afe890ddfe7a7f155090 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 15:24:19 +0200 Subject: [PATCH 30/85] socket.io lib as submodule --- .gitmodules | 3 +++ libs/socket.io-client | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 libs/socket.io-client diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..84c7e5c9 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "libs/socket.io-client"] + path = libs/socket.io-client + url = https://github.com/socketio/socket.io-client-cpp.git diff --git a/libs/socket.io-client b/libs/socket.io-client new file mode 160000 index 00000000..4d83a7a4 --- /dev/null +++ b/libs/socket.io-client @@ -0,0 +1 @@ +Subproject commit 4d83a7a40421c4d37d2813c54c46752f8e1de3d2 From 5156f5d596119d005e26437f2438ce730d5e33a0 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 15:48:16 +0200 Subject: [PATCH 31/85] Revert "socket.io lib as submodule" This reverts commit 463f038f06572289fbf0afe890ddfe7a7f155090. --- .gitmodules | 3 --- libs/socket.io-client | 1 - 2 files changed, 4 deletions(-) delete mode 100644 .gitmodules delete mode 160000 libs/socket.io-client diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 84c7e5c9..00000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "libs/socket.io-client"] - path = libs/socket.io-client - url = https://github.com/socketio/socket.io-client-cpp.git diff --git a/libs/socket.io-client b/libs/socket.io-client deleted file mode 160000 index 4d83a7a4..00000000 --- a/libs/socket.io-client +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4d83a7a40421c4d37d2813c54c46752f8e1de3d2 From c0bf16d7a8d668de89218647b50b8289fa7926b5 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 17:24:47 +0200 Subject: [PATCH 32/85] neat_new_flow refactoring --- neat_bsd.h | 1 - neat_core.c | 99 ++++++++++++++++++++++++++--------------------------- 2 files changed, 48 insertions(+), 52 deletions(-) diff --git a/neat_bsd.h b/neat_bsd.h index 8d1d7056..9c488bae 100644 --- a/neat_bsd.h +++ b/neat_bsd.h @@ -7,5 +7,4 @@ int udp6_fd; \ uv_udp_t uv_route_handle; \ char *route_buf - #endif diff --git a/neat_core.c b/neat_core.c index 7afd23e5..768ed1f2 100644 --- a/neat_core.c +++ b/neat_core.c @@ -6587,80 +6587,74 @@ neat_shutdown(struct neat_ctx *ctx, struct neat_flow *flow) return flow->shutdownfx(ctx, flow); } -neat_flow -*neat_new_flow(neat_ctx *ctx) +neat_flow * +neat_new_flow(neat_ctx *ctx) { - neat_flow *rv; + neat_flow *flow; neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); - rv = (neat_flow *)calloc (1, sizeof (neat_flow)); - if (!rv) { - goto error; + flow = calloc (1, sizeof (struct neat_flow)); + if (flow == NULL) { + return NULL; } - rv->ctx = ctx; - rv->writefx = neat_write_to_lower_layer; - rv->readfx = neat_read_from_lower_layer; - rv->acceptfx = neat_accept_via_kernel; - rv->connectfx = neat_connect; - rv->closefx = neat_close_socket; - rv->listenfx = NULL; // TODO: Consider reimplementing - rv->shutdownfx = neat_shutdown_via_kernel; + flow->state = NEAT_FLOW_CLOSED; + flow->ctx = ctx; + flow->writefx = neat_write_to_lower_layer; + flow->readfx = neat_read_from_lower_layer; + flow->acceptfx = neat_accept_via_kernel; + flow->connectfx = neat_connect; + flow->closefx = neat_close_socket; + flow->listenfx = NULL; // TODO: Consider reimplementing + flow->shutdownfx = neat_shutdown_via_kernel; #if defined(USRSCTP_SUPPORT) - rv->acceptusrsctpfx = neat_accept_via_usrsctp; + flow->acceptusrsctpfx = neat_accept_via_usrsctp; #endif - TAILQ_INIT(&(rv->listen_sockets)); - TAILQ_INIT(&rv->bufferedMessages); + TAILQ_INIT(&(flow->listen_sockets)); + TAILQ_INIT(&flow->bufferedMessages); + #ifdef SCTP_MULTISTREAMING - TAILQ_INIT(&rv->multistream_read_queue); + TAILQ_INIT(&flow->multistream_read_queue); #endif // SCTP_MULTISTREAMING - rv->properties = json_object(); - rv->user_ips = NULL; - rv->security_needed = 0; + flow->properties = json_object(); + flow->user_ips = NULL; + flow->security_needed = 0; - rv->socket = calloc(1, sizeof(struct neat_pollable_socket)); - if (!rv->socket) { - goto error; + flow->socket = calloc(1, sizeof(struct neat_pollable_socket)); + if (flow->socket == NULL) { + free(flow); + return NULL; } - rv->socket->flow = rv; - rv->socket->fd = 0; - rv->readBufferSize = 0; + flow->socket->flow = flow; + flow->socket->fd = 0; + flow->readBufferSize = 0; #if defined(USRSCTP_SUPPORT) - rv->socket->usrsctp_socket = NULL; - if (neat_base_stack(rv->socket->stack) == NEAT_STACK_SCTP) { - rv->socket->fd = -1; + flow->socket->usrsctp_socket = NULL; + if (neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) { + flow->socket->fd = -1; } #endif - rv->socket->handle = (uv_poll_t *) calloc(1, sizeof(uv_poll_t)); - if (!rv->socket->handle) { - goto error; + flow->socket->handle = calloc(1, sizeof(uv_poll_t)); + if (flow->socket->handle == NULL) { + free(flow->socket); + free(flow); + return NULL; } - rv->socket->handle->loop = NULL; - rv->socket->handle->type = UV_UNKNOWN_HANDLE; + flow->socket->handle->loop = NULL; + flow->socket->handle->type = UV_UNKNOWN_HANDLE; /* Initialise flow statistics */ - rv->flow_stats.bytes_sent = 0; - rv->flow_stats.bytes_received = 0; + flow->flow_stats.bytes_sent = 0; + flow->flow_stats.bytes_received = 0; - LIST_INSERT_HEAD(&ctx->flows, rv, next_flow); + LIST_INSERT_HEAD(&ctx->flows, flow, next_flow); - return rv; -error: - if (rv) { - if (rv->socket) { - if (rv->socket->handle) { - free(rv->socket->handle); - } - free(rv->socket); - } - free(rv); - } - return NULL; + return flow; } // Notify application about congestion via callback @@ -6767,9 +6761,12 @@ neat_notify_close(neat_flow *flow) const int stream_id = NEAT_INVALID_STREAM; //READYCALLBACKSTRUCT expects this: neat_error_code code = NEAT_ERROR_OK; + + assert(flow); + neat_ctx *ctx = flow->ctx; - neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); + neat_log(ctx, NEAT_LOG_DEBUG, "%s - state: %d", __func__, flow->state); if (flow->state == NEAT_FLOW_CLOSED) { neat_log(ctx, NEAT_LOG_WARNING, "%s - flow already closed - skipping", __func__); From a5ee69b84f4512d8f7832a3885e7fd05fb4d6f8c Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 17:49:35 +0200 Subject: [PATCH 33/85] minor refactoring --- neat_core.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/neat_core.c b/neat_core.c index 768ed1f2..43bb83aa 100644 --- a/neat_core.c +++ b/neat_core.c @@ -664,8 +664,8 @@ free_cb(uv_handle_t *handle) static int neat_close_socket(struct neat_ctx *ctx, struct neat_flow *flow) { - struct neat_pollable_socket *s; - struct neat_pollable_socket *stemp; + struct neat_pollable_socket *socket; + struct neat_pollable_socket *socket_temp; neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); @@ -676,9 +676,9 @@ neat_close_socket(struct neat_ctx *ctx, struct neat_flow *flow) } #endif - TAILQ_FOREACH_SAFE(s, &(flow->listen_sockets), next, stemp) { - neat_close_via_kernel_2(ctx, s->fd); - free(s); + TAILQ_FOREACH_SAFE(socket, &(flow->listen_sockets), next, socket_temp) { + neat_close_via_kernel_2(ctx, socket->fd); + free(socket); } neat_close_via_kernel(flow->ctx, flow); @@ -840,7 +840,6 @@ neat_error_code neat_set_operations(neat_ctx *ctx, neat_flow *flow, { neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); - flow->ownedByCore = 0; flow->operations = ops; if (flow->socket == NULL) { @@ -4406,8 +4405,9 @@ accept_resolve_cb(struct neat_resolver_results *results, } listen_socket = calloc(1, sizeof(*listen_socket)); - if (!listen_socket) + if (listen_socket == NULL) { return NEAT_ERROR_OUT_OF_MEMORY; + } listen_socket->flow = flow; listen_socket->stack = neat_base_stack(stacks[i]); @@ -4440,8 +4440,9 @@ accept_resolve_cb(struct neat_resolver_results *results, listen_socket->fd = fd; handle = calloc(1, sizeof(*handle)); - if (!handle) + if (handle == NULL) { return NEAT_ERROR_OUT_OF_MEMORY; + } listen_socket->handle = handle; handle->data = listen_socket; From 0f146c73eac3159534874db91fcfe931191f5cd3 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 18:15:07 +0200 Subject: [PATCH 34/85] improve closing mechanism --- neat_core.c | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/neat_core.c b/neat_core.c index 43bb83aa..9c6e364e 100644 --- a/neat_core.c +++ b/neat_core.c @@ -618,7 +618,7 @@ synchronous_free(neat_flow *flow) } static void -free_cb(uv_handle_t *handle) +socket_handle_free_cb(uv_handle_t *handle) { struct neat_pollable_socket *pollable_socket = handle->data; #ifdef SCTP_MULTISTREAMING @@ -658,7 +658,17 @@ free_cb(uv_handle_t *handle) } else { synchronous_free(pollable_socket->flow); } +} + +static void +listen_socket_handle_free_cb(uv_handle_t *handle) +{ + struct neat_pollable_socket *pollable_socket = handle->data; + //struct neat_ctx *ctx = pollable_socket->flow->ctx; + //neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); + free(pollable_socket); + free(handle); } static int @@ -676,11 +686,12 @@ neat_close_socket(struct neat_ctx *ctx, struct neat_flow *flow) } #endif + // close all listening sockets TAILQ_FOREACH_SAFE(socket, &(flow->listen_sockets), next, socket_temp) { neat_close_via_kernel_2(ctx, socket->fd); - free(socket); } + // close active socket neat_close_via_kernel(flow->ctx, flow); return 0; } @@ -689,6 +700,9 @@ void neat_free_flow(neat_flow *flow) { struct neat_ctx *ctx = flow->ctx; + struct neat_pollable_socket *listen_socket; + struct neat_pollable_socket *listen_socket_temp; + neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); LIST_REMOVE(flow, next_flow); @@ -702,15 +716,25 @@ neat_free_flow(neat_flow *flow) neat_free_candidates(ctx, flow->candidate_list); + // close all listening sockets + TAILQ_FOREACH_SAFE(listen_socket, &(flow->listen_sockets), next, listen_socket_temp) { + if (!uv_is_closing((uv_handle_t *)listen_socket->handle)) { + neat_log(ctx, NEAT_LOG_DEBUG, "%s - closing handle and waiting for socket_handle_free_cb", __func__); + uv_close((uv_handle_t *)(listen_socket->handle), listen_socket_handle_free_cb); + } else { + neat_log(ctx, NEAT_LOG_DEBUG, "%s - handle is already closing", __func__); + } + } + // close handles for active flow if (flow->socket->handle != NULL && flow->socket->handle->type != UV_UNKNOWN_HANDLE #ifdef SCTP_MULTISTREAMING && (!flow->socket->multistream || flow->socket->sctp_streams_used == 0) #endif ) { if (!uv_is_closing((uv_handle_t *)flow->socket->handle)) { - neat_log(ctx, NEAT_LOG_DEBUG, "%s - closing handle and waiting for free_cb", __func__); - uv_close((uv_handle_t *)(flow->socket->handle), free_cb); + neat_log(ctx, NEAT_LOG_DEBUG, "%s - closing handle and waiting for socket_handle_free_cb", __func__); + uv_close((uv_handle_t *)(flow->socket->handle), socket_handle_free_cb); } else { neat_log(ctx, NEAT_LOG_DEBUG, "%s - handle is already closing", __func__); } @@ -4404,7 +4428,7 @@ accept_resolve_cb(struct neat_resolver_results *results, #endif } - listen_socket = calloc(1, sizeof(*listen_socket)); + listen_socket = calloc(1, sizeof(struct neat_pollable_socket)); if (listen_socket == NULL) { return NEAT_ERROR_OUT_OF_MEMORY; } From 64406ce17f582f0a337d093c42a3b67b73a627a3 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 18:30:21 +0200 Subject: [PATCH 35/85] http client: stop event loop if error --- examples/client_http_get.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client_http_get.c b/examples/client_http_get.c index 905ca29d..a4f18658 100644 --- a/examples/client_http_get.c +++ b/examples/client_http_get.c @@ -80,7 +80,7 @@ on_error(struct neat_flow_operations *opCB) result = EXIT_FAILURE; - neat_close(opCB->ctx, opCB->flow); + neat_stop_event_loop(opCB->ctx, opCB->flow); return NEAT_OK; } From b162dd780983b3c17c9b1df4e9344d816557c639 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 18:31:56 +0200 Subject: [PATCH 36/85] http client: stop event loop if error --- examples/client_http_get.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client_http_get.c b/examples/client_http_get.c index a4f18658..f608cb81 100644 --- a/examples/client_http_get.c +++ b/examples/client_http_get.c @@ -80,7 +80,7 @@ on_error(struct neat_flow_operations *opCB) result = EXIT_FAILURE; - neat_stop_event_loop(opCB->ctx, opCB->flow); + neat_stop_event_loop(opCB->ctx); return NEAT_OK; } From 8b1351404419cfbeb2196b7df4b7ae6aa35037c0 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 9 May 2017 19:16:19 +0200 Subject: [PATCH 37/85] closing improvements --- CMakeLists.txt | 2 +- examples/tneat.c | 10 ++++------ neat_core.c | 17 +++++++++++------ 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 80d676ea..fb097373 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -179,7 +179,7 @@ IF (FLOW_GROUPS) ADD_DEFINITIONS(-DFLOW_GROUPS) ENDIF() -OPTION(SCTP_MULTISTREAMING "Include support for SCTP multistreaming" 1) +OPTION(SCTP_MULTISTREAMING "Include support for SCTP multistreaming" 0) IF (SCTP_MULTISTREAMING) ADD_DEFINITIONS(-DSCTP_MULTISTREAMING) ENDIF() diff --git a/examples/tneat.c b/examples/tneat.c index ad634ce7..db061627 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -139,13 +139,11 @@ on_all_written(struct neat_flow_operations *opCB) if ((config_runtime_max > 0 && time_elapsed >= config_runtime_max) || (config_message_count > 0 && tnf->snd.calls >= config_message_count)) { neat_close(opCB->ctx, opCB->flow); + } else { + opCB->on_writable = on_writable; + opCB->on_all_written = NULL; + neat_set_operations(opCB->ctx, opCB->flow, opCB); } - - - - opCB->on_writable = on_writable; - opCB->on_all_written = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); return NEAT_OK; } diff --git a/neat_core.c b/neat_core.c index 9c6e364e..315d38a8 100644 --- a/neat_core.c +++ b/neat_core.c @@ -625,8 +625,10 @@ socket_handle_free_cb(uv_handle_t *handle) struct neat_flow *flow = NULL; struct neat_flow *prev_flow = NULL; #endif - struct neat_ctx *ctx = pollable_socket->flow->ctx; - neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); + + assert(pollable_socket); + //struct neat_ctx *ctx = pollable_socket->flow->ctx; + //neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); if (pollable_socket->multistream) { #ifdef SCTP_MULTISTREAMING @@ -648,7 +650,7 @@ socket_handle_free_cb(uv_handle_t *handle) assert(pollable_socket->sctp_streams_used == 0); - neat_log(ctx, NEAT_LOG_DEBUG, "%s - all multistreams closed - freeing socket", __func__); + //neat_log(ctx, NEAT_LOG_DEBUG, "%s - all multistreams closed - freeing socket", __func__); free(pollable_socket->handle); free(pollable_socket); #else @@ -719,10 +721,10 @@ neat_free_flow(neat_flow *flow) // close all listening sockets TAILQ_FOREACH_SAFE(listen_socket, &(flow->listen_sockets), next, listen_socket_temp) { if (!uv_is_closing((uv_handle_t *)listen_socket->handle)) { - neat_log(ctx, NEAT_LOG_DEBUG, "%s - closing handle and waiting for socket_handle_free_cb", __func__); + neat_log(ctx, NEAT_LOG_DEBUG, "%s - closing listening handle and waiting for listen_socket_handle_free_cb", __func__); uv_close((uv_handle_t *)(listen_socket->handle), listen_socket_handle_free_cb); } else { - neat_log(ctx, NEAT_LOG_DEBUG, "%s - handle is already closing", __func__); + neat_log(ctx, NEAT_LOG_DEBUG, "%s - listen handle is already closing", __func__); } } @@ -864,7 +866,7 @@ neat_error_code neat_set_operations(neat_ctx *ctx, neat_flow *flow, { neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); - flow->operations = ops; + flow->operations = ops; if (flow->socket == NULL) { return NEAT_OK; @@ -6848,6 +6850,9 @@ neat_close(struct neat_ctx *ctx, struct neat_flow *flow) neat_log(ctx, NEAT_LOG_DEBUG, "%s - stopping polling", __func__); uv_poll_stop(flow->socket->handle); } + + neat_close_socket(ctx, flow); + #ifdef SCTP_MULTISTREAMING } #endif From 5e9b6d265346ef33771be37991531c608883e6ed Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 10 May 2017 01:00:04 +0200 Subject: [PATCH 38/85] usrsctp fix --- neat_core.c | 56 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/neat_core.c b/neat_core.c index 315d38a8..b9726efd 100644 --- a/neat_core.c +++ b/neat_core.c @@ -1280,9 +1280,7 @@ resize_read_buffer(neat_flow *flow) } static int -io_readable(neat_ctx *ctx, neat_flow *flow, - struct neat_pollable_socket *socket, - neat_error_code code) +io_readable(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *socket, neat_error_code code) { struct sockaddr_storage peerAddr; socklen_t peerAddrLen = sizeof(struct sockaddr_storage); @@ -6419,43 +6417,53 @@ handle_upcall(struct socket *sock, void *arg, int flags) { struct neat_pollable_socket *pollable_socket = arg; neat_flow *flow = pollable_socket->flow; + neat_ctx *ctx; + int events = 0; + neat_error_code code; neat_log(flow->ctx, NEAT_LOG_DEBUG, "%s", __func__); assert(flow); - if (flow) { - neat_ctx *ctx = flow->ctx; - neat_log(flow->ctx, NEAT_LOG_DEBUG, "%s", __func__); + ctx = flow->ctx; + events = usrsctp_get_events(sock); - int events = usrsctp_get_events(sock); - - if ((events & SCTP_EVENT_READ) && flow->acceptPending) { - do_accept(ctx, flow, pollable_socket); - return; - } + if ((events & SCTP_EVENT_READ) && flow->acceptPending) { + do_accept(ctx, flow, pollable_socket); + return; + } - // remove "on_writable" callback check - if (events & SCTP_EVENT_WRITE && flow->operations->on_writable) { - if (flow->firstWritePending) - flow->firstWritePending = 0; + // remove "on_writable" callback check + if (events & SCTP_EVENT_WRITE) { + if (flow->state == NEAT_FLOW_OPEN) { + flow->firstWritePending = 0; io_writable(ctx, flow, NEAT_OK); + } else { + neat_log(ctx, NEAT_LOG_WARNING, "%s - io_writable and flow in wrong state"); } - if (events & SCTP_EVENT_READ && flow->operations->on_readable) { - neat_error_code code; + } + + if (events & SCTP_EVENT_READ && flow->operations->on_readable) { + if (flow->state == NEAT_FLOW_OPEN) { do { code = io_readable(ctx, flow, pollable_socket, NEAT_OK); - } while (code == READ_OK); - if (code == READ_WITH_ZERO && flow->operations && flow->operations->on_readable) + } while (code == READ_OK && flow->state == NEAT_FLOW_OPEN); + + if (code == READ_WITH_ZERO && flow->operations && flow->operations->on_readable) { flow->operations->on_readable(flow->operations); + } + } else { + neat_log(ctx, NEAT_LOG_WARNING, "%s - io_readable and flow in wrong state"); } + } - // xxx why two times? - events = usrsctp_get_events(sock); - if (events & SCTP_EVENT_WRITE && flow->operations->on_writable) - io_writable(ctx, flow, NEAT_OK); + // xxx why two times? + events = usrsctp_get_events(sock); + if (events & SCTP_EVENT_WRITE && flow->operations->on_writable) { + io_writable(ctx, flow, NEAT_OK); } + } static int From 3ef0e2640243b4cebe419fe9ddbaac23f7a158af Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 10 May 2017 09:53:41 +0200 Subject: [PATCH 39/85] tneat and client_run_once refactoring --- examples/client_http_run_once.c | 53 +++++++++++++++++---------------- examples/tneat.c | 18 +++++------ neat_core.c | 3 +- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/examples/client_http_run_once.c b/examples/client_http_run_once.c index ecf2daeb..cfc707cc 100644 --- a/examples/client_http_run_once.c +++ b/examples/client_http_run_once.c @@ -103,26 +103,31 @@ on_readable(struct neat_flow_operations *opCB) return on_error(opCB); } - if (!bytes_read) { // eof - struct user_flow *f = opCB->userData; - ctx_flows[f->ctx_id] = ctx_flows[f->ctx_id] - 1; - streams_going--; /* one stream less */ - fprintf(stderr, "[Flow %u ended, %d to go, %d for ctx %d]\n", f->id, streams_going, ctx_flows[f->ctx_id], f->ctx_id); - fflush(stdout); - opCB->on_readable = NULL; // do not read more - neat_set_operations(opCB->ctx, opCB->flow, opCB); - - if (ctx_flows[f->ctx_id] == 0) { - fprintf(stderr, "%s - no flows left for ctx, stopping event loop\n", __func__); - neat_stop_event_loop(opCB->ctx); - } - - } else if (bytes_read > 0) { + if (bytes_read > 0) { //fwrite(buffer, sizeof(char), bytes_read, stdout); } + return 0; } +static neat_error_code +on_close(struct neat_flow_operations *opCB) { + struct user_flow *f = opCB->userData; + ctx_flows[f->ctx_id] = ctx_flows[f->ctx_id] - 1; + streams_going--; /* one stream less */ + fprintf(stderr, "[Flow %u ended, %d to go, %d for ctx %d]\n", f->id, streams_going, ctx_flows[f->ctx_id], f->ctx_id); + fflush(stdout); + opCB->on_readable = NULL; // do not read more + neat_set_operations(opCB->ctx, opCB->flow, opCB); + + if (ctx_flows[f->ctx_id] == 0) { + fprintf(stderr, "%s - no flows left for ctx, stopping event loop\n", __func__); + //neat_stop_event_loop(opCB->ctx); + } + + return NEAT_OK; +} + static neat_error_code on_writable(struct neat_flow_operations *opCB) { @@ -199,8 +204,7 @@ main(int argc, char *argv[]) break; case 'v': config_log_level = atoi(optarg); - fprintf(stderr, "%s - option - log level: %d\n", - __func__, config_log_level); + fprintf(stderr, "%s - option - log level: %d\n", __func__, config_log_level); break; case 's': config_property = config_property_sctp; @@ -257,6 +261,8 @@ main(int argc, char *argv[]) ops[i].on_connected = on_connected; ops[i].on_error = on_error; + ops[i].on_close = on_close; + flows[i].id = streams_going; flows[i].ctx_id = c; @@ -315,19 +321,14 @@ main(int argc, char *argv[]) } cleanup: fprintf(stderr, "Cleanup!\n"); - for (i = 0, c = 0; i < num_flows; i++, c++) { - if (c >= num_ctxs) { - c = 0; - } - if (flows[i].flow != NULL) { - neat_close(ctx[c], flows[i].flow); - } - } + for (c = 0; c < num_ctxs; c++) { - neat_free_ctx(ctx[c]); + neat_free_ctx(ctx[c]); } + if (buffer) { free(buffer); } + exit(result); } diff --git a/examples/tneat.c b/examples/tneat.c index db061627..1497ef67 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -35,7 +35,7 @@ static uint16_t config_port = 23232; static uint16_t config_log_level = 1; static uint16_t config_num_flows = 1; static uint16_t config_max_flows = 100; -static uint16_t close_after = 2; +static uint16_t config_max_server_runs = 0; static char *config_property = "\ {\ \"transport\": [\ @@ -51,6 +51,7 @@ static char *config_property = "\ }"; static uint32_t flows_active = 0; +static uint32_t server_runs = 0; static char *cert_file = NULL; static char *key_file = NULL; @@ -116,6 +117,7 @@ on_error(struct neat_flow_operations *opCB) { fprintf(stderr, "%s()\n", __func__); + neat_stop_event_loop(opCB->ctx); return NEAT_OK; } @@ -155,6 +157,7 @@ on_writable(struct neat_flow_operations *opCB) { struct tneat_flow *tnf = opCB->userData; neat_error_code code; + if (config_log_level >= 2) { fprintf(stderr, "%s()\n", __func__); } @@ -169,7 +172,7 @@ on_writable(struct neat_flow_operations *opCB) opCB->on_all_written = on_all_written; neat_set_operations(opCB->ctx, opCB->flow, opCB); - // increase stats + // update stats tnf->snd.calls++; tnf->snd.bytes += config_snd_buffer_size; gettimeofday(&(tnf->snd.tv_last), NULL); @@ -188,7 +191,6 @@ on_writable(struct neat_flow_operations *opCB) } code = neat_write(opCB->ctx, opCB->flow, tnf->snd.buffer, config_snd_buffer_size, NULL, 0); - if (code != NEAT_OK) { fprintf(stderr, "%s - neat_write error: code %d\n", __func__, (int)code); return on_error(opCB); @@ -204,8 +206,6 @@ on_readable(struct neat_flow_operations *opCB) uint32_t buffer_filled; neat_error_code code; - - if (config_log_level >= 2) { fprintf(stderr, "%s()\n", __func__); } @@ -221,8 +221,9 @@ on_readable(struct neat_flow_operations *opCB) } } + // we got data! if (buffer_filled > 0) { - // we got data! + if (tnf->rcv.calls == 0) { gettimeofday(&(tnf->rcv.tv_first), NULL); } @@ -237,7 +238,6 @@ on_readable(struct neat_flow_operations *opCB) printf("\n"); } } - // peer disconnected } return NEAT_OK; @@ -343,8 +343,8 @@ on_close(struct neat_flow_operations *opCB) neat_stop_event_loop(opCB->ctx); } } else { - close_after--; - if (!close_after) { + server_runs++; + if (config_max_server_runs > 0 && server_runs >= config_max_server_runs) { fprintf(stderr, "%s - stopping event loop\n", __func__); neat_stop_event_loop(opCB->ctx); } diff --git a/neat_core.c b/neat_core.c index b9726efd..cd8d0447 100644 --- a/neat_core.c +++ b/neat_core.c @@ -588,12 +588,14 @@ synchronous_free(neat_flow *flow) if (flow->cc_algorithm) { free((char*)flow->cc_algorithm); } + if (flow->resolver_results) { neat_log(flow->ctx, NEAT_LOG_DEBUG, "%s - neat_resolver_free_results", __func__); neat_resolver_free_results(flow->resolver_results); } else { neat_log(flow->ctx, NEAT_LOG_DEBUG, "%s - NOT neat_resolver_free_results", __func__); } + if (flow->ownedByCore) { free(flow->operations); } @@ -613,7 +615,6 @@ synchronous_free(neat_flow *flow) free(flow->socket); } - free(flow); } From 6b96f0737c19237ace11717c64876132379b7642 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 10 May 2017 10:00:51 +0200 Subject: [PATCH 40/85] client_http_run_once polish --- examples/client_http_run_once.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/examples/client_http_run_once.c b/examples/client_http_run_once.c index cfc707cc..aa1bd6f8 100644 --- a/examples/client_http_run_once.c +++ b/examples/client_http_run_once.c @@ -258,12 +258,10 @@ main(int argc, char *argv[]) ctx_flows[c]++; - ops[i].on_connected = on_connected; ops[i].on_error = on_error; ops[i].on_close = on_close; - flows[i].id = streams_going; flows[i].ctx_id = c; ops[i].userData = &flows[i]; @@ -284,7 +282,7 @@ main(int argc, char *argv[]) descriptors to become readable to know when to ask NEAT to run another loop ONCE on everything that it might have to work on. */ - for (c=0; c 0) { /* there's stuff to do on one or more contexts, do them all */ ; - } - else { + } else { fprintf(stderr, "Waiting...\n"); } - for (c=0; c Date: Wed, 10 May 2017 15:01:53 +0200 Subject: [PATCH 41/85] CMAKE changes --- CMakeLists.txt | 41 ++++++++++++++++++++++------------------- examples/CMakeLists.txt | 3 --- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fb097373..4f6be76c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,30 +9,30 @@ set(BUILD_VERSION ${BUILD_MAJOR}.${BUILD_MINOR}.${BUILD_PATCH}) # INSTALLATION_DIRECTORIES ################################################# -SET(CMAKE_INSTALL_BINDIR_DOC "user executables [PREFIX/bin]") -SET(CMAKE_INSTALL_SBINDIR_DOC "system admin executables [EPREFIX/sbin]") -SET(CMAKE_INSTALL_DATADIR_DOC "read-only architecture-independent data [PREFIX/share]") -SET(CMAKE_INSTALL_SYSCONFDIR_DOC "read-only single-machine data [PREFIX/etc]") -SET(CMAKE_INSTALL_LOCALSTATEDIR_DOC "modifiable single-machine data [PREFIX/var]") -SET(CMAKE_INSTALL_LIBDIR_DOC "object code libraries [PREFIX/lib]") -SET(CMAKE_INSTALL_INCLUDEDIR_DOC "read-only includes [PREFIX/include]") -SET(CMAKE_INSTALL_MANDIR_DOC "man documentation [PREFIX/share/man]") - -SET(CMAKE_INSTALL_BINDIR "bin" +SET(CMAKE_INSTALL_BINDIR_DOC "user executables [PREFIX/bin/neat]") +SET(CMAKE_INSTALL_SBINDIR_DOC "system admin executables [EPREFIX/sbin/neat]") +SET(CMAKE_INSTALL_DATADIR_DOC "read-only architecture-independent data [PREFIX/share/neat]") +SET(CMAKE_INSTALL_SYSCONFDIR_DOC "read-only single-machine data [PREFIX/etc/neat]") +SET(CMAKE_INSTALL_LOCALSTATEDIR_DOC "modifiable single-machine data [PREFIX/var/neat]") +SET(CMAKE_INSTALL_LIBDIR_DOC "object code libraries [PREFIX/lib/neat]") +SET(CMAKE_INSTALL_INCLUDEDIR_DOC "read-only includes [PREFIX/include/neat]") +SET(CMAKE_INSTALL_MANDIR_DOC "man documentation [PREFIX/share/man/neat]") + +SET(CMAKE_INSTALL_BINDIR "bin/neat" CACHE PATH ${CMAKE_INSTALL_BINDIR_DOC}) -SET(CMAKE_INSTALL_SBINDIR "sbin" +SET(CMAKE_INSTALL_SBINDIR "sbin/neat" CACHE PATH ${CMAKE_INSTALL_SBINDIR_DOC}) -SET(CMAKE_INSTALL_DATADIR "share" +SET(CMAKE_INSTALL_DATADIR "share/neat" CACHE PATH ${CMAKE_INSTALL_DATADIR_DOC}) -SET(CMAKE_INSTALL_SYSCONFDIR "etc" +SET(CMAKE_INSTALL_SYSCONFDIR "etc/neat" CACHE PATH ${CMAKE_INSTALL_SYSCONFDIR_DOC}) -SET(CMAKE_INSTALL_LOCALSTATEDIR "var" +SET(CMAKE_INSTALL_LOCALSTATEDIR "var/neat" CACHE PATH ${CMAKE_INSTALL_LOCALSTATEDIR_DOC}) -SET(CMAKE_INSTALL_LIBDIR "lib" +SET(CMAKE_INSTALL_LIBDIR "lib/neat" CACHE PATH ${CMAKE_INSTALL_LIBDIR_DOC}) -SET(CMAKE_INSTALL_INCLUDEDIR "include" +SET(CMAKE_INSTALL_INCLUDEDIR "include/neat" CACHE PATH ${CMAKE_INSTALL_INCLUDEDIR_DOC}) -SET(CMAKE_INSTALL_MANDIR "share/man" +SET(CMAKE_INSTALL_MANDIR "share/man/neat" CACHE PATH ${CMAKE_INSTALL_MANDIR_DOC}) # PACKAGING @@ -47,7 +47,6 @@ include(CPack) add_custom_target(dist COMMAND ${CMAKE_MAKE_PROGRAM} clean package_source) - INCLUDE(CheckIncludeFile) INCLUDE(CheckStructHasMember) INCLUDE(CheckTypeSize) @@ -186,6 +185,8 @@ ENDIF() OPTION(OPENSSL_SUPPORT "Include support for OpenSSL" 1) +OPTION(SOCKET_API "Include the socket API" 0) + # REQUIREMENTS ################################################# CHECK_INCLUDE_FILE(uv.h HAVE_SYS_UV_H) @@ -335,6 +336,8 @@ INSTALL(FILES ${neat_headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) # INCLUDE EXAMPLES AND TESTS FOLDER ################################################# -ADD_SUBDIRECTORY(socketapi) +IF (SOCKET_API) + ADD_SUBDIRECTORY(socketapi) +ENDIF() ADD_SUBDIRECTORY(examples) ADD_SUBDIRECTORY(tests) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 0d378f71..65fae0c8 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -36,9 +36,6 @@ FOREACH (source_file ${neat_programs}) util.c ) TARGET_LINK_LIBRARIES(${source_file_we} neat) - INSTALL(TARGETS ${source_file_we} - RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR}/libneat - BUNDLE DESTINATION ${CMAKE_INSTALL_LIBDIR}/libneat) ENDFOREACH () # COPY EXAMPLE PROPERTY FILES From 85d671c7db930c3899d2381b578b0c16c060287b Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 10 May 2017 15:13:45 +0200 Subject: [PATCH 42/85] revert CMAKE path changes --- CMakeLists.txt | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f6be76c..f36bc6c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,30 +9,30 @@ set(BUILD_VERSION ${BUILD_MAJOR}.${BUILD_MINOR}.${BUILD_PATCH}) # INSTALLATION_DIRECTORIES ################################################# -SET(CMAKE_INSTALL_BINDIR_DOC "user executables [PREFIX/bin/neat]") -SET(CMAKE_INSTALL_SBINDIR_DOC "system admin executables [EPREFIX/sbin/neat]") -SET(CMAKE_INSTALL_DATADIR_DOC "read-only architecture-independent data [PREFIX/share/neat]") -SET(CMAKE_INSTALL_SYSCONFDIR_DOC "read-only single-machine data [PREFIX/etc/neat]") -SET(CMAKE_INSTALL_LOCALSTATEDIR_DOC "modifiable single-machine data [PREFIX/var/neat]") -SET(CMAKE_INSTALL_LIBDIR_DOC "object code libraries [PREFIX/lib/neat]") -SET(CMAKE_INSTALL_INCLUDEDIR_DOC "read-only includes [PREFIX/include/neat]") -SET(CMAKE_INSTALL_MANDIR_DOC "man documentation [PREFIX/share/man/neat]") - -SET(CMAKE_INSTALL_BINDIR "bin/neat" +SET(CMAKE_INSTALL_BINDIR_DOC "user executables [PREFIX/bin]") +SET(CMAKE_INSTALL_SBINDIR_DOC "system admin executables [EPREFIX/sbin]") +SET(CMAKE_INSTALL_DATADIR_DOC "read-only architecture-independent data [PREFIX/share]") +SET(CMAKE_INSTALL_SYSCONFDIR_DOC "read-only single-machine data [PREFIX/etc]") +SET(CMAKE_INSTALL_LOCALSTATEDIR_DOC "modifiable single-machine data [PREFIX/var]") +SET(CMAKE_INSTALL_LIBDIR_DOC "object code libraries [PREFIX/lib]") +SET(CMAKE_INSTALL_INCLUDEDIR_DOC "read-only includes [PREFIX/include]") +SET(CMAKE_INSTALL_MANDIR_DOC "man documentation [PREFIX/share/man]") + +SET(CMAKE_INSTALL_BINDIR "bin" CACHE PATH ${CMAKE_INSTALL_BINDIR_DOC}) -SET(CMAKE_INSTALL_SBINDIR "sbin/neat" +SET(CMAKE_INSTALL_SBINDIR "sbin" CACHE PATH ${CMAKE_INSTALL_SBINDIR_DOC}) -SET(CMAKE_INSTALL_DATADIR "share/neat" +SET(CMAKE_INSTALL_DATADIR "share" CACHE PATH ${CMAKE_INSTALL_DATADIR_DOC}) -SET(CMAKE_INSTALL_SYSCONFDIR "etc/neat" +SET(CMAKE_INSTALL_SYSCONFDIR "etc" CACHE PATH ${CMAKE_INSTALL_SYSCONFDIR_DOC}) -SET(CMAKE_INSTALL_LOCALSTATEDIR "var/neat" +SET(CMAKE_INSTALL_LOCALSTATEDIR "var" CACHE PATH ${CMAKE_INSTALL_LOCALSTATEDIR_DOC}) -SET(CMAKE_INSTALL_LIBDIR "lib/neat" +SET(CMAKE_INSTALL_LIBDIR "lib" CACHE PATH ${CMAKE_INSTALL_LIBDIR_DOC}) -SET(CMAKE_INSTALL_INCLUDEDIR "include/neat" +SET(CMAKE_INSTALL_INCLUDEDIR "include" CACHE PATH ${CMAKE_INSTALL_INCLUDEDIR_DOC}) -SET(CMAKE_INSTALL_MANDIR "share/man/neat" +SET(CMAKE_INSTALL_MANDIR "share/man" CACHE PATH ${CMAKE_INSTALL_MANDIR_DOC}) # PACKAGING From 254ed55a0a77231252d1719cc070a7e40191b488 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 10 May 2017 15:54:03 +0200 Subject: [PATCH 43/85] message boundary preservation added --- examples/CMakeLists.txt | 2 + examples/client_http_get.c | 4 ++ examples/prop_message_mode.json | 20 ++++++ examples/prop_streaming_mode.json | 20 ++++++ neat_core.c | 104 +++++++++++++++++++++--------- neat_internal.h | 28 ++++---- neat_resolver.c | 5 +- policy/pib/example/mbp.policy | 17 +++++ 8 files changed, 156 insertions(+), 44 deletions(-) create mode 100644 examples/prop_message_mode.json create mode 100644 examples/prop_streaming_mode.json create mode 100644 policy/pib/example/mbp.policy diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 0d378f71..49f2c292 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -24,6 +24,8 @@ LIST(APPEND neat_property_examples prop_sctp_multihomed.json prop_tcp.json prop_tcp_security.json + prop_streaming_mode.json + prop_message_mode.json ) # BUILD EACH PROGRAM diff --git a/examples/client_http_get.c b/examples/client_http_get.c index f608cb81..c69f4737 100644 --- a/examples/client_http_get.c +++ b/examples/client_http_get.c @@ -56,6 +56,10 @@ static char *config_property = "\ \"multihoming\": {\ \"value\": true,\ \"precedence\": 1\ + },\ + \"transport_type\": {\ + \"value\": \"message\",\ + \"precedence\": 1\ }\ }"; static unsigned char *buffer = NULL; diff --git a/examples/prop_message_mode.json b/examples/prop_message_mode.json new file mode 100644 index 00000000..bebb5f2b --- /dev/null +++ b/examples/prop_message_mode.json @@ -0,0 +1,20 @@ +{ + "transport": [ + { + "value": "SCTP", + "precedence": 1 + }, + { + "value": "SCTP/UDP", + "precedence": 1 + }, + { + "value": "TCP", + "precedence": 1 + } + ], + "transport_type": { + "value": "message", + "precedence": 1 + } +} diff --git a/examples/prop_streaming_mode.json b/examples/prop_streaming_mode.json new file mode 100644 index 00000000..dc1b2b98 --- /dev/null +++ b/examples/prop_streaming_mode.json @@ -0,0 +1,20 @@ +{ + "transport": [ + { + "value": "SCTP", + "precedence": 1 + }, + { + "value": "SCTP/UDP", + "precedence": 1 + }, + { + "value": "TCP", + "precedence": 1 + } + ], + "transport_type": { + "value": "stream", + "precedence": 1 + } +} diff --git a/neat_core.c b/neat_core.c index cd8d0447..a25b3858 100644 --- a/neat_core.c +++ b/neat_core.c @@ -1213,6 +1213,7 @@ handle_sctp_event(neat_flow *flow, union sctp_notification *notfn) break; case SCTP_SHUTDOWN_EVENT: neat_log(ctx, NEAT_LOG_DEBUG, "Got SCTP shutdown event"); + flow->eofSeen = 1; return READ_WITH_ZERO; break; case SCTP_ADAPTATION_INDICATION: @@ -1249,7 +1250,7 @@ handle_sctp_event(neat_flow *flow, union sctp_notification *notfn) } #endif // defined(HAVE_NETINET_SCTP_H) || defined(USRSCTP_SUPPORT) -int +static int resize_read_buffer(neat_flow *flow) { ssize_t spaceFree; @@ -1496,10 +1497,6 @@ io_readable(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *socket, msghdr.msg_flags = 0; -#ifdef MSG_NOTIFICATION - msghdr.msg_flags |= MSG_NOTIFICATION; -#endif // MSG_NOTIFICATION - if ((n = recvmsg(socket->fd, &msghdr, 0)) < 0) { #ifdef SCTP_MULTISTREAMING if (multistream_buffer) { @@ -1510,6 +1507,12 @@ io_readable(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *socket, return READ_WITH_ERROR; } + // felix XXX polish me! + if (n == 0) { + flow->eofSeen = 1; + } + + #if (defined(SCTP_RCVINFO) || defined (SCTP_SNDRCV)) for (cmsg = CMSG_FIRSTHDR(&msghdr); cmsg != NULL; cmsg = CMSG_NXTHDR(&msghdr, cmsg)) { if (cmsg->cmsg_len == 0) { @@ -1698,7 +1701,7 @@ io_readable(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *socket, flow->readBufferMsgComplete = 1; } - if (!flow->readBufferMsgComplete) { + if (!flow->readBufferMsgComplete && flow->preserveMessageBoundaries) { neat_log(ctx, NEAT_LOG_WARNING, "%s - READ_WITH_ERROR 12", __func__); return READ_WITH_ERROR; } @@ -2517,11 +2520,12 @@ do_accept(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *listen_so neat_log(ctx, NEAT_LOG_INFO, "%s - write_size %d - read_size %d", __func__, listen_socket->write_size, listen_socket->read_size); - newFlow->ctx = ctx; - newFlow->ownedByCore = 1; - newFlow->isServer = 1; - newFlow->isSCTPMultihoming = flow->isSCTPMultihoming; - newFlow->security_needed = flow->security_needed; + newFlow->ctx = ctx; + newFlow->ownedByCore = 1; + newFlow->isServer = 1; + newFlow->isSCTPMultihoming = flow->isSCTPMultihoming; + newFlow->security_needed = flow->security_needed; + newFlow->eofSeen = 0; newFlow->operations = calloc (sizeof(struct neat_flow_operations), 1); if (newFlow->operations == NULL) { @@ -3423,6 +3427,11 @@ open_resolve_cb(struct neat_resolver_results *results, uint8_t code, for (unsigned int i = 0; i < nr_of_stacks; ++i) { // struct neat_he_candidate *tmp; + if (flow->preserveMessageBoundaries && + (neat_base_stack(stacks[i]) != NEAT_STACK_SCTP && stacks[i] != NEAT_STACK_UDP && stacks[i] != NEAT_STACK_UDPLITE)) { + continue; + } + struct neat_he_candidate *candidate = calloc(1, sizeof(*candidate)); if (!candidate) return NEAT_ERROR_OUT_OF_MEMORY; @@ -4077,7 +4086,10 @@ neat_open(neat_ctx *ctx, neat_flow *flow, const char *name, uint16_t port, int group = 0; float priority = 0.5f; const char *cc_algorithm = NULL; - json_t *multihoming = NULL, *val = NULL, *security = NULL; + json_t *multihoming = NULL; + json_t *val = NULL; + json_t *security = NULL; + json_t *transport_type = NULL; neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); @@ -4104,12 +4116,16 @@ neat_open(neat_ctx *ctx, neat_flow *flow, const char *name, uint16_t port, } flow->name = strdup(name); - if (flow->name == NULL) + if (flow->name == NULL) { return NEAT_ERROR_OUT_OF_MEMORY; - flow->port = port; + } + + flow->port = port; //flow->stream_count = stream_count; - flow->group = group; - flow->priority = priority; + flow->group = group; + flow->priority = priority; + flow->eofSeen = 0; + if ((multihoming = json_object_get(flow->properties, "multihoming")) != NULL && (val = json_object_get(multihoming, "value")) != NULL && json_typeof(val) == JSON_TRUE) @@ -4119,6 +4135,15 @@ neat_open(neat_ctx *ctx, neat_flow *flow, const char *name, uint16_t port, flow->isSCTPMultihoming = 0; } + if ((transport_type = json_object_get(flow->properties, "transport_type")) != NULL && + (val = json_object_get(transport_type, "value")) != NULL && + !strcmp(json_string_value(val), "message")) { + flow->preserveMessageBoundaries = 1; + } else { + flow->preserveMessageBoundaries = 0; + } + + if ((security = json_object_get(flow->properties, "security")) != NULL && (val = json_object_get(security, "value")) != NULL && json_typeof(val) == JSON_TRUE) @@ -5111,21 +5136,42 @@ neat_read_from_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, assert(false); #endif // SCTP_MULTISTREAMING - } else { - if (!flow->readBufferMsgComplete) { - return NEAT_ERROR_WOULD_BLOCK; - } - if (flow->readBufferSize > amt) { - neat_log(ctx, NEAT_LOG_DEBUG, "%s: Message too big", __func__); - return NEAT_ERROR_MESSAGE_TOO_BIG; + } else { + if (flow->preserveMessageBoundaries) { + if (!flow->readBufferMsgComplete) { + return NEAT_ERROR_WOULD_BLOCK; + } + if (flow->readBufferSize > amt) { + neat_log(ctx, NEAT_LOG_DEBUG, "%s: Message too big", __func__); + return NEAT_ERROR_MESSAGE_TOO_BIG; + } + } else if (flow->readBufferSize == 0) { + neat_log(ctx, NEAT_LOG_DEBUG, "%s nothing scheduled", __func__); + if (flow->eofSeen) { + flow->eofSeen = 0; + printf("eofSeen: return NEAT_OK\n"); + return NEAT_OK; + } else { + return NEAT_ERROR_WOULD_BLOCK; + } + } + + assert(flow->readBuffer); + if (flow->readBufferSize > amt) { + /* this can only happen if message boundaries are not preserved */ + *actualAmt = amt; + memcpy(buffer, flow->readBuffer, amt); + /* This is very inefficient, we should also use a offset */ + memmove(flow->readBuffer, flow->readBuffer + amt, flow->readBufferSize - amt); + flow->readBufferSize -= amt; + } else { + *actualAmt = flow->readBufferSize; + memcpy(buffer, flow->readBuffer, flow->readBufferSize); + flow->readBufferSize = 0; + flow->readBufferMsgComplete = 0; + } } - assert(flow->readBuffer); - memcpy(buffer, flow->readBuffer, flow->readBufferSize); - *actualAmt = flow->readBufferSize; - flow->readBufferSize = 0; - flow->readBufferMsgComplete = 0; - } goto end; } diff --git a/neat_internal.h b/neat_internal.h index 0aef7490..ed19bc6d 100644 --- a/neat_internal.h +++ b/neat_internal.h @@ -302,19 +302,21 @@ struct neat_flow neat_accept_usrsctp_impl acceptusrsctpfx; #endif - unsigned int hefirstConnect : 1; - unsigned int firstWritePending : 1; - unsigned int acceptPending : 1; - unsigned int isPolling : 1; - unsigned int ownedByCore : 1; - unsigned int everConnected : 1; - unsigned int isDraining : 1; - unsigned int isServer : 1; // i.e. created via accept() - unsigned int isSCTPMultihoming : 1; - unsigned int security_needed : 1; - unsigned int isSCTPIdata : 1; - unsigned int isClosing : 1; - unsigned int notifyDrainPending : 1; + unsigned int hefirstConnect : 1; + unsigned int firstWritePending : 1; + unsigned int acceptPending : 1; + unsigned int isPolling : 1; + unsigned int ownedByCore : 1; + unsigned int everConnected : 1; + unsigned int isDraining : 1; + unsigned int isServer : 1; // i.e. created via accept() + unsigned int isSCTPMultihoming : 1; + unsigned int security_needed : 1; + unsigned int isSCTPIdata : 1; + unsigned int isClosing : 1; + unsigned int notifyDrainPending : 1; + unsigned int preserveMessageBoundaries : 1; + unsigned int eofSeen : 1; unsigned int streams_requested; diff --git a/neat_resolver.c b/neat_resolver.c index e12aedbe..d650a216 100644 --- a/neat_resolver.c +++ b/neat_resolver.c @@ -874,8 +874,9 @@ neat_resolve(struct neat_resolver *resolver, } request = calloc(sizeof(struct neat_resolver_request), 1); - if (!request) - return RETVAL_FAILURE; + if (!request) { + return RETVAL_FAILURE; + } request->family = family; request->dst_port = htons(port); diff --git a/policy/pib/example/mbp.policy b/policy/pib/example/mbp.policy new file mode 100644 index 00000000..5c398bf2 --- /dev/null +++ b/policy/pib/example/mbp.policy @@ -0,0 +1,17 @@ +{ + "uid":"mbp_transport", + "description":"select a transport protocol that preserves message boundaries", + "priority": 2, + "replace_matched": false, + "match":{ + "transport_type": { + "value": "message" + } + }, + "properties":{ + "transport": { + "value": ["SCTP", "SCTP/UDP", "UDP", UDPLite], + "precedence": 2 + } + } +} From c9f68d525ac6ef85f25b6cb7ea0200400110250f Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 10 May 2017 16:15:08 +0200 Subject: [PATCH 44/85] do not require message preserving for HTTP --- examples/client_http_get.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/client_http_get.c b/examples/client_http_get.c index c69f4737..f608cb81 100644 --- a/examples/client_http_get.c +++ b/examples/client_http_get.c @@ -56,10 +56,6 @@ static char *config_property = "\ \"multihoming\": {\ \"value\": true,\ \"precedence\": 1\ - },\ - \"transport_type\": {\ - \"value\": \"message\",\ - \"precedence\": 1\ }\ }"; static unsigned char *buffer = NULL; From 3516f9e657967426e75d9e861001e2dcf9c59ae4 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 10 May 2017 23:44:01 +0200 Subject: [PATCH 45/85] improve usrsctp support --- neat_core.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/neat_core.c b/neat_core.c index a25b3858..be1311fc 100644 --- a/neat_core.c +++ b/neat_core.c @@ -1214,6 +1214,7 @@ handle_sctp_event(neat_flow *flow, union sctp_notification *notfn) case SCTP_SHUTDOWN_EVENT: neat_log(ctx, NEAT_LOG_DEBUG, "Got SCTP shutdown event"); flow->eofSeen = 1; + neat_notify_close(flow); return READ_WITH_ZERO; break; case SCTP_ADAPTATION_INDICATION: @@ -1593,10 +1594,10 @@ io_readable(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *socket, sctp_event_ret = handle_sctp_event(flow, (union sctp_notification*)(multistream_buffer)); free(multistream_buffer); } else { - sctp_event_ret = handle_sctp_event(flow, (union sctp_notification*)(flow->readBuffer+ flow->readBufferSize)); + sctp_event_ret = handle_sctp_event(flow, (union sctp_notification*)(flow->readBuffer + flow->readBufferSize)); } #else // SCTP_MULTISTREAM - sctp_event_ret = handle_sctp_event(flow, (union sctp_notification*)(flow->readBuffer+ flow->readBufferSize)); + sctp_event_ret = handle_sctp_event(flow, (union sctp_notification*)(flow->readBuffer + flow->readBufferSize)); #endif // SCTP_MULTISTREAM // We don't update readBufferSize, so buffer is implicitly "freed" @@ -6497,7 +6498,7 @@ handle_upcall(struct socket *sock, void *arg, int flags) code = io_readable(ctx, flow, pollable_socket, NEAT_OK); } while (code == READ_OK && flow->state == NEAT_FLOW_OPEN); - if (code == READ_WITH_ZERO && flow->operations && flow->operations->on_readable) { + if (code == READ_WITH_ZERO && flow->operations && flow->operations->on_readable && flow->state == NEAT_FLOW_OPEN) { flow->operations->on_readable(flow->operations); } } else { @@ -6507,7 +6508,7 @@ handle_upcall(struct socket *sock, void *arg, int flags) // xxx why two times? events = usrsctp_get_events(sock); - if (events & SCTP_EVENT_WRITE && flow->operations->on_writable) { + if (events & SCTP_EVENT_WRITE && flow->operations->on_writable && flow->state == NEAT_FLOW_OPEN) { io_writable(ctx, flow, NEAT_OK); } From 53ff18e51a035c9219badfd62e1bb4ab79270603 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 10 May 2017 23:56:44 +0200 Subject: [PATCH 46/85] address use after free bug --- neat_core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neat_core.c b/neat_core.c index be1311fc..64c82a82 100644 --- a/neat_core.c +++ b/neat_core.c @@ -1601,7 +1601,8 @@ io_readable(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *socket, #endif // SCTP_MULTISTREAM // We don't update readBufferSize, so buffer is implicitly "freed" - if (sctp_event_ret == READ_WITH_ZERO) { + if (sctp_event_ret == READ_WITH_ZERO && flow->state == NEAT_FLOW_OPEN) { + assert(flow); flow->readBufferMsgComplete = 1; } From 87796e76d6895b65b46e39e574d000e31fa008b7 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Thu, 11 May 2017 00:03:26 +0200 Subject: [PATCH 47/85] address use after free - second try --- neat_core.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/neat_core.c b/neat_core.c index 64c82a82..17e72cdf 100644 --- a/neat_core.c +++ b/neat_core.c @@ -1213,7 +1213,8 @@ handle_sctp_event(neat_flow *flow, union sctp_notification *notfn) break; case SCTP_SHUTDOWN_EVENT: neat_log(ctx, NEAT_LOG_DEBUG, "Got SCTP shutdown event"); - flow->eofSeen = 1; + flow->eofSeen = 1; + flow->readBufferMsgComplete = 1; neat_notify_close(flow); return READ_WITH_ZERO; break; @@ -1600,12 +1601,6 @@ io_readable(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *socket, sctp_event_ret = handle_sctp_event(flow, (union sctp_notification*)(flow->readBuffer + flow->readBufferSize)); #endif // SCTP_MULTISTREAM - // We don't update readBufferSize, so buffer is implicitly "freed" - if (sctp_event_ret == READ_WITH_ZERO && flow->state == NEAT_FLOW_OPEN) { - assert(flow); - flow->readBufferMsgComplete = 1; - } - return sctp_event_ret; } From dabb8a6c8a7e0ca1e0b397a54a877063cbdf971f Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Thu, 11 May 2017 00:29:27 +0200 Subject: [PATCH 48/85] examples updated --- examples/client.c | 25 +++++------------ examples/server_chargen.c | 43 +++++++++++++++-------------- examples/server_daytime.c | 28 +++++++------------ examples/server_discard.c | 25 +++++++---------- examples/server_echo.c | 57 +++++++++++++++++++-------------------- 5 files changed, 74 insertions(+), 104 deletions(-) diff --git a/examples/client.c b/examples/client.c index 5bdd508e..1ced1d6a 100644 --- a/examples/client.c +++ b/examples/client.c @@ -183,20 +183,12 @@ on_readable(struct neat_flow_operations *opCB) } } - // all fine - if (buffer_filled > 0) { - if (config_log_level >= 1) { - fprintf(stderr, "%s - received %d bytes on stream id %d\n", __func__, buffer_filled, opCB->stream_id); - } - fwrite(buffer_rcv, sizeof(char), buffer_filled, stdout); - fflush(stdout); - } else { - fprintf(stderr, "%s - nothing more to read\n", __func__); - ops.on_readable = NULL; - neat_set_operations(opCB->ctx, opCB->flow, &ops); - neat_close(opCB->ctx, opCB->flow); + if (config_log_level >= 1) { + fprintf(stderr, "%s - received %d bytes on stream id %d\n", __func__, buffer_filled, opCB->stream_id); } + fwrite(buffer_rcv, sizeof(char), buffer_filled, stdout); + fflush(stdout); return NEAT_OK; } @@ -255,12 +247,6 @@ on_connected(struct neat_flow_operations *opCB) int rc; uv_loop_t *loop; - /* - if (config_log_level >= 1) { - printf("%s - available streams : %d\n", __func__, opCB->flow->stream_count); - } - */ - last_stream = 0; loop = neat_get_event_loop(opCB->ctx); @@ -281,8 +267,9 @@ on_connected(struct neat_flow_operations *opCB) } } - if (config_timeout) + if (config_timeout) { neat_change_timeout(opCB->ctx, opCB->flow, config_timeout); + } return NEAT_OK; } diff --git a/examples/server_chargen.c b/examples/server_chargen.c index 5761e530..f78c4089 100644 --- a/examples/server_chargen.c +++ b/examples/server_chargen.c @@ -33,8 +33,8 @@ static char *config_property = "{\ }\ ]\ }"; -static uint16_t config_log_level = 1; -static uint16_t chargen_offset = 0; +static uint16_t config_log_level = 1; +static uint16_t chargen_offset = 0; #define BUFFERSIZE 32 @@ -90,23 +90,14 @@ static neat_error_code on_readable(struct neat_flow_operations *opCB) } } - if (buffer_filled > 0) { - if (config_log_level >= 1) { - printf("data received - %d byte\n", buffer_filled); - } - if (config_log_level >= 2) { - fwrite(buffer, sizeof(char), buffer_filled, stdout); - printf("\n"); - fflush(stdout); - } - } else { // peer disconnected - if (config_log_level >= 1) { - printf("peer disconnected\n"); - } - opCB->on_readable = NULL; - opCB->on_writable = NULL; - opCB->on_all_written = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); + + if (config_log_level >= 1) { + printf("data received - %d byte\n", buffer_filled); + } + if (config_log_level >= 2) { + fwrite(buffer, sizeof(char), buffer_filled, stdout); + printf("\n"); + fflush(stdout); } return NEAT_OK; @@ -125,9 +116,7 @@ on_all_written(struct neat_flow_operations *opCB) return NEAT_OK; } -/* - //XXX behave more like specified in the rfc (UDP, TCP) -*/ + static neat_error_code on_writable(struct neat_flow_operations *opCB) { @@ -161,6 +150,13 @@ on_writable(struct neat_flow_operations *opCB) return NEAT_OK; } +static neat_error_code +on_close(struct neat_flow_operations *opCB) +{ + fprintf(stderr, "%s - flow closed OK!\n", __func__); + return NEAT_OK; +} + static neat_error_code on_connected(struct neat_flow_operations *opCB) @@ -176,11 +172,14 @@ on_connected(struct neat_flow_operations *opCB) opCB->on_readable = on_readable; opCB->on_writable = on_writable; opCB->on_all_written = NULL; + opCB->on_close = on_close; neat_set_operations(opCB->ctx, opCB->flow, opCB); return NEAT_OK; } + + int main(int argc, char *argv[]) { diff --git a/examples/server_daytime.c b/examples/server_daytime.c index 39681c78..9124b1eb 100644 --- a/examples/server_daytime.c +++ b/examples/server_daytime.c @@ -90,24 +90,16 @@ on_readable(struct neat_flow_operations *opCB) } } - if (buffer_filled > 0) { - if (config_log_level >= 1) { - printf("received data - %d byte\n", buffer_filled); - } - if (config_log_level >= 2) { - fwrite(buffer, sizeof(char), buffer_filled, stdout); - printf("\n"); - fflush(stdout); - } - } else { // peer disconnected - if (config_log_level >= 1) { - printf("peer disconnected\n"); - } - opCB->on_readable = NULL; - opCB->on_writable = NULL; - opCB->on_all_written = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); + + if (config_log_level >= 1) { + printf("received data - %d byte\n", buffer_filled); + } + if (config_log_level >= 2) { + fwrite(buffer, sizeof(char), buffer_filled, stdout); + printf("\n"); + fflush(stdout); } + return NEAT_OK; } @@ -121,7 +113,7 @@ on_all_written(struct neat_flow_operations *opCB) opCB->on_writable = NULL; opCB->on_all_written = NULL; neat_set_operations(opCB->ctx, opCB->flow, opCB); - neat_shutdown(opCB->ctx, opCB->flow); + neat_close(opCB->ctx, opCB->flow); return NEAT_OK; } diff --git a/examples/server_discard.c b/examples/server_discard.c index 1bcdba14..3bb63f32 100644 --- a/examples/server_discard.c +++ b/examples/server_discard.c @@ -94,23 +94,16 @@ on_readable(struct neat_flow_operations *opCB) } } - if (buffer_filled > 0) { - if (config_log_level >= 1) { - printf("received data - %d byte\n", buffer_filled); - } - if (config_log_level >= 2) { - fwrite(buffer, sizeof(char), buffer_filled, stdout); - printf("\n"); - fflush(stdout); - } - } else { - if (config_log_level >= 1) { - printf("peer disconnected\n"); - } - opCB->on_readable = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); - neat_close(opCB->ctx, opCB->flow); + + if (config_log_level >= 1) { + printf("received data - %d byte\n", buffer_filled); + } + if (config_log_level >= 2) { + fwrite(buffer, sizeof(char), buffer_filled, stdout); + printf("\n"); + fflush(stdout); } + return NEAT_OK; } diff --git a/examples/server_echo.c b/examples/server_echo.c index 6aa95eb9..7e0da007 100644 --- a/examples/server_echo.c +++ b/examples/server_echo.c @@ -106,37 +106,35 @@ on_readable(struct neat_flow_operations *opCB) } } - // we got some data - if (ef->bytes > 0) { - if (config_log_level >= 1) { - printf("received data - %d bytes on stream %d\n", ef->bytes, opCB->stream_id); - } - if (config_log_level >= 2) { - fwrite(ef->buffer, sizeof(char), ef->bytes, stdout); - printf("\n"); - fflush(stdout); - } + if (config_log_level >= 1) { + printf("received data - %d bytes on stream %d\n", ef->bytes, opCB->stream_id); + } + if (config_log_level >= 2) { + fwrite(ef->buffer, sizeof(char), ef->bytes, stdout); + printf("\n"); + fflush(stdout); + } - // remember stream_id - ef->stream_id = opCB->stream_id; + // remember stream_id + ef->stream_id = opCB->stream_id; - // echo data - opCB->on_readable = NULL; - opCB->on_writable = on_writable; - opCB->on_all_written = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); - // peer disconnected - stop callbacks and free ressources - } else { - if (config_log_level >= 1) { - printf("peer disconnected\n"); - } - opCB->on_readable = NULL; - opCB->on_writable = NULL; - opCB->on_all_written = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); - free(ef->buffer); - free(ef); - } + // echo data + opCB->on_readable = NULL; + opCB->on_writable = on_writable; + opCB->on_all_written = NULL; + neat_set_operations(opCB->ctx, opCB->flow, opCB); + + return NEAT_OK; +} + +static neat_error_code +on_close(struct neat_flow_operations *opCB) +{ + struct echo_flow *ef = opCB->userData; + fprintf(stderr, "%s - flow closed OK!\n", __func__); + + free(ef->buffer); + free(ef); return NEAT_OK; } @@ -150,6 +148,7 @@ on_all_written(struct neat_flow_operations *opCB) opCB->on_readable = on_readable; opCB->on_writable = NULL; opCB->on_all_written = NULL; + opCB->on_close = on_close; neat_set_operations(opCB->ctx, opCB->flow, opCB); return NEAT_OK; } From f4d0cfed507cb186a1ae194ee5d9f0fe4f88aab6 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Thu, 11 May 2017 00:48:22 +0200 Subject: [PATCH 49/85] client improvements --- examples/client.c | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/examples/client.c b/examples/client.c index 1ced1d6a..7855b96a 100644 --- a/examples/client.c +++ b/examples/client.c @@ -319,35 +319,9 @@ tty_read(uv_stream_t *stream, ssize_t buffer_filled, const uv_buf_t *buffer) if (!uv_is_closing((uv_handle_t*) &tty)) { uv_close((uv_handle_t*) &tty, NULL); } - neat_shutdown(ctx, flow); - } else if (strncmp(buffer->base, "close\n", buffer_filled) == 0) { - if (config_log_level >= 1) { - fprintf(stderr, "%s - tty_read - CLOSE\n", __func__); - } - uv_read_stop(stream); - ops.on_writable = NULL; - neat_set_operations(ctx, flow, &ops); - if (!uv_is_closing((uv_handle_t*) &tty)) { - uv_close((uv_handle_t*) &tty, NULL); - } neat_close(ctx, flow); - buffer_filled = UV_EOF; - } else if (strncmp(buffer->base, "abort\n", buffer_filled) == 0) { - if (config_log_level >= 1) { - fprintf(stderr, "%s - tty_read - ABORT\n", __func__); - } - uv_read_stop(stream); - ops.on_writable = NULL; - neat_set_operations(ctx, flow, &ops); - if (!uv_is_closing((uv_handle_t*) &tty)) { - uv_close((uv_handle_t*) &tty, NULL); - } - neat_abort(ctx, flow); - buffer_filled = UV_EOF; } - fprintf(stderr, "%s - felix - marker\n", __func__); - // all fine if (buffer_filled > 0 && buffer_filled != UV_EOF) { // copy input to app buffer From 444dbb3a98e700f18a3cd80be2cba017fa78ad3a Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Thu, 11 May 2017 00:50:02 +0200 Subject: [PATCH 50/85] docs updated --- docs/neat_close.md | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/docs/neat_close.md b/docs/neat_close.md index eb879654..34d3b273 100644 --- a/docs/neat_close.md +++ b/docs/neat_close.md @@ -1,15 +1,6 @@ # neat_close +Initiates the closing procedure for a flow. -Close this flow and free all associated data. - -This is the *last call*: -* all flow related data (e.g. `userData`) should be freed in advance -* no flow related *callbacks* are fired -* all data in the *receive buffer* gets discarded -* all data in the *send buffer* will be transmitted -* initiates a *graceful connection shutdown* - -**This function must always the be called to free ressources** ### Syntax From 681b786f7542f2daac29d1de94be918b29db81d1 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Fri, 12 May 2017 01:02:10 +0200 Subject: [PATCH 51/85] improve closing procedure for usrsctp --- neat_core.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/neat_core.c b/neat_core.c index 17e72cdf..eaea9bb1 100644 --- a/neat_core.c +++ b/neat_core.c @@ -1215,7 +1215,7 @@ handle_sctp_event(neat_flow *flow, union sctp_notification *notfn) neat_log(ctx, NEAT_LOG_DEBUG, "Got SCTP shutdown event"); flow->eofSeen = 1; flow->readBufferMsgComplete = 1; - neat_notify_close(flow); + //neat_notify_close(flow); return READ_WITH_ZERO; break; case SCTP_ADAPTATION_INDICATION: @@ -1571,8 +1571,10 @@ io_readable(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *socket, &infotype, &(msghdr.msg_flags)); if (n < 0) { - /* if (errno == EAGAIN) - return READ_OK;*/ + if (errno == EAGAIN) { + //return READ_OK; + neat_log(ctx, NEAT_LOG_WARNING, "%s - usrsctp_recvv error EAGAIN", __func__); + } neat_log(ctx, NEAT_LOG_WARNING, "%s - READ_WITH_ERROR 10 - usrsctp_recvv error", __func__); return READ_WITH_ERROR; } @@ -6488,14 +6490,15 @@ handle_upcall(struct socket *sock, void *arg, int flags) } - if (events & SCTP_EVENT_READ && flow->operations->on_readable) { + if (events & SCTP_EVENT_READ) { if (flow->state == NEAT_FLOW_OPEN) { do { code = io_readable(ctx, flow, pollable_socket, NEAT_OK); - } while (code == READ_OK && flow->state == NEAT_FLOW_OPEN); + } while (code == READ_OK); - if (code == READ_WITH_ZERO && flow->operations && flow->operations->on_readable && flow->state == NEAT_FLOW_OPEN) { - flow->operations->on_readable(flow->operations); + if (code == READ_WITH_ZERO) { + neat_notify_close(flow); + return; } } else { neat_log(ctx, NEAT_LOG_WARNING, "%s - io_readable and flow in wrong state"); From b3add11d531d153a6cde9da16a5c8e5eaf2a96bd Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 17 May 2017 18:43:40 +0200 Subject: [PATCH 52/85] close socket for unsused candidates --- neat_core.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/neat_core.c b/neat_core.c index 7d22a286..6094b5e7 100644 --- a/neat_core.c +++ b/neat_core.c @@ -254,7 +254,7 @@ static void neat_walk_cb(uv_handle_t *handle, void *ctx) // Bug fix (karlgrin, 170323). if ((handle == NULL) || (handle->data == NULL)) return; - + if (!uv_is_closing(handle)) { // If this assert triggers, then some handle is not being closed // correctly. A handle with a data pointer should already be closed @@ -415,8 +415,8 @@ uint8_t neat_remove_event_cb(struct neat_ctx *nc, uint8_t event_type, return RETVAL_SUCCESS; } -void neat_run_event_cb(struct neat_ctx *nc, uint8_t event_type, - void *data) +void +neat_run_event_cb(struct neat_ctx *nc, uint8_t event_type, void *data) { struct neat_event_cbs *cb_list_head = NULL; struct neat_event_cb *cb_itr = NULL; @@ -2221,6 +2221,7 @@ he_connected_cb(uv_poll_t *handle, int status, int events) send_result_connection_attempt_to_pm(flow->ctx, flow, he_res, false); } + close(candidate->pollable_socket->fd); uv_poll_stop(handle); uv_close((uv_handle_t*)handle, free_he_handle_cb); From ca160d55d9b7d8d556bede515a1a441734358789 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 17 May 2017 22:50:53 +0200 Subject: [PATCH 53/85] minor refactoring --- neat_core.c | 14 +++++++------- tests/run.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/neat_core.c b/neat_core.c index 6094b5e7..8ed38b7e 100644 --- a/neat_core.c +++ b/neat_core.c @@ -5292,7 +5292,8 @@ neat_connect(struct neat_he_candidate *candidate, uv_poll_cb callback_fx) #ifdef TCP_CONGESTION const char *algo; #endif - int enable = 1, retval; + int enable = 1; + int retval; socklen_t len = 0; int size = 0, protocol; #ifdef __linux__ @@ -5455,8 +5456,7 @@ neat_connect(struct neat_he_candidate *candidate, uv_poll_cb callback_fx) candidate->pollable_socket->write_size = 0; } len = (socklen_t)sizeof(int); - if (getsockopt(candidate->pollable_socket->fd, - SOL_SOCKET, SO_RCVBUF, &size, &len) == 0) { + if (getsockopt(candidate->pollable_socket->fd, SOL_SOCKET, SO_RCVBUF, &size, &len) == 0) { candidate->pollable_socket->read_size = size; } else { candidate->pollable_socket->read_size = 0; @@ -5465,19 +5465,19 @@ neat_connect(struct neat_he_candidate *candidate, uv_poll_cb callback_fx) switch (candidate->pollable_socket->stack) { case NEAT_STACK_TCP: if (setsockopt(candidate->pollable_socket->fd, IPPROTO_TCP, TCP_NODELAY, &enable, sizeof(enable) < 0)) { - neat_log(ctx, NEAT_LOG_DEBUG, "Call to setsockopt(TCP_NODELAY) failed"); + neat_log(ctx, NEAT_LOG_WARNING, "%s - Call to setsockopt(TCP_NODELAY) failed", __func__); } #if defined(__FreeBSD__) && defined(FLOW_GROUPS) group = candidate->pollable_socket->flow->group; if (setsockopt(candidate->pollable_socket->fd, IPPROTO_TCP, 8192 /* Group ID */, &group, sizeof(group)) != 0) { - neat_log(ctx, NEAT_LOG_DEBUG, "Unable to set flow group: %s", strerror(errno)); + neat_log(ctx, NEAT_LOG_WARNING, "%s - Unable to set flow group: %s", __func__, strerror(errno)); } // Map the priority range to some integer range prio = candidate->pollable_socket->flow->priority * 255; if (setsockopt(candidate->pollable_socket->fd, IPPROTO_TCP, 4096 /* Priority */, &prio, sizeof(prio)) != 0) { - neat_log(ctx, NEAT_LOG_DEBUG, "Unable to set flow priority: %s", strerror(errno)); + neat_log(ctx, NEAT_LOG_WARNING, "%s - Unable to set flow priority: %s", __func__, strerror(errno)); } #endif @@ -5485,7 +5485,7 @@ neat_connect(struct neat_he_candidate *candidate, uv_poll_cb callback_fx) if (candidate->pollable_socket->flow->cc_algorithm) { algo = candidate->pollable_socket->flow->cc_algorithm; if (setsockopt(candidate->pollable_socket->fd, IPPROTO_TCP, TCP_CONGESTION, algo, strlen(algo)) != 0) { - neat_log(ctx, NEAT_LOG_DEBUG, "Unable to set CC algorithm: %s", strerror(errno)); + neat_log(ctx, NEAT_LOG_WARNING, "%s - Unable to set CC algorithm: %s", __func__, strerror(errno)); } } #endif diff --git a/tests/run.py b/tests/run.py index f0d88837..3316fc61 100755 --- a/tests/run.py +++ b/tests/run.py @@ -21,7 +21,7 @@ tests_general.append([1, 0, workdir + 'client_http_get -u /cgi-bin/he -v 2 buildbot.nplab.de']) tests_general.append([0, 0, workdir + 'client_http_get -n 2 -u /files/4M bsd10.nplab.de']) if (platform.system() == "FreeBSD") or (platform.system() == "Linux"): - tests_general.append([1, 0, workdir + 'client_http_get -P ' + workdir + 'prop_tcp_security.json -p 443 -v 2 www.neat-project.org']) + tests_general.append([1, 0, workdir + 'client_http_get -P ' + workdir + 'prop_tcp_security.json -p 443 -v 2 ec.europa.eu']) tests_general.append([0, 0, workdir + 'tneat -v 2 -P ' + workdir + 'prop_sctp_dtls.json interop.fh-muenster.de']) #tests_general.append([0, 0, 'python3.5 ../../policy/pmtests.py']) From dcaa48262e7fcc988740e237e0cf03c1987286bc Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Thu, 18 May 2017 14:07:59 +0200 Subject: [PATCH 54/85] reduce timeout span and improve closing procedure --- neat_core.c | 17 +++++++++++++---- neat_resolver.h | 4 ++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/neat_core.c b/neat_core.c index 8ed38b7e..d55883e2 100644 --- a/neat_core.c +++ b/neat_core.c @@ -495,6 +495,8 @@ on_handle_closed_candidate(uv_handle_t *handle) void neat_free_candidate(struct neat_ctx *ctx, struct neat_he_candidate *candidate) { + struct neat_he_sockopt *sockopt; + struct neat_he_sockopt *tmp; neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); if (candidate == NULL) { @@ -508,11 +510,14 @@ neat_free_candidate(struct neat_ctx *ctx, struct neat_he_candidate *candidate) free(candidate->pollable_socket->dst_address); free(candidate->pollable_socket->src_address); + close(candidate->pollable_socket->fd); + if (!TAILQ_EMPTY(&(candidate->sock_opts))) { - struct neat_he_sockopt *sockopt, *tmp; + TAILQ_FOREACH_SAFE(sockopt, (&candidate->sock_opts), next, tmp) { - if (sockopt->type == NEAT_SOCKOPT_STRING) + if (sockopt->type == NEAT_SOCKOPT_STRING) { free(sockopt->value.s_val); + } TAILQ_REMOVE((&candidate->sock_opts), sockopt, next); } } @@ -553,12 +558,14 @@ neat_free_candidate(struct neat_ctx *ctx, struct neat_he_candidate *candidate) void neat_free_candidates(struct neat_ctx *ctx, struct neat_he_candidates *candidates) { - struct neat_he_candidate *candidate, *tmp; + struct neat_he_candidate *candidate; + struct neat_he_candidate *tmp; neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); - if (candidates == NULL) + if (candidates == NULL) { return; + } TAILQ_FOREACH_SAFE(candidate, candidates, next, tmp) { neat_free_candidate(ctx, candidate); @@ -721,6 +728,8 @@ neat_free_flow(neat_flow *flow) #endif neat_free_candidates(ctx, flow->candidate_list); + flow->candidate_list = NULL; + // close all listening sockets TAILQ_FOREACH_SAFE(listen_socket, &(flow->listen_sockets), next, listen_socket_temp) { diff --git a/neat_resolver.h b/neat_resolver.h index 01c18980..59daa3af 100644 --- a/neat_resolver.h +++ b/neat_resolver.h @@ -9,9 +9,9 @@ #include "neat_addr.h" //Timeout for complete DNS query -#define DNS_TIMEOUT 10000 +#define DNS_TIMEOUT 5000 //Timeout after first good reply -#define DNS_RESOLVED_TIMEOUT 1000 +#define DNS_RESOLVED_TIMEOUT 100 #define DNS_LITERAL_TIMEOUT 1 #define DNS_BUF_SIZE 1472 #define MAX_NUM_RESOLVED 3 From 083faee9a5f8debc0d74661306ecc61b9fdcf42b Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Thu, 18 May 2017 14:39:29 +0200 Subject: [PATCH 55/85] polish - no functional changes --- neat_resolver.c | 13 ++++--------- neat_resolver_helpers.c | 3 +-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/neat_resolver.c b/neat_resolver.c index d650a216..02f1cbcb 100644 --- a/neat_resolver.c +++ b/neat_resolver.c @@ -322,8 +322,7 @@ neat_resolver_populate_results(struct neat_resolver_request *request, } static void -neat_resolver_timeout_shared(uv_timer_t *handle, - uint8_t literal) +neat_resolver_timeout_shared(uv_timer_t *handle, uint8_t literal) { struct neat_resolver_request *request = handle->data; struct neat_resolver_results *result_list; @@ -811,15 +810,12 @@ neat_start_request(struct neat_resolver *resolver, //node is a literal, so we will just wait a short while for address list to //be populated if (is_literal) { - uv_timer_start(&(request->timeout_handle), - neat_resolver_literal_timeout_cb, - DNS_LITERAL_TIMEOUT, 0); + uv_timer_start(&(request->timeout_handle), neat_resolver_literal_timeout_cb, DNS_LITERAL_TIMEOUT, 0); return; } //Start the resolver timeout, this includes fetching addresses - uv_timer_start(&(request->timeout_handle), neat_resolver_timeout_cb, - resolver->dns_t1, 0); + uv_timer_start(&(request->timeout_handle), neat_resolver_timeout_cb, resolver->dns_t1, 0); //No point starting to query if we don't have any source addresses if (!resolver->nc->src_addr_cnt) { @@ -886,8 +882,7 @@ neat_resolve(struct neat_resolver *resolver, uv_timer_init(resolver->nc->loop, &(request->timeout_handle)); request->timeout_handle.data = request; - is_literal = neat_resolver_helpers_check_for_literal(&(request->family), - node); + is_literal = neat_resolver_helpers_check_for_literal(&(request->family), node); if (is_literal < 0) { free(request); diff --git a/neat_resolver_helpers.c b/neat_resolver_helpers.c index 853db8b2..8487d780 100644 --- a/neat_resolver_helpers.c +++ b/neat_resolver_helpers.c @@ -34,8 +34,7 @@ neat_resolver_helpers_addr_internal(struct sockaddr_storage *addr) //Check if node is an IP literal or not. Returns -1 on failure, 0 if not //literal, 1 if literal int8_t -neat_resolver_helpers_check_for_literal(uint8_t *family, - const char *node) +neat_resolver_helpers_check_for_literal(uint8_t *family, const char *node) { struct in6_addr dummy_addr; int32_t v4_literal = 0, v6_literal = 0; From 627da838acf68ad130092a60c000ea0c9a34e7cd Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Fri, 19 May 2017 12:25:43 +0200 Subject: [PATCH 56/85] http server refactoring - wip --- examples/CMakeLists.txt | 1 + examples/picohttpparser.c | 620 ++++++++++++++++++++++++++++++++++++++ examples/picohttpparser.h | 89 ++++++ examples/server_http.c | 251 ++++++++++++--- examples/util.c | 15 +- 5 files changed, 929 insertions(+), 47 deletions(-) create mode 100755 examples/picohttpparser.c create mode 100755 examples/picohttpparser.h diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 3d232ff0..d16899a7 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -36,6 +36,7 @@ FOREACH (source_file ${neat_programs}) ${source_file_we} ${source_file} util.c + picohttpparser.c ) TARGET_LINK_LIBRARIES(${source_file_we} neat) ENDFOREACH () diff --git a/examples/picohttpparser.c b/examples/picohttpparser.c new file mode 100755 index 00000000..0c319c98 --- /dev/null +++ b/examples/picohttpparser.c @@ -0,0 +1,620 @@ +/* + * Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase, + * Shigeo Mitsunari + * + * The software is licensed under either the MIT License (below) or the Perl + * license. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include +#include +#include +#ifdef __SSE4_2__ +#ifdef _MSC_VER +#include +#else +#include +#endif +#endif +#include "picohttpparser.h" + +/* $Id: a707070d11d499609f99d09f97535642cec910a8 $ */ + +#if __GNUC__ >= 3 +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif + +#ifdef _MSC_VER +#define ALIGNED(n) _declspec(align(n)) +#else +#define ALIGNED(n) __attribute__((aligned(n))) +#endif + +#define IS_PRINTABLE_ASCII(c) ((unsigned char)(c)-040u < 0137u) + +#define CHECK_EOF() \ + if (buf == buf_end) { \ + *ret = -2; \ + return NULL; \ + } + +#define EXPECT_CHAR_NO_CHECK(ch) \ + if (*buf++ != ch) { \ + *ret = -1; \ + return NULL; \ + } + +#define EXPECT_CHAR(ch) \ + CHECK_EOF(); \ + EXPECT_CHAR_NO_CHECK(ch); + +#define ADVANCE_TOKEN(tok, toklen) \ + do { \ + const char *tok_start = buf; \ + static const char ALIGNED(16) ranges2[] = "\000\040\177\177"; \ + int found2; \ + buf = findchar_fast(buf, buf_end, ranges2, sizeof(ranges2) - 1, &found2); \ + if (!found2) { \ + CHECK_EOF(); \ + } \ + while (1) { \ + if (*buf == ' ') { \ + break; \ + } else if (unlikely(!IS_PRINTABLE_ASCII(*buf))) { \ + if ((unsigned char)*buf < '\040' || *buf == '\177') { \ + *ret = -1; \ + return NULL; \ + } \ + } \ + ++buf; \ + CHECK_EOF(); \ + } \ + tok = tok_start; \ + toklen = buf - tok_start; \ + } while (0) + +static const char *token_char_map = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\1\0\1\1\1\1\1\0\0\1\1\0\1\1\0\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0" + "\0\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\1\1" + "\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\1\0\1\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; + +static const char *findchar_fast(const char *buf, const char *buf_end, const char *ranges, size_t ranges_size, int *found) +{ + *found = 0; +#if __SSE4_2__ + if (likely(buf_end - buf >= 16)) { + __m128i ranges16 = _mm_loadu_si128((const __m128i *)ranges); + + size_t left = (buf_end - buf) & ~15; + do { + __m128i b16 = _mm_loadu_si128((const __m128i *)buf); + int r = _mm_cmpestri(ranges16, ranges_size, b16, 16, _SIDD_LEAST_SIGNIFICANT | _SIDD_CMP_RANGES | _SIDD_UBYTE_OPS); + if (unlikely(r != 16)) { + buf += r; + *found = 1; + break; + } + buf += 16; + left -= 16; + } while (likely(left != 0)); + } +#else + /* suppress unused parameter warning */ + (void)buf_end; + (void)ranges; + (void)ranges_size; +#endif + return buf; +} + +static const char *get_token_to_eol(const char *buf, const char *buf_end, const char **token, size_t *token_len, int *ret) +{ + const char *token_start = buf; + +#ifdef __SSE4_2__ + static const char ranges1[] = "\0\010" + /* allow HT */ + "\012\037" + /* allow SP and up to but not including DEL */ + "\177\177" + /* allow chars w. MSB set */ + ; + int found; + buf = findchar_fast(buf, buf_end, ranges1, sizeof(ranges1) - 1, &found); + if (found) + goto FOUND_CTL; +#else + /* find non-printable char within the next 8 bytes, this is the hottest code; manually inlined */ + while (likely(buf_end - buf >= 8)) { +#define DOIT() \ + do { \ + if (unlikely(!IS_PRINTABLE_ASCII(*buf))) \ + goto NonPrintable; \ + ++buf; \ + } while (0) + DOIT(); + DOIT(); + DOIT(); + DOIT(); + DOIT(); + DOIT(); + DOIT(); + DOIT(); +#undef DOIT + continue; + NonPrintable: + if ((likely((unsigned char)*buf < '\040') && likely(*buf != '\011')) || unlikely(*buf == '\177')) { + goto FOUND_CTL; + } + ++buf; + } +#endif + for (;; ++buf) { + CHECK_EOF(); + if (unlikely(!IS_PRINTABLE_ASCII(*buf))) { + if ((likely((unsigned char)*buf < '\040') && likely(*buf != '\011')) || unlikely(*buf == '\177')) { + goto FOUND_CTL; + } + } + } +FOUND_CTL: + if (likely(*buf == '\015')) { + ++buf; + EXPECT_CHAR('\012'); + *token_len = buf - 2 - token_start; + } else if (*buf == '\012') { + *token_len = buf - token_start; + ++buf; + } else { + *ret = -1; + return NULL; + } + *token = token_start; + + return buf; +} + +static const char *is_complete(const char *buf, const char *buf_end, size_t last_len, int *ret) +{ + int ret_cnt = 0; + buf = last_len < 3 ? buf : buf + last_len - 3; + + while (1) { + CHECK_EOF(); + if (*buf == '\015') { + ++buf; + CHECK_EOF(); + EXPECT_CHAR('\012'); + ++ret_cnt; + } else if (*buf == '\012') { + ++buf; + ++ret_cnt; + } else { + ++buf; + ret_cnt = 0; + } + if (ret_cnt == 2) { + return buf; + } + } + + //*ret = -2; + return NULL; +} + +#define PARSE_INT(valp_, mul_) \ + if (*buf < '0' || '9' < *buf) { \ + buf++; \ + *ret = -1; \ + return NULL; \ + } \ + *(valp_) = (mul_) * (*buf++ - '0'); + +#define PARSE_INT_3(valp_) \ + do { \ + int res_ = 0; \ + PARSE_INT(&res_, 100) \ + *valp_ = res_; \ + PARSE_INT(&res_, 10) \ + *valp_ += res_; \ + PARSE_INT(&res_, 1) \ + *valp_ += res_; \ + } while (0) + +/* returned pointer is always within [buf, buf_end), or null */ +static const char *parse_http_version(const char *buf, const char *buf_end, int *minor_version, int *ret) +{ + /* we want at least [HTTP/1.] to try to parse */ + if (buf_end - buf < 9) { + *ret = -2; + return NULL; + } + EXPECT_CHAR_NO_CHECK('H'); + EXPECT_CHAR_NO_CHECK('T'); + EXPECT_CHAR_NO_CHECK('T'); + EXPECT_CHAR_NO_CHECK('P'); + EXPECT_CHAR_NO_CHECK('/'); + EXPECT_CHAR_NO_CHECK('1'); + EXPECT_CHAR_NO_CHECK('.'); + PARSE_INT(minor_version, 1); + return buf; +} + +static const char *parse_headers(const char *buf, const char *buf_end, struct phr_header *headers, size_t *num_headers, + size_t max_headers, int *ret) +{ + for (;; ++*num_headers) { + CHECK_EOF(); + if (*buf == '\015') { + ++buf; + EXPECT_CHAR('\012'); + break; + } else if (*buf == '\012') { + ++buf; + break; + } + if (*num_headers == max_headers) { + *ret = -1; + return NULL; + } + if (!(*num_headers != 0 && (*buf == ' ' || *buf == '\t'))) { + /* parsing name, but do not discard SP before colon, see + * http://www.mozilla.org/security/announce/2006/mfsa2006-33.html */ + headers[*num_headers].name = buf; + static const char ALIGNED(16) ranges1[] = "\x00 " /* control chars and up to SP */ + "\"\"" /* 0x22 */ + "()" /* 0x28,0x29 */ + ",," /* 0x2c */ + "//" /* 0x2f */ + ":@" /* 0x3a-0x40 */ + "[]" /* 0x5b-0x5d */ + "{\377"; /* 0x7b-0xff */ + int found; + buf = findchar_fast(buf, buf_end, ranges1, sizeof(ranges1) - 1, &found); + if (!found) { + CHECK_EOF(); + } + while (1) { + if (*buf == ':') { + break; + } else if (!token_char_map[(unsigned char)*buf]) { + *ret = -1; + return NULL; + } + ++buf; + CHECK_EOF(); + } + if ((headers[*num_headers].name_len = buf - headers[*num_headers].name) == 0) { + *ret = -1; + return NULL; + } + ++buf; + for (;; ++buf) { + CHECK_EOF(); + if (!(*buf == ' ' || *buf == '\t')) { + break; + } + } + } else { + headers[*num_headers].name = NULL; + headers[*num_headers].name_len = 0; + } + if ((buf = get_token_to_eol(buf, buf_end, &headers[*num_headers].value, &headers[*num_headers].value_len, ret)) == NULL) { + return NULL; + } + } + return buf; +} + +static const char *parse_request(const char *buf, const char *buf_end, const char **method, size_t *method_len, const char **path, + size_t *path_len, int *minor_version, struct phr_header *headers, size_t *num_headers, + size_t max_headers, int *ret) +{ + /* skip first empty line (some clients add CRLF after POST content) */ + CHECK_EOF(); + if (*buf == '\015') { + ++buf; + EXPECT_CHAR('\012'); + } else if (*buf == '\012') { + ++buf; + } + + /* parse request line */ + ADVANCE_TOKEN(*method, *method_len); + ++buf; + ADVANCE_TOKEN(*path, *path_len); + ++buf; + if ((buf = parse_http_version(buf, buf_end, minor_version, ret)) == NULL) { + return NULL; + } + if (*buf == '\015') { + ++buf; + EXPECT_CHAR('\012'); + } else if (*buf == '\012') { + ++buf; + } else { + *ret = -1; + return NULL; + } + + return parse_headers(buf, buf_end, headers, num_headers, max_headers, ret); +} + +int phr_parse_request(const char *buf_start, size_t len, const char **method, size_t *method_len, const char **path, + size_t *path_len, int *minor_version, struct phr_header *headers, size_t *num_headers, size_t last_len) +{ + const char *buf = buf_start, *buf_end = buf_start + len; + size_t max_headers = *num_headers; + int r; + + *method = NULL; + *method_len = 0; + *path = NULL; + *path_len = 0; + *minor_version = -1; + *num_headers = 0; + + /* if last_len != 0, check if the request is complete (a fast countermeasure + againt slowloris */ + if (last_len != 0 && is_complete(buf, buf_end, last_len, &r) == NULL) { + return r; + } + + if ((buf = parse_request(buf, buf_end, method, method_len, path, path_len, minor_version, headers, num_headers, max_headers, + &r)) == NULL) { + return r; + } + + return (int)(buf - buf_start); +} + +static const char *parse_response(const char *buf, const char *buf_end, int *minor_version, int *status, const char **msg, + size_t *msg_len, struct phr_header *headers, size_t *num_headers, size_t max_headers, int *ret) +{ + /* parse "HTTP/1.x" */ + if ((buf = parse_http_version(buf, buf_end, minor_version, ret)) == NULL) { + return NULL; + } + /* skip space */ + if (*buf++ != ' ') { + *ret = -1; + return NULL; + } + /* parse status code, we want at least [:digit:][:digit:][:digit:] to try to parse */ + if (buf_end - buf < 4) { + *ret = -2; + return NULL; + } + PARSE_INT_3(status); + + /* skip space */ + if (*buf++ != ' ') { + *ret = -1; + return NULL; + } + /* get message */ + if ((buf = get_token_to_eol(buf, buf_end, msg, msg_len, ret)) == NULL) { + return NULL; + } + + return parse_headers(buf, buf_end, headers, num_headers, max_headers, ret); +} + +int phr_parse_response(const char *buf_start, size_t len, int *minor_version, int *status, const char **msg, size_t *msg_len, + struct phr_header *headers, size_t *num_headers, size_t last_len) +{ + const char *buf = buf_start, *buf_end = buf + len; + size_t max_headers = *num_headers; + int r; + + *minor_version = -1; + *status = 0; + *msg = NULL; + *msg_len = 0; + *num_headers = 0; + + /* if last_len != 0, check if the response is complete (a fast countermeasure + against slowloris */ + if (last_len != 0 && is_complete(buf, buf_end, last_len, &r) == NULL) { + return r; + } + + if ((buf = parse_response(buf, buf_end, minor_version, status, msg, msg_len, headers, num_headers, max_headers, &r)) == NULL) { + return r; + } + + return (int)(buf - buf_start); +} + +int phr_parse_headers(const char *buf_start, size_t len, struct phr_header *headers, size_t *num_headers, size_t last_len) +{ + const char *buf = buf_start, *buf_end = buf + len; + size_t max_headers = *num_headers; + int r; + + *num_headers = 0; + + /* if last_len != 0, check if the response is complete (a fast countermeasure + against slowloris */ + if (last_len != 0 && is_complete(buf, buf_end, last_len, &r) == NULL) { + return r; + } + + if ((buf = parse_headers(buf, buf_end, headers, num_headers, max_headers, &r)) == NULL) { + return r; + } + + return (int)(buf - buf_start); +} + +enum { + CHUNKED_IN_CHUNK_SIZE, + CHUNKED_IN_CHUNK_EXT, + CHUNKED_IN_CHUNK_DATA, + CHUNKED_IN_CHUNK_CRLF, + CHUNKED_IN_TRAILERS_LINE_HEAD, + CHUNKED_IN_TRAILERS_LINE_MIDDLE +}; + +static int decode_hex(int ch) +{ + if ('0' <= ch && ch <= '9') { + return ch - '0'; + } else if ('A' <= ch && ch <= 'F') { + return ch - 'A' + 0xa; + } else if ('a' <= ch && ch <= 'f') { + return ch - 'a' + 0xa; + } else { + return -1; + } +} + +ssize_t phr_decode_chunked(struct phr_chunked_decoder *decoder, char *buf, size_t *_bufsz) +{ + size_t dst = 0, src = 0, bufsz = *_bufsz; + ssize_t ret = -2; /* incomplete */ + + while (1) { + switch (decoder->_state) { + case CHUNKED_IN_CHUNK_SIZE: + for (;; ++src) { + int v; + if (src == bufsz) + goto Exit; + if ((v = decode_hex(buf[src])) == -1) { + if (decoder->_hex_count == 0) { + ret = -1; + goto Exit; + } + break; + } + if (decoder->_hex_count == sizeof(size_t) * 2) { + ret = -1; + goto Exit; + } + decoder->bytes_left_in_chunk = decoder->bytes_left_in_chunk * 16 + v; + ++decoder->_hex_count; + } + decoder->_hex_count = 0; + decoder->_state = CHUNKED_IN_CHUNK_EXT; + /* fallthru */ + case CHUNKED_IN_CHUNK_EXT: + /* RFC 7230 A.2 "Line folding in chunk extensions is disallowed" */ + for (;; ++src) { + if (src == bufsz) + goto Exit; + if (buf[src] == '\012') + break; + } + ++src; + if (decoder->bytes_left_in_chunk == 0) { + if (decoder->consume_trailer) { + decoder->_state = CHUNKED_IN_TRAILERS_LINE_HEAD; + break; + } else { + goto Complete; + } + } + decoder->_state = CHUNKED_IN_CHUNK_DATA; + /* fallthru */ + case CHUNKED_IN_CHUNK_DATA: { + size_t avail = bufsz - src; + if (avail < decoder->bytes_left_in_chunk) { + if (dst != src) + memmove(buf + dst, buf + src, avail); + src += avail; + dst += avail; + decoder->bytes_left_in_chunk -= avail; + goto Exit; + } + if (dst != src) + memmove(buf + dst, buf + src, decoder->bytes_left_in_chunk); + src += decoder->bytes_left_in_chunk; + dst += decoder->bytes_left_in_chunk; + decoder->bytes_left_in_chunk = 0; + decoder->_state = CHUNKED_IN_CHUNK_CRLF; + } + /* fallthru */ + case CHUNKED_IN_CHUNK_CRLF: + for (;; ++src) { + if (src == bufsz) + goto Exit; + if (buf[src] != '\015') + break; + } + if (buf[src] != '\012') { + ret = -1; + goto Exit; + } + ++src; + decoder->_state = CHUNKED_IN_CHUNK_SIZE; + break; + case CHUNKED_IN_TRAILERS_LINE_HEAD: + for (;; ++src) { + if (src == bufsz) + goto Exit; + if (buf[src] != '\015') + break; + } + if (buf[src++] == '\012') + goto Complete; + decoder->_state = CHUNKED_IN_TRAILERS_LINE_MIDDLE; + /* fallthru */ + case CHUNKED_IN_TRAILERS_LINE_MIDDLE: + for (;; ++src) { + if (src == bufsz) + goto Exit; + if (buf[src] == '\012') + break; + } + ++src; + decoder->_state = CHUNKED_IN_TRAILERS_LINE_HEAD; + break; + default: + assert(!"decoder is corrupt"); + } + } + +Complete: + ret = bufsz - src; +Exit: + if (dst != src) + memmove(buf + dst, buf + src, bufsz - src); + *_bufsz = dst; + return ret; +} + +int phr_decode_chunked_is_in_data(struct phr_chunked_decoder *decoder) +{ + return decoder->_state == CHUNKED_IN_CHUNK_DATA; +} + +#undef CHECK_EOF +#undef EXPECT_CHAR +#undef ADVANCE_TOKEN diff --git a/examples/picohttpparser.h b/examples/picohttpparser.h new file mode 100755 index 00000000..a8fad71d --- /dev/null +++ b/examples/picohttpparser.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase, + * Shigeo Mitsunari + * + * The software is licensed under either the MIT License (below) or the Perl + * license. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#ifndef picohttpparser_h +#define picohttpparser_h + +#include + +#ifdef _MSC_VER +#define ssize_t intptr_t +#endif + +/* $Id: 67fd3ee74103ada60258d8a16e868f483abcca87 $ */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* contains name and value of a header (name == NULL if is a continuing line + * of a multiline header */ +struct phr_header { + const char *name; + size_t name_len; + const char *value; + size_t value_len; +}; + +/* returns number of bytes consumed if successful, -2 if request is partial, + * -1 if failed */ +int phr_parse_request(const char *buf, size_t len, const char **method, size_t *method_len, const char **path, size_t *path_len, + int *minor_version, struct phr_header *headers, size_t *num_headers, size_t last_len); + +/* ditto */ +int phr_parse_response(const char *_buf, size_t len, int *minor_version, int *status, const char **msg, size_t *msg_len, + struct phr_header *headers, size_t *num_headers, size_t last_len); + +/* ditto */ +int phr_parse_headers(const char *buf, size_t len, struct phr_header *headers, size_t *num_headers, size_t last_len); + +/* should be zero-filled before start */ +struct phr_chunked_decoder { + size_t bytes_left_in_chunk; /* number of bytes left in current chunk */ + char consume_trailer; /* if trailing headers should be consumed */ + char _hex_count; + char _state; +}; + +/* the function rewrites the buffer given as (buf, bufsz) removing the chunked- + * encoding headers. When the function returns without an error, bufsz is + * updated to the length of the decoded data available. Applications should + * repeatedly call the function while it returns -2 (incomplete) every time + * supplying newly arrived data. If the end of the chunked-encoded data is + * found, the function returns a non-negative number indicating the number of + * octets left undecoded at the tail of the supplied buffer. Returns -1 on + * error. + */ +ssize_t phr_decode_chunked(struct phr_chunked_decoder *decoder, char *buf, size_t *bufsz); + +/* returns if the chunked decoder is in middle of chunked data */ +int phr_decode_chunked_is_in_data(struct phr_chunked_decoder *decoder); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/examples/server_http.c b/examples/server_http.c index 32a7ddad..de553dc8 100644 --- a/examples/server_http.c +++ b/examples/server_http.c @@ -5,15 +5,16 @@ #include #include #include -#include "util.h" #include +#include + +#include "util.h" +#include "picohttpparser.h" /********************************************************************** http server - server_daytime [OPTIONS] - **********************************************************************/ static char *config_property = "{\ @@ -29,13 +30,118 @@ static char *config_property = "{\ ]\ }"; static uint16_t config_log_level = 1; -static const char *response_header = "HTTP/1.0 200 OK\r\nUser-agent: libneat\r\nConnection: close\r\n\r\n"; -static const char *response_body_a = "NEAT Webserver

Welcome to NEAT!

"; -static const char *response_body_b = ""; #define BUFFERSIZE 33768 +static char *http_header_connection_close = "Connection: Close"; +static char *http_header_connection_keep_alive = "Connection: Keep-Alive"; + + static neat_error_code on_writable(struct neat_flow_operations *opCB); +struct http_flow { + unsigned char buffer[BUFFERSIZE]; + char *method; + char *path; + int minor_version; + int pret; + struct phr_header headers[100]; + size_t buffer_len; + size_t buffer_len_prev; + size_t method_len; + size_t path_len; + size_t num_headers; + uint8_t keep_alive; +}; + +static int +prepare_http_response(struct http_flow *http_flow, unsigned char **buffer, uint32_t *buffer_len) { + + int header_length = 0; + unsigned char *header_buffer = NULL; + int payload_length = 0; + unsigned char *payload_buffer = NULL; + int i = 0; + char misc_buffer[512]; + + // iterate through header fields + for (i = 0; i < (int)http_flow->num_headers; i++) { + // build string from name/value pair + snprintf(misc_buffer, 512, "%.*s: %.*s", + (int)http_flow->headers[i].name_len, + http_flow->headers[i].name, + (int)http_flow->headers[i].value_len, + http_flow->headers[i].value); + + if (strncasecmp(misc_buffer, http_header_connection_close, strlen(http_header_connection_close)) == 0) { + printf(">>>>>>>>>>> CLOSE !!!!!!!!!!!!!!!!\n"); + } else if (strncasecmp(misc_buffer, http_header_connection_keep_alive, strlen(http_header_connection_keep_alive)) == 0) { + printf(">>>>>>>>>>> KEEP-ALIVE !!!!!!!!!!!!!!!!\n"); + http_flow->keep_alive = 1; + } + + } + + // XXX needs refactoring - just shit ... shame on me... :/ + if (http_flow->path_len == 1 && http_flow->path[0] == '/') { + // request root "/" --> index.html + snprintf(misc_buffer, sizeof(misc_buffer), "index.html"); + } else if (http_flow->path_len > 1 && http_flow->path[0] == '/') { + // path begins with "/" + snprintf(misc_buffer, sizeof(misc_buffer), "%.*s", (int)http_flow->path_len - 1, http_flow->path + 1); + } else { + // path does not begin with "/" + snprintf(misc_buffer, sizeof(misc_buffer), "%.*s", (int)http_flow->path_len, http_flow->path); + } + + payload_length = read_file(misc_buffer, (char **) &payload_buffer); + + // error when reading file - read inde.html + if (payload_length < 0) { + fprintf(stderr, "%s - reading failed - delivering index.html\n", __func__); + snprintf(misc_buffer, sizeof(misc_buffer), "index.html"); + payload_length = read_file(misc_buffer, (char **) &payload_buffer); + } + + if (payload_length < 0 ) { + // we have a serious problem here... + exit(EXIT_FAILURE); + } + + // prepare response header + header_length = asprintf((char **)&header_buffer, + "HTTP/1.1 200 OK\r\n" + "Server: NEAT super fancy webserver\r\n" + "Content-Length: %u\r\n" + "Connection: %s\r\n" + "\r\n", + payload_length, + http_flow->keep_alive ? "Keep-Alive" : "Close"); + + if (header_length == -1) { + fprintf(stderr, "%s - asprintf failed\n", __func__); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "%s\n", header_buffer); + + header_buffer = realloc(header_buffer, header_length + payload_length); + + if (header_buffer == NULL) { + fprintf(stderr, "%s - realloc failed\n", __func__); + } + + memcpy(header_buffer + header_length, payload_buffer, payload_length); + + *buffer = header_buffer; + *buffer_len = header_length + payload_length; + + fprintf(stderr, "%s - pointer: %p - len: %u\n",__func__, (void *)*buffer, *buffer_len); + + return NEAT_OK; +} + + + /* print usage and exit */ @@ -69,14 +175,14 @@ on_readable(struct neat_flow_operations *opCB) { // data is available to read neat_error_code code; - unsigned char buffer[BUFFERSIZE]; - uint32_t buffer_filled; + struct http_flow *http_flow = opCB->userData; + uint32_t buffer_filled = 0; if (config_log_level >= 2) { fprintf(stderr, "%s()\n", __func__); } - code = neat_read(opCB->ctx, opCB->flow, buffer, BUFFERSIZE, &buffer_filled, NULL, 0); + code = neat_read(opCB->ctx, opCB->flow, http_flow->buffer + http_flow->buffer_len, BUFFERSIZE - http_flow->buffer_len, &buffer_filled, NULL, 0); if (code != NEAT_OK) { if (code == NEAT_ERROR_WOULD_BLOCK) { return NEAT_OK; @@ -86,43 +192,72 @@ on_readable(struct neat_flow_operations *opCB) } } - if (buffer_filled > 0) { - if (config_log_level >= 1) { - printf("received data - %d byte\n", buffer_filled); - } - if (config_log_level >= 2) { - fwrite(buffer, sizeof(char), buffer_filled, stdout); - printf("\n"); - fflush(stdout); - } - + if (config_log_level >= 1) { + printf("received data - %d byte\n", buffer_filled); + } +#if 0 + if (config_log_level >= 2) { + fwrite(http_flow->buffer, sizeof(char), buffer_filled, stdout); + printf("\n"); + fflush(stdout); + } +#endif + + http_flow->buffer_len_prev = http_flow->buffer_len; + http_flow->buffer_len += buffer_filled; + http_flow->num_headers = sizeof(http_flow->headers) / sizeof(http_flow->headers[0]); + + http_flow->pret = phr_parse_request((const char *) http_flow->buffer, + http_flow->buffer_len, + (const char **) &(http_flow->method), + &(http_flow->method_len), + (const char **) &(http_flow->path), + &(http_flow->path_len), + &(http_flow->minor_version), + http_flow->headers, + &(http_flow->num_headers), + http_flow->buffer_len_prev); + + if (http_flow->pret > 0) { + fprintf(stderr, "%s - request parsed!\n", __func__); opCB->on_writable = on_writable; neat_set_operations(opCB->ctx, opCB->flow, opCB); + return NEAT_OK; + } else if (http_flow->pret == -1) { + fprintf(stderr, "%s - error : parsing request!\n", __func__); + neat_close(opCB->ctx, opCB->flow); + return NEAT_OK; + } - } else { // peer disconnected - if (config_log_level >= 1) { - printf("peer disconnected\n"); - } - opCB->on_readable = NULL; - opCB->on_writable = NULL; - opCB->on_all_written = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); + assert(http_flow->pret == -2); + if (http_flow->buffer_len == sizeof(http_flow->buffer)) { + fprintf(stderr, "%s - error : request to long!!\n", __func__); neat_close(opCB->ctx, opCB->flow); + return NEAT_OK; } + + neat_set_operations(opCB->ctx, opCB->flow, opCB); return NEAT_OK; } static neat_error_code on_all_written(struct neat_flow_operations *opCB) { + struct http_flow *http_flow = opCB->userData; + if (config_log_level >= 2) { fprintf(stderr, "%s()\n", __func__); } - opCB->on_writable = NULL; - opCB->on_all_written = NULL; - neat_set_operations(opCB->ctx, opCB->flow, opCB); - neat_shutdown(opCB->ctx, opCB->flow); + if (http_flow->keep_alive == 1) { + memset(http_flow, 0, sizeof(struct http_flow)); + opCB->on_all_written = NULL; + opCB->on_readable = on_readable; + neat_set_operations(opCB->ctx, opCB->flow, opCB); + } else { + neat_close(opCB->ctx, opCB->flow); + } + return NEAT_OK; } @@ -130,40 +265,60 @@ static neat_error_code on_writable(struct neat_flow_operations *opCB) { neat_error_code code; - char *stats = NULL; - char buffer[BUFFERSIZE]; + struct http_flow *http_flow = opCB->userData; + unsigned char *buffer = NULL; + uint32_t buffer_len = 0; if (config_log_level >= 2) { fprintf(stderr, "%s()\n", __func__); } - code = neat_get_stats(opCB->ctx, &stats); - if (code != NEAT_OK){ - fprintf(stderr, "%s - neat_get_stats() failed\n", __func__); - return on_error(opCB); + // print request information + printf(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); + printf("request is %d bytes long\n", http_flow->pret); + printf("method is %.*s\n", (int)http_flow->method_len, http_flow->method); + printf("path is %.*s\n", (int)http_flow->path_len, http_flow->path); + printf("HTTP version is 1.%d\n", http_flow->minor_version); + printf("headers:\n"); + for (int i = 0; i != (int)http_flow->num_headers; ++i) { + printf("%.*s: %.*s\n", + (int)http_flow->headers[i].name_len, + http_flow->headers[i].name, + (int)http_flow->headers[i].value_len, + http_flow->headers[i].value); + } + printf(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); - if (config_log_level >= 2) { - fprintf(stderr, "%s", stats); + if (prepare_http_response(http_flow, &buffer, &buffer_len) != NEAT_OK) { + exit(-1); } - snprintf(buffer, BUFFERSIZE, "%s%s%s%s%s%s\r\n", response_header, response_body_a, "
", stats, "
", response_body_b); + printf("%s - buffer len %d - pointer %p\n", __func__, buffer_len, (void *) buffer); - code = neat_write(opCB->ctx, opCB->flow, (const unsigned char *) buffer, strlen(buffer), NULL, 0); + code = neat_write(opCB->ctx, opCB->flow, buffer, buffer_len, NULL, 0); if (code != NEAT_OK) { fprintf(stderr, "%s - neat_write failed - code: %d\n", __func__, (int)code); return on_error(opCB); } opCB->on_writable = NULL; - opCB->on_all_written = NULL; + opCB->on_all_written = on_all_written; neat_set_operations(opCB->ctx, opCB->flow, opCB); - neat_shutdown(opCB->ctx, opCB->flow); + return NEAT_OK; } +static neat_error_code +on_close(struct neat_flow_operations *opCB) +{ + fprintf(stderr, "%s - flow closed OK!\n", __func__); + free(opCB->userData); + return NEAT_OK; +} + static neat_error_code on_connected(struct neat_flow_operations *opCB) { @@ -175,9 +330,15 @@ on_connected(struct neat_flow_operations *opCB) printf("peer connected\n"); } + if ((opCB->userData = calloc(1, sizeof(struct http_flow))) == NULL) { + fprintf(stderr, "%s - could not allocate http_flow\n", __func__); + exit(EXIT_FAILURE); + } + opCB->on_readable = on_readable; opCB->on_writable = NULL; opCB->on_all_written = NULL; + opCB->on_close = on_close; neat_set_operations(opCB->ctx, opCB->flow, opCB); return NEAT_OK; @@ -274,6 +435,10 @@ main(int argc, char *argv[]) goto cleanup; } + if (chdir("htdocs")) { + printf("%s - chdir failed\n", __func__); + } + neat_start_event_loop(ctx, NEAT_RUN_DEFAULT); // cleanup diff --git a/examples/util.c b/examples/util.c index c3869a9f..7eb302be 100644 --- a/examples/util.c +++ b/examples/util.c @@ -5,18 +5,21 @@ #include #include #include +#include int read_file(const char *filename, const char **bufptr) { - int rc; + int rc = -1; struct stat stat_buffer; char *buffer = NULL; FILE *f = NULL; size_t file_size, offset = 0; - if ((rc = stat(filename, &stat_buffer)) < 0) + if ((rc = stat(filename, &stat_buffer)) < 0) { + fprintf(stderr, "%s - stat failed - {%s}\n", __func__, filename); goto error; + } assert(stat_buffer.st_size >= 0); @@ -25,19 +28,23 @@ read_file(const char *filename, const char **bufptr) buffer = (char*)malloc(file_size + 1); if (!buffer) { rc = -ENOMEM; + fprintf(stderr, "%s - malloc failed\n", __func__); goto error; } f = fopen(filename, "r"); if (!f) { rc = -EIO; + fprintf(stderr, "%s - fopen failed\n", __func__); goto error; } do { size_t bytes_read = fread(buffer + offset, 1, file_size - offset, f); - if (bytes_read < file_size - offset && ferror(f)) + if (bytes_read < file_size - offset && ferror(f)) { + fprintf(stderr, "%s - fread failed\n", __func__); goto error; + } offset += bytes_read; } while (offset < file_size); @@ -46,7 +53,7 @@ read_file(const char *filename, const char **bufptr) buffer[file_size] = 0; *bufptr = buffer; - return 0; + return file_size; error: if (buffer) free(buffer); From 7783d58e8c7a825502668bc1b501361322f9701b Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Fri, 19 May 2017 02:47:45 +0200 Subject: [PATCH 57/85] http server refactoring --- examples/server_http.c | 127 +++++++++++++++++++++++++---------------- 1 file changed, 78 insertions(+), 49 deletions(-) diff --git a/examples/server_http.c b/examples/server_http.c index de553dc8..3558dbe9 100644 --- a/examples/server_http.c +++ b/examples/server_http.c @@ -8,6 +8,7 @@ #include #include + #include "util.h" #include "picohttpparser.h" @@ -29,10 +30,12 @@ static char *config_property = "{\ }\ ]\ }"; -static uint16_t config_log_level = 1; +static uint8_t config_log_level = 1; +static uint8_t config_keep_alive = 0; #define BUFFERSIZE 33768 +struct neat_ctx *ctx = NULL; + -static char *http_header_connection_close = "Connection: Close"; static char *http_header_connection_keep_alive = "Connection: Keep-Alive"; @@ -53,6 +56,15 @@ struct http_flow { uint8_t keep_alive; }; +void +sig_handler(int signo) { + + if (signo == SIGINT) { + printf("received SIGINT - stopping event loop\n"); + neat_stop_event_loop(ctx); + } +} + static int prepare_http_response(struct http_flow *http_flow, unsigned char **buffer, uint32_t *buffer_len) { @@ -63,7 +75,11 @@ prepare_http_response(struct http_flow *http_flow, unsigned char **buffer, uint3 int i = 0; char misc_buffer[512]; - // iterate through header fields + if (config_log_level >= 2) { + fprintf(stderr, "%s()\n", __func__); + } + + // iterate header fields for (i = 0; i < (int)http_flow->num_headers; i++) { // build string from name/value pair snprintf(misc_buffer, 512, "%.*s: %.*s", @@ -72,10 +88,9 @@ prepare_http_response(struct http_flow *http_flow, unsigned char **buffer, uint3 (int)http_flow->headers[i].value_len, http_flow->headers[i].value); - if (strncasecmp(misc_buffer, http_header_connection_close, strlen(http_header_connection_close)) == 0) { - printf(">>>>>>>>>>> CLOSE !!!!!!!!!!!!!!!!\n"); - } else if (strncasecmp(misc_buffer, http_header_connection_keep_alive, strlen(http_header_connection_keep_alive)) == 0) { - printf(">>>>>>>>>>> KEEP-ALIVE !!!!!!!!!!!!!!!!\n"); + // we have a Keep-Alive connection + if (strncasecmp(misc_buffer, http_header_connection_keep_alive, strlen(http_header_connection_keep_alive)) == 0 && + config_keep_alive == 1) { http_flow->keep_alive = 1; } @@ -93,17 +108,22 @@ prepare_http_response(struct http_flow *http_flow, unsigned char **buffer, uint3 snprintf(misc_buffer, sizeof(misc_buffer), "%.*s", (int)http_flow->path_len, http_flow->path); } + // try to read requested file payload_length = read_file(misc_buffer, (char **) &payload_buffer); - // error when reading file - read inde.html + if (payload_length < 0) { - fprintf(stderr, "%s - reading failed - delivering index.html\n", __func__); + // error when reading file - read index.html + if (config_log_level >= 1) { + fprintf(stderr, "%s - reading >>%s<< failed - delivering index.html\n", __func__, misc_buffer); + } snprintf(misc_buffer, sizeof(misc_buffer), "index.html"); payload_length = read_file(misc_buffer, (char **) &payload_buffer); } if (payload_length < 0 ) { // we have a serious problem here... + fprintf(stderr, "%s - read_file failed\n", __func__); exit(EXIT_FAILURE); } @@ -122,21 +142,25 @@ prepare_http_response(struct http_flow *http_flow, unsigned char **buffer, uint3 exit(EXIT_FAILURE); } - fprintf(stderr, "%s\n", header_buffer); + if (config_log_level >= 1) { + // print response information + fprintf(stderr, "\n\n%s\n", header_buffer); + } + header_buffer = realloc(header_buffer, header_length + payload_length); if (header_buffer == NULL) { fprintf(stderr, "%s - realloc failed\n", __func__); + exit(EXIT_FAILURE); } memcpy(header_buffer + header_length, payload_buffer, payload_length); + free(payload_buffer); *buffer = header_buffer; *buffer_len = header_length + payload_length; - fprintf(stderr, "%s - pointer: %p - len: %u\n",__func__, (void *)*buffer, *buffer_len); - return NEAT_OK; } @@ -163,10 +187,8 @@ print_usage() static neat_error_code on_error(struct neat_flow_operations *opCB) { - if (config_log_level >= 2) { - fprintf(stderr, "%s()\n", __func__); - } + fprintf(stderr, "%s()\n", __func__); exit(EXIT_FAILURE); } @@ -193,19 +215,13 @@ on_readable(struct neat_flow_operations *opCB) } if (config_log_level >= 1) { - printf("received data - %d byte\n", buffer_filled); + printf("%s - read %d byte\n", __func__, buffer_filled); } -#if 0 - if (config_log_level >= 2) { - fwrite(http_flow->buffer, sizeof(char), buffer_filled, stdout); - printf("\n"); - fflush(stdout); - } -#endif - http_flow->buffer_len_prev = http_flow->buffer_len; - http_flow->buffer_len += buffer_filled; - http_flow->num_headers = sizeof(http_flow->headers) / sizeof(http_flow->headers[0]); + + http_flow->buffer_len_prev = http_flow->buffer_len; + http_flow->buffer_len += buffer_filled; + http_flow->num_headers = sizeof(http_flow->headers) / sizeof(http_flow->headers[0]); http_flow->pret = phr_parse_request((const char *) http_flow->buffer, http_flow->buffer_len, @@ -219,7 +235,7 @@ on_readable(struct neat_flow_operations *opCB) http_flow->buffer_len_prev); if (http_flow->pret > 0) { - fprintf(stderr, "%s - request parsed!\n", __func__); + // request parsed successfully opCB->on_writable = on_writable; neat_set_operations(opCB->ctx, opCB->flow, opCB); return NEAT_OK; @@ -236,6 +252,7 @@ on_readable(struct neat_flow_operations *opCB) return NEAT_OK; } + // continue reading neat_set_operations(opCB->ctx, opCB->flow, opCB); return NEAT_OK; } @@ -273,29 +290,28 @@ on_writable(struct neat_flow_operations *opCB) fprintf(stderr, "%s()\n", __func__); } - // print request information - printf(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); - printf("request is %d bytes long\n", http_flow->pret); - printf("method is %.*s\n", (int)http_flow->method_len, http_flow->method); - printf("path is %.*s\n", (int)http_flow->path_len, http_flow->path); - printf("HTTP version is 1.%d\n", http_flow->minor_version); - printf("headers:\n"); - for (int i = 0; i != (int)http_flow->num_headers; ++i) { - printf("%.*s: %.*s\n", - (int)http_flow->headers[i].name_len, - http_flow->headers[i].name, - (int)http_flow->headers[i].value_len, - http_flow->headers[i].value); - + if (config_log_level >= 1) { + // print request information + printf("#######################################################\n"); + printf("request is %d bytes long\n", http_flow->pret); + printf("method is %.*s\n", (int)http_flow->method_len, http_flow->method); + printf("path is %.*s\n", (int)http_flow->path_len, http_flow->path); + printf("HTTP version is 1.%d\n", http_flow->minor_version); + printf("headers:\n"); + for (int i = 0; i != (int)http_flow->num_headers; ++i) { + printf("%.*s: %.*s\n", + (int)http_flow->headers[i].name_len, + http_flow->headers[i].name, + (int)http_flow->headers[i].value_len, + http_flow->headers[i].value); + } + printf("#######################################################\n"); } - printf(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); if (prepare_http_response(http_flow, &buffer, &buffer_len) != NEAT_OK) { - exit(-1); + exit(EXIT_FAILURE); } - printf("%s - buffer len %d - pointer %p\n", __func__, buffer_len, (void *) buffer); - code = neat_write(opCB->ctx, opCB->flow, buffer, buffer_len, NULL, 0); if (code != NEAT_OK) { @@ -303,6 +319,8 @@ on_writable(struct neat_flow_operations *opCB) return on_error(opCB); } + free(buffer); + opCB->on_writable = NULL; opCB->on_all_written = on_all_written; neat_set_operations(opCB->ctx, opCB->flow, opCB); @@ -314,7 +332,9 @@ on_writable(struct neat_flow_operations *opCB) static neat_error_code on_close(struct neat_flow_operations *opCB) { - fprintf(stderr, "%s - flow closed OK!\n", __func__); + if (config_log_level >= 1) { + fprintf(stderr, "%s - flow closed OK!\n", __func__); + } free(opCB->userData); return NEAT_OK; } @@ -350,7 +370,6 @@ main(int argc, char *argv[]) // uint64_t prop; int arg, result; char *arg_property = NULL; - struct neat_ctx *ctx = NULL; struct neat_flow *flow = NULL; struct neat_flow_operations ops; @@ -358,7 +377,7 @@ main(int argc, char *argv[]) result = EXIT_SUCCESS; - while ((arg = getopt(argc, argv, "P:v:")) != -1) { + while ((arg = getopt(argc, argv, "P:v:k")) != -1) { switch(arg) { case 'P': if (read_file(optarg, &arg_property) < 0) { @@ -377,6 +396,12 @@ main(int argc, char *argv[]) printf("option - log level: %d\n", config_log_level); } break; + case 'k': + config_keep_alive = 1; + if (config_log_level >= 1) { + printf("option - Keep-Alive: %d\n", config_keep_alive); + } + break; default: print_usage(); goto cleanup; @@ -436,7 +461,11 @@ main(int argc, char *argv[]) } if (chdir("htdocs")) { - printf("%s - chdir failed\n", __func__); + fprintf(stderr, "%s - chdir failed\n", __func__); + } + + if (signal(SIGINT, sig_handler) == SIG_ERR) { + fprintf(stderr, "%s - can not register SIGINT\n", __func__); } neat_start_event_loop(ctx, NEAT_RUN_DEFAULT); From 7ade86d1acc3085176bc61dfe82b67ece157b725 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Fri, 19 May 2017 03:39:57 +0200 Subject: [PATCH 58/85] avoid asprintf --- examples/server_http.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/examples/server_http.c b/examples/server_http.c index 3558dbe9..cb4867a8 100644 --- a/examples/server_http.c +++ b/examples/server_http.c @@ -69,12 +69,13 @@ static int prepare_http_response(struct http_flow *http_flow, unsigned char **buffer, uint32_t *buffer_len) { int header_length = 0; - unsigned char *header_buffer = NULL; int payload_length = 0; unsigned char *payload_buffer = NULL; + unsigned char *header_buffer = NULL; int i = 0; char misc_buffer[512]; + if (config_log_level >= 2) { fprintf(stderr, "%s()\n", __func__); } @@ -127,8 +128,14 @@ prepare_http_response(struct http_flow *http_flow, unsigned char **buffer, uint3 exit(EXIT_FAILURE); } + header_buffer = malloc(512); + if (header_buffer == NULL) { + fprintf(stderr, "%s - malloc failed\n", __func__); + exit(EXIT_FAILURE); + } + // prepare response header - header_length = asprintf((char **)&header_buffer, + header_length = snprintf((char *) header_buffer, 1024, "HTTP/1.1 200 OK\r\n" "Server: NEAT super fancy webserver\r\n" "Content-Length: %u\r\n" From 15e8983504f82f4f0e5d765a3b68065a4e97161c Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Fri, 19 May 2017 03:50:23 +0200 Subject: [PATCH 59/85] buffersize via macro --- examples/server_http.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/server_http.c b/examples/server_http.c index cb4867a8..600906f3 100644 --- a/examples/server_http.c +++ b/examples/server_http.c @@ -33,6 +33,7 @@ static char *config_property = "{\ static uint8_t config_log_level = 1; static uint8_t config_keep_alive = 0; #define BUFFERSIZE 33768 +#define BUFFERSIZE_SMALL 1024 struct neat_ctx *ctx = NULL; @@ -73,7 +74,7 @@ prepare_http_response(struct http_flow *http_flow, unsigned char **buffer, uint3 unsigned char *payload_buffer = NULL; unsigned char *header_buffer = NULL; int i = 0; - char misc_buffer[512]; + char misc_buffer[BUFFERSIZE_SMALL]; if (config_log_level >= 2) { @@ -83,7 +84,7 @@ prepare_http_response(struct http_flow *http_flow, unsigned char **buffer, uint3 // iterate header fields for (i = 0; i < (int)http_flow->num_headers; i++) { // build string from name/value pair - snprintf(misc_buffer, 512, "%.*s: %.*s", + snprintf(misc_buffer, BUFFERSIZE_SMALL, "%.*s: %.*s", (int)http_flow->headers[i].name_len, http_flow->headers[i].name, (int)http_flow->headers[i].value_len, @@ -128,14 +129,14 @@ prepare_http_response(struct http_flow *http_flow, unsigned char **buffer, uint3 exit(EXIT_FAILURE); } - header_buffer = malloc(512); + header_buffer = malloc(BUFFERSIZE_SMALL); if (header_buffer == NULL) { fprintf(stderr, "%s - malloc failed\n", __func__); exit(EXIT_FAILURE); } // prepare response header - header_length = snprintf((char *) header_buffer, 1024, + header_length = snprintf((char *) header_buffer, BUFFERSIZE_SMALL, "HTTP/1.1 200 OK\r\n" "Server: NEAT super fancy webserver\r\n" "Content-Length: %u\r\n" From 0ccfa9b7eede30a2636169cae8614d373f7fc9f2 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Fri, 19 May 2017 05:24:07 +0200 Subject: [PATCH 60/85] fix explicit EOR on server side --- neat_core.c | 108 +++++++++++++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 57 deletions(-) diff --git a/neat_core.c b/neat_core.c index d55883e2..ee559fa9 100644 --- a/neat_core.c +++ b/neat_core.c @@ -198,7 +198,7 @@ neat_start_event_loop(struct neat_ctx *nc, neat_run_mode run_mode) void neat_stop_event_loop(struct neat_ctx *nc) { - neat_log(nc, NEAT_LOG_DEBUG, "%s", __func__); + //neat_log(nc, NEAT_LOG_DEBUG, "%s", __func__); uv_stop(nc->loop); } @@ -2197,9 +2197,6 @@ he_connected_cb(uv_poll_t *handle, int status, int events) // TODO: // flow->ctx = he_ctx->nc; - - - //flow->isSCTPExplicitEOR = candidate->isSCTPExplicitEOR; flow->isPolling = 1; send_result_connection_attempt_to_pm(flow->ctx, flow, he_res, true); @@ -4707,11 +4704,11 @@ neat_write_flush(struct neat_ctx *ctx, struct neat_flow *flow) sndinfo = (struct sctp_sndinfo *)CMSG_DATA(cmsg); memset(sndinfo, 0, sizeof(struct sctp_sndinfo)); sndinfo->snd_sid = msg->stream_id; -#if defined(SCTP_EOR) +#if defined(SCTP_EXPLICIT_EOR) if ((flow->socket->sctp_explicit_eor) && (len == msg->bufferedSize)) { sndinfo->snd_flags |= SCTP_EOR; } -#endif // defined(SCTP_EOR) +#endif // defined(SCTP_EXPLICIT_EOR) #elif defined (SCTP_SNDRCV) msghdr.msg_control = cmsgbuf; msghdr.msg_controllen = CMSG_SPACE(sizeof(struct sctp_sndrcvinfo)); @@ -4722,11 +4719,11 @@ neat_write_flush(struct neat_ctx *ctx, struct neat_flow *flow) sndrcvinfo = (struct sctp_sndrcvinfo *)CMSG_DATA(cmsg); memset(sndrcvinfo, 0, sizeof(struct sctp_sndrcvinfo)); sndrcvinfo->sinfo_stream = msg->stream_id; -#if defined(SCTP_EOR) - if ((flow->isSCTPExplicitEOR) && (len == msg->bufferedSize)) { +#if defined(SCTP_EXPLICIT_EOR) + if ((flow->socket->sctp_explicit_eor) && (len == msg->bufferedSize)) { sndrcvinfo->sinfo_flags |= SCTP_EOR; } -#endif // defined(SCTP_EOR) +#endif // defined(SCTP_EXPLICIT_EOR) #else // defined(SCTP_SNDINFO) msghdr.msg_control = NULL; msghdr.msg_controllen = 0; @@ -4848,6 +4845,7 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, size_t len; int atomic; neat_error_code code = NEAT_OK; + int flags = 0; #ifdef NEAT_SCTP_DTLS struct security_data *private = NULL; #endif @@ -4950,11 +4948,11 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, } else { len = amt; } - iov.iov_len = len; - msghdr.msg_name = NULL; - msghdr.msg_namelen = 0; - msghdr.msg_iov = &iov; - msghdr.msg_iovlen = 1; + iov.iov_len = len; + msghdr.msg_name = NULL; + msghdr.msg_namelen = 0; + msghdr.msg_iov = &iov; + msghdr.msg_iovlen = 1; if (neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) { #ifdef NEAT_SCTP_DTLS @@ -4981,11 +4979,11 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, sndinfo->snd_sid = stream_id; } -#if defined(SCTP_EOR) +#if defined(SCTP_EXPLICIT_EOR) if ((flow->socket->sctp_explicit_eor) && (len == amt)) { sndinfo->snd_flags |= SCTP_EOR; } -#endif +#endif // defined(SCTP_EXPLICIT_EOR) #elif defined (SCTP_SNDRCV) msghdr.msg_control = cmsgbuf; msghdr.msg_controllen = CMSG_SPACE(sizeof(struct sctp_sndrcvinfo)); @@ -4999,11 +4997,12 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, if (stream_id) { sndrcvinfo->sinfo_stream = stream_id; } -#if defined(SCTP_EOR) - if ((flow->isSCTPExplicitEOR) && (len == amt)) { + +#if defined(SCTP_EXPLICIT_EOR) + if ((flow->socket->sctp_explicit_eor) && (len == amt)) { sndrcvinfo->sinfo_flags |= SCTP_EOR; } -#endif +#endif // defined(SCTP_EXPLICIT_EOR) #else msghdr.msg_control = NULL; msghdr.msg_controllen = 0; @@ -5020,31 +5019,28 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, if (flow->security_needed && neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) { rv = SSL_write(private->ssl, buffer, len); } else { - rv = sendmsg(flow->socket->fd, (const struct msghdr *)&msghdr, -#ifndef MSG_NOSIGNAL - 0 -#else - MSG_NOSIGNAL #endif - ); - } -#else - rv = sendmsg(flow->socket->fd, (const struct msghdr *)&msghdr, + #ifndef MSG_NOSIGNAL - 0 -#else - MSG_NOSIGNAL -#endif - ); + flags = 0; +#else // MSG_NOSIGNAL + flags = MSG_NOSIGNAL; +#endif // MSG_NOSIGNAL + + rv = sendmsg(flow->socket->fd, (const struct msghdr *)&msghdr, flags); +#ifdef NEAT_SCTP_DTLS + } #endif + } else { #if defined(USRSCTP_SUPPORT) neat_log(ctx, NEAT_LOG_INFO, "%s - send %zd bytes on flow %p and socket %p", __func__, len, (void *)flow, (void *)flow->socket->usrsctp_socket); rv = usrsctp_sendv(flow->socket->usrsctp_socket, buffer, len, NULL, 0, (void *)sndinfo, (socklen_t)sizeof(struct sctp_sndinfo), SCTP_SENDV_SNDINFO, 0); - if (rv < 0) + if (rv < 0) { perror("usrsctp_sendv"); + } #endif } #ifdef IPPROTO_SCTP @@ -5531,23 +5527,22 @@ neat_connect(struct neat_he_candidate *candidate, uv_poll_cb callback_fx) IPPROTO_SCTP, SCTP_EXPLICIT_EOR, &enable, - sizeof(int)) == 0) + sizeof(int)) == 0) { candidate->pollable_socket->sctp_explicit_eor = 1; + } } else { #ifndef NEAT_SCTP_DTLS if (setsockopt(candidate->pollable_socket->fd, IPPROTO_SCTP, SCTP_EXPLICIT_EOR, &enable, - sizeof(int)) == 0) + sizeof(int)) == 0) { candidate->pollable_socket->sctp_explicit_eor = 1; - else - candidate->pollable_socket->sctp_explicit_eor = 0; -#else - candidate->pollable_socket->sctp_explicit_eor = 0; -#endif + } +#endif // NEAT_SCTP_DTLS } -#endif +#endif // SCTP_EXPLICIT_EOR + #ifndef USRSCTP_SUPPORT // Subscribe to events needed for callbacks neat_sctp_init_events(candidate->pollable_socket->fd); @@ -5760,8 +5755,7 @@ neat_close_via_kernel_2(struct neat_ctx *ctx, int fd) } static int -neat_listen_via_kernel(struct neat_ctx *ctx, struct neat_flow *flow, - struct neat_pollable_socket *listen_socket) +neat_listen_via_kernel(struct neat_ctx *ctx, struct neat_flow *flow, struct neat_pollable_socket *listen_socket) { // TODO: This function should not write to any fields in neat_flow int enable = 1; @@ -5915,7 +5909,7 @@ neat_listen_via_kernel(struct neat_ctx *ctx, struct neat_flow *flow, } neat_log(ctx, NEAT_LOG_DEBUG, "Offering %d SCTP streams in/out", initmsg.sinit_num_ostreams); #endif // defined(SCTP_INITMSG) && !defined(USRSCTP_SUPPORT) - flow->socket->write_limit = flow->socket->write_size / 4; + listen_socket->write_limit = listen_socket->write_size / 4; #ifdef SCTP_NODELAY if (setsockopt(fd, IPPROTO_SCTP, SCTP_NODELAY, &enable, sizeof(int)) != 0) neat_log(ctx, NEAT_LOG_DEBUG, "Unable to set socket option IPPROTO_SCTP:SCTP_NODELAY"); @@ -5923,21 +5917,20 @@ neat_listen_via_kernel(struct neat_ctx *ctx, struct neat_flow *flow, #ifdef SCTP_EXPLICIT_EOR if (!flow->security_needed) { if (setsockopt(fd, IPPROTO_SCTP, SCTP_EXPLICIT_EOR, &enable, sizeof(int)) == 0) { - flow->socket->sctp_explicit_eor = 1; + listen_socket->sctp_explicit_eor = 1; } else { neat_log(ctx, NEAT_LOG_DEBUG, "Unable to set socket option IPPROTO_SCTP:SCTP_EXPLICIT_EOR"); } } else { #ifndef NEAT_SCTP_DTLS - if (setsockopt(fd, IPPROTO_SCTP, SCTP_EXPLICIT_EOR, &enable, sizeof(int)) == 0) - flow->socket->sctp_explicit_eor = 1; - else - flow->socket->sctp_explicit_eor = 0; -#else - flow->socket->sctp_explicit_eor = 0; -#endif + if (setsockopt(fd, IPPROTO_SCTP, SCTP_EXPLICIT_EOR, &enable, sizeof(int)) == 0) { + listen_socket->sctp_explicit_eor = 1; + } else { + neat_log(ctx, NEAT_LOG_DEBUG, "Unable to set socket option IPPROTO_SCTP:SCTP_EXPLICIT_EOR"); + } +#endif // NEAT_SCTP_DTLS } -#endif +#endif // SCTP_EXPLICIT_EOR break; default: break; @@ -6222,8 +6215,9 @@ neat_connect_via_usrsctp(struct neat_he_candidate *candidate) usrsctp_setsockopt(candidate->pollable_socket->usrsctp_socket, IPPROTO_SCTP, SCTP_NODELAY, &enable, sizeof(int)); #endif #ifdef SCTP_EXPLICIT_EOR - if (usrsctp_setsockopt(candidate->pollable_socket->usrsctp_socket, IPPROTO_SCTP, SCTP_EXPLICIT_EOR, &enable, sizeof(int)) == 0) + if (usrsctp_setsockopt(candidate->pollable_socket->usrsctp_socket, IPPROTO_SCTP, SCTP_EXPLICIT_EOR, &enable, sizeof(int)) == 0) { candidate->pollable_socket->sctp_explicit_eor = 1; + } #endif if (candidate->pollable_socket->flow->isSCTPMultihoming && neat_base_stack(candidate->pollable_socket->stack) == NEAT_STACK_SCTP && candidate->pollable_socket->nr_local_addr > 0) { @@ -7298,7 +7292,7 @@ neat_sctp_open_stream(struct neat_pollable_socket *socket, uint16_t sid) //memset(sndinfo, 0, sizeof(struct sctp_sndinfo)); sndinfo->snd_sid = sid; sndinfo->snd_ppid = htonl(1207); -#if defined(SCTP_EOR) +#if defined(SCTP_EXPLICIT_EOR) sndinfo->snd_flags |= SCTP_EOR; #endif #elif defined (SCTP_SNDRCV) @@ -7313,7 +7307,7 @@ neat_sctp_open_stream(struct neat_pollable_socket *socket, uint16_t sid) //memset(sndrcvinfo, 0, sizeof(struct sctp_sndrcvinfo)); sndrcvinfo->sinfo_stream = sid; sndrcvinfo->sinfo_ppid = htonl(1207); -#if defined(SCTP_EOR) +#if defined(SCTP_EXPLICIT_EOR) sndrcvinfo->sinfo_flags |= SCTP_EOR; #endif #else From 9e4ef4907a218b71a0f8689a79185fb5ac74b8ff Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Sun, 21 May 2017 00:32:51 +0200 Subject: [PATCH 61/85] minor refactoring --- examples/picohttpparser.c | 0 examples/picohttpparser.h | 0 neat_core.c | 215 +++++++++++++++++++------------------- neat_security.c | 6 +- 4 files changed, 111 insertions(+), 110 deletions(-) mode change 100755 => 100644 examples/picohttpparser.c mode change 100755 => 100644 examples/picohttpparser.h diff --git a/examples/picohttpparser.c b/examples/picohttpparser.c old mode 100755 new mode 100644 diff --git a/examples/picohttpparser.h b/examples/picohttpparser.h old mode 100755 new mode 100644 diff --git a/neat_core.c b/neat_core.c index ee559fa9..b9de35c0 100644 --- a/neat_core.c +++ b/neat_core.c @@ -2552,11 +2552,11 @@ do_accept(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *listen_so newFlow->operations->userData = flow->operations->userData; #ifdef NEAT_SCTP_DTLS - if (flow->security_needed && newFlow->socket->stack == NEAT_STACK_SCTP) { - copy_dtls_data(newFlow->socket, listen_socket); - struct security_data *server = (struct security_data *) listen_socket->dtls_data->userData; - SSL_CTX_up_ref(server->ctx); - } + if (flow->security_needed && newFlow->socket->stack == NEAT_STACK_SCTP) { + copy_dtls_data(newFlow->socket, listen_socket); + struct security_data *server = (struct security_data *) listen_socket->dtls_data->userData; + SSL_CTX_up_ref(server->ctx); + } #endif #if defined(SO_NOSIGPIPE) @@ -2568,118 +2568,120 @@ do_accept(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *listen_so #endif // defined(SO_NOSIGPIPE) switch (newFlow->socket->stack) { - case NEAT_STACK_SCTP_UDP: - case NEAT_STACK_SCTP: + case NEAT_STACK_SCTP_UDP: + case NEAT_STACK_SCTP: #if defined(USRSCTP_SUPPORT) - newFlow->socket->usrsctp_socket = newFlow->acceptusrsctpfx(ctx, newFlow, listen_socket); - if (!newFlow->socket->usrsctp_socket) { - neat_free_flow(newFlow); - return NULL; - } else { - neat_log(ctx, NEAT_LOG_DEBUG, "USRSCTP io_connected"); - io_connected(ctx, newFlow, NEAT_OK); - neat_sctp_init_events(newFlow->socket->usrsctp_socket); - newFlow->acceptPending = 0; - } + newFlow->socket->usrsctp_socket = newFlow->acceptusrsctpfx(ctx, newFlow, listen_socket); + if (!newFlow->socket->usrsctp_socket) { + neat_free_flow(newFlow); + return NULL; + } else { + neat_log(ctx, NEAT_LOG_DEBUG, "USRSCTP io_connected"); + io_connected(ctx, newFlow, NEAT_OK); + neat_sctp_init_events(newFlow->socket->usrsctp_socket); + newFlow->acceptPending = 0; + } #else - neat_log(ctx, NEAT_LOG_DEBUG, "Creating new SCTP socket"); - newFlow->socket->fd = newFlow->acceptfx(ctx, newFlow, listen_socket->fd); - if (newFlow->socket->fd == -1) { - neat_free_flow(newFlow); - return NULL; - } else { + neat_log(ctx, NEAT_LOG_DEBUG, "Creating new SCTP socket"); + newFlow->socket->fd = newFlow->acceptfx(ctx, newFlow, listen_socket->fd); + if (newFlow->socket->fd == -1) { + neat_free_flow(newFlow); + return NULL; + } else { #ifndef USRSCTP_SUPPORT - // Subscribe to events needed for callbacks - neat_sctp_init_events(newFlow->socket->fd); + // Subscribe to events needed for callbacks + neat_sctp_init_events(newFlow->socket->fd); #endif - uv_poll_init(ctx->loop, newFlow->socket->handle, newFlow->socket->fd); // makes fd nb as side effect - newFlow->socket->handle->data = newFlow->socket; + uv_poll_init(ctx->loop, newFlow->socket->handle, newFlow->socket->fd); // makes fd nb as side effect + newFlow->socket->handle->data = newFlow->socket; #ifdef NEAT_SCTP_DTLS - if (newFlow->security_needed) { - struct security_data *private = (struct security_data *) newFlow->socket->dtls_data->userData; - private->ssl = SSL_new(private->ctx); - private->dtlsBIO = BIO_new_dgram_sctp(newFlow->socket->fd, BIO_CLOSE); - if (private->dtlsBIO != NULL) { - SSL_set_bio(private->ssl, private->dtlsBIO, private->dtlsBIO); - } else { - neat_log(ctx, NEAT_LOG_ERROR, "Creating new BIO failed"); + if (newFlow->security_needed) { + struct security_data *private = (struct security_data *) newFlow->socket->dtls_data->userData; + private->ssl = SSL_new(private->ctx); + private->dtlsBIO = BIO_new_dgram_sctp(newFlow->socket->fd, BIO_CLOSE); + if (private->dtlsBIO != NULL) { + SSL_set_bio(private->ssl, private->dtlsBIO, private->dtlsBIO); + } else { + neat_log(ctx, NEAT_LOG_ERROR, "Creating new BIO failed"); + } } - } #endif - io_connected(ctx, newFlow, NEAT_OK); - uvpollable_cb(newFlow->socket->handle, NEAT_OK, 0); - } + io_connected(ctx, newFlow, NEAT_OK); + uvpollable_cb(newFlow->socket->handle, NEAT_OK, 0); + } #if defined(SCTP_RECVRCVINFO) - // Enable anciliarry data when receiving data from SCTP - optval = 1; - rc = setsockopt(newFlow->socket->fd, IPPROTO_SCTP, SCTP_RECVRCVINFO, &optval, sizeof(optval)); - if (rc < 0) - neat_log(ctx, NEAT_LOG_DEBUG, "Call to setsockopt(SCTP_RECVRCVINFO) failed"); + // Enable anciliarry data when receiving data from SCTP + optval = 1; + rc = setsockopt(newFlow->socket->fd, IPPROTO_SCTP, SCTP_RECVRCVINFO, &optval, sizeof(optval)); + if (rc < 0) { + neat_log(ctx, NEAT_LOG_DEBUG, "Call to setsockopt(SCTP_RECVRCVINFO) failed"); + } #endif // defined(SCTP_RECVRCVINFO) #if defined(SCTP_RECVNXTINFO) - // Enable anciliarry data when receiving data from SCTP - optval = 1; - rc = setsockopt(newFlow->socket->fd, IPPROTO_SCTP, SCTP_RECVNXTINFO, &optval, sizeof(optval)); - if (rc < 0) - neat_log(ctx, NEAT_LOG_DEBUG, "Call to setsockopt(SCTP_RECVNXTINFO) failed"); + // Enable anciliarry data when receiving data from SCTP + optval = 1; + rc = setsockopt(newFlow->socket->fd, IPPROTO_SCTP, SCTP_RECVNXTINFO, &optval, sizeof(optval)); + if (rc < 0) { + neat_log(ctx, NEAT_LOG_DEBUG, "Call to setsockopt(SCTP_RECVNXTINFO) failed"); + } #endif // defined(SCTP_RECVNXTINFO) #endif - break; - case NEAT_STACK_UDP: - neat_log(ctx, NEAT_LOG_DEBUG, "Creating new UDP socket"); - newFlow->socket->fd = socket(newFlow->socket->family, newFlow->socket->type, IPPROTO_UDP); + break; + case NEAT_STACK_UDP: + neat_log(ctx, NEAT_LOG_DEBUG, "Creating new UDP socket"); + newFlow->socket->fd = socket(newFlow->socket->family, newFlow->socket->type, IPPROTO_UDP); - if (newFlow->socket->fd == -1) { - neat_free_flow(newFlow); - return NULL; - } else { + if (newFlow->socket->fd == -1) { + neat_free_flow(newFlow); + return NULL; + } else { - optval = 1; - setsockopt(newFlow->socket->fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); - setsockopt(newFlow->socket->fd, SOL_SOCKET, SO_REUSEPORT, &optval, sizeof(optval)); + optval = 1; + setsockopt(newFlow->socket->fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); + setsockopt(newFlow->socket->fd, SOL_SOCKET, SO_REUSEPORT, &optval, sizeof(optval)); - bind(newFlow->socket->fd, (struct sockaddr*) &newFlow->socket->src_sockaddr, sizeof(struct sockaddr)); - connect(newFlow->socket->fd, (struct sockaddr*) &newFlow->socket->dst_sockaddr, sizeof(struct sockaddr)); + bind(newFlow->socket->fd, (struct sockaddr*) &newFlow->socket->src_sockaddr, sizeof(struct sockaddr)); + connect(newFlow->socket->fd, (struct sockaddr*) &newFlow->socket->dst_sockaddr, sizeof(struct sockaddr)); - newFlow->everConnected = 1; + newFlow->everConnected = 1; - uv_poll_init(ctx->loop, newFlow->socket->handle, newFlow->socket->fd); // makes fd nb as side effect + uv_poll_init(ctx->loop, newFlow->socket->handle, newFlow->socket->fd); // makes fd nb as side effect - newFlow->socket->handle->data = newFlow->socket; + newFlow->socket->handle->data = newFlow->socket; - io_connected(ctx, newFlow, NEAT_OK); - uvpollable_cb(newFlow->socket->handle, NEAT_OK, 0); - } - break; - case NEAT_STACK_UDPLITE: + io_connected(ctx, newFlow, NEAT_OK); + uvpollable_cb(newFlow->socket->handle, NEAT_OK, 0); + } + break; + case NEAT_STACK_UDPLITE: #if defined(__NetBSD__) || defined(__APPLE__) - assert(0); // Should not reach this point + assert(0); // Should not reach this point #else - neat_log(ctx, NEAT_LOG_DEBUG, "Creating new UDPLite socket"); - newFlow->socket->fd = socket(newFlow->socket->family, newFlow->socket->type, IPPROTO_UDPLITE); + neat_log(ctx, NEAT_LOG_DEBUG, "Creating new UDPLite socket"); + newFlow->socket->fd = socket(newFlow->socket->family, newFlow->socket->type, IPPROTO_UDPLITE); - if (newFlow->socket->fd == -1) { - neat_free_flow(newFlow); - return NULL; - } else { - setsockopt(newFlow->socket->fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); - setsockopt(newFlow->socket->fd, SOL_SOCKET, SO_REUSEPORT, &optval, sizeof(optval)); + if (newFlow->socket->fd == -1) { + neat_free_flow(newFlow); + return NULL; + } else { + setsockopt(newFlow->socket->fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); + setsockopt(newFlow->socket->fd, SOL_SOCKET, SO_REUSEPORT, &optval, sizeof(optval)); - bind(newFlow->socket->fd, (struct sockaddr*) &newFlow->socket->src_sockaddr, sizeof(struct sockaddr)); - connect(newFlow->socket->fd, (struct sockaddr*) &newFlow->socket->dst_sockaddr, sizeof(struct sockaddr)); + bind(newFlow->socket->fd, (struct sockaddr*) &newFlow->socket->src_sockaddr, sizeof(struct sockaddr)); + connect(newFlow->socket->fd, (struct sockaddr*) &newFlow->socket->dst_sockaddr, sizeof(struct sockaddr)); - newFlow->everConnected = 1; + newFlow->everConnected = 1; - uv_poll_init(ctx->loop, newFlow->socket->handle, newFlow->socket->fd); // makes fd nb as side effect + uv_poll_init(ctx->loop, newFlow->socket->handle, newFlow->socket->fd); // makes fd nb as side effect - newFlow->socket->handle->data = newFlow->socket; + newFlow->socket->handle->data = newFlow->socket; - io_connected(ctx, newFlow, NEAT_OK); - uvpollable_cb(newFlow->socket->handle, NEAT_OK, 0); - } + io_connected(ctx, newFlow, NEAT_OK); + uvpollable_cb(newFlow->socket->handle, NEAT_OK, 0); + } #endif - break; + break; default: newFlow->socket->fd = newFlow->acceptfx(ctx, newFlow, listen_socket->fd); if (newFlow->socket->fd == -1) { @@ -2713,8 +2715,7 @@ do_accept(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *listen_so newFlow->acceptPending = 0; // xxx patrick? - if ((false) && - (newFlow->socket->stack == NEAT_STACK_TCP)) { + if ((newFlow->security_needed) && (newFlow->socket->stack == NEAT_STACK_TCP)) { neat_log(ctx, NEAT_LOG_DEBUG, "TCP Server Security"); if (neat_security_install(newFlow->ctx, newFlow) != NEAT_OK) { neat_io_error(flow->ctx, flow, NEAT_ERROR_SECURITY); @@ -2728,26 +2729,26 @@ do_accept(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *listen_so switch (newFlow->socket->stack) { #if defined(IPPROTO_SCTP) && defined(SCTP_STATUS) - case NEAT_STACK_SCTP: - optlen = sizeof(status); - rc = getsockopt(newFlow->socket->fd, IPPROTO_SCTP, SCTP_STATUS, &status, &optlen); - if (rc < 0) { - neat_log(ctx, NEAT_LOG_DEBUG, "Call to getsockopt(SCTP_STATUS) failed"); - newFlow->socket->sctp_streams_available = 1; - } else { - newFlow->socket->sctp_streams_available = MIN(status.sstat_instrms, status.sstat_outstrms); - } + case NEAT_STACK_SCTP: + optlen = sizeof(status); + rc = getsockopt(newFlow->socket->fd, IPPROTO_SCTP, SCTP_STATUS, &status, &optlen); + if (rc < 0) { + neat_log(ctx, NEAT_LOG_DEBUG, "Call to getsockopt(SCTP_STATUS) failed"); + newFlow->socket->sctp_streams_available = 1; + } else { + newFlow->socket->sctp_streams_available = MIN(status.sstat_instrms, status.sstat_outstrms); + } #ifdef SCTP_MULTISTREAMING - newFlow->socket->sctp_streams_used = 1; - newFlow->multistream_id = 0; + newFlow->socket->sctp_streams_used = 1; + newFlow->multistream_id = 0; #endif - // number of outbound streams == number of inbound streams - neat_log(ctx, NEAT_LOG_DEBUG, "%s - SCTP - number of streams: %d", __func__, newFlow->socket->sctp_streams_available); - break; + // number of outbound streams == number of inbound streams + neat_log(ctx, NEAT_LOG_DEBUG, "%s - SCTP - number of streams: %d", __func__, newFlow->socket->sctp_streams_available); + break; #endif - default: - //newFlow->sockestream_count = 1; - break; + default: + //newFlow->sockestream_count = 1; + break; } return newFlow; diff --git a/neat_security.c b/neat_security.c index 2ca20fa1..fdc6c357 100644 --- a/neat_security.c +++ b/neat_security.c @@ -138,9 +138,9 @@ neat_security_handshake(struct neat_flow_operations *opCB) filter->readfx == neat_security_filter_read) { struct security_data *private = (struct security_data *) filter->userData; // pop application functions back onto stack - opCB->on_writable = private->pushed_on_writable; - opCB->on_readable = private->pushed_on_readable; - opCB->on_connected = private->pushed_on_connected; + opCB->on_writable = private->pushed_on_writable; + opCB->on_readable = private->pushed_on_readable; + opCB->on_connected = private->pushed_on_connected; neat_set_operations(opCB->ctx, opCB->flow, opCB); // call on_connected From 450233fb8949942cccbe3b476ccfd29d5300fa8b Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 23 May 2017 01:23:42 +0200 Subject: [PATCH 62/85] https support for webserver --- examples/server_http.c | 122 ++++++++++++++++----- neat_security.c | 235 ++++++++++++++++++++++++----------------- 2 files changed, 231 insertions(+), 126 deletions(-) diff --git a/examples/server_http.c b/examples/server_http.c index 600906f3..50e21b54 100644 --- a/examples/server_http.c +++ b/examples/server_http.c @@ -11,6 +11,7 @@ #include "util.h" #include "picohttpparser.h" +#define QUOTE(...) #__VA_ARGS__ /********************************************************************** @@ -18,20 +19,40 @@ **********************************************************************/ -static char *config_property = "{\ - \"transport\": [\ - {\ - \"value\": \"SCTP\",\ - \"precedence\": 1\ - },\ - {\ - \"value\": \"TCP\",\ - \"precedence\": 1\ - }\ - ]\ -}"; +static char *config_property = QUOTE( + { + "transport": [ + { + "value": "SCTP", + "precedence": 1 + }, + { + "value": "TCP", + "precedence": 1 + } + ] + } +); + +static char *config_property_https = QUOTE( + { + "transport": [ + { + "value": "TCP", + "precedence": 1 + } + ], + "security" : + { + "value": true, + "precedence": 2 + } + } +); static uint8_t config_log_level = 1; static uint8_t config_keep_alive = 0; +static uint8_t config_https = 1; +static char *htdocs_directory = "htdocs"; // without trailing slash!! #define BUFFERSIZE 33768 #define BUFFERSIZE_SMALL 1024 struct neat_ctx *ctx = NULL; @@ -39,7 +60,6 @@ struct neat_ctx *ctx = NULL; static char *http_header_connection_keep_alive = "Connection: Keep-Alive"; - static neat_error_code on_writable(struct neat_flow_operations *opCB); struct http_flow { @@ -101,13 +121,13 @@ prepare_http_response(struct http_flow *http_flow, unsigned char **buffer, uint3 // XXX needs refactoring - just shit ... shame on me... :/ if (http_flow->path_len == 1 && http_flow->path[0] == '/') { // request root "/" --> index.html - snprintf(misc_buffer, sizeof(misc_buffer), "index.html"); + snprintf(misc_buffer, sizeof(misc_buffer), "%s/index.html", htdocs_directory); } else if (http_flow->path_len > 1 && http_flow->path[0] == '/') { // path begins with "/" - snprintf(misc_buffer, sizeof(misc_buffer), "%.*s", (int)http_flow->path_len - 1, http_flow->path + 1); + snprintf(misc_buffer, sizeof(misc_buffer), "%s/%.*s", htdocs_directory, (int)http_flow->path_len - 1, http_flow->path + 1); } else { // path does not begin with "/" - snprintf(misc_buffer, sizeof(misc_buffer), "%.*s", (int)http_flow->path_len, http_flow->path); + snprintf(misc_buffer, sizeof(misc_buffer), "%s/%.*s", htdocs_directory, (int)http_flow->path_len, http_flow->path); } // try to read requested file @@ -119,7 +139,7 @@ prepare_http_response(struct http_flow *http_flow, unsigned char **buffer, uint3 if (config_log_level >= 1) { fprintf(stderr, "%s - reading >>%s<< failed - delivering index.html\n", __func__, misc_buffer); } - snprintf(misc_buffer, sizeof(misc_buffer), "index.html"); + snprintf(misc_buffer, sizeof(misc_buffer), "htdocs/index.html"); payload_length = read_file(misc_buffer, (char **) &payload_buffer); } @@ -197,6 +217,7 @@ on_error(struct neat_flow_operations *opCB) { fprintf(stderr, "%s()\n", __func__); + //return 0; exit(EXIT_FAILURE); } @@ -377,11 +398,14 @@ main(int argc, char *argv[]) { // uint64_t prop; int arg, result; - char *arg_property = NULL; - struct neat_flow *flow = NULL; - struct neat_flow_operations ops; + char *arg_property = NULL; + struct neat_flow *flow_http = NULL; + struct neat_flow *flow_https = NULL; + struct neat_flow_operations ops_http; + struct neat_flow_operations ops_https; - memset(&ops, 0, sizeof(ops)); + memset(&ops_http, 0, sizeof(struct neat_flow_operations)); + memset(&ops_https, 0, sizeof(struct neat_flow_operations)); result = EXIT_SUCCESS; @@ -438,40 +462,80 @@ main(int argc, char *argv[]) } // new neat flow - if ((flow = neat_new_flow(ctx)) == NULL) { + if ((flow_http = neat_new_flow(ctx)) == NULL) { fprintf(stderr, "%s - neat_new_flow failed\n", __func__); result = EXIT_FAILURE; goto cleanup; } // set properties - if (neat_set_property(ctx, flow, arg_property ? arg_property : config_property)) { + if (neat_set_property(ctx, flow_http, arg_property ? arg_property : config_property)) { fprintf(stderr, "%s - neat_set_property failed\n", __func__); result = EXIT_FAILURE; goto cleanup; } // set callbacks - ops.on_connected = on_connected; - ops.on_error = on_error; + ops_http.on_connected = on_connected; + ops_http.on_error = on_error; - if (neat_set_operations(ctx, flow, &ops)) { + if (neat_set_operations(ctx, flow_http, &ops_http)) { fprintf(stderr, "%s - neat_set_operations failed\n", __func__); result = EXIT_FAILURE; goto cleanup; } // wait for on_connected or on_error to be invoked - if (neat_accept(ctx, flow, 8080, NULL, 0)) { + if (neat_accept(ctx, flow_http, 8080, NULL, 0)) { fprintf(stderr, "%s - neat_accept failed\n", __func__); result = EXIT_FAILURE; goto cleanup; } - if (chdir("htdocs")) { - fprintf(stderr, "%s - chdir failed\n", __func__); + if (config_https) { + // new neat flow + if ((flow_https = neat_new_flow(ctx)) == NULL) { + fprintf(stderr, "%s - neat_new_flow failed\n", __func__); + result = EXIT_FAILURE; + goto cleanup; + } + + // set properties + if (neat_set_property(ctx, flow_https, config_property_https)) { + fprintf(stderr, "%s - neat_set_property failed\n", __func__); + result = EXIT_FAILURE; + goto cleanup; + } + + // set callbacks + ops_https.on_connected = on_connected; + ops_https.on_error = on_error; + + if (neat_set_operations(ctx, flow_https, &ops_https)) { + fprintf(stderr, "%s - neat_set_operations failed\n", __func__); + result = EXIT_FAILURE; + goto cleanup; + } + + if (neat_secure_identity(ctx, flow_https, "cert.pem", NEAT_CERT_KEY_PEM)) { + fprintf(stderr, "%s - neat_get_secure_identity failed\n", __func__); + result = EXIT_FAILURE; + goto cleanup; + } + + // wait for on_connected or on_error to be invoked + if (neat_accept(ctx, flow_https, 8081, NULL, 0)) { + fprintf(stderr, "%s - neat_accept failed\n", __func__); + result = EXIT_FAILURE; + goto cleanup; + } + } + //if (chdir("htdocs")) { + // fprintf(stderr, "%s - chdir failed\n", __func__); + //} + if (signal(SIGINT, sig_handler) == SIG_ERR) { fprintf(stderr, "%s - can not register SIGINT\n", __func__); } diff --git a/neat_security.c b/neat_security.c index fdc6c357..bc5d838d 100644 --- a/neat_security.c +++ b/neat_security.c @@ -44,14 +44,19 @@ neat_security_filter_dtor(struct neat_iofilter *filter) } static neat_error_code -drain_output(struct neat_ctx *ctx, struct neat_flow *flow, - struct neat_iofilter *filter, struct neat_tlv optional[], unsigned int opt_count) +drain_output(struct neat_ctx *ctx, + struct neat_flow *flow, + struct neat_iofilter *filter, + struct neat_tlv optional[], + unsigned int opt_count) { neat_error_code rv; struct security_data *private; private = (struct security_data *) filter->userData; int didFilterWrite = 0; + neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); + if (!private->outCipherBufferUsed) { return NEAT_OK; } @@ -92,6 +97,7 @@ gather_input(struct neat_ctx *ctx, struct neat_flow *flow, struct neat_iofilter *filter, struct neat_tlv optional[], unsigned int opt_count) { neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); + struct security_data *private = (struct security_data *) filter->userData; uint32_t actualAmt; uint32_t avail = CIPHER_BUFFER_SIZE - private->inCipherBufferUsed; @@ -112,16 +118,22 @@ gather_input(struct neat_ctx *ctx, struct neat_flow *flow, // todo filters! } static neat_error_code -neat_security_filter_write(struct neat_ctx *ctx, struct neat_flow *flow, +neat_security_filter_write(struct neat_ctx *ctx, + struct neat_flow *flow, struct neat_iofilter *filter, - const unsigned char *buffer, uint32_t amt, - struct neat_tlv optional[], unsigned int opt_count); + const unsigned char *buffer, + uint32_t amt, + struct neat_tlv optional[], + unsigned int opt_count); static neat_error_code -neat_security_filter_read(struct neat_ctx *ctx, struct neat_flow *flow, +neat_security_filter_read(struct neat_ctx *ctx, + struct neat_flow *flow, struct neat_iofilter *filter, - unsigned char *buffer, uint32_t amt, + unsigned char *buffer, + uint32_t amt, uint32_t *actualAmt, - struct neat_tlv optional[], unsigned int opt_count); + struct neat_tlv optional[], + unsigned int opt_count); static neat_error_code neat_security_handshake(struct neat_flow_operations *opCB) @@ -159,18 +171,25 @@ neat_security_handshake(struct neat_flow_operations *opCB) } static neat_error_code -handshake(struct neat_ctx *ctx, struct neat_flow *flow, - struct neat_iofilter *filter, struct neat_tlv optional[], unsigned int opt_count) +handshake(struct neat_ctx *ctx, + struct neat_flow *flow, + struct neat_iofilter *filter, + struct neat_tlv optional[], + unsigned int opt_count) { neat_error_code rv; struct security_data *private; private = (struct security_data *) filter->userData; + + neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); + if (SSL_is_init_finished(private->ssl)) { return NEAT_OK; } int err = SSL_do_handshake(private->ssl); if (err == 1) { + neat_log(ctx, NEAT_LOG_WARNING, "%s - handshake failed", __func__); return NEAT_OK; } @@ -184,11 +203,13 @@ handshake(struct neat_ctx *ctx, struct neat_flow *flow, flow->operations->on_readable = NULL; neat_set_operations(ctx, flow, flow->operations); } else if (err != SSL_ERROR_NONE) { - // ERR_print_errors_fp(stderr); + neat_log(ctx, NEAT_LOG_WARNING, "%s - handshake error", __func__); + ERR_print_errors_fp(stderr); return NEAT_ERROR_SECURITY; } if (SSL_is_init_finished(private->ssl)) { + neat_log(ctx, NEAT_LOG_WARNING, "%s - SSL_is_init_finished", __func__); return NEAT_OK; } @@ -230,10 +251,13 @@ handshake(struct neat_ctx *ctx, struct neat_flow *flow, } static neat_error_code -neat_security_filter_write(struct neat_ctx *ctx, struct neat_flow *flow, +neat_security_filter_write(struct neat_ctx *ctx, + struct neat_flow *flow, struct neat_iofilter *filter, - const unsigned char *buffer, uint32_t amt, - struct neat_tlv optional[], unsigned int opt_count) + const unsigned char *buffer, + uint32_t amt, + struct neat_tlv optional[], + unsigned int opt_count) { neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); neat_error_code rv; @@ -274,11 +298,14 @@ neat_security_filter_write(struct neat_ctx *ctx, struct neat_flow *flow, } static neat_error_code -neat_security_filter_read(struct neat_ctx *ctx, struct neat_flow *flow, +neat_security_filter_read(struct neat_ctx *ctx, + struct neat_flow *flow, struct neat_iofilter *filter, - unsigned char *buffer, uint32_t amt, + unsigned char *buffer, + uint32_t amt, uint32_t *actualAmt, - struct neat_tlv optional[], unsigned int opt_count) + struct neat_tlv optional[], + unsigned int opt_count) { neat_log(ctx, NEAT_LOG_DEBUG, "%s %d", __func__, *actualAmt); struct security_data *private; @@ -337,6 +364,8 @@ neat_security_install(neat_ctx *ctx, neat_flow *flow) #define client_method() TLSv1_2_client_method() #define server_method() TLSv1_2_server_method() #endif + //ERR_load_crypto_strings(); + //SSL_load_error_strings(); int isClient = !flow->isServer; if (flow->socket->stack == NEAT_STACK_TCP) { @@ -366,9 +395,13 @@ neat_security_install(neat_ctx *ctx, neat_flow *flow) return NEAT_ERROR_SECURITY; } - if ((SSL_CTX_use_certificate_file(private->ctx, flow->server_pem, SSL_FILETYPE_PEM) < 0) || - (SSL_CTX_use_PrivateKey_file(private->ctx, flow->server_pem, SSL_FILETYPE_PEM) < 0 )) { - neat_log(ctx, NEAT_LOG_ERROR, "unable to use cert or private key"); + if (SSL_CTX_use_certificate_file(private->ctx, flow->server_pem, SSL_FILETYPE_PEM) != 1) { + neat_log(ctx, NEAT_LOG_ERROR, "unable to use SSL_CTX_use_certificate_file : %s", flow->server_pem); + ERR_print_errors_fp(stderr); + return NEAT_ERROR_SECURITY; + } + if (SSL_CTX_use_PrivateKey_file(private->ctx, flow->server_pem, SSL_FILETYPE_PEM) != 1) { + neat_log(ctx, NEAT_LOG_ERROR, "unable to use SSL_CTX_use_PrivateKey_file : %s", flow->server_pem); return NEAT_ERROR_SECURITY; } } @@ -423,6 +456,8 @@ neat_dtls_dtor(struct neat_dtls_data *dtls) struct security_data *private; private = (struct security_data *) dtls->userData; + neat_log(NEAT_LOG_DEBUG, ctx, "%s", __func__); + // private->outputBIO and private->inputBIO are freed by SSL_free(private->ssl) if (private && private->ssl) { SSL_free(private->ssl); @@ -441,75 +476,77 @@ neat_dtls_dtor(struct neat_dtls_data *dtls) void handle_notifications(BIO *bio, void *context, void *buf) { - struct sctp_assoc_change *sac; - struct sctp_send_failed *ssf; - struct sctp_paddr_change *spc; - struct sctp_remote_error *sre; - union sctp_notification *snp = buf; - char addrbuf[INET6_ADDRSTRLEN]; - const char *ap; - union { - struct sockaddr_in s4; - struct sockaddr_in6 s6; - struct sockaddr_storage ss; - } addr; - - switch (snp->sn_header.sn_type) { - case SCTP_ASSOC_CHANGE: - sac = &snp->sn_assoc_change; - printf("NOTIFICATION: assoc_change: state=%hu, error=%hu, instr=%hu outstr=%hu\n", - sac->sac_state, sac->sac_error, sac->sac_inbound_streams, sac->sac_outbound_streams); - break; - - case SCTP_PEER_ADDR_CHANGE: - spc = &snp->sn_paddr_change; - addr.ss = spc->spc_aaddr; - if (addr.ss.ss_family == AF_INET) { - ap = inet_ntop(AF_INET, &addr.s4.sin_addr, addrbuf, INET6_ADDRSTRLEN); - } else { - ap = inet_ntop(AF_INET6, &addr.s6.sin6_addr, addrbuf, INET6_ADDRSTRLEN); - } - printf("NOTIFICATION: intf_change: %s state=%d, error=%d\n", ap, spc->spc_state, spc->spc_error); - break; - - case SCTP_REMOTE_ERROR: - sre = &snp->sn_remote_error; - printf("NOTIFICATION: remote_error: err=%hu len=%hu\n", ntohs(sre->sre_error), ntohs(sre->sre_length)); - break; - - case SCTP_SEND_FAILED: - ssf = &snp->sn_send_failed; - printf("NOTIFICATION: sendfailed: len=%u err=%d\n", ssf->ssf_length, ssf->ssf_error); - break; - - case SCTP_SHUTDOWN_EVENT: - printf("NOTIFICATION: shutdown event\n"); - break; - - case SCTP_ADAPTATION_INDICATION: - printf("NOTIFICATION: adaptation event\n"); - break; - - case SCTP_PARTIAL_DELIVERY_EVENT: - printf("NOTIFICATION: partial delivery\n"); - break; + struct sctp_assoc_change *sac; + struct sctp_send_failed *ssf; + struct sctp_paddr_change *spc; + struct sctp_remote_error *sre; + union sctp_notification *snp = buf; + char addrbuf[INET6_ADDRSTRLEN]; + const char *ap; + union { + struct sockaddr_in s4; + struct sockaddr_in6 s6; + struct sockaddr_storage ss; + } addr; + + neat_log(NEAT_LOG_DEBUG, ctx, "%s", __func__); + + switch (snp->sn_header.sn_type) { + case SCTP_ASSOC_CHANGE: + sac = &snp->sn_assoc_change; + printf("NOTIFICATION: assoc_change: state=%hu, error=%hu, instr=%hu outstr=%hu\n", + sac->sac_state, sac->sac_error, sac->sac_inbound_streams, sac->sac_outbound_streams); + break; + + case SCTP_PEER_ADDR_CHANGE: + spc = &snp->sn_paddr_change; + addr.ss = spc->spc_aaddr; + if (addr.ss.ss_family == AF_INET) { + ap = inet_ntop(AF_INET, &addr.s4.sin_addr, addrbuf, INET6_ADDRSTRLEN); + } else { + ap = inet_ntop(AF_INET6, &addr.s6.sin6_addr, addrbuf, INET6_ADDRSTRLEN); + } + printf("NOTIFICATION: intf_change: %s state=%d, error=%d\n", ap, spc->spc_state, spc->spc_error); + break; + + case SCTP_REMOTE_ERROR: + sre = &snp->sn_remote_error; + printf("NOTIFICATION: remote_error: err=%hu len=%hu\n", ntohs(sre->sre_error), ntohs(sre->sre_length)); + break; + + case SCTP_SEND_FAILED: + ssf = &snp->sn_send_failed; + printf("NOTIFICATION: sendfailed: len=%u err=%d\n", ssf->ssf_length, ssf->ssf_error); + break; + + case SCTP_SHUTDOWN_EVENT: + printf("NOTIFICATION: shutdown event\n"); + break; + + case SCTP_ADAPTATION_INDICATION: + printf("NOTIFICATION: adaptation event\n"); + break; + + case SCTP_PARTIAL_DELIVERY_EVENT: + printf("NOTIFICATION: partial delivery\n"); + break; #ifdef SCTP_AUTHENTICATION_EVENT - case SCTP_AUTHENTICATION_EVENT: - printf("NOTIFICATION: authentication event\n"); - break; + case SCTP_AUTHENTICATION_EVENT: + printf("NOTIFICATION: authentication event\n"); + break; #endif #ifdef SCTP_SENDER_DRY_EVENT - case SCTP_SENDER_DRY_EVENT: - printf("NOTIFICATION: sender dry event\n"); - break; + case SCTP_SENDER_DRY_EVENT: + printf("NOTIFICATION: sender dry event\n"); + break; #endif - default: - printf("NOTIFICATION: unknown type: %hu\n", snp->sn_header.sn_type); - break; - } + default: + printf("NOTIFICATION: unknown type: %hu\n", snp->sn_header.sn_type); + break; + } } static neat_error_code @@ -523,6 +560,7 @@ neat_dtls_handshake(struct neat_flow_operations *opCB) if (private->state == DTLS_CONNECTING && ((!opCB->flow->isServer && !SSL_in_connect_init(private->ssl)) || ((opCB->flow->isServer && !SSL_in_accept_init(private->ssl))))) { + neat_log(opCB->ctx, NEAT_LOG_DEBUG, "%s: SSL connection established", __func__); private->state = DTLS_CONNECTED; opCB->flow->socket->handle->data = opCB->flow->socket; @@ -562,7 +600,7 @@ neat_dtls_install(neat_ctx *ctx, struct neat_pollable_socket *sock) tls_init_trust_list(private->ctx); } else { private->ctx = SSL_CTX_new(DTLS_server_method()); - // SSL_CTX_set_ecdh_auto(private->ctx, 1); + // SSL_CTX_set_ecdh_auto(private->ctx, 1); if (!(sock->flow->cert_pem)) { neat_log(ctx, NEAT_LOG_ERROR, "Server certificate file not set via neat_secure_identity()"); @@ -570,15 +608,18 @@ neat_dtls_install(neat_ctx *ctx, struct neat_pollable_socket *sock) free(private); return NEAT_ERROR_SECURITY; } + if (!(sock->flow->key_pem)) { neat_log(ctx, NEAT_LOG_ERROR, "Server key file not set via neat_secure_identity()"); free(dtls); free(private); return NEAT_ERROR_SECURITY; } + int pid = getpid(); - if( !SSL_CTX_set_session_id_context(private->ctx, (void*)&pid, sizeof pid) ) - perror("SSL_CTX_set_session_id_context"); + if (!SSL_CTX_set_session_id_context(private->ctx, (void*)&pid, sizeof pid)) { + perror("SSL_CTX_set_session_id_context"); + } if ((SSL_CTX_use_certificate_chain_file(private->ctx, sock->flow->cert_pem) < 0) || (SSL_CTX_use_PrivateKey_file(private->ctx, sock->flow->key_pem, SSL_FILETYPE_PEM) < 0 )) { @@ -720,18 +761,18 @@ neat_security_install(neat_ctx *ctx, neat_flow *flow) neat_error_code neat_secure_identity(neat_ctx *ctx, neat_flow *flow, const char *filename, int pemType) { switch (pemType) { - case NEAT_CERT_PEM: - free(flow->cert_pem); - flow->cert_pem = strdup(filename); - break; - case NEAT_KEY_PEM: - free(flow->key_pem); - flow->key_pem = strdup(filename); - break; - case NEAT_CERT_KEY_PEM: - free(flow->server_pem); - flow->server_pem = strdup(filename); - break; + case NEAT_CERT_PEM: + free(flow->cert_pem); + flow->cert_pem = strdup(filename); + break; + case NEAT_KEY_PEM: + free(flow->key_pem); + flow->key_pem = strdup(filename); + break; + case NEAT_CERT_KEY_PEM: + free(flow->server_pem); + flow->server_pem = strdup(filename); + break; } return NEAT_OK; } From b1c8fddf4761b8aad9f3e50193d36a376278e16c Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 23 May 2017 01:32:16 +0200 Subject: [PATCH 63/85] fix compile error --- neat_security.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/neat_security.c b/neat_security.c index bc5d838d..ffe487da 100644 --- a/neat_security.c +++ b/neat_security.c @@ -456,8 +456,6 @@ neat_dtls_dtor(struct neat_dtls_data *dtls) struct security_data *private; private = (struct security_data *) dtls->userData; - neat_log(NEAT_LOG_DEBUG, ctx, "%s", __func__); - // private->outputBIO and private->inputBIO are freed by SSL_free(private->ssl) if (private && private->ssl) { SSL_free(private->ssl); From 813846f2e4032aeb5f43beab96f21b9a60b26543 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 23 May 2017 01:34:24 +0200 Subject: [PATCH 64/85] fix compile error --- neat_security.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/neat_security.c b/neat_security.c index ffe487da..e80404c3 100644 --- a/neat_security.c +++ b/neat_security.c @@ -487,8 +487,6 @@ void handle_notifications(BIO *bio, void *context, void *buf) { struct sockaddr_storage ss; } addr; - neat_log(NEAT_LOG_DEBUG, ctx, "%s", __func__); - switch (snp->sn_header.sn_type) { case SCTP_ASSOC_CHANGE: sac = &snp->sn_assoc_change; From a82eaa64c75f5ecadda9b6753f00fd7123ca74f9 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Sun, 11 Jun 2017 23:24:29 +0200 Subject: [PATCH 65/85] calloc fix --- neat_core.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/neat_core.c b/neat_core.c index e76324ee..e2b1bdc3 100644 --- a/neat_core.c +++ b/neat_core.c @@ -121,7 +121,7 @@ neat_init_ctx() struct neat_ctx *nc; struct neat_ctx *ctx = NULL; - nc = calloc(sizeof(struct neat_ctx), 1); + nc = calloc(1, sizeof(struct neat_ctx)); if (!nc) { return NULL; @@ -436,7 +436,7 @@ neat_run_event_cb(struct neat_ctx *nc, uint8_t event_type, void *data) struct neat_iofilter * insert_neat_iofilter(neat_ctx *ctx, neat_flow *flow) { - struct neat_iofilter *filter = calloc (1, sizeof (struct neat_iofilter)); + struct neat_iofilter *filter = calloc(1, sizeof (struct neat_iofilter)); if (filter) { filter->next = flow->iofilters; flow->iofilters = filter; @@ -1645,7 +1645,7 @@ io_readable(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *socket, multistream_flow->ctx = ctx; multistream_flow->ownedByCore = 1; multistream_flow->isServer = 1; - multistream_flow->operations = calloc (sizeof(struct neat_flow_operations), 1); + multistream_flow->operations = calloc(1, sizeof(struct neat_flow_operations)); if (!multistream_flow->operations) return READ_WITH_ERROR; multistream_flow->operations->on_connected = listen_flow->operations->on_connected; @@ -2298,7 +2298,7 @@ void uvpollable_cb(uv_poll_t *handle, int status, int events) int so_error = 0; unsigned int len = sizeof(so_error); if (getsockopt(flow->socket->fd, SOL_SOCKET, SO_ERROR, &so_error, &len) < 0) { - neat_log(ctx, NEAT_LOG_DEBUG, "Call to getsockopt failed: %s", strerror(errno)); + neat_log(ctx, NEAT_LOG_DEBUG, "Call to getsockopt failed for FD : %d - %s", flow->socket->fd, strerror(errno)); neat_io_error(ctx, flow, NEAT_ERROR_INTERNAL); return; } @@ -2537,7 +2537,7 @@ do_accept(neat_ctx *ctx, neat_flow *flow, struct neat_pollable_socket *listen_so newFlow->security_needed = flow->security_needed; newFlow->eofSeen = 0; - newFlow->operations = calloc (sizeof(struct neat_flow_operations), 1); + newFlow->operations = calloc(1, sizeof(struct neat_flow_operations)); if (newFlow->operations == NULL) { neat_io_error(ctx, flow, NEAT_ERROR_OUT_OF_MEMORY); return NULL; @@ -6684,7 +6684,7 @@ neat_new_flow(neat_ctx *ctx) neat_flow *flow; neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); - flow = calloc (1, sizeof (struct neat_flow)); + flow = calloc(1, sizeof (struct neat_flow)); if (flow == NULL) { return NULL; } From f6664c6bafd21bcda62b3038e05b994d77fb787f Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Sun, 11 Jun 2017 23:25:52 +0200 Subject: [PATCH 66/85] tneat - loop --- examples/tneat.c | 101 +++++++++++++++++++++++++++++------------------ examples/util.c | 1 + 2 files changed, 64 insertions(+), 38 deletions(-) diff --git a/examples/tneat.c b/examples/tneat.c index 1497ef67..67dffe56 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -25,13 +25,17 @@ /* default values */ +#define NEAT_MODE_CLIENT 1 +#define NEAT_MODE_SERVER 2 +#define NEAT_MODE_LOOP 3 + static uint32_t config_rcv_buffer_size = 10240; static uint32_t config_snd_buffer_size = 1024; -static uint32_t config_message_count = 10; +static uint32_t config_message_count = 1; static uint32_t config_runtime_max = 0; -static uint16_t config_active = 0; +static uint16_t config_mode = 0; static uint16_t config_chargen_offset = 0; -static uint16_t config_port = 23232; +static uint16_t config_port = 1988; static uint16_t config_log_level = 1; static uint16_t config_num_flows = 1; static uint16_t config_max_flows = 100; @@ -80,8 +84,9 @@ struct tneat_flow_direction { }; struct tneat_flow { - struct tneat_flow_direction rcv; - struct tneat_flow_direction snd; + uint8_t active; + struct tneat_flow_direction rcv; + struct tneat_flow_direction snd; }; static neat_error_code on_writable(struct neat_flow_operations *opCB); @@ -107,6 +112,7 @@ print_usage() printf("\t- v \tlog level 0..3 (%d)\n", config_log_level); printf("\t- c \tpath to server certificate (%s)\n", cert_file); printf("\t- k \tpath to server key (%s)\n", key_file); + printf("\t- L \tloop mode - tneat talking to itself\n"); } /* @@ -254,13 +260,11 @@ on_connected(struct neat_flow_operations *opCB) fprintf(stderr, "%s() - connection established\n", __func__); } - if ((opCB->userData = calloc(1, sizeof(struct tneat_flow))) == NULL) { + if ((tnf = calloc(1, sizeof(struct tneat_flow))) == NULL) { fprintf(stderr, "%s - could not allocate tneat_flow\n", __func__); exit(EXIT_FAILURE); } - tnf = opCB->userData; - if ((tnf->snd.buffer = malloc(config_snd_buffer_size)) == NULL) { fprintf(stderr, "%s - could not allocate send buffer\n", __func__); exit(EXIT_FAILURE); @@ -277,9 +281,16 @@ on_connected(struct neat_flow_operations *opCB) tnf->rcv.calls = 0; tnf->rcv.bytes = 0; + // hacky but quick and easy solution + if (opCB->userData) { + tnf->active = 1; + } + + opCB->userData = tnf; + // set callbacks opCB->on_readable = on_readable; - if (config_active) { + if (tnf->active) { opCB->on_writable = on_writable; } neat_set_operations(opCB->ctx, opCB->flow, opCB); @@ -297,7 +308,7 @@ on_close(struct neat_flow_operations *opCB) fprintf(stderr, "%s\n", __func__); - if (!config_active) { + if (tnf->active) { // print statistics timersub(&(tnf->rcv.tv_last), &(tnf->rcv.tv_first), &diff_time); time_elapsed = diff_time.tv_sec + (double)diff_time.tv_usec/1000000.0; @@ -307,7 +318,9 @@ on_close(struct neat_flow_operations *opCB) printf("\tbytes\t\t: %u\n", tnf->rcv.bytes); printf("\trcv-calls\t: %u\n", tnf->rcv.calls); printf("\tduration\t: %.2fs\n", time_elapsed); - printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->rcv.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); + if (time_elapsed) { + printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->rcv.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); + } } else { // print statistics @@ -318,7 +331,9 @@ on_close(struct neat_flow_operations *opCB) printf("\tbytes\t\t: %u\n", tnf->snd.bytes); printf("\tsnd-calls\t: %u\n", tnf->snd.calls); printf("\tduration\t: %.2fs\n", time_elapsed); - printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->snd.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); + if (time_elapsed > 0) { + printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->snd.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); + } } if (tnf->snd.buffer) { @@ -336,9 +351,9 @@ on_close(struct neat_flow_operations *opCB) fprintf(stderr, "%s - flow closed OK!\n", __func__); // stop event loop if we are active part - if (config_active) { + if (tnf->active) { flows_active--; - if (!flows_active) { + if (!flows_active && config_mode != NEAT_MODE_LOOP) { fprintf(stderr, "%s - stopping event loop\n", __func__); neat_stop_event_loop(opCB->ctx); } @@ -359,7 +374,8 @@ main(int argc, char *argv[]) struct neat_ctx *ctx = NULL; int i = 0; - struct neat_flow *flows[config_max_flows]; + struct neat_flow *flows_client[config_max_flows]; + struct neat_flow *flow_server; struct neat_flow_operations ops[config_max_flows]; int arg, result; @@ -369,7 +385,7 @@ main(int argc, char *argv[]) result = EXIT_SUCCESS; - while ((arg = getopt(argc, argv, "l:n:p:P:R:T:v:c:k:")) != -1) { + while ((arg = getopt(argc, argv, "l:n:p:P:R:T:v:c:k:L")) != -1) { switch(arg) { case 'l': config_snd_buffer_size = atoi(optarg); @@ -429,6 +445,12 @@ main(int argc, char *argv[]) printf("option - server key file: %s\n", key_file); } break; + case 'L': + config_mode = NEAT_MODE_LOOP; + if (config_log_level >= 1) { + printf("option - LOOP MODE\n"); + } + break; default: print_usage(); goto cleanup; @@ -436,16 +458,18 @@ main(int argc, char *argv[]) } } - if (optind == argc) { - config_active = 0; - printf("role: passive\n"); - } else if (optind + 1 == argc) { - config_active = 1; - printf("role: active\n"); - } else { - fprintf(stderr, "%s - argument error\n", __func__); - print_usage(); - goto cleanup; + if (config_mode != NEAT_MODE_LOOP) { + if (optind == argc) { + config_mode = NEAT_MODE_SERVER; + printf("role: passive\n"); + } else if (optind + 1 == argc) { + config_mode = NEAT_MODE_CLIENT; + printf("role: active\n"); + } else { + fprintf(stderr, "%s - argument error\n", __func__); + print_usage(); + goto cleanup; + } } if ((ctx = neat_init_ctx()) == NULL) { @@ -462,16 +486,16 @@ main(int argc, char *argv[]) neat_log_level(ctx, NEAT_LOG_DEBUG); } - if (config_active) { + if (config_mode == NEAT_MODE_CLIENT || config_mode == NEAT_MODE_LOOP) { for (i = 0; i < config_num_flows; i++) { - if ((flows[i] = neat_new_flow(ctx)) == NULL) { + if ((flows_client[i] = neat_new_flow(ctx)) == NULL) { fprintf(stderr, "could not initialize context\n"); result = EXIT_FAILURE; goto cleanup; } // set properties - if (neat_set_property(ctx, flows[i], arg_property)) { + if (neat_set_property(ctx, flows_client[i], arg_property)) { fprintf(stderr, "%s - error: neat_set_property\n", __func__); result = EXIT_FAILURE; goto cleanup; @@ -481,10 +505,10 @@ main(int argc, char *argv[]) ops[i].on_error = on_error; ops[i].on_close = on_close; ops[i].userData = &result; // allow on_error to modify the result variable - neat_set_operations(ctx, flows[i], &(ops[i])); + neat_set_operations(ctx, flows_client[i], &(ops[i])); // wait for on_connected or on_error to be invoked - if (neat_open(ctx, flows[i], argv[optind], config_port, NULL, 0) != NEAT_OK) { + if (neat_open(ctx, flows_client[i], argv[optind], config_port, NULL, 0) != NEAT_OK) { fprintf(stderr, "Could not open flow\n"); exit(EXIT_FAILURE); } else { @@ -492,11 +516,12 @@ main(int argc, char *argv[]) flows_active++; } } + } - } else { + if (config_mode == NEAT_MODE_SERVER || config_mode == NEAT_MODE_LOOP) { // new neat flow - if ((flows[0] = neat_new_flow(ctx)) == NULL) { + if ((flow_server = neat_new_flow(ctx)) == NULL) { fprintf(stderr, "%s - neat_new_flow failed\n", __func__); result = EXIT_FAILURE; goto cleanup; @@ -506,26 +531,26 @@ main(int argc, char *argv[]) ops[0].on_error = on_error; ops[0].on_close = on_close; - if (neat_set_operations(ctx, flows[0], &(ops[0]))) { + if (neat_set_operations(ctx, flow_server, &(ops[0]))) { fprintf(stderr, "%s - neat_set_operations failed\n", __func__); result = EXIT_FAILURE; goto cleanup; } - if (cert_file && neat_secure_identity(ctx, flows[0], cert_file, NEAT_CERT_PEM)) { + if (cert_file && neat_secure_identity(ctx, flow_server, cert_file, NEAT_CERT_PEM)) { fprintf(stderr, "%s - neat_get_secure_identity failed\n", __func__); result = EXIT_FAILURE; goto cleanup; } - if (key_file && neat_secure_identity(ctx, flows[0], key_file, NEAT_KEY_PEM)) { + if (key_file && neat_secure_identity(ctx, flow_server, key_file, NEAT_KEY_PEM)) { fprintf(stderr, "%s - neat_get_secure_identity failed\n", __func__); result = EXIT_FAILURE; goto cleanup; } // set properties - if (neat_set_property(ctx, flows[0], arg_property)) { + if (neat_set_property(ctx, flow_server, arg_property)) { fprintf(stderr, "%s - neat_set_property failed\n", __func__); result = EXIT_FAILURE; goto cleanup; @@ -533,7 +558,7 @@ main(int argc, char *argv[]) // wait for on_connected or on_error to be invoked - if (neat_accept(ctx, flows[0], config_port, NULL, 0)) { + if (neat_accept(ctx, flow_server, config_port, NULL, 0)) { fprintf(stderr, "%s - neat_accept failed\n", __func__); result = EXIT_FAILURE; goto cleanup; diff --git a/examples/util.c b/examples/util.c index 7eb302be..d4ecfdfb 100644 --- a/examples/util.c +++ b/examples/util.c @@ -78,6 +78,7 @@ char if (i > 8) { fprintf(stderr, "%s - YB should be enough - something went wrong\n", __func__); + exit(EXIT_FAILURE); } } snprintf(buffer, buffersize, "%.*f %s", i, bytes, units[i]); From bd6465c428f645bcade4c69b0cb7065d5c4f913b Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Sun, 11 Jun 2017 23:38:26 +0200 Subject: [PATCH 67/85] tneat - fixes --- examples/tneat.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/tneat.c b/examples/tneat.c index 67dffe56..9fc7cd4e 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -318,7 +318,7 @@ on_close(struct neat_flow_operations *opCB) printf("\tbytes\t\t: %u\n", tnf->rcv.bytes); printf("\trcv-calls\t: %u\n", tnf->rcv.calls); printf("\tduration\t: %.2fs\n", time_elapsed); - if (time_elapsed) { + if (time_elapsed > 0.0) { printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->rcv.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); } @@ -331,7 +331,7 @@ on_close(struct neat_flow_operations *opCB) printf("\tbytes\t\t: %u\n", tnf->snd.bytes); printf("\tsnd-calls\t: %u\n", tnf->snd.calls); printf("\tduration\t: %.2fs\n", time_elapsed); - if (time_elapsed > 0) { + if (time_elapsed > 0.0) { printf("\tbandwidth\t: %s/s\n", filesize_human(tnf->snd.bytes/time_elapsed, buffer_filesize_human, sizeof(buffer_filesize_human))); } } From 6998e07acec93ee285c34da50797b63432e9fa0d Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 12 Jun 2017 00:15:34 +0200 Subject: [PATCH 68/85] tneat - use after free fix --- examples/tneat.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/tneat.c b/examples/tneat.c index 9fc7cd4e..f90c4470 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -336,20 +336,6 @@ on_close(struct neat_flow_operations *opCB) } } - if (tnf->snd.buffer) { - free(tnf->snd.buffer); - } - - if (tnf->rcv.buffer) { - free(tnf->rcv.buffer); - } - - if (tnf) { - free(tnf); - } - - fprintf(stderr, "%s - flow closed OK!\n", __func__); - // stop event loop if we are active part if (tnf->active) { flows_active--; @@ -365,6 +351,20 @@ on_close(struct neat_flow_operations *opCB) } } + if (tnf->snd.buffer) { + free(tnf->snd.buffer); + } + + if (tnf->rcv.buffer) { + free(tnf->rcv.buffer); + } + + if (tnf) { + free(tnf); + } + + fprintf(stderr, "%s - flow closed OK!\n", __func__); + return NEAT_OK; } From d8f44d4e9a2425b0c28a0a7f89bf225c5ee22b91 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 12 Jun 2017 10:30:27 +0200 Subject: [PATCH 69/85] bb sync --- examples/tneat.c | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/examples/tneat.c b/examples/tneat.c index f90c4470..a77db9ae 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -308,7 +308,7 @@ on_close(struct neat_flow_operations *opCB) fprintf(stderr, "%s\n", __func__); - if (tnf->active) { + if (tnf->active == 0) { // print statistics timersub(&(tnf->rcv.tv_last), &(tnf->rcv.tv_first), &diff_time); time_elapsed = diff_time.tv_sec + (double)diff_time.tv_usec/1000000.0; @@ -345,7 +345,7 @@ on_close(struct neat_flow_operations *opCB) } } else { server_runs++; - if (config_max_server_runs > 0 && server_runs >= config_max_server_runs) { + if ((config_max_server_runs > 0 && server_runs >= config_max_server_runs) || (config_mode == NEAT_MODE_LOOP && !flows_active)) { fprintf(stderr, "%s - stopping event loop\n", __func__); neat_stop_event_loop(opCB->ctx); } @@ -376,12 +376,14 @@ main(int argc, char *argv[]) struct neat_flow *flows_client[config_max_flows]; struct neat_flow *flow_server; - struct neat_flow_operations ops[config_max_flows]; + struct neat_flow_operations ops_client[config_max_flows]; + struct neat_flow_operations op_server; int arg, result; char *arg_property = config_property; - memset(&ops, 0, sizeof(ops)); + memset(&ops_client, 0, sizeof(ops_client)); + memset(&op_server, 0, sizeof(op_server)); result = EXIT_SUCCESS; @@ -501,20 +503,20 @@ main(int argc, char *argv[]) goto cleanup; } - ops[i].on_connected = on_connected; - ops[i].on_error = on_error; - ops[i].on_close = on_close; - ops[i].userData = &result; // allow on_error to modify the result variable - neat_set_operations(ctx, flows_client[i], &(ops[i])); + ops_client[i].on_connected = on_connected; + ops_client[i].on_error = on_error; + ops_client[i].on_close = on_close; + ops_client[i].userData = &result; // allow on_error to modify the result variable + neat_set_operations(ctx, flows_client[i], &(ops_client[i])); // wait for on_connected or on_error to be invoked if (neat_open(ctx, flows_client[i], argv[optind], config_port, NULL, 0) != NEAT_OK) { fprintf(stderr, "Could not open flow\n"); exit(EXIT_FAILURE); - } else { - fprintf(stderr, "Opened flow %d\n", i); - flows_active++; } + + fprintf(stderr, "Opened flow %d\n", i); + flows_active++; } } @@ -527,11 +529,11 @@ main(int argc, char *argv[]) goto cleanup; } - ops[0].on_connected = on_connected; - ops[0].on_error = on_error; - ops[0].on_close = on_close; + op_server.on_connected = on_connected; + op_server.on_error = on_error; + op_server.on_close = on_close; - if (neat_set_operations(ctx, flow_server, &(ops[0]))) { + if (neat_set_operations(ctx, flow_server, &(op_server))) { fprintf(stderr, "%s - neat_set_operations failed\n", __func__); result = EXIT_FAILURE; goto cleanup; From 00c037fbff5f1f4fa5140dac3a5c3fb26a2d46ec Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 12 Jun 2017 10:42:40 +0200 Subject: [PATCH 70/85] tneat - portfix --- examples/tneat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tneat.c b/examples/tneat.c index a77db9ae..8deb9202 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -35,7 +35,7 @@ static uint32_t config_message_count = 1; static uint32_t config_runtime_max = 0; static uint16_t config_mode = 0; static uint16_t config_chargen_offset = 0; -static uint16_t config_port = 1988; +static uint16_t config_port = 23232; static uint16_t config_log_level = 1; static uint16_t config_num_flows = 1; static uint16_t config_max_flows = 100; From 5c17638bbcbf79b479df91ff127c781406ad41b8 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 12 Jun 2017 12:10:56 +0200 Subject: [PATCH 71/85] bb tests --- tests/run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/run.py b/tests/run.py index 3316fc61..42d3fda2 100755 --- a/tests/run.py +++ b/tests/run.py @@ -20,6 +20,8 @@ tests_general.append([1, 0, workdir + 'client_http_get -u /cgi-bin/he -v 2 not.resolvable.neat']) tests_general.append([1, 0, workdir + 'client_http_get -u /cgi-bin/he -v 2 buildbot.nplab.de']) tests_general.append([0, 0, workdir + 'client_http_get -n 2 -u /files/4M bsd10.nplab.de']) +tests_general.append([0, 0, workdir + 'tneat -n 1000 -v 3 -L -P ' + workdir + 'prop_tcp.json 127.0.0.1']) +tests_general.append([0, 0, workdir + 'tneat -n 1000 -v 3 -L -P ' + workdir + 'prop_sctp.json 127.0.0.1']) if (platform.system() == "FreeBSD") or (platform.system() == "Linux"): tests_general.append([1, 0, workdir + 'client_http_get -P ' + workdir + 'prop_tcp_security.json -p 443 -v 2 ec.europa.eu']) tests_general.append([0, 0, workdir + 'tneat -v 2 -P ' + workdir + 'prop_sctp_dtls.json interop.fh-muenster.de']) From aa3db21c51c39616b3dff9dc425e554abb7b13ad Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 12 Jun 2017 12:15:17 +0200 Subject: [PATCH 72/85] sctp tests only for supported platforms --- tests/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run.py b/tests/run.py index 42d3fda2..287b2e4c 100755 --- a/tests/run.py +++ b/tests/run.py @@ -21,10 +21,10 @@ tests_general.append([1, 0, workdir + 'client_http_get -u /cgi-bin/he -v 2 buildbot.nplab.de']) tests_general.append([0, 0, workdir + 'client_http_get -n 2 -u /files/4M bsd10.nplab.de']) tests_general.append([0, 0, workdir + 'tneat -n 1000 -v 3 -L -P ' + workdir + 'prop_tcp.json 127.0.0.1']) -tests_general.append([0, 0, workdir + 'tneat -n 1000 -v 3 -L -P ' + workdir + 'prop_sctp.json 127.0.0.1']) if (platform.system() == "FreeBSD") or (platform.system() == "Linux"): tests_general.append([1, 0, workdir + 'client_http_get -P ' + workdir + 'prop_tcp_security.json -p 443 -v 2 ec.europa.eu']) tests_general.append([0, 0, workdir + 'tneat -v 2 -P ' + workdir + 'prop_sctp_dtls.json interop.fh-muenster.de']) + tests_general.append([0, 0, workdir + 'tneat -n 1000 -v 3 -L -P ' + workdir + 'prop_sctp.json 127.0.0.1']) #tests_general.append([0, 0, 'python3.5 ../../policy/pmtests.py']) From 6aa2fbc92178025c84ac545a0ce2ea420f10d8e3 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 12 Jun 2017 12:23:24 +0200 Subject: [PATCH 73/85] Enable multistreaming by default --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fd6c550b..22d26250 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -178,7 +178,7 @@ IF (FLOW_GROUPS) ADD_DEFINITIONS(-DFLOW_GROUPS) ENDIF() -OPTION(SCTP_MULTISTREAMING "Include support for SCTP multistreaming" 0) +OPTION(SCTP_MULTISTREAMING "Include support for SCTP multistreaming" 1) IF (SCTP_MULTISTREAMING) ADD_DEFINITIONS(-DSCTP_MULTISTREAMING) ENDIF() From 961e75858dd6b8ddc2d27eb119f95632a1ff0165 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Wed, 14 Jun 2017 14:56:37 +0200 Subject: [PATCH 74/85] calloc and tneat fix --- examples/tneat.c | 2 +- neat_linux.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/tneat.c b/examples/tneat.c index 8deb9202..c8d0d46d 100644 --- a/examples/tneat.c +++ b/examples/tneat.c @@ -37,7 +37,7 @@ static uint16_t config_mode = 0; static uint16_t config_chargen_offset = 0; static uint16_t config_port = 23232; static uint16_t config_log_level = 1; -static uint16_t config_num_flows = 1; +static uint16_t config_num_flows = 10; static uint16_t config_max_flows = 100; static uint16_t config_max_server_runs = 0; static char *config_property = "\ diff --git a/neat_linux.c b/neat_linux.c index 0ab55eb1..d3d18c45 100644 --- a/neat_linux.c +++ b/neat_linux.c @@ -148,7 +148,7 @@ static void neat_linux_cleanup(struct neat_ctx *nc) struct neat_ctx *neat_linux_init_ctx(struct neat_ctx *ctx) { //TODO: Consider allocator function - if ((ctx->mnl_rcv_buf = calloc(MNL_SOCKET_BUFFER_SIZE, 1)) == NULL) { + if ((ctx->mnl_rcv_buf = calloc(1, MNL_SOCKET_BUFFER_SIZE)) == NULL) { neat_log(ctx, NEAT_LOG_ERROR, "Failed to allocate netlink buffer", __func__); return NULL; } From c68d43320b2dbcc37977d5cc78603ec42c732873 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Thu, 15 Jun 2017 11:33:48 +0200 Subject: [PATCH 75/85] CMAKE changes --- CMakeLists.txt | 2 +- examples/CMakeLists.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 22d26250..fd6c550b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -178,7 +178,7 @@ IF (FLOW_GROUPS) ADD_DEFINITIONS(-DFLOW_GROUPS) ENDIF() -OPTION(SCTP_MULTISTREAMING "Include support for SCTP multistreaming" 1) +OPTION(SCTP_MULTISTREAMING "Include support for SCTP multistreaming" 0) IF (SCTP_MULTISTREAMING) ADD_DEFINITIONS(-DSCTP_MULTISTREAMING) ENDIF() diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index d16899a7..93e47587 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -13,6 +13,9 @@ LIST(APPEND neat_programs tneat.c peer.c msbench.c + minimal_client.c + minimal_server.c + minimal_server2.c ) LIST(APPEND neat_property_examples From 0efe8699ad8165f892f9eed9bde01c207a821014 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Thu, 15 Jun 2017 16:29:43 +0200 Subject: [PATCH 76/85] bb test --- neat_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neat_core.c b/neat_core.c index 8ff78738..cf16786e 100644 --- a/neat_core.c +++ b/neat_core.c @@ -669,7 +669,7 @@ socket_handle_free_cb(uv_handle_t *handle) #endif } else { - synchronous_free(pollable_socket->flow); + //synchronous_free(pollable_socket->flow); } } From 987ce709e169941b8484b04c843cebf5446f6752 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Fri, 16 Jun 2017 02:04:15 +0200 Subject: [PATCH 77/85] revert bb test --- neat_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neat_core.c b/neat_core.c index cf16786e..8ff78738 100644 --- a/neat_core.c +++ b/neat_core.c @@ -669,7 +669,7 @@ socket_handle_free_cb(uv_handle_t *handle) #endif } else { - //synchronous_free(pollable_socket->flow); + synchronous_free(pollable_socket->flow); } } From 7db24965e1f1ffc15bbce435db649668dc376298 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Fri, 16 Jun 2017 02:10:27 +0200 Subject: [PATCH 78/85] tneat tests less verbose --- tests/run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/run.py b/tests/run.py index 287b2e4c..927ec124 100755 --- a/tests/run.py +++ b/tests/run.py @@ -20,11 +20,11 @@ tests_general.append([1, 0, workdir + 'client_http_get -u /cgi-bin/he -v 2 not.resolvable.neat']) tests_general.append([1, 0, workdir + 'client_http_get -u /cgi-bin/he -v 2 buildbot.nplab.de']) tests_general.append([0, 0, workdir + 'client_http_get -n 2 -u /files/4M bsd10.nplab.de']) -tests_general.append([0, 0, workdir + 'tneat -n 1000 -v 3 -L -P ' + workdir + 'prop_tcp.json 127.0.0.1']) +tests_general.append([0, 0, workdir + 'tneat -n 1000 -v 0 -L -P ' + workdir + 'prop_tcp.json 127.0.0.1']) if (platform.system() == "FreeBSD") or (platform.system() == "Linux"): tests_general.append([1, 0, workdir + 'client_http_get -P ' + workdir + 'prop_tcp_security.json -p 443 -v 2 ec.europa.eu']) - tests_general.append([0, 0, workdir + 'tneat -v 2 -P ' + workdir + 'prop_sctp_dtls.json interop.fh-muenster.de']) - tests_general.append([0, 0, workdir + 'tneat -n 1000 -v 3 -L -P ' + workdir + 'prop_sctp.json 127.0.0.1']) + tests_general.append([0, 0, workdir + 'tneat -v 1 -P ' + workdir + 'prop_sctp_dtls.json interop.fh-muenster.de']) + tests_general.append([0, 0, workdir + 'tneat -n 1000 -v 0 -L -P ' + workdir + 'prop_sctp.json 127.0.0.1']) #tests_general.append([0, 0, 'python3.5 ../../policy/pmtests.py']) From ad5501da3673333ea8b1cdb09ca1a18190e071de Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 19 Jun 2017 11:36:52 +0200 Subject: [PATCH 79/85] tneat self test for sctp removed --- tests/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run.py b/tests/run.py index 927ec124..59f6382e 100755 --- a/tests/run.py +++ b/tests/run.py @@ -24,7 +24,7 @@ if (platform.system() == "FreeBSD") or (platform.system() == "Linux"): tests_general.append([1, 0, workdir + 'client_http_get -P ' + workdir + 'prop_tcp_security.json -p 443 -v 2 ec.europa.eu']) tests_general.append([0, 0, workdir + 'tneat -v 1 -P ' + workdir + 'prop_sctp_dtls.json interop.fh-muenster.de']) - tests_general.append([0, 0, workdir + 'tneat -n 1000 -v 0 -L -P ' + workdir + 'prop_sctp.json 127.0.0.1']) + #tests_general.append([0, 0, workdir + 'tneat -n 1000 -v 0 -L -P ' + workdir + 'prop_sctp.json 127.0.0.1']) #tests_general.append([0, 0, 'python3.5 ../../policy/pmtests.py']) From 055517207d2974e1c639a400b72a52b7ee37c095 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 19 Jun 2017 11:38:43 +0200 Subject: [PATCH 80/85] tneat self test for tcp removed --- tests/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run.py b/tests/run.py index 59f6382e..c3109465 100755 --- a/tests/run.py +++ b/tests/run.py @@ -20,7 +20,7 @@ tests_general.append([1, 0, workdir + 'client_http_get -u /cgi-bin/he -v 2 not.resolvable.neat']) tests_general.append([1, 0, workdir + 'client_http_get -u /cgi-bin/he -v 2 buildbot.nplab.de']) tests_general.append([0, 0, workdir + 'client_http_get -n 2 -u /files/4M bsd10.nplab.de']) -tests_general.append([0, 0, workdir + 'tneat -n 1000 -v 0 -L -P ' + workdir + 'prop_tcp.json 127.0.0.1']) +#tests_general.append([0, 0, workdir + 'tneat -n 1000 -v 0 -L -P ' + workdir + 'prop_tcp.json 127.0.0.1']) if (platform.system() == "FreeBSD") or (platform.system() == "Linux"): tests_general.append([1, 0, workdir + 'client_http_get -P ' + workdir + 'prop_tcp_security.json -p 443 -v 2 ec.europa.eu']) tests_general.append([0, 0, workdir + 'tneat -v 1 -P ' + workdir + 'prop_sctp_dtls.json interop.fh-muenster.de']) From de7e22a92469bd452035a6e67801d414e752b680 Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 19 Jun 2017 16:32:33 +0100 Subject: [PATCH 81/85] added feature: unordered transmission --- neat_core.c | 234 ++++++++++++++++++++++++++++-------------------- neat_internal.h | 3 +- 2 files changed, 137 insertions(+), 100 deletions(-) diff --git a/neat_core.c b/neat_core.c index 8ff78738..785fc639 100644 --- a/neat_core.c +++ b/neat_core.c @@ -1127,6 +1127,7 @@ handle_sctp_assoc_change(neat_flow *flow, struct sctp_assoc_change *sac) #ifdef SCTP_ASSOC_SUPPORTS_PR case SCTP_ASSOC_SUPPORTS_PR: neat_log(ctx, NEAT_LOG_DEBUG, "\t- PR"); + flow->socket->sctp_partial_reliability = 1; break; #endif // SCTP_ASSOC_SUPPORTS_PR @@ -4488,106 +4489,116 @@ neat_write_flush(struct neat_ctx *ctx, struct neat_flow *flow) TAILQ_FOREACH_SAFE(msg, &flow->bufferedMessages, message_next, next_msg) { do { #ifdef NEAT_SCTP_DTLS - if ((flow->socket->fd != -1) && flow->security_needed && neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) { - struct security_data *private = (struct security_data *) flow->socket->dtls_data->userData; - struct bio_dgram_sctp_sndinfo sinfo; - memset(&sinfo, 0, sizeof(struct bio_dgram_sctp_sndinfo)); - BIO_ctrl(private->dtlsBIO, BIO_CTRL_DGRAM_SCTP_SET_SNDINFO, sizeof(struct bio_dgram_sctp_sndinfo), &sinfo); - socklen_t size = SSL_write(private->ssl, msg->buffered + msg->bufferedOffset, msg->bufferedSize); - if (SSL_get_error(private->ssl, size) == SSL_ERROR_WANT_WRITE || SSL_get_error(private->ssl, size) == SSL_ERROR_WANT_READ) { - uvpollable_cb(flow->socket->handle, NEAT_OK, UV_WRITABLE | UV_READABLE); - } else if (size > 0) { - msg->bufferedOffset += size; - msg->bufferedSize -= size; - } + if ((flow->socket->fd != -1) && flow->security_needed && neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) { + struct security_data *private = (struct security_data *) flow->socket->dtls_data->userData; + struct bio_dgram_sctp_sndinfo sinfo; + memset(&sinfo, 0, sizeof(struct bio_dgram_sctp_sndinfo)); + BIO_ctrl(private->dtlsBIO, BIO_CTRL_DGRAM_SCTP_SET_SNDINFO, sizeof(struct bio_dgram_sctp_sndinfo), &sinfo); + socklen_t size = SSL_write(private->ssl, msg->buffered + msg->bufferedOffset, msg->bufferedSize); + if (SSL_get_error(private->ssl, size) == SSL_ERROR_WANT_WRITE || SSL_get_error(private->ssl, size) == SSL_ERROR_WANT_READ) { + uvpollable_cb(flow->socket->handle, NEAT_OK, UV_WRITABLE | UV_READABLE); + } else if (size > 0) { + msg->bufferedOffset += size; + msg->bufferedSize -= size; } -#endif - if (!flow->security_needed || !(neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP)) { - iov.iov_base = msg->buffered + msg->bufferedOffset; - if ((neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) && - (flow->socket->sctp_explicit_eor) && - (flow->socket->write_limit > 0) && - (msg->bufferedSize > flow->socket->write_limit)) { - len = flow->socket->write_limit; - } else { - len = msg->bufferedSize; } - iov.iov_len = len; - msghdr.msg_name = NULL; - msghdr.msg_namelen = 0; - msghdr.msg_iov = &iov; - msghdr.msg_iovlen = 1; +#endif + if (!flow->security_needed || !(neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP)) { + iov.iov_base = msg->buffered + msg->bufferedOffset; + if ((neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) && + (flow->socket->sctp_explicit_eor) && + (flow->socket->write_limit > 0) && + (msg->bufferedSize > flow->socket->write_limit)) { + len = flow->socket->write_limit; + } else { + len = msg->bufferedSize; + } + iov.iov_len = len; + msghdr.msg_name = NULL; + msghdr.msg_namelen = 0; + msghdr.msg_iov = &iov; + msghdr.msg_iovlen = 1; - if (neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) { + if (neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) { #if defined(SCTP_SNDINFO) - msghdr.msg_control = cmsgbuf; - msghdr.msg_controllen = CMSG_SPACE(sizeof(struct sctp_sndinfo)); - cmsg = (struct cmsghdr *)cmsgbuf; - cmsg->cmsg_level = IPPROTO_SCTP; - cmsg->cmsg_type = SCTP_SNDINFO; - cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndinfo)); - sndinfo = (struct sctp_sndinfo *)CMSG_DATA(cmsg); - memset(sndinfo, 0, sizeof(struct sctp_sndinfo)); - sndinfo->snd_sid = msg->stream_id; + msghdr.msg_control = cmsgbuf; + msghdr.msg_controllen = CMSG_SPACE(sizeof(struct sctp_sndinfo)); + cmsg = (struct cmsghdr *)cmsgbuf; + cmsg->cmsg_level = IPPROTO_SCTP; + cmsg->cmsg_type = SCTP_SNDINFO; + cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndinfo)); + sndinfo = (struct sctp_sndinfo *)CMSG_DATA(cmsg); + memset(sndinfo, 0, sizeof(struct sctp_sndinfo)); + sndinfo->snd_sid = msg->stream_id; + + if (unordered) { + sndinfo->snd_flags |= SCTP_UNORDERED; + } + #if defined(SCTP_EXPLICIT_EOR) - if ((flow->socket->sctp_explicit_eor) && (len == msg->bufferedSize)) { - sndinfo->snd_flags |= SCTP_EOR; - } + if ((flow->socket->sctp_explicit_eor) && (len == msg->bufferedSize)) { + sndinfo->snd_flags |= SCTP_EOR; + } #endif // defined(SCTP_EXPLICIT_EOR) #elif defined (SCTP_SNDRCV) - msghdr.msg_control = cmsgbuf; - msghdr.msg_controllen = CMSG_SPACE(sizeof(struct sctp_sndrcvinfo)); - cmsg = (struct cmsghdr *)cmsgbuf; - cmsg->cmsg_level = IPPROTO_SCTP; - cmsg->cmsg_type = SCTP_SNDRCV; - cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndrcvinfo)); - sndrcvinfo = (struct sctp_sndrcvinfo *)CMSG_DATA(cmsg); - memset(sndrcvinfo, 0, sizeof(struct sctp_sndrcvinfo)); - sndrcvinfo->sinfo_stream = msg->stream_id; + msghdr.msg_control = cmsgbuf; + msghdr.msg_controllen = CMSG_SPACE(sizeof(struct sctp_sndrcvinfo)); + cmsg = (struct cmsghdr *)cmsgbuf; + cmsg->cmsg_level = IPPROTO_SCTP; + cmsg->cmsg_type = SCTP_SNDRCV; + cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndrcvinfo)); + sndrcvinfo = (struct sctp_sndrcvinfo *)CMSG_DATA(cmsg); + memset(sndrcvinfo, 0, sizeof(struct sctp_sndrcvinfo)); + sndrcvinfo->sinfo_stream = msg->stream_id; + + if (unordered) { + sndrcvinfo->sinfo_flags |= SCTP_UNORDERED; + } + #if defined(SCTP_EXPLICIT_EOR) - if ((flow->socket->sctp_explicit_eor) && (len == msg->bufferedSize)) { - sndrcvinfo->sinfo_flags |= SCTP_EOR; - } + if ((flow->socket->sctp_explicit_eor) && (len == msg->bufferedSize)) { + sndrcvinfo->sinfo_flags |= SCTP_EOR; + } #endif // defined(SCTP_EXPLICIT_EOR) #else // defined(SCTP_SNDINFO) - msghdr.msg_control = NULL; - msghdr.msg_controllen = 0; + msghdr.msg_control = NULL; + msghdr.msg_controllen = 0; #endif // defined(SCTP_SNDINFO) - } else { - msghdr.msg_control = NULL; - msghdr.msg_controllen = 0; - } - - msghdr.msg_flags = 0; - if (flow->socket->fd != -1) { - rv = sendmsg(flow->socket->fd, (const struct msghdr *)&msghdr, 0); - } else { -#if defined(USRSCTP_SUPPORT) - if (neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) { - neat_log(ctx, NEAT_LOG_INFO, "%s - send %zd bytes on flow %p and socket %p", __func__, msg->bufferedSize, (void *)flow, (void *)flow->socket->usrsctp_socket); - rv = usrsctp_sendv(flow->socket->usrsctp_socket, msg->buffered + msg->bufferedOffset, msg->bufferedSize, - (struct sockaddr *) (flow->sockAddr), 1, (void *)sndinfo, - (socklen_t)sizeof(struct sctp_sndinfo), SCTP_SENDV_SNDINFO, - 0); } else { - neat_log(ctx, NEAT_LOG_ERROR, "%s - fd == -1 and no SCTP used ... error!", __func__); + msghdr.msg_control = NULL; + msghdr.msg_controllen = 0; } + + msghdr.msg_flags = 0; + if (flow->socket->fd != -1) { + rv = sendmsg(flow->socket->fd, (const struct msghdr *)&msghdr, 0); + } else { +#if defined(USRSCTP_SUPPORT) + if (neat_base_stack(flow->socket->stack) == NEAT_STACK_SCTP) { + neat_log(ctx, NEAT_LOG_INFO, "%s - send %zd bytes on flow %p and socket %p", __func__, msg->bufferedSize, (void *)flow, (void *)flow->socket->usrsctp_socket); + rv = usrsctp_sendv(flow->socket->usrsctp_socket, msg->buffered + msg->bufferedOffset, msg->bufferedSize, + (struct sockaddr *) (flow->sockAddr), 1, (void *)sndinfo, + (socklen_t)sizeof(struct sctp_sndinfo), SCTP_SENDV_SNDINFO, + 0); + } else { + neat_log(ctx, NEAT_LOG_ERROR, "%s - fd == -1 and no SCTP used ... error!", __func__); + } #else - neat_log(ctx, NEAT_LOG_ERROR, "%s - fd == -1 and not usrsctp support - fixme!", __func__); - assert(false); + neat_log(ctx, NEAT_LOG_ERROR, "%s - fd == -1 and not usrsctp support - fixme!", __func__); + assert(false); #endif - } - if (!flow->security_needed) { - if (rv < 0) { - if (errno == EWOULDBLOCK) { - return NEAT_ERROR_WOULD_BLOCK; - } else { - return NEAT_ERROR_IO; + } + if (!flow->security_needed) { + if (rv < 0) { + if (errno == EWOULDBLOCK) { + return NEAT_ERROR_WOULD_BLOCK; + } else { + return NEAT_ERROR_IO; + } } + msg->bufferedOffset += rv; + msg->bufferedSize -= rv; } - msg->bufferedOffset += rv; - msg->bufferedSize -= rv; - } } } while (msg->bufferedSize > 0); @@ -4602,8 +4613,14 @@ neat_write_flush(struct neat_ctx *ctx, struct neat_flow *flow) } static neat_error_code -neat_write_fillbuffer(struct neat_ctx *ctx, struct neat_flow *flow, - const unsigned char *buffer, uint32_t amt, int stream_id) +neat_write_fillbuffer(struct neat_ctx *ctx, + struct neat_flow *flow, + const unsigned char *buffer, + uint32_t amt, + int stream_id, + uint8_t unordered, + uint8_t pr_method, + uint32_t pr_value) { struct neat_buffered_message *msg; neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); @@ -4622,8 +4639,11 @@ neat_write_fillbuffer(struct neat_ctx *ctx, struct neat_flow *flow, msg->buffered = NULL; msg->bufferedOffset = 0; msg->bufferedSize = 0; - msg->bufferedAllocation= 0; + msg->bufferedAllocation = 0; msg->stream_id = stream_id; + msg->unordered = unordered; + msg->pr_method = pr_method; + msg->pr_value = pr_value; TAILQ_INSERT_TAIL(&flow->bufferedMessages, msg, message_next); } else { assert(stream_id == 0); @@ -4631,8 +4651,7 @@ neat_write_fillbuffer(struct neat_ctx *ctx, struct neat_flow *flow, } // check if there is room to buffer without extending allocation if ((msg->bufferedOffset + msg->bufferedSize + amt) <= msg->bufferedAllocation) { - memcpy(msg->buffered + msg->bufferedOffset + msg->bufferedSize, - buffer, amt); + memcpy(msg->buffered + msg->bufferedOffset + msg->bufferedSize, buffer, amt); msg->bufferedSize += amt; return NEAT_OK; } @@ -4679,12 +4698,12 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, int has_stream_id = 0; // int has_context = 0; // int context = 0; - // int has_pr_method = 0; - // int pr_method = 0; - // int has_pr_value = 0; - // int pr_value = 0; - // int has_unordered = 0; - // int unordered = 0; + int has_pr_method = 0; + int pr_method = 0; + int has_pr_value = 0; + int pr_value = 0; + int has_unordered = 0; + int unordered = 0; // int has_priority = 0; // float priority = 0.5f; // int has_dest_addr = 0; @@ -4696,7 +4715,14 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, struct msghdr msghdr; struct iovec iov; #if defined(SCTP_SNDINFO) - char cmsgbuf[CMSG_SPACE(sizeof(struct sctp_sndinfo))]; + char cmsgbuf[CMSG_SPACE(sizeof(struct sctp_sndinfo) + +#if defined(SCTP_PRINFO) + CMSG_SPACE(sizeof(struct sctp_sndinfo)) +#else // defined(SCTP_PRINFO) + 0 +#endif // defined(SCTP_PRINFO) + ]; + struct sctp_sndinfo *sndinfo = NULL; memset(&cmsgbuf, 0, sizeof(cmsgbuf)); #elif defined (SCTP_SNDRCV) @@ -4710,9 +4736,9 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, HANDLE_OPTIONAL_ARGUMENTS_START() OPTIONAL_INTEGER_PRESENT(NEAT_TAG_STREAM_ID, stream_id, has_stream_id) // OPTIONAL_INTEGER_PRESENT(NEAT_TAG_CONTEXT, context, has_context) - // OPTIONAL_INTEGER_PRESENT(NEAT_TAG_PARTIAL_RELIABILITY_METHOD, pr_method, has_pr_method) - // OPTIONAL_INTEGER_PRESENT(NEAT_TAG_PARTIAL_RELIABILITY_VALUE, pr_value, has_pr_value) - // OPTIONAL_INTEGER_PRESENT(NEAT_TAG_UNORDERED, unordered, has_unordered) + OPTIONAL_INTEGER_PRESENT(NEAT_TAG_PARTIAL_RELIABILITY_METHOD, pr_method, has_pr_method) + OPTIONAL_INTEGER_PRESENT(NEAT_TAG_PARTIAL_RELIABILITY_VALUE, pr_value, has_pr_value) + OPTIONAL_INTEGER_PRESENT(NEAT_TAG_UNORDERED, unordered, has_unordered) // OPTIONAL_FLOAT_PRESENT( NEAT_TAG_PRIORITY, priority, has_priority) // OPTIONAL_STRING_PRESENT( NEAT_TAG_DESTINATION_IP_ADDRESS, dest_addr, has_dest_addr) HANDLE_OPTIONAL_ARGUMENTS_END(); @@ -4804,11 +4830,17 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, sndinfo->snd_sid = stream_id; } + if (unordered) { + sndinfo->snd_flags |= SCTP_UNORDERED; + } + #if defined(SCTP_EXPLICIT_EOR) if ((flow->socket->sctp_explicit_eor) && (len == amt)) { sndinfo->snd_flags |= SCTP_EOR; } #endif // defined(SCTP_EXPLICIT_EOR) + + #elif defined (SCTP_SNDRCV) msghdr.msg_control = cmsgbuf; msghdr.msg_controllen = CMSG_SPACE(sizeof(struct sctp_sndrcvinfo)); @@ -4823,6 +4855,10 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, sndrcvinfo->sinfo_stream = stream_id; } + if (unordered) { + sndrcvinfo->sinfo_flags |= SCTP_UNORDERED; + } + #if defined(SCTP_EXPLICIT_EOR) if ((flow->socket->sctp_explicit_eor) && (len == amt)) { sndrcvinfo->sinfo_flags |= SCTP_EOR; @@ -4891,7 +4927,7 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, /* Update flow statistics with the sent bytes */ flow->flow_stats.bytes_sent += rv; - code = neat_write_fillbuffer(ctx, flow, buffer, amt, stream_id); + code = neat_write_fillbuffer(ctx, flow, buffer, amt, stream_id, unordered, pr_method, pr_value); if (code != NEAT_OK) { return code; } diff --git a/neat_internal.h b/neat_internal.h index 5f558b95..3aa6d070 100644 --- a/neat_internal.h +++ b/neat_internal.h @@ -234,6 +234,7 @@ struct neat_pollable_socket uint8_t multistream; // multistreaming active unsigned int sctp_explicit_eor : 1; + unsigned int sctp_partial_reliability : 1; uint16_t sctp_streams_available; // available streams #ifdef SCTP_MULTISTREAMING uint8_t sctp_notification_wait; // wait for all notifications @@ -342,7 +343,7 @@ struct neat_flow size_t multistream_read_queue_size; neat_flow_states multistream_state; -#endif +#endif // SCTP_MULTISTREAMING }; typedef struct neat_flow neat_flow; From adbcf09096d3b6f58cde830187dc0a68867e0b8f Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Mon, 19 Jun 2017 19:09:51 +0200 Subject: [PATCH 82/85] PR extension for SCTP --- neat_core.c | 54 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/neat_core.c b/neat_core.c index 785fc639..19c1cbc9 100644 --- a/neat_core.c +++ b/neat_core.c @@ -4531,7 +4531,7 @@ neat_write_flush(struct neat_ctx *ctx, struct neat_flow *flow) memset(sndinfo, 0, sizeof(struct sctp_sndinfo)); sndinfo->snd_sid = msg->stream_id; - if (unordered) { + if (msg->unordered) { sndinfo->snd_flags |= SCTP_UNORDERED; } @@ -4551,7 +4551,7 @@ neat_write_flush(struct neat_ctx *ctx, struct neat_flow *flow) memset(sndrcvinfo, 0, sizeof(struct sctp_sndrcvinfo)); sndrcvinfo->sinfo_stream = msg->stream_id; - if (unordered) { + if (msg->unordered) { sndrcvinfo->sinfo_flags |= SCTP_UNORDERED; } @@ -4708,28 +4708,36 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, // float priority = 0.5f; // int has_dest_addr = 0; // const char *dest_addr = ""; - -#if defined(SCTP_SNDINFO) || defined (SCTP_SNDRCV) - struct cmsghdr *cmsg; -#endif struct msghdr msghdr; struct iovec iov; #if defined(SCTP_SNDINFO) - char cmsgbuf[CMSG_SPACE(sizeof(struct sctp_sndinfo) + + char cmsgbuf[CMSG_SPACE(sizeof(struct sctp_sndinfo)) + #if defined(SCTP_PRINFO) - CMSG_SPACE(sizeof(struct sctp_sndinfo)) + CMSG_SPACE(sizeof(struct sctp_prinfo)) #else // defined(SCTP_PRINFO) 0 #endif // defined(SCTP_PRINFO) - ]; - + ]; struct sctp_sndinfo *sndinfo = NULL; - memset(&cmsgbuf, 0, sizeof(cmsgbuf)); -#elif defined (SCTP_SNDRCV) - char cmsgbuf[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))]; +#elif defined(SCTP_SNDRCV) + char cmsgbuf[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo)) + +#if defined(SCTP_PRINFO) + CMSG_SPACE(sizeof(struct sctp_prinfo)) +#else // defined(SCTP_PRINFO) + 0 +#endif // defined(SCTP_PRINFO) + ]; struct sctp_sndrcvinfo *sndrcvinfo; +#endif //defined(SCTP_SNDINFO) || defined (SCTP_SNDRCV) + +#if defined(SCTP_SNDINFO) || defined (SCTP_SNDRCV) + struct cmsghdr *cmsg; memset(&cmsgbuf, 0, sizeof(cmsgbuf)); -#endif +#if defined(SCTP_PRINFO) + struct sctp_prinfo *prinfo; +#endif // defined(SCTP_PRINFO) +#endif // defined(SCTP_SNDINFO) || defined (SCTP_SNDRCV) + neat_log(ctx, NEAT_LOG_DEBUG, "%s", __func__); @@ -4818,21 +4826,31 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, if (!flow->security_needed) { #if defined(SCTP_SNDINFO) msghdr.msg_control = cmsgbuf; - msghdr.msg_controllen = CMSG_SPACE(sizeof(struct sctp_sndinfo)); + msghdr.msg_controllen = sizeof(cmsgbuf); cmsg = (struct cmsghdr *)cmsgbuf; cmsg->cmsg_level = IPPROTO_SCTP; cmsg->cmsg_type = SCTP_SNDINFO; cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndinfo)); sndinfo = (struct sctp_sndinfo *)CMSG_DATA(cmsg); memset(sndinfo, 0, sizeof(struct sctp_sndinfo)); - if (stream_id) { sndinfo->snd_sid = stream_id; } - if (unordered) { sndinfo->snd_flags |= SCTP_UNORDERED; } + cmsg = (struct cmsghdr *)((caddr_t)cmsg + CMSG_SPACE(sizeof(struct sctp_sndinfo))); + + +#if defined(SCTP_PRINFO) + cmsg->cmsg_level = IPPROTO_SCTP; + cmsg->cmsg_type = SCTP_PRINFO; + cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_prinfo)); + prinfo = (struct sctp_prinfo *)CMSG_DATA(cmsg); + memset(prinfo, 0, sizeof(struct sctp_prinfo)); + prinfo->pr_policy = pr_method; + prinfo->pr_value = pr_value; +#endif #if defined(SCTP_EXPLICIT_EOR) if ((flow->socket->sctp_explicit_eor) && (len == amt)) { @@ -4843,7 +4861,7 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, #elif defined (SCTP_SNDRCV) msghdr.msg_control = cmsgbuf; - msghdr.msg_controllen = CMSG_SPACE(sizeof(struct sctp_sndrcvinfo)); + msghdr.msg_controllen = sizeof(cmsgbuf); cmsg = (struct cmsghdr *)cmsgbuf; cmsg->cmsg_level = IPPROTO_SCTP; cmsg->cmsg_type = SCTP_SNDRCV; From db8df46833a30bd44f7829496a67b3a96b51cf4e Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 20 Jun 2017 01:48:15 +0200 Subject: [PATCH 83/85] partial reliability fix for linux --- neat_core.c | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/neat_core.c b/neat_core.c index 19c1cbc9..932cd2ce 100644 --- a/neat_core.c +++ b/neat_core.c @@ -4751,6 +4751,7 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, // OPTIONAL_STRING_PRESENT( NEAT_TAG_DESTINATION_IP_ADDRESS, dest_addr, has_dest_addr) HANDLE_OPTIONAL_ARGUMENTS_END(); + if (has_stream_id && stream_id < 0) { neat_log(ctx, NEAT_LOG_DEBUG, "%s - invalid stream id: Must be 0 or greater", __func__); return NEAT_ERROR_BAD_ARGUMENT; @@ -4771,6 +4772,12 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, } #endif // SCTP_MULTISTREAMING + if (has_pr_method && has_pr_value) { +#if !defined(SCTP_PRINFO) + neat_log(ctx, NEAT_LOG_WARNING, "%s - partial reliability options set but not supported"); +#endif + } + switch (flow->socket->stack) { case NEAT_STACK_TCP: case NEAT_STACK_MPTCP: @@ -4833,10 +4840,10 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndinfo)); sndinfo = (struct sctp_sndinfo *)CMSG_DATA(cmsg); memset(sndinfo, 0, sizeof(struct sctp_sndinfo)); - if (stream_id) { + if (has_stream_id && stream_id) { sndinfo->snd_sid = stream_id; } - if (unordered) { + if (has_unordered && unordered) { sndinfo->snd_flags |= SCTP_UNORDERED; } cmsg = (struct cmsghdr *)((caddr_t)cmsg + CMSG_SPACE(sizeof(struct sctp_sndinfo))); @@ -4848,8 +4855,10 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_prinfo)); prinfo = (struct sctp_prinfo *)CMSG_DATA(cmsg); memset(prinfo, 0, sizeof(struct sctp_prinfo)); - prinfo->pr_policy = pr_method; - prinfo->pr_value = pr_value; + if (has_pr_method && has_pr_value) { + prinfo->pr_policy = pr_method; + prinfo->pr_value = pr_value; + } #endif #if defined(SCTP_EXPLICIT_EOR) @@ -4869,14 +4878,27 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, sndrcvinfo = (struct sctp_sndrcvinfo *)CMSG_DATA(cmsg); memset(sndrcvinfo, 0, sizeof(struct sctp_sndrcvinfo)); - if (stream_id) { + if (has_stream_id && stream_id) { sndrcvinfo->sinfo_stream = stream_id; } - if (unordered) { + if (has_unordered && unordered) { sndrcvinfo->sinfo_flags |= SCTP_UNORDERED; } +#if defined(SCTP_PRINFO) + cmsg->cmsg_level = IPPROTO_SCTP; + cmsg->cmsg_type = SCTP_PRINFO; + cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_prinfo)); + prinfo = (struct sctp_prinfo *)CMSG_DATA(cmsg); + memset(prinfo, 0, sizeof(struct sctp_prinfo)); + + if (has_pr_value && has_pr_method) { + prinfo->pr_policy = pr_method; + prinfo->pr_value = pr_value; + } +#endif + #if defined(SCTP_EXPLICIT_EOR) if ((flow->socket->sctp_explicit_eor) && (len == amt)) { sndrcvinfo->sinfo_flags |= SCTP_EOR; From 07f3e7055e5337e99824e5ea4f177e103f4b8a3d Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 20 Jun 2017 00:58:26 +0100 Subject: [PATCH 84/85] compile fix for netbsd --- neat_core.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/neat_core.c b/neat_core.c index 932cd2ce..e96540b3 100644 --- a/neat_core.c +++ b/neat_core.c @@ -4777,7 +4777,13 @@ neat_write_to_lower_layer(struct neat_ctx *ctx, struct neat_flow *flow, neat_log(ctx, NEAT_LOG_WARNING, "%s - partial reliability options set but not supported"); #endif } - + + if (has_unordered) { +#if !defined(SCTP_SNDRCV) && !defined(SCTP_SNDINFO) + neat_log(ctx, NEAT_LOG_WARNING, "%s - unordered delivery requested but not supported"); +#endif + } + switch (flow->socket->stack) { case NEAT_STACK_TCP: case NEAT_STACK_MPTCP: From 2f44d61e079c354b6d17763e7b679032d33b4fcd Mon Sep 17 00:00:00 2001 From: Felix Weinrank Date: Tue, 20 Jun 2017 09:50:54 +0100 Subject: [PATCH 85/85] fix merge error for client_http_get.c --- examples/client_http_get.c | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/client_http_get.c b/examples/client_http_get.c index d48ee231..7a08bf87 100644 --- a/examples/client_http_get.c +++ b/examples/client_http_get.c @@ -70,6 +70,7 @@ struct stat_flow { struct timeval tv_last; struct timeval tv_delta; uint16_t protocol; + uv_timer_t timer; struct neat_flow *flow; };