From c45fc6ab7c44196aa77bc5da903a3b79250c8d9c Mon Sep 17 00:00:00 2001 From: Kenneth Shelton Date: Fri, 28 Jul 2017 15:02:17 +0000 Subject: [PATCH 1/4] Fix UAF when defining new custom types after rules have been created Added . support to repeat. This causes items to be placed in json object and that object to be merged into parent Added flag to repeat cause parser to fail if name already exists in json object. This allows for some basic conditional logic. --- doc/configuration.rst | 5 +++ src/parser.c | 97 ++++++++++++++++++++++++++----------------- src/parser.h | 7 +++- src/pdag.c | 94 +++++++++++++++++++++++++++++++---------- src/pdag.h | 13 +++--- src/samp.c | 8 ++-- tests/json_eq.c | 4 +- 7 files changed, 157 insertions(+), 71 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index dba9d036..329dde24 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1117,6 +1117,11 @@ generate such messages (note the flags part):: Aug 18 13:18:45 192.168.0.1 %ASA-6-106015: Deny TCP (no connection) from 10.252.88.66/443 to 10.79.249.222/52746 flags RST on interface outside +option.failOnDuplicate +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +If set to "True", causes parsers to fail if the name already exists +in the tree. A no-op unless using .. to cause a merger + cee-syslog ########## This parses cee syslog from the message. This format has been defined diff --git a/src/parser.c b/src/parser.c index 1352f20c..70505cce 100644 --- a/src/parser.c +++ b/src/parser.c @@ -83,6 +83,7 @@ hParseInt(const unsigned char **buf, size_t *lenBuf) * @param[in] npb->strLen length of the to-be-parsed string * @param[in] offs an offset into the string * @param[in] pointer to parser data block + * @param[in] pointer to current parser's name * @param[out] parsed bytes * @param[out] value ptr to json object containing parsed data * (can be unused, but if used *value MUST be NULL on entry) @@ -99,6 +100,7 @@ int ln_v2_parse##ParserName( \ npb_t *const npb, \ size_t *const offs, \ __attribute__((unused)) void *const pdata, \ + __attribute__((unused)) const char *parser_name, \ size_t *parsed, \ struct json_object **value) \ { \ @@ -1763,7 +1765,7 @@ PARSER_Parse(CiscoInterfaceSpec) int bHaveIP = 0; size_t lenIP; size_t idxIP = i; - if(ln_v2_parseIPv4(npb, &i, NULL, &lenIP, NULL) == 0) { + if(ln_v2_parseIPv4(npb, &i, NULL, parser_name, &lenIP, NULL) == 0) { bHaveIP = 1; i += lenIP - 1; /* position on delimiter */ } else { @@ -1783,14 +1785,14 @@ PARSER_Parse(CiscoInterfaceSpec) /* we now utilize our other parser helpers */ if(!bHaveIP) { idxIP = i; - if(ln_v2_parseIPv4(npb, &i, NULL, &lenIP, NULL) != 0) goto done; + if(ln_v2_parseIPv4(npb, &i, NULL, parser_name, &lenIP, NULL) != 0) goto done; i += lenIP; } if(i == npb->strLen || c[i] != '/') goto done; ++i; /* skip slash */ const size_t idxPort = i; size_t lenPort; - if(ln_v2_parseNumber(npb, &i, NULL, &lenPort, NULL) != 0) goto done; + if(ln_v2_parseNumber(npb, &i, NULL, parser_name, &lenPort, NULL) != 0) goto done; i += lenPort; if(i == npb->strLen) goto success; @@ -1803,12 +1805,12 @@ PARSER_Parse(CiscoInterfaceSpec) if(i+5 < npb->strLen && c[i] == ' ' && c[i+1] == '(') { size_t iTmp = i+2; /* skip over " (" */ idxIP2 = iTmp; - if(ln_v2_parseIPv4(npb, &iTmp, NULL, &lenIP2, NULL) == 0) { + if(ln_v2_parseIPv4(npb, &iTmp, NULL, parser_name, &lenIP2, NULL) == 0) { iTmp += lenIP2; if(i < npb->strLen || c[iTmp] == '/') { ++iTmp; /* skip slash */ idxPort2 = iTmp; - if(ln_v2_parseNumber(npb, &iTmp, NULL, &lenPort2, NULL) == 0) { + if(ln_v2_parseNumber(npb, &iTmp, NULL, parser_name, &lenPort2, NULL) == 0) { iTmp += lenPort2; if(iTmp < npb->strLen && c[iTmp] == ')') { i = iTmp + 1; /* match, so use new index */ @@ -2167,7 +2169,7 @@ PARSER_Parse(IPv6) /* prevent pure IPv4 address to be recognized */ if(beginBlock == *offs) goto done; i = beginBlock; - if(ln_v2_parseIPv4(npb, &i, NULL, &ipv4_parsed, NULL) != 0) + if(ln_v2_parseIPv4(npb, &i, NULL, parser_name, &ipv4_parsed, NULL) != 0) goto done; i += ipv4_parsed; } @@ -3005,18 +3007,21 @@ PARSER_Parse(Repeat) size_t strtoffs = *offs; size_t lastKnownGood = strtoffs; struct json_object *json_arr = NULL; + struct json_object *parsed_value = NULL; const size_t parsedTo_save = npb->parsedTo; + int mergeResults = parser_name != NULL && parser_name[0] == '.' && parser_name[1] == '\0'; do { - struct json_object *parsed_value = json_object_new_object(); + if(parsed_value == NULL) { + parsed_value = json_object_new_object(); + } r = ln_normalizeRec(npb, data->parser, strtoffs, 1, - parsed_value, &endNode); + parsed_value, &endNode, data->failOnDuplicate, NULL, parser_name); strtoffs = npb->parsedTo; LN_DBGPRINTF(npb->ctx, "repeat parser returns %d, parsed %zu, json: %s", r, npb->parsedTo, json_object_to_json_string(parsed_value)); if(r != 0) { - json_object_put(parsed_value); if(data->permitMismatchInParser) { strtoffs = lastKnownGood; /* go back to final match */ LN_DBGPRINTF(npb->ctx, "mismatch in repeat, " @@ -3027,37 +3032,43 @@ PARSER_Parse(Repeat) } } - if(json_arr == NULL) { - json_arr = json_object_new_array(); - } + if (!mergeResults) { + if(json_arr == NULL) { + json_arr = json_object_new_array(); + } - /* check for name=".", which means we need to place the - * value only into to array. As we do not have direct - * access to the key, we loop over our result as a work- - * around. - */ - struct json_object *toAdd = parsed_value; - struct json_object_iterator it = json_object_iter_begin(parsed_value); - struct json_object_iterator itEnd = json_object_iter_end(parsed_value); - while (!json_object_iter_equal(&it, &itEnd)) { - const char *key = json_object_iter_peek_name(&it); - struct json_object *const val = json_object_iter_peek_value(&it); - if(key[0] == '.' && key[1] == '\0') { - json_object_get(val); /* inc refcount! */ - toAdd = val; + /* check for name=".", which means we need to place the + * value only into to array. As we do not have direct + * access to the key, we loop over our result as a work- + * around. + */ + struct json_object *toAdd = parsed_value; + struct json_object_iterator it = json_object_iter_begin(parsed_value); + struct json_object_iterator itEnd = json_object_iter_end(parsed_value); + while (!json_object_iter_equal(&it, &itEnd)) { + const char *key = json_object_iter_peek_name(&it); + struct json_object *const val = json_object_iter_peek_value(&it); + if(key[0] == '.' && key[1] == '\0') { + json_object_get(val); /* inc refcount! */ + toAdd = val; + } + json_object_iter_next(&it); } - json_object_iter_next(&it); - } - json_object_array_add(json_arr, toAdd); - if(toAdd != parsed_value) - json_object_put(parsed_value); - LN_DBGPRINTF(npb->ctx, "arr: %s", json_object_to_json_string(json_arr)); + json_object_array_add(json_arr, toAdd); + + if(toAdd != parsed_value) + json_object_put(parsed_value); + LN_DBGPRINTF(npb->ctx, "arr: %s", json_object_to_json_string(json_arr)); + + // If we are an array, we need to get a new value and we don't want to free at end + parsed_value = NULL; + } /* now check if we shall continue */ npb->parsedTo = 0; lastKnownGood = strtoffs; /* record pos in case of fail in while */ - r = ln_normalizeRec(npb, data->while_cond, strtoffs, 1, NULL, &endNode); + r = ln_normalizeRec(npb, data->while_cond, strtoffs, 1, NULL, &endNode, 0, NULL, parser_name); LN_DBGPRINTF(npb->ctx, "repeat while returns %d, parsed %zu", r, npb->parsedTo); if(r == 0) @@ -3068,15 +3079,25 @@ PARSER_Parse(Repeat) /* success, persist */ *parsed = strtoffs - *offs; if(value == NULL) { - json_object_put(json_arr); + if (json_arr != NULL) { + json_object_put(json_arr); + } + if (parsed_value != NULL) { + json_object_put(parsed_value); + } } else { - *value = json_arr; + *value = !mergeResults ? json_arr : parsed_value; } npb->parsedTo = parsedTo_save; r = 0; /* success */ done: - if(r != 0 && json_arr != NULL) { - json_object_put(json_arr); + if(r != 0) { + if (json_arr != NULL) { + json_object_put(json_arr); + } + if (parsed_value != NULL) { + json_object_put(parsed_value); + } } return r; } @@ -3110,6 +3131,8 @@ PARSER_Construct(Repeat) endnode->flags.isTerminal = 1; } else if(!strcasecmp(key, "option.permitMismatchInParser")) { data->permitMismatchInParser = json_object_get_boolean(val); + } else if(!strcasecmp(key, "option.failOnDuplicate")) { + data->failOnDuplicate = json_object_get_boolean(val); } else { ln_errprintf(ctx, 0, "invalid param for hexnumber: %s", json_object_to_json_string(val)); diff --git a/src/parser.h b/src/parser.h index 38be62d1..87549af9 100644 --- a/src/parser.h +++ b/src/parser.h @@ -39,11 +39,13 @@ // TODO #warning check how to handle "value" - does it need to be set to NULL? #define PARSERDEF_NO_DATA(parser) \ - int ln_v2_parse##parser(npb_t *npb, size_t *offs, void *const, size_t *parsed, struct json_object **value) + int ln_v2_parse##parser(npb_t *npb, size_t *offs, void *const, const char *parser_name, \ + size_t *parsed, struct json_object **value) #define PARSERDEF(parser) \ int ln_construct##parser(ln_ctx ctx, json_object *const json, void **pdata); \ - int ln_v2_parse##parser(npb_t *npb, size_t *offs, void *const, size_t *parsed, struct json_object **value); \ + int ln_v2_parse##parser(npb_t *npb, size_t *offs, void *const, const char *parser_name, \ + size_t *parsed, struct json_object **value); \ void ln_destruct##parser(ln_ctx ctx, void *const pdata) PARSERDEF(RFC5424Date); @@ -89,6 +91,7 @@ struct data_Repeat { ln_pdag *parser; ln_pdag *while_cond; int permitMismatchInParser; + int failOnDuplicate; }; #endif /* #ifndef LIBLOGNORM_PARSER_H_INCLUDED */ diff --git a/src/pdag.c b/src/pdag.c index 0ddbcf80..508a3b66 100644 --- a/src/pdag.c +++ b/src/pdag.c @@ -132,18 +132,21 @@ ln_parserName2ID(const char *const __restrict__ name) /* find type pdag in table. If "bAdd" is set, add it if not * already present, a new entry will be added. * Returns NULL on error, ptr to type pdag entry otherwise + * + * We return the index here so that previously parsed rules/types before a realloc + * don't end up with a pointer to freed memory */ -struct ln_type_pdag * +int ln_pdagFindType(ln_ctx ctx, const char *const __restrict__ name, const int bAdd) { - struct ln_type_pdag *td = NULL; + int td = -1; int i; LN_DBGPRINTF(ctx, "ln_pdagFindType, name '%s', bAdd: %d, nTypes %d", name, bAdd, ctx->nTypes); for(i = 0 ; i < ctx->nTypes ; ++i) { if(!strcmp(ctx->type_pdags[i].name, name)) { - td = ctx->type_pdags + i; + td = i; goto done; } } @@ -162,10 +165,10 @@ ln_pdagFindType(ln_ctx ctx, const char *const __restrict__ name, const int bAdd) goto done; } ctx->type_pdags = newarr; - td = ctx->type_pdags + ctx->nTypes; - ++ctx->nTypes; - td->name = strdup(name); - td->pdag = ln_newPDAG(ctx); + /* td now is index of new member and nTypes is index+1 (count) */ + td = ctx->nTypes++; + ctx->type_pdags[td].name = strdup(name); + ctx->type_pdags[td].pdag = ln_newPDAG(ctx); done: return td; } @@ -203,7 +206,7 @@ ln_newParser(ln_ctx ctx, json_object *json; const char *val; prsid_t prsid; - struct ln_type_pdag *custType = NULL; + int custType = -1; const char *name = NULL; const char *textconf = json_object_to_json_string(prscnf); int parserPrio; @@ -219,7 +222,7 @@ ln_newParser(ln_ctx ctx, prsid = PRS_CUSTOM_TYPE; custType = ln_pdagFindType(ctx, val, 0); parserPrio = 16; /* hopefully relatively specific... */ - if(custType == NULL) { + if(custType < 0) { ln_errprintf(ctx, 0, "unknown user-defined type '%s'", val); goto done; } @@ -1316,6 +1319,37 @@ addUnparsedField(const char *str, const size_t strLen, const size_t offs, struct } +static int +checkDuplicate(const ln_parser_t *const prs, + struct json_object *json, + struct json_object *value, + const char *check) +{ + int r = 0; + + if (NULL != value && NULL != json) { + struct json_object_iterator it = json_object_iter_begin(value); + struct json_object_iterator itEnd = json_object_iter_end(value); + while (!json_object_iter_equal(&it, &itEnd)) { + const char *key = json_object_iter_peek_name(&it); + if(key[0] == '.' && key[1] == '.' && key[2] == '\0') { + key = prs->name; + } + if (json_object_object_get_ex(json, key, NULL)) { + r = 1; + break; + } + json_object_iter_next(&it); + } + } + + if (r == 0 && NULL != json && NULL != check && json_object_object_get_ex(json, check, NULL)) { + r = 1; + } + + return r; +} + /* Do some fixup to the json that we cannot do on a lower layer */ static int fixJSON(struct ln_pdag *dag, @@ -1396,12 +1430,16 @@ tryParser(npb_t *const __restrict__ npb, size_t *offs, size_t *const __restrict__ pParsed, struct json_object **value, - const ln_parser_t *const prs + const ln_parser_t *const prs, + int failOnDuplicate, + struct json_object *cur_json_object, + const char *parser_name ) { - int r; + int r = LN_WRONGPARSER; struct ln_pdag *endNode = NULL; size_t parsedTo = npb->parsedTo; + struct ln_type_pdag *custType = NULL;; # ifdef ADVANCED_STATS char hdr[16]; const size_t lenhdr @@ -1433,12 +1471,17 @@ tryParser(npb_t *const __restrict__ npb, # endif if(prs->prsid == PRS_CUSTOM_TYPE) { + if (prs->custType < 0 || prs->custType >= dag->ctx->nTypes) { + LN_DBGPRINTF(dag->ctx, "tryParser: Invalid custom type index: %d (%d types)", prs->custType, dag->ctx->nTypes); + goto done; + } if(*value == NULL) *value = json_object_new_object(); - LN_DBGPRINTF(dag->ctx, "calling custom parser '%s'", prs->custType->name); - r = ln_normalizeRec(npb, prs->custType->pdag, *offs, 1, *value, &endNode); + custType = &dag->ctx->type_pdags[prs->custType]; + LN_DBGPRINTF(dag->ctx, "calling custom parser '%s'", custType->name); + r = ln_normalizeRec(npb, custType->pdag, *offs, 1, *value, &endNode, failOnDuplicate, cur_json_object, parser_name); LN_DBGPRINTF(dag->ctx, "called CUSTOM PARSER '%s', result %d, " - "offs %zd, *pParsed %zd", prs->custType->name, r, *offs, *pParsed); + "offs %zd, *pParsed %zd", custType->name, r, *offs, *pParsed); *pParsed = npb->parsedTo - *offs; #ifdef ADVANCED_STATS es_addBuf(&npb->astats.exec_path, hdr, lenhdr); @@ -1446,8 +1489,9 @@ tryParser(npb_t *const __restrict__ npb, #endif } else { r = parser_lookup_table[prs->prsid].parser(npb, - offs, prs->parser_data, pParsed, (prs->name == NULL) ? NULL : value); + offs, prs->parser_data, parser_name, pParsed, (prs->name == NULL) ? NULL : value); } +done: LN_DBGPRINTF(npb->ctx, "parser lookup returns %d, pParsed %zu", r, *pParsed); npb->parsedTo = parsedTo; @@ -1534,7 +1578,10 @@ ln_normalizeRec(npb_t *const __restrict__ npb, const size_t offs, const int bPartialMatch, struct json_object *json, - struct ln_pdag **endNode + struct ln_pdag **endNode, + int failOnDuplicate, + struct json_object *cur_json_object, + const char *parser_name ) { int r = LN_WRONGPARSER; @@ -1544,7 +1591,7 @@ ln_normalizeRec(npb_t *const __restrict__ npb, size_t parsedTo = npb->parsedTo; size_t parsed = 0; struct json_object *value; - + LN_DBGPRINTF(dag->ctx, "%zu: enter parser, dag node %p, json %p", offs, dag, json); ++dag->stats.called; @@ -1554,8 +1601,12 @@ LN_DBGPRINTF(dag->ctx, "%zu: enter parser, dag node %p, json %p", offs, dag, jso #endif /* now try the parsers */ - for(iprs = 0 ; iprs < dag->nparsers && r != 0 ; ++iprs) { + for(iprs = 0 ; iprs < dag->nparsers && r != 0; ++iprs) { const ln_parser_t *const prs = dag->parsers + iprs; + if (failOnDuplicate && checkDuplicate(prs, cur_json_object, NULL, prs->name)) { + LN_DBGPRINTF(dag->ctx, "parser field '%s' already exists with skip duplicate set, skipping", prs->name); + continue; + } if(dag->ctx->debug) { LN_DBGPRINTF(dag->ctx, "%zu/%d:trying '%s' parser for field '%s', " "data '%s'", @@ -1566,15 +1617,16 @@ LN_DBGPRINTF(dag->ctx, "%zu: enter parser, dag node %p, json %p", offs, dag, jso } i = offs; value = NULL; - localR = tryParser(npb, dag, &i, &parsed, &value, prs); + localR = tryParser(npb, dag, &i, &parsed, &value, prs, failOnDuplicate, json, prs->name); if(localR == 0) { parsedTo = i + parsed; /* potential hit, need to verify */ LN_DBGPRINTF(dag->ctx, "%zu: potential hit, trying subtree %p", offs, prs->node); r = ln_normalizeRec(npb, prs->node, parsedTo, - bPartialMatch, json, endNode); + bPartialMatch, json, endNode, failOnDuplicate, cur_json_object, parser_name); LN_DBGPRINTF(dag->ctx, "%zu: subtree returns %d, parsedTo %zu", offs, r, parsedTo); + if(r == 0) { LN_DBGPRINTF(dag->ctx, "%zu: parser matches at %zu", offs, i); CHKR(fixJSON(dag, &value, json, prs)); @@ -1644,7 +1696,7 @@ ln_normalize(ln_ctx ctx, const char *str, const size_t strLen, struct json_objec CHKN(*json_p = json_object_new_object()); } - r = ln_normalizeRec(&npb, ctx->pdag, 0, 0, *json_p, &endNode); + r = ln_normalizeRec(&npb, ctx->pdag, 0, 0, *json_p, &endNode, 0, NULL, NULL); if(ctx->debug) { if(r == 0) { diff --git a/src/pdag.h b/src/pdag.h index a9a1a3d7..508bc6b0 100644 --- a/src/pdag.h +++ b/src/pdag.h @@ -82,8 +82,8 @@ struct ln_parser_s { prsid_t prsid; /**< parser ID (for lookup table) */ ln_pdag *node; /**< node to branch to if parser succeeded */ void *parser_data; /**< opaque data that the field-parser understands */ - struct ln_type_pdag *custType; /**< points to custom type, if such is used */ - int prio; /**< priority (combination of user- and parser-specific parts) */ + int custType; /**< index of custom type, if such is used */ + int prio; /**< priority (combination of user- and parser-specific parts) */ const char *name; /**< field name */ const char *conf; /**< configuration as printable json for comparison reasons */ }; @@ -92,7 +92,7 @@ struct ln_parser_info { const char *name; /**< parser name as used in rule base */ int prio; /**< parser specific prio in range 0..255 */ int (*construct)(ln_ctx ctx, json_object *const json, void **); - int (*parser)(npb_t *npb, size_t*, void *const, + int (*parser)(npb_t *npb, size_t*, void *const, const char *, size_t*, struct json_object **); /**< parser to use */ void (*destruct)(ln_ctx, void *const); /* note: destructor is only needed if parser data exists */ #ifdef ADVANCED_STATS @@ -252,7 +252,7 @@ int ln_pdagOptimize(ln_ctx ctx); void ln_fullPdagStats(ln_ctx ctx, FILE *const fp, const int); ln_parser_t * ln_newLiteralParser(ln_ctx ctx, char lit); ln_parser_t* ln_newParser(ln_ctx ctx, json_object *const prscnf); -struct ln_type_pdag * ln_pdagFindType(ln_ctx ctx, const char *const __restrict__ name, const int bAdd); +int ln_pdagFindType(ln_ctx ctx, const char *const __restrict__ name, const int bAdd); void ln_fullPDagStatsDOT(ln_ctx ctx, FILE *const fp); /* friends */ @@ -262,7 +262,10 @@ ln_normalizeRec(npb_t *const __restrict__ npb, const size_t offs, const int bPartialMatch, struct json_object *json, - struct ln_pdag **endNode + struct ln_pdag **endNode, + int failOnDuplicate, + json_object *cur_json_object, + const char *parser_name ); #endif /* #ifndef LOGNORM_PDAG_H_INCLUDED */ diff --git a/src/samp.c b/src/samp.c index 705d6a09..c340bf07 100644 --- a/src/samp.c +++ b/src/samp.c @@ -645,9 +645,9 @@ processType(ln_ctx ctx, // TODO: optimize CHKN(str = es_newStr(lenBuf)); CHKR(es_addBuf(&str, (char*)buf + offs, lenBuf - offs)); - struct ln_type_pdag *const td = ln_pdagFindType(ctx, typename, 1); - CHKN(td); - addSampToTree(ctx, str, td->pdag, NULL); + int td = ln_pdagFindType(ctx, typename, 1); + CHKN((td >= 0 && td < ctx->nTypes) ? ctx->type_pdags[td].pdag : NULL); + addSampToTree(ctx, str, ctx->type_pdags[td].pdag, NULL); es_deleteStr(str); r = 0; done: return r; @@ -907,7 +907,7 @@ ln_processSamp(ln_ctx ctx, const char *buf, const size_t lenBuf) * Read a character from our sample source. */ static int -ln_sampReadChar(const ln_ctx ctx, FILE *const __restrict__ repo, const char **inpbuf) +ln_sampReadChar(const ln_ctx __attribute__((unused)) ctx, FILE *const __restrict__ repo, const char **inpbuf) { int c; assert((repo != NULL && inpbuf == NULL) || (repo == NULL && inpbuf != NULL)); diff --git a/tests/json_eq.c b/tests/json_eq.c index 5875ac43..036a42e1 100644 --- a/tests/json_eq.c +++ b/tests/json_eq.c @@ -30,9 +30,9 @@ static int arr_eq(obj* expected, obj* actual) { int actual_len = json_object_array_length(actual); if (expected_len != actual_len) return 0; for (int i = 0; i < expected_len; i++) { - obj* exp = json_object_array_get_idx(expected, i); + obj* _exp = json_object_array_get_idx(expected, i); obj* act = json_object_array_get_idx(actual, i); - eql &= eq(exp, act); + eql &= eq(_exp, act); } return eql; } From b9bb504d21a5c2004330aba17cac3508553f5e97 Mon Sep 17 00:00:00 2001 From: Kenneth Shelton Date: Fri, 28 Jul 2017 16:18:40 +0000 Subject: [PATCH 2/4] Fix test. After while loop there should be 2 spaces, not 3. --- src/parser.c | 2 ++ tests/repeat_mismatch_in_while.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/parser.c b/src/parser.c index 70505cce..be49fbbf 100644 --- a/src/parser.c +++ b/src/parser.c @@ -3022,6 +3022,8 @@ PARSER_Parse(Repeat) r, npb->parsedTo, json_object_to_json_string(parsed_value)); if(r != 0) { + json_object_put(parsed_value); + parsed_value = NULL; if(data->permitMismatchInParser) { strtoffs = lastKnownGood; /* go back to final match */ LN_DBGPRINTF(npb->ctx, "mismatch in repeat, " diff --git a/tests/repeat_mismatch_in_while.sh b/tests/repeat_mismatch_in_while.sh index 843b9752..91fbf707 100755 --- a/tests/repeat_mismatch_in_while.sh +++ b/tests/repeat_mismatch_in_while.sh @@ -24,7 +24,7 @@ assert_output_json_eq '{ "originalmsg": "Aug 18 13:18:45 192.168.99.2 %ASA-6-106 reset_rules add_rule 'version=2' add_rule 'prefix=%timestamp:date-rfc3164% %hostname:word%' -add_rule 'rule=cisco,fwblock: \x25ASA-6-106015\x3a Deny %proto:word% (no connection) from %source:cisco-interface-spec% to %dest:cisco-interface-spec% flags %flags:repeat{ "option.permitMismatchInParser":true, "parser": {"type":"word", "name":"."}, "while":{"type":"literal", "text":" "} }%\x20 on interface %srciface:word%' +add_rule 'rule=cisco,fwblock: \x25ASA-6-106015\x3a Deny %proto:word% (no connection) from %source:cisco-interface-spec% to %dest:cisco-interface-spec% flags %flags:repeat{ "option.permitMismatchInParser":true, "parser": {"type":"word", "name":"."}, "while":{"type":"literal", "text":" "} }%\x20 on interface %srciface:word%' echo step 2 execute 'Aug 18 13:18:45 192.168.99.2 %ASA-6-106015: Deny TCP (no connection) from 173.252.88.66/443 to 76.79.249.222/52746 flags RST on interface outside' From 30302590d887daee40b89818bd319a070564ecd5 Mon Sep 17 00:00:00 2001 From: Kenneth Shelton Date: Wed, 9 Aug 2017 18:24:02 +0000 Subject: [PATCH 3/4] Made ipv6 behave more like ipv4 (Doesn't have to end in space) Fixed null ptr deref --- src/parser.c | 9 +++++++-- tests/field_ipv6.sh | 2 +- tests/field_ipv6_jsoncnf.sh | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/parser.c b/src/parser.c index be49fbbf..de24c35e 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1625,7 +1625,9 @@ PARSER_Parse(OpQuotedString) /* create JSON value to save quoted string contents */ CHKN(cstr = strndup((char*)c + *offs + 1, *parsed - 2)); } - CHKN(*value = json_object_new_string(cstr)); + if (value != NULL) { + CHKN(*value = json_object_new_string(cstr)); + } r = 0; /* success */ done: @@ -2143,11 +2145,14 @@ PARSER_Parse(IPv6) if(skipIPv6AddrBlock(npb, &i) != 0) goto done; nBlocks++; if(i == npb->strLen) goto chk_ok; - if(isspace(c[i])) goto chk_ok; + /* no more valid chars, check address */ + if(c[i] != ':' && c[i] != '.') goto chk_ok; if(c[i] == '.'){ /* IPv4 processing! */ hasIPv4 = 1; break; } + /* maximum blocks consumed and not ipv4, check if valid */ + if (nBlocks == 8) goto chk_ok; if(c[i] != ':') goto done; i++; /* "eat" ':' */ if(i == npb->strLen) goto chk_ok; diff --git a/tests/field_ipv6.sh b/tests/field_ipv6.sh index 3887a55d..57327d87 100755 --- a/tests/field_ipv6.sh +++ b/tests/field_ipv6.sh @@ -41,7 +41,7 @@ execute 'ABCD:EF01:2345:6789:ABCD:EF01:2345::6789' # :: with too many blocks assert_output_json_eq '{ "originalmsg": "ABCD:EF01:2345:6789:ABCD:EF01:2345::6789", "unparsed-data": "ABCD:EF01:2345:6789:ABCD:EF01:2345::6789" }' execute 'ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798' # too many blocks (9) -assert_output_json_eq '{"originalmsg": "ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798", "unparsed-data": "ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798" }' +assert_output_json_eq '{"originalmsg": "ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798", "unparsed-data": ":6798" }' execute ':0:0:0:0:0:0:1' # missing first digit assert_output_json_eq '{ "originalmsg": ":0:0:0:0:0:0:1", "unparsed-data": ":0:0:0:0:0:0:1" }' diff --git a/tests/field_ipv6_jsoncnf.sh b/tests/field_ipv6_jsoncnf.sh index 3a75059a..f20d0384 100755 --- a/tests/field_ipv6_jsoncnf.sh +++ b/tests/field_ipv6_jsoncnf.sh @@ -41,7 +41,7 @@ execute 'ABCD:EF01:2345:6789:ABCD:EF01:2345::6789' # :: with too many blocks assert_output_json_eq '{ "originalmsg": "ABCD:EF01:2345:6789:ABCD:EF01:2345::6789", "unparsed-data": "ABCD:EF01:2345:6789:ABCD:EF01:2345::6789" }' execute 'ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798' # too many blocks (9) -assert_output_json_eq '{"originalmsg": "ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798", "unparsed-data": "ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798" }' +assert_output_json_eq '{"originalmsg": "ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798", "unparsed-data": ":6798" }' execute ':0:0:0:0:0:0:1' # missing first digit assert_output_json_eq '{ "originalmsg": ":0:0:0:0:0:0:1", "unparsed-data": ":0:0:0:0:0:0:1" }' From f5163ee8976e3e77bb59deef6864b5aa72c9f79c Mon Sep 17 00:00:00 2001 From: Kenneth Shelton Date: Fri, 11 Aug 2017 15:16:21 +0000 Subject: [PATCH 4/4] Fix backtracking --- src/parser.c | 5 +++++ src/pdag.c | 24 ++++++++++++++++-------- src/pdag.h | 1 + 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/parser.c b/src/parser.c index de24c35e..9c6411da 100644 --- a/src/parser.c +++ b/src/parser.c @@ -3010,10 +3010,12 @@ PARSER_Parse(Repeat) struct data_Repeat *const data = (struct data_Repeat*) pdata; struct ln_pdag *endNode = NULL; size_t strtoffs = *offs; + size_t lastMatch = strtoffs; size_t lastKnownGood = strtoffs; struct json_object *json_arr = NULL; struct json_object *parsed_value = NULL; const size_t parsedTo_save = npb->parsedTo; + const size_t longestParsedTo_save = npb->longestParsedTo; int mergeResults = parser_name != NULL && parser_name[0] == '.' && parser_name[1] == '\0'; do { @@ -3035,6 +3037,8 @@ PARSER_Parse(Repeat) "parse ptr back to %zd", strtoffs); goto success; } else { + // Reset longest match + npb->longestParsedTo = lastMatch > longestParsedTo_save ? lastMatch : longestParsedTo_save; goto done; } } @@ -3074,6 +3078,7 @@ PARSER_Parse(Repeat) /* now check if we shall continue */ npb->parsedTo = 0; + lastMatch = lastKnownGood; lastKnownGood = strtoffs; /* record pos in case of fail in while */ r = ln_normalizeRec(npb, data->while_cond, strtoffs, 1, NULL, &endNode, 0, NULL, parser_name); LN_DBGPRINTF(npb->ctx, "repeat while returns %d, parsed %zu", diff --git a/src/pdag.c b/src/pdag.c index 508a3b66..b65adb34 100644 --- a/src/pdag.c +++ b/src/pdag.c @@ -1365,6 +1365,7 @@ fixJSON(struct ln_pdag *dag, /* Free the unneeded value */ json_object_put(*value); } + *value = NULL; } else if(prs->name[0] == '.' && prs->name[1] == '\0') { if(json_object_get_type(*value) == json_type_object) { struct json_object_iterator it = json_object_iter_begin(*value); @@ -1419,6 +1420,7 @@ fixJSON(struct ln_pdag *dag, } } r = 0; + *value = NULL; return r; } @@ -1590,7 +1592,7 @@ ln_normalizeRec(npb_t *const __restrict__ npb, size_t iprs; size_t parsedTo = npb->parsedTo; size_t parsed = 0; - struct json_object *value; + struct json_object *value = NULL; LN_DBGPRINTF(dag->ctx, "%zu: enter parser, dag node %p, json %p", offs, dag, json); @@ -1616,7 +1618,6 @@ LN_DBGPRINTF(dag->ctx, "%zu: enter parser, dag node %p, json %p", offs, dag, jso : "UNKNOWN"); } i = offs; - value = NULL; localR = tryParser(npb, dag, &i, &parsed, &value, prs, failOnDuplicate, json, prs->name); if(localR == 0) { parsedTo = i + parsed; @@ -1630,9 +1631,14 @@ LN_DBGPRINTF(dag->ctx, "%zu: enter parser, dag node %p, json %p", offs, dag, jso if(r == 0) { LN_DBGPRINTF(dag->ctx, "%zu: parser matches at %zu", offs, i); CHKR(fixJSON(dag, &value, json, prs)); + value = NULL; if(npb->ctx->opts & LN_CTXOPT_ADD_RULE) { add_rule_to_mockup(npb, prs); } + /* did we have a longer parser --> then update */ + if(parsedTo > npb->parsedTo) + npb->parsedTo = parsedTo; + } else { ++dag->stats.backtracked; #ifdef ADVANCED_STATS @@ -1641,14 +1647,16 @@ LN_DBGPRINTF(dag->ctx, "%zu: enter parser, dag node %p, json %p", offs, dag, jso #endif LN_DBGPRINTF(dag->ctx, "%zu nonmatch, backtracking required, parsed to=%zu", offs, parsedTo); - if (value != NULL) { /* Free the value if it was created */ - json_object_put(value); - } } } + if (value != NULL) { /* Free the value if it was created */ + json_object_put(value); + value = NULL; + } + /* did we have a longer parser --> then update */ - if(parsedTo > npb->parsedTo) - npb->parsedTo = parsedTo; + if(parsedTo > npb->longestParsedTo) + npb->longestParsedTo = parsedTo; LN_DBGPRINTF(dag->ctx, "parsedTo %zu, *pParsedTo %zu", parsedTo, npb->parsedTo); } @@ -1727,7 +1735,7 @@ ln_normalize(ln_ctx ctx, const char *str, const size_t strLen, struct json_objec addRuleMetadata(&npb, *json_p, endNode); r = 0; } else { - addUnparsedField(str, strLen, npb.parsedTo, *json_p); + addUnparsedField(str, strLen, npb.longestParsedTo, *json_p); } if(ctx->opts & LN_CTXOPT_ADD_RULE) { diff --git a/src/pdag.h b/src/pdag.h index 508bc6b0..63ee75c2 100644 --- a/src/pdag.h +++ b/src/pdag.h @@ -159,6 +159,7 @@ struct npb { const char *str; /**< to-be-normalized message */ size_t strLen; /**< length of it */ size_t parsedTo; /**< up to which byte could this be parsed? */ + size_t longestParsedTo; /**< up to which byte could this be parsed? */ es_str_t *rule; /**< a mock-up of the rule used to parse */ es_str_t *exec_path; #ifdef ADVANCED_STATS