From 8a704d742e268a34000ec4b1795b229f9b0fd530 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 12 Jun 2023 14:22:56 -0400 Subject: [PATCH 01/51] tests/endids/Makefile: Move SRC definition out of the loop. Re-adding it during each pass of the for loop led to a bunch of warnings during the bulid. --- tests/endids/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/endids/Makefile b/tests/endids/Makefile index cee444d3f..5a299d8bc 100755 --- a/tests/endids/Makefile +++ b/tests/endids/Makefile @@ -8,10 +8,11 @@ TEST_OUTDIR.tests/endids = ${BUILD}/tests/endids INCDIR.${TEST_SRCDIR.tests/endids}/endids${n}.c += src/adt .endfor +SRC += ${TEST_SRCDIR.tests/endids}/utils.c + .for n in ${TEST.tests/endids:T:R:C/^endids//} test:: ${TEST_OUTDIR.tests/endids}/res${n} SRC += ${TEST_SRCDIR.tests/endids}/endids${n}.c -SRC += ${TEST_SRCDIR.tests/endids}/utils.c CFLAGS.${TEST_SRCDIR.tests/endids}/endids${n}.c += -UNDEBUG ${TEST_OUTDIR.tests/endids}/run${n}: ${TEST_OUTDIR.tests/endids}/endids${n}.o ${TEST_OUTDIR.tests/endids}/utils.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a From b9b4cde6d48f205c44acf0cc38cf97df5d4dc0f0 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 13 Jun 2023 16:13:24 -0400 Subject: [PATCH 02/51] Move definitions into include/adt/common.h. Add BUILD_FOR_FUZZER. Previously these were split between src/libfsm/internal.h and include/common/check.h, but some of the internal.h definitions were used in libre. Also, add `BUILD_FOR_FUZZER`, which sets more restrictive limits for cases that lead to uninteresting failure modes that get reported over and over during fuzzing. --- include/adt/common.h | 102 +++++++++++++++++++++++++++++++++++++ include/adt/hash.h | 2 +- include/common/check.h | 21 -------- src/adt/internedstateset.c | 2 +- src/adt/siphash.c | 2 +- src/libfsm/internal.h | 74 +-------------------------- src/libre/ast_analysis.c | 15 ++++++ src/libre/parser.act | 2 +- 8 files changed, 122 insertions(+), 98 deletions(-) create mode 100644 include/adt/common.h delete mode 100644 include/common/check.h diff --git a/include/adt/common.h b/include/adt/common.h new file mode 100644 index 000000000..635b3a495 --- /dev/null +++ b/include/adt/common.h @@ -0,0 +1,102 @@ +#ifndef LIBFSM_COMMON_H +#define LIBFSM_COMMON_H + +/* Internal definitions shared between adt, libfsm, and libre */ + +#if defined(__clang__) +/* Newer versions of clang's UBSan emit warnings about *all* unsigned + * integer overflows. While they are defined behavior, overflow can + * cause bugs. This macro ignores them for a particular function. + * Overflow/rollover is expected in when hashing, for example. */ +#define SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() \ + __attribute__((no_sanitize("integer"))) +#else +#define SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() +#endif + +/* If set to non-zero, do extra intensive integrity checks, often inside + * some inner loops, which are far too expensive for production. */ +#ifndef EXPENSIVE_CHECKS +#define EXPENSIVE_CHECKS 0 +#endif + +/* If set to non-zero, this build should reject inputs as unsupported + * that lead to uninteresting failures while fuzzing -- for example, + * it's not surprising that `(some regex){1000000}` can hit the + * allocator limit, but once the fuzzer finds that it will produce + * variants of the failure over and over. */ +#ifndef BUILD_FOR_FUZZER +#define BUILD_FOR_FUZZER 0 +#endif + +/* If non-zero, expand the timer macros defined below, otherwise + * they compile away. */ +#ifndef TRACK_TIMES +#define TRACK_TIMES 0 +#endif + +#if EXPENSIVE_CHECKS && TRACK_TIMES +#error benchmarking with EXPENSIVE_CHECKS +#endif + +#if TRACK_TIMES +#include +#define TIMER_LOG_THRESHOLD 100 + +#define INIT_TIMERS() struct timeval pre, post +#define INIT_TIMERS_NAMED(PREFIX) struct timeval PREFIX ## _pre, PREFIX ## _post +#define TIME(T) \ + if (gettimeofday(T, NULL) == -1) { assert(!"gettimeofday"); } +#define DIFF_MSEC(LABEL, PRE, POST, ACCUM) \ + do { \ + size_t *accum = ACCUM; \ + const size_t diff_usec = \ + (1000000*(POST.tv_sec - PRE.tv_sec) \ + + (POST.tv_usec - PRE.tv_usec)); \ + const size_t diff_msec = diff_usec/1000; \ + if (diff_msec >= TIMER_LOG_THRESHOLD \ + || TRACK_TIMES > 1) { \ + fprintf(stderr, "%s: %zu msec%s\n", LABEL, \ + diff_msec, \ + diff_msec >= 100 ? " #### OVER 100" : ""); \ + } \ + if (accum != NULL) { \ + (*accum) += diff_usec; \ + } \ + } while(0) +#define DIFF_MSEC_ALWAYS(LABEL, PRE, POST, ACCUM) \ + do { \ + size_t *accum = ACCUM; \ + const size_t diff_usec = \ + (1000000*(POST.tv_sec - PRE.tv_sec) \ + + (POST.tv_usec - PRE.tv_usec)); \ + const size_t diff_msec = diff_usec/1000; \ + fprintf(stderr, "%s: %zu msec%s\n", LABEL, diff_msec, \ + diff_msec >= 100 ? " #### OVER 100" : ""); \ + if (accum != NULL) { \ + (*accum) += diff_usec; \ + } \ + } while(0) + +#define DIFF_USEC_ALWAYS(LABEL, PRE, POST, ACCUM) \ + do { \ + size_t *accum = ACCUM; \ + const size_t diff_usec = \ + (1000000*(POST.tv_sec - PRE.tv_sec) \ + + (POST.tv_usec - PRE.tv_usec)); \ + fprintf(stderr, "%s: %zu usec\n", LABEL, diff_usec); \ + if (accum != NULL) { \ + (*accum) += diff_usec; \ + } \ + } while(0) + +#else +#define INIT_TIMERS() +#define INIT_TIMERS_NAMED(PREFIX) +#define TIME(T) +#define DIFF_MSEC(A, B, C, D) +#define DIFF_MSEC_ALWAYS(A, B, C, D) +#define DIFF_USEC_ALWAYS(A, B, C, D) +#endif + +#endif diff --git a/include/adt/hash.h b/include/adt/hash.h index 23ac079a9..d4ed1227d 100644 --- a/include/adt/hash.h +++ b/include/adt/hash.h @@ -9,7 +9,7 @@ #include -#include "common/check.h" +#include "adt/common.h" /* 32 and 64-bit approximations of the golden ratio. */ #define FSM_PHI_32 0x9e3779b9UL diff --git a/include/common/check.h b/include/common/check.h deleted file mode 100644 index 5fba4c92f..000000000 --- a/include/common/check.h +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2021 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#ifndef LIBFSM_CHECK_H -#define LIBFSM_CHECK_H - -#if defined(__clang__) -/* Newer versions of clang's UBSan emit warnings about *all* unsigned - * integer overflows. While they are defined behavior, overflow can - * cause bugs. This macro ignores them for a particular function. - * Overflow/rollover is expected in when hashing, for example. */ -#define SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() \ - __attribute__((no_sanitize("integer"))) -#else -#define SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() -#endif - -#endif diff --git a/src/adt/internedstateset.c b/src/adt/internedstateset.c index 04743d0b9..e6e3a2b8c 100644 --- a/src/adt/internedstateset.c +++ b/src/adt/internedstateset.c @@ -18,7 +18,7 @@ #include #include -#include "common/check.h" +#include "adt/common.h" #include diff --git a/src/adt/siphash.c b/src/adt/siphash.c index af2133e59..71e3909f1 100644 --- a/src/adt/siphash.c +++ b/src/adt/siphash.c @@ -20,7 +20,7 @@ #include /* Added to suppress UBSan warning for (expected) unsigned integer overflow. */ -#include "common/check.h" +#include "adt/common.h" /* default: SipHash-2-4 */ #define cROUNDS 2 diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index 4e25b3950..6e77510a7 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -13,17 +13,13 @@ #include #include -#include "common/check.h" +#include struct bm; struct edge_set; struct state_set; struct state_array; -/* If set to non-zero, do extra intensive integrity checks, often inside - * some inner loops, which are far too expensive for production. */ -#define EXPENSIVE_CHECKS 0 - /* * The alphabet (Sigma) for libfsm's FSM is arbitrary octets. * These octets may or may not spell out UTF-8 sequences, @@ -48,74 +44,6 @@ struct state_array; #define FSM_CAPTURE_MAX INT_MAX -#ifndef TRACK_TIMES -#define TRACK_TIMES 0 -#endif - -#if EXPENSIVE_CHECKS && TRACK_TIMES -#error benchmarking with EXPENSIVE_CHECKS -#endif - -#if TRACK_TIMES -#include -#define TIMER_LOG_THRESHOLD 100 - -#define INIT_TIMERS() struct timeval pre, post -#define INIT_TIMERS_NAMED(PREFIX) struct timeval PREFIX ## _pre, PREFIX ## _post -#define TIME(T) \ - if (gettimeofday(T, NULL) == -1) { assert(!"gettimeofday"); } -#define DIFF_MSEC(LABEL, PRE, POST, ACCUM) \ - do { \ - size_t *accum = ACCUM; \ - const size_t diff_usec = \ - (1000000*(POST.tv_sec - PRE.tv_sec) \ - + (POST.tv_usec - PRE.tv_usec)); \ - const size_t diff_msec = diff_usec/1000; \ - if (diff_msec >= TIMER_LOG_THRESHOLD \ - || TRACK_TIMES > 1) { \ - fprintf(stderr, "%s: %zu msec%s\n", LABEL, \ - diff_msec, \ - diff_msec >= 100 ? " #### OVER 100" : ""); \ - } \ - if (accum != NULL) { \ - (*accum) += diff_usec; \ - } \ - } while(0) -#define DIFF_MSEC_ALWAYS(LABEL, PRE, POST, ACCUM) \ - do { \ - size_t *accum = ACCUM; \ - const size_t diff_usec = \ - (1000000*(POST.tv_sec - PRE.tv_sec) \ - + (POST.tv_usec - PRE.tv_usec)); \ - const size_t diff_msec = diff_usec/1000; \ - fprintf(stderr, "%s: %zu msec%s\n", LABEL, diff_msec, \ - diff_msec >= 100 ? " #### OVER 100" : ""); \ - if (accum != NULL) { \ - (*accum) += diff_usec; \ - } \ - } while(0) - -#define DIFF_USEC_ALWAYS(LABEL, PRE, POST, ACCUM) \ - do { \ - size_t *accum = ACCUM; \ - const size_t diff_usec = \ - (1000000*(POST.tv_sec - PRE.tv_sec) \ - + (POST.tv_usec - PRE.tv_usec)); \ - fprintf(stderr, "%s: %zu usec\n", LABEL, diff_usec); \ - if (accum != NULL) { \ - (*accum) += diff_usec; \ - } \ - } while(0) - -#else -#define INIT_TIMERS() -#define INIT_TIMERS_NAMED(PREFIX) -#define TIME(T) -#define DIFF_MSEC(A, B, C, D) -#define DIFF_MSEC_ALWAYS(A, B, C, D) -#define DIFF_USEC_ALWAYS(A, B, C, D) -#endif - struct fsm_edge { fsm_state_t state; /* destination */ unsigned char symbol; diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 945eeb814..3298f62c2 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -17,6 +17,8 @@ #include "ast.h" #include "ast_analysis.h" +#include + #define LOG_ANALYSIS 0 #define LOG_FIRST_ANALYSIS (0 + LOG_ANALYSIS) #define LOG_REPEATED_GROUPS (0 + LOG_ANALYSIS) @@ -204,6 +206,19 @@ analysis_iter(struct ast_expr *n) (void *)n, (void *)e); set_flags(n, AST_FLAG_NULLABLE); } + + /* When building for fuzzing, ignore uninteresting + * failures from regexes like '.{1000000}' that use + * repetition to hit memory limits. */ +#if BUILD_FOR_FUZZER + if ((n->u.repeat.max != AST_COUNT_UNBOUNDED && n->u.repeat.max >= 10) + || (n->u.repeat.min >= 10)) { + fprintf(stderr, "%s: rejecting regex with {count} >= 10 (%u)\n", + __func__, n->u.repeat.max); + return AST_ANALYSIS_ERROR_MEMORY; + } +#endif + break; } diff --git a/src/libre/parser.act b/src/libre/parser.act index fceee8fb9..b836510ae 100644 --- a/src/libre/parser.act +++ b/src/libre/parser.act @@ -31,6 +31,7 @@ #include #include + #include #include "libre/class.h" #include "libre/class_lookup.h" @@ -1167,7 +1168,6 @@ ADVANCE_LEXER; -#define BUILD_FOR_FUZZER 0 #if BUILD_FOR_FUZZER /* these errors currently are not handled properly */ if (act_state->lex_tok == TOK_ERROR) { From 8a9298e345e799022432498fb03335e0797e782a Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 13 Jun 2023 16:19:23 -0400 Subject: [PATCH 03/51] Update generated parser code, due to BUILD_FOR_FUZZER change. --- src/libre/dialect/glob/parser.c | 48 ++-- src/libre/dialect/glob/parser.h | 4 +- src/libre/dialect/like/parser.c | 48 ++-- src/libre/dialect/like/parser.h | 4 +- src/libre/dialect/literal/parser.c | 32 +-- src/libre/dialect/literal/parser.h | 4 +- src/libre/dialect/native/parser.c | 268 +++++++++--------- src/libre/dialect/native/parser.h | 4 +- src/libre/dialect/pcre/parser.c | 434 ++++++++++++++--------------- src/libre/dialect/pcre/parser.h | 4 +- src/libre/dialect/sql/parser.c | 206 +++++++------- src/libre/dialect/sql/parser.h | 4 +- 12 files changed, 530 insertions(+), 530 deletions(-) diff --git a/src/libre/dialect/glob/parser.c b/src/libre/dialect/glob/parser.c index 04a031b96..6c039b0a4 100644 --- a/src/libre/dialect/glob/parser.c +++ b/src/libre/dialect/glob/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 149 "src/libre/parser.act" +#line 22 "src/libre/parser.act" #include @@ -22,6 +22,7 @@ #include #include + #include #include "libre/class.h" #include "libre/class_lookup.h" @@ -287,7 +288,7 @@ return s; } -#line 291 "src/libre/dialect/glob/parser.c" +#line 292 "src/libre/dialect/glob/parser.c" #ifndef ERROR_TERMINAL @@ -328,7 +329,7 @@ ZL2_list_Hof_Hatoms:; goto ZL1; } -#line 332 "src/libre/dialect/glob/parser.c" +#line 333 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 116 */ @@ -340,7 +341,7 @@ ZL2_list_Hof_Hatoms:; goto ZL2_list_Hof_Hatoms; /* END OF INLINE: list-of-atoms */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -366,12 +367,12 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 811 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 375 "src/libre/dialect/glob/parser.c" +#line 376 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ @@ -383,7 +384,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 387 "src/libre/dialect/glob/parser.c" +#line 388 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -396,7 +397,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: CHAR */ { -#line 592 "src/libre/parser.act" +#line 589 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -409,7 +410,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ZIc = lex_state->buf.a[0]; -#line 413 "src/libre/dialect/glob/parser.c" +#line 414 "src/libre/dialect/glob/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -422,7 +423,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 426 "src/libre/dialect/glob/parser.c" +#line 427 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -436,12 +437,12 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 811 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 445 "src/libre/dialect/glob/parser.c" +#line 446 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ @@ -453,7 +454,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 457 "src/libre/dialect/glob/parser.c" +#line 458 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ @@ -462,7 +463,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 466 "src/libre/dialect/glob/parser.c" +#line 467 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ @@ -481,7 +482,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 485 "src/libre/dialect/glob/parser.c" +#line 486 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -496,14 +497,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 721 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 507 "src/libre/dialect/glob/parser.c" +#line 508 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -515,7 +516,7 @@ ZL1:; goto ZL2; } -#line 519 "src/libre/dialect/glob/parser.c" +#line 520 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -550,7 +551,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast goto ZL1; } -#line 554 "src/libre/dialect/glob/parser.c" +#line 555 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZInode); @@ -571,7 +572,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast goto ZL1; } -#line 575 "src/libre/dialect/glob/parser.c" +#line 576 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -595,14 +596,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 770 "src/libre/parser.act" +#line 767 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 606 "src/libre/dialect/glob/parser.c" +#line 607 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -620,7 +621,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1090 "src/libre/parser.act" static int @@ -702,7 +703,6 @@ ZL0:; ADVANCE_LEXER; -#define BUILD_FOR_FUZZER 0 #if BUILD_FOR_FUZZER /* these errors currently are not handled properly */ if (act_state->lex_tok == TOK_ERROR) { diff --git a/src/libre/dialect/glob/parser.h b/src/libre/dialect/glob/parser.h index d9cb909c2..4b44fa3bd 100644 --- a/src/libre/dialect/glob/parser.h +++ b/src/libre/dialect/glob/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 310 "src/libre/parser.act" +#line 300 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__glob(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1249 "src/libre/parser.act" +#line 1247 "src/libre/parser.act" #line 35 "src/libre/dialect/glob/parser.h" diff --git a/src/libre/dialect/like/parser.c b/src/libre/dialect/like/parser.c index 0be88437a..eb42f63d4 100644 --- a/src/libre/dialect/like/parser.c +++ b/src/libre/dialect/like/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 149 "src/libre/parser.act" +#line 22 "src/libre/parser.act" #include @@ -22,6 +22,7 @@ #include #include + #include #include "libre/class.h" #include "libre/class_lookup.h" @@ -287,7 +288,7 @@ return s; } -#line 291 "src/libre/dialect/like/parser.c" +#line 292 "src/libre/dialect/like/parser.c" #ifndef ERROR_TERMINAL @@ -328,7 +329,7 @@ ZL2_list_Hof_Hatoms:; goto ZL1; } -#line 332 "src/libre/dialect/like/parser.c" +#line 333 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 116 */ @@ -340,7 +341,7 @@ ZL2_list_Hof_Hatoms:; goto ZL2_list_Hof_Hatoms; /* END OF INLINE: list-of-atoms */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -366,12 +367,12 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 811 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 375 "src/libre/dialect/like/parser.c" +#line 376 "src/libre/dialect/like/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ @@ -383,7 +384,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 387 "src/libre/dialect/like/parser.c" +#line 388 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -396,7 +397,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: CHAR */ { -#line 592 "src/libre/parser.act" +#line 589 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -409,7 +410,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ZIc = lex_state->buf.a[0]; -#line 413 "src/libre/dialect/like/parser.c" +#line 414 "src/libre/dialect/like/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -422,7 +423,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 426 "src/libre/dialect/like/parser.c" +#line 427 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -436,12 +437,12 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 811 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 445 "src/libre/dialect/like/parser.c" +#line 446 "src/libre/dialect/like/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ @@ -453,7 +454,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 457 "src/libre/dialect/like/parser.c" +#line 458 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ @@ -462,7 +463,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 466 "src/libre/dialect/like/parser.c" +#line 467 "src/libre/dialect/like/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ @@ -481,7 +482,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 485 "src/libre/dialect/like/parser.c" +#line 486 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -496,14 +497,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 721 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 507 "src/libre/dialect/like/parser.c" +#line 508 "src/libre/dialect/like/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -515,7 +516,7 @@ ZL1:; goto ZL2; } -#line 519 "src/libre/dialect/like/parser.c" +#line 520 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -550,7 +551,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast goto ZL1; } -#line 554 "src/libre/dialect/like/parser.c" +#line 555 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZInode); @@ -571,7 +572,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast goto ZL1; } -#line 575 "src/libre/dialect/like/parser.c" +#line 576 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -595,14 +596,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 770 "src/libre/parser.act" +#line 767 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 606 "src/libre/dialect/like/parser.c" +#line 607 "src/libre/dialect/like/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -620,7 +621,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1090 "src/libre/parser.act" static int @@ -702,7 +703,6 @@ ZL0:; ADVANCE_LEXER; -#define BUILD_FOR_FUZZER 0 #if BUILD_FOR_FUZZER /* these errors currently are not handled properly */ if (act_state->lex_tok == TOK_ERROR) { diff --git a/src/libre/dialect/like/parser.h b/src/libre/dialect/like/parser.h index 5aa0aa4dc..2e3c7b1dd 100644 --- a/src/libre/dialect/like/parser.h +++ b/src/libre/dialect/like/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 310 "src/libre/parser.act" +#line 300 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__like(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1249 "src/libre/parser.act" +#line 1247 "src/libre/parser.act" #line 35 "src/libre/dialect/like/parser.h" diff --git a/src/libre/dialect/literal/parser.c b/src/libre/dialect/literal/parser.c index bebcc0c03..9b85216e3 100644 --- a/src/libre/dialect/literal/parser.c +++ b/src/libre/dialect/literal/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 149 "src/libre/parser.act" +#line 22 "src/libre/parser.act" #include @@ -22,6 +22,7 @@ #include #include + #include #include "libre/class.h" #include "libre/class_lookup.h" @@ -287,7 +288,7 @@ return s; } -#line 291 "src/libre/dialect/literal/parser.c" +#line 292 "src/libre/dialect/literal/parser.c" #ifndef ERROR_TERMINAL @@ -328,7 +329,7 @@ ZL2_list_Hof_Hatoms:; goto ZL1; } -#line 332 "src/libre/dialect/literal/parser.c" +#line 333 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 115 */ @@ -340,7 +341,7 @@ ZL2_list_Hof_Hatoms:; goto ZL2_list_Hof_Hatoms; /* END OF INLINE: list-of-atoms */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -376,7 +377,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 380 "src/libre/dialect/literal/parser.c" +#line 381 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZInode); @@ -397,7 +398,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 401 "src/libre/dialect/literal/parser.c" +#line 402 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -421,14 +422,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 770 "src/libre/parser.act" +#line 767 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 432 "src/libre/dialect/literal/parser.c" +#line 433 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -461,7 +462,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 592 "src/libre/parser.act" +#line 589 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -474,7 +475,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ZIc = lex_state->buf.a[0]; -#line 478 "src/libre/dialect/literal/parser.c" +#line 479 "src/libre/dialect/literal/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -491,7 +492,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 495 "src/libre/dialect/literal/parser.c" +#line 496 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -500,14 +501,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 721 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 511 "src/libre/dialect/literal/parser.c" +#line 512 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -519,7 +520,7 @@ ZL1:; goto ZL2; } -#line 523 "src/libre/dialect/literal/parser.c" +#line 524 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -533,7 +534,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1090 "src/libre/parser.act" static int @@ -615,7 +616,6 @@ ZL0:; ADVANCE_LEXER; -#define BUILD_FOR_FUZZER 0 #if BUILD_FOR_FUZZER /* these errors currently are not handled properly */ if (act_state->lex_tok == TOK_ERROR) { diff --git a/src/libre/dialect/literal/parser.h b/src/libre/dialect/literal/parser.h index a4ab8031e..55fff6039 100644 --- a/src/libre/dialect/literal/parser.h +++ b/src/libre/dialect/literal/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 310 "src/libre/parser.act" +#line 300 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__literal(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1249 "src/libre/parser.act" +#line 1247 "src/libre/parser.act" #line 35 "src/libre/dialect/literal/parser.h" diff --git a/src/libre/dialect/native/parser.c b/src/libre/dialect/native/parser.c index 1149b0e7f..1ddf136f3 100644 --- a/src/libre/dialect/native/parser.c +++ b/src/libre/dialect/native/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 149 "src/libre/parser.act" +#line 22 "src/libre/parser.act" #include @@ -22,6 +22,7 @@ #include #include + #include #include "libre/class.h" #include "libre/class_lookup.h" @@ -287,7 +288,7 @@ return s; } -#line 291 "src/libre/dialect/native/parser.c" +#line 292 "src/libre/dialect/native/parser.c" #ifndef ERROR_TERMINAL @@ -344,7 +345,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CHAR */ { -#line 592 "src/libre/parser.act" +#line 589 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -357,7 +358,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = lex_state->buf.a[0]; -#line 361 "src/libre/dialect/native/parser.c" +#line 362 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -367,7 +368,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 409 "src/libre/parser.act" +#line 405 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -393,7 +394,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 397 "src/libre/dialect/native/parser.c" +#line 398 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -403,7 +404,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 548 "src/libre/parser.act" +#line 541 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -451,7 +452,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 455 "src/libre/dialect/native/parser.c" +#line 456 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -461,7 +462,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 505 "src/libre/parser.act" +#line 498 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -504,7 +505,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 508 "src/libre/dialect/native/parser.c" +#line 509 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -522,7 +523,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(ZIc); -#line 526 "src/libre/dialect/native/parser.c" +#line 527 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -557,7 +558,7 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI218; (void) ZIend; -#line 561 "src/libre/dialect/native/parser.c" +#line 562 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; @@ -568,12 +569,12 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 mark(&act_state->countstart, &(*ZI266)); mark(&act_state->countend, &(ZIend)); -#line 572 "src/libre/dialect/native/parser.c" +#line 573 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 855 "src/libre/parser.act" +#line 853 "src/libre/parser.act" struct ast_pos ast_start, ast_end; @@ -593,7 +594,7 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), &ast_start, (*ZIm), &ast_end); -#line 597 "src/libre/dialect/native/parser.c" +#line 598 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -609,7 +610,7 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 649 "src/libre/parser.act" +#line 641 "src/libre/parser.act" unsigned long u; char *e; @@ -629,7 +630,7 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 633 "src/libre/dialect/native/parser.c" +#line 634 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -649,7 +650,7 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI221; (void) ZIend; -#line 653 "src/libre/dialect/native/parser.c" +#line 654 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -664,12 +665,12 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 mark(&act_state->countstart, &(*ZI266)); mark(&act_state->countend, &(ZIend)); -#line 668 "src/libre/dialect/native/parser.c" +#line 669 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 855 "src/libre/parser.act" +#line 853 "src/libre/parser.act" struct ast_pos ast_start, ast_end; @@ -689,7 +690,7 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); -#line 693 "src/libre/dialect/native/parser.c" +#line 694 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -730,14 +731,14 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; goto ZL1; } -#line 734 "src/libre/dialect/native/parser.c" +#line 735 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: expr::character-class::list-of-class-terms */ goto ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms; /* END OF INLINE: expr::character-class::list-of-class-terms */ } - /*UNREACHED*/ + /* UNREACHED */ case (ERROR_TERMINAL): return; default: @@ -774,7 +775,7 @@ p_155(flags flags, lex_state lex_state, act_state act_state, err err) (void) ZI157; (void) ZI158; -#line 778 "src/libre/dialect/native/parser.c" +#line 779 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -788,14 +789,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 735 "src/libre/parser.act" +#line 732 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; } goto ZL2; -#line 799 "src/libre/dialect/native/parser.c" +#line 800 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -829,7 +830,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL1; } -#line 833 "src/libre/dialect/native/parser.c" +#line 834 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 225 */ @@ -843,7 +844,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL2_expr_C_Clist_Hof_Hpieces; /* END OF INLINE: expr::list-of-pieces */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -877,7 +878,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CHAR */ { -#line 592 "src/libre/parser.act" +#line 589 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -890,7 +891,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[0]; -#line 894 "src/libre/dialect/native/parser.c" +#line 895 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -903,7 +904,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: ESC */ { -#line 409 "src/libre/parser.act" +#line 405 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -929,7 +930,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI112; (void) ZI113; -#line 933 "src/libre/dialect/native/parser.c" +#line 934 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -942,7 +943,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: HEX */ { -#line 548 "src/libre/parser.act" +#line 541 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -990,7 +991,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 994 "src/libre/dialect/native/parser.c" +#line 995 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1003,7 +1004,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: OCT */ { -#line 505 "src/libre/parser.act" +#line 498 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1046,7 +1047,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1050 "src/libre/dialect/native/parser.c" +#line 1051 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1066,7 +1067,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err goto ZL1; } -#line 1070 "src/libre/dialect/native/parser.c" +#line 1071 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1092,7 +1093,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 592 "src/libre/parser.act" +#line 589 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1105,7 +1106,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI262 = lex_state->buf.a[0]; -#line 1109 "src/libre/dialect/native/parser.c" +#line 1110 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1124,7 +1125,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: ESC */ { -#line 409 "src/libre/parser.act" +#line 405 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1150,7 +1151,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI251; (void) ZI252; -#line 1154 "src/libre/dialect/native/parser.c" +#line 1155 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1169,7 +1170,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: HEX */ { -#line 548 "src/libre/parser.act" +#line 541 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1217,7 +1218,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI258 = (char) (unsigned char) u; -#line 1221 "src/libre/dialect/native/parser.c" +#line 1222 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1250,7 +1251,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI247; (void) ZI248; -#line 1254 "src/libre/dialect/native/parser.c" +#line 1255 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; @@ -1269,7 +1270,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: OCT */ { -#line 505 "src/libre/parser.act" +#line 498 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1312,7 +1313,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI254 = (char) (unsigned char) u; -#line 1316 "src/libre/dialect/native/parser.c" +#line 1317 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1367,7 +1368,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1371 "src/libre/dialect/native/parser.c" +#line 1372 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1382,7 +1383,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1386 "src/libre/dialect/native/parser.c" +#line 1387 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1426,7 +1427,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI168; -#line 1430 "src/libre/dialect/native/parser.c" +#line 1431 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; @@ -1439,7 +1440,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1443 "src/libre/dialect/native/parser.c" +#line 1444 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -1467,7 +1468,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI187; -#line 1471 "src/libre/dialect/native/parser.c" +#line 1472 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; @@ -1480,7 +1481,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1484 "src/libre/dialect/native/parser.c" +#line 1485 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -1490,7 +1491,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZIcbrak) = ']'; -#line 1494 "src/libre/dialect/native/parser.c" +#line 1495 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_196 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1506,7 +1507,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1510 "src/libre/dialect/native/parser.c" +#line 1511 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1530,7 +1531,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI179; -#line 1534 "src/libre/dialect/native/parser.c" +#line 1535 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; @@ -1543,13 +1544,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1547 "src/libre/dialect/native/parser.c" +#line 1548 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1028 "src/libre/parser.act" +#line 1000 "src/libre/parser.act" struct ast_expr *any; @@ -1587,7 +1588,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1591 "src/libre/dialect/native/parser.c" +#line 1592 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ p_181 (flags, lex_state, act_state, err, &ZItmp); @@ -1614,7 +1615,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI194; -#line 1618 "src/libre/dialect/native/parser.c" +#line 1619 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; @@ -1627,13 +1628,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1631 "src/libre/dialect/native/parser.c" +#line 1632 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1028 "src/libre/parser.act" +#line 1000 "src/libre/parser.act" struct ast_expr *any; @@ -1671,7 +1672,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1675 "src/libre/dialect/native/parser.c" +#line 1676 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ @@ -1680,7 +1681,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZIcbrak) = ']'; -#line 1684 "src/libre/dialect/native/parser.c" +#line 1685 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_196 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1696,7 +1697,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1700 "src/libre/dialect/native/parser.c" +#line 1701 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1731,7 +1732,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI202; (void) ZIend; -#line 1735 "src/libre/dialect/native/parser.c" +#line 1736 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; @@ -1742,7 +1743,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1746 "src/libre/dialect/native/parser.c" +#line 1747 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1765,7 +1766,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI204; (void) ZIend; -#line 1769 "src/libre/dialect/native/parser.c" +#line 1770 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; @@ -1778,7 +1779,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL4; } -#line 1782 "src/libre/dialect/native/parser.c" +#line 1783 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ @@ -1789,7 +1790,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL4; } -#line 1793 "src/libre/dialect/native/parser.c" +#line 1794 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ @@ -1799,7 +1800,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1803 "src/libre/dialect/native/parser.c" +#line 1804 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1812,14 +1813,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 742 "src/libre/parser.act" +#line 739 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 1823 "src/libre/dialect/native/parser.c" +#line 1824 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-closegroup */ ZIend = ZIstart; @@ -1829,7 +1830,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF INLINE: 200 */ /* BEGINNING OF ACTION: mark-expr */ { -#line 798 "src/libre/parser.act" +#line 796 "src/libre/parser.act" struct ast_pos ast_start, ast_end; @@ -1844,7 +1845,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZItmp)->u.class.end = ast_end; */ -#line 1848 "src/libre/dialect/native/parser.c" +#line 1849 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-expr */ } @@ -1886,7 +1887,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st (void) ZI150; (void) ZIend; -#line 1890 "src/libre/dialect/native/parser.c" +#line 1891 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1897,7 +1898,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(ZIc); -#line 1901 "src/libre/dialect/native/parser.c" +#line 1902 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -1952,7 +1953,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZIrstart; (void) ZI182; -#line 1956 "src/libre/dialect/native/parser.c" +#line 1957 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1970,7 +1971,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1974 "src/libre/dialect/native/parser.c" +#line 1975 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1991,7 +1992,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(ZIc); -#line 1995 "src/libre/dialect/native/parser.c" +#line 1996 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -2006,7 +2007,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI185; (void) ZI186; -#line 2010 "src/libre/dialect/native/parser.c" +#line 2011 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2017,7 +2018,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1041 "src/libre/parser.act" +#line 1038 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -2050,7 +2051,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2054 "src/libre/dialect/native/parser.c" +#line 2055 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2066,7 +2067,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2070 "src/libre/dialect/native/parser.c" +#line 2071 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -2116,7 +2117,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2120 "src/libre/dialect/native/parser.c" +#line 2121 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2147,7 +2148,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex goto ZL1; } -#line 2151 "src/libre/dialect/native/parser.c" +#line 2152 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2168,7 +2169,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex goto ZL1; } -#line 2172 "src/libre/dialect/native/parser.c" +#line 2173 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2181,14 +2182,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 728 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 2192 "src/libre/dialect/native/parser.c" +#line 2193 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -2200,7 +2201,7 @@ ZL1:; goto ZL2; } -#line 2204 "src/libre/dialect/native/parser.c" +#line 2205 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2248,14 +2249,14 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 770 "src/libre/parser.act" +#line 767 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 2259 "src/libre/dialect/native/parser.c" +#line 2260 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -2288,7 +2289,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 2292 "src/libre/dialect/native/parser.c" +#line 2293 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2310,7 +2311,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(*ZIcbrak); -#line 2314 "src/libre/dialect/native/parser.c" +#line 2315 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -2325,7 +2326,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (void) ZI198; (void) ZI199; -#line 2329 "src/libre/dialect/native/parser.c" +#line 2330 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2341,12 +2342,12 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(*ZIcbrak); -#line 2345 "src/libre/dialect/native/parser.c" +#line 2346 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1041 "src/libre/parser.act" +#line 1038 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -2379,7 +2380,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 2383 "src/libre/dialect/native/parser.c" +#line 2384 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2466,7 +2467,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL1; } -#line 2470 "src/libre/dialect/native/parser.c" +#line 2471 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 231 */ @@ -2479,7 +2480,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL2_expr_C_Clist_Hof_Halts; /* END OF INLINE: expr::list-of-alts */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -2491,14 +2492,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 728 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 2502 "src/libre/dialect/native/parser.c" +#line 2503 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -2531,7 +2532,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI266; (void) ZI267; -#line 2535 "src/libre/dialect/native/parser.c" +#line 2536 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -2539,7 +2540,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 649 "src/libre/parser.act" +#line 641 "src/libre/parser.act" unsigned long u; char *e; @@ -2559,7 +2560,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 2563 "src/libre/dialect/native/parser.c" +#line 2564 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -2583,7 +2584,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, NULL, 1, NULL); -#line 2587 "src/libre/dialect/native/parser.c" +#line 2588 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -2597,7 +2598,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 2601 "src/libre/dialect/native/parser.c" +#line 2602 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -2611,7 +2612,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 2615 "src/libre/dialect/native/parser.c" +#line 2616 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -2624,7 +2625,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, NULL, 1, NULL); -#line 2628 "src/libre/dialect/native/parser.c" +#line 2629 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2637,14 +2638,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 714 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 2648 "src/libre/dialect/native/parser.c" +#line 2649 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ @@ -2653,7 +2654,7 @@ ZL1:; (ZIc) = ast_make_count(1, NULL, 1, NULL); -#line 2657 "src/libre/dialect/native/parser.c" +#line 2658 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2678,12 +2679,12 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 811 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 2687 "src/libre/dialect/native/parser.c" +#line 2688 "src/libre/dialect/native/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ @@ -2695,7 +2696,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 2699 "src/libre/dialect/native/parser.c" +#line 2700 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2712,7 +2713,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 2716 "src/libre/dialect/native/parser.c" +#line 2717 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -2729,7 +2730,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIid) = act_state->group_id++; -#line 2733 "src/libre/dialect/native/parser.c" +#line 2734 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -2746,7 +2747,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 2750 "src/libre/dialect/native/parser.c" +#line 2751 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -2770,7 +2771,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 2774 "src/libre/dialect/native/parser.c" +#line 2775 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -2803,14 +2804,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 721 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 2814 "src/libre/dialect/native/parser.c" +#line 2815 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -2822,7 +2823,7 @@ ZL1:; goto ZL2; } -#line 2826 "src/libre/dialect/native/parser.c" +#line 2827 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2851,7 +2852,7 @@ p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 2855 "src/libre/dialect/native/parser.c" +#line 2856 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2869,7 +2870,7 @@ p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI246); -#line 2873 "src/libre/dialect/native/parser.c" +#line 2874 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ p_155 (flags, lex_state, act_state, err); @@ -2885,12 +2886,12 @@ p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla mark(&act_state->rangestart, &(*ZI247)); mark(&act_state->rangeend, &(ZIend)); -#line 2889 "src/libre/dialect/native/parser.c" +#line 2890 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1041 "src/libre/parser.act" +#line 1038 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -2923,7 +2924,7 @@ p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 2927 "src/libre/dialect/native/parser.c" +#line 2928 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2957,7 +2958,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 2961 "src/libre/dialect/native/parser.c" +#line 2962 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2991,7 +2992,7 @@ p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 2995 "src/libre/dialect/native/parser.c" +#line 2996 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3009,7 +3010,7 @@ p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(*ZI250); -#line 3013 "src/libre/dialect/native/parser.c" +#line 3014 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_155 (flags, lex_state, act_state, err); @@ -3025,12 +3026,12 @@ p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI mark(&act_state->rangestart, &(*ZI251)); mark(&act_state->rangeend, &(ZIend)); -#line 3029 "src/libre/dialect/native/parser.c" +#line 3030 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1041 "src/libre/parser.act" +#line 1038 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -3063,7 +3064,7 @@ p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 3067 "src/libre/dialect/native/parser.c" +#line 3068 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3081,7 +3082,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1090 "src/libre/parser.act" static int @@ -3163,7 +3164,6 @@ ZL0:; ADVANCE_LEXER; -#define BUILD_FOR_FUZZER 0 #if BUILD_FOR_FUZZER /* these errors currently are not handled properly */ if (act_state->lex_tok == TOK_ERROR) { diff --git a/src/libre/dialect/native/parser.h b/src/libre/dialect/native/parser.h index 92f89ad55..959bf3acc 100644 --- a/src/libre/dialect/native/parser.h +++ b/src/libre/dialect/native/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 310 "src/libre/parser.act" +#line 300 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__native(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1249 "src/libre/parser.act" +#line 1247 "src/libre/parser.act" #line 35 "src/libre/dialect/native/parser.h" diff --git a/src/libre/dialect/pcre/parser.c b/src/libre/dialect/pcre/parser.c index 1574eda04..bccd5bbba 100644 --- a/src/libre/dialect/pcre/parser.c +++ b/src/libre/dialect/pcre/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 149 "src/libre/parser.act" +#line 22 "src/libre/parser.act" #include @@ -22,6 +22,7 @@ #include #include + #include #include "libre/class.h" #include "libre/class_lookup.h" @@ -287,7 +288,7 @@ return s; } -#line 291 "src/libre/dialect/pcre/parser.c" +#line 292 "src/libre/dialect/pcre/parser.c" #ifndef ERROR_TERMINAL @@ -347,7 +348,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ZIc = RE_EXTENDED; -#line 351 "src/libre/dialect/pcre/parser.c" +#line 352 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_EXTENDED */ ADVANCE_LEXER; @@ -357,7 +358,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta (ZIo) = (ZIi) | (ZIc); -#line 361 "src/libre/dialect/pcre/parser.c" +#line 362 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -378,7 +379,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ZIc = RE_ICASE; -#line 382 "src/libre/dialect/pcre/parser.c" +#line 383 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_INSENSITIVE */ ADVANCE_LEXER; @@ -388,7 +389,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta (ZIo) = (ZIi) | (ZIc); -#line 392 "src/libre/dialect/pcre/parser.c" +#line 393 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -403,7 +404,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ZIc = RE_SINGLE; -#line 407 "src/libre/dialect/pcre/parser.c" +#line 408 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_SINGLE */ ADVANCE_LEXER; @@ -413,7 +414,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta (ZIo) = (ZIi) | (ZIc); -#line 417 "src/libre/dialect/pcre/parser.c" +#line 418 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -424,14 +425,14 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ZIo = ZIi; /* BEGINNING OF ACTION: err-unknown-flag */ { -#line 756 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EFLAG; } goto ZL1; -#line 435 "src/libre/dialect/pcre/parser.c" +#line 436 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unknown-flag */ } @@ -469,7 +470,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CHAR */ { -#line 592 "src/libre/parser.act" +#line 589 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -482,7 +483,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = lex_state->buf.a[0]; -#line 486 "src/libre/dialect/pcre/parser.c" +#line 487 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -492,7 +493,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CONTROL */ { -#line 466 "src/libre/parser.act" +#line 461 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -529,20 +530,20 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 533 "src/libre/dialect/pcre/parser.c" +#line 534 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 777 "src/libre/parser.act" +#line 774 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 546 "src/libre/dialect/pcre/parser.c" +#line 547 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -551,7 +552,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 409 "src/libre/parser.act" +#line 405 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -577,7 +578,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 581 "src/libre/dialect/pcre/parser.c" +#line 582 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -587,7 +588,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 548 "src/libre/parser.act" +#line 541 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -635,7 +636,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 639 "src/libre/dialect/pcre/parser.c" +#line 640 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -645,7 +646,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 505 "src/libre/parser.act" +#line 498 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -688,7 +689,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 692 "src/libre/dialect/pcre/parser.c" +#line 693 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -698,7 +699,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 447 "src/libre/parser.act" +#line 445 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -714,20 +715,20 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 718 "src/libre/dialect/pcre/parser.c" +#line 719 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 777 "src/libre/parser.act" +#line 774 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 731 "src/libre/dialect/pcre/parser.c" +#line 732 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -744,7 +745,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(ZIc); -#line 748 "src/libre/dialect/pcre/parser.c" +#line 749 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -781,14 +782,14 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; goto ZL1; } -#line 785 "src/libre/dialect/pcre/parser.c" +#line 786 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: expr::character-class::list-of-class-terms */ goto ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms; /* END OF INLINE: expr::character-class::list-of-class-terms */ } - /*UNREACHED*/ + /* UNREACHED */ case (ERROR_TERMINAL): return; default: @@ -841,7 +842,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL1; } -#line 845 "src/libre/dialect/pcre/parser.c" +#line 846 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ } @@ -864,7 +865,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL2_expr_C_Clist_Hof_Hpieces; /* END OF INLINE: expr::list-of-pieces */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -894,7 +895,7 @@ p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 898 "src/libre/dialect/pcre/parser.c" +#line 899 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -912,7 +913,7 @@ p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI293); -#line 916 "src/libre/dialect/pcre/parser.c" +#line 917 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ p_169 (flags, lex_state, act_state, err); @@ -928,12 +929,12 @@ p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla mark(&act_state->rangestart, &(*ZI294)); mark(&act_state->rangeend, &(ZIend)); -#line 932 "src/libre/dialect/pcre/parser.c" +#line 933 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1041 "src/libre/parser.act" +#line 1038 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -966,7 +967,7 @@ p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 970 "src/libre/dialect/pcre/parser.c" +#line 971 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -1007,7 +1008,7 @@ p_169(flags flags, lex_state lex_state, act_state act_state, err err) (void) ZI171; (void) ZI172; -#line 1011 "src/libre/dialect/pcre/parser.c" +#line 1012 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -1021,14 +1022,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 735 "src/libre/parser.act" +#line 732 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; } goto ZL2; -#line 1032 "src/libre/dialect/pcre/parser.c" +#line 1033 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -1060,7 +1061,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CHAR */ { -#line 592 "src/libre/parser.act" +#line 589 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1073,7 +1074,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[0]; -#line 1077 "src/libre/dialect/pcre/parser.c" +#line 1078 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1086,7 +1087,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CONTROL */ { -#line 466 "src/libre/parser.act" +#line 461 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1123,7 +1124,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI124; (void) ZI125; -#line 1127 "src/libre/dialect/pcre/parser.c" +#line 1128 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; @@ -1136,7 +1137,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: ESC */ { -#line 409 "src/libre/parser.act" +#line 405 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1162,7 +1163,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI114; (void) ZI115; -#line 1166 "src/libre/dialect/pcre/parser.c" +#line 1167 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1175,7 +1176,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: HEX */ { -#line 548 "src/libre/parser.act" +#line 541 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1223,7 +1224,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1227 "src/libre/dialect/pcre/parser.c" +#line 1228 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1236,7 +1237,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: NOESC */ { -#line 435 "src/libre/parser.act" +#line 431 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1250,7 +1251,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI116; (void) ZI117; -#line 1254 "src/libre/dialect/pcre/parser.c" +#line 1255 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; @@ -1263,7 +1264,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: OCT */ { -#line 505 "src/libre/parser.act" +#line 498 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1306,7 +1307,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1310 "src/libre/dialect/pcre/parser.c" +#line 1311 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1319,7 +1320,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 447 "src/libre/parser.act" +#line 445 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1335,20 +1336,20 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI126; (void) ZI127; -#line 1339 "src/libre/dialect/pcre/parser.c" +#line 1340 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 777 "src/libre/parser.act" +#line 774 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1352 "src/libre/dialect/pcre/parser.c" +#line 1353 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -1367,7 +1368,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err goto ZL1; } -#line 1371 "src/libre/dialect/pcre/parser.c" +#line 1372 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1393,7 +1394,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 592 "src/libre/parser.act" +#line 589 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1406,7 +1407,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI309 = lex_state->buf.a[0]; -#line 1410 "src/libre/dialect/pcre/parser.c" +#line 1411 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1425,7 +1426,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CONTROL */ { -#line 466 "src/libre/parser.act" +#line 461 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1462,20 +1463,20 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI314; (void) ZI315; -#line 1466 "src/libre/dialect/pcre/parser.c" +#line 1467 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 777 "src/libre/parser.act" +#line 774 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1479 "src/libre/dialect/pcre/parser.c" +#line 1480 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ p_320 (flags, lex_state, act_state, err, &ZI313, &ZI314, &ZInode); @@ -1493,7 +1494,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: ESC */ { -#line 409 "src/libre/parser.act" +#line 405 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1519,7 +1520,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI298; (void) ZI299; -#line 1523 "src/libre/dialect/pcre/parser.c" +#line 1524 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1538,7 +1539,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: HEX */ { -#line 548 "src/libre/parser.act" +#line 541 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1586,7 +1587,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI305 = (char) (unsigned char) u; -#line 1590 "src/libre/dialect/pcre/parser.c" +#line 1591 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1619,7 +1620,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI294; (void) ZI295; -#line 1623 "src/libre/dialect/pcre/parser.c" +#line 1624 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; @@ -1638,7 +1639,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NOESC */ { -#line 435 "src/libre/parser.act" +#line 431 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1652,7 +1653,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI135; (void) ZI136; -#line 1656 "src/libre/dialect/pcre/parser.c" +#line 1657 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; @@ -1665,7 +1666,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } -#line 1669 "src/libre/dialect/pcre/parser.c" +#line 1670 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1678,7 +1679,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: OCT */ { -#line 505 "src/libre/parser.act" +#line 498 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1721,7 +1722,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI301 = (char) (unsigned char) u; -#line 1725 "src/libre/dialect/pcre/parser.c" +#line 1726 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1740,7 +1741,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 447 "src/libre/parser.act" +#line 445 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1756,20 +1757,20 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI318; (void) ZI319; -#line 1760 "src/libre/dialect/pcre/parser.c" +#line 1761 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 777 "src/libre/parser.act" +#line 774 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1773 "src/libre/dialect/pcre/parser.c" +#line 1774 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ p_320 (flags, lex_state, act_state, err, &ZI317, &ZI318, &ZInode); @@ -1808,14 +1809,14 @@ p_expr_C_Ccomment(flags flags, lex_state lex_state, act_state act_state, err err ADVANCE_LEXER; /* BEGINNING OF ACTION: err-invalid-comment */ { -#line 700 "src/libre/parser.act" +#line 697 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EBADCOMMENT; } goto ZL1; -#line 1819 "src/libre/dialect/pcre/parser.c" +#line 1820 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-invalid-comment */ } @@ -1856,7 +1857,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1860 "src/libre/dialect/pcre/parser.c" +#line 1861 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1871,7 +1872,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1875 "src/libre/dialect/pcre/parser.c" +#line 1876 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1915,7 +1916,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI182; -#line 1919 "src/libre/dialect/pcre/parser.c" +#line 1920 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; @@ -1928,7 +1929,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1932 "src/libre/dialect/pcre/parser.c" +#line 1933 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -1956,7 +1957,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI201; -#line 1960 "src/libre/dialect/pcre/parser.c" +#line 1961 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; @@ -1969,7 +1970,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1973 "src/libre/dialect/pcre/parser.c" +#line 1974 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -1979,7 +1980,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZIcbrak) = ']'; -#line 1983 "src/libre/dialect/pcre/parser.c" +#line 1984 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_210 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1995,7 +1996,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1999 "src/libre/dialect/pcre/parser.c" +#line 2000 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2019,7 +2020,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI193; -#line 2023 "src/libre/dialect/pcre/parser.c" +#line 2024 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; @@ -2032,13 +2033,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2036 "src/libre/dialect/pcre/parser.c" +#line 2037 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1028 "src/libre/parser.act" +#line 1000 "src/libre/parser.act" struct ast_expr *any; @@ -2076,7 +2077,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2080 "src/libre/dialect/pcre/parser.c" +#line 2081 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ p_195 (flags, lex_state, act_state, err, &ZItmp); @@ -2103,7 +2104,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI208; -#line 2107 "src/libre/dialect/pcre/parser.c" +#line 2108 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; @@ -2116,13 +2117,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2120 "src/libre/dialect/pcre/parser.c" +#line 2121 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1028 "src/libre/parser.act" +#line 1000 "src/libre/parser.act" struct ast_expr *any; @@ -2160,7 +2161,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2164 "src/libre/dialect/pcre/parser.c" +#line 2165 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ @@ -2169,7 +2170,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZIcbrak) = ']'; -#line 2173 "src/libre/dialect/pcre/parser.c" +#line 2174 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_210 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -2185,7 +2186,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2189 "src/libre/dialect/pcre/parser.c" +#line 2190 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2220,7 +2221,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI216; (void) ZIend; -#line 2224 "src/libre/dialect/pcre/parser.c" +#line 2225 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; @@ -2231,7 +2232,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2235 "src/libre/dialect/pcre/parser.c" +#line 2236 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2254,7 +2255,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI218; (void) ZIend; -#line 2258 "src/libre/dialect/pcre/parser.c" +#line 2259 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; @@ -2267,7 +2268,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL4; } -#line 2271 "src/libre/dialect/pcre/parser.c" +#line 2272 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ @@ -2278,7 +2279,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL4; } -#line 2282 "src/libre/dialect/pcre/parser.c" +#line 2283 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ @@ -2288,7 +2289,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2292 "src/libre/dialect/pcre/parser.c" +#line 2293 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2301,14 +2302,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 742 "src/libre/parser.act" +#line 739 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 2312 "src/libre/dialect/pcre/parser.c" +#line 2313 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closegroup */ ZIend = ZIstart; @@ -2318,7 +2319,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF INLINE: 214 */ /* BEGINNING OF ACTION: mark-expr */ { -#line 798 "src/libre/parser.act" +#line 796 "src/libre/parser.act" struct ast_pos ast_start, ast_end; @@ -2333,7 +2334,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZItmp)->u.class.end = ast_end; */ -#line 2337 "src/libre/dialect/pcre/parser.c" +#line 2338 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-expr */ } @@ -2375,7 +2376,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st (void) ZI164; (void) ZIend; -#line 2379 "src/libre/dialect/pcre/parser.c" +#line 2380 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2386,7 +2387,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(ZIc); -#line 2390 "src/libre/dialect/pcre/parser.c" +#line 2391 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -2452,7 +2453,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, (ZIc) = ast_make_count(1, NULL, 1, NULL); -#line 2456 "src/libre/dialect/pcre/parser.c" +#line 2457 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ /* BEGINNING OF ACTION: ast-make-piece */ @@ -2471,7 +2472,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2475 "src/libre/dialect/pcre/parser.c" +#line 2476 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2508,7 +2509,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 2512 "src/libre/dialect/pcre/parser.c" +#line 2513 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2526,7 +2527,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(*ZI317); -#line 2530 "src/libre/dialect/pcre/parser.c" +#line 2531 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_169 (flags, lex_state, act_state, err); @@ -2542,12 +2543,12 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI mark(&act_state->rangestart, &(*ZI318)); mark(&act_state->rangeend, &(ZIend)); -#line 2546 "src/libre/dialect/pcre/parser.c" +#line 2547 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1041 "src/libre/parser.act" +#line 1038 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -2580,7 +2581,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 2584 "src/libre/dialect/pcre/parser.c" +#line 2585 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2614,7 +2615,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex goto ZL1; } -#line 2618 "src/libre/dialect/pcre/parser.c" +#line 2619 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2628,14 +2629,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 728 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 2639 "src/libre/dialect/pcre/parser.c" +#line 2640 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -2647,7 +2648,7 @@ ZL1:; goto ZL2; } -#line 2651 "src/libre/dialect/pcre/parser.c" +#line 2652 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2682,7 +2683,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZIrstart; (void) ZI196; -#line 2686 "src/libre/dialect/pcre/parser.c" +#line 2687 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2700,7 +2701,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2704 "src/libre/dialect/pcre/parser.c" +#line 2705 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2721,7 +2722,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(ZIc); -#line 2725 "src/libre/dialect/pcre/parser.c" +#line 2726 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -2736,7 +2737,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI199; (void) ZI200; -#line 2740 "src/libre/dialect/pcre/parser.c" +#line 2741 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2747,7 +2748,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1041 "src/libre/parser.act" +#line 1038 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -2780,7 +2781,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2784 "src/libre/dialect/pcre/parser.c" +#line 2785 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2796,7 +2797,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2800 "src/libre/dialect/pcre/parser.c" +#line 2801 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -2833,7 +2834,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI258; (void) ZIend; -#line 2837 "src/libre/dialect/pcre/parser.c" +#line 2838 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; @@ -2844,12 +2845,12 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 mark(&act_state->countstart, &(*ZI321)); mark(&act_state->countend, &(ZIend)); -#line 2848 "src/libre/dialect/pcre/parser.c" +#line 2849 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 855 "src/libre/parser.act" +#line 853 "src/libre/parser.act" struct ast_pos ast_start, ast_end; @@ -2869,7 +2870,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), &ast_start, (*ZIm), &ast_end); -#line 2873 "src/libre/dialect/pcre/parser.c" +#line 2874 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2919,7 +2920,7 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI263; (void) ZIend; -#line 2923 "src/libre/dialect/pcre/parser.c" +#line 2924 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; @@ -2930,7 +2931,7 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 mark(&act_state->countstart, &(*ZI321)); mark(&act_state->countend, &(ZIend)); -#line 2934 "src/libre/dialect/pcre/parser.c" +#line 2935 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-unbounded */ @@ -2939,12 +2940,12 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIn) = AST_COUNT_UNBOUNDED; -#line 2943 "src/libre/dialect/pcre/parser.c" +#line 2944 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-unbounded */ /* BEGINNING OF ACTION: count-range */ { -#line 855 "src/libre/parser.act" +#line 853 "src/libre/parser.act" struct ast_pos ast_start, ast_end; @@ -2964,7 +2965,7 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); -#line 2968 "src/libre/dialect/pcre/parser.c" +#line 2969 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2977,7 +2978,7 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: COUNT */ { -#line 649 "src/libre/parser.act" +#line 641 "src/libre/parser.act" unsigned long u; char *e; @@ -2997,7 +2998,7 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ZIn = (unsigned int) u; -#line 3001 "src/libre/dialect/pcre/parser.c" +#line 3002 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ ADVANCE_LEXER; @@ -3013,7 +3014,7 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI261; (void) ZIend; -#line 3017 "src/libre/dialect/pcre/parser.c" +#line 3018 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -3028,12 +3029,12 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 mark(&act_state->countstart, &(*ZI321)); mark(&act_state->countend, &(ZIend)); -#line 3032 "src/libre/dialect/pcre/parser.c" +#line 3033 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 855 "src/libre/parser.act" +#line 853 "src/libre/parser.act" struct ast_pos ast_start, ast_end; @@ -3053,7 +3054,7 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); -#line 3057 "src/libre/dialect/pcre/parser.c" +#line 3058 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -3098,7 +3099,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, (ZIempty__pos) = RE_FLAGS_NONE; -#line 3102 "src/libre/dialect/pcre/parser.c" +#line 3103 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF ACTION: re-flag-none */ @@ -3107,7 +3108,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, (ZIempty__neg) = RE_FLAGS_NONE; -#line 3111 "src/libre/dialect/pcre/parser.c" +#line 3112 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF INLINE: 233 */ @@ -3160,7 +3161,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 964 "src/libre/parser.act" +#line 960 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3169,7 +3170,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3173 "src/libre/dialect/pcre/parser.c" +#line 3174 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -3181,7 +3182,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL5; } -#line 3185 "src/libre/dialect/pcre/parser.c" +#line 3186 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3198,12 +3199,12 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, (ZIflags) = *flags; -#line 3202 "src/libre/dialect/pcre/parser.c" +#line 3203 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 964 "src/libre/parser.act" +#line 960 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3212,7 +3213,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3216 "src/libre/dialect/pcre/parser.c" +#line 3217 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3226,7 +3227,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags = (ZIflags); -#line 3230 "src/libre/dialect/pcre/parser.c" +#line 3231 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ switch (CURRENT_TERMINAL) { @@ -3247,14 +3248,14 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, { /* BEGINNING OF ACTION: err-expected-closeflags */ { -#line 763 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEFLAGS; } goto ZL1; -#line 3258 "src/libre/dialect/pcre/parser.c" +#line 3259 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closeflags */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -3266,7 +3267,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 3270 "src/libre/dialect/pcre/parser.c" +#line 3271 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3314,7 +3315,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state goto ZL1; } -#line 3318 "src/libre/dialect/pcre/parser.c" +#line 3319 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ /* BEGINNING OF INLINE: 266 */ @@ -3325,14 +3326,14 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 777 "src/libre/parser.act" +#line 774 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 3336 "src/libre/dialect/pcre/parser.c" +#line 3337 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3342,14 +3343,14 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 777 "src/libre/parser.act" +#line 774 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 3353 "src/libre/dialect/pcre/parser.c" +#line 3354 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3448,7 +3449,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t (void) ZIstart; (void) ZIend; -#line 3452 "src/libre/dialect/pcre/parser.c" +#line 3453 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -3465,7 +3466,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t goto ZL1; } -#line 3469 "src/libre/dialect/pcre/parser.c" +#line 3470 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3496,7 +3497,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 3500 "src/libre/dialect/pcre/parser.c" +#line 3501 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3518,7 +3519,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(*ZIcbrak); -#line 3522 "src/libre/dialect/pcre/parser.c" +#line 3523 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -3533,7 +3534,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (void) ZI212; (void) ZI213; -#line 3537 "src/libre/dialect/pcre/parser.c" +#line 3538 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -3549,12 +3550,12 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(*ZIcbrak); -#line 3553 "src/libre/dialect/pcre/parser.c" +#line 3554 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1041 "src/libre/parser.act" +#line 1038 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -3587,7 +3588,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 3591 "src/libre/dialect/pcre/parser.c" +#line 3592 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3626,7 +3627,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL1; } -#line 3630 "src/libre/dialect/pcre/parser.c" +#line 3631 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 280 */ @@ -3639,7 +3640,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL2_expr_C_Clist_Hof_Halts; /* END OF INLINE: expr::list-of-alts */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -3651,14 +3652,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 728 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 3662 "src/libre/dialect/pcre/parser.c" +#line 3663 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -3691,7 +3692,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI321; (void) ZI322; -#line 3695 "src/libre/dialect/pcre/parser.c" +#line 3696 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -3699,7 +3700,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 649 "src/libre/parser.act" +#line 641 "src/libre/parser.act" unsigned long u; char *e; @@ -3719,7 +3720,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 3723 "src/libre/dialect/pcre/parser.c" +#line 3724 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -3743,7 +3744,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, NULL, 1, NULL); -#line 3747 "src/libre/dialect/pcre/parser.c" +#line 3748 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -3757,7 +3758,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 3761 "src/libre/dialect/pcre/parser.c" +#line 3762 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -3771,7 +3772,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 3775 "src/libre/dialect/pcre/parser.c" +#line 3776 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -3786,14 +3787,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 714 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 3797 "src/libre/dialect/pcre/parser.c" +#line 3798 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ @@ -3802,7 +3803,7 @@ ZL1:; (ZIc) = ast_make_count(1, NULL, 1, NULL); -#line 3806 "src/libre/dialect/pcre/parser.c" +#line 3807 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ } @@ -3835,7 +3836,7 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast (ZIid) = act_state->group_id++; -#line 3839 "src/libre/dialect/pcre/parser.c" +#line 3840 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3852,7 +3853,7 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast goto ZL1; } -#line 3856 "src/libre/dialect/pcre/parser.c" +#line 3857 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ } @@ -3874,14 +3875,14 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 770 "src/libre/parser.act" +#line 767 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 3885 "src/libre/dialect/pcre/parser.c" +#line 3886 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -3910,12 +3911,12 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 811 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 3919 "src/libre/dialect/pcre/parser.c" +#line 3920 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ @@ -3927,7 +3928,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 3931 "src/libre/dialect/pcre/parser.c" +#line 3932 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3944,7 +3945,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 3948 "src/libre/dialect/pcre/parser.c" +#line 3949 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -3964,7 +3965,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIe)->u.anchor.is_end_nl = 1; } -#line 3968 "src/libre/dialect/pcre/parser.c" +#line 3969 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end-nl */ } @@ -3982,12 +3983,12 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-bsr */ { -#line 817 "src/libre/parser.act" +#line 816 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIclass__bsr) = &class_bsr; -#line 3991 "src/libre/dialect/pcre/parser.c" +#line 3992 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-bsr */ /* BEGINNING OF ACTION: ast-make-named */ @@ -3999,7 +4000,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4003 "src/libre/dialect/pcre/parser.c" +#line 4004 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: ast-make-concat */ @@ -4011,7 +4012,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4015 "src/libre/dialect/pcre/parser.c" +#line 4016 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ /* BEGINNING OF ACTION: make-literal-cr */ @@ -4020,7 +4021,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIcr) = '\r'; -#line 4024 "src/libre/dialect/pcre/parser.c" +#line 4025 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cr */ /* BEGINNING OF ACTION: ast-make-literal */ @@ -4032,7 +4033,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4036 "src/libre/dialect/pcre/parser.c" +#line 4037 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: make-literal-nl */ @@ -4041,7 +4042,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZInl) = '\n'; -#line 4045 "src/libre/dialect/pcre/parser.c" +#line 4046 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-nl */ /* BEGINNING OF ACTION: ast-make-literal */ @@ -4053,7 +4054,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4057 "src/libre/dialect/pcre/parser.c" +#line 4058 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-concat */ @@ -4064,7 +4065,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4068 "src/libre/dialect/pcre/parser.c" +#line 4069 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-add-concat */ @@ -4075,7 +4076,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4079 "src/libre/dialect/pcre/parser.c" +#line 4080 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-make-alt */ @@ -4087,7 +4088,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4091 "src/libre/dialect/pcre/parser.c" +#line 4092 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ @@ -4098,7 +4099,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4102 "src/libre/dialect/pcre/parser.c" +#line 4103 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-add-alt */ @@ -4109,7 +4110,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4113 "src/libre/dialect/pcre/parser.c" +#line 4114 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -4127,7 +4128,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIflags) = *flags; -#line 4131 "src/libre/dialect/pcre/parser.c" +#line 4132 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: make-group-id */ @@ -4136,7 +4137,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIid) = act_state->group_id++; -#line 4140 "src/libre/dialect/pcre/parser.c" +#line 4141 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -4150,7 +4151,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e *flags = (ZIflags); -#line 4154 "src/libre/dialect/pcre/parser.c" +#line 4155 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ /* BEGINNING OF ACTION: ast-make-group */ @@ -4162,7 +4163,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4166 "src/libre/dialect/pcre/parser.c" +#line 4167 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -4186,7 +4187,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4190 "src/libre/dialect/pcre/parser.c" +#line 4191 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -4238,14 +4239,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 721 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 4249 "src/libre/dialect/pcre/parser.c" +#line 4250 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -4257,7 +4258,7 @@ ZL1:; goto ZL2; } -#line 4261 "src/libre/dialect/pcre/parser.c" +#line 4262 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4290,7 +4291,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 4294 "src/libre/dialect/pcre/parser.c" +#line 4295 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -4311,7 +4312,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 4315 "src/libre/dialect/pcre/parser.c" +#line 4316 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4354,7 +4355,7 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t goto ZL1; } -#line 4358 "src/libre/dialect/pcre/parser.c" +#line 4359 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ @@ -4365,12 +4366,12 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t goto ZL1; } -#line 4369 "src/libre/dialect/pcre/parser.c" +#line 4370 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-expr */ { -#line 798 "src/libre/parser.act" +#line 796 "src/libre/parser.act" struct ast_pos ast_start, ast_end; @@ -4385,7 +4386,7 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t (ZInode)->u.class.end = ast_end; */ -#line 4389 "src/libre/dialect/pcre/parser.c" +#line 4390 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-expr */ } @@ -4399,7 +4400,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1090 "src/libre/parser.act" static int @@ -4481,7 +4482,6 @@ ZL0:; ADVANCE_LEXER; -#define BUILD_FOR_FUZZER 0 #if BUILD_FOR_FUZZER /* these errors currently are not handled properly */ if (act_state->lex_tok == TOK_ERROR) { diff --git a/src/libre/dialect/pcre/parser.h b/src/libre/dialect/pcre/parser.h index 73016415c..20b24942d 100644 --- a/src/libre/dialect/pcre/parser.h +++ b/src/libre/dialect/pcre/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 310 "src/libre/parser.act" +#line 300 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__pcre(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1249 "src/libre/parser.act" +#line 1247 "src/libre/parser.act" #line 35 "src/libre/dialect/pcre/parser.h" diff --git a/src/libre/dialect/sql/parser.c b/src/libre/dialect/sql/parser.c index abf84de29..993fcfc26 100644 --- a/src/libre/dialect/sql/parser.c +++ b/src/libre/dialect/sql/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 149 "src/libre/parser.act" +#line 22 "src/libre/parser.act" #include @@ -22,6 +22,7 @@ #include #include + #include #include "libre/class.h" #include "libre/class_lookup.h" @@ -287,7 +288,7 @@ return s; } -#line 291 "src/libre/dialect/sql/parser.c" +#line 292 "src/libre/dialect/sql/parser.c" #ifndef ERROR_TERMINAL @@ -335,7 +336,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ (void) ZI205; -#line 339 "src/libre/dialect/sql/parser.c" +#line 340 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; @@ -365,7 +366,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ (void) ZI115; (void) ZI116; -#line 369 "src/libre/dialect/sql/parser.c" +#line 370 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -378,7 +379,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ goto ZL1; } -#line 382 "src/libre/dialect/sql/parser.c" +#line 383 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ @@ -389,7 +390,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ goto ZL1; } -#line 393 "src/libre/dialect/sql/parser.c" +#line 394 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -441,14 +442,14 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 770 "src/libre/parser.act" +#line 767 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 452 "src/libre/dialect/sql/parser.c" +#line 453 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -490,7 +491,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; goto ZL4; } -#line 494 "src/libre/dialect/sql/parser.c" +#line 495 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -499,14 +500,14 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; { /* BEGINNING OF ACTION: err-expected-term */ { -#line 707 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXTERM; } goto ZL1; -#line 510 "src/libre/dialect/sql/parser.c" +#line 511 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-term */ } @@ -522,7 +523,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; goto ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms; /* END OF INLINE: expr::character-class::list-of-class-terms */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -566,7 +567,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL1; } -#line 570 "src/libre/dialect/sql/parser.c" +#line 571 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 184 */ @@ -579,7 +580,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL2_expr_C_Clist_Hof_Hpieces; /* END OF INLINE: expr::list-of-pieces */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -606,7 +607,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 592 "src/libre/parser.act" +#line 589 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -619,7 +620,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI207 = lex_state->buf.a[0]; -#line 623 "src/libre/dialect/sql/parser.c" +#line 624 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -678,7 +679,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIopen__start; (void) ZIopen__end; -#line 682 "src/libre/dialect/sql/parser.c" +#line 683 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENGROUP */ break; @@ -695,7 +696,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 699 "src/libre/dialect/sql/parser.c" +#line 700 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZIclass; @@ -723,7 +724,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIclose__start; (void) ZIclose__end; -#line 727 "src/libre/dialect/sql/parser.c" +#line 728 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; @@ -734,12 +735,12 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIopen__end)); -#line 738 "src/libre/dialect/sql/parser.c" +#line 739 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-group */ /* BEGINNING OF ACTION: mark-expr */ { -#line 798 "src/libre/parser.act" +#line 796 "src/libre/parser.act" struct ast_pos ast_start, ast_end; @@ -754,7 +755,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZItmp)->u.class.end = ast_end; */ -#line 758 "src/libre/dialect/sql/parser.c" +#line 759 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-expr */ ZInode = ZIclass; @@ -774,7 +775,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI159; -#line 778 "src/libre/dialect/sql/parser.c" +#line 779 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; @@ -787,7 +788,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL3; } -#line 791 "src/libre/dialect/sql/parser.c" +#line 792 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZImask__tmp = ZImask; @@ -819,7 +820,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIclose__start; (void) ZIclose__end; -#line 823 "src/libre/dialect/sql/parser.c" +#line 824 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ break; @@ -834,12 +835,12 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state mark(&act_state->groupstart, &(ZIclose__start)); mark(&act_state->groupend, &(ZIclose__end)); -#line 838 "src/libre/dialect/sql/parser.c" +#line 839 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-group */ /* BEGINNING OF ACTION: mark-expr */ { -#line 798 "src/libre/parser.act" +#line 796 "src/libre/parser.act" struct ast_pos ast_start, ast_end; @@ -854,12 +855,12 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZItmp)->u.class.end = ast_end; */ -#line 858 "src/libre/dialect/sql/parser.c" +#line 859 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-expr */ /* BEGINNING OF ACTION: mark-expr */ { -#line 798 "src/libre/parser.act" +#line 796 "src/libre/parser.act" struct ast_pos ast_start, ast_end; @@ -874,7 +875,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZImask__tmp)->u.class.end = ast_end; */ -#line 878 "src/libre/dialect/sql/parser.c" +#line 879 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-expr */ } @@ -883,14 +884,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 742 "src/libre/parser.act" +#line 739 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL3; -#line 894 "src/libre/dialect/sql/parser.c" +#line 895 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ } @@ -906,7 +907,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL3; } -#line 910 "src/libre/dialect/sql/parser.c" +#line 911 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-subtract */ } @@ -922,14 +923,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 742 "src/libre/parser.act" +#line 739 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 933 "src/libre/dialect/sql/parser.c" +#line 934 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -941,7 +942,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 945 "src/libre/dialect/sql/parser.c" +#line 946 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -991,7 +992,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 995 "src/libre/dialect/sql/parser.c" +#line 996 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1021,7 +1022,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex goto ZL1; } -#line 1025 "src/libre/dialect/sql/parser.c" +#line 1026 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -1035,14 +1036,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 728 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 1046 "src/libre/dialect/sql/parser.c" +#line 1047 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -1054,7 +1055,7 @@ ZL1:; goto ZL2; } -#line 1058 "src/libre/dialect/sql/parser.c" +#line 1059 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -1089,7 +1090,7 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI118; (void) ZI119; -#line 1093 "src/libre/dialect/sql/parser.c" +#line 1094 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1102,7 +1103,7 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1106 "src/libre/dialect/sql/parser.c" +#line 1107 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ @@ -1113,12 +1114,12 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1117 "src/libre/dialect/sql/parser.c" +#line 1118 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1028 "src/libre/parser.act" +#line 1000 "src/libre/parser.act" struct ast_expr *any; @@ -1156,7 +1157,7 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1160 "src/libre/dialect/sql/parser.c" +#line 1161 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1165,7 +1166,7 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1028 "src/libre/parser.act" +#line 1000 "src/libre/parser.act" struct ast_expr *any; @@ -1203,7 +1204,7 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1207 "src/libre/dialect/sql/parser.c" +#line 1208 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1234,7 +1235,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 1238 "src/libre/dialect/sql/parser.c" +#line 1239 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1257,7 +1258,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI (ZIa).type = AST_ENDPOINT_LITERAL; (ZIa).u.literal.c = (unsigned char)(*ZI207); -#line 1261 "src/libre/dialect/sql/parser.c" +#line 1262 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -1272,7 +1273,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI (void) ZI138; (void) ZI139; -#line 1276 "src/libre/dialect/sql/parser.c" +#line 1277 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1280,7 +1281,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 592 "src/libre/parser.act" +#line 589 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1293,7 +1294,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI ZIcz = lex_state->buf.a[0]; -#line 1297 "src/libre/dialect/sql/parser.c" +#line 1298 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -1308,7 +1309,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI (ZIz).type = AST_ENDPOINT_LITERAL; (ZIz).u.literal.c = (unsigned char)(ZIcz); -#line 1312 "src/libre/dialect/sql/parser.c" +#line 1313 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: mark-range */ @@ -1318,12 +1319,12 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI mark(&act_state->rangestart, &(*ZI208)); mark(&act_state->rangeend, &(ZIend)); -#line 1322 "src/libre/dialect/sql/parser.c" +#line 1323 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1041 "src/libre/parser.act" +#line 1038 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -1356,7 +1357,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 1360 "src/libre/dialect/sql/parser.c" +#line 1361 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -1393,7 +1394,7 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI177; (void) ZIend; -#line 1397 "src/libre/dialect/sql/parser.c" +#line 1398 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; @@ -1404,12 +1405,12 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 mark(&act_state->countstart, &(*ZI211)); mark(&act_state->countend, &(ZIend)); -#line 1408 "src/libre/dialect/sql/parser.c" +#line 1409 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 855 "src/libre/parser.act" +#line 853 "src/libre/parser.act" struct ast_pos ast_start, ast_end; @@ -1429,7 +1430,7 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), &ast_start, (*ZIm), &ast_end); -#line 1433 "src/libre/dialect/sql/parser.c" +#line 1434 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1445,7 +1446,7 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 649 "src/libre/parser.act" +#line 641 "src/libre/parser.act" unsigned long u; char *e; @@ -1465,7 +1466,7 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 1469 "src/libre/dialect/sql/parser.c" +#line 1470 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1485,7 +1486,7 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI180; (void) ZIend; -#line 1489 "src/libre/dialect/sql/parser.c" +#line 1490 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -1500,12 +1501,12 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 mark(&act_state->countstart, &(*ZI211)); mark(&act_state->countend, &(ZIend)); -#line 1504 "src/libre/dialect/sql/parser.c" +#line 1505 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 855 "src/libre/parser.act" +#line 853 "src/libre/parser.act" struct ast_pos ast_start, ast_end; @@ -1525,7 +1526,7 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); -#line 1529 "src/libre/dialect/sql/parser.c" +#line 1530 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1566,7 +1567,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL1; } -#line 1570 "src/libre/dialect/sql/parser.c" +#line 1571 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 190 */ @@ -1579,7 +1580,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL2_expr_C_Clist_Hof_Halts; /* END OF INLINE: expr::list-of-alts */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -1591,14 +1592,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 728 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 1602 "src/libre/dialect/sql/parser.c" +#line 1603 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -1631,7 +1632,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI211; (void) ZI212; -#line 1635 "src/libre/dialect/sql/parser.c" +#line 1636 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -1639,7 +1640,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 649 "src/libre/parser.act" +#line 641 "src/libre/parser.act" unsigned long u; char *e; @@ -1659,7 +1660,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 1663 "src/libre/dialect/sql/parser.c" +#line 1664 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1683,7 +1684,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, NULL, 1, NULL); -#line 1687 "src/libre/dialect/sql/parser.c" +#line 1688 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -1697,7 +1698,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 1701 "src/libre/dialect/sql/parser.c" +#line 1702 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -1711,7 +1712,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 1715 "src/libre/dialect/sql/parser.c" +#line 1716 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -1724,7 +1725,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, NULL, 1, NULL); -#line 1728 "src/libre/dialect/sql/parser.c" +#line 1729 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1737,14 +1738,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 714 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 1748 "src/libre/dialect/sql/parser.c" +#line 1749 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ @@ -1753,7 +1754,7 @@ ZL1:; (ZIc) = ast_make_count(1, NULL, 1, NULL); -#line 1757 "src/libre/dialect/sql/parser.c" +#line 1758 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1778,12 +1779,12 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 811 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1787 "src/libre/dialect/sql/parser.c" +#line 1788 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ @@ -1795,7 +1796,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1799 "src/libre/dialect/sql/parser.c" +#line 1800 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -1808,7 +1809,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* BEGINNING OF EXTRACT: CHAR */ { -#line 592 "src/libre/parser.act" +#line 589 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1821,7 +1822,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ZIa = lex_state->buf.a[0]; -#line 1825 "src/libre/dialect/sql/parser.c" +#line 1826 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1834,7 +1835,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1838 "src/libre/dialect/sql/parser.c" +#line 1839 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1848,12 +1849,12 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 811 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1857 "src/libre/dialect/sql/parser.c" +#line 1858 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ @@ -1865,7 +1866,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1869 "src/libre/dialect/sql/parser.c" +#line 1870 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ @@ -1874,7 +1875,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 1878 "src/libre/dialect/sql/parser.c" +#line 1879 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ @@ -1893,7 +1894,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1897 "src/libre/dialect/sql/parser.c" +#line 1898 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1910,7 +1911,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIid) = act_state->group_id++; -#line 1914 "src/libre/dialect/sql/parser.c" +#line 1915 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -1927,7 +1928,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1931 "src/libre/dialect/sql/parser.c" +#line 1932 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -1958,14 +1959,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 721 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 1969 "src/libre/dialect/sql/parser.c" +#line 1970 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -1977,7 +1978,7 @@ ZL1:; goto ZL2; } -#line 1981 "src/libre/dialect/sql/parser.c" +#line 1982 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2020,7 +2021,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act (void) ZI129; (void) ZI130; -#line 2024 "src/libre/dialect/sql/parser.c" +#line 2025 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -2037,7 +2038,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act goto ZL1; } -#line 2041 "src/libre/dialect/sql/parser.c" +#line 2042 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2067,7 +2068,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 2071 "src/libre/dialect/sql/parser.c" +#line 2072 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2088,7 +2089,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 2092 "src/libre/dialect/sql/parser.c" +#line 2093 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2106,7 +2107,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1090 "src/libre/parser.act" static int @@ -2188,7 +2189,6 @@ ZL0:; ADVANCE_LEXER; -#define BUILD_FOR_FUZZER 0 #if BUILD_FOR_FUZZER /* these errors currently are not handled properly */ if (act_state->lex_tok == TOK_ERROR) { diff --git a/src/libre/dialect/sql/parser.h b/src/libre/dialect/sql/parser.h index 44a0a564d..63ed8e773 100644 --- a/src/libre/dialect/sql/parser.h +++ b/src/libre/dialect/sql/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 310 "src/libre/parser.act" +#line 300 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__sql(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1249 "src/libre/parser.act" +#line 1247 "src/libre/parser.act" #line 35 "src/libre/dialect/sql/parser.h" From 0cb89efeda518b45454beb332b8e44e610f6a265 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 13 Jun 2023 16:19:48 -0400 Subject: [PATCH 04/51] ast_compile: Remove direct access to fsm->statecount. This is a step closer to removing: #include "libfsm/internal.h" /* XXX */ but ast_compile.c still needs it for the internal interface `fsm_mergeab`. If either that or `fsm_unionxy` is reworked to make a clearer library boundary, libre could stop depending on internal details of libfsm. --- src/libre/ast_compile.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libre/ast_compile.c b/src/libre/ast_compile.c index e0f321037..502faf8b4 100644 --- a/src/libre/ast_compile.c +++ b/src/libre/ast_compile.c @@ -159,7 +159,8 @@ fsm_unionxy(struct fsm *a, struct fsm *b, fsm_state_t x, fsm_state_t y) { size_t i; - for (i = 0; i < b->statecount; i++) { + const size_t statecount = fsm_countstates(b); + for (i = 0; i < statecount; i++) { fsm_setend(b, i, 0); } } From fb5b74ddb5cfa0b3a2b42e343e3c5beb58030ea2 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 13 Jun 2023 16:23:03 -0400 Subject: [PATCH 05/51] bugfix: Ensure combine_info is zeroed during early return. Otherwise, this can lead to uninitialized memory in combine_info when either a or b has zero states. Found via scan-build. --- src/libfsm/union.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 455dc8359..60e1fbaff 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -33,6 +33,8 @@ fsm_union(struct fsm *a, struct fsm *b, combine_info = &combine_info_internal; } + memset(combine_info, 0x00, sizeof(*combine_info)); + assert(a != NULL); assert(b != NULL); From 51483146c849178c675b89f516cfbdb7e8003810 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 13 Jun 2023 16:24:22 -0400 Subject: [PATCH 06/51] bugfix: Eliminate possible write to null pointer. Found by scan-build. --- src/libfsm/vm/ir.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfsm/vm/ir.c b/src/libfsm/vm/ir.c index 6ec2bfa0a..8c03d94f9 100644 --- a/src/libfsm/vm/ir.c +++ b/src/libfsm/vm/ir.c @@ -301,9 +301,9 @@ opasm_new(struct dfavm_assembler_ir *a, enum dfavm_op_instr instr, enum dfavm_op op->instr = instr; op->cmp_arg = arg; - } - op->ir_state = ir_state; + op->ir_state = ir_state; + } return op; } From 656b32f602ba57e9266e943b5ebd1b8bb13f2554 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 14 May 2023 12:21:51 -0700 Subject: [PATCH 07/51] Missing API comments. --- include/fsm/print.h | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/include/fsm/print.h b/include/fsm/print.h index 0035bc125..790e620bc 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -15,23 +15,24 @@ struct fsm; * Print an FSM to the given file stream. The output is written in the format * specified. The available formats are: * - * fsm_print_api - C code which calls the fsm(3) API - * fsm_print_awk - awk code (gawk dialect) - * fsm_print_c - ISO C90 code - * fsm_print_dot - Graphviz Dot format, intended for rendering graphically - * fsm_print_fsm - fsm(5) .fsm format, suitable for parsing by fsm(1) - * fsm_print_ir - Codegen IR as Dot - * fsm_print_irjson - Codegen IR as JSON - * fsm_print_json - JavaScript Object Notation - * fsm_print_vmc - ISO C90 code, VM style - * fsm_print_vmdot - Graphviz Dot format, showing VM opcodes - * fsm_print_rust - Rust code - * fsm_print_sh - Shell script (bash dialect) - * fsm_print_go - Go code + * fsm_print_api - C code which calls the fsm(3) API + * fsm_print_awk - awk code (gawk dialect) + * fsm_print_c - ISO C90 code + * fsm_print_dot - Graphviz Dot format, intended for rendering graphically + * fsm_print_fsm - fsm(5) .fsm format, suitable for parsing by fsm(1) + * fsm_print_ir - Codegen IR as Dot + * fsm_print_irjson - Codegen IR as JSON + * fsm_print_json - JavaScript Object Notation + * fsm_print_vmc - ISO C90 code, VM style + * fsm_print_vmdot - Graphviz Dot format, showing VM opcodes + * fsm_print_rust - Rust code + * fsm_print_sh - Shell script (bash dialect) + * fsm_print_go - Go code + * fsm_print_goasm/vmasm_* - Assembly in various dialects + * fsm_print_vmops_* - VM opcodes as a datastructure * * The output options may be NULL, indicating to use defaults. * - * TODO: what to return? * TODO: explain constraints * * Returns 0, or -1 on error and errno will be set. An errno of ENOTSUP means From e6c11fb0d8f4a24476f947e73c424ce882f0c1d8 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 14 May 2023 12:22:50 -0700 Subject: [PATCH 08/51] Another pass at ABI fixes for rust, and introduce rust for CI. --- .github/workflows/ci.yml | 2 +- src/retest/main.c | 6 +++++- src/retest/runner.c | 8 ++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8f30907c..1cb5199fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -328,7 +328,7 @@ jobs: cc: [ clang, gcc ] make: [ bmake ] debug: [ DEBUG, EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op - lang: [ "vm -x v1", "vm -x v2", asm, c, vmc, vmops, go, goasm ] + lang: [ "vm -x v1", "vm -x v2", asm, c, rust, vmc, vmops, go, goasm ] exclude: - os: macos cc: gcc # it's clang anyway diff --git a/src/retest/main.c b/src/retest/main.c index 4d2cb5445..78d870115 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -172,9 +172,11 @@ usage(void) fprintf(stderr, " -l \n"); fprintf(stderr, " sets implementation type:\n"); fprintf(stderr, " vm interpret vm instructions (default)\n"); - fprintf(stderr, " asm generate assembly and assemble\n"); + fprintf(stderr, " asm/goasm generate assembly and assemble\n"); fprintf(stderr, " c compile as per fsm_print_c()\n"); fprintf(stderr, " vmc compile as per fsm_print_vmc()\n"); + fprintf(stderr, " vmops compile as per fsm_print_vmops_{c,h,main}()\n"); + fprintf(stderr, " rust compile as per fsm_print_rust()\n"); fprintf(stderr, "\n"); fprintf(stderr, " -x \n"); @@ -1264,6 +1266,8 @@ main(int argc, char *argv[]) impl = IMPL_GO; } else if (strcmp(optarg, "goasm") == 0) { impl = IMPL_GOASM; + } else if (strcmp(optarg, "rust") == 0) { + impl = IMPL_RUST; } else { fprintf(stderr, "unknown argument to -l: %s\n", optarg); usage(); diff --git a/src/retest/runner.c b/src/retest/runner.c index cd94c6007..927583613 100644 --- a/src/retest/runner.c +++ b/src/retest/runner.c @@ -114,10 +114,14 @@ print(const struct fsm *fsm, enum implementation impl, fprintf(f, "use std::slice;\n"); fprintf(f, "\n"); + // XXX: fsm_main shouldn't return Option, Option or a bitfield type fprintf(f, "#[no_mangle]\n"); fprintf(f, "pub extern \"C\" fn retest_trampoline(ptr: *const c_uchar, len: usize) -> i64 {\n"); fprintf(f, " let a: &[u8] = unsafe { slice::from_raw_parts(ptr, len as usize) };\n"); - fprintf(f, " fsm_main(a).unwrap_or(-1)\n"); + fprintf(f, " match fsm_main(a) {\n"); + fprintf(f, " Some(u) => u as i64,\n"); // XXX: we lose info for the i64 ABI + fprintf(f, " None => -1,\n"); + fprintf(f, " }\n"); fprintf(f, "}\n"); } @@ -152,7 +156,7 @@ compile(enum implementation impl, case IMPL_RUST: if (0 != systemf("%s %s --crate-type dylib %s -o %s", - "rustc", "--edition 2018", + "rustc", "--edition 2021", tmp_src, tmp_so)) { return 0; From 64a4bc1f0b8adbf75e174a1129a4af526fb6e0a0 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 16 May 2023 11:12:24 -0700 Subject: [PATCH 09/51] Missing assertion (gcc complains about this but not clang. --- fuzz/target.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fuzz/target.c b/fuzz/target.c index d85cf2ec3..7e7c8ff09 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -1,11 +1,11 @@ #include #include +#include #include #include #include #include -#include #include #include #include @@ -390,4 +390,6 @@ harness_fuzzer_target(const uint8_t *data, size_t size) return res; } } + + assert(!"unreached"); } From dc0c7287b7b163b88a643d178f4b90947f9e1773 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 16 May 2023 11:18:56 -0700 Subject: [PATCH 10/51] Assert on errno == 0 just to catch any pre-existing missing error handling. --- fuzz/target.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fuzz/target.c b/fuzz/target.c index 7e7c8ff09..6757adfda 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -283,7 +283,6 @@ fuzz_all_print_functions(FILE *f, const char *pattern, bool det, bool min, const }; fsm = re_comp(RE_PCRE, scanner_next, &s, &options, RE_MULTI, &err); - if (fsm == NULL) { /* ignore invalid regexp syntax, etc. */ return EXIT_SUCCESS; @@ -303,6 +302,9 @@ fuzz_all_print_functions(FILE *f, const char *pattern, bool det, bool min, const } } + /* if errno isn't zero already, I want to know why */ + assert(errno == 0); + /* see if this triggers any asserts */ int r = 0; r |= fsm_print_api(f, fsm); From b066001122edf140c610884b42520dfb8891f838 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 17 May 2023 09:31:20 -0700 Subject: [PATCH 11/51] Missing argument list for __asan_default_options() --- src/lx/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lx/main.c b/src/lx/main.c index 516634b83..cbcfeea76 100644 --- a/src/lx/main.c +++ b/src/lx/main.c @@ -966,4 +966,4 @@ main(int argc, char *argv[]) /* XXX: we're not interested in leaks in lx for the moment; ASAN is applied * for the libraries only. cleanup for lx's own structures should be addressed * at some point; excluding this is sloppy, but it's less important right now. */ -const char * __asan_default_options() { return "detect_leaks=0"; } +const char * __asan_default_options(void) { return "detect_leaks=0"; } From d759723daa57efbba67218447b4b409edece0e1a Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 16 May 2023 18:57:44 -0700 Subject: [PATCH 12/51] Update kmkf for -fsanitize=fuzzer-no-link --- share/git/kmkf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/git/kmkf b/share/git/kmkf index 6d1994deb..7d0ff701e 160000 --- a/share/git/kmkf +++ b/share/git/kmkf @@ -1 +1 @@ -Subproject commit 6d1994deb38b36f7462f99b1becff830f509e7e1 +Subproject commit 7d0ff701ec84a8f6b0db9da2d60886fb25a578c3 From 2dcca178fe1273c49830c49bda4cb7d54accc2d2 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 15 May 2023 23:26:47 -0700 Subject: [PATCH 13/51] Run fuzzing in CI. --- .github/workflows/ci.yml | 79 +++++++++++++++++++++++++++++++++++++++- Makefile | 8 ++++ 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cb5199fe..1d1caf491 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,7 @@ jobs: strategy: fail-fast: true matrix: - san: [ NO_SANITIZER, ASAN, UBSAN, MSAN, EFENCE ] # NO_SANITIZER=1 is a no-op + san: [ NO_SANITIZER, ASAN, UBSAN, MSAN, EFENCE, FUZZER ] # NO_SANITIZER=1 is a no-op os: [ ubuntu ] cc: [ clang, gcc ] make: [ bmake ] # we test makefiles separately @@ -149,6 +149,8 @@ jobs: san: MSAN # not supported - os: macos make: pmake # not packaged + - san: FUZZER + cc: gcc # -fsanitize=fuzzer is clang-only steps: - name: Fetch checkout @@ -186,13 +188,25 @@ jobs: id: cpu-cores - name: Make - if: steps.cache-build.outputs.cache-hit != 'true' + if: matrix.san != 'FUZZER' && steps.cache-build.outputs.cache-hit != 'true' run: | # note: lexer.h first, because parser.? depends on it find . -name 'lexer.?' -exec touch '{}' \; # workaround for git checkout timestamps find . -name 'parser.?' -exec touch '{}' \; # workaround for git checkout timestamps ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.san }}=1 ${{ matrix.debug }}=1 PKGCONF=pkg-config CC=${{ matrix.cc }} NODOC=1 + # We aren't building the CLI executables here, just the fuzzer + # matrix.san=FUZZER implies UBSAN and ASAN (but not MSAN, MSAN is incompatible with ASAN) + # XXX: needing to explicitly mkdir here is a makefile bug + - name: Make (Fuzzer) + if: matrix.san == 'FUZZER' && steps.cache-build.outputs.cache-hit != 'true' + run: | + # note: lexer.h first, because parser.? depends on it + find . -name 'lexer.?' -exec touch '{}' \; # workaround for git checkout timestamps + find . -name 'parser.?' -exec touch '{}' \; # workaround for git checkout timestamps + ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.san }}=1 ${{ matrix.debug }}=1 UBSAN=1 ASAN=1 PKGCONF=pkg-config CC=${{ matrix.cc }} NODOC=1 mkdir + ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.san }}=1 ${{ matrix.debug }}=1 UBSAN=1 ASAN=1 PKGCONF=pkg-config CC=${{ matrix.cc }} NODOC=1 fuzz + # testing different bmake dialects # the goal here is to excercise the build system, not the code # we don't care about e.g. different compilers here @@ -315,6 +329,67 @@ jobs: # I don't want to build SID just for sake of its -l test run: ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.san }}=1 ${{ matrix.debug }}=1 PKGCONF=pkg-config SID='true; echo sid' CC=${{ matrix.cc }} NODOC=1 LX=../${{ env.build }}/bin/lx test + test_fuzz: + name: "Fuzz (mode ${{ matrix.mode }}) ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" + runs-on: ${{ matrix.os }}-latest + needs: [ build ] + + strategy: + fail-fast: false + matrix: + san: [ FUZZER ] + os: [ ubuntu ] + cc: [ clang ] + make: [ bmake ] + debug: [ DEBUG, EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op + mode: [ p, m ] + exclude: + - os: macos + cc: gcc # it's clang anyway + + steps: + - name: Fetch checkout + uses: actions/cache@v3 + id: cache-checkout + with: + path: ${{ env.wc }} + key: checkout-${{ github.sha }} + + - name: Dependencies (Ubuntu) + if: matrix.os == 'ubuntu' + run: | + uname -a + sudo apt-get install bmake + ${{ matrix.cc }} --version + + - name: Dependencies (MacOS) + if: matrix.os == 'macos' + run: | + uname -a + brew update + brew install bmake + ${{ matrix.cc }} --version + + - name: Fetch build + uses: actions/cache@v3 + id: cache-build + with: + path: ${{ env.build }} + key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v1 + id: cpu-cores + + - name: Fuzz + env: + MODE: ${{ env.mode }} + ASAN_OPTIONS: detect_leaks=0:halt_on_error=1 + UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 + run: | + mkdir -p seeds + ./${{ env.build }}/fuzz/fuzzer -jobs=$((${{ steps.cpu-cores.outputs.count }} + 1)) -max_total_time=60 seeds + test_pcre: name: "Test (PCRE suite) ${{ matrix.lang }} ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" runs-on: ${{ matrix.os }}-latest diff --git a/Makefile b/Makefile index eea459b6c..91e2415f9 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,9 @@ BUILD_IMPOSSIBLE="attempting to use .OBJDIR other than .CURDIR" # targets all:: mkdir .WAIT dep .WAIT lib prog +.if make(fuzz) +fuzz:: mkdir +.endif doc:: mkdir dep:: gen:: @@ -153,6 +156,11 @@ INCDIR += include .include .endif +# XXX: workaround for CI in github where the linker doesn't support dwarf-5? +.if defined(FUZZER) +CFLAGS += -gdwarf-4 +.endif + # these are internal tools for development; we don't install them to $PREFIX STAGE_BUILD := ${STAGE_BUILD:Nbin/retest} STAGE_BUILD := ${STAGE_BUILD:Nbin/reperf} From d5c6132d0ecb822455502ddbfdf47f2090a2dff2 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 18 May 2023 17:51:50 -0700 Subject: [PATCH 14/51] abort() on unreachable codepaths for -DNDEBUG --- src/retest/runner.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/retest/runner.c b/src/retest/runner.c index 927583613..c5c2cde82 100644 --- a/src/retest/runner.c +++ b/src/retest/runner.c @@ -99,7 +99,7 @@ print(const struct fsm *fsm, enum implementation impl, case IMPL_INTERPRET: assert(!"unreached"); - break; + abort(); } if (e == -1) { @@ -297,7 +297,7 @@ runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation case IMPL_INTERPRET: assert(!"unreached"); - break; + abort(); } if (!print(fsm, r->impl, tmp_src)) { From d0da06ab27779528ebcec1e6af6e63cafd7fd292 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 19 May 2023 18:52:52 -0700 Subject: [PATCH 15/51] Missing header. --- src/libfsm/minimise.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index 7d95b78ad..60cec48c5 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -17,6 +17,10 @@ #include #include +#if EXPENSIVE_CHECKS +#include +#endif + #include #include #include From 61664691acbbb1839144741a04d1d08e23216659 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 19 May 2023 11:11:59 -0700 Subject: [PATCH 16/51] Add an explicit "default" mode for fuzzing. --- .github/workflows/ci.yml | 2 +- fuzz/target.c | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d1caf491..387a68d08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -342,7 +342,7 @@ jobs: cc: [ clang ] make: [ bmake ] debug: [ DEBUG, EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op - mode: [ p, m ] + mode: [ m, p, d ] exclude: - os: macos cc: gcc # it's clang anyway diff --git a/fuzz/target.c b/fuzz/target.c index 6757adfda..facac9f05 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -337,16 +337,17 @@ static enum run_mode get_run_mode(void) { const char *mode = getenv("MODE"); - if (mode != NULL) { - switch (mode[0]) { - case 'm': return MODE_SHUFFLE_MINIMISE; - case 'p': return MODE_ALL_PRINT_FUNCTIONS; - default: - break; - } + if (mode == NULL) { + return MODE_DEFAULT; } - return MODE_DEFAULT; + switch (mode[0]) { + case 'm': return MODE_SHUFFLE_MINIMISE; + case 'p': return MODE_ALL_PRINT_FUNCTIONS; + case 'd': + default: + return MODE_DEFAULT; + } } static FILE *dev_null = NULL; @@ -354,8 +355,6 @@ static FILE *dev_null = NULL; int harness_fuzzer_target(const uint8_t *data, size_t size) { - enum run_mode run_mode = get_run_mode(); - if (size < 1) { return EXIT_SUCCESS; } @@ -368,7 +367,7 @@ harness_fuzzer_target(const uint8_t *data, size_t size) const char *pattern = (const char *)data_buf; - switch (run_mode) { + switch (get_run_mode()) { case MODE_DEFAULT: return build_and_codegen(pattern); From b8212f98ff4d0bf3ea0f5bdf380358848ebc38e0 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 19 May 2023 18:14:11 -0700 Subject: [PATCH 17/51] Ignore timeouts when fuzzing under EXPENSIVE_CHECKS. --- fuzz/target.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fuzz/target.c b/fuzz/target.c index facac9f05..7b6f8fee2 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -121,7 +121,9 @@ build(const char *pattern) total_usec += delta_usec; if (total_usec > TIMEOUT_USEC) { +#ifndef EXPENSIVE_CHECKS assert(!"timeout"); +#endif } return fsm; From e2f5431a520e37c37df71b556d0eafad3d2ec2d2 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 19 May 2023 10:29:22 -0700 Subject: [PATCH 18/51] Cache seeds between CI runs. The fuzzing itself is run unconditionally; the idea is the cache is restored each time, and then each run adds a little more. I'm also presenting the corpus as an asset so it can be grabbed to run fuzzing locally. --- .github/workflows/ci.yml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 387a68d08..1d9f50bcf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,7 @@ on: [ push, pull_request ] env: pcre2: pcre2-10.40 wc: wc + seeds: seeds build: build cvtpcre: build/test/retest prefix: prefix @@ -377,6 +378,18 @@ jobs: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} + # note we do the fuzzing unconditionally; each run adds to the corpus + - name: Cache seeds (mode ${{ matrix.mode }}) + uses: actions/cache@v3 + id: cache-seeds + with: + path: ${{ env.seeds }}-${{ matrix.mode }} + key: seeds-${{ matrix.mode }} + + - name: mkdir seeds + if: steps.cache-seeds.outputs.cache-hit != 'true' + run: mkdir -p ${{ env.seeds }}-${{ matrix.mode }} + - name: Get number of CPU cores uses: SimenB/github-actions-cpu-cores@v1 id: cpu-cores @@ -386,9 +399,14 @@ jobs: MODE: ${{ env.mode }} ASAN_OPTIONS: detect_leaks=0:halt_on_error=1 UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 - run: | - mkdir -p seeds - ./${{ env.build }}/fuzz/fuzzer -jobs=$((${{ steps.cpu-cores.outputs.count }} + 1)) -max_total_time=60 seeds + run: ./${{ env.build }}/fuzz/fuzzer -jobs=$((${{ steps.cpu-cores.outputs.count }} + 1)) -max_total_time=60 ${{ env.seeds }}-${{ matrix.mode }} + + # nothing to do with the caching, I'm uploading the seeds so a developer can grab them to fuzz locally + - name: Upload seeds (mode ${{ matrix.mode }}) + uses: actions/upload-artifact@v3 + with: + name: seeds-${{ matrix.mode }} + path: ${{ env.seeds }}-${{ matrix.mode }} test_pcre: name: "Test (PCRE suite) ${{ matrix.lang }} ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" From 33cc19677943633698e27030cb75f3ef669b9d86 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 19 May 2023 16:31:50 -0700 Subject: [PATCH 19/51] Consolidate `DEBUG` under `EXPENSIVE_CHECKS`, but only for CI. It's a little awkward to have CI bundle two things together for the way I've written it, so I've been iterating over them as a set. But that's taking way too long, so here I'm combining the two. I think this is semantically identical to -DNDEBUG; we'd never want to test with just assertions without these expensive checks, and we'd never want to run in production with them. But after some discussion we're keeping them separate for practical reasons. The main idea is that EXPENSIVE_CHECKS is supposed to be for stuff that doesn't change the algorithmic complexity of an operation. That means probably more stuff should move under EXPENSIVE_CHECKS rather than as assertions. --- .github/workflows/ci.yml | 14 +++++++------- Makefile | 7 +++++++ include/adt/common.h | 4 ++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d9f50bcf..297ce3075 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-DEBUG-ASAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-ubuntu-gcc-EXPENSIVE_CHECKS-ASAN-${{ github.sha }}-${{ env.pcre2 }} - name: Fetch build if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -79,7 +79,7 @@ jobs: id: cache-build with: path: ${{ env.build }} - key: build-bmake-ubuntu-gcc-DEBUG-ASAN-${{ github.sha }} # arbitary build, just for cvtpcre + key: build-bmake-ubuntu-gcc-EXPENSIVE_CHECKS-ASAN-${{ github.sha }} # arbitary build, just for cvtpcre - name: Convert PCRE suite if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -140,7 +140,7 @@ jobs: os: [ ubuntu ] cc: [ clang, gcc ] make: [ bmake ] # we test makefiles separately - debug: [ DEBUG, EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op + debug: [ EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op, EXPENSIVE_CHECKS implies DEBUG exclude: - os: macos cc: gcc # it's clang anyway @@ -281,7 +281,7 @@ jobs: os: [ ubuntu ] cc: [ clang, gcc ] make: [ bmake ] - debug: [ DEBUG, EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op + debug: [ EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op, EXPENSIVE_CHECKS implies DEBUG exclude: - os: macos cc: gcc # it's clang anyway @@ -342,7 +342,7 @@ jobs: os: [ ubuntu ] cc: [ clang ] make: [ bmake ] - debug: [ DEBUG, EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op + debug: [ EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op, EXPENSIVE_CHECKS implies DEBUG mode: [ m, p, d ] exclude: - os: macos @@ -420,7 +420,7 @@ jobs: os: [ ubuntu ] cc: [ clang, gcc ] make: [ bmake ] - debug: [ DEBUG, EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op + debug: [ EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op, EXPENSIVE_CHECKS implies DEBUG lang: [ "vm -x v1", "vm -x v2", asm, c, rust, vmc, vmops, go, goasm ] exclude: - os: macos @@ -464,7 +464,7 @@ jobs: id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-DEBUG-ASAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-ubuntu-gcc-EXPENSIVE_CHECKS-ASAN-${{ github.sha }}-${{ env.pcre2 }} - name: Run PCRE suite (${{ matrix.lang }}) run: CC=${{ matrix.cc }} ./${{ env.build }}/bin/retest -O1 -l ${{ matrix.lang }} ${{ env.cvtpcre }}/*.tst diff --git a/Makefile b/Makefile index 91e2415f9..84e41d811 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,13 @@ RE ?= re BUILD ?= build PREFIX ?= /usr/local +# libfsm has EXPENSIVE_CHECKS which are a superset of assertions; +# this is here just so CI can only set one flag at a time. +.if defined(EXPENSIVE_CHECKS) +CFLAGS += -DEXPENSIVE_CHECKS +DEBUG ?= 1 +.endif + # ${unix} is an arbitrary variable set by sys.mk .if defined(unix) .BEGIN:: diff --git a/include/adt/common.h b/include/adt/common.h index 635b3a495..8604edacd 100644 --- a/include/adt/common.h +++ b/include/adt/common.h @@ -20,6 +20,10 @@ #define EXPENSIVE_CHECKS 0 #endif +#if EXPENSIVE_CHECKS && NDEBUG +#error NDEBUG with EXPENSIVE_CHECKS +#endif + /* If set to non-zero, this build should reject inputs as unsupported * that lead to uninteresting failures while fuzzing -- for example, * it's not surprising that `(some regex){1000000}` can hit the From af8f8feda7634cc1b274a6f18a2333ca52ae5d04 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 19 May 2023 18:40:23 -0700 Subject: [PATCH 20/51] -fork for fuzzing, this ignores OOMs by default. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 297ce3075..403c7847e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -399,7 +399,7 @@ jobs: MODE: ${{ env.mode }} ASAN_OPTIONS: detect_leaks=0:halt_on_error=1 UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 - run: ./${{ env.build }}/fuzz/fuzzer -jobs=$((${{ steps.cpu-cores.outputs.count }} + 1)) -max_total_time=60 ${{ env.seeds }}-${{ matrix.mode }} + run: ./${{ env.build }}/fuzz/fuzzer -fork=$((${{ steps.cpu-cores.outputs.count }} + 1)) -max_total_time=60 ${{ env.seeds }}-${{ matrix.mode }} # nothing to do with the caching, I'm uploading the seeds so a developer can grab them to fuzz locally - name: Upload seeds (mode ${{ matrix.mode }}) From 89b3e019c7d9e0081edce3346bab1bbf475d395b Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 19 May 2023 19:04:05 -0700 Subject: [PATCH 21/51] Apparently ASAN_OPTIONS= and UBSAN_OPTIONS= do actually both go into an environment variable named `UBSAN_OPTIONS` --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 403c7847e..c123b3eaf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -397,8 +397,7 @@ jobs: - name: Fuzz env: MODE: ${{ env.mode }} - ASAN_OPTIONS: detect_leaks=0:halt_on_error=1 - UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 + UBSAN_OPTIONS: ASAN_OPTIONS=detect_leaks=0:halt_on_error=1 UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 run: ./${{ env.build }}/fuzz/fuzzer -fork=$((${{ steps.cpu-cores.outputs.count }} + 1)) -max_total_time=60 ${{ env.seeds }}-${{ matrix.mode }} # nothing to do with the caching, I'm uploading the seeds so a developer can grab them to fuzz locally From 784d98c3b07f7e51908a42afe0c0dc723d955aa6 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 19 May 2023 21:57:20 -0700 Subject: [PATCH 22/51] Update kmkf for DWARF-4 workaround. --- Makefile | 5 ----- share/git/kmkf | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 84e41d811..75c87cb2f 100644 --- a/Makefile +++ b/Makefile @@ -163,11 +163,6 @@ INCDIR += include .include .endif -# XXX: workaround for CI in github where the linker doesn't support dwarf-5? -.if defined(FUZZER) -CFLAGS += -gdwarf-4 -.endif - # these are internal tools for development; we don't install them to $PREFIX STAGE_BUILD := ${STAGE_BUILD:Nbin/retest} STAGE_BUILD := ${STAGE_BUILD:Nbin/reperf} diff --git a/share/git/kmkf b/share/git/kmkf index 7d0ff701e..bc42704b8 160000 --- a/share/git/kmkf +++ b/share/git/kmkf @@ -1 +1 @@ -Subproject commit 7d0ff701ec84a8f6b0db9da2d60886fb25a578c3 +Subproject commit bc42704b8e7e8ce5bedbe8fb6c780da595b2eecb From d939bfaf3766ee7c2893a58f7480cdbc48792ad9 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 19 May 2023 23:16:22 -0700 Subject: [PATCH 23/51] Two attempts to deal with timeouts when fuzzing. --- .github/workflows/ci.yml | 1 + fuzz/target.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c123b3eaf..a1da1bf0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -333,6 +333,7 @@ jobs: test_fuzz: name: "Fuzz (mode ${{ matrix.mode }}) ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" runs-on: ${{ matrix.os }}-latest + timeout-minutes: 5 # this should never be reached, it's a safeguard for bugs in the fuzzer itself needs: [ build ] strategy: diff --git a/fuzz/target.c b/fuzz/target.c index 7b6f8fee2..87086b929 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -123,6 +123,9 @@ build(const char *pattern) if (total_usec > TIMEOUT_USEC) { #ifndef EXPENSIVE_CHECKS assert(!"timeout"); +#else + fprintf(stderr, "exiting zero due to timeout under EXPENSIVE_CHECKS\n"); + exit(0); #endif } From cf4eae676cb660f16f3648a1c1213cef44342317 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 20 May 2023 10:01:14 -0700 Subject: [PATCH 24/51] Retry on error, workaround for kmkf#14 --- .github/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1da1bf0a..e1261dae4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -266,8 +266,16 @@ jobs: # Same for lx run: ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.debug }}=1 PKGCONF=pkg-config SID='true; echo sid' LX='true; echo lx' CC=${{ matrix.cc }} NODOC=1 test + # there's an unfixed intermittent makefile bug under -j for + # kmkf duplicate install targets, it's not interesting for libfsm's CI, + # so I'm retrying on error here. # github.com/katef/kmkf/issues/14 - name: Install - run: ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.debug }}=1 PKGCONF=pkg-config PREFIX=../${{ env.prefix }} NODOC=1 install + uses: nick-fields/retry@v2.8.3 + with: + timeout_seconds: 10 # required, but not a problem for the kmkf bug + max_attempts: 3 + retry_on: error + command: ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.debug }}=1 PKGCONF=pkg-config PREFIX=../${{ env.prefix }} NODOC=1 install test_san: name: "Test (Sanitizers) ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" From 0ea03dc4f0068fb2bdc16822c63417a301544995 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 21 May 2023 09:30:06 -0700 Subject: [PATCH 25/51] Combine ASAN and UBSAN just to save some time. --- .github/workflows/ci.yml | 16 ++++++++-------- Makefile | 6 ++++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1261dae4..fe6a72bff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-EXPENSIVE_CHECKS-ASAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-ubuntu-gcc-EXPENSIVE_CHECKS-AUSAN-${{ github.sha }}-${{ env.pcre2 }} - name: Fetch build if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -79,7 +79,7 @@ jobs: id: cache-build with: path: ${{ env.build }} - key: build-bmake-ubuntu-gcc-EXPENSIVE_CHECKS-ASAN-${{ github.sha }} # arbitary build, just for cvtpcre + key: build-bmake-ubuntu-gcc-EXPENSIVE_CHECKS-AUSAN-${{ github.sha }} # arbitary build, just for cvtpcre - name: Convert PCRE suite if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -136,7 +136,7 @@ jobs: strategy: fail-fast: true matrix: - san: [ NO_SANITIZER, ASAN, UBSAN, MSAN, EFENCE, FUZZER ] # NO_SANITIZER=1 is a no-op + san: [ NO_SANITIZER, AUSAN, MSAN, EFENCE, FUZZER ] # NO_SANITIZER=1 is a no-op os: [ ubuntu ] cc: [ clang, gcc ] make: [ bmake ] # we test makefiles separately @@ -205,8 +205,8 @@ jobs: # note: lexer.h first, because parser.? depends on it find . -name 'lexer.?' -exec touch '{}' \; # workaround for git checkout timestamps find . -name 'parser.?' -exec touch '{}' \; # workaround for git checkout timestamps - ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.san }}=1 ${{ matrix.debug }}=1 UBSAN=1 ASAN=1 PKGCONF=pkg-config CC=${{ matrix.cc }} NODOC=1 mkdir - ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.san }}=1 ${{ matrix.debug }}=1 UBSAN=1 ASAN=1 PKGCONF=pkg-config CC=${{ matrix.cc }} NODOC=1 fuzz + ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.san }}=1 ${{ matrix.debug }}=1 AUSAN=1 PKGCONF=pkg-config CC=${{ matrix.cc }} NODOC=1 mkdir + ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.san }}=1 ${{ matrix.debug }}=1 AUSAN=1 PKGCONF=pkg-config CC=${{ matrix.cc }} NODOC=1 fuzz # testing different bmake dialects # the goal here is to excercise the build system, not the code @@ -285,7 +285,7 @@ jobs: strategy: fail-fast: false matrix: - san: [ ASAN, UBSAN, MSAN, EFENCE ] + san: [ AUSAN, MSAN, EFENCE ] os: [ ubuntu ] cc: [ clang, gcc ] make: [ bmake ] @@ -424,7 +424,7 @@ jobs: strategy: fail-fast: false matrix: - san: [ ASAN, UBSAN, MSAN, EFENCE ] + san: [ AUSAN, MSAN, EFENCE ] os: [ ubuntu ] cc: [ clang, gcc ] make: [ bmake ] @@ -472,7 +472,7 @@ jobs: id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-EXPENSIVE_CHECKS-ASAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-ubuntu-gcc-EXPENSIVE_CHECKS-AUSAN-${{ github.sha }}-${{ env.pcre2 }} - name: Run PCRE suite (${{ matrix.lang }}) run: CC=${{ matrix.cc }} ./${{ env.build }}/bin/retest -O1 -l ${{ matrix.lang }} ${{ env.cvtpcre }}/*.tst diff --git a/Makefile b/Makefile index 75c87cb2f..8d742883e 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,12 @@ CFLAGS += -DEXPENSIVE_CHECKS DEBUG ?= 1 .endif +# combined just to save time in CI +.if defined(AUSAN) +ASAN ?= 1 +UBSAN ?= 1 +.endif + # ${unix} is an arbitrary variable set by sys.mk .if defined(unix) .BEGIN:: From 4ee299ddfb23c2e38a11bcc9417083b6a7842f46 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 21 May 2023 08:54:33 -0700 Subject: [PATCH 26/51] No need to rebuild the entire repo just to test makefiles. --- .github/workflows/ci.yml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe6a72bff..ea6478a19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -214,15 +214,16 @@ jobs: test_makefiles: name: "Test (Makefiles) ${{ matrix.make }} ${{ matrix.os }} ${{ matrix.debug }}" runs-on: ${{ matrix.os }}-latest - needs: [ checkout ] + needs: [ checkout, build ] strategy: fail-fast: false matrix: + san: [ NO_SANITIZER ] # NO_SANITIZER=1 is a no-op os: [ ubuntu ] cc: [ clang ] make: [ bmake, pmake ] - debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op + debug: [ EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op exclude: - os: macos make: pmake # not packaged @@ -235,6 +236,24 @@ jobs: path: ${{ env.wc }} key: checkout-${{ github.sha }} + # An arbitary build. + - name: Fetch build + uses: actions/cache@v3 + id: cache-build + with: + path: ${{ env.build }} + key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} + + # We don't need to build the entire repo to know that the makefiles work, + # I'm just deleting a couple of .o files and rebuilding those instead. + - name: Delete something + if: steps.cache-build.outputs.cache-hit == 'true' + run: find ${{ env.build }} -type f -name '*.o' | sort -r | head -5 | xargs rm + + - name: Outdate something + if: steps.cache-build.outputs.cache-hit == 'true' + run: find ${{ env.wc }} -type f -name '*.c' | sort -r | head -5 | xargs touch + - name: Dependencies (Ubuntu) if: matrix.os == 'ubuntu' run: | From c732e007c069ef61f9b499b48eee12b51ccea8d1 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 20 May 2023 10:28:05 -0700 Subject: [PATCH 27/51] A cheesy attempt at parallelisation. --- src/retest/main.c | 66 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/src/retest/main.c b/src/retest/main.c index 78d870115..bd08773fc 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -8,6 +8,8 @@ #define _POSIX_C_SOURCE 200809L #endif +#include + #include #include @@ -1230,6 +1232,9 @@ main(int argc, char *argv[]) max_test_errors = 0; + /* because we fork() for each test file */ + setvbuf(stderr, NULL, _IOLBF, BUFSIZ); + { int c; @@ -1342,35 +1347,60 @@ main(int argc, char *argv[]) { int r, i; - struct error_record erec; + for (i = 0; i < argc; i++) { + struct error_record erec; + int nerrs; - r = 0; + switch (fork()) { + case -1: + perror("fork"); + return EXIT_FAILURE; - if (error_record_init(&erec) != 0) { - fprintf(stderr, "error initializing error state: %s\n", strerror(errno)); - return EXIT_FAILURE; - } + case 0: + break; - for (i = 0; i < argc; i++) { - int nerrs; + default: + continue; + } + + if (error_record_init(&erec) != 0) { + fprintf(stderr, "error initializing error state: %s\n", strerror(errno)); + return EXIT_FAILURE; + } nerrs = process_test_file(argv[i], dialect, impl, max_test_errors, &erec); - if (nerrs > 0) { - r |= 1; - continue; + if (erec.len > 0) { + error_record_print(stderr, &erec); + fprintf(stderr, "%s: %u errors\n", argv[i], (unsigned) erec.len); + } else { + fprintf(stderr, "%s: no errors\n", argv[i]); } - } - if (erec.len > 0) { - error_record_print(stderr, &erec); - fprintf(stderr, "%u errors\n", (unsigned) erec.len); - } else { - fprintf(stderr, "no errors\n"); + error_record_finalize(&erec); + + exit(nerrs > 0 ? EXIT_FAILURE : EXIT_SUCCESS); + +/* XXX: would prefer to get errors to parent, summarize once. pipe is probably simplest. +or: wait in child for signal, parent signals child to summarize errors when ready */ } - error_record_finalize(&erec); + r = 0; + + for (i = 0; i < argc; i++) { + int status; + + pid_t pid = wait(&status); + if (pid == -1) { + perror("wait"); + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + r |= 1; + } + } +/* TODO: summarise which argv[] .tst files failed */ return r; } From dd9863696a02fd93ed4fdd65644a2dc6c4932c93 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 21 May 2023 22:37:42 -0700 Subject: [PATCH 28/51] No need to strdup here. --- src/retest/main.c | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/retest/main.c b/src/retest/main.c index bd08773fc..b6b4c52f7 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -442,7 +442,7 @@ parse_escapes(char *s, char **errpos, int *lenp) } struct single_error_record { - char *filename; + const char *filename; /* owned by argv[] storage */ char *regexp; char *flags; char *failed_match; @@ -568,7 +568,8 @@ dup_str(const char *s, int *err) } static int -error_record_add(struct error_record *erec, enum error_type type, const char *fn, const char *re, const char *flags, const char *failed_match, unsigned int line, enum re_dialect dialect) +error_record_add(struct error_record *erec, enum error_type type, + const char *filename, const char *re, const char *flags, const char *failed_match, unsigned int line, enum re_dialect dialect) { size_t ind; int err; @@ -595,7 +596,7 @@ error_record_add(struct error_record *erec, enum error_type type, const char *fn /* add record */ err = 0; - erec->errors[ind].filename = dup_str(fn,&err); + erec->errors[ind].filename = filename; erec->errors[ind].regexp = re != NULL ? dup_str_esc(re,&err) : NULL; erec->errors[ind].flags = flags != NULL ? dup_str_esc(flags,&err) : NULL; erec->errors[ind].failed_match = failed_match != NULL ? dup_str(failed_match,&err) : NULL; @@ -613,6 +614,9 @@ single_error_record_print(FILE *f, const struct single_error_record *err) { const char *dialect_name; + assert(f != NULL); + assert(err != NULL); + dialect_name = dialect_to_name(err->dialect); fprintf(f, "(%s:%u) ", err->filename, err->line); @@ -650,6 +654,10 @@ static void error_record_print(FILE *f, const struct error_record *erec) { size_t i,n; + + assert(f != NULL); + assert(erec != NULL); + n = erec->len; for (i=0; i < n; i++) { fprintf(f, "[%4u/%4u] ", (unsigned) i + 1, (unsigned) n); @@ -668,7 +676,6 @@ error_record_finalize(struct error_record *erec) n = erec->len; for (i=0; i < n; i++) { - free(erec->errors[i].filename); free(erec->errors[i].regexp); free(erec->errors[i].flags); free(erec->errors[i].failed_match); @@ -730,7 +737,8 @@ flagstring(enum re_flags flags, char buf[16]) * c. '#' lines are comments */ static int -process_test_file(const char *fname, enum re_dialect default_dialect, enum implementation impl, int max_errors, struct error_record *erec) +process_test_file(const char *filename, + enum re_dialect default_dialect, enum implementation impl, int max_errors, struct error_record *erec) { static const struct fsm_runner init_runner; @@ -764,7 +772,7 @@ process_test_file(const char *fname, enum re_dialect default_dialect, enum imple /* XXX - fix this */ opt.comments = 0; - f = xopen(fname); + f = xopen(filename); num_regexps = 0; num_re_errors = 0; @@ -951,7 +959,7 @@ process_test_file(const char *fname, enum re_dialect default_dialect, enum imple /* ignore errors */ error_record_add(erec, - ERROR_ESCAPE_SEQUENCE, fname, regexp, flagdesc, NULL, linenum, dialect); + ERROR_ESCAPE_SEQUENCE, filename, regexp, flagdesc, NULL, linenum, dialect); num_re_errors++; continue; @@ -976,7 +984,7 @@ process_test_file(const char *fname, enum re_dialect default_dialect, enum imple /* ignore errors */ error_record_add(erec, - ERROR_PARSING_REGEXP, fname, regexp, flagdesc, NULL, linenum, dialect); + ERROR_PARSING_REGEXP, filename, regexp, flagdesc, NULL, linenum, dialect); /* don't exit; instead we leave vm==NULL so we * skip to next regexp ... */ @@ -1009,7 +1017,7 @@ process_test_file(const char *fname, enum re_dialect default_dialect, enum imple /* ignore errors */ error_record_add(erec, - ERROR_WATCHDOG, fname, regexp, flagdesc, NULL, linenum, dialect); + ERROR_WATCHDOG, filename, regexp, flagdesc, NULL, linenum, dialect); /* try to free */ fsm_free(fsm); @@ -1026,7 +1034,7 @@ process_test_file(const char *fname, enum re_dialect default_dialect, enum imple /* ignore errors */ error_record_add(erec, - ERROR_DETERMINISING, fname, regexp, flagdesc, NULL, linenum, dialect); + ERROR_DETERMINISING, filename, regexp, flagdesc, NULL, linenum, dialect); /* try to free */ fsm_free(fsm); @@ -1062,7 +1070,7 @@ process_test_file(const char *fname, enum re_dialect default_dialect, enum imple linenum, dialect_name, regexp, flagdesc, strerror(errno)); /* ignore errors */ - error_record_add(erec, ret, fname, regexp, flagdesc, NULL, linenum, dialect); + error_record_add(erec, ret, filename, regexp, flagdesc, NULL, linenum, dialect); /* don't exit; instead we leave vm==NULL so we * skip to next regexp ... */ @@ -1084,7 +1092,7 @@ process_test_file(const char *fname, enum re_dialect default_dialect, enum imple fprintf(stderr, "line %d: unrecognized record type '%c': %s\n", linenum, s[0], s); /* ignore errors */ error_record_add(erec, - ERROR_BAD_RECORD_TYPE, fname, regexp, flagdesc, NULL, linenum, dialect); + ERROR_BAD_RECORD_TYPE, filename, regexp, flagdesc, NULL, linenum, dialect); num_errors++; continue; @@ -1105,7 +1113,7 @@ process_test_file(const char *fname, enum re_dialect default_dialect, enum imple /* ignore errors */ error_record_add(erec, - ERROR_ESCAPE_SEQUENCE, fname, regexp, flagdesc, NULL, linenum, dialect); + ERROR_ESCAPE_SEQUENCE, filename, regexp, flagdesc, NULL, linenum, dialect); num_errors++; continue; @@ -1137,7 +1145,7 @@ process_test_file(const char *fname, enum re_dialect default_dialect, enum imple (matching) ? ERROR_SHOULD_MATCH : ERROR_SHOULD_NOT_MATCH, - fname, regexp, flagdesc, orig, linenum, dialect); + filename, regexp, flagdesc, orig, linenum, dialect); if (max_errors > 0 && num_errors >= max_errors) { @@ -1156,7 +1164,7 @@ process_test_file(const char *fname, enum re_dialect default_dialect, enum imple } if (ferror(f)) { - fprintf(stderr, "line %d: error reading %s: %s\n", linenum, fname, strerror(errno)); + fprintf(stderr, "line %d: error reading %s: %s\n", linenum, filename, strerror(errno)); if (regexp != NULL) { num_errors++; } @@ -1165,8 +1173,9 @@ process_test_file(const char *fname, enum re_dialect default_dialect, enum imple fclose(f); finish: - printf("%s: %d regexps, %d test cases\n", fname, num_regexps, num_test_cases); - printf("%s: %d re errors, %d errors\n", fname, num_re_errors, num_errors); + + printf("%s: %d regexps, %d test cases\n", filename, num_regexps, num_test_cases); + printf("%s: %d re errors, %d errors\n", filename, num_re_errors, num_errors); if (max_errors > 0 && num_errors >= max_errors) { exit(EXIT_FAILURE); From f216cb45c650094d0828ae9417918ed234dabf11 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 1 Jun 2023 18:32:53 -0700 Subject: [PATCH 29/51] Just DEBUG for tests and fuzzing, not EXPENSIVE_CHECKS. --- .github/workflows/ci.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea6478a19..2c216388d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-EXPENSIVE_CHECKS-AUSAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} - name: Fetch build if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -79,7 +79,7 @@ jobs: id: cache-build with: path: ${{ env.build }} - key: build-bmake-ubuntu-gcc-EXPENSIVE_CHECKS-AUSAN-${{ github.sha }} # arbitary build, just for cvtpcre + key: build-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }} # arbitary build, just for cvtpcre - name: Convert PCRE suite if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -140,7 +140,7 @@ jobs: os: [ ubuntu ] cc: [ clang, gcc ] make: [ bmake ] # we test makefiles separately - debug: [ EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op, EXPENSIVE_CHECKS implies DEBUG + debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op exclude: - os: macos cc: gcc # it's clang anyway @@ -211,6 +211,9 @@ jobs: # testing different bmake dialects # the goal here is to excercise the build system, not the code # we don't care about e.g. different compilers here + # + # I'm including EXPENSIVE_CHECKS here just so we have some coverage + # of the build during CI, even if we don't run that during tests. test_makefiles: name: "Test (Makefiles) ${{ matrix.make }} ${{ matrix.os }} ${{ matrix.debug }}" runs-on: ${{ matrix.os }}-latest @@ -223,7 +226,7 @@ jobs: os: [ ubuntu ] cc: [ clang ] make: [ bmake, pmake ] - debug: [ EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op + debug: [ EXPENSIVE_CHECKS, DEBUG, RELEASE ] # RELEASE=1 is a no-op exclude: - os: macos make: pmake # not packaged @@ -308,7 +311,7 @@ jobs: os: [ ubuntu ] cc: [ clang, gcc ] make: [ bmake ] - debug: [ EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op, EXPENSIVE_CHECKS implies DEBUG + debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op exclude: - os: macos cc: gcc # it's clang anyway @@ -370,7 +373,7 @@ jobs: os: [ ubuntu ] cc: [ clang ] make: [ bmake ] - debug: [ EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op, EXPENSIVE_CHECKS implies DEBUG + debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op mode: [ m, p, d ] exclude: - os: macos @@ -447,7 +450,7 @@ jobs: os: [ ubuntu ] cc: [ clang, gcc ] make: [ bmake ] - debug: [ EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op, EXPENSIVE_CHECKS implies DEBUG + debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op lang: [ "vm -x v1", "vm -x v2", asm, c, rust, vmc, vmops, go, goasm ] exclude: - os: macos @@ -491,7 +494,7 @@ jobs: id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-EXPENSIVE_CHECKS-AUSAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} - name: Run PCRE suite (${{ matrix.lang }}) run: CC=${{ matrix.cc }} ./${{ env.build }}/bin/retest -O1 -l ${{ matrix.lang }} ${{ env.cvtpcre }}/*.tst From b5d710e325bdb5d7067ba5464c4550f982f0ec63 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 2 Jun 2023 02:16:30 -0700 Subject: [PATCH 30/51] Explicit cache save/restore actions for seeds. Here I'm also saving the seeds unconditionally on error, so we don't need to re-find the same seeds for each bug. Hopefully this should make fuzzing reproducible in CI. --- .github/workflows/ci.yml | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c216388d..429ec2523 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -410,12 +410,12 @@ jobs: key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} # note we do the fuzzing unconditionally; each run adds to the corpus - - name: Cache seeds (mode ${{ matrix.mode }}) - uses: actions/cache@v3 + - name: Restore seeds (mode ${{ matrix.mode }}) + uses: actions/cache/restore@v3 id: cache-seeds with: path: ${{ env.seeds }}-${{ matrix.mode }} - key: seeds-${{ matrix.mode }} + key: seeds-${{ matrix.mode }}-${{ matrix.debug }} - name: mkdir seeds if: steps.cache-seeds.outputs.cache-hit != 'true' @@ -431,11 +431,34 @@ jobs: UBSAN_OPTIONS: ASAN_OPTIONS=detect_leaks=0:halt_on_error=1 UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 run: ./${{ env.build }}/fuzz/fuzzer -fork=$((${{ steps.cpu-cores.outputs.count }} + 1)) -max_total_time=60 ${{ env.seeds }}-${{ matrix.mode }} + # saving the cache would fail because this key already exists, + # so I'm just explicitly purging by key here. + # the key contains "-${{ matrix.debug }}" so we don't lose seeds + # found from another instance in the matrix when purging. + - name: Purge cached seeds (mode ${{ matrix.mode }}-${{ matrix.debug }}) + if: steps.cache-seeds.outputs.cache-hit == 'true' + run: | + set +e + gh extension install actions/gh-actions-cache + gh actions-cache delete ${{ steps.cache-seeds.outputs.cache-primary-key }} -R ${{ github.repository }} --confirm + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # if: always() means we keep these even on error, so we don't need to re-find + # the same seeds for a given bug. + # The explicit cache/restore and cache/save actions are just for that. + - name: Save seeds (mode ${{ matrix.mode }}-${{ matrix.debug }}) + uses: actions/cache/save@v3 + if: always() + with: + path: ${{ env.seeds }}-${{ matrix.mode }} + key: ${{ steps.cache-seeds.outputs.cache-primary-key }} + # nothing to do with the caching, I'm uploading the seeds so a developer can grab them to fuzz locally - - name: Upload seeds (mode ${{ matrix.mode }}) + - name: Upload seeds (mode ${{ matrix.mode }}-${{ matrix.debug }}) uses: actions/upload-artifact@v3 with: - name: seeds-${{ matrix.mode }} + name: seeds-${{ matrix.mode }}-${{ matrix.debug }} path: ${{ env.seeds }}-${{ matrix.mode }} test_pcre: From 4e5bcce34bfb44fe70a0f4958f8ab22f10151ff0 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 3 Jun 2023 12:27:37 -0700 Subject: [PATCH 31/51] None of these are valid rust byte escapes. https://static.rust-lang.org/doc/master/reference.html#literals --- src/print/rust.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/print/rust.c b/src/print/rust.c index 745f9875d..d5a547e18 100644 --- a/src/print/rust.c +++ b/src/print/rust.c @@ -29,13 +29,9 @@ rust_escputc_char(FILE *f, const struct fsm_options *opt, char c) case '\\': return fputs("\\\\", f); case '\'': return fputs("\\\'", f); - case '\a': return fputs("\\a", f); - case '\b': return fputs("\\b", f); - case '\f': return fputs("\\f", f); case '\n': return fputs("\\n", f); case '\r': return fputs("\\r", f); case '\t': return fputs("\\t", f); - case '\v': return fputs("\\v", f); default: break; From 7a432074da222194a6aaeae8704d9541e3c08bfa Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 3 Jun 2023 14:08:29 -0700 Subject: [PATCH 32/51] Only `mut` when we modify a variable. --- src/libfsm/print/rust.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index faef05558..0ef12cd31 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -216,8 +216,19 @@ fsm_print_rustfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, " }\n"); fprintf(f, "\n"); - fprintf(f, " let mut l = Ls;\n"); - fprintf(f, "\n"); + { + int has_branch = 0; + + for (op = a.linked; op != NULL; op = op->next) { + if (op->instr == VM_OP_BRANCH) { + has_branch = 1; + break; + } + } + + fprintf(f, " let %sl = Ls;\n", has_branch ? "mut " : ""); + fprintf(f, "\n"); + } fprintf(f, " loop {\n"); fprintf(f, " match l {\n"); From 479d103357f4e1a8596437895bbc6ddf237a1da0 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 3 Jun 2023 15:17:25 -0700 Subject: [PATCH 33/51] Mark unused input when there are no fetch instructions. --- src/libfsm/print/rust.c | 92 +++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 41 deletions(-) diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index 0ef12cd31..e249c6aad 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -65,6 +65,18 @@ cmp_operator(int cmp) } } +static int +has_op(const struct dfavm_op_ir *op, enum dfavm_op_instr instr) +{ + for ( ; op != NULL; op = op->next) { + if (op->instr == instr) { + return 1; + } + } + + return 0; +} + static void print_label(FILE *f, const struct dfavm_op_ir *op) { @@ -129,25 +141,20 @@ print_fetch(FILE *f) /* TODO: eventually to be non-static */ static int -fsm_print_rustfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, +fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, + const struct ir *ir, const struct fsm_options *opt, const char *cp, int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), const void *leaf_opaque) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; struct dfavm_op_ir *op; bool fallthrough; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; - assert(f != NULL); - assert(ir != NULL); + assert(a != NULL); assert(opt != NULL); assert(cp != NULL); - a = zero; - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. * We'll need that in order to be able to use the leaf callback here. */ (void) leaf; @@ -156,10 +163,6 @@ fsm_print_rustfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; - } - /* * We only output labels for ops which are branched to. This gives * gaps in the sequence for ops which don't need a label. @@ -170,8 +173,8 @@ fsm_print_rustfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, l = START; - for (op = a.linked; op != NULL; op = op->next) { - if (op == a.linked || op->num_incoming > 0) { + for (op = a->linked; op != NULL; op = op->next) { + if (op == a->linked || op->num_incoming > 0) { op->index = l++; } } @@ -180,8 +183,8 @@ fsm_print_rustfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, /* * Only declare variables if we're actually going to use them. */ - if (a.linked->cmp == VM_CMP_ALWAYS && a.linked->instr == VM_OP_STOP) { - assert(a.linked->next == NULL); + if (a->linked->cmp == VM_CMP_ALWAYS && a->linked->instr == VM_OP_STOP) { + assert(a->linked->next == NULL); fprintf(f, "\n"); } else { switch (opt->io) { @@ -205,8 +208,8 @@ fsm_print_rustfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, } fprintf(f, " pub enum Label {\n "); - for (op = a.linked; op != NULL; op = op->next) { - if (op == a.linked || op->num_incoming > 0) { + for (op = a->linked; op != NULL; op = op->next) { + if (op == a->linked || op->num_incoming > 0) { fprintf(f, " "); print_label(f, op); fprintf(f, ","); @@ -216,28 +219,17 @@ fsm_print_rustfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, " }\n"); fprintf(f, "\n"); - { - int has_branch = 0; - - for (op = a.linked; op != NULL; op = op->next) { - if (op->instr == VM_OP_BRANCH) { - has_branch = 1; - break; - } - } - - fprintf(f, " let %sl = Ls;\n", has_branch ? "mut " : ""); - fprintf(f, "\n"); - } + fprintf(f, " let %sl = Ls;\n", has_op(a->linked, VM_OP_BRANCH) ? "mut " : ""); + fprintf(f, "\n"); fprintf(f, " loop {\n"); fprintf(f, " match l {\n"); fallthrough = true; - for (op = a.linked; op != NULL; op = op->next) { - if (op == a.linked || op->num_incoming > 0) { - if (op != a.linked) { + for (op = a->linked; op != NULL; op = op->next) { + if (op == a->linked || op->num_incoming > 0) { + if (op != a->linked) { if (fallthrough) { fprintf(f, " "); print_jump(f, op); @@ -362,8 +354,6 @@ fsm_print_rustfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, " }\n"); fprintf(f, " }\n"); - dfavm_opasm_finalize_op(&a); - return 0; } @@ -371,14 +361,24 @@ static int fsm_print_rust_complete(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *prefix, const char *cp) { + static const struct dfavm_assembler_ir zero; + static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; + struct dfavm_assembler_ir a; + assert(f != NULL); assert(ir != NULL); assert(opt != NULL); + a = zero; + + if (!dfavm_compile_ir(&a, ir, vm_opts)) { + return -1; + } + if (opt->fragment) { - fsm_print_rustfrag(f, ir, opt, cp, + fsm_print_rustfrag(f, &a, ir, opt, cp, opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque); - return -1; + goto error; } fprintf(f, "\n"); @@ -394,13 +394,15 @@ fsm_print_rust_complete(FILE *f, const struct ir *ir, case FSM_IO_STR: /* e.g. dbg!(fsm_main("xabces")); */ - fprintf(f, "(input: &str) -> Option {\n"); + fprintf(f, "(%sinput: &str) -> Option {\n", + has_op(a.linked, VM_OP_FETCH) ? "" : "_"); fprintf(f, " use Label::*;\n"); break; case FSM_IO_PAIR: /* e.g. dbg!(fsm_main("xabces".as_bytes())); */ - fprintf(f, "(input: &[u8]) -> Option {\n"); + fprintf(f, "(%sinput: &[u8]) -> Option {\n", + has_op(a.linked, VM_OP_FETCH) ? "" : "_"); fprintf(f, " use Label::*;\n"); break; @@ -409,17 +411,25 @@ fsm_print_rust_complete(FILE *f, const struct ir *ir, exit(EXIT_FAILURE); } - fsm_print_rustfrag(f, ir, opt, cp, + fsm_print_rustfrag(f, &a, ir, opt, cp, opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque); fprintf(f, "}\n"); fprintf(f, "\n"); + dfavm_opasm_finalize_op(&a); + if (ferror(f)) { return -1; } return 0; + +error: + + dfavm_opasm_finalize_op(&a); + + return -1; } int From 75bb88d95bc25117d79ff7cc349bce5aa94b9999 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 16:52:35 -0400 Subject: [PATCH 34/51] state_set: Improve `state_set_search` performance, correct result. The description says "Return where an item would be, if it were inserted", but it was returning the last element <= rather than the first element >=, then the call to `state_set_cmpval` later was shifting i by 1 for that specific case. Handle it correctly inside the search function instead. Two other all call sites need to check whether the result refers to the append position (one past the end of the array) before checking `set->a[i] == state`, update them. Add a fast path upfront: It's VERY common to append states in order to the state array, so before we binary search each first compare against the last entry (unless empty). --- src/adt/stateset.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index c1cff9933..4d4de9438 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -138,7 +138,8 @@ state_set_cmp(const struct state_set *a, const struct state_set *b) } /* - * Return where an item would be, if it were inserted + * Return where an item would be, if it were inserted. + * When insertion would append this returns one past the array. */ static size_t state_set_search(const struct state_set *set, fsm_state_t state) @@ -150,6 +151,11 @@ state_set_search(const struct state_set *set, fsm_state_t state) assert(!IS_SINGLETON(set)); assert(set->a != NULL); + /* fast path: append case */ + if (set->i > 0 && state > set->a[set->i - 1]) { + return set->i; + } + start = mid = 0; end = set->i; @@ -161,6 +167,12 @@ state_set_search(const struct state_set *set, fsm_state_t state) end = mid; } else if (r > 0) { start = mid + 1; + /* update mid if we're about to halt, because + * we're looking for the first position >= state, + * not the last position <= */ + if (start == end) { + mid = start; + } } else { return mid; } @@ -242,7 +254,7 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, */ if (!state_set_empty(set)) { i = state_set_search(set, state); - if (set->a[i] == state) { + if (i < set->i && set->a[i] == state) { return 1; } } @@ -261,9 +273,6 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, set->n *= 2; } - if (state_set_cmpval(state, set->a[i]) > 0) { - i++; - } if (i <= set->i) { memmove(&set->a[i + 1], &set->a[i], (set->i - i) * (sizeof *set->a)); @@ -470,7 +479,7 @@ state_set_remove(struct state_set **setp, fsm_state_t state) } i = state_set_search(set, state); - if (set->a[i] == state) { + if (i < set->i && set->a[i] == state) { if (i < set->i) { memmove(&set->a[i], &set->a[i + 1], (set->i - i - 1) * (sizeof *set->a)); } @@ -524,7 +533,7 @@ state_set_contains(const struct state_set *set, fsm_state_t state) } i = state_set_search(set, state); - if (set->a[i] == state) { + if (i < set->i && set->a[i] == state) { return 1; } From 709b8cc11c5f425e6962386cd43ad36ae6eb2d68 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 16:57:13 -0400 Subject: [PATCH 35/51] stateset: Avoid memmove of size 0. --- src/adt/stateset.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index 4d4de9438..645424555 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -273,8 +273,7 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, set->n *= 2; } - - if (i <= set->i) { + if (i < set->i) { memmove(&set->a[i + 1], &set->a[i], (set->i - i) * (sizeof *set->a)); } From cead0d99915faad077e3e22dd09761a61556c8e0 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:09:01 -0400 Subject: [PATCH 36/51] stateset: Add note about potentially expensive assertion. In -O0 this can become pretty expensive (~25% of overall runtime for `time ./re -rpcre -C '^[ab]{0,2000}$'`), but when built with -O3 very little overhead remains. I'm adding this comment because every time I see this it seems to me like it should have `EXPENSIVE_CHECKS` around it, but profiling is telling me it really doesn't matter. --- src/adt/stateset.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index 645424555..7839234d2 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -284,6 +284,8 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, set->i = 1; } + /* This assert can be pretty expensive in -O0 but in -O3 it has very + * little impact on the overall runtime. */ assert(state_set_contains(set, state)); return 1; From cbfeddd9c54178fe34aae0606a619e9592c6dd69 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:11:07 -0400 Subject: [PATCH 37/51] stateset: Comment struct fields. --- src/adt/stateset.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index 7839234d2..8d73a9b5f 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -44,8 +44,8 @@ struct state_set { const struct fsm_alloc *alloc; fsm_state_t *a; - size_t i; - size_t n; + size_t i; /* used */ + size_t n; /* ceil */ }; int From c3dab77fc7785e3ec4df9e4fb60c6c760ca66b0c Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:12:04 -0400 Subject: [PATCH 38/51] edgeset: Fix indentation for `#if`'d block. --- src/adt/edgeset.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index c718727ca..f1a9e9f31 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -223,9 +223,9 @@ edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, assert(set->count <= set->ceil); #if LOG_BITSET - fprintf(stderr, " -- edge_set_add: symbols [0x%lx, 0x%lx, 0x%lx, 0x%lx] -> state %d on %p\n", - symbols[0], symbols[1], symbols[2], symbols[3], - state, (void *)set); + fprintf(stderr, " -- edge_set_add: symbols [0x%lx, 0x%lx, 0x%lx, 0x%lx] -> state %d on %p\n", + symbols[0], symbols[1], symbols[2], symbols[3], + state, (void *)set); #endif /* Linear search for a group with the same destination From 1ada07cce13fd2982056963e5b1e2bae59b235e9 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:12:31 -0400 Subject: [PATCH 39/51] edgeset: Switch from linear to binary searching in edge_set_add_bulk. This is a major hotspot when doing epsilon removal over large runs of potentially skipped states (as might appear from `^[ab]{0,2000}$`). Add a fast path for appending, which is also very common. Extract the edge set destination search into its own function, `find_state_position`, and add a `#define` to switch between linear search, binary search, or calling both and comparing the result. I will remove linear search in the next commit, but am checking this in as an intermediate step for checking & benchmarking. --- src/adt/edgeset.c | 182 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 165 insertions(+), 17 deletions(-) diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index f1a9e9f31..818b78493 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -11,6 +11,7 @@ #include #define LOG_BITSET 0 +#define LOG_BSEARCH 0 #include "libfsm/internal.h" /* XXX: for allocating struct fsm_edge, and the edges array */ @@ -184,6 +185,157 @@ edge_set_advise_growth(struct edge_set **pset, const struct fsm_alloc *alloc, return 1; } +enum fsp_res { + FSP_FOUND_INSERT_POSITION, + FSP_FOUND_VALUE_PRESENT, +}; + +/* Use binary search to find the first position N where set->groups[N].to >= state, + * which includes the position immediately following the last entry. Return an enum + * which indicates whether state is already present. */ +static enum fsp_res +find_state_position_bsearch(const struct edge_set *set, fsm_state_t state, size_t *dst) +{ + size_t lo = 0, hi = set->count; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: looking for %d in %p (count %zu)\n", + __func__, state, (void *)set, set->count); + } + +#if EXPENSIVE_CHECKS + /* invariant: input is unique and sorted */ + for (size_t i = 1; i < set->count; i++) { + assert(set->groups[i - 1].to < set->groups[i].to); + } +#endif + + if (set->count == 0) { + if (LOG_BSEARCH) { + fprintf(stderr, "%s: empty, returning 0\n", __func__); + } + *dst = 0; + return FSP_FOUND_INSERT_POSITION; + } else { + if (LOG_BSEARCH) { + fprintf(stderr, "%s: fast path: looking for %d, set->groups[last].to %d\n", + __func__, state, set->groups[hi - 1].to); + } + + /* Check the last entry so we can append in constant time. */ + const fsm_state_t last = set->groups[hi - 1].to; + if (state > last) { + *dst = hi; + return FSP_FOUND_INSERT_POSITION; + } else if (state == last) { + *dst = hi - 1; + return FSP_FOUND_VALUE_PRESENT; + } + } + + size_t mid; + while (lo < hi) { /* lo <= mid < hi */ + mid = lo + (hi - lo)/2; /* avoid overflow */ + const struct edge_group *eg = &set->groups[mid]; + const fsm_state_t cur = eg->to; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: lo %zu, hi %zu, mid %zu, cur %d, looking for %d\n", + __func__, lo, hi, mid, cur, state); + } + + if (state == cur) { + *dst = mid; + return FSP_FOUND_VALUE_PRESENT; + } else if (state > cur) { + lo = mid + 1; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: new lo %zd\n", __func__, lo); + } + + /* Update mid if we're about to halt, because we're looking + * for the first position >= state, not the last position <=. */ + if (lo == hi) { + mid = lo; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: special case, updating mid to %zd\n", __func__, mid); + } + } + } else if (state < cur) { + hi = mid; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: new hi %zd\n", __func__, hi); + } + } + } + + if (LOG_BSEARCH) { + fprintf(stderr, "%s: halting at %zd (looking for %d, cur %d)\n", + __func__, mid, state, set->groups[mid].to); + } + + /* dst is now the first position > state (== case is handled above), + * which may be one past the end of the array. */ + assert(mid == set->count || set->groups[mid].to > state); + *dst = mid; + return FSP_FOUND_INSERT_POSITION; +} + +static enum fsp_res +find_state_position_linear(const struct edge_set *set, fsm_state_t state, size_t *dst) +{ + /* Linear search for a group with the same destination + * state, or the position where that group would go. */ + size_t i; + for (i = 0; i < set->count; i++) { + const struct edge_group *eg = &set->groups[i]; + if (eg->to == state) { + *dst = i; + return FSP_FOUND_VALUE_PRESENT; + } else if (eg->to > state) { + break; /* will shift down and insert below */ + } else { + continue; + } + } + + *dst = i; + return FSP_FOUND_INSERT_POSITION; +} + +/* Find the state in the edge set, or where it would be inserted if not present. */ +static enum fsp_res +find_state_position(const struct edge_set *set, fsm_state_t state, size_t *dst) +{ + /* 0: linear, 1: bsearch, -1: call both, to check result */ +#define USE_BSEARCH 1 + + switch (USE_BSEARCH) { + case 0: + return find_state_position_linear(set, state, dst); + case 1: + return find_state_position_bsearch(set, state, dst); + case -1: + { + size_t dst_linear, dst_bsearch; + enum fsp_res res_linear = find_state_position_linear(set, state, &dst_linear); + enum fsp_res res_bsearch = find_state_position_bsearch(set, state, &dst_bsearch); + + if (res_linear != res_bsearch || dst_linear != dst_bsearch) { + fprintf(stderr, "%s: disagreement for state %d: linear res %d, dst %zu, bsearch res %d, dst %zu\n", + __func__, state, + res_linear, dst_linear, + res_bsearch, dst_bsearch); + for (size_t i = 0; i < set->count; i++) { + fprintf(stderr, "set->groups[%zu].to: %d\n", i, set->groups[i].to); + } + } + assert(res_linear == res_bsearch); + assert(dst_linear == dst_bsearch); + *dst = dst_linear; + return res_linear; + } + } +} + int edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, uint64_t symbols[256/64], fsm_state_t state) @@ -228,25 +380,21 @@ edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, state, (void *)set); #endif - /* Linear search for a group with the same destination - * state, or the position where that group would go. */ - for (i = 0; i < set->count; i++) { + switch (find_state_position(set, state, &i)) { + case FSP_FOUND_VALUE_PRESENT: + assert(i < set->count); + /* This API does not indicate whether that + * symbol -> to edge was already present. */ eg = &set->groups[i]; - - if (eg->to == state) { - /* This API does not indicate whether that - * symbol -> to edge was already present. */ - size_t i; - for (i = 0; i < 256/64; i++) { - eg->symbols[i] |= symbols[i]; - } - dump_edge_set(set); - return 1; - } else if (eg->to > state) { - break; /* will shift down and insert below */ - } else { - continue; + for (i = 0; i < 256/64; i++) { + eg->symbols[i] |= symbols[i]; } + dump_edge_set(set); + return 1; + + break; + case FSP_FOUND_INSERT_POSITION: + break; /* continue below */ } /* insert/append at i */ From 7122d2fe739e106be8b422206b15e681027acd11 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:19:24 -0400 Subject: [PATCH 40/51] edgeset: Commit to using binary search. When I run `time ./re -rpcre -C '^[ab]{0,2000}$'` locally for -O3: - linear search: 2.991s - binary search: 1.521s --- src/adt/edgeset.c | 59 +---------------------------------------------- 1 file changed, 1 insertion(+), 58 deletions(-) diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index 818b78493..dd3b8853c 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -194,7 +194,7 @@ enum fsp_res { * which includes the position immediately following the last entry. Return an enum * which indicates whether state is already present. */ static enum fsp_res -find_state_position_bsearch(const struct edge_set *set, fsm_state_t state, size_t *dst) +find_state_position(const struct edge_set *set, fsm_state_t state, size_t *dst) { size_t lo = 0, hi = set->count; if (LOG_BSEARCH) { @@ -279,63 +279,6 @@ find_state_position_bsearch(const struct edge_set *set, fsm_state_t state, size_ return FSP_FOUND_INSERT_POSITION; } -static enum fsp_res -find_state_position_linear(const struct edge_set *set, fsm_state_t state, size_t *dst) -{ - /* Linear search for a group with the same destination - * state, or the position where that group would go. */ - size_t i; - for (i = 0; i < set->count; i++) { - const struct edge_group *eg = &set->groups[i]; - if (eg->to == state) { - *dst = i; - return FSP_FOUND_VALUE_PRESENT; - } else if (eg->to > state) { - break; /* will shift down and insert below */ - } else { - continue; - } - } - - *dst = i; - return FSP_FOUND_INSERT_POSITION; -} - -/* Find the state in the edge set, or where it would be inserted if not present. */ -static enum fsp_res -find_state_position(const struct edge_set *set, fsm_state_t state, size_t *dst) -{ - /* 0: linear, 1: bsearch, -1: call both, to check result */ -#define USE_BSEARCH 1 - - switch (USE_BSEARCH) { - case 0: - return find_state_position_linear(set, state, dst); - case 1: - return find_state_position_bsearch(set, state, dst); - case -1: - { - size_t dst_linear, dst_bsearch; - enum fsp_res res_linear = find_state_position_linear(set, state, &dst_linear); - enum fsp_res res_bsearch = find_state_position_bsearch(set, state, &dst_bsearch); - - if (res_linear != res_bsearch || dst_linear != dst_bsearch) { - fprintf(stderr, "%s: disagreement for state %d: linear res %d, dst %zu, bsearch res %d, dst %zu\n", - __func__, state, - res_linear, dst_linear, - res_bsearch, dst_bsearch); - for (size_t i = 0; i < set->count; i++) { - fprintf(stderr, "set->groups[%zu].to: %d\n", i, set->groups[i].to); - } - } - assert(res_linear == res_bsearch); - assert(dst_linear == dst_bsearch); - *dst = dst_linear; - return res_linear; - } - } -} - int edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, uint64_t symbols[256/64], fsm_state_t state) From 937585a955d277231d558fe5ae20b0cc40c87eda Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 12:08:44 -0400 Subject: [PATCH 41/51] determinise: Drastically reduce calls to qsort. After the other changes in this PR, calls to qsort from `sort_and_dedup_dst_buf` are one of the largest remaining hotspots in the profile. We can often avoid calling qsort, though: - If there is <= 1 entry, just return, it's sorted. - Otherwise, first do a sweep through the array noting the min and max values. Unless there is a huge range between them, it's much faster to build a bitset from them in a small (max 10KB) stack-allocated array and then unpack the bitset (now sorted and unique). Only the needed portion of the array is initialized. I have not done a lot of experimentation to find a cutoff point where the bitset becomes slower than qsort (it may be much larger), I picked 10KB because it's likely to be safe to stack-allocate. I tried changing the bitset unpacking to use an 8 or 16 bit mask and jump forward faster through large sub-word ranges of 0 bits, but any improvement was lost among random variation, so I decided it wasn't worth the extra complexity. We already skip whole words that are 0. --- src/libfsm/determinise.c | 95 ++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 18 deletions(-) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 56e135afd..ac15f05a8 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -2016,28 +2016,87 @@ static void sort_and_dedup_dst_buf(fsm_state_t *buf, size_t *used) { const size_t orig_used = *used; - qsort(buf, orig_used, sizeof(buf[0]), cmp_fsm_state_t); - - /* squash out duplicates */ - size_t rd = 1; - size_t wr = 1; - while (rd < orig_used) { - if (buf[rd - 1] == buf[rd]) { - rd++; /* skip */ - } else { - buf[wr] = buf[rd]; - rd++; - wr++; - } + + if (orig_used <= 1) { + return; /* no change */ } - *used = wr; -#if EXPENSIVE_CHECKS - assert(wr <= orig_used); - for (size_t i = 1; i < *used; i++) { - assert(buf[i - 1] < buf[i]); + /* Figure out what the min and max values are, because + * when the difference between them is not too large it + * can be significantly faster to avoid qsort here. */ + fsm_state_t min = (fsm_state_t)-1; + fsm_state_t max = 0; + for (size_t i = 0; i < orig_used; i++) { + const fsm_state_t cur = buf[i]; + if (cur < min) { min = cur; } + if (cur > max) { max = cur; } } + + /* If there's only one unique value, then we're done. */ + if (min == max) { + buf[0] = min; + *used = 1; + return; + } + +/* 81920 = 10 KB buffer on the stack. This must be divisible by 64. + * Set to 0 to disable. */ +#define QSORT_CUTOFF 81920 + + if (QSORT_CUTOFF == 0 || max - min > QSORT_CUTOFF) { + /* If the bitset would be very large but sparse due to + * extreme values, then fall back on using qsort and + * then sweeping over the array to squash out + * duplicates. */ + qsort(buf, orig_used, sizeof(buf[0]), cmp_fsm_state_t); + + /* squash out duplicates */ + size_t rd = 1; + size_t wr = 1; + while (rd < orig_used) { + if (buf[rd - 1] == buf[rd]) { + rd++; /* skip */ + } else { + buf[wr] = buf[rd]; + rd++; + wr++; + } + } + + *used = wr; +#if EXPENSIVE_CHECKS + assert(wr <= orig_used); + for (size_t i = 1; i < *used; i++) { + assert(buf[i - 1] < buf[i]); + } #endif + } else { + /* Convert the array into a bitset and back, which sorts + * and deduplicates in the process. Add 1 to avoid a zero- + * zero-length array error if QSORT_CUTOFF is 0. */ + uint64_t bitset[QSORT_CUTOFF/64 + 1]; + const size_t words = u64bitset_words(max - min); + memset(bitset, 0x00, words * sizeof(bitset[0])); + + for (size_t i = 0; i < orig_used; i++) { + u64bitset_set(bitset, buf[i] - min); + } + + size_t dst = 0; + for (size_t i = 0; i < words; i++) { + const uint64_t w = bitset[i]; + if (w != 0) { /* skip empty words */ + uint64_t bit = 0x1; + for (size_t b_i = 0; b_i < 64; b_i++, bit <<= 1) { + if (w & bit) { + buf[dst] = 64*i + b_i + min; + dst++; + } + } + } + } + *used = dst; + } } static int From 30e34ef9a739c78f788753919d68d6925171a0d5 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 12:17:56 -0400 Subject: [PATCH 42/51] edgeset: Remove stale comment. --- src/adt/edgeset.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index dd3b8853c..9658213c8 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -326,8 +326,6 @@ edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, switch (find_state_position(set, state, &i)) { case FSP_FOUND_VALUE_PRESENT: assert(i < set->count); - /* This API does not indicate whether that - * symbol -> to edge was already present. */ eg = &set->groups[i]; for (i = 0; i < 256/64; i++) { eg->symbols[i] |= symbols[i]; From cf6051f26135802784795465232369e5dfc66c66 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 12:46:35 -0400 Subject: [PATCH 43/51] UBSan: Avoid implicit signed/unsigned conversion. --- src/libre/ast_compile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libre/ast_compile.c b/src/libre/ast_compile.c index 502faf8b4..aa0902d55 100644 --- a/src/libre/ast_compile.c +++ b/src/libre/ast_compile.c @@ -208,11 +208,11 @@ addedge_literal(struct comp_env *env, enum re_flags re_flags, assert(to < env->fsm->statecount); if (re_flags & RE_ICASE) { - if (!fsm_addedge_literal(fsm, from, to, tolower((unsigned char) c))) { + if (!fsm_addedge_literal(fsm, from, to, (char)tolower((unsigned char) c))) { return 0; } - if (!fsm_addedge_literal(fsm, from, to, toupper((unsigned char) c))) { + if (!fsm_addedge_literal(fsm, from, to, (char)toupper((unsigned char) c))) { return 0; } } else { From c1e12828b2e79a11b61978d809a0e020f5d8a6e6 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 12:49:22 -0400 Subject: [PATCH 44/51] UBSan: Avoid implicit signed/unsigned conversion. --- src/retest/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/retest/main.c b/src/retest/main.c index b6b4c52f7..e01c93e7c 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -393,7 +393,7 @@ parse_escapes(char *s, char **errpos, int *lenp) ndig++; } else { - s[j++] = ccode; + s[j++] = (char)ccode; st = ST_BARE; if (!hexcurly) { From 6eff0f98ce625e93ba025f0f4c7358153708cc92 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 13:17:12 -0400 Subject: [PATCH 45/51] bugfix: The range is min..max inclusive, so add 1. If min and max are exactly 64 states apart the upper value was getting silently dropped due to an incorrect `words` value here. One of the patterns in the PCRE suite triggers this: ./re -rpcre '(?:c|d)(?:)(?:aaaaaaaa(?:)(?:bbbbbbbb)(?:bbbbbbbb(?:))(?:bbbbbbbb(?:)(?:bbbbbbbb)))' "caaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" This should match, but did not. --- src/libfsm/determinise.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index ac15f05a8..3dbdc429e 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -2075,7 +2075,7 @@ sort_and_dedup_dst_buf(fsm_state_t *buf, size_t *used) * and deduplicates in the process. Add 1 to avoid a zero- * zero-length array error if QSORT_CUTOFF is 0. */ uint64_t bitset[QSORT_CUTOFF/64 + 1]; - const size_t words = u64bitset_words(max - min); + const size_t words = u64bitset_words(max - min + 1); memset(bitset, 0x00, words * sizeof(bitset[0])); for (size_t i = 0; i < orig_used; i++) { From 98ee906d15426150020070db975778bf5f97a565 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 15 Jun 2023 14:00:25 -0400 Subject: [PATCH 46/51] Address a couple warnings from scan-build. determinise: It's not possible to find a cached result in the hash table without allocating a to-set buffer first, so assert that it will be non-NULL. fsm_findmode: This should never be used on a state without edges. vm/v1.c and vm/v2.c: Free allocated return value on error. --- src/libfsm/determinise.c | 2 ++ src/libfsm/mode.c | 4 ++++ src/libfsm/vm/v1.c | 4 +++- src/libfsm/vm/v2.c | 5 ++++- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 3dbdc429e..9c77f3a42 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -1339,6 +1339,7 @@ to_set_htab_check(struct analyze_closures_env *env, if (b->count == 0) { return 0; /* empty bucket -> not found */ } else if (b->count == count) { + assert(env->to_sets.buf != NULL); assert(b->offset + count <= env->to_sets.used); const fsm_state_t *ids = &env->to_sets.buf[b->offset]; if (0 == memcmp(ids, dst, count * sizeof(dst[0]))) { @@ -1465,6 +1466,7 @@ save_to_set(struct analyze_closures_env *env, env->to_sets.ceil = nceil; env->to_sets.buf = nbuf; } + assert(env->to_sets.buf != NULL); #if LOG_TO_SET static size_t to_set_id; diff --git a/src/libfsm/mode.c b/src/libfsm/mode.c index 76c60b8ad..87af0bdf9 100644 --- a/src/libfsm/mode.c +++ b/src/libfsm/mode.c @@ -28,6 +28,7 @@ fsm_findmode(const struct fsm *fsm, fsm_state_t state, unsigned int *freq) } mode; mode.freq = 1; + mode.state = (fsm_state_t)-1; edge_set_group_iter_reset(fsm->states[state].edges, EDGE_GROUP_ITER_ALL, &iter); while (edge_set_group_iter_next(&iter, &info)) { @@ -46,6 +47,9 @@ fsm_findmode(const struct fsm *fsm, fsm_state_t state, unsigned int *freq) *freq = mode.freq; } + /* It's not meaningful to call this on a state without edges. */ + assert(mode.state != (fsm_state_t)-1); + assert(mode.freq >= 1); return mode.state; } diff --git a/src/libfsm/vm/v1.c b/src/libfsm/vm/v1.c index a326b88d8..de1f6ea93 100644 --- a/src/libfsm/vm/v1.c +++ b/src/libfsm/vm/v1.c @@ -217,7 +217,9 @@ encode_opasm_v1(const struct dfavm_vm_op *instr, size_t ninstr, size_t total_byt return ret; error: - /* XXX - cleanup */ + if (ret != NULL) { + free(ret); + } return NULL; } diff --git a/src/libfsm/vm/v2.c b/src/libfsm/vm/v2.c index c85edff98..07eb12ef4 100644 --- a/src/libfsm/vm/v2.c +++ b/src/libfsm/vm/v2.c @@ -155,7 +155,10 @@ encode_opasm_v2(const struct dfavm_vm_op *instr, size_t ninstr) return ret; error: - /* XXX - cleanup */ + if (ret != NULL) { + free(ret); + } + return NULL; } From 51892e3745fd642f6bb8a6acd31b69cf065a43b3 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 16 Feb 2023 13:52:31 -0500 Subject: [PATCH 47/51] Add src/adt/idmap.c, a state -> ID set map. --- Makefile | 1 + include/adt/idmap.h | 58 ++++++ src/adt/Makefile | 1 + src/adt/idmap.c | 392 ++++++++++++++++++++++++++++++++++++++ tests/idmap/Makefile | 19 ++ tests/idmap/idmap_basic.c | 136 +++++++++++++ 6 files changed, 607 insertions(+) create mode 100644 include/adt/idmap.h create mode 100644 src/adt/idmap.c create mode 100644 tests/idmap/Makefile create mode 100644 tests/idmap/idmap_basic.c diff --git a/Makefile b/Makefile index 8d742883e..f1f4f1396 100644 --- a/Makefile +++ b/Makefile @@ -108,6 +108,7 @@ SUBDIR += src SUBDIR += tests/capture SUBDIR += tests/complement SUBDIR += tests/gen +SUBDIR += tests/idmap SUBDIR += tests/intersect #SUBDIR += tests/ir # XXX: fragile due to state numbering SUBDIR += tests/eclosure diff --git a/include/adt/idmap.h b/include/adt/idmap.h new file mode 100644 index 000000000..064fd15d1 --- /dev/null +++ b/include/adt/idmap.h @@ -0,0 +1,58 @@ +#ifndef IDMAP_H +#define IDMAP_H + +/* Mapping between one fsm_state_t and a set of + * unsigned IDs. The implementation assumes that both + * IDs are sequentially assigned and don't need a sparse + * mapping -- it will handle 10 -> [1, 3, 47] well, but + * not 1000000 -> [14, 524288, 1073741823]. */ + +#include + +#include "fsm/fsm.h" +#include "fsm/alloc.h" + +struct idmap; /* Opaque handle. */ + +struct idmap * +idmap_new(const struct fsm_alloc *alloc); + +void +idmap_free(struct idmap *m); + +/* Associate a value with a state (if not already present.) + * Returns 1 on success, or 0 on allocation failure. */ +int +idmap_set(struct idmap *m, fsm_state_t state_id, unsigned value); + +/* How many values are associated with an ID? */ +size_t +idmap_get_value_count(const struct idmap *m, fsm_state_t state_id); + +/* Get the values associated with an ID. + * + * Returns 1 on success and writes them into the buffer, in ascending + * order, with the count in *written (if non-NULL). + * + * Returns 0 on error (insufficient buffer space). */ +int +idmap_get(const struct idmap *m, fsm_state_t state_id, + size_t buf_size, unsigned *buf, size_t *written); + +/* Iterator callback. */ +typedef void +idmap_iter_fun(fsm_state_t state_id, unsigned value, void *opaque); + +/* Iterate over the ID map. State IDs may be yielded out of order, + * values will be in ascending order. */ +void +idmap_iter(const struct idmap *m, + idmap_iter_fun *cb, void *opaque); + +/* Iterate over the values associated with a single state + * (in ascending order). */ +void +idmap_iter_for_state(const struct idmap *m, fsm_state_t state_id, + idmap_iter_fun *cb, void *opaque); + +#endif diff --git a/src/adt/Makefile b/src/adt/Makefile index 05199f2dc..64ad7429f 100644 --- a/src/adt/Makefile +++ b/src/adt/Makefile @@ -2,6 +2,7 @@ SRC += src/adt/alloc.c SRC += src/adt/bitmap.c +SRC += src/adt/idmap.c SRC += src/adt/internedstateset.c SRC += src/adt/priq.c SRC += src/adt/path.c diff --git a/src/adt/idmap.c b/src/adt/idmap.c new file mode 100644 index 000000000..ca169b71e --- /dev/null +++ b/src/adt/idmap.c @@ -0,0 +1,392 @@ +/* + * Copyright 2021 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include "adt/idmap.h" + +#include "adt/alloc.h" +#include "adt/hash.h" +#include "adt/u64bitset.h" + +#include +#include +#include + +#define NO_STATE ((fsm_state_t)-1) + +#define DEF_BUCKET_COUNT 4 + +struct idmap { + const struct fsm_alloc *alloc; + unsigned bucket_count; + unsigned buckets_used; + + /* All buckets' values are assumed to be large + * enough to store this value, and they will all + * grow as necessary. */ + unsigned max_value; + + /* Basic linear-probing, add-only hash table. */ + struct idmap_bucket { + fsm_state_t state; /* Key. NO_STATE when empty. */ + + /* values[] is always either NULL or has at least + * max_value + 1 bits; all grow on demand. */ + uint64_t *values; + } *buckets; +}; + +static unsigned +value_words(unsigned max_value) { + if (max_value == 0) { + /* Still allocate one word, for storing 0. */ + return 1; + } else { + return u64bitset_words(max_value); + } +} + +struct idmap * +idmap_new(const struct fsm_alloc *alloc) +{ + struct idmap *res = NULL; + struct idmap_bucket *buckets = NULL; + + res = f_malloc(alloc, sizeof(*res)); + if (res == NULL) { + goto cleanup; + } + + buckets = f_calloc(alloc, + DEF_BUCKET_COUNT, sizeof(buckets[0])); + if (buckets == NULL) { + goto cleanup; + } + + for (size_t i = 0; i < DEF_BUCKET_COUNT; i++) { + buckets[i].state = NO_STATE; + } + + res->alloc = alloc; + res->buckets_used = 0; + res->bucket_count = DEF_BUCKET_COUNT; + res->max_value = 0; + res->buckets = buckets; + + return res; + +cleanup: + f_free(alloc, res); + f_free(alloc, buckets); + return NULL; +} + +void +idmap_free(struct idmap *m) +{ + if (m == NULL) { + return; + } + + for (size_t i = 0; i < m->bucket_count; i++) { + if (m->buckets[i].state == NO_STATE) { + continue; + } + f_free(m->alloc, m->buckets[i].values); + } + + f_free(m->alloc, m->buckets); + f_free(m->alloc, m); +} + +static int +grow_bucket_values(struct idmap *m, unsigned old_words, unsigned new_words) +{ + assert(new_words > old_words); + + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + struct idmap_bucket *b = &m->buckets[b_i]; + if (b->state == NO_STATE) { + assert(b->values == NULL); + continue; + } + + uint64_t *nv = f_calloc(m->alloc, + new_words, sizeof(nv[0])); + if (nv == NULL) { + return 0; + } + + for (size_t w_i = 0; w_i < old_words; w_i++) { + nv[w_i] = b->values[w_i]; + } + f_free(m->alloc, b->values); + b->values = nv; + } + return 1; +} + +static int +grow_buckets(struct idmap *m) +{ + const size_t ocount = m->bucket_count; + const size_t ncount = 2*ocount; + assert(ncount > m->bucket_count); + + struct idmap_bucket *nbuckets = f_calloc(m->alloc, + ncount, sizeof(nbuckets[0])); + if (nbuckets == NULL) { + return 0; + } + for (size_t nb_i = 0; nb_i < ncount; nb_i++) { + nbuckets[nb_i].state = NO_STATE; + } + + const size_t nmask = ncount - 1; + + for (size_t ob_i = 0; ob_i < ocount; ob_i++) { + const struct idmap_bucket *ob = &m->buckets[ob_i]; + if (ob->state == NO_STATE) { + continue; + } + + const uint64_t h = hash_id(ob->state); + for (size_t nb_i = 0; nb_i < ncount; nb_i++) { + struct idmap_bucket *nb = &nbuckets[(h + nb_i) & nmask]; + if (nb->state == NO_STATE) { + nb->state = ob->state; + nb->values = ob->values; + break; + } else { + assert(nb->state != ob->state); + /* collision */ + continue; + } + } + } + + f_free(m->alloc, m->buckets); + + m->buckets = nbuckets; + m->bucket_count = ncount; + + return 1; +} + +int +idmap_set(struct idmap *m, fsm_state_t state_id, + unsigned value) +{ + assert(state_id != NO_STATE); + + const uint64_t h = hash_id(state_id); + if (value > m->max_value) { + const unsigned ovw = value_words(m->max_value); + const unsigned nvw = value_words(value); + /* If this value won't fit in the existing value + * arrays, then grow them all. We do not track the + * number of bits in each individual array. */ + if (nvw > ovw && !grow_bucket_values(m, ovw, nvw)) { + return 0; + } + m->max_value = value; + } + + assert(m->max_value >= value); + + if (m->buckets_used >= m->bucket_count/2) { + if (!grow_buckets(m)) { + return 0; + } + } + + const uint64_t mask = m->bucket_count - 1; + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + struct idmap_bucket *b = &m->buckets[(h + b_i) & mask]; + if (b->state == state_id) { + assert(b->values != NULL); + u64bitset_set(b->values, value); + return 1; + } else if (b->state == NO_STATE) { + b->state = state_id; + assert(b->values == NULL); + + const unsigned vw = value_words(m->max_value); + b->values = f_calloc(m->alloc, + vw, sizeof(b->values[0])); + if (b->values == NULL) { + return 0; + } + m->buckets_used++; + + u64bitset_set(b->values, value); + return 1; + } else { + continue; /* collision */ + } + + } + + assert(!"unreachable"); + return 0; +} + +static const struct idmap_bucket * +get_bucket(const struct idmap *m, fsm_state_t state_id) +{ + const uint64_t h = hash_id(state_id); + const uint64_t mask = m->bucket_count - 1; + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + const struct idmap_bucket *b = &m->buckets[(h + b_i) & mask]; + if (b->state == NO_STATE) { + return NULL; + } else if (b->state == state_id) { + return b; + } + } + + return NULL; +} + +size_t +idmap_get_value_count(const struct idmap *m, fsm_state_t state_id) +{ + const struct idmap_bucket *b = get_bucket(m, state_id); + if (b == NULL) { + return 0; + } + assert(b->values != NULL); + + size_t res = 0; + const size_t words = value_words(m->max_value); + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + /* This could use popcount64(w). */ + if (w == 0) { + continue; + } + for (uint64_t bit = 1; bit; bit <<= 1) { + if (w & bit) { + res++; + } + } + } + + return res; +} + +int +idmap_get(const struct idmap *m, fsm_state_t state_id, + size_t buf_size, unsigned *buf, size_t *written) +{ + const struct idmap_bucket *b = get_bucket(m, state_id); + if (b == NULL) { + if (written != NULL) { + *written = 0; + } + return 1; + } + + size_t buf_offset = 0; + const size_t words = value_words(m->max_value); + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + if (w == 0) { + continue; + } + + for (uint64_t b_i = 0; b_i < 64; b_i++) { + if (w & ((uint64_t)1 << b_i)) { + if (buf_offset * sizeof(buf[0]) >= buf_size) { + return 0; + } + buf[buf_offset] = 64*w_i + b_i; + buf_offset++; + } + } + } + + if (written != NULL) { + *written = buf_offset; + } + return 1; +} + +void +idmap_iter(const struct idmap *m, + idmap_iter_fun *cb, void *opaque) +{ + const size_t words = value_words(m->max_value); + + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + const struct idmap_bucket *b = &m->buckets[b_i]; + if (b->state == NO_STATE) { + continue; + } + + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + if (w == 0) { + continue; + } + for (uint64_t b_i = 0; b_i < 64; b_i++) { + if (w & ((uint64_t)1 << b_i)) { + const unsigned v = 64*w_i + b_i; + cb(b->state, v, opaque); + } + } + } + } +} + +void +idmap_iter_for_state(const struct idmap *m, fsm_state_t state_id, + idmap_iter_fun *cb, void *opaque) +{ + const size_t words = value_words(m->max_value); + const struct idmap_bucket *b = get_bucket(m, state_id); + if (b == NULL) { + return; + } + + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + if (w == 0) { + continue; + } + /* if N contiguous bits are all zero, skip them all at once */ +#define BLOCK_BITS 16 + uint64_t block = ((uint64_t)1 << BLOCK_BITS) - 1; + size_t block_count = 0; + + uint64_t b_i = 0; + while (b_i < 64) { + if ((w & block) == 0) { + block <<= BLOCK_BITS; + b_i += BLOCK_BITS; + continue; + } + + if (w & ((uint64_t)1 << b_i)) { + const unsigned v = 64*w_i + b_i; + cb(b->state, v, opaque); + block_count++; + } + b_i++; + block <<= 1; + } + +#define CHECK 0 +#if CHECK + size_t check_count = 0; + for (uint64_t b_i = 0; b_i < 64; b_i++) { + if (w & ((uint64_t)1 << b_i)) { + check_count++; + } + } + assert(block_count == check_count); +#endif + } +} diff --git a/tests/idmap/Makefile b/tests/idmap/Makefile new file mode 100644 index 000000000..aee01f565 --- /dev/null +++ b/tests/idmap/Makefile @@ -0,0 +1,19 @@ +.include "../../share/mk/top.mk" + +TEST.tests/idmap != ls -1 tests/idmap/idmap*.c +TEST_SRCDIR.tests/idmap = tests/idmap +TEST_OUTDIR.tests/idmap = ${BUILD}/tests/idmap + +.for n in ${TEST.tests/idmap:T:R:C/^idmap//} +INCDIR.${TEST_SRCDIR.tests/idmap}/idmap${n}.c += src/adt +.endfor + +.for n in ${TEST.tests/idmap:T:R:C/^idmap//} +test:: ${TEST_OUTDIR.tests/idmap}/res${n} +SRC += ${TEST_SRCDIR.tests/idmap}/idmap${n}.c +CFLAGS.${TEST_SRCDIR.tests/idmap}/idmap${n}.c += -UNDEBUG -D_DEFAULT_SOURCE -std=c99 +${TEST_OUTDIR.tests/idmap}/run${n}: ${TEST_OUTDIR.tests/idmap}/idmap${n}.o ${BUILD}/lib/adt.o + ${CC} ${CFLAGS} ${CFLAGS.${TEST_SRCDIR.tests/idmap}/idmap${n}.c} -o ${TEST_OUTDIR.tests/idmap}/run${n} ${TEST_OUTDIR.tests/idmap}/idmap${n}.o ${BUILD}/lib/adt.o +${TEST_OUTDIR.tests/idmap}/res${n}: ${TEST_OUTDIR.tests/idmap}/run${n} + ( ${TEST_OUTDIR.tests/idmap}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/idmap}/res${n} +.endfor diff --git a/tests/idmap/idmap_basic.c b/tests/idmap/idmap_basic.c new file mode 100644 index 000000000..19f44d56e --- /dev/null +++ b/tests/idmap/idmap_basic.c @@ -0,0 +1,136 @@ +/* + * Copyright 2021 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include + +#include + +#define DEF_LIMIT 10 +#define DEF_SEED 0 + +/* Thes numbers were chose to get a reasonable variety, + * but also some duplicated values as the input grows. */ +#define MAX_GEN_VALUES 23 +#define ID_MASK ((1 << 9) - 1) +#define VALUE_MASK ((1 << 10) - 1) + +static void +dump_cb(fsm_state_t state_id, unsigned value, void *opaque) +{ + /* fprintf(stderr, " -- state %d, value %u\n", state_id, value); */ + assert(state_id <= ID_MASK); + assert(value <= VALUE_MASK); + (void)opaque; +} + +static int +cmp_u(const void *pa, const void *pb) +{ + const unsigned a = *(unsigned *)pa; + const unsigned b = *(unsigned *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +int main(int argc, char **argv) { + const size_t limit = (argc > 1 ? atoi(argv[1]) : DEF_LIMIT); + const unsigned seed = (argc > 2 ? atoi(argv[2]) : DEF_SEED); + + (void)argc; + (void)argv; + struct idmap *m = idmap_new(NULL); + + srandom(seed); + + /* Fill the table with random data */ + for (size_t id_i = 0; id_i < limit; id_i++) { + const fsm_state_t id = (fsm_state_t)(random() & ID_MASK); + const size_t value_count = random() % MAX_GEN_VALUES; + + for (size_t v_i = 0; v_i < value_count; v_i++) { + const unsigned v = random() & VALUE_MASK; + if (!idmap_set(m, id, v)) { + assert(!"failed to set"); + } + } + } + + idmap_iter(m, dump_cb, NULL); + + srandom(seed); + + size_t got_buf_ceil = MAX_GEN_VALUES; + unsigned *got_buf = malloc(got_buf_ceil * sizeof(got_buf[0])); + assert(got_buf != NULL); + + /* Reset the PRNG and read back the same data. */ + for (size_t id_i = 0; id_i < limit; id_i++) { + const fsm_state_t id = (fsm_state_t)(random() & ID_MASK); + const size_t generated_value_count = random() % MAX_GEN_VALUES; + + /* Note: This can occasionally differ from + * generated_value_count, because the same id or values + * may have been generated more than once. As long as + * all the values match, it's fine. */ + const size_t value_count = idmap_get_value_count(m, id); + + if (value_count > got_buf_ceil) { + size_t nceil = got_buf_ceil; + while (nceil <= value_count) { + nceil *= 2; + } + free(got_buf); + got_buf = malloc(nceil * sizeof(got_buf[0])); + assert(got_buf != NULL); + got_buf_ceil = nceil; + } + + size_t written; + if (!idmap_get(m, id, + got_buf_ceil * sizeof(got_buf[0]), got_buf, + &written)) { + assert(!"failed to get"); + } + assert(written == value_count); + + unsigned gen_buf[MAX_GEN_VALUES]; + + for (size_t v_i = 0; v_i < generated_value_count; v_i++) { + const unsigned v = random() & VALUE_MASK; + gen_buf[v_i] = v; + } + qsort(gen_buf, generated_value_count, sizeof(gen_buf[0]), cmp_u); + + /* Every generated value should appear in the buffer. + * There may be more in the buffer; ignore them. */ + size_t v_i = 0; + for (size_t gen_i = 0; gen_i < generated_value_count; gen_i++) { + int found = 0; + const unsigned gv = gen_buf[gen_i]; + assert(value_count <= got_buf_ceil); + /* got_buf should be sorted, so we can pick up where we left off */ + while (v_i < value_count) { + if (gv == got_buf[v_i]) { + /* Intentionally don't increment v_i on match, + * because gen_buf can repeat values. */ + found = 1; + break; + } + v_i++; + } + if (!found) { + fprintf(stderr, "NOT FOUND: state %d -- value: %u\n", + id, gv); + return EXIT_FAILURE; + } + } + } + + free(got_buf); + idmap_free(m); + return EXIT_SUCCESS; +} From 7c6644f53b94c3be7246a67eaa985b03dfb0d526 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 10 Jul 2023 14:02:31 -0400 Subject: [PATCH 48/51] Remove theft test harness for deleted ADT (ipriq). --- theft/Makefile | 1 - theft/fuzz_adt_ipriq.c | 197 ----------------------------------------- 2 files changed, 198 deletions(-) delete mode 100644 theft/fuzz_adt_ipriq.c diff --git a/theft/Makefile b/theft/Makefile index 0d38d8cfc..921c482a9 100644 --- a/theft/Makefile +++ b/theft/Makefile @@ -6,7 +6,6 @@ SRC += theft/util.c SRC += theft/wrap.c SRC += theft/fuzz_adt_edge_set.c -SRC += theft/fuzz_adt_ipriq.c SRC += theft/fuzz_adt_priq.c SRC += theft/fuzz_capture_string_set.c SRC += theft/fuzz_literals.c diff --git a/theft/fuzz_adt_ipriq.c b/theft/fuzz_adt_ipriq.c deleted file mode 100644 index 1847ef6ce..000000000 --- a/theft/fuzz_adt_ipriq.c +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright 2021 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include "type_info_adt_ipriq.h" - -#include -#include - -struct model { - size_t used; - size_t entries[]; -}; - -static enum ipriq_cmp_res -cmp_size_t(size_t a, size_t b, void *opaque) -{ - (void)opaque; - return a < b ? IPRIQ_CMP_LT : - a > b ? IPRIQ_CMP_GT : IPRIQ_CMP_EQ; -} - -static int exec_add(size_t x, struct model *m, struct ipriq *pq) -{ - if (!ipriq_add(pq, x)) { - return 0; - } - - m->entries[m->used] = x; - m->used++; - return 1; -} - -static int find_min_pos(const struct model *m, size_t *pos) -{ - size_t i; - if (m->used == 0) { - return 0; - } - - size_t res, min; - res = 0; - min = m->entries[0]; - - for (i = 1; i < m->used; i++) { - if (m->entries[i] < min) { - res = i; - min = m->entries[i]; - } - } - *pos = res; - return 1; -} - -static int exec_peek(struct model *m, struct ipriq *pq) -{ - size_t res; - - if (!ipriq_peek(pq, &res)) { - return m->used == 0; - } - - size_t pos; - if (!find_min_pos(m, &pos)) { - assert(!"unreachable (peek)"); - } - - return res == m->entries[pos]; -} - -static int exec_pop(struct model *m, struct ipriq *pq) -{ - size_t res; - - if (!ipriq_pop(pq, &res)) { - return m->used == 0; - } - - size_t pos; - if (!find_min_pos(m, &pos)) { - assert(!"unreachable (pop)"); - } - - if (res != m->entries[pos]) { - return 0; - } - - assert(m->used > 0); - if (pos < m->used - 1) { - m->entries[pos] = m->entries[m->used - 1]; - } - m->used--; - return 1; -} - -static enum theft_trial_res -compare_against_model(const struct ipriq_scenario *scen) -{ - enum theft_trial_res res = THEFT_TRIAL_FAIL; - size_t i; - - struct model *m = malloc(sizeof(*m) - + scen->count * sizeof(m->entries[0])); - if (m == NULL) { - return THEFT_TRIAL_ERROR; - } - m->used = 0; - - struct ipriq *pq = ipriq_new(NULL, cmp_size_t, NULL); - if (pq == NULL) { - return THEFT_TRIAL_ERROR; - } - - for (i = 0; i < scen->count; i++) { - const struct ipriq_op *op = &scen->ops[i]; - - switch (op->t) { - case IPRIQ_OP_ADD: - if (!exec_add(op->u.add.x, m, pq)) { - goto cleanup; - } - break; - - case IPRIQ_OP_PEEK: - if (!exec_peek(m, pq)) { - goto cleanup; - } - break; - - case IPRIQ_OP_POP: - if (!exec_pop(m, pq)) { - goto cleanup; - } - break; - - default: - assert(false); break; - } - } - - res = THEFT_TRIAL_PASS; - -cleanup: - free(m); - - return res; -} - -static enum theft_trial_res -prop_ipriq_model(struct theft *t, void *arg1) -{ - const struct ipriq_scenario *scen = arg1; - (void)t; - return compare_against_model(scen); -} - -static bool -test_ipriq(theft_seed seed, uintptr_t limit) -{ - enum theft_run_res res; - - struct ipriq_hook_env env = { - .tag = 'I', - .limit = limit, - }; - - struct theft_run_config config = { - .name = __func__, - .prop1 = prop_ipriq_model, - .type_info = { &type_info_adt_ipriq }, - .trials = 1000, - .hooks = { - .trial_pre = theft_hook_first_fail_halt, - .env = &env, - }, - .fork = { - .enable = true, - }, - - .seed = seed, - }; - - (void)limit; - - res = theft_run(&config); - printf("%s: %s\n", __func__, theft_run_res_str(res)); - - return res == THEFT_RUN_PASS; -} - -void -register_test_adt_ipriq(void) -{ - reg_test1("adt_ipriq", test_ipriq, 10000); -} From c646868316ac8281ab38c33e2f9dae5cb7a5f7d8 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 2 Jun 2023 11:41:28 -0400 Subject: [PATCH 49/51] Add pcre-anchor test for anchoring edge case. --- tests/pcre-anchor/in81.re | 1 + tests/pcre-anchor/out81.fsm | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 tests/pcre-anchor/in81.re create mode 100644 tests/pcre-anchor/out81.fsm diff --git a/tests/pcre-anchor/in81.re b/tests/pcre-anchor/in81.re new file mode 100644 index 000000000..8b5fad7c3 --- /dev/null +++ b/tests/pcre-anchor/in81.re @@ -0,0 +1 @@ +($x)* \ No newline at end of file diff --git a/tests/pcre-anchor/out81.fsm b/tests/pcre-anchor/out81.fsm new file mode 100644 index 000000000..2cdc2f023 --- /dev/null +++ b/tests/pcre-anchor/out81.fsm @@ -0,0 +1,5 @@ +0 -> 0 ?; +0 -> 1 "\n"; + +start: 0; +end: 0, 1; \ No newline at end of file From 0789d614a22e2302132fb80f1fde4f6a056f793c Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 31 May 2023 17:14:35 -0400 Subject: [PATCH 50/51] fuzz/run_fuzzer: Run single seed file when given as argument. --- fuzz/run_fuzzer | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fuzz/run_fuzzer b/fuzz/run_fuzzer index be8ba1d95..429ffa961 100755 --- a/fuzz/run_fuzzer +++ b/fuzz/run_fuzzer @@ -4,6 +4,8 @@ BUILD=../build FUZZER=${BUILD}/fuzz/fuzzer SEEDS=${BUILD}/fuzz/fuzzer_seeds +ARG=$1 + SECONDS=${SECONDS:-60} WORKERS=${WORKERS:-4} SEEDS=${SEEDS:-seeds} @@ -25,5 +27,9 @@ if [ ! -d "${SEEDS}" ]; then mkdir -p "${SEEDS}" fi -echo "\n==== ${FUZZER}" -${FUZZER} -jobs=${WORKERS} -workers=${WORKERS} -max_total_time=${SECONDS} ${SEEDS} +if [ -z "${ARG}" ]; then + echo "\n==== ${FUZZER}" + exec ${FUZZER} -jobs=${WORKERS} -workers=${WORKERS} -max_total_time=${SECONDS} ${SEEDS} +else + exec ${FUZZER} ${ARG} +fi \ No newline at end of file From 1ca3726322abed8c02f7f16bffe8b32683189605 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 12 Sep 2023 12:14:20 +0100 Subject: [PATCH 51/51] Don't purge the seed cache for PRs syncing clones. --- .github/workflows/ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 429ec2523..27fdf78f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -409,8 +409,15 @@ jobs: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} - # note we do the fuzzing unconditionally; each run adds to the corpus + # note we do the fuzzing unconditionally; each run adds to the corpus. + # + # We only run fuzzing for PRs in the base repo, this prevents attempting + # to purge the seed cache from a PR syncing a forked repo, which fails + # due to a permissions error (I'm unsure why, I think PRs from clones can't + # purge a cache in CI presumably for security/DoS reasons). PRs from clones + # still run fuzzing, just from empty, and do not save their seeds. - name: Restore seeds (mode ${{ matrix.mode }}) + if: github.repository == 'katef/libfsm' uses: actions/cache/restore@v3 id: cache-seeds with: