Merge pull request katef#73 from katef/theft-seed

Expose theft's seed to the fuzzing CLI

Makefile

@@ -45,6 +45,7 @@ SUBDIR += tests/glob
 SUBDIR += tests/like
 SUBDIR += tests/literal
 SUBDIR += tests/minimise
+SUBDIR += tests/pcre
 SUBDIR += tests/reverse
 SUBDIR += tests/union
 SUBDIR += tests/set

fuzz/Makefile

@@ -9,7 +9,6 @@ SRC += fuzz/fuzz_adt_priq.c
 SRC += fuzz/fuzz_adt_set.c
 SRC += fuzz/fuzz_literals.c
 SRC += fuzz/fuzz_nfa.c
-SRC += fuzz/fuzz_nfa_slow.c
 SRC += fuzz/fuzz_re_parser.c
 SRC += fuzz/fuzz_re_parser_literal.c
 SRC += fuzz/fuzz_re_parser_pcre.c
@@ -30,22 +29,10 @@ PROPNAME.fuzz_libfsm += adt
 PROPNAME.fuzz_libfsm += nfa_edge_order_should_not_matter
 PROPNAME.fuzz_libfsm += nfa_minimise_should_not_add_states
 PROPNAME.fuzz_libfsm += nfa_operations_should_not_impact_matching
-PROPNAME.fuzz_libfsm += nfa_regress_minimise_false_positive
-PROPNAME.fuzz_libfsm += nfa_regress_slow_determinise
-# PROPNAME.fuzz_libfsm += nfa_slow_determinise # TODO: too slow for now
 
 PROPNAME.fuzz_libfsm += re_parser_literal
 PROPNAME.fuzz_libfsm += re_parser_pcre
-
 PROPNAME.fuzz_libfsm += re_pcre_minimize
-# PROPNAME.fuzz_libfsm += re_regress1
-# PROPNAME.fuzz_libfsm += re_regress2
-# PROPNAME.fuzz_libfsm += re_regress3
-# PROPNAME.fuzz_libfsm += re_regress4
-# PROPNAME.fuzz_libfsm += re_regress5
-# PROPNAME.fuzz_libfsm += re_regress6
-# PROPNAME.fuzz_libfsm += re_regress_aBx
-PROPNAME.fuzz_libfsm += re_regress
 
 PROPNAME.fuzz_libfsm += union_literals
 

fuzz/fuzz_adt_priq.c

@@ -67,7 +67,7 @@ repeat_with_verbose(const struct theft_hook_trial_post_info *info,
 }
 
 static bool
-test_priq_operations(uintptr_t limit)
+test_priq_operations(uintptr_t limit, theft_seed seed)
 {
 	enum theft_run_res res;
 
@@ -120,7 +120,7 @@ test_priq_operations(uintptr_t limit)
 			.enable = true,
 		},
 
-		.seed = theft_seed_of_time(),
+		.seed = seed,
 		.always_seed_count = sizeof always_seeds / sizeof *always_seeds,
 		.always_seeds = always_seeds
 	};

fuzz/fuzz_adt_set.c

@@ -50,7 +50,7 @@ repeat_with_verbose(const struct theft_hook_trial_post_info *info,
 }
 
 static bool
-test_set_operations(uintptr_t limit2)
+test_set_operations(theft_seed seed, uintptr_t limit2)
 {
 	enum theft_run_res res;
 
@@ -68,7 +68,7 @@ test_set_operations(uintptr_t limit2)
 			.trial_post = repeat_with_verbose,
 			.env = &env,
 		},
-		.seed = theft_seed_of_time(),
+		.seed = seed,
 		.trials = 100000,
 		.fork = {
 			.enable = true,
@@ -83,7 +83,7 @@ test_set_operations(uintptr_t limit2)
 }
 
 static bool
-test_set_equality(void)
+test_set_equality(theft_seed seed)
 {
 	enum theft_run_res res;
 
@@ -100,7 +100,7 @@ test_set_equality(void)
 			.trial_post = repeat_with_verbose,
 			.env = &env,
 		},
-		.seed = theft_seed_of_time(),
+		.seed = seed,
 		.trials = 10000,
 		.fork = {
 			.enable = true,
@@ -116,7 +116,8 @@ test_set_equality(void)
 	return res == THEFT_RUN_PASS;
 }
 
-static enum theft_trial_res prop_set_equality(struct theft *t, void *arg1)
+static enum
+theft_trial_res prop_set_equality(struct theft *t, void *arg1)
 {
 	struct set_hook_env *env;
 	struct set_sequence *seq = arg1;

fuzz/fuzz_libfsm.h

@@ -50,8 +50,8 @@ struct string_pair {
 void print_or_hexdump(FILE *f, const uint8_t *buf, size_t size);
 void hexdump(FILE *f, const uint8_t *buf, size_t size);
 
-typedef bool test_fun(void);
-typedef bool test_fun1(uintptr_t arg);
+typedef bool test_fun(theft_seed);
+typedef bool test_fun1(theft_seed, uintptr_t arg);
 
 void reg_test(const char *name, test_fun *test);
 void reg_test1(const char *name, test_fun1 *test, uintptr_t arg);
@@ -75,15 +75,6 @@ enum theft_hook_trial_post_res
 trial_post_inc_verbosity(const struct theft_hook_trial_post_info *info,
     void *penv);
 
-bool test_re_parser_literal(uint8_t verbosity,
-    const uint8_t *re, size_t re_size,
-    size_t count, const struct string_pair *pairs);
-
-bool test_re_parser_pcre(uint8_t verbosity,
-    const uint8_t *re_string, size_t re_size,
-    size_t pos_count, const struct string_pair *pos_pairs,
-    size_t neg_count, const struct string_pair *neg_pairs);
-
 size_t delta_msec(const struct timeval *pre, const struct timeval *post);
 
 size_t *gen_permutation_vector(size_t length, uint32_t seed);

fuzz/fuzz_literals.c

@@ -25,7 +25,7 @@ static const struct fsm_options opt = {
 };
 
 static bool
-test_union_literals(void)
+test_union_literals(theft_seed seed)
 {
 	enum theft_run_res res;
 	struct theft_type_info arg_info;
@@ -46,7 +46,7 @@ test_union_literals(void)
 			.env = &env,
 		},
 
-		.seed = theft_seed_of_time(),
+		.seed = seed,
 		.fork = {
 			.enable = true,
 		},

fuzz/fuzz_nfa.c

@@ -9,9 +9,6 @@
 #include "type_info_nfa.h"
 #include "type_info_fsm_literal.h"
 
-/* In test_nfa_slow.c */
-bool test_nfa_regress_slow_determinise(void);
-
 static struct fsm *
 nfa_of_spec(struct nfa_spec *spec, bool shuffle);
 
@@ -25,8 +22,6 @@ static enum theft_trial_res
 prop_nfa_edge_order_should_not_matter(struct theft *t,
 	void *arg1);
 static enum theft_trial_res
-prop_slow_determinise(struct theft *t, void *arg1);
-static enum theft_trial_res
 prop_nfa_minimise_should_not_add_states(struct theft *t,
 	void *arg1);
 
@@ -70,7 +65,7 @@ shrink_pre(const struct theft_hook_shrink_pre_info *info,
 }
 
 static bool
-nfa_operations_should_not_impact_matching(void)
+nfa_operations_should_not_impact_matching(theft_seed seed)
 {
 	enum theft_run_res res;
 
@@ -98,7 +93,7 @@ nfa_operations_should_not_impact_matching(void)
 			.env = &env,
 		},
 
-		.seed = theft_seed_of_time(),
+		.seed = seed,
 		.fork = {
 			.enable = true,
 			//.timeout = 1000,
@@ -402,7 +397,8 @@ nfa_of_spec(struct nfa_spec *spec, bool shuffle)
 	return nfa;
 }
 
-static bool nfa_edge_order_should_not_matter(void)
+static bool
+nfa_edge_order_should_not_matter(theft_seed seed)
 {
 	enum theft_run_res res;
 
@@ -426,7 +422,7 @@ static bool nfa_edge_order_should_not_matter(void)
 			.env = &env,
 		},
 
-		.seed = theft_seed_of_time(),
+		.seed = seed,
 		.fork = {
 			.enable = true,
 		},
@@ -493,146 +489,7 @@ prop_nfa_edge_order_should_not_matter(struct theft *t,
 }
 
 static bool
-nfa_regress_minimise_false_positive(void)
-{
-	struct fsm *nfa;
-	struct fsm_state *st;
-	bool match;
-#define STATE_COUNT 2
-	struct fsm_state *states[STATE_COUNT];
-	uint8_t str[] = { 0x00 };
-	struct string_pair pair = {
-		.string = str,
-		.size   = sizeof str
-	};
-
-	nfa = fsm_new(&test_nfa_fsm_options);
-	assert(nfa); /* XXX */
-
-	for (size_t i = 0; i < STATE_COUNT; i++) {
-		states[i] = fsm_addstate(nfa);
-		assert(states[i]); /* XXX */
-	}
-
-	fsm_setstart(nfa, states[0]);
-
-	if (!fsm_addedge_any(nfa, states[1], states[1])) {
-		assert(false);
-	}
-
-	fsm_setend(nfa, states[STATE_COUNT - 1], 1);
-#undef STATE_COUNT
-
-	printf("BEFORE fsm_minimise\n");
-	fsm_print(nfa, stdout, FSM_OUT_DOT);
-
-	if (!fsm_minimise(nfa)) {
-		assert(false);
-	}
-
-	printf("AFTER fsm_minimise\n");
-	fsm_print(nfa, stdout, FSM_OUT_DOT);
-
-	st = wrap_fsm_exec(nfa, &pair);
-
-	match = (st != NULL);
-
-	fsm_free(nfa);
-
-	return !match;
-}
-
-static bool
-nfa_slow_determinise(void)
-{
-	enum theft_run_res res;
-
-	//seed = 0xe0083991257118b0LLU;
-
-	struct test_env env = {
-		.tag = 'E',
-		.verbosity = 0,
-		.shrink_timeout = 60,
-	};
-
-	struct theft_run_config cfg = {
-		.name = __func__,
-		.prop1 = prop_slow_determinise,
-		.type_info = { &type_info_nfa, },
-		.hooks = {
-			.trial_pre = trial_pre_fail_once,
-			.trial_post = trial_post_inc_verbosity,
-			.shrink_pre = shrink_pre,
-			.env = &env,
-		},
-		.seed = theft_seed_of_time(),
-		.fork = {
-			.enable = true,
-		},
-	};
-
-	res = theft_run(&cfg);
-
-	return res == THEFT_RUN_PASS;
-}
-
-static enum theft_trial_res
-prop_slow_determinise(struct theft *t, void *arg1)
-{
-	struct nfa_spec *nfa_spec = arg1;
-	enum theft_trial_res res;
-	struct timeval pre, post;
-	struct test_env *env;
-	struct fsm *nfa;
-	uint8_t verbosity;
-	size_t elapsed_msec;
-
-	env = theft_hook_get_env(t);
-	assert(env->tag == 'E');
-	verbosity = env->verbosity;
-
-	assert(nfa_spec->tag == 'N');
-
-	if (nfa_spec->state_count == 0) {
-		return THEFT_TRIAL_SKIP;
-	}
-
-	nfa = nfa_of_spec(nfa_spec, false);
-	if (nfa == NULL) {
-		return THEFT_TRIAL_ERROR;
-	}
-
-	if (-1 == gettimeofday(&pre, NULL)) {
-		assert(false);
-		return false;
-	}
-	if (!fsm_determinise(nfa)) {
-		fprintf(stdout, "FAIL: determinise\n");
-		return THEFT_TRIAL_ERROR;
-	}
-	if (-1 == gettimeofday(&post, NULL)) {
-		assert(false);
-		return false;
-	}
-	elapsed_msec = 1000 * (post.tv_sec - pre.tv_sec)
-		+ (post.tv_usec / 1000 - pre.tv_usec / 1000);
-
-	if (verbosity > 0) {
-		fprintf(stdout, "%s: fsm_determinise took %zd msec\n",
-			__func__, elapsed_msec);
-	}
-
-	res = elapsed_msec < 2000
-		? THEFT_TRIAL_PASS
-		: THEFT_TRIAL_FAIL;
-
-	fsm_free(nfa);
-
-	return res;
-}
-
-static bool
-nfa_minimise_should_not_add_states(void)
+nfa_minimise_should_not_add_states(theft_seed seed)
 {
 	enum theft_run_res res;
 
@@ -652,7 +509,7 @@ nfa_minimise_should_not_add_states(void)
 			.shrink_pre = shrink_pre,
 			.env = &env,
 		},
-		.seed = theft_seed_of_time(),
+		.seed = seed,
 		.fork = {
 			.enable = true,
 		},
@@ -716,13 +573,7 @@ register_test_nfa(void)
 		nfa_operations_should_not_impact_matching);
 	reg_test("nfa_edge_order_should_not_matter",
 		nfa_edge_order_should_not_matter);
-	reg_test("nfa_regress_minimise_false_positive",
-		nfa_regress_minimise_false_positive);
-	reg_test("nfa_slow_determinise",
-		nfa_slow_determinise);
-	reg_test("nfa_regress_slow_determinise",
-		test_nfa_regress_slow_determinise);
-
-	reg_test("nfa_minimise_should_not_add_states", nfa_minimise_should_not_add_states);
+	reg_test("nfa_minimise_should_not_add_states",
+		nfa_minimise_should_not_add_states);
 }