99-test_fuzz.t: Clean up and re-organize such that sub-tests could be…

… split easily

Reviewed-by: Richard Levitte <[email protected]>
(Merged from openssl#12359)

fuzz/README.md

@@ -99,7 +99,7 @@ Reproducing issues
 
 If a fuzzer generates a reproducible error, you can reproduce the problem using
 the fuzz/*-test binaries and the file generated by the fuzzer. They binaries
-don't need to be build for fuzzing, there is no need to set CC or the call
+don't need to be built for fuzzing, there is no need to set CC or the call
 config with enable-fuzz-* or -fsanitize-coverage, but some of the other options
 above might be needed. For instance the enable-asan or enable-ubsan option might
 be useful to show you when the problem happens. For the client and server fuzzer
@@ -110,6 +110,20 @@ To reproduce the crash you can run:
 
     fuzz/$FUZZER-test $file
 
+To do all the tests of a specific fuzzer such as asn1 you can run
+
+    fuzz/asn1-test fuzz/corpora/asn1
+or
+    make test TESTS=fuzz_test FUZZ_TESTS=asn1
+
+To run several fuzz tests you can use for instance:
+
+    make test TESTS=test_fuzz FUZZ_TESTS="cmp cms"
+
+To run all fuzz tests you can use:
+
+    make test TESTS=test_fuzz
+
 Random numbers
 --------------
 

test/README.md

@@ -121,6 +121,10 @@ Run all tests in test groups 80 to 99 except for tests in group 90:
 
     $ make TESTS='[89]? -90' test
 
+To run specific fuzz tests you can use for instance:
+
+    $ make test TESTS=test_fuzz FUZZ_TESTS="cmp cms"
+
 To stochastically verify that the algorithm that produces uniformly distributed
 random numbers is operating correctly (with a false positive rate of 0.01%):
 

test/recipes/99-test_fuzz.t

@@ -9,35 +9,30 @@
 use strict;
 use warnings;
 
-use OpenSSL::Glob;
 use OpenSSL::Test qw/:DEFAULT srctop_file/;
 use OpenSSL::Test::Utils;
 
 setup("test_fuzz");
 
-my @fuzzers = ('asn1', 'asn1parse', 'bignum', 'bndiv', 'client', 'conf', 'crl', 'server', 'x509');
-if (!disabled("cmp")) {
-    push @fuzzers, 'cmp';
+my @fuzzers = ();
+@fuzzers = split /\s+/, $ENV{FUZZ_TESTS} if $ENV{FUZZ_TESTS};
+
+if (!@fuzzers) {
+    @fuzzers = (
+        # those commented here as very slow could be moved to separate runs
+        'asn1', # very slow
+        'asn1parse', 'bignum', 'bndiv', 'conf','crl',
+        'client', # very slow
+        'server', # very slow
+        'x509'
+        );
+    push @fuzzers, 'cmp' if !disabled("cmp");
+    push @fuzzers, 'cms' if !disabled("cms");
+    push @fuzzers, 'ct' if !disabled("ct");
 }
-if (!disabled("cms")) {
-    push @fuzzers, 'cms';
-}
-if (!disabled("ct")) {
-    push @fuzzers, 'ct';
-}
-plan tests => scalar @fuzzers;
 
-foreach my $f (@fuzzers) {
-    subtest "Fuzzing $f" => sub {
-        my @dirs = glob(srctop_file('fuzz', 'corpora', $f));
-        push @dirs, glob(srctop_file('fuzz', 'corpora', "$f-*"));
+plan tests => scalar @fuzzers + 1; # one more due to below require_ok(...)
 
-        plan skip_all => "No corpora for $f-test" unless @dirs;
+require_ok(srctop_file('test','recipes','fuzz.pl'));
 
-        plan tests => scalar @dirs;
-
-        foreach (@dirs) {
-            ok(run(fuzz(["$f-test", $_])));
-        }
-    }
-}
+&fuzz_tests(@fuzzers);

test/recipes/fuzz.pl

@@ -0,0 +1,31 @@
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Glob;
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+
+sub fuzz_tests {
+    my @fuzzers = @_;
+
+    foreach my $f (@fuzzers) {
+        subtest "Fuzzing $f" => sub {
+            my @dir = glob(srctop_file('fuzz', 'corpora', "$f"));
+
+            plan skip_all => "No directory fuzz/corpora/$f" unless @dir;
+            plan tests => scalar @dir; # likely 1
+
+            foreach (@dir) {
+                ok(run(fuzz(["$f-test", $_])));
+            }
+        }
+    }
+}
+
+1;