From a7da4d488d55f68de50a96bd3027cd9fc650d444 Mon Sep 17 00:00:00 2001 From: Nicola Tuveri Date: Wed, 4 Nov 2020 15:39:42 +0200 Subject: [PATCH] [test/recipes] Split test_fuzz into separate recipes When using `HARNESS_JOBS` to run the tests in parallel, no matter the level of parallelism that can be used, the monolithic `test_fuzz` takes a long time to run, conditioning the duration of the whole build. This commit splits the single `test_fuzz` recipe into separate recipes for each fuzzer. The previous mechanism to select individual fuzz tests using the `FUZZ_TESTS` environment variable is also dropped (and documentation updated). Reviewed-by: Richard Levitte (Merged from https://github.com/openssl/openssl/pull/13307) --- fuzz/README.md | 6 ++--- test/README.md | 6 ++--- test/recipes/99-test_fuzz.t | 38 --------------------------- test/recipes/99-test_fuzz_asn1.t | 22 ++++++++++++++++ test/recipes/99-test_fuzz_asn1parse.t | 22 ++++++++++++++++ test/recipes/99-test_fuzz_bignum.t | 22 ++++++++++++++++ test/recipes/99-test_fuzz_bndiv.t | 22 ++++++++++++++++ test/recipes/99-test_fuzz_client.t | 22 ++++++++++++++++ test/recipes/99-test_fuzz_cmp.t | 25 ++++++++++++++++++ test/recipes/99-test_fuzz_cms.t | 25 ++++++++++++++++++ test/recipes/99-test_fuzz_conf.t | 22 ++++++++++++++++ test/recipes/99-test_fuzz_crl.t | 22 ++++++++++++++++ test/recipes/99-test_fuzz_ct.t | 25 ++++++++++++++++++ test/recipes/99-test_fuzz_server.t | 22 ++++++++++++++++ test/recipes/99-test_fuzz_x509.t | 22 ++++++++++++++++ test/recipes/fuzz.pl | 21 ++++++--------- 16 files changed, 287 insertions(+), 57 deletions(-) delete mode 100644 test/recipes/99-test_fuzz.t create mode 100644 test/recipes/99-test_fuzz_asn1.t create mode 100644 test/recipes/99-test_fuzz_asn1parse.t create mode 100644 test/recipes/99-test_fuzz_bignum.t create mode 100644 test/recipes/99-test_fuzz_bndiv.t create mode 100644 test/recipes/99-test_fuzz_client.t create mode 100644 test/recipes/99-test_fuzz_cmp.t create mode 100644 test/recipes/99-test_fuzz_cms.t create mode 100644 test/recipes/99-test_fuzz_conf.t create mode 100644 test/recipes/99-test_fuzz_crl.t create mode 100644 test/recipes/99-test_fuzz_ct.t create mode 100644 test/recipes/99-test_fuzz_server.t create mode 100644 test/recipes/99-test_fuzz_x509.t diff --git a/fuzz/README.md b/fuzz/README.md index deb7a43168475..6cc7811ad003e 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -114,15 +114,15 @@ To do all the tests of a specific fuzzer such as asn1 you can run fuzz/asn1-test fuzz/corpora/asn1 or - make test TESTS=fuzz_test FUZZ_TESTS=asn1 + make test TESTS=fuzz_test_asn1 To run several fuzz tests you can use for instance: - make test TESTS=test_fuzz FUZZ_TESTS="cmp cms" + make test TESTS='test_fuzz_cmp test_fuzz_cms' To run all fuzz tests you can use: - make test TESTS=test_fuzz + make test TESTS='test_fuzz_*' Random numbers -------------- diff --git a/test/README.md b/test/README.md index f4f0574aef797..43f8471120edc 100644 --- a/test/README.md +++ b/test/README.md @@ -98,11 +98,11 @@ it's VMS style wildcards) Run all tests except for the fuzz tests: - $ make TESTS=-test_fuzz test + $ make TESTS='-test_fuzz*' test or, if you want to be explicit: - $ make TESTS='alltests -test_fuzz' test + $ make TESTS='alltests -test_fuzz*' test Run all tests that have a name starting with "test_ssl" but not those starting with "test_ssl_": @@ -123,7 +123,7 @@ Run all tests in test groups 80 to 99 except for tests in group 90: To run specific fuzz tests you can use for instance: - $ make test TESTS=test_fuzz FUZZ_TESTS="cmp cms" + $ make test TESTS='test_fuzz_cmp test_fuzz_cms' To stochastically verify that the algorithm that produces uniformly distributed random numbers is operating correctly (with a false positive rate of 0.01%): diff --git a/test/recipes/99-test_fuzz.t b/test/recipes/99-test_fuzz.t deleted file mode 100644 index 8bacad47de487..0000000000000 --- a/test/recipes/99-test_fuzz.t +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env perl -# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. -# -# Licensed under the Apache License 2.0 (the "License"). You may not use -# this file except in compliance with the License. You can obtain a copy -# in the file LICENSE in the source distribution or at -# https://www.openssl.org/source/license.html - -use strict; -use warnings; - -use OpenSSL::Test qw/:DEFAULT srctop_file/; -use OpenSSL::Test::Utils; - -setup("test_fuzz"); - -my @fuzzers = (); -@fuzzers = split /\s+/, $ENV{FUZZ_TESTS} if $ENV{FUZZ_TESTS}; - -if (!@fuzzers) { - @fuzzers = ( - # those commented here as very slow could be moved to separate runs - 'asn1', # very slow - 'asn1parse', 'bignum', 'bndiv', 'conf','crl', - 'client', # very slow - 'server', # very slow - 'x509' - ); - push @fuzzers, 'cmp' if !disabled("cmp"); - push @fuzzers, 'cms' if !disabled("cms"); - push @fuzzers, 'ct' if !disabled("ct"); -} - -plan tests => scalar @fuzzers + 1; # one more due to below require_ok(...) - -require_ok(srctop_file('test','recipes','fuzz.pl')); - -&fuzz_tests(@fuzzers); diff --git a/test/recipes/99-test_fuzz_asn1.t b/test/recipes/99-test_fuzz_asn1.t new file mode 100644 index 0000000000000..41fc541e9e7ea --- /dev/null +++ b/test/recipes/99-test_fuzz_asn1.t @@ -0,0 +1,22 @@ +#!/usr/bin/env perl +# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; +use warnings; + +use OpenSSL::Test qw/:DEFAULT srctop_file/; +use OpenSSL::Test::Utils; + +my $fuzzer = "asn1"; +setup("test_fuzz_${fuzzer}"); + +plan tests => 2; # one more due to below require_ok(...) + +require_ok(srctop_file('test','recipes','fuzz.pl')); + +fuzz_ok($fuzzer); diff --git a/test/recipes/99-test_fuzz_asn1parse.t b/test/recipes/99-test_fuzz_asn1parse.t new file mode 100644 index 0000000000000..8a008bb89b01d --- /dev/null +++ b/test/recipes/99-test_fuzz_asn1parse.t @@ -0,0 +1,22 @@ +#!/usr/bin/env perl +# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; +use warnings; + +use OpenSSL::Test qw/:DEFAULT srctop_file/; +use OpenSSL::Test::Utils; + +my $fuzzer = "asn1parse"; +setup("test_fuzz_${fuzzer}"); + +plan tests => 2; # one more due to below require_ok(...) + +require_ok(srctop_file('test','recipes','fuzz.pl')); + +fuzz_ok($fuzzer); diff --git a/test/recipes/99-test_fuzz_bignum.t b/test/recipes/99-test_fuzz_bignum.t new file mode 100644 index 0000000000000..190c37bc8f980 --- /dev/null +++ b/test/recipes/99-test_fuzz_bignum.t @@ -0,0 +1,22 @@ +#!/usr/bin/env perl +# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; +use warnings; + +use OpenSSL::Test qw/:DEFAULT srctop_file/; +use OpenSSL::Test::Utils; + +my $fuzzer = "bignum"; +setup("test_fuzz_${fuzzer}"); + +plan tests => 2; # one more due to below require_ok(...) + +require_ok(srctop_file('test','recipes','fuzz.pl')); + +fuzz_ok($fuzzer); diff --git a/test/recipes/99-test_fuzz_bndiv.t b/test/recipes/99-test_fuzz_bndiv.t new file mode 100644 index 0000000000000..4932840b7cd0e --- /dev/null +++ b/test/recipes/99-test_fuzz_bndiv.t @@ -0,0 +1,22 @@ +#!/usr/bin/env perl +# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; +use warnings; + +use OpenSSL::Test qw/:DEFAULT srctop_file/; +use OpenSSL::Test::Utils; + +my $fuzzer = "bndiv"; +setup("test_fuzz_${fuzzer}"); + +plan tests => 2; # one more due to below require_ok(...) + +require_ok(srctop_file('test','recipes','fuzz.pl')); + +fuzz_ok($fuzzer); diff --git a/test/recipes/99-test_fuzz_client.t b/test/recipes/99-test_fuzz_client.t new file mode 100644 index 0000000000000..5d147cf9b6027 --- /dev/null +++ b/test/recipes/99-test_fuzz_client.t @@ -0,0 +1,22 @@ +#!/usr/bin/env perl +# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; +use warnings; + +use OpenSSL::Test qw/:DEFAULT srctop_file/; +use OpenSSL::Test::Utils; + +my $fuzzer = "client"; +setup("test_fuzz_${fuzzer}"); + +plan tests => 2; # one more due to below require_ok(...) + +require_ok(srctop_file('test','recipes','fuzz.pl')); + +fuzz_ok($fuzzer); diff --git a/test/recipes/99-test_fuzz_cmp.t b/test/recipes/99-test_fuzz_cmp.t new file mode 100644 index 0000000000000..0fc1d156da06a --- /dev/null +++ b/test/recipes/99-test_fuzz_cmp.t @@ -0,0 +1,25 @@ +#!/usr/bin/env perl +# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; +use warnings; + +use OpenSSL::Test qw/:DEFAULT srctop_file/; +use OpenSSL::Test::Utils; + +my $fuzzer = "cmp"; +setup("test_fuzz_${fuzzer}"); + +plan skip_all => "This test requires $fuzzer support" + if disabled($fuzzer); + +plan tests => 2; # one more due to below require_ok(...) + +require_ok(srctop_file('test','recipes','fuzz.pl')); + +fuzz_ok($fuzzer); diff --git a/test/recipes/99-test_fuzz_cms.t b/test/recipes/99-test_fuzz_cms.t new file mode 100644 index 0000000000000..9c76f46a3daf0 --- /dev/null +++ b/test/recipes/99-test_fuzz_cms.t @@ -0,0 +1,25 @@ +#!/usr/bin/env perl +# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; +use warnings; + +use OpenSSL::Test qw/:DEFAULT srctop_file/; +use OpenSSL::Test::Utils; + +my $fuzzer = "cms"; +setup("test_fuzz_${fuzzer}"); + +plan skip_all => "This test requires $fuzzer support" + if disabled($fuzzer); + +plan tests => 2; # one more due to below require_ok(...) + +require_ok(srctop_file('test','recipes','fuzz.pl')); + +fuzz_ok($fuzzer); diff --git a/test/recipes/99-test_fuzz_conf.t b/test/recipes/99-test_fuzz_conf.t new file mode 100644 index 0000000000000..c96565f259e73 --- /dev/null +++ b/test/recipes/99-test_fuzz_conf.t @@ -0,0 +1,22 @@ +#!/usr/bin/env perl +# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; +use warnings; + +use OpenSSL::Test qw/:DEFAULT srctop_file/; +use OpenSSL::Test::Utils; + +my $fuzzer = "conf"; +setup("test_fuzz_${fuzzer}"); + +plan tests => 2; # one more due to below require_ok(...) + +require_ok(srctop_file('test','recipes','fuzz.pl')); + +fuzz_ok($fuzzer); diff --git a/test/recipes/99-test_fuzz_crl.t b/test/recipes/99-test_fuzz_crl.t new file mode 100644 index 0000000000000..1a5281a072853 --- /dev/null +++ b/test/recipes/99-test_fuzz_crl.t @@ -0,0 +1,22 @@ +#!/usr/bin/env perl +# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; +use warnings; + +use OpenSSL::Test qw/:DEFAULT srctop_file/; +use OpenSSL::Test::Utils; + +my $fuzzer = "crl"; +setup("test_fuzz_${fuzzer}"); + +plan tests => 2; # one more due to below require_ok(...) + +require_ok(srctop_file('test','recipes','fuzz.pl')); + +fuzz_ok($fuzzer); diff --git a/test/recipes/99-test_fuzz_ct.t b/test/recipes/99-test_fuzz_ct.t new file mode 100644 index 0000000000000..bbfb4ace938db --- /dev/null +++ b/test/recipes/99-test_fuzz_ct.t @@ -0,0 +1,25 @@ +#!/usr/bin/env perl +# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; +use warnings; + +use OpenSSL::Test qw/:DEFAULT srctop_file/; +use OpenSSL::Test::Utils; + +my $fuzzer = "ct"; +setup("test_fuzz_${fuzzer}"); + +plan skip_all => "This test requires $fuzzer support" + if disabled($fuzzer); + +plan tests => 2; # one more due to below require_ok(...) + +require_ok(srctop_file('test','recipes','fuzz.pl')); + +fuzz_ok($fuzzer); diff --git a/test/recipes/99-test_fuzz_server.t b/test/recipes/99-test_fuzz_server.t new file mode 100644 index 0000000000000..0d0f021387cad --- /dev/null +++ b/test/recipes/99-test_fuzz_server.t @@ -0,0 +1,22 @@ +#!/usr/bin/env perl +# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; +use warnings; + +use OpenSSL::Test qw/:DEFAULT srctop_file/; +use OpenSSL::Test::Utils; + +my $fuzzer = "server"; +setup("test_fuzz_${fuzzer}"); + +plan tests => 2; # one more due to below require_ok(...) + +require_ok(srctop_file('test','recipes','fuzz.pl')); + +fuzz_ok($fuzzer); diff --git a/test/recipes/99-test_fuzz_x509.t b/test/recipes/99-test_fuzz_x509.t new file mode 100644 index 0000000000000..9a1e3a19cadce --- /dev/null +++ b/test/recipes/99-test_fuzz_x509.t @@ -0,0 +1,22 @@ +#!/usr/bin/env perl +# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; +use warnings; + +use OpenSSL::Test qw/:DEFAULT srctop_file/; +use OpenSSL::Test::Utils; + +my $fuzzer = "x509"; +setup("test_fuzz_${fuzzer}"); + +plan tests => 2; # one more due to below require_ok(...) + +require_ok(srctop_file('test','recipes','fuzz.pl')); + +fuzz_ok($fuzzer); diff --git a/test/recipes/fuzz.pl b/test/recipes/fuzz.pl index 795d85c1dfccb..3f03eef4f7240 100644 --- a/test/recipes/fuzz.pl +++ b/test/recipes/fuzz.pl @@ -9,22 +9,17 @@ use warnings; use OpenSSL::Glob; -use OpenSSL::Test qw/:DEFAULT srctop_file/; +use OpenSSL::Test qw/:DEFAULT srctop_dir/; -sub fuzz_tests { - my @fuzzers = @_; +sub fuzz_ok { + die "Only one argument accepted" if scalar @_ != 1; - foreach my $f (@fuzzers) { - subtest "Fuzzing $f" => sub { - my @dir = glob(srctop_file('fuzz', 'corpora', "$f")); + my $f = $_[0]; + my $d = srctop_dir('fuzz', 'corpora', $f); - plan skip_all => "No directory fuzz/corpora/$f" unless @dir; - plan tests => scalar @dir; # likely 1 - - foreach (@dir) { - ok(run(fuzz(["$f-test", $_]))); - } - } + SKIP: { + skip "No directory $d", 1 unless -d $d; + ok(run(fuzz(["$f-test", $d])), "Fuzzing $f"); } }