From 35d0e5fefc419bddcbe09d7fc163d8cd3417125b Mon Sep 17 00:00:00 2001 From: "djm@openbsd.org" Date: Wed, 17 Oct 2018 23:28:05 +0000 Subject: upstream: add some knobs: UNITTEST_FAST?= no # Skip slow tests (e.g. less intensive fuzzing). UNITTEST_SLOW?= no # Include slower tests (e.g. more intensive fuzzing). UNITTEST_VERBOSE?= no # Verbose test output (inc. per-test names). useful if you want to run the tests as a smoke test to exercise the functionality without waiting for all the fuzzers to run. OpenBSD-Regress-ID: e04d82ebec86068198cd903acf1c67563c57315e --- regress/unittests/Makefile.inc | 18 ++++++- regress/unittests/sshbuf/Makefile | 5 +- regress/unittests/sshbuf/test_sshbuf_fuzz.c | 9 +++- regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c | 12 +++-- regress/unittests/sshkey/Makefile | 4 +- regress/unittests/sshkey/test_fuzz.c | 57 +++++++++++++++------- regress/unittests/test_helper/test_helper.c | 28 +++++++++-- regress/unittests/test_helper/test_helper.h | 4 +- 8 files changed, 103 insertions(+), 34 deletions(-) (limited to 'regress/unittests') diff --git a/regress/unittests/Makefile.inc b/regress/unittests/Makefile.inc index d662a46bc..428ef6836 100644 --- a/regress/unittests/Makefile.inc +++ b/regress/unittests/Makefile.inc @@ -1,10 +1,15 @@ -# $OpenBSD: Makefile.inc,v 1.12 2017/12/21 00:41:22 djm Exp $ +# $OpenBSD: Makefile.inc,v 1.13 2018/10/17 23:28:05 djm Exp $ REGRESS_FAIL_EARLY?= yes .include .include +# User-settable options +UNITTEST_FAST?= no # Skip slow tests (e.g. less intensive fuzzing). +UNITTEST_SLOW?= no # Include slower tests (e.g. more intensive fuzzing). +UNITTEST_VERBOSE?= no # Verbose test output (inc. per-test names). + MALLOC_OPTIONS?= CFGJRSUX TEST_ENV?= MALLOC_OPTIONS=${MALLOC_OPTIONS} @@ -68,3 +73,14 @@ DPADD+= ${LIBUTIL} LDADD+= -lcrypto DPADD+= ${LIBCRYPTO} .endif + +UNITTEST_ARGS?= + +.if (${UNITTEST_VERBOSE:L} != "no") +UNITTEST_ARGS+= -v +.endif +.if (${UNITTEST_FAST:L} != "no") +UNITTEST_ARGS+= -f +.elif (${UNITTEST_SLOW:L} != "no") +UNITTEST_ARGS+= -F +.endif diff --git a/regress/unittests/sshbuf/Makefile b/regress/unittests/sshbuf/Makefile index 81d4f27a6..0e8e9fd10 100644 --- a/regress/unittests/sshbuf/Makefile +++ b/regress/unittests/sshbuf/Makefile @@ -1,4 +1,4 @@ -# $OpenBSD: Makefile,v 1.6 2017/12/21 00:41:22 djm Exp $ +# $OpenBSD: Makefile,v 1.7 2018/10/17 23:28:05 djm Exp $ .include @@ -17,6 +17,5 @@ SRCS+=sshbuf-getput-basic.c sshbuf-getput-crypto.c sshbuf-misc.c sshbuf.c SRCS+=atomicio.c run-regress-${PROG}: ${PROG} - env ${TEST_ENV} ./${PROG} - + env ${TEST_ENV} ./${PROG} ${UNITTEST_ARGS} diff --git a/regress/unittests/sshbuf/test_sshbuf_fuzz.c b/regress/unittests/sshbuf/test_sshbuf_fuzz.c index c52376b53..e236c82f9 100644 --- a/regress/unittests/sshbuf/test_sshbuf_fuzz.c +++ b/regress/unittests/sshbuf/test_sshbuf_fuzz.c @@ -1,4 +1,4 @@ -/* $OpenBSD: test_sshbuf_fuzz.c,v 1.1 2014/04/30 05:32:00 djm Exp $ */ +/* $OpenBSD: test_sshbuf_fuzz.c,v 1.2 2018/10/17 23:28:05 djm Exp $ */ /* * Regress test for sshbuf.h buffer API * @@ -30,10 +30,15 @@ sshbuf_fuzz_tests(void) { struct sshbuf *p1; u_char *dp; - size_t sz, sz2, i; + size_t sz, sz2, i, ntests = NUM_FUZZ_TESTS; u_int32_t r; int ret; + if (test_is_fast()) + ntests >>= 2; + if (test_is_slow()) + ntests <<= 2; + /* NB. uses sshbuf internals */ TEST_START("fuzz alloc/dealloc"); p1 = sshbuf_new(); diff --git a/regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c b/regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c index c6b5c29d1..7c7cb2bfd 100644 --- a/regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c +++ b/regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c @@ -1,4 +1,4 @@ -/* $OpenBSD: test_sshbuf_getput_fuzz.c,v 1.2 2014/05/02 02:54:00 djm Exp $ */ +/* $OpenBSD: test_sshbuf_getput_fuzz.c,v 1.3 2018/10/17 23:28:05 djm Exp $ */ /* * Regress test for sshbuf.h buffer API * @@ -115,11 +115,15 @@ sshbuf_getput_fuzz_tests(void) 0x55, 0x0f, 0x69, 0xd8, 0x0e, 0xc2, 0x3c, 0xd4, }; struct fuzz *fuzz; + u_int fuzzers = FUZZ_1_BIT_FLIP | FUZZ_2_BIT_FLIP | + FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP | + FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END; + + if (test_is_fast()) + fuzzers &= ~(FUZZ_2_BYTE_FLIP|FUZZ_2_BIT_FLIP); TEST_START("fuzz blob parsing"); - fuzz = fuzz_begin(FUZZ_1_BIT_FLIP | FUZZ_2_BIT_FLIP | - FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP | - FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END, blob, sizeof(blob)); + fuzz = fuzz_begin(fuzzers, blob, sizeof(blob)); TEST_ONERROR(onerror, fuzz); for(; !fuzz_done(fuzz); fuzz_next(fuzz)) attempt_parse_blob(blob, sizeof(blob)); diff --git a/regress/unittests/sshkey/Makefile b/regress/unittests/sshkey/Makefile index 1c940bec6..aa731df1c 100644 --- a/regress/unittests/sshkey/Makefile +++ b/regress/unittests/sshkey/Makefile @@ -1,4 +1,4 @@ -# $OpenBSD: Makefile,v 1.5 2017/12/21 00:41:22 djm Exp $ +# $OpenBSD: Makefile,v 1.6 2018/10/17 23:28:05 djm Exp $ PROG=test_sshkey SRCS=tests.c test_sshkey.c test_file.c test_fuzz.c common.c @@ -18,7 +18,7 @@ SRCS+=digest-openssl.c REGRESS_TARGETS=run-regress-${PROG} run-regress-${PROG}: ${PROG} - env ${TEST_ENV} ./${PROG} -d ${.CURDIR}/testdata + env ${TEST_ENV} ./${PROG} ${UNITTEST_ARGS} -d ${.CURDIR}/testdata .include diff --git a/regress/unittests/sshkey/test_fuzz.c b/regress/unittests/sshkey/test_fuzz.c index d3b0c92b4..5953de595 100644 --- a/regress/unittests/sshkey/test_fuzz.c +++ b/regress/unittests/sshkey/test_fuzz.c @@ -1,4 +1,4 @@ -/* $OpenBSD: test_fuzz.c,v 1.8 2017/12/21 00:41:22 djm Exp $ */ +/* $OpenBSD: test_fuzz.c,v 1.9 2018/10/17 23:28:05 djm Exp $ */ /* * Fuzz tests for key parsing * @@ -51,14 +51,16 @@ public_fuzz(struct sshkey *k) struct sshkey *k1; struct sshbuf *buf; struct fuzz *fuzz; + u_int fuzzers = FUZZ_1_BIT_FLIP | FUZZ_1_BYTE_FLIP | + FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END; + if (test_is_fast()) + fuzzers &= ~FUZZ_1_BIT_FLIP; + if (test_is_slow()) + fuzzers |= FUZZ_2_BIT_FLIP | FUZZ_2_BYTE_FLIP; ASSERT_PTR_NE(buf = sshbuf_new(), NULL); ASSERT_INT_EQ(sshkey_putb(k, buf), 0); - /* XXX need a way to run the tests in "slow, but complete" mode */ - fuzz = fuzz_begin(FUZZ_1_BIT_FLIP | /* XXX too slow FUZZ_2_BIT_FLIP | */ - FUZZ_1_BYTE_FLIP | /* XXX too slow FUZZ_2_BYTE_FLIP | */ - FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END, - sshbuf_mutable_ptr(buf), sshbuf_len(buf)); + fuzz = fuzz_begin(fuzzers, sshbuf_mutable_ptr(buf), sshbuf_len(buf)); ASSERT_INT_EQ(sshkey_from_blob(sshbuf_ptr(buf), sshbuf_len(buf), &k1), 0); sshkey_free(k1); @@ -77,12 +79,17 @@ sig_fuzz(struct sshkey *k, const char *sig_alg) struct fuzz *fuzz; u_char *sig, c[] = "some junk to be signed"; size_t l; + u_int fuzzers = FUZZ_1_BIT_FLIP | FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP | + FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END; + + if (test_is_fast()) + fuzzers &= ~FUZZ_2_BYTE_FLIP; + if (test_is_slow()) + fuzzers |= FUZZ_2_BIT_FLIP; ASSERT_INT_EQ(sshkey_sign(k, &sig, &l, c, sizeof(c), sig_alg, 0), 0); ASSERT_SIZE_T_GT(l, 0); - fuzz = fuzz_begin(FUZZ_1_BIT_FLIP | /* too slow FUZZ_2_BIT_FLIP | */ - FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP | - FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END, sig, l); + fuzz = fuzz_begin(fuzzers, sig, l); ASSERT_INT_EQ(sshkey_verify(k, sig, l, c, sizeof(c), NULL, 0), 0); free(sig); TEST_ONERROR(onerror, fuzz); @@ -96,13 +103,15 @@ sig_fuzz(struct sshkey *k, const char *sig_alg) fuzz_cleanup(fuzz); } +#define NUM_FAST_BASE64_TESTS 1024 + void sshkey_fuzz_tests(void) { struct sshkey *k1; struct sshbuf *buf, *fuzzed; struct fuzz *fuzz; - int r; + int r, i; TEST_START("fuzz RSA private"); @@ -114,12 +123,14 @@ sshkey_fuzz_tests(void) sshbuf_free(buf); ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL); TEST_ONERROR(onerror, fuzz); - for(; !fuzz_done(fuzz); fuzz_next(fuzz)) { + for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) { r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz)); ASSERT_INT_EQ(r, 0); if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0) sshkey_free(k1); sshbuf_reset(fuzzed); + if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS) + break; } sshbuf_free(fuzzed); fuzz_cleanup(fuzz); @@ -134,12 +145,14 @@ sshkey_fuzz_tests(void) sshbuf_free(buf); ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL); TEST_ONERROR(onerror, fuzz); - for(; !fuzz_done(fuzz); fuzz_next(fuzz)) { + for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) { r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz)); ASSERT_INT_EQ(r, 0); if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0) sshkey_free(k1); sshbuf_reset(fuzzed); + if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS) + break; } sshbuf_free(fuzzed); fuzz_cleanup(fuzz); @@ -154,12 +167,14 @@ sshkey_fuzz_tests(void) sshbuf_free(buf); ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL); TEST_ONERROR(onerror, fuzz); - for(; !fuzz_done(fuzz); fuzz_next(fuzz)) { + for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) { r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz)); ASSERT_INT_EQ(r, 0); if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0) sshkey_free(k1); sshbuf_reset(fuzzed); + if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS) + break; } sshbuf_free(fuzzed); fuzz_cleanup(fuzz); @@ -174,12 +189,14 @@ sshkey_fuzz_tests(void) sshbuf_free(buf); ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL); TEST_ONERROR(onerror, fuzz); - for(; !fuzz_done(fuzz); fuzz_next(fuzz)) { + for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) { r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz)); ASSERT_INT_EQ(r, 0); if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0) sshkey_free(k1); sshbuf_reset(fuzzed); + if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS) + break; } sshbuf_free(fuzzed); fuzz_cleanup(fuzz); @@ -195,12 +212,14 @@ sshkey_fuzz_tests(void) sshbuf_free(buf); ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL); TEST_ONERROR(onerror, fuzz); - for(; !fuzz_done(fuzz); fuzz_next(fuzz)) { + for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) { r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz)); ASSERT_INT_EQ(r, 0); if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0) sshkey_free(k1); sshbuf_reset(fuzzed); + if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS) + break; } sshbuf_free(fuzzed); fuzz_cleanup(fuzz); @@ -215,12 +234,14 @@ sshkey_fuzz_tests(void) sshbuf_free(buf); ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL); TEST_ONERROR(onerror, fuzz); - for(; !fuzz_done(fuzz); fuzz_next(fuzz)) { + for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) { r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz)); ASSERT_INT_EQ(r, 0); if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0) sshkey_free(k1); sshbuf_reset(fuzzed); + if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS) + break; } sshbuf_free(fuzzed); fuzz_cleanup(fuzz); @@ -236,12 +257,14 @@ sshkey_fuzz_tests(void) sshbuf_free(buf); ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL); TEST_ONERROR(onerror, fuzz); - for(; !fuzz_done(fuzz); fuzz_next(fuzz)) { + for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) { r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz)); ASSERT_INT_EQ(r, 0); if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0) sshkey_free(k1); sshbuf_reset(fuzzed); + if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS) + break; } sshbuf_free(fuzzed); fuzz_cleanup(fuzz); diff --git a/regress/unittests/test_helper/test_helper.c b/regress/unittests/test_helper/test_helper.c index 4cc70852c..6200ccd58 100644 --- a/regress/unittests/test_helper/test_helper.c +++ b/regress/unittests/test_helper/test_helper.c @@ -1,4 +1,4 @@ -/* $OpenBSD: test_helper.c,v 1.8 2018/02/08 08:46:20 djm Exp $ */ +/* $OpenBSD: test_helper.c,v 1.9 2018/10/17 23:28:05 djm Exp $ */ /* * Copyright (c) 2011 Damien Miller * @@ -115,6 +115,8 @@ static test_onerror_func_t *test_onerror = NULL; static void *onerror_ctx = NULL; static const char *data_dir = NULL; static char subtest_info[512]; +static int fast = 0; +static int slow = 0; int main(int argc, char **argv) @@ -134,8 +136,14 @@ main(int argc, char **argv) } } - while ((ch = getopt(argc, argv, "vqd:")) != -1) { + while ((ch = getopt(argc, argv, "Ffvqd:")) != -1) { switch (ch) { + case 'F': + slow = 1; + break; + case 'f': + fast = 1; + break; case 'd': data_dir = optarg; break; @@ -167,17 +175,29 @@ main(int argc, char **argv) } int -test_is_verbose() +test_is_verbose(void) { return verbose_mode; } int -test_is_quiet() +test_is_quiet(void) { return quiet_mode; } +int +test_is_fast(void) +{ + return fast; +} + +int +test_is_slow(void) +{ + return slow; +} + const char * test_data_file(const char *name) { diff --git a/regress/unittests/test_helper/test_helper.h b/regress/unittests/test_helper/test_helper.h index 6da0066e9..1f893c8dd 100644 --- a/regress/unittests/test_helper/test_helper.h +++ b/regress/unittests/test_helper/test_helper.h @@ -1,4 +1,4 @@ -/* $OpenBSD: test_helper.h,v 1.8 2018/02/08 08:46:20 djm Exp $ */ +/* $OpenBSD: test_helper.h,v 1.9 2018/10/17 23:28:05 djm Exp $ */ /* * Copyright (c) 2011 Damien Miller * @@ -45,6 +45,8 @@ void set_onerror_func(test_onerror_func_t *f, void *ctx); void test_done(void); int test_is_verbose(void); int test_is_quiet(void); +int test_is_fast(void); +int test_is_slow(void); void test_subtest_info(const char *fmt, ...) __attribute__((format(printf, 1, 2))); void ssl_err_check(const char *file, int line); -- cgit v1.2.3