upstream: add some knobs:

UNITTEST_FAST?= no     # Skip slow tests (e.g. less intensive fuzzing).
UNITTEST_SLOW?= no     # Include slower tests (e.g. more intensive fuzzing).
UNITTEST_VERBOSE?= no  # Verbose test output (inc. per-test names).

useful if you want to run the tests as a smoke test to exercise the
functionality without waiting for all the fuzzers to run.

OpenBSD-Regress-ID: e04d82ebec86068198cd903acf1c67563c57315e

8 files changed, 103 insertions, 34 deletions

diff --git a/regress/unittests/Makefile.inc b/regress/unittests/Makefile.inc
index d662a46bc..428ef6836 100644
--- a/regress/unittests/Makefile.inc
+++ b/regress/unittests/Makefile.inc
@@ -1,10 +1,15 @@
-#       $OpenBSD: Makefile.inc,v 1.12 2017/12/21 00:41:22 djm Exp $
+#       $OpenBSD: Makefile.inc,v 1.13 2018/10/17 23:28:05 djm Exp $
 
 REGRESS_FAIL_EARLY?=    yes
 
 .include <bsd.own.mk>
 .include <bsd.obj.mk>
 
+# User-settable options
+UNITTEST_FAST?= no      # Skip slow tests (e.g. less intensive fuzzing).
+UNITTEST_SLOW?= no      # Include slower tests (e.g. more intensive fuzzing).
+UNITTEST_VERBOSE?= no   # Verbose test output (inc. per-test names).
+
 MALLOC_OPTIONS?=        CFGJRSUX
 TEST_ENV?=              MALLOC_OPTIONS=${MALLOC_OPTIONS}
 
@@ -68,3 +73,14 @@ DPADD+= ${LIBUTIL}
 LDADD+= -lcrypto
 DPADD+= ${LIBCRYPTO}
 .endif
+
+UNITTEST_ARGS?=
+
+.if (${UNITTEST_VERBOSE:L} != "no")
+UNITTEST_ARGS+= -v
+.endif
+.if (${UNITTEST_FAST:L} != "no")
+UNITTEST_ARGS+= -f
+.elif (${UNITTEST_SLOW:L} != "no")
+UNITTEST_ARGS+= -F
+.endif
diff --git a/regress/unittests/sshbuf/Makefile b/regress/unittests/sshbuf/Makefile
index 81d4f27a6..0e8e9fd10 100644
--- a/regress/unittests/sshbuf/Makefile
+++ b/regress/unittests/sshbuf/Makefile
@@ -1,4 +1,4 @@
-#       $OpenBSD: Makefile,v 1.6 2017/12/21 00:41:22 djm Exp $
+#       $OpenBSD: Makefile,v 1.7 2018/10/17 23:28:05 djm Exp $
 
 .include <bsd.regress.mk>
 
@@ -17,6 +17,5 @@ SRCS+=sshbuf-getput-basic.c sshbuf-getput-crypto.c sshbuf-misc.c sshbuf.c
 SRCS+=atomicio.c
 
 run-regress-${PROG}: ${PROG}
-        env ${TEST_ENV} ./${PROG}
- 
+        env ${TEST_ENV} ./${PROG} ${UNITTEST_ARGS}
 
diff --git a/regress/unittests/sshbuf/test_sshbuf_fuzz.c b/regress/unittests/sshbuf/test_sshbuf_fuzz.c
index c52376b53..e236c82f9 100644
--- a/regress/unittests/sshbuf/test_sshbuf_fuzz.c
+++ b/regress/unittests/sshbuf/test_sshbuf_fuzz.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: test_sshbuf_fuzz.c,v 1.1 2014/04/30 05:32:00 djm Exp $ */
+/*      $OpenBSD: test_sshbuf_fuzz.c,v 1.2 2018/10/17 23:28:05 djm Exp $ */
 /*
  * Regress test for sshbuf.h buffer API
  *
@@ -30,10 +30,15 @@ sshbuf_fuzz_tests(void)
 {
         struct sshbuf *p1;
         u_char *dp;
-        size_t sz, sz2, i;
+        size_t sz, sz2, i, ntests = NUM_FUZZ_TESTS;
         u_int32_t r;
         int ret;
 
+        if (test_is_fast())
+                ntests >>= 2;
+        if (test_is_slow())
+                ntests <<= 2;
+
         /* NB. uses sshbuf internals */
         TEST_START("fuzz alloc/dealloc");
         p1 = sshbuf_new();
diff --git a/regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c b/regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c
index c6b5c29d1..7c7cb2bfd 100644
--- a/regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c
+++ b/regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: test_sshbuf_getput_fuzz.c,v 1.2 2014/05/02 02:54:00 djm Exp $ */
+/*      $OpenBSD: test_sshbuf_getput_fuzz.c,v 1.3 2018/10/17 23:28:05 djm Exp $ */
 /*
  * Regress test for sshbuf.h buffer API
  *
@@ -115,11 +115,15 @@ sshbuf_getput_fuzz_tests(void)
                 0x55, 0x0f, 0x69, 0xd8, 0x0e, 0xc2, 0x3c, 0xd4,
         };
         struct fuzz *fuzz;
+        u_int fuzzers = FUZZ_1_BIT_FLIP | FUZZ_2_BIT_FLIP |
+            FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP |
+            FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END;
+
+        if (test_is_fast())
+                fuzzers &= ~(FUZZ_2_BYTE_FLIP|FUZZ_2_BIT_FLIP);
 
         TEST_START("fuzz blob parsing");
-        fuzz = fuzz_begin(FUZZ_1_BIT_FLIP | FUZZ_2_BIT_FLIP |
-            FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP |
-            FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END, blob, sizeof(blob));
+        fuzz = fuzz_begin(fuzzers, blob, sizeof(blob));
         TEST_ONERROR(onerror, fuzz);
         for(; !fuzz_done(fuzz); fuzz_next(fuzz))
                 attempt_parse_blob(blob, sizeof(blob));
diff --git a/regress/unittests/sshkey/Makefile b/regress/unittests/sshkey/Makefile
index 1c940bec6..aa731df1c 100644
--- a/regress/unittests/sshkey/Makefile
+++ b/regress/unittests/sshkey/Makefile
@@ -1,4 +1,4 @@
-#       $OpenBSD: Makefile,v 1.5 2017/12/21 00:41:22 djm Exp $
+#       $OpenBSD: Makefile,v 1.6 2018/10/17 23:28:05 djm Exp $
 
 PROG=test_sshkey
 SRCS=tests.c test_sshkey.c test_file.c test_fuzz.c common.c
@@ -18,7 +18,7 @@ SRCS+=digest-openssl.c
 REGRESS_TARGETS=run-regress-${PROG}
 
 run-regress-${PROG}: ${PROG}
-        env ${TEST_ENV} ./${PROG} -d ${.CURDIR}/testdata
+        env ${TEST_ENV} ./${PROG} ${UNITTEST_ARGS} -d ${.CURDIR}/testdata
 
 .include <bsd.regress.mk>
 
diff --git a/regress/unittests/sshkey/test_fuzz.c b/regress/unittests/sshkey/test_fuzz.c
index d3b0c92b4..5953de595 100644
--- a/regress/unittests/sshkey/test_fuzz.c
+++ b/regress/unittests/sshkey/test_fuzz.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: test_fuzz.c,v 1.8 2017/12/21 00:41:22 djm Exp $ */
+/*      $OpenBSD: test_fuzz.c,v 1.9 2018/10/17 23:28:05 djm Exp $ */
 /*
  * Fuzz tests for key parsing
  *
@@ -51,14 +51,16 @@ public_fuzz(struct sshkey *k)
         struct sshkey *k1;
         struct sshbuf *buf;
         struct fuzz *fuzz;
+        u_int fuzzers = FUZZ_1_BIT_FLIP | FUZZ_1_BYTE_FLIP |
+            FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END;
 
+        if (test_is_fast())
+                fuzzers &= ~FUZZ_1_BIT_FLIP;
+        if (test_is_slow())
+                fuzzers |= FUZZ_2_BIT_FLIP | FUZZ_2_BYTE_FLIP;
         ASSERT_PTR_NE(buf = sshbuf_new(), NULL);
         ASSERT_INT_EQ(sshkey_putb(k, buf), 0);
-        /* XXX need a way to run the tests in "slow, but complete" mode */
-        fuzz = fuzz_begin(FUZZ_1_BIT_FLIP | /* XXX too slow FUZZ_2_BIT_FLIP | */
-            FUZZ_1_BYTE_FLIP | /* XXX too slow FUZZ_2_BYTE_FLIP | */
-            FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END,
-            sshbuf_mutable_ptr(buf), sshbuf_len(buf));
+        fuzz = fuzz_begin(fuzzers, sshbuf_mutable_ptr(buf), sshbuf_len(buf));
         ASSERT_INT_EQ(sshkey_from_blob(sshbuf_ptr(buf), sshbuf_len(buf),
             &k1), 0);
         sshkey_free(k1);
@@ -77,12 +79,17 @@ sig_fuzz(struct sshkey *k, const char *sig_alg)
         struct fuzz *fuzz;
         u_char *sig, c[] = "some junk to be signed";
         size_t l;
+        u_int fuzzers = FUZZ_1_BIT_FLIP | FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP |
+            FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END;
+
+        if (test_is_fast())
+                fuzzers &= ~FUZZ_2_BYTE_FLIP;
+        if (test_is_slow())
+                fuzzers |= FUZZ_2_BIT_FLIP;
 
         ASSERT_INT_EQ(sshkey_sign(k, &sig, &l, c, sizeof(c), sig_alg, 0), 0);
         ASSERT_SIZE_T_GT(l, 0);
-        fuzz = fuzz_begin(FUZZ_1_BIT_FLIP | /* too slow FUZZ_2_BIT_FLIP | */
-            FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP |
-            FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END, sig, l);
+        fuzz = fuzz_begin(fuzzers, sig, l);
         ASSERT_INT_EQ(sshkey_verify(k, sig, l, c, sizeof(c), NULL, 0), 0);
         free(sig);
         TEST_ONERROR(onerror, fuzz);
@@ -96,13 +103,15 @@ sig_fuzz(struct sshkey *k, const char *sig_alg)
         fuzz_cleanup(fuzz);
 }
 
+#define NUM_FAST_BASE64_TESTS   1024
+
 void
 sshkey_fuzz_tests(void)
 {
         struct sshkey *k1;
         struct sshbuf *buf, *fuzzed;
         struct fuzz *fuzz;
-        int r;
+        int r, i;
 
 
         TEST_START("fuzz RSA private");
@@ -114,12 +123,14 @@ sshkey_fuzz_tests(void)
         sshbuf_free(buf);
         ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
         TEST_ONERROR(onerror, fuzz);
-        for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+        for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                 r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                 ASSERT_INT_EQ(r, 0);
                 if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                         sshkey_free(k1);
                 sshbuf_reset(fuzzed);
+                if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                        break;
         }
         sshbuf_free(fuzzed);
         fuzz_cleanup(fuzz);
@@ -134,12 +145,14 @@ sshkey_fuzz_tests(void)
         sshbuf_free(buf);
         ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
         TEST_ONERROR(onerror, fuzz);
-        for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+        for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                 r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                 ASSERT_INT_EQ(r, 0);
                 if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                         sshkey_free(k1);
                 sshbuf_reset(fuzzed);
+                if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                        break;
         }
         sshbuf_free(fuzzed);
         fuzz_cleanup(fuzz);
@@ -154,12 +167,14 @@ sshkey_fuzz_tests(void)
         sshbuf_free(buf);
         ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
         TEST_ONERROR(onerror, fuzz);
-        for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+        for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                 r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                 ASSERT_INT_EQ(r, 0);
                 if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                         sshkey_free(k1);
                 sshbuf_reset(fuzzed);
+                if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                        break;
         }
         sshbuf_free(fuzzed);
         fuzz_cleanup(fuzz);
@@ -174,12 +189,14 @@ sshkey_fuzz_tests(void)
         sshbuf_free(buf);
         ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
         TEST_ONERROR(onerror, fuzz);
-        for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+        for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                 r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                 ASSERT_INT_EQ(r, 0);
                 if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                         sshkey_free(k1);
                 sshbuf_reset(fuzzed);
+                if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                        break;
         }
         sshbuf_free(fuzzed);
         fuzz_cleanup(fuzz);
@@ -195,12 +212,14 @@ sshkey_fuzz_tests(void)
         sshbuf_free(buf);
         ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
         TEST_ONERROR(onerror, fuzz);
-        for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+        for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                 r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                 ASSERT_INT_EQ(r, 0);
                 if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                         sshkey_free(k1);
                 sshbuf_reset(fuzzed);
+                if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                        break;
         }
         sshbuf_free(fuzzed);
         fuzz_cleanup(fuzz);
@@ -215,12 +234,14 @@ sshkey_fuzz_tests(void)
         sshbuf_free(buf);
         ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
         TEST_ONERROR(onerror, fuzz);
-        for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+        for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                 r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                 ASSERT_INT_EQ(r, 0);
                 if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                         sshkey_free(k1);
                 sshbuf_reset(fuzzed);
+                if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                        break;
         }
         sshbuf_free(fuzzed);
         fuzz_cleanup(fuzz);
@@ -236,12 +257,14 @@ sshkey_fuzz_tests(void)
         sshbuf_free(buf);
         ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
         TEST_ONERROR(onerror, fuzz);
-        for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+        for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                 r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                 ASSERT_INT_EQ(r, 0);
                 if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                         sshkey_free(k1);
                 sshbuf_reset(fuzzed);
+                if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                        break;
         }
         sshbuf_free(fuzzed);
         fuzz_cleanup(fuzz);
diff --git a/regress/unittests/test_helper/test_helper.c b/regress/unittests/test_helper/test_helper.c
index 4cc70852c..6200ccd58 100644
--- a/regress/unittests/test_helper/test_helper.c
+++ b/regress/unittests/test_helper/test_helper.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: test_helper.c,v 1.8 2018/02/08 08:46:20 djm Exp $     */
+/*      $OpenBSD: test_helper.c,v 1.9 2018/10/17 23:28:05 djm Exp $     */
 /*
  * Copyright (c) 2011 Damien Miller <djm@mindrot.org>
  *
@@ -115,6 +115,8 @@ static test_onerror_func_t *test_onerror = NULL;
 static void *onerror_ctx = NULL;
 static const char *data_dir = NULL;
 static char subtest_info[512];
+static int fast = 0;
+static int slow = 0;
 
 int
 main(int argc, char **argv)
@@ -134,8 +136,14 @@ main(int argc, char **argv)
                 }
         }
 
-        while ((ch = getopt(argc, argv, "vqd:")) != -1) {
+        while ((ch = getopt(argc, argv, "Ffvqd:")) != -1) {
                 switch (ch) {
+                case 'F':
+                        slow = 1;
+                        break;
+                case 'f':
+                        fast = 1;
+                        break;
                 case 'd':
                         data_dir = optarg;
                         break;
@@ -167,17 +175,29 @@ main(int argc, char **argv)
 }
 
 int
-test_is_verbose()
+test_is_verbose(void)
 {
         return verbose_mode;
 }
 
 int
-test_is_quiet()
+test_is_quiet(void)
 {
         return quiet_mode;
 }
 
+int
+test_is_fast(void)
+{
+        return fast;
+}
+
+int
+test_is_slow(void)
+{
+        return slow;
+}
+
 const char *
 test_data_file(const char *name)
 {
diff --git a/regress/unittests/test_helper/test_helper.h b/regress/unittests/test_helper/test_helper.h
index 6da0066e9..1f893c8dd 100644
--- a/regress/unittests/test_helper/test_helper.h
+++ b/regress/unittests/test_helper/test_helper.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: test_helper.h,v 1.8 2018/02/08 08:46:20 djm Exp $     */
+/*      $OpenBSD: test_helper.h,v 1.9 2018/10/17 23:28:05 djm Exp $     */
 /*
  * Copyright (c) 2011 Damien Miller <djm@mindrot.org>
  *
@@ -45,6 +45,8 @@ void set_onerror_func(test_onerror_func_t *f, void *ctx);
 void test_done(void);
 int test_is_verbose(void);
 int test_is_quiet(void);
+int test_is_fast(void);
+int test_is_slow(void);
 void test_subtest_info(const char *fmt, ...)
     __attribute__((format(printf, 1, 2)));
 void ssl_err_check(const char *file, int line);