Commit 8c57b68b80 for openssl.org
commit 8c57b68b80e2e3798d5503d3e63dbd6b0e73310a
Author: Jakub Zelenka <jakub.zelenka@openssl.foundation>
Date: Tue Aug 18 14:55:48 2026 +0200
test: keep budgeted fuzz mfail runs within their time budget
The budgeted mfail run measured the whole corpus in counting mode and
always ran every corpus file, so a large corpus could blow the budget
or cause the injection phase to be skipped entirely.
Estimate the per file cost from a small probe instead and reduce the
number of corpus files when the budget cannot fit the whole corpus
with a minimal injection count. The test-corpus runner collects and
sorts the corpus and walks it in bit reversed index order, so that
any prefix stays an evenly spread sample. The recipe passes just the
corpus directory and controls the selection through the new
OPENSSL_TEST_CORPUS_MAX_FILES variable.
Since an estimate can be far off for corpora with strongly uneven per
file cost, enforce the budget through the new
OPENSSL_TEST_CORPUS_MAX_TIME variable. The runner stops starting new
files and injection passes once the time limit is reached and reports
the number of skipped files. Both variables are documented in
openssl-env.pod.
With the budget enforced, lower it in the CI mfail job so a passing
pull request job finishes within 15 minutes including the build. The
job timeout stays at 30 minutes to leave room for the leak bisection
diagnostics on failure, which run outside the budget. Also add a
daily job with a much larger budget that covers more corpus files and
injection points per file.
This also reverts the temporary corpus trimming in the CI mfail job
since the enforced budget covers it.
Assisted-by: Claude:claude-fable-5
Reviewed-by: Nikola Pajkovsky <nikolap@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.foundation>
Merge-date: Wed Nov 4 17:49:35 2026
Merged-from: https://github.com/openssl/openssl/pull/32412
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1711870239..3210df3b81 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -477,20 +477,13 @@ jobs:
fuzz_tests_mfail:
runs-on: ubuntu-latest
- timeout-minutes: 15
+ timeout-minutes: 30
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: checkout fuzz/corpora submodule
run: git submodule update --init --depth 1 fuzz/corpora
- - name: trim corpora to a few seeds per fuzzer
- env:
- FUZZ_CORPUS_KEEP: 5
- run: |
- for d in fuzz/corpora/*/; do
- find "$d" -maxdepth 1 -type f | tail -n +$((FUZZ_CORPUS_KEEP + 1)) | xargs -r rm -f
- done
- name: Adjust ASLR for sanitizer
run: sudo sysctl -w vm.mmap_rnd_bits=28
- name: config
@@ -504,6 +497,9 @@ jobs:
- name: make
run: make -s -j4
- name: make test (fuzz with mfail)
+ env:
+ OSSL_FUZZ_TEST_BUDGET: 600
+ OSSL_FUZZ_TEST_JOBS: 4
run: .github/workflows/make-test OPENSSL_TEST_RAND_ORDER=0 TESTS="test_fuzz*"
- name: save artifacts
if: success() || failure()
diff --git a/.github/workflows/run-checker-daily.yml b/.github/workflows/run-checker-daily.yml
index 6a422a49ef..52ab8cd3c4 100644
--- a/.github/workflows/run-checker-daily.yml
+++ b/.github/workflows/run-checker-daily.yml
@@ -538,3 +538,42 @@ jobs:
with:
name: "ci@bn_debug"
path: artifacts.tar.gz
+
+ fuzz_tests_mfail_deep:
+ needs: [validate-dispatch-inputs]
+ if: |
+ !cancelled() &&
+ (needs.validate-dispatch-inputs.result == 'success' || needs.validate-dispatch-inputs.result == 'skipped')
+ runs-on: ubuntu-latest
+ timeout-minutes: 300
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ persist-credentials: false
+ ref: ${{ github.event.inputs.head_sha || github.sha }}
+ - name: checkout fuzz/corpora submodule
+ run: git submodule update --init --depth 1 fuzz/corpora
+ - name: Adjust ASLR for sanitizer
+ run: sudo sysctl -w vm.mmap_rnd_bits=28
+ - name: config
+ run: |
+ ./config --strict-warnings --banner=Configured --debug \
+ -DPEDANTIC -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION \
+ enable-asan enable-ec_explicit_curves enable-ubsan \
+ enable-rc5 enable-md2 enable-ec_nistp_64_gcc_128 \
+ enable-weak-ssl-ciphers enable-nextprotoneg
+ perl configdata.pm --dump
+ - name: make
+ run: make -s -j4
+ - name: make test (fuzz with mfail, deep run)
+ env:
+ OSSL_FUZZ_TEST_BUDGET: 10800
+ OSSL_FUZZ_TEST_JOBS: 4
+ run: .github/workflows/make-test OPENSSL_TEST_RAND_ORDER=0 TESTS="test_fuzz*"
+ - name: save artifacts
+ if: success() || failure()
+ uses: actions/upload-artifact@v5
+ with:
+ name: "ci@fuzz_tests_mfail_deep"
+ path: artifacts.tar.gz
+ if-no-files-found: ignore
diff --git a/doc/man7/openssl-env.pod b/doc/man7/openssl-env.pod
index a1b68f893f..e5ee825afb 100644
--- a/doc/man7/openssl-env.pod
+++ b/doc/man7/openssl-env.pod
@@ -135,6 +135,27 @@ Default is 0. For more info see L<CRYPTO_secure_malloc_init(3)>.
This variable is not considered security-sensitive.
+=item B<OPENSSL_TEST_CORPUS_MAX_FILES>
+
+This test-only environment variable is recognised by the fuzz test corpus
+runner. When set to a positive integer, the runner processes at most that
+many corpus files, selected evenly spread over the sorted corpus. It is
+used by the fuzz test recipes to keep budgeted runs within their time
+budget and should generally not be set by users.
+
+This variable is not considered security-sensitive.
+
+=item B<OPENSSL_TEST_CORPUS_MAX_TIME>
+
+This test-only environment variable is recognised by the fuzz test corpus
+runner. When set to a positive number of seconds, the runner stops starting
+new corpus files and failure injection passes once that much processing
+time has elapsed and reports the number of skipped files. It is used by
+the fuzz test recipes to keep budgeted runs within their time budget and
+should generally not be set by users.
+
+This variable is not considered security-sensitive.
+
=item B<OPENSSL_TEST_LIBCTX>
This test-only environment variable, that is recognised by the L<openssl(1)>
diff --git a/fuzz/test-corpus.c b/fuzz/test-corpus.c
index a908f69644..5decce1f8d 100644
--- a/fuzz/test-corpus.c
+++ b/fuzz/test-corpus.c
@@ -42,6 +42,16 @@ static double secs_since(clock_t start)
return (double)(clock() - start) / CLOCKS_PER_SEC;
}
+static clock_t corpus_start;
+static double corpus_max_time;
+
+/* stop starting new work once the corpus time limit has passed */
+static int corpus_max_time_hit(void)
+{
+ return corpus_max_time > 0
+ && secs_since(corpus_start) >= corpus_max_time;
+}
+
static void run_baseline(const unsigned char *buf, size_t s)
{
FuzzerTestOneInput(buf, s);
@@ -51,7 +61,7 @@ static void run_mfail(const unsigned char *buf, size_t s,
const char *path, int file_idx)
{
mfail_init(file_idx, MFAIL_FLAG_COUNT);
- while (mfail_has_next()) {
+ while (!corpus_max_time_hit() && mfail_has_next()) {
if (mfail_get_phase() == MFAIL_PHASE_COUNTING)
fprintf(stderr,
"# MFAIL_BEGIN file_idx=%d phase=count\n", file_idx);
@@ -107,11 +117,99 @@ static void testfile(const char *pathname, int file_idx)
fclose(f);
}
+static char **corpus_paths;
+static int corpus_total;
+static int corpus_capacity;
+
+static int corpus_add(const char *pathname)
+{
+ char *copy;
+ struct stat st;
+
+ /* only collect regular files */
+ if (stat(pathname, &st) < 0 || !S_ISREG(st.st_mode))
+ return 1;
+
+ if (corpus_total >= corpus_capacity) {
+ int capacity = corpus_capacity > 0 ? corpus_capacity * 2 : 64;
+ char **paths = realloc(corpus_paths, capacity * sizeof(*paths));
+
+ if (paths == NULL)
+ return 0;
+ corpus_paths = paths;
+ corpus_capacity = capacity;
+ }
+ copy = malloc(strlen(pathname) + 1);
+ if (copy == NULL)
+ return 0;
+ strcpy(copy, pathname);
+ corpus_paths[corpus_total++] = copy;
+ return 1;
+}
+
+/* collect the file or all the directory entries the argument points to */
+static int corpus_collect(const char *arg)
+{
+ size_t dirname_len = strlen(arg);
+ const char *filename = NULL;
+ char *pathname = NULL;
+ OPENSSL_DIR_CTX *ctx = NULL;
+ int wasdir = 0;
+ int ok = 1;
+
+ /*
+ * We start with trying to read the given path as a directory.
+ */
+ while (ok && (filename = OPENSSL_DIR_read(&ctx, arg)) != NULL) {
+ wasdir = 1;
+ if (pathname == NULL) {
+ pathname = malloc(PATH_MAX);
+ if (pathname == NULL)
+ break;
+ strcpy(pathname, arg);
+#ifdef __VMS
+ if (strchr(":<]", pathname[dirname_len - 1]) == NULL)
+#endif
+ pathname[dirname_len++] = '/';
+ pathname[dirname_len] = '\0';
+ }
+ strcpy(pathname + dirname_len, filename);
+ ok = corpus_add(pathname);
+ }
+ OPENSSL_DIR_end(&ctx);
+
+ /* If it wasn't a directory, treat it as a file instead */
+ if (ok && !wasdir)
+ ok = corpus_add(arg);
+
+ free(pathname);
+ return ok;
+}
+
+static int corpus_path_cmp(const void *a, const void *b)
+{
+ return strcmp(*(const char *const *)a, *(const char *const *)b);
+}
+
int main(int argc, char **argv)
{
- int n, mfi_rc;
+ int n, mfi_rc, bits, selected, chosen, skipped;
+ unsigned int i, r, v;
int file_idx = 0;
- clock_t corpus_start;
+ const char *max_time_env = getenv("OPENSSL_TEST_CORPUS_MAX_TIME");
+ const char *max_files_env = getenv("OPENSSL_TEST_CORPUS_MAX_FILES");
+ int max_files = 0;
+
+ if (max_time_env != NULL && *max_time_env != '\0') {
+ corpus_max_time = strtod(max_time_env, NULL);
+ if (corpus_max_time < 0)
+ corpus_max_time = 0;
+ }
+ if (max_files_env != NULL && *max_files_env != '\0') {
+ max_files = (int)strtol(max_files_env, NULL, 10);
+ if (max_files < 0)
+ max_files = 0;
+ }
mfi_rc = mfail_install(1);
if (mfi_rc < 0) {
@@ -129,45 +227,58 @@ int main(int argc, char **argv)
return 1;
}
- corpus_start = clock();
-
for (n = 1; n < argc; ++n) {
- size_t dirname_len = strlen(argv[n]);
- const char *filename = NULL;
- char *pathname = NULL;
- OPENSSL_DIR_CTX *ctx = NULL;
- int wasdir = 0;
-
- /*
- * We start with trying to read the given path as a directory.
- */
- while ((filename = OPENSSL_DIR_read(&ctx, argv[n])) != NULL) {
- wasdir = 1;
- if (pathname == NULL) {
- pathname = malloc(PATH_MAX);
- if (pathname == NULL)
- break;
- strcpy(pathname, argv[n]);
-#ifdef __VMS
- if (strchr(":<]", pathname[dirname_len - 1]) == NULL)
-#endif
- pathname[dirname_len++] = '/';
- pathname[dirname_len] = '\0';
- }
- strcpy(pathname + dirname_len, filename);
- testfile(pathname, file_idx++);
+ if (!corpus_collect(argv[n])) {
+ fprintf(stderr, "failed to collect corpus files\n");
+ return 1;
}
- OPENSSL_DIR_end(&ctx);
+ }
+ if (corpus_total > 1)
+ qsort(corpus_paths, corpus_total, sizeof(*corpus_paths),
+ corpus_path_cmp);
+
+ selected = corpus_total;
+ if (max_files > 0 && max_files < selected)
+ selected = max_files;
- /* If it wasn't a directory, treat it as a file instead */
- if (!wasdir)
- testfile(argv[n], file_idx++);
+ corpus_start = clock();
- free(pathname);
+ /*
+ * Walk the sorted list in bit reversed index order so that both the
+ * selection and a possible time limit cut keep the processed files
+ * evenly spread over the whole corpus.
+ */
+ bits = 0;
+ while ((1 << bits) < corpus_total)
+ bits++;
+ chosen = 0;
+ skipped = 0;
+ for (i = 0; chosen < selected && i < (1U << bits); i++) {
+ for (r = 0, v = i, n = 0; n < bits; n++, v >>= 1)
+ r = (r << 1) | (v & 1);
+ if (r >= (unsigned int)corpus_total)
+ continue;
+ chosen++;
+ if (corpus_max_time_hit()) {
+ skipped++;
+ continue;
+ }
+ testfile(corpus_paths[r], file_idx++);
}
- if (!mfail_is_installed() || mfail_is_count_only())
+ if (skipped > 0)
+ fprintf(stderr, "# CORPUS_MAX_TIME reached: %d files skipped\n",
+ skipped);
+
+ if (!mfail_is_installed() || mfail_is_count_only()) {
+ fprintf(stderr, "# corpus_files: total=%d selected=%d\n",
+ corpus_total, selected);
fprintf(stderr, "# corpus_time: %.6f\n", secs_since(corpus_start));
+ }
+
+ for (n = 0; n < corpus_total; n++)
+ free(corpus_paths[n]);
+ free(corpus_paths);
FuzzerCleanup();
return 0;
diff --git a/test/recipes/fuzz.pl b/test/recipes/fuzz.pl
index e5182a362c..d3a7e2399e 100644
--- a/test/recipes/fuzz.pl
+++ b/test/recipes/fuzz.pl
@@ -27,6 +27,9 @@ sub fuzz_print_backtrace {
my $bt_log = result_file("$f-backtrace.stderr.log");
local $ENV{OPENSSL_TEST_MFAIL_BACKTRACE} = 1;
local $ENV{OPENSSL_TEST_MFAIL_POINT} = $point if defined $point;
+ # never limit the single file rerun used for diagnostics
+ delete local $ENV{OPENSSL_TEST_CORPUS_MAX_TIME};
+ delete local $ENV{OPENSSL_TEST_CORPUS_MAX_FILES};
run(fuzz(["$f-test", $path], stderr => $bt_log));
diag("- backtrace at injection point:");
@@ -108,17 +111,18 @@ sub fuzz_run_count_only {
my $exit_ok = run(fuzz(["$f-test", $d], stderr => $log));
fuzz_dump_log($log);
- my ($corpus_time, $cur, @allocs) = (0, undef);
+ my ($corpus_time, $total, $cur, @allocs) = (0, 0, undef);
if (open(my $fh, '<', $log)) {
while (my $line = <$fh>) {
$corpus_time = $1 if $line =~ /^#\s*corpus_time:\s*([\d.]+)/;
+ $total = $1 if $line =~ /^#\s*corpus_files:\s*total=(\d+)/;
$cur = $1 if $line =~ /^#\s*CORPUS_FILE\s+file_idx=(\d+)/;
push @allocs, $1 + 0 if defined $cur
&& $line =~ /:\s*(\d+)\s+allocations\s*$/;
}
close $fh;
}
- return ($exit_ok, $corpus_time, \@allocs);
+ return ($exit_ok, $corpus_time, \@allocs, $total);
}
# find path and point of the reported leak for easy recreation
@@ -142,6 +146,8 @@ sub fuzz_mfail_bisect {
delete local $ENV{OPENSSL_TEST_MFAIL_COUNT};
delete local $ENV{OPENSSL_TEST_MFAIL_START};
delete local $ENV{OPENSSL_TEST_MFAIL_POINT};
+ delete local $ENV{OPENSSL_TEST_CORPUS_MAX_TIME};
+ delete local $ENV{OPENSSL_TEST_CORPUS_MAX_FILES};
diag("bisecting mfail leak across isolated point reruns");
@@ -228,6 +234,19 @@ sub fuzz_per_test_budget {
return $per_test;
}
+# report when the run was cut short by the corpus time limit
+sub fuzz_check_max_time {
+ my ($f, $log) = @_;
+ return unless open(my $fh, '<', $log);
+ while (my $line = <$fh>) {
+ if ($line =~ /^#\s*(CORPUS_MAX_TIME reached.*?)\s*$/) {
+ diag("$f: $1");
+ last;
+ }
+ }
+ close $fh;
+}
+
sub fuzz_ok {
my ($f, %opts) = @_;
my $d = srctop_dir('fuzz', 'corpora', $f);
@@ -245,13 +264,32 @@ sub fuzz_ok {
return;
}
- # baseline run to measure the corpus run time
- my ($ok, $corpus_time, $allocs) = fuzz_run_count_only($f, $d);
+ # probe a small evenly spread sample, selected by the test runner,
+ # to estimate the per-file run time
+ my ($ok, $probe_time, $allocs, $num_files);
+ {
+ local $ENV{OPENSSL_TEST_CORPUS_MAX_FILES} = 16;
+ # bound the probe in case even its few files are too slow
+ local $ENV{OPENSSL_TEST_CORPUS_MAX_TIME} =
+ sprintf("%.3f", 0.25 * $target);
+ ($ok, $probe_time, $allocs, $num_files) =
+ fuzz_run_count_only($f, $d);
+ }
unless ($ok) {
ok(0, "Fuzzing $f (count-only)");
return;
}
+ # empty corpus, nothing to select, just run it as it is
+ if ($num_files <= 0) {
+ ok(run(fuzz(["$f-test", $d])), "Fuzzing $f");
+ return;
+ }
+
+ # average over the files the probe managed to process
+ my $probe_n = scalar @$allocs;
+ $probe_n = 1 if $probe_n < 1;
+
# get the maximum allocations in instance and count total
my $total_allocs = 0;
my $max_k = 0;
@@ -259,44 +297,54 @@ sub fuzz_ok {
$total_allocs += $_;
$max_k = $_ if $_ > $max_k;
}
- my $num_files = scalar @$allocs;
- diag(sprintf("%s: count-only %.3fs, allocs=%d, files=%d, max=%d",
- $f, $corpus_time, $total_allocs, $num_files, $max_k));
-
- # baseline alone consumed the budget, nothing left for mfail
- if ($corpus_time <= 0 || $corpus_time >= $target) {
- ok(1, "Fuzzing $f (no mfail budget; "
- . "corpus=${corpus_time}s, target=${target}s)");
- return;
- }
+ diag(sprintf("%s: probe %d/%d files in %.3fs, allocs=%d, max=%d",
+ $f, $probe_n, $num_files, $probe_time, $total_allocs, $max_k));
# no allocations counted, can't size the mfail run
- if ($total_allocs <= 0 || $num_files <= 0) {
+ if ($total_allocs <= 0) {
ok(1, "Fuzzing $f (no allocations counted)");
return;
}
- # number of mfail iterations that fit alongside the baseline:
- # ~corpus_time * (1 + count / 2) <= target
- my $count = int(2 * ($target - $corpus_time) / $corpus_time);
+ # average per-file time; the clock may be too coarse to measure
+ my $avg = $probe_time / $probe_n;
+ $avg = 0.001 if $avg <= 0;
+ my $remaining = $target - $probe_time;
+ $remaining = 0 if $remaining < 0;
+
+ # a file with count injections costs ~avg * (1 + count / 2), so
+ # running mfail over n files fits: n * avg * (1 + count / 2) <= budget
+ my $min_count = 3;
+ my $use_n = $num_files;
+ my $count = int(2 * ($remaining / ($avg * $num_files) - 1));
+ if ($count < $min_count) {
+ # budget too low for the whole corpus, drop some files instead
+ $count = $min_count;
+ $use_n = int($remaining / ($avg * (1 + $count / 2)));
+ # always exercise mfail on at least one file
+ ($use_n, $count) = (1, 1) if $use_n < 1;
+ }
# never exceed max(K_i); injections beyond that are wasted
$count = $max_k if $count > $max_k;
- if ($count <= 0) {
- ok(1, "Fuzzing $f (budget too small for mfail)");
- return;
- }
- diag("$f: running mfail with count=$count");
+
+ $use_n = $num_files if $use_n > $num_files;
+ diag(sprintf("%s: running mfail on %d/%d files with count=%d",
+ $f, $use_n, $num_files, $count));
local $ENV{OPENSSL_TEST_MFAIL_COUNT} = $count;
+ # the test runner selects an evenly spread subset of this size
+ local $ENV{OPENSSL_TEST_CORPUS_MAX_FILES} = $use_n;
+ # hard stop in case the probe estimate was too optimistic
+ local $ENV{OPENSSL_TEST_CORPUS_MAX_TIME} =
+ sprintf("%.3f", $remaining > 1 ? $remaining : 1);
my $main_log = "$f-mfail.stderr.log";
my ($passed, $leaks, undef, $log) = fuzz_run($f, $d, $main_log, 1);
- unless ($passed) {
- fuzz_mfail_bisect($f, $log) if $leaks;
- ok(0, "Fuzzing $f (mfail count=$count, per-test=${per_test}s)");
- return;
- }
- ok(1, "Fuzzing $f (mfail count=$count, per-test=${per_test}s)");
+ fuzz_check_max_time($f, $log);
+ fuzz_mfail_bisect($f, $log) if !$passed && $leaks;
+ ok($passed, sprintf("Fuzzing %s (mfail count=%d, files=%d/%d, "
+ . "per-test=%.3fs)", $f, $count, $use_n, $num_files,
+ $per_test));
}
}