mirror of
https://git.gnunet.org/libmicrohttpd.git
synced 2026-09-25 04:09:31 +03:00
expand test suite, add CI/CD logic as done in other GNUnet projects
This commit is contained in:
Christian Grothoff
@@ -55,3 +55,6 @@ src/microhttpd_ws/test_websocket
|
||||
/po/configure.ac
|
||||
*.exe.manifest
|
||||
/libmicrohttpd-*.*.*.tar.*
|
||||
|
||||
# Reproducers written by the src/fuzz harnesses (MHD_FUZZ_CRASH_DIR)
|
||||
crashes/
|
||||
|
||||
+22
-1
@@ -5832,6 +5832,26 @@ AS_VAR_IF([use_heavy_tests],["yes"],
|
||||
AM_CONDITIONAL([RUN_ZZUF_TESTS],[test "x$run_zzuf_tests" = "xyes"])
|
||||
AM_CONDITIONAL([FORCE_USE_ZZUF_SOCAT],[test "x$zzuf_socat_mandatory" = "xyes"])
|
||||
|
||||
# Structure-aware in-process fuzzing harnesses (src/fuzz)
|
||||
AC_MSG_CHECKING([[whether to build the fuzzing harnesses]])
|
||||
AC_ARG_ENABLE([[fuzzing]],
|
||||
[AS_HELP_STRING([[--enable-fuzzing]],
|
||||
[build the in-process fuzzing harnesses in src/fuzz and run them ]
|
||||
[as part of "make check"; requires a static build of the library ]
|
||||
[and is only really useful together with --enable-asserts and ]
|
||||
[--enable-sanitizers=address,undefined [no]])],
|
||||
[], [enable_fuzzing="no"])
|
||||
AS_VAR_IF([enable_fuzzing], ["yes"],
|
||||
[
|
||||
AS_VAR_IF([enable_static], ["no"],
|
||||
[AC_MSG_RESULT([[no]])
|
||||
AC_MSG_ERROR([[--enable-fuzzing requires --enable-static]])],
|
||||
[AC_MSG_RESULT([[yes]])])
|
||||
],
|
||||
[enable_fuzzing="no"
|
||||
AC_MSG_RESULT([[no]])])
|
||||
AM_CONDITIONAL([ENABLE_FUZZING], [[test "x$enable_fuzzing" = "xyes"]])
|
||||
|
||||
# Final flags that may interfere with autoconf detections
|
||||
AS_CASE([${enable_build_type}],[debug|debugger],
|
||||
[ # Debug build or build for walking with debugger
|
||||
@@ -5933,7 +5953,8 @@ src/examples/Makefile
|
||||
src/tools/Makefile
|
||||
src/testcurl/Makefile
|
||||
src/testcurl/https/Makefile
|
||||
src/testzzuf/Makefile])
|
||||
src/testzzuf/Makefile
|
||||
src/fuzz/Makefile])
|
||||
AC_OUTPUT
|
||||
|
||||
# Finally: summary
|
||||
|
||||
@@ -1,2 +1,5 @@
|
||||
# This Makefile.am is in the public domain
|
||||
EXTRA_DIST = xcc ascebc mhd.png mhd.svg mhd_logo.png
|
||||
EXTRA_DIST += ci
|
||||
EXTRA_DIST += run-option-matrix.sh
|
||||
EXTRA_DIST += oss-fuzz
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# Output of the 6-coverage job.
|
||||
/artifacts/
|
||||
@@ -0,0 +1,53 @@
|
||||
FROM docker.io/library/debian:trixie
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Everything needed to bootstrap, configure, build and "make check"
|
||||
# GNU libmicrohttpd. Unlike most GNU Taler / GNUnet components MHD has
|
||||
# no database, no libgnunet and no libjansson dependency; the only hard
|
||||
# external dependencies are a C compiler and the autotools. Everything
|
||||
# else is optional and only unlocks parts of the test suite.
|
||||
RUN apt-get update -yqq && \
|
||||
apt-get upgrade -yqq && \
|
||||
apt-get install -yqq \
|
||||
git \
|
||||
ca-certificates \
|
||||
build-essential \
|
||||
make \
|
||||
autoconf \
|
||||
automake \
|
||||
libtool \
|
||||
pkg-config \
|
||||
texinfo \
|
||||
file \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Optional dependencies:
|
||||
# libcurl*-dev src/testcurl and src/testzzuf (RUN_LIBCURL_TESTS)
|
||||
# libgnutls28-dev + libgcrypt20-dev HTTPS support (--enable-https=auto)
|
||||
# zlib1g-dev used by some of the example/test code
|
||||
# zzuf, socat src/testzzuf (--enable-heavy-tests)
|
||||
# lcov the coverage job
|
||||
RUN apt-get update -yqq && \
|
||||
apt-get install -yqq \
|
||||
curl \
|
||||
libcurl4-gnutls-dev \
|
||||
libgnutls28-dev \
|
||||
libgcrypt20-dev \
|
||||
zlib1g-dev \
|
||||
zzuf \
|
||||
socat \
|
||||
lcov \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 32-bit toolchain for the "-m32" half of the build-configuration matrix
|
||||
# (job 3-build-matrix). Only available on x86; on other architectures the
|
||||
# matrix job logs a SKIP for the 32-bit combinations instead of failing.
|
||||
RUN apt-get update -yqq && \
|
||||
(apt-get install -yqq gcc-multilib libc6-dev-i386 \
|
||||
|| echo "no 32-bit toolchain for this architecture, -m32 jobs will SKIP") \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /workdir
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,92 @@
|
||||
GNU libmicrohttpd -- containerised CI jobs
|
||||
==========================================
|
||||
|
||||
Each job is a directory under contrib/ci/jobs/. It holds a config.ini
|
||||
describing the container to use and an executable job.sh that is the
|
||||
entry point inside that container. contrib/ci/ci.sh is the driver.
|
||||
|
||||
Requirements
|
||||
------------
|
||||
|
||||
podman (the driver calls it directly, not docker)
|
||||
qemu-user-static only for foreign architectures; without it
|
||||
"podman run --arch ..." fails in creative ways.
|
||||
See https://wiki.archlinux.org/title/Podman#Foreign_architectures
|
||||
|
||||
Nothing else has to be installed on the host: every job builds and runs
|
||||
inside a debian:trixie container described by contrib/ci/Containerfile
|
||||
(or contrib/ci/<arch>.Containerfile, if one exists for the architecture
|
||||
the job asks for).
|
||||
|
||||
Running
|
||||
-------
|
||||
|
||||
One job, from the top of the source tree:
|
||||
|
||||
contrib/ci/ci.sh 2-test
|
||||
|
||||
All of them, in numeric order:
|
||||
|
||||
contrib/ci/run-all-jobs
|
||||
|
||||
The source tree is bind-mounted into the container at /workdir, so the
|
||||
jobs build in your checkout and leave their output there.
|
||||
|
||||
The jobs
|
||||
--------
|
||||
|
||||
0-codespell spell check. Runs in nixery.dev/shell/codespell, does
|
||||
not build a container. Advisory only
|
||||
(HALT_ON_FAILURE = False).
|
||||
1-build ./bootstrap && ./configure && make. The smoke build:
|
||||
default configuration, nothing enabled.
|
||||
2-test --enable-asserts --enable-sanitizers=address,undefined
|
||||
plus the full "make check". Dumps the logs of every
|
||||
failing test.
|
||||
3-build-matrix the build-configuration matrix: asserts on/off x
|
||||
sanitizers on/off x six digest/auth configurations x
|
||||
{64-bit, 32-bit}. Clean out-of-tree build and
|
||||
"make check" per combination, with a summary table at
|
||||
the end.
|
||||
4-fuzz a few seconds per harness of the structure-aware
|
||||
fuzzers in src/fuzz, plus a replay of the whole
|
||||
committed corpus.
|
||||
5-zzuf src/testzzuf, the bit-level fuzzing suite, over a
|
||||
moderate sweep of zzuf seeds.
|
||||
6-coverage --enable-coverage build, full "make check", lcov +
|
||||
genhtml. Writes the HTML report to
|
||||
contrib/ci/artifacts/coverage/$CI_COMMIT_REF/html/ and
|
||||
prints the path; it uploads nowhere.
|
||||
|
||||
Every push, and nightly
|
||||
-----------------------
|
||||
|
||||
Run on every push:
|
||||
|
||||
0-codespell, 1-build, 2-test, 4-fuzz
|
||||
|
||||
That is the "--enable-asserts --enable-sanitizers" configuration that
|
||||
TESTING.md proposal P3 asks for on every push, plus a bounded fuzz run so
|
||||
a regression is caught in the pull request.
|
||||
|
||||
Run nightly:
|
||||
|
||||
3-build-matrix, 5-zzuf, 6-coverage
|
||||
|
||||
These are the expensive ones. 3-build-matrix is 48 builds; 5-zzuf and
|
||||
6-coverage each run the heavy-test suite.
|
||||
|
||||
Why 3-build-matrix exists
|
||||
-------------------------
|
||||
|
||||
MAX_DIGEST in src/microhttpd/digestauth.c is 16 in an MD5-only build and
|
||||
32 otherwise, so the out-of-bounds write fixed in commit 5a73c1ae
|
||||
overflows by a different amount - or not at all - depending only on
|
||||
configure flags. Testing one configuration cannot answer whether a fix
|
||||
holds in every build a distribution ships. See TESTING.md, proposal P3.
|
||||
|
||||
Not part of CI: OSS-Fuzz
|
||||
------------------------
|
||||
|
||||
contrib/oss-fuzz/ holds the OSS-Fuzz integration. It is a separate,
|
||||
externally driven path and is deliberately not run from these jobs.
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/bin/bash
|
||||
set -exvuo pipefail
|
||||
|
||||
# Run a single CI job in a container.
|
||||
#
|
||||
# contrib/ci/ci.sh <job-name> [arch]
|
||||
#
|
||||
# Requires podman
|
||||
# Fails if not found in PATH
|
||||
OCI_RUNTIME=$(which podman)
|
||||
REPO_NAME=$(basename "${PWD}")
|
||||
JOB_NAME="${1}"
|
||||
JOB_ARCH=$((grep CONTAINER_ARCH contrib/ci/jobs/${JOB_NAME}/config.ini | cut -d' ' -f 3) || echo "${2:-amd64}")
|
||||
JOB_CONTAINER=$((grep CONTAINER_NAME contrib/ci/jobs/${JOB_NAME}/config.ini | cut -d' ' -f 3) || echo "localhost/${REPO_NAME}:${JOB_ARCH}")
|
||||
CONTAINER_BUILD=$((grep CONTAINER_BUILD contrib/ci/jobs/${JOB_NAME}/config.ini | cut -d' ' -f 3) || echo "True")
|
||||
CONTAINERFILE="contrib/ci/$JOB_ARCH.Containerfile"
|
||||
|
||||
if ! [[ -f "$CONTAINERFILE" ]];
|
||||
then
|
||||
CONTAINERFILE="$(dirname "$CONTAINERFILE")/Containerfile"
|
||||
fi
|
||||
|
||||
echo "Image name: ${JOB_CONTAINER}" 2>&1
|
||||
echo "Containerfile: ${CONTAINERFILE}" 2>&1
|
||||
|
||||
if [ "${CONTAINER_BUILD}" = "True" ] ;
|
||||
then
|
||||
"${OCI_RUNTIME}" build \
|
||||
--arch "${JOB_ARCH}" \
|
||||
-t "${JOB_CONTAINER}" \
|
||||
-f "$CONTAINERFILE" .
|
||||
fi
|
||||
|
||||
"${OCI_RUNTIME}" run \
|
||||
--rm \
|
||||
-ti \
|
||||
--arch "${JOB_ARCH}" \
|
||||
--env CI_COMMIT_REF="$(git rev-parse HEAD)" \
|
||||
--volume "${PWD}":/workdir \
|
||||
--workdir /workdir \
|
||||
"${JOB_CONTAINER}" \
|
||||
contrib/ci/jobs/"${JOB_NAME}"/job.sh
|
||||
@@ -0,0 +1,6 @@
|
||||
[build]
|
||||
HALT_ON_FAILURE = False
|
||||
WARN_ON_FAILURE = True
|
||||
CONTAINER_BUILD = False
|
||||
CONTAINER_NAME = nixery.dev/shell/codespell
|
||||
CONTAINER_ARCH = amd64
|
||||
@@ -0,0 +1,40 @@
|
||||
# List of "words" that codespell should ignore in the libmicrohttpd sources.
|
||||
#
|
||||
# Note: The word sensitivity depends on how the to-be-ignored word is
|
||||
# spelled in codespell_lib/data/dictionary.txt. F.e. if there is a word
|
||||
# 'foo' and you add 'Foo' _here_, codespell will continue to complain
|
||||
# about 'Foo'.
|
||||
#
|
||||
# Entries here must be *deliberate*: an identifier, a protocol token, a
|
||||
# truncated string used by a test, or a spelling this project has chosen.
|
||||
# Real typos belong in a patch, not in this file.
|
||||
|
||||
# "TE" is an HTTP header field name (RFC 9110 10.1.4), and the fuzzers
|
||||
# and the API headers name it.
|
||||
TE
|
||||
|
||||
# HTTP method token registered by RFC 3253 (WebDAV versioning).
|
||||
CHECKIN
|
||||
|
||||
# The RFC 7578 / RFC 2388 spelling used verbatim in postprocessor.c.
|
||||
IMPLEMENTORS
|
||||
|
||||
# Local variable name for "content length" all over connection.c,
|
||||
# postprocessor.c and the fuzzers.
|
||||
clen
|
||||
|
||||
# Appears inside autoconf quoting in configure.ac.
|
||||
fo
|
||||
|
||||
# Deliberately truncated tokens used by the string-parser unit tests
|
||||
# (test_str_token*.c) and by src/testcurl/test_tricky.c, which sends
|
||||
# intentionally malformed requests.
|
||||
strin
|
||||
Strin
|
||||
Thi
|
||||
|
||||
# libmicrohttpd consistently hyphenates this in its API documentation.
|
||||
re-use
|
||||
re-used
|
||||
re-uses
|
||||
re-using
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -exuo pipefail
|
||||
|
||||
job_dir=$(dirname "${BASH_SOURCE[0]}")
|
||||
skip=$(cat "${job_dir}"/skip.txt)
|
||||
|
||||
echo "Current directory: $(pwd)"
|
||||
|
||||
codespell -d -I "${job_dir}"/dictionary.txt -S ${skip//$'\n'/,} "$@"
|
||||
@@ -0,0 +1,77 @@
|
||||
*/.git/**
|
||||
*.git
|
||||
ABOUT-NLS
|
||||
ChangeLog
|
||||
*build-aux*
|
||||
autom4te.cache
|
||||
config.guess
|
||||
config.log
|
||||
config.status
|
||||
config.sub
|
||||
configure
|
||||
configure~
|
||||
configure.lineno
|
||||
depcomp
|
||||
install-sh
|
||||
ltmain.sh
|
||||
*libtool*
|
||||
missing
|
||||
test-driver
|
||||
compile
|
||||
*.in
|
||||
*.m4
|
||||
*/m4/*
|
||||
*/doc/*
|
||||
*.texi
|
||||
*.info
|
||||
*.info-*
|
||||
*/po/*
|
||||
*.po
|
||||
*.pot
|
||||
*.sed
|
||||
*.sin
|
||||
*.header
|
||||
*/contrib/ascebc/*
|
||||
*/contrib/fixes-autoconf/*
|
||||
*/contrib/fixes-libtool/*
|
||||
*.patch
|
||||
*.png
|
||||
*.PNG
|
||||
*.svg
|
||||
*.jpg
|
||||
*.jpeg
|
||||
*.gif
|
||||
*.pdf
|
||||
*.crt
|
||||
*.key
|
||||
*.pem
|
||||
*key
|
||||
*/keys/*
|
||||
*.o
|
||||
*.lo
|
||||
*.la
|
||||
*.a
|
||||
*.so
|
||||
*.so.*
|
||||
.deps
|
||||
.libs
|
||||
*.log
|
||||
*.trs
|
||||
*.gcda
|
||||
*.gcno
|
||||
*.info.tmp
|
||||
*.vcxproj
|
||||
*.filters
|
||||
*.sln
|
||||
*.props
|
||||
*.rpath
|
||||
*.hint
|
||||
*.tag
|
||||
*.gz
|
||||
*.tgz
|
||||
*.bz2
|
||||
*.xz
|
||||
*.zip
|
||||
tags
|
||||
TAGS
|
||||
*~
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
# The smoke build: exactly what a user typing the three commands from
|
||||
# README/INSTALL gets. No asserts, no sanitizers, default everything.
|
||||
# If this breaks, nothing else in the pipeline is worth running.
|
||||
#
|
||||
# "configure" prints a "Configuration Summary:" block at the end; it is
|
||||
# the quickest way to see in the CI log which optional features (HTTPS,
|
||||
# digest algorithms, libcurl tests) were actually compiled in.
|
||||
|
||||
./bootstrap
|
||||
./configure
|
||||
make -j"$(nproc)"
|
||||
@@ -0,0 +1,6 @@
|
||||
[build]
|
||||
HALT_ON_FAILURE = True
|
||||
WARN_ON_FAILURE = True
|
||||
CONTAINER_BUILD = True
|
||||
CONTAINER_NAME = localhost/libmicrohttpd
|
||||
CONTAINER_ARCH = amd64
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
job_dir=$(dirname "${BASH_SOURCE[0]}")
|
||||
|
||||
. "${job_dir}"/build.sh
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
# The every-push configuration of TESTING.md P3:
|
||||
#
|
||||
# --enable-asserts --enable-sanitizers=address,undefined
|
||||
#
|
||||
# Both halves matter. mhd_assert() is compiled out of an ordinary build,
|
||||
# so the invariants the library documents are only actually checked with
|
||||
# --enable-asserts; and three of the four v1.0.7 findings were memory
|
||||
# safety defects that ASAN reports on the spot but that an unsanitised
|
||||
# build happily runs through. Anything that is not this configuration is
|
||||
# nightly (see job 3-build-matrix).
|
||||
|
||||
./bootstrap
|
||||
./configure \
|
||||
--enable-asserts \
|
||||
--enable-sanitizers=address,undefined
|
||||
make -j"$(nproc)"
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
check_command()
|
||||
{
|
||||
make -j"$(nproc)" check
|
||||
}
|
||||
|
||||
print_logs()
|
||||
{
|
||||
set +e
|
||||
echo "###############################################################"
|
||||
echo "## make check FAILED - dumping the logs of every failing test"
|
||||
echo "###############################################################"
|
||||
# Automake writes one .log per test plus a per-directory
|
||||
# test-suite.log; the per-test logs carry the actual diagnostics.
|
||||
find . -name 'test-suite.log' -print -exec cat {} \;
|
||||
find . -name '*.log' -path '*/src/*' -newer config.status -print0 |
|
||||
while IFS= read -r -d '' logfile; do
|
||||
case "${logfile}" in
|
||||
*/config.log) continue ;;
|
||||
esac
|
||||
# Only the logs of tests that did not pass: automake writes a
|
||||
# matching .trs with ":test-result: PASS" for successful ones.
|
||||
trs="${logfile%.log}.trs"
|
||||
if [ -f "${trs}" ] && grep -q ':test-result: PASS' "${trs}"; then
|
||||
continue
|
||||
fi
|
||||
echo "--------------------------------------------------------"
|
||||
echo "## ${logfile}"
|
||||
echo "--------------------------------------------------------"
|
||||
cat "${logfile}"
|
||||
done
|
||||
set -e
|
||||
}
|
||||
|
||||
if ! check_command ; then
|
||||
print_logs
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,6 @@
|
||||
[build]
|
||||
HALT_ON_FAILURE = True
|
||||
WARN_ON_FAILURE = True
|
||||
CONTAINER_BUILD = True
|
||||
CONTAINER_NAME = localhost/libmicrohttpd
|
||||
CONTAINER_ARCH = amd64
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
job_dir=$(dirname "${BASH_SOURCE[0]}")
|
||||
|
||||
. "${job_dir}"/1-build.sh
|
||||
. "${job_dir}"/2-test.sh
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
# Every combination of the matrix is a *clean out-of-tree* build, and
|
||||
# autoconf refuses those while the source directory itself still carries
|
||||
# a config.status. A CI checkout is clean; a developer running this job
|
||||
# by hand on a working tree usually is not.
|
||||
if [ -f config.status ] || [ -f Makefile ]; then
|
||||
make distclean || true
|
||||
fi
|
||||
if [ -f config.status ]; then
|
||||
echo "ERROR: source directory is still configured;" \
|
||||
"run 'make distclean' by hand first." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
./bootstrap
|
||||
Executable
+258
@@ -0,0 +1,258 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# The build-configuration matrix of TESTING.md, proposal P3.
|
||||
#
|
||||
# Why this job exists, concretely:
|
||||
#
|
||||
# MAX_DIGEST in src/microhttpd/digestauth.c is the size of the largest
|
||||
# digest the build supports. It is 16 in an MD5-only build and 32 when
|
||||
# SHA-256 or SHA-512/256 are compiled in. The out-of-bounds stack
|
||||
# write fixed in commit 5a73c1ae wrote up to 64 bytes into a
|
||||
# MAX_DIGEST-sized array, so *the very same request overflows by a
|
||||
# different amount - or not at all - depending purely on the configure
|
||||
# flags*. A CI that only ever builds the default configuration cannot
|
||||
# see that, and cannot tell whether a fix holds in every shipped build.
|
||||
# Distributions do ship reduced builds, so the reduced builds have to
|
||||
# be tested.
|
||||
#
|
||||
# The same argument applies to --enable-asserts (mhd_assert() is
|
||||
# compiled out of a normal build, so every invariant the library
|
||||
# documents is unchecked there), to the sanitizers (they are the only
|
||||
# oracle that turns a silent memory-safety defect into a test failure)
|
||||
# and to -m32 (pointer and size_t width change the arithmetic in the
|
||||
# read-buffer shift-back path of 29eaa56b).
|
||||
#
|
||||
# The sweep is
|
||||
#
|
||||
# {asserts on, asserts off}
|
||||
# x {sanitizers on, sanitizers off}
|
||||
# x {all digests, MD5 only, SHA-256 only, SHA-512/256 only,
|
||||
# --disable-dauth, --disable-bauth}
|
||||
# x {native 64-bit, 32-bit via -m32}
|
||||
#
|
||||
# = 48 combinations, each a clean out-of-tree build plus a full
|
||||
# "make check". This is a nightly job; the every-push subset is job
|
||||
# 2-test.
|
||||
#
|
||||
# Combinations that the environment cannot support (no 32-bit toolchain,
|
||||
# no sanitizer runtime, no 32-bit sanitizer runtime) are reported as SKIP
|
||||
# with the reason, never as a failure - a missing i386 libasan is a
|
||||
# property of the runner, not a defect in MHD.
|
||||
#
|
||||
# Knobs:
|
||||
# MHD_CI_MATRIX_FILTER only run combinations whose tag matches this
|
||||
# extended regular expression
|
||||
# MHD_CI_MATRIX_JOBS -j level for each build (default: nproc)
|
||||
# MHD_CI_MATRIX_KEEP set to 1 to keep the build trees for triage
|
||||
#
|
||||
# No "set -e": a failing combination must be recorded and the sweep must
|
||||
# continue. No "set -x" either (job.sh switches it on): with 48 builds
|
||||
# the trace drowns the summary, and every step prints its own banner.
|
||||
set +x
|
||||
set -uo pipefail
|
||||
|
||||
srcdir="$(pwd)"
|
||||
probe_dir="$(mktemp -d)"
|
||||
trap 'rm -rf "${probe_dir}"' EXIT
|
||||
|
||||
jobs="${MHD_CI_MATRIX_JOBS:-$(nproc)}"
|
||||
filter="${MHD_CI_MATRIX_FILTER:-.}"
|
||||
keep="${MHD_CI_MATRIX_KEEP:-0}"
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Environment probes
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
# Compile *and run* a trivial program. Running it matters: a machine can
|
||||
# have a working 32-bit compiler and 32-bit headers while being unable to
|
||||
# execute i386 binaries at all (missing loader, restricted container,
|
||||
# CONFIG_IA32_EMULATION off). Probing with a compile alone would then
|
||||
# turn every 32-bit combination into a wall of bogus SIGSEGV failures
|
||||
# instead of one honest SKIP.
|
||||
probe_cc()
|
||||
{
|
||||
printf 'int main(void){return 0;}\n' > "${probe_dir}/probe.c"
|
||||
# shellcheck disable=SC2086
|
||||
${CC:-gcc} "$@" -o "${probe_dir}/probe" "${probe_dir}/probe.c" \
|
||||
> "${probe_dir}/probe.log" 2>&1 || return 1
|
||||
"${probe_dir}/probe" >> "${probe_dir}/probe.log" 2>&1
|
||||
}
|
||||
|
||||
have_m32=no
|
||||
have_san=no
|
||||
have_m32_san=no
|
||||
have_m32_curl=no
|
||||
have_m32_gnutls=no
|
||||
|
||||
probe_cc -m32 && have_m32=yes
|
||||
probe_cc -fsanitize=address,undefined && have_san=yes
|
||||
if [ "${have_m32}" = "yes" ]; then
|
||||
probe_cc -m32 -fsanitize=address,undefined && have_m32_san=yes
|
||||
probe_cc -m32 -lcurl && have_m32_curl=yes
|
||||
probe_cc -m32 -lgnutls && have_m32_gnutls=yes
|
||||
fi
|
||||
|
||||
echo "== environment probes =============================================="
|
||||
echo " gcc -m32 .............................. ${have_m32}"
|
||||
echo " gcc -fsanitize=address,undefined ...... ${have_san}"
|
||||
echo " gcc -m32 -fsanitize=address,undefined . ${have_m32_san}"
|
||||
echo " 32-bit libcurl ........................ ${have_m32_curl}"
|
||||
echo " 32-bit gnutls ......................... ${have_m32_gnutls}"
|
||||
echo "===================================================================="
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# The axes. Every option spelling below is the one configure.ac really
|
||||
# defines; --enable-md5 and --enable-sha256 take a TYPE argument
|
||||
# (yes/no/builtin/tlslib) while --enable-sha512-256 is a plain
|
||||
# enable/disable switch, and --enable-sanitizers takes a comma separated
|
||||
# list.
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
# tag configure arguments
|
||||
digest_tags=(all md5 sha256 sha512-256 nodauth nobauth)
|
||||
digest_args_all=""
|
||||
digest_args_md5="--enable-md5=builtin --enable-sha256=no --disable-sha512-256"
|
||||
digest_args_sha256="--enable-md5=no --enable-sha256=builtin --disable-sha512-256"
|
||||
digest_args_sha512_256="--enable-md5=no --enable-sha256=no --enable-sha512-256"
|
||||
digest_args_nodauth="--disable-dauth"
|
||||
digest_args_nobauth="--disable-bauth"
|
||||
|
||||
digest_args()
|
||||
{
|
||||
case "$1" in
|
||||
all) echo "${digest_args_all}" ;;
|
||||
md5) echo "${digest_args_md5}" ;;
|
||||
sha256) echo "${digest_args_sha256}" ;;
|
||||
sha512-256) echo "${digest_args_sha512_256}" ;;
|
||||
nodauth) echo "${digest_args_nodauth}" ;;
|
||||
nobauth) echo "${digest_args_nobauth}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
results=()
|
||||
failures=0
|
||||
|
||||
record()
|
||||
{
|
||||
results+=("$(printf '%-34s %-5s %s' "$1" "$2" "${3:-}")")
|
||||
}
|
||||
|
||||
run_combination()
|
||||
{
|
||||
local tag="$1" bits="$2" asserts="$3" san="$4" digest="$5"
|
||||
local builddir="${srcdir}/build-${tag}"
|
||||
local -a args=()
|
||||
|
||||
if ! printf '%s' "${tag}" | grep -Eq -- "${filter}"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# --- skip decisions -------------------------------------------
|
||||
if [ "${bits}" = "32" ] && [ "${have_m32}" != "yes" ]; then
|
||||
record "${tag}" SKIP "no 32-bit toolchain (gcc -m32 fails)"
|
||||
return 0
|
||||
fi
|
||||
if [ "${san}" = "on" ] && [ "${bits}" = "64" ] && [ "${have_san}" != "yes" ]; then
|
||||
record "${tag}" SKIP "no sanitizer runtime for this compiler"
|
||||
return 0
|
||||
fi
|
||||
if [ "${san}" = "on" ] && [ "${bits}" = "32" ] && [ "${have_m32_san}" != "yes" ]; then
|
||||
record "${tag}" SKIP "no 32-bit sanitizer runtime (i386 libasan)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# --- configure arguments --------------------------------------
|
||||
if [ "${asserts}" = "on" ]; then
|
||||
args+=(--enable-asserts)
|
||||
else
|
||||
args+=(--disable-asserts)
|
||||
fi
|
||||
if [ "${san}" = "on" ]; then
|
||||
# shellcheck disable=SC2054 # one argument, the comma is part of it
|
||||
args+=(--enable-sanitizers=address,undefined)
|
||||
fi
|
||||
# Word splitting is intended here: digest_args() returns a
|
||||
# whitespace separated list of configure arguments.
|
||||
# shellcheck disable=SC2207
|
||||
args+=($(digest_args "${digest}"))
|
||||
|
||||
if [ "${bits}" = "32" ]; then
|
||||
args+=(CC="${CC:-gcc} -m32")
|
||||
args+=(--build=x86_64-pc-linux-gnu --host=i686-pc-linux-gnu)
|
||||
# The 32-bit development libraries of libcurl and GnuTLS are
|
||||
# usually not installed next to a 64-bit toolchain; turn the
|
||||
# corresponding parts of the suite off explicitly rather than
|
||||
# letting configure fail a link test and print a warning.
|
||||
[ "${have_m32_curl}" = "yes" ] || args+=(--disable-curl)
|
||||
[ "${have_m32_gnutls}" = "yes" ] || args+=(--disable-https)
|
||||
# src/examples/json_echo links against libjansson, which is
|
||||
# not part of MHD's own dependency set and is essentially
|
||||
# never installed for i386 next to an amd64 toolchain. The
|
||||
# examples are not built by "make check", so dropping them
|
||||
# costs no coverage and keeps the 32-bit leg buildable.
|
||||
args+=(--disable-examples)
|
||||
fi
|
||||
|
||||
# --- clean out-of-tree build ----------------------------------
|
||||
rm -rf "${builddir}"
|
||||
mkdir -p "${builddir}"
|
||||
|
||||
echo "===================================================================="
|
||||
echo "== ${tag}"
|
||||
echo "== ../configure ${args[*]}"
|
||||
echo "===================================================================="
|
||||
|
||||
if ! ( cd "${builddir}" && ../configure "${args[@]}" ); then
|
||||
record "${tag}" FAIL "configure failed"
|
||||
failures=$((failures + 1))
|
||||
return 0
|
||||
fi
|
||||
if ! ( cd "${builddir}" && make -j"${jobs}" ); then
|
||||
record "${tag}" FAIL "build failed"
|
||||
failures=$((failures + 1))
|
||||
return 0
|
||||
fi
|
||||
if ! ( cd "${builddir}" && make -j"${jobs}" check ); then
|
||||
record "${tag}" FAIL "make check failed"
|
||||
failures=$((failures + 1))
|
||||
echo "## test logs for ${tag}:"
|
||||
find "${builddir}" -name 'test-suite.log' -print -exec cat {} \; || true
|
||||
return 0
|
||||
fi
|
||||
|
||||
record "${tag}" PASS ""
|
||||
[ "${keep}" = "1" ] || rm -rf "${builddir}"
|
||||
return 0
|
||||
}
|
||||
|
||||
for bits in 64 32; do
|
||||
for asserts in on off; do
|
||||
for san in on off; do
|
||||
for digest in "${digest_tags[@]}"; do
|
||||
run_combination \
|
||||
"${bits}bit-asserts-${asserts}-san-${san}-${digest}" \
|
||||
"${bits}" "${asserts}" "${san}" "${digest}"
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------
|
||||
echo
|
||||
echo "===================================================================="
|
||||
echo "== build-configuration matrix summary"
|
||||
echo "===================================================================="
|
||||
printf '%-34s %-5s %s\n' "COMBINATION" "STATE" "NOTE"
|
||||
for line in "${results[@]}"; do
|
||||
echo "${line}"
|
||||
done
|
||||
echo "===================================================================="
|
||||
echo "== ${failures} failing combination(s)"
|
||||
echo "===================================================================="
|
||||
|
||||
if [ "${failures}" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
@@ -0,0 +1,6 @@
|
||||
[build]
|
||||
HALT_ON_FAILURE = False
|
||||
WARN_ON_FAILURE = True
|
||||
CONTAINER_BUILD = True
|
||||
CONTAINER_NAME = localhost/libmicrohttpd
|
||||
CONTAINER_ARCH = amd64
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
job_dir=$(dirname "${BASH_SOURCE[0]}")
|
||||
|
||||
. "${job_dir}"/1-bootstrap.sh
|
||||
. "${job_dir}"/2-matrix.sh
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
# The in-process fuzzing harnesses of src/fuzz. See src/fuzz/README and
|
||||
# src/fuzz/BUILD-INTEGRATION.md.
|
||||
#
|
||||
# --enable-fuzzing requires --enable-static (configure enforces it):
|
||||
# fuzz_str and fuzz_auth_header call functions that are internal to the
|
||||
# library and therefore not exported from the shared object, so the
|
||||
# harnesses link against the static archive. --enable-static is the
|
||||
# default, but say it explicitly so the job does not silently break if
|
||||
# that default ever changes.
|
||||
#
|
||||
# The harnesses carry three oracles - the sanitizers, an
|
||||
# MHD_set_panic_func() tripwire and a body-framing oracle - and only the
|
||||
# first of those needs the sanitizers to be switched on, so this job is
|
||||
# always built with them.
|
||||
|
||||
./bootstrap
|
||||
./configure \
|
||||
--enable-fuzzing \
|
||||
--enable-static \
|
||||
--enable-asserts \
|
||||
--enable-sanitizers=address,undefined
|
||||
make -j"$(nproc)"
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
# A short, *bounded* fuzz run: this job has to finish in seconds, not
|
||||
# hours, so that a regression is caught in the pull request rather than
|
||||
# the next morning. Long, corpus-accumulating sessions are a different
|
||||
# thing entirely and are not run here.
|
||||
#
|
||||
# MHD_FUZZ_ITERATIONS, MHD_FUZZ_SEED, MHD_FUZZ_MIN_DISCIPLINE and
|
||||
# MHD_FUZZ_MIN_MEM_LIMIT are ordinary make variables of
|
||||
# src/fuzz/Makefile.am and can be overridden on the command line. The
|
||||
# defaults cover the full discipline and memory-limit range.
|
||||
#
|
||||
# Every run is exactly reproducible from (harness, seed), so the seed is
|
||||
# pinned here. Point MHD_CI_FUZZ_SEED at $RANDOM in a nightly job if a
|
||||
# varying seed is wanted; a failure then prints a copy-pasteable replay
|
||||
# command.
|
||||
|
||||
FUZZ_ITERATIONS="${MHD_CI_FUZZ_ITERATIONS:-20000}"
|
||||
FUZZ_SEED="${MHD_CI_FUZZ_SEED:-1}"
|
||||
|
||||
make -C src/fuzz check \
|
||||
MHD_FUZZ_ITERATIONS="${FUZZ_ITERATIONS}" \
|
||||
MHD_FUZZ_SEED="${FUZZ_SEED}"
|
||||
|
||||
# Replay the whole committed seed corpus - including
|
||||
# corpus/known-findings/ - through every harness. This is the part that
|
||||
# proves a fixed crash stays fixed, and it is cheap.
|
||||
make -C src/fuzz check-corpus
|
||||
@@ -0,0 +1,6 @@
|
||||
[build]
|
||||
HALT_ON_FAILURE = True
|
||||
WARN_ON_FAILURE = True
|
||||
CONTAINER_BUILD = True
|
||||
CONTAINER_NAME = localhost/libmicrohttpd
|
||||
CONTAINER_ARCH = amd64
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
job_dir=$(dirname "${BASH_SOURCE[0]}")
|
||||
|
||||
. "${job_dir}"/1-build.sh
|
||||
. "${job_dir}"/2-fuzz.sh
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
# src/testzzuf is only built when RUN_ZZUF_TESTS is true, which requires
|
||||
# --enable-heavy-tests *and* a usable libcurl (see src/Makefile.am and
|
||||
# src/testzzuf/README). The option spelling is
|
||||
#
|
||||
# --enable-heavy-tests[=SCOPE] with SCOPE in {basic, full}
|
||||
#
|
||||
# ("yes" is an alias for "basic", "all" for "full"; anything else is a
|
||||
# configure error). "basic" runs 10 client iterations per daemon, "full"
|
||||
# runs 200 - far too slow for a routine job, so CI uses "basic" and a
|
||||
# seed sweep instead.
|
||||
#
|
||||
# The sanitizer build is the one worth running: enabling sanitizers
|
||||
# automatically forces the socat relay mode, because zzuf and the
|
||||
# sanitizers cannot both interpose on the C library. socat mode is also
|
||||
# the only mode in which the deterministic chunk-extension self-check
|
||||
# runs, since it needs an unfuzzed channel to MHD.
|
||||
|
||||
./bootstrap
|
||||
./configure \
|
||||
--enable-heavy-tests=basic \
|
||||
--enable-asserts \
|
||||
--enable-sanitizers=address,undefined
|
||||
make -j"$(nproc)"
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
# Sweep a moderate range of zzuf seeds. A single seed proves almost
|
||||
# nothing; a huge range takes correspondingly long, so the recommended
|
||||
# pattern from src/testzzuf/README is a moderate range per job with
|
||||
# different jobs given different ranges. ZZUF_SEED_START takes
|
||||
# precedence over ZZUF_SEED.
|
||||
#
|
||||
# On failure both runner scripts print a block containing the failing
|
||||
# seed and a copy-pasteable replay command; grep the job log for
|
||||
# "FUZZING TEST FAILED".
|
||||
|
||||
ZZUF_START="${MHD_CI_ZZUF_SEED_START:-0}"
|
||||
ZZUF_STOP="${MHD_CI_ZZUF_SEED_STOP:-64}"
|
||||
|
||||
make -C src/testzzuf check \
|
||||
ZZUF_SEED_START="${ZZUF_START}" \
|
||||
ZZUF_SEED_STOP="${ZZUF_STOP}"
|
||||
@@ -0,0 +1,6 @@
|
||||
[build]
|
||||
HALT_ON_FAILURE = False
|
||||
WARN_ON_FAILURE = True
|
||||
CONTAINER_BUILD = True
|
||||
CONTAINER_NAME = localhost/libmicrohttpd
|
||||
CONTAINER_ARCH = amd64
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
job_dir=$(dirname "${BASH_SOURCE[0]}")
|
||||
|
||||
. "${job_dir}"/1-build.sh
|
||||
. "${job_dir}"/2-zzuf.sh
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
# --enable-coverage adds -fprofile-arcs -ftest-coverage. Optimisation is
|
||||
# turned off and debug info on so that the line attribution in the report
|
||||
# is meaningful.
|
||||
#
|
||||
# Heavy tests are enabled because the interesting question this report
|
||||
# answers is "which parts of the parser does the *whole* suite never
|
||||
# reach" - and src/testzzuf is a large part of that suite. Sanitizers
|
||||
# are deliberately NOT enabled here: they change the code that is
|
||||
# generated and slow the run down, and the coverage numbers are what this
|
||||
# job is for.
|
||||
|
||||
./bootstrap
|
||||
./configure CFLAGS="-ggdb -O0" \
|
||||
--enable-coverage \
|
||||
--enable-asserts \
|
||||
--enable-heavy-tests=basic
|
||||
make -j"$(nproc)"
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
# A failing test still produces useful coverage data, so this step does
|
||||
# not abort the job; the coverage report is the deliverable here and
|
||||
# job 2-test is the one that gates on test results.
|
||||
make -j"$(nproc)" check || echo "WARNING: 'make check' failed; the coverage report below is still generated, but it describes a failing run."
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
# Produce an HTML coverage report into a local artifacts directory.
|
||||
#
|
||||
# Unlike the GNU Taler jobs this deliberately does *not* upload anywhere:
|
||||
# there is no rsync target for libmicrohttpd, and a CI job that silently
|
||||
# depends on a private host is a job that breaks for everyone else. The
|
||||
# report is left in the working tree; wire it up to whatever artifact
|
||||
# store the runner has.
|
||||
|
||||
ARTIFACT_DIR="${MHD_CI_ARTIFACT_DIR:-contrib/ci/artifacts/coverage/${CI_COMMIT_REF:-local}}"
|
||||
|
||||
mkdir -p "${ARTIFACT_DIR}"
|
||||
|
||||
# lcov 2.x turns a lot of things that used to be warnings into errors and
|
||||
# needs them switched off explicitly; lcov 1.x does not know some of
|
||||
# those keywords at all. Try the strict-lcov invocation first and fall
|
||||
# back to the plain one, so the job works on either.
|
||||
lcov_try()
|
||||
{
|
||||
lcov "$@" --ignore-errors mismatch,negative,source,empty,unused \
|
||||
|| lcov "$@"
|
||||
}
|
||||
|
||||
lcov_try --capture \
|
||||
--directory . \
|
||||
--output-file "${ARTIFACT_DIR}/coverage.info"
|
||||
|
||||
# Only MHD's own sources are interesting; drop system headers and the
|
||||
# test programs themselves.
|
||||
lcov_try --remove "${ARTIFACT_DIR}/coverage.info" \
|
||||
'/usr/*' \
|
||||
"${PWD}/src/testcurl/*" \
|
||||
"${PWD}/src/testzzuf/*" \
|
||||
"${PWD}/src/fuzz/*" \
|
||||
"${PWD}/src/examples/*" \
|
||||
--output-file "${ARTIFACT_DIR}/coverage-lib.info" \
|
||||
|| cp "${ARTIFACT_DIR}/coverage.info" "${ARTIFACT_DIR}/coverage-lib.info"
|
||||
|
||||
genhtml "${ARTIFACT_DIR}/coverage-lib.info" \
|
||||
--output-directory "${ARTIFACT_DIR}/html" \
|
||||
--title "GNU libmicrohttpd ${CI_COMMIT_REF:-local}" \
|
||||
--legend \
|
||||
--ignore-errors source,empty \
|
||||
|| genhtml "${ARTIFACT_DIR}/coverage-lib.info" \
|
||||
--output-directory "${ARTIFACT_DIR}/html" \
|
||||
--title "GNU libmicrohttpd ${CI_COMMIT_REF:-local}" \
|
||||
--legend
|
||||
|
||||
echo "===================================================================="
|
||||
echo "== coverage report written to:"
|
||||
echo "== ${PWD}/${ARTIFACT_DIR}/html/index.html"
|
||||
echo "== raw tracefiles:"
|
||||
echo "== ${PWD}/${ARTIFACT_DIR}/coverage.info (everything)"
|
||||
echo "== ${PWD}/${ARTIFACT_DIR}/coverage-lib.info (library only)"
|
||||
echo "===================================================================="
|
||||
lcov_try --summary "${ARTIFACT_DIR}/coverage-lib.info" || true
|
||||
@@ -0,0 +1,6 @@
|
||||
[build]
|
||||
HALT_ON_FAILURE = False
|
||||
WARN_ON_FAILURE = True
|
||||
CONTAINER_BUILD = True
|
||||
CONTAINER_NAME = localhost/libmicrohttpd
|
||||
CONTAINER_ARCH = amd64
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
set -exuo pipefail
|
||||
|
||||
job_dir=$(dirname "${BASH_SOURCE[0]}")
|
||||
|
||||
. "${job_dir}"/1-build.sh
|
||||
. "${job_dir}"/2-test.sh
|
||||
. "${job_dir}"/3-coverage.sh
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
set -eax
|
||||
for JOB in $(ls $(dirname $0)/jobs | sort -n); do
|
||||
$(dirname $0)/ci.sh $JOB;
|
||||
done;
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/sh
|
||||
set -ex
|
||||
# This file is in the public domain.
|
||||
# Determines the current version of our code.
|
||||
# Shared between various jobs.
|
||||
|
||||
BRANCH=$(git name-rev --name-only HEAD)
|
||||
if [ -z "${BRANCH}" ]; then
|
||||
exit 1
|
||||
else
|
||||
# "Unshallow" our checkout, but only our current branch, and exclude the submodules.
|
||||
git fetch --no-recurse-submodules --tags --depth=1000 origin "${BRANCH}"
|
||||
RECENT_VERSION_TAG=$(git describe --tags --match 'v*.*.*' --exclude '*-dev*' --always --abbrev=0 HEAD || exit 1)
|
||||
commits="$(git rev-list ${RECENT_VERSION_TAG}..HEAD --count)"
|
||||
if [ "${commits}" = "0" ]; then
|
||||
git describe --tag HEAD | sed -r 's/^v//' || exit 1
|
||||
else
|
||||
echo $(echo ${RECENT_VERSION_TAG} | sed -r 's/^v//')-${commits}-$(git rev-parse --short=8 HEAD)
|
||||
fi
|
||||
fi
|
||||
@@ -0,0 +1,44 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
################################################################################
|
||||
#
|
||||
# OSS-Fuzz Dockerfile for GNU libmicrohttpd.
|
||||
#
|
||||
# This file becomes projects/libmicrohttpd/Dockerfile in the google/oss-fuzz
|
||||
# repository; the Apache-2.0 header above is the one OSS-Fuzz requires for
|
||||
# files living in that repository.
|
||||
|
||||
FROM gcr.io/oss-fuzz-base/base-builder
|
||||
|
||||
# libmicrohttpd is an autotools project and the git checkout ships no
|
||||
# 'configure', so the autotools themselves are build dependencies.
|
||||
# No other dependency is needed: build.sh configures with --disable-https,
|
||||
# --disable-curl and --disable-doc, which removes GnuTLS, libcurl and
|
||||
# texinfo from the picture (see build.sh for the reasoning).
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
autoconf \
|
||||
automake \
|
||||
libtool \
|
||||
make \
|
||||
pkg-config \
|
||||
zip && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN git clone --depth 1 https://git.gnunet.org/libmicrohttpd.git libmicrohttpd
|
||||
|
||||
WORKDIR $SRC/libmicrohttpd
|
||||
|
||||
COPY build.sh $SRC/
|
||||
@@ -0,0 +1,361 @@
|
||||
GNU libmicrohttpd -- OSS-Fuzz integration
|
||||
========================================
|
||||
|
||||
This directory contains everything needed to run the in-process fuzzing
|
||||
harnesses of `src/fuzz/` on Google's OSS-Fuzz service: the build script,
|
||||
the project metadata, the container recipe, the seed-corpus packager,
|
||||
per-harness dictionaries and per-harness `.options` files.
|
||||
|
||||
It is the "continuous" half of `TESTING.md` section **P4**; `src/fuzz/`
|
||||
is the other half. Read `src/fuzz/README` first -- it explains what the
|
||||
four harnesses do, what they have already found, and how to run them
|
||||
without clang.
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
0. This is deliberately NOT part of contrib/ci/
|
||||
-------------------------------------------------------------------
|
||||
|
||||
**OSS-Fuzz is not a CI job and nothing here is wired into
|
||||
`contrib/ci/jobs/`.** That is on purpose, at the maintainer's request:
|
||||
|
||||
* an OSS-Fuzz run is a *continuous, hosted, unbounded* campaign owned
|
||||
by Google's infrastructure, not a bounded per-push check. A CI job
|
||||
must terminate in minutes and must be reproducible offline; neither
|
||||
is true here;
|
||||
* the artifacts in this directory are consumed by the
|
||||
`google/oss-fuzz` repository (`projects/libmicrohttpd/`), not by
|
||||
this tree's build system;
|
||||
* running these builds needs Docker, the OSS-Fuzz base images and
|
||||
network access.
|
||||
|
||||
The bounded, in-tree fuzzing that *does* belong in CI already exists and
|
||||
is unrelated to this directory: `make -C src/fuzz check` (a few seconds
|
||||
per harness) and `make -C src/fuzz check-corpus` (corpus replay). Wire
|
||||
*those* into `contrib/ci/`, never this.
|
||||
|
||||
Nothing in this directory is referenced by any `Makefile.am`; adding it
|
||||
to the build system is not required and not wanted.
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
1. Layout
|
||||
-------------------------------------------------------------------
|
||||
|
||||
build.sh the OSS-Fuzz build script
|
||||
project.yaml OSS-Fuzz project metadata
|
||||
Dockerfile the base-builder container recipe
|
||||
make_seed_corpus.sh packages src/fuzz/corpus/ into
|
||||
$OUT/<fuzzer>_seed_corpus.zip
|
||||
fuzz_request.options per-harness libFuzzer options (max_len, dict)
|
||||
fuzz_str.options
|
||||
fuzz_auth_header.options
|
||||
fuzz_postprocessor.options
|
||||
dicts/fuzz_request.dict per-harness fuzzing dictionaries
|
||||
dicts/fuzz_str.dict
|
||||
dicts/fuzz_auth_header.dict
|
||||
dicts/fuzz_postprocessor.dict
|
||||
README this file
|
||||
|
||||
Of these, only `build.sh`, `project.yaml` and `Dockerfile` are copied
|
||||
into `google/oss-fuzz`; everything else is read out of the cloned
|
||||
libmicrohttpd checkout at build time, which is why the dictionaries and
|
||||
the corpus packager live here and not in the OSS-Fuzz repository.
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
2. The four fuzz targets
|
||||
-------------------------------------------------------------------
|
||||
|
||||
fuzz_request a real struct MHD_Daemon driven over a
|
||||
socketpair; consumes daemon options *and*
|
||||
stream segmentation from the input
|
||||
fuzz_str the mhd_str.c primitives with exactly-sized
|
||||
output buffers
|
||||
fuzz_auth_header MHD_get_rq_dauth_params_() /
|
||||
MHD_get_rq_bauth_params_()
|
||||
fuzz_postprocessor MHD_post_process()
|
||||
|
||||
All four are single translation units that export
|
||||
|
||||
int LLVMFuzzerTestOneInput (const uint8_t *data, size_t size);
|
||||
|
||||
unconditionally. `-DFUZZ_NO_MAIN` (which `build.sh` passes) removes only
|
||||
the standalone driver's `main()` from `fuzz_common.h`; the fuzz target
|
||||
itself is never conditionally compiled. There is no
|
||||
`LLVMFuzzerInitialize()` and none is needed: the two tuning knobs of
|
||||
`fuzz_request` (`MHD_FUZZ_MIN_DISCIPLINE`, `MHD_FUZZ_MIN_MEM_LIMIT`) are
|
||||
read lazily with `getenv()` on the first call and default to the *full*
|
||||
range (-3 and 0), which is what a fuzzing service should explore.
|
||||
|
||||
### max_len
|
||||
|
||||
`max_len` in the `.options` files is set to the point beyond which the
|
||||
harness ignores the extra bytes, so libFuzzer does not waste its budget:
|
||||
|
||||
fuzz_request 8192 4 configuration bytes + length-prefixed send
|
||||
segments (2-byte header, payload <= 0x3FFF,
|
||||
at most 96 segments over at most 8
|
||||
connections). 8 KiB comfortably holds the
|
||||
two-request %%NONCE%% digest handshake, a
|
||||
chunked body with extensions and trailers,
|
||||
and a body-oracle declaration. Larger
|
||||
inputs are safe -- every segment is bounded
|
||||
independently -- but buy almost nothing.
|
||||
fuzz_str 514 2 selector bytes + the payload, which the
|
||||
harness truncates to 512.
|
||||
fuzz_auth_header 4097 1 selector byte + the Authorization header
|
||||
value, truncated to 4096.
|
||||
fuzz_postprocessor 8196 4 selector bytes + the POST body, truncated
|
||||
to 8192.
|
||||
|
||||
### close_fd_mask
|
||||
|
||||
Deliberately **not** set. The harnesses are quiet by default (MHD's
|
||||
error log is only enabled when `MHD_FUZZ_VERBOSE` is set, which never
|
||||
happens under libFuzzer), so there is no output to suppress -- while
|
||||
`fuzz_report_finding()` writes the description of a non-memory-safety
|
||||
finding straight to fd 2 and `MHD_PANIC()` writes to stderr too. Closing
|
||||
those fds would throw away exactly the diagnostics that make a report
|
||||
actionable.
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
3. What build.sh configures, and why
|
||||
-------------------------------------------------------------------
|
||||
|
||||
--enable-static --disable-shared --with-pic
|
||||
fuzz_str and fuzz_auth_header call MHD-internal symbols compiled
|
||||
with hidden visibility; they are not exported from
|
||||
libmicrohttpd.so and can only be reached through the static
|
||||
archive. OSS-Fuzz also requires the binaries in $OUT to be
|
||||
self-contained.
|
||||
--enable-fuzzing
|
||||
configures src/fuzz/Makefile. Not strictly needed (build.sh
|
||||
compiles the harnesses itself) but it keeps this build equivalent
|
||||
to the documented developer build and fails loudly if src/fuzz/
|
||||
ever stops being wired into configure.ac.
|
||||
--enable-asserts
|
||||
keeps mhd_assert() alive. Assertions reachable from network input
|
||||
are remote aborts; findings K1-K6 in `src/fuzz/README` section 6
|
||||
are all of that kind and are invisible without this.
|
||||
--disable-https
|
||||
the harnesses never speak TLS (they hand MHD an already-connected
|
||||
AF_UNIX socketpair through MHD_add_connection() and never set
|
||||
MHD_USE_TLS), so HTTPS adds no coverage; it would drag in GnuTLS,
|
||||
which OSS-Fuzz fuzzes separately, and it would make the
|
||||
MemorySanitizer build impossible without an MSan-instrumented
|
||||
GnuTLS. With HTTPS off, libc and libpthread are the only
|
||||
external dependencies.
|
||||
--disable-curl --disable-doc --disable-examples --disable-tools
|
||||
not needed, and they only add build time and dependencies.
|
||||
--enable-build-type=neutral
|
||||
the default, stated explicitly: "neutral" is the only build type
|
||||
that does not inject its own optimisation/debug flags, so the
|
||||
$CFLAGS supplied by OSS-Fuzz survive unmodified.
|
||||
|
||||
`build.sh` never sets `--enable-sanitizers` or `--enable-coverage`:
|
||||
OSS-Fuzz provides instrumentation through `$CFLAGS`/`$CXXFLAGS`, and a
|
||||
second, configure-generated `-fsanitize=` set is a classic way to break
|
||||
an OSS-Fuzz build. `$CFLAGS` is passed through to `configure` and to
|
||||
every harness compilation verbatim.
|
||||
|
||||
The build is out-of-tree (`$WORK/mhd-build`), so `build.sh` never
|
||||
modifies the checkout. The git checkout ships no `configure`, so
|
||||
`./bootstrap` runs first; because `bootstrap` ends in an `|| echo ...`
|
||||
chain and therefore cannot be trusted to return a failure status,
|
||||
`build.sh` checks for the product and falls back to `autoreconf -fi`.
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
4. Building and running locally with infra/helper.py
|
||||
-------------------------------------------------------------------
|
||||
|
||||
Prerequisites: Docker, python3, and a checkout of `google/oss-fuzz`.
|
||||
|
||||
git clone --depth 1 https://github.com/google/oss-fuzz
|
||||
cd oss-fuzz
|
||||
mkdir -p projects/libmicrohttpd
|
||||
cp /path/to/libmicrohttpd/contrib/oss-fuzz/build.sh projects/libmicrohttpd/
|
||||
cp /path/to/libmicrohttpd/contrib/oss-fuzz/project.yaml projects/libmicrohttpd/
|
||||
cp /path/to/libmicrohttpd/contrib/oss-fuzz/Dockerfile projects/libmicrohttpd/
|
||||
|
||||
Build the image and the targets:
|
||||
|
||||
python3 infra/helper.py build_image libmicrohttpd
|
||||
python3 infra/helper.py build_fuzzers --sanitizer address libmicrohttpd
|
||||
python3 infra/helper.py check_build libmicrohttpd
|
||||
|
||||
`build_fuzzers` accepts an optional path to a local source tree as its
|
||||
last argument, which is how you test uncommitted changes:
|
||||
|
||||
python3 infra/helper.py build_fuzzers --sanitizer address \
|
||||
libmicrohttpd /path/to/libmicrohttpd
|
||||
|
||||
Repeat for the other sanitizers and engines before submitting:
|
||||
|
||||
python3 infra/helper.py build_fuzzers --sanitizer undefined libmicrohttpd
|
||||
python3 infra/helper.py build_fuzzers --sanitizer memory libmicrohttpd
|
||||
python3 infra/helper.py build_fuzzers --engine afl libmicrohttpd
|
||||
python3 infra/helper.py build_fuzzers --engine honggfuzz libmicrohttpd
|
||||
|
||||
Run one:
|
||||
|
||||
python3 infra/helper.py run_fuzzer libmicrohttpd fuzz_request
|
||||
python3 infra/helper.py run_fuzzer libmicrohttpd fuzz_str
|
||||
python3 infra/helper.py run_fuzzer libmicrohttpd fuzz_auth_header
|
||||
python3 infra/helper.py run_fuzzer libmicrohttpd fuzz_postprocessor
|
||||
|
||||
`run_fuzzer` passes anything after the target name to libFuzzer, and
|
||||
takes `--corpus-dir` for a persistent corpus:
|
||||
|
||||
python3 infra/helper.py run_fuzzer --corpus-dir=/tmp/mhd-corpus \
|
||||
libmicrohttpd fuzz_request -- -max_total_time=600 -rss_limit_mb=4096
|
||||
|
||||
Coverage report (needs a coverage build):
|
||||
|
||||
python3 infra/helper.py build_fuzzers --sanitizer coverage libmicrohttpd
|
||||
python3 infra/helper.py coverage libmicrohttpd --fuzz-target fuzz_request
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
5. What to do with a report
|
||||
-------------------------------------------------------------------
|
||||
|
||||
An OSS-Fuzz report contains a *testcase* (the raw input bytes) and a
|
||||
stack trace. Download the testcase from the report, then:
|
||||
|
||||
python3 infra/helper.py reproduce libmicrohttpd fuzz_request ./testcase
|
||||
|
||||
The same bytes can also be replayed with the in-tree standalone driver,
|
||||
which needs no Docker and no clang:
|
||||
|
||||
./configure --enable-fuzzing --enable-static --enable-asserts \
|
||||
--enable-sanitizers=address,undefined
|
||||
make -C src/fuzz check_PROGRAMS # or: make -C src/fuzz check
|
||||
src/fuzz/fuzz_request --file=./testcase
|
||||
|
||||
Minimising a libFuzzer crash:
|
||||
|
||||
python3 infra/helper.py shell libmicrohttpd
|
||||
# inside the container:
|
||||
/out/fuzz_request -minimize_crash=1 -runs=100000 /testcase
|
||||
|
||||
Once fixed, add the minimised input to `src/fuzz/corpus/` (or to
|
||||
`src/fuzz/corpus/known-findings/` if it stays interesting as a named
|
||||
regression) and commit it: `make_seed_corpus.sh` picks it up
|
||||
automatically on the next OSS-Fuzz build, so the case is re-run forever.
|
||||
|
||||
Note that `fuzz_request` is **not** perfectly deterministic: MHD's digest
|
||||
nonces embed a millisecond timestamp, and whether a nonce counts as stale
|
||||
therefore depends on the wall clock. Finding K6 is of that kind and
|
||||
reproduces in only a fraction of replays. If ClusterFuzz marks a
|
||||
testcase "unreproducible" but the trace points at nonce handling, replay
|
||||
it in a loop before dismissing it.
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
6. Seed corpora
|
||||
-------------------------------------------------------------------
|
||||
|
||||
`make_seed_corpus.sh` builds one zip per target:
|
||||
|
||||
$OUT/fuzz_request_seed_corpus.zip
|
||||
$OUT/fuzz_str_seed_corpus.zip
|
||||
$OUT/fuzz_auth_header_seed_corpus.zip
|
||||
$OUT/fuzz_postprocessor_seed_corpus.zip
|
||||
|
||||
`src/fuzz/corpus/` is organised per harness by file-name prefix
|
||||
(`fuzz_<harness>-NN.bin`); inputs are *not* interchangeable between
|
||||
harnesses, because byte 0 selects a different thing in each, so each zip
|
||||
gets only its own prefix. `src/fuzz/corpus/README` is documentation and
|
||||
is excluded.
|
||||
|
||||
`src/fuzz/corpus/known-findings/K1..K6*.bin` are byte-exact reproducers
|
||||
for the six findings documented in `src/fuzz/README` section 6. They are
|
||||
all `fuzz_request` inputs and are added to that target's seed corpus
|
||||
(prefixed `known-finding-`), which is what turns them into permanent
|
||||
regression tests: ClusterFuzz keeps every seed in the corpus and replays
|
||||
it on every run.
|
||||
|
||||
Regenerating the corpus from the harnesses' built-in seeds:
|
||||
|
||||
make -C src/fuzz refresh-corpus # ./fuzz_<name> --write-corpus=corpus
|
||||
|
||||
That only rewrites the `fuzz_<harness>-NN.bin` files. `known-findings/`
|
||||
is hand-maintained and is never touched by it.
|
||||
|
||||
The script can be run by hand:
|
||||
|
||||
contrib/oss-fuzz/make_seed_corpus.sh . /tmp/out
|
||||
unzip -l /tmp/out/fuzz_request_seed_corpus.zip
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
7. Dictionaries
|
||||
-------------------------------------------------------------------
|
||||
|
||||
`dicts/*.dict` are libFuzzer/AFL dictionaries (`name="value"`, with only
|
||||
`\\`, `\"` and `\xAB` as escapes -- CR and LF are written `\x0d`,
|
||||
`\x0a`). They are installed to `$OUT/<fuzzer>.dict` and referenced from
|
||||
`$OUT/<fuzzer>.options`, which is how ClusterFuzz picks them up.
|
||||
|
||||
They cover: HTTP methods and versions; framing headers
|
||||
(`Transfer-Encoding`, `Content-Length`, `chunked`, conflicting and
|
||||
malformed variants); chunk-size lines and chunk-extension syntax
|
||||
(`;ext`, `;ext=val`, `;ext="quoted"`, unterminated quotes) -- the exact
|
||||
grammar of commit `c13f4c64`; digest-auth parameters (`algorithm=`,
|
||||
`qop=`, `nonce=`, `realm=`, `userhash=`, `username*=`, `nc=`,
|
||||
`response=`) with the algorithm tokens `MD5`, `SHA-256`, `SHA-512-256`
|
||||
and their `-sess` variants plus deliberately unknown ones, and `auth` /
|
||||
`auth-int`; over-long hex `response=` values (commit `5a73c1ae`);
|
||||
percent-encoding, including truncated, invalid, double and overlong
|
||||
forms; base64; and the `multipart/form-data` and
|
||||
`application/x-www-form-urlencoded` vocabulary for the post processor.
|
||||
|
||||
`fuzz_request.dict` additionally contains the literal `%%NONCE%%`
|
||||
placeholder, which the harness rewrites at send time into the most recent
|
||||
nonce the daemon issued. Without it the digest `response=` code path is
|
||||
statistically unreachable (see `src/fuzz/README` section 2.3), so it is
|
||||
the single most valuable token in the file.
|
||||
|
||||
The token list in `fuzz_common.h` (`fuzz_interesting_str`) is the
|
||||
generator's equivalent; the two are intentionally similar but not
|
||||
generated from each other.
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
8. Known limitations
|
||||
-------------------------------------------------------------------
|
||||
|
||||
* **The request-body oracle is inactive under libFuzzer.** The
|
||||
smuggling oracle of `fuzz_request` (see `src/fuzz/README` section
|
||||
2.4) compares what MHD delivers to the application against ground
|
||||
truth declared in an `op 1` segment of the input. It is gated on
|
||||
`fuzz_pristine`, which the standalone driver sets for un-mutated
|
||||
inputs and which is never set when `FUZZ_NO_MAIN` is defined -- i.e.
|
||||
it is off for every OSS-Fuzz execution. That is correct and
|
||||
intentional: libFuzzer mutates the request without mutating the
|
||||
declaration, so the ground truth would be wrong and every mutated
|
||||
input would look like a finding. The consequence is that OSS-Fuzz
|
||||
catches memory-safety bugs, UB, panics and assertion failures, but
|
||||
*not* pure framing/desync defects of the `c13f4c64` kind. Those stay
|
||||
the job of the in-tree driver (`make -C src/fuzz check`), which is
|
||||
another reason to keep running it in CI.
|
||||
* **i386 is not claimed.** MHD's digest buffer sizes depend on the
|
||||
word size (`TESTING.md` section P3), so a 32-bit build is genuinely
|
||||
interesting, but OSS-Fuzz supports i386 only for ASan + libFuzzer and
|
||||
that configuration has not been verified here. Add
|
||||
`- i386` to `architectures:` once it has been.
|
||||
* **centipede is not claimed** for the same reason: it is a supported
|
||||
OSS-Fuzz engine but these targets have not been tried with it.
|
||||
* **Nondeterminism.** See the note about digest nonce timestamps in
|
||||
section 5.
|
||||
* The `[libfuzzer]` section of the `.options` files is the only one
|
||||
used. ClusterFuzz also understands other sections (for sanitizer
|
||||
options and, in some versions, environment variables), but nothing
|
||||
here depends on that, and the exact set of supported sections is not
|
||||
documented in the OSS-Fuzz repository -- if you ever need to pin
|
||||
`MHD_FUZZ_MIN_DISCIPLINE` or `MHD_FUZZ_MIN_MEM_LIMIT` for the hosted
|
||||
runs, verify the mechanism against the ClusterFuzz sources first
|
||||
rather than assuming it works.
|
||||
Executable
+206
@@ -0,0 +1,206 @@
|
||||
#!/bin/bash -eu
|
||||
#
|
||||
# OSS-Fuzz build script for GNU libmicrohttpd.
|
||||
#
|
||||
# This file is in the public domain.
|
||||
#
|
||||
# It is executed inside the OSS-Fuzz base-builder image, which exports:
|
||||
#
|
||||
# $SRC parent directory of the checked-out sources
|
||||
# ($SRC/libmicrohttpd, see Dockerfile)
|
||||
# $WORK scratch directory for build artifacts
|
||||
# $OUT where the finished fuzz targets must be installed
|
||||
# $CC $CXX the instrumented compilers
|
||||
# $CFLAGS $CXXFLAGS sanitizer + coverage flags; MUST be honoured and
|
||||
# MUST NOT be replaced
|
||||
# $LIB_FUZZING_ENGINE the fuzzing engine to link against ("-fsanitize=fuzzer",
|
||||
# a path to libFuzzingEngine.a, the AFL driver, ...)
|
||||
# $SANITIZER address | undefined | memory | coverage
|
||||
# $FUZZING_ENGINE libfuzzer | afl | honggfuzz | centipede | none
|
||||
#
|
||||
# The same script can be run outside OSS-Fuzz for a local smoke test; every
|
||||
# variable above has a defensive default below.
|
||||
#
|
||||
# See contrib/oss-fuzz/README for the full story and for the local
|
||||
# infra/helper.py recipe.
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defaults, so that the script is also runnable by hand
|
||||
# ---------------------------------------------------------------------------
|
||||
SRC="${SRC:-$(cd "$(dirname "$0")/../../.." && pwd)}"
|
||||
WORK="${WORK:-${SRC}/work}"
|
||||
OUT="${OUT:-${SRC}/out}"
|
||||
CC="${CC:-clang}"
|
||||
CXX="${CXX:-clang++}"
|
||||
CFLAGS="${CFLAGS:--O1 -fno-omit-frame-pointer -gline-tables-only}"
|
||||
CXXFLAGS="${CXXFLAGS:-${CFLAGS}}"
|
||||
LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE:--fsanitize=fuzzer}"
|
||||
SANITIZER="${SANITIZER:-address}"
|
||||
|
||||
# Directory holding the libmicrohttpd sources. OSS-Fuzz clones them to
|
||||
# $SRC/libmicrohttpd (see Dockerfile); allow an override for local runs.
|
||||
MHD_SRC="${MHD_SRC:-${SRC}/libmicrohttpd}"
|
||||
|
||||
# Out-of-tree build directory. Keeping the build out of the source tree
|
||||
# means build.sh never modifies the checkout, which matters for the
|
||||
# "run build.sh twice" and "reproduce against a pristine tree" cases.
|
||||
BUILD="${WORK}/mhd-build"
|
||||
|
||||
FUZZERS="fuzz_request fuzz_str fuzz_auth_header fuzz_postprocessor"
|
||||
|
||||
mkdir -p "${WORK}" "${OUT}" "${BUILD}"
|
||||
|
||||
echo "=== libmicrohttpd OSS-Fuzz build ==="
|
||||
echo " MHD_SRC = ${MHD_SRC}"
|
||||
echo " BUILD = ${BUILD}"
|
||||
echo " OUT = ${OUT}"
|
||||
echo " SANITIZER = ${SANITIZER}"
|
||||
echo " LIB_FUZZING_ENGINE = ${LIB_FUZZING_ENGINE}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Bootstrap (the git checkout ships no 'configure')
|
||||
# ---------------------------------------------------------------------------
|
||||
cd "${MHD_SRC}"
|
||||
if [ ! -x ./configure ]; then
|
||||
echo "--- bootstrapping ---"
|
||||
# ./bootstrap swallows its own failures (it ends in an '|| echo ...'
|
||||
# chain), so its exit status cannot be trusted; check for the product
|
||||
# and fall back to autoreconf.
|
||||
./bootstrap || true
|
||||
if [ ! -x ./configure ]; then
|
||||
autoreconf -fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Configure
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rationale for each flag:
|
||||
#
|
||||
# --enable-static --disable-shared
|
||||
# fuzz_str and fuzz_auth_header call MHD-internal symbols
|
||||
# (MHD_hex_to_bin(), MHD_get_rq_dauth_params_(), MHD_pool_create(),
|
||||
# ...) that are compiled with hidden visibility and are therefore
|
||||
# NOT exported from libmicrohttpd.so. Only the static archive can
|
||||
# be linked. Static linking is also what OSS-Fuzz wants: the
|
||||
# target binaries in $OUT must not depend on anything outside $OUT.
|
||||
# --with-pic
|
||||
# keep the static objects position independent so they can be
|
||||
# linked into the (PIE) fuzz targets regardless of compiler default.
|
||||
# --enable-fuzzing
|
||||
# configures src/fuzz/Makefile. Not strictly needed here (the
|
||||
# harnesses are compiled by hand below) but it keeps this build
|
||||
# equivalent to the documented developer build, and it makes
|
||||
# configure fail loudly if src/fuzz/ ever stops being wired up.
|
||||
# --enable-asserts
|
||||
# keeps mhd_assert() alive. Assertions on attacker-reachable paths
|
||||
# are exactly what this campaign is meant to find; without them
|
||||
# findings K1-K6 (src/fuzz/README section 6) are invisible.
|
||||
# --disable-https
|
||||
# deliberate. The harnesses never speak TLS: they hand MHD an
|
||||
# already-connected AF_UNIX socketpair via MHD_add_connection() and
|
||||
# never set MHD_USE_TLS. Enabling HTTPS would (a) add nothing to
|
||||
# coverage, (b) drag GnuTLS - which OSS-Fuzz fuzzes separately -
|
||||
# into the link, and (c) make the MemorySanitizer build impossible
|
||||
# without an MSan-instrumented GnuTLS. With HTTPS off the only
|
||||
# external dependencies are libc and libpthread, so all three
|
||||
# sanitizers are usable.
|
||||
# --disable-curl
|
||||
# the curl-based test suite is not built here and pulling libcurl in
|
||||
# would create the same uninstrumented-dependency problem.
|
||||
# --disable-doc --disable-examples --disable-tools
|
||||
# nothing of that is needed and it only costs build time (and
|
||||
# texinfo/pandoc dependencies).
|
||||
# --disable-dependency-tracking
|
||||
# one-shot build, no need for .deps.
|
||||
# --enable-build-type=neutral
|
||||
# the default, stated explicitly: "neutral" is the one build type
|
||||
# that does NOT inject its own optimisation/debug flags, so the
|
||||
# $CFLAGS handed to us by OSS-Fuzz survive unmodified.
|
||||
#
|
||||
# NOTE: no --enable-sanitizers and no --enable-coverage. OSS-Fuzz supplies
|
||||
# the sanitizer and coverage instrumentation through $CFLAGS; letting
|
||||
# configure add a second, possibly conflicting -fsanitize= set is a classic
|
||||
# way to break an OSS-Fuzz build.
|
||||
cd "${BUILD}"
|
||||
"${MHD_SRC}/configure" \
|
||||
--enable-static \
|
||||
--disable-shared \
|
||||
--with-pic \
|
||||
--enable-fuzzing \
|
||||
--enable-asserts \
|
||||
--disable-https \
|
||||
--disable-curl \
|
||||
--disable-doc \
|
||||
--disable-examples \
|
||||
--disable-tools \
|
||||
--disable-dependency-tracking \
|
||||
--enable-build-type=neutral \
|
||||
CC="${CC}" \
|
||||
CFLAGS="${CFLAGS}" \
|
||||
LDFLAGS="${LDFLAGS:-}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Build the library only
|
||||
# ---------------------------------------------------------------------------
|
||||
# Building just src/microhttpd avoids compiling the (large) test suite and
|
||||
# the src/fuzz check_PROGRAMS, which would be built with the standalone
|
||||
# driver's main() and are useless here.
|
||||
make -j"$(nproc)" -C src/microhttpd libmicrohttpd.la
|
||||
|
||||
MHD_LIB="${BUILD}/src/microhttpd/.libs/libmicrohttpd.a"
|
||||
test -f "${MHD_LIB}" || {
|
||||
echo "ERROR: ${MHD_LIB} was not produced" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Compile the harnesses as libFuzzer translation units
|
||||
# ---------------------------------------------------------------------------
|
||||
# -DFUZZ_NO_MAIN drops the standalone driver's main() from fuzz_common.h;
|
||||
# LLVMFuzzerTestOneInput() itself is unconditional in every harness.
|
||||
#
|
||||
# Include path:
|
||||
# -I${BUILD} for the generated MHD_config.h
|
||||
# -I${MHD_SRC} for the in-tree headers next to configure.ac
|
||||
# -I${MHD_SRC}/src/include for microhttpd.h
|
||||
# -I${MHD_SRC}/src/microhttpd for internal.h, mhd_str.h, gen_auth.h, ...
|
||||
# -I${MHD_SRC}/src/fuzz for fuzz_common.h
|
||||
MHD_INCLUDES=(
|
||||
-I"${BUILD}"
|
||||
-I"${MHD_SRC}"
|
||||
-I"${MHD_SRC}/src/include"
|
||||
-I"${MHD_SRC}/src/microhttpd"
|
||||
-I"${MHD_SRC}/src/fuzz"
|
||||
)
|
||||
|
||||
for fuzzer in ${FUZZERS}; do
|
||||
echo "--- building ${fuzzer} ---"
|
||||
# shellcheck disable=SC2086
|
||||
$CC $CFLAGS \
|
||||
-DFUZZ_NO_MAIN \
|
||||
"${MHD_INCLUDES[@]}" \
|
||||
-c "${MHD_SRC}/src/fuzz/${fuzzer}.c" \
|
||||
-o "${WORK}/${fuzzer}.o"
|
||||
# Link with $CXX: $LIB_FUZZING_ENGINE is a C++ archive for most engines.
|
||||
# shellcheck disable=SC2086
|
||||
$CXX $CXXFLAGS \
|
||||
"${WORK}/${fuzzer}.o" \
|
||||
-o "${OUT}/${fuzzer}" \
|
||||
$LIB_FUZZING_ENGINE \
|
||||
"${MHD_LIB}" \
|
||||
-lpthread
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. Seed corpora, dictionaries and .options files
|
||||
# ---------------------------------------------------------------------------
|
||||
"${MHD_SRC}/contrib/oss-fuzz/make_seed_corpus.sh" "${MHD_SRC}" "${OUT}"
|
||||
|
||||
for fuzzer in ${FUZZERS}; do
|
||||
cp "${MHD_SRC}/contrib/oss-fuzz/dicts/${fuzzer}.dict" "${OUT}/${fuzzer}.dict"
|
||||
cp "${MHD_SRC}/contrib/oss-fuzz/${fuzzer}.options" "${OUT}/${fuzzer}.options"
|
||||
done
|
||||
|
||||
echo "=== done; contents of \$OUT ==="
|
||||
ls -la "${OUT}"
|
||||
@@ -0,0 +1,85 @@
|
||||
# libFuzzer dictionary for fuzz_auth_header (GNU libmicrohttpd).
|
||||
#
|
||||
# The harness feeds byte 1.. as the raw value of an "Authorization:" header
|
||||
# into MHD_get_rq_dauth_params_() / MHD_get_rq_bauth_params_(), so every
|
||||
# token here is a fragment of that header value. Byte 0 is a selector and
|
||||
# is not covered by the dictionary.
|
||||
|
||||
# --- schemes ------------------------------------------------------------
|
||||
s_digest="Digest "
|
||||
s_basic="Basic "
|
||||
s_bearer="Bearer "
|
||||
s_negotiate="Negotiate "
|
||||
s_digest_nosp="Digest"
|
||||
s_digest_tab="Digest\x09"
|
||||
|
||||
# --- separators / quoting ------------------------------------------------
|
||||
comma=","
|
||||
comma_sp=", "
|
||||
equals="="
|
||||
quote="\""
|
||||
empty_quoted="\"\""
|
||||
backslash="\\"
|
||||
escaped_quote="\\\""
|
||||
escaped_bs="\\\\"
|
||||
unterminated_quote="\"aaaa"
|
||||
sp=" "
|
||||
tab="\x09"
|
||||
|
||||
# --- digest parameters ---------------------------------------------------
|
||||
p_username="username="
|
||||
p_username_q="username=\"user\""
|
||||
p_username_star="username*="
|
||||
p_username_star_v="username*=UTF-8''user"
|
||||
p_username_star_bad="username*=''"
|
||||
p_userhash="userhash="
|
||||
p_userhash_true="userhash=true"
|
||||
p_userhash_false="userhash=false"
|
||||
p_userhash_q="userhash=\"true\""
|
||||
p_realm="realm="
|
||||
p_realm_q="realm=\"TestRealm\""
|
||||
p_realm_empty="realm=\"\""
|
||||
p_nonce="nonce="
|
||||
p_nonce_q="nonce=\"0000\""
|
||||
p_uri="uri="
|
||||
p_uri_q="uri=\"/a\""
|
||||
p_response="response="
|
||||
p_response_q="response=\"00\""
|
||||
p_cnonce="cnonce="
|
||||
p_cnonce_q="cnonce=\"abcd\""
|
||||
p_nc="nc="
|
||||
p_nc_v="nc=00000001"
|
||||
p_nc_bad="nc=zzzzzzzz"
|
||||
p_opaque="opaque="
|
||||
p_qop="qop="
|
||||
p_algorithm="algorithm="
|
||||
p_unknown="frobnicate="
|
||||
|
||||
# --- algorithm tokens ----------------------------------------------------
|
||||
a_md5="MD5"
|
||||
a_md5_sess="MD5-sess"
|
||||
a_sha256="SHA-256"
|
||||
a_sha256_sess="SHA-256-sess"
|
||||
a_sha512_256="SHA-512-256"
|
||||
a_sha512_256_sess="SHA-512-256-sess"
|
||||
a_sha512="SHA-512"
|
||||
a_sha1="SHA-1"
|
||||
a_case="sha-256"
|
||||
|
||||
# --- qop tokens ----------------------------------------------------------
|
||||
q_auth="auth"
|
||||
q_auth_int="auth-int"
|
||||
q_both="auth,auth-int"
|
||||
q_quoted="qop=\"auth\""
|
||||
q_unknown="auth-xyz"
|
||||
|
||||
# --- values --------------------------------------------------------------
|
||||
hex16="0123456789abcdef"
|
||||
hex32="0123456789abcdef0123456789abcdef"
|
||||
hex64="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
hex128="00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
|
||||
b64_user_pass="dXNlcjpwYXNz"
|
||||
b64_pad="YQ=="
|
||||
b64_bad="!!!!"
|
||||
pct="%41"
|
||||
pct_bad="%zz"
|
||||
@@ -0,0 +1,61 @@
|
||||
# libFuzzer dictionary for fuzz_postprocessor (GNU libmicrohttpd).
|
||||
#
|
||||
# Bytes 0-3 of the input select the Content-Type shape, the post-processor
|
||||
# buffer size, the chunking pattern and the boundary length; byte 4.. is the
|
||||
# POST body. These tokens are the body syntax of the two encodings
|
||||
# MHD_post_process() understands.
|
||||
|
||||
# --- application/x-www-form-urlencoded ----------------------------------
|
||||
amp="&"
|
||||
equals="="
|
||||
plus="+"
|
||||
semi=";"
|
||||
kv="a=b"
|
||||
kv_empty="a="
|
||||
kv_noeq="a"
|
||||
kv_only_eq="="
|
||||
kv_amp_amp="&&"
|
||||
pct_a="%41"
|
||||
pct_space="%20"
|
||||
pct_nul="%00"
|
||||
pct_cr="%0d"
|
||||
pct_lf="%0a"
|
||||
pct_pct="%25"
|
||||
pct_bad="%zz"
|
||||
pct_trunc="%4"
|
||||
pct_utf8="%C3%A4"
|
||||
pct_overlong="%C0%80"
|
||||
|
||||
# --- multipart/form-data -------------------------------------------------
|
||||
dashdash="--"
|
||||
crlf="\x0d\x0a"
|
||||
crlfcrlf="\x0d\x0a\x0d\x0a"
|
||||
bare_lf="\x0a"
|
||||
bare_cr="\x0d"
|
||||
b_open="--b\x0d\x0a"
|
||||
b_close="--b--\x0d\x0a"
|
||||
b_close_nocrlf="--b--"
|
||||
b_long="--boundary-boundary-boundary\x0d\x0a"
|
||||
cd="Content-Disposition: "
|
||||
cd_form="Content-Disposition: form-data; name=\"a\"\x0d\x0a"
|
||||
cd_file="Content-Disposition: form-data; name=\"a\"; filename=\"f\"\x0d\x0a"
|
||||
cd_attach="Content-Disposition: attachment; filename=\"f\"\x0d\x0a"
|
||||
cd_noname="Content-Disposition: form-data\x0d\x0a"
|
||||
cd_empty_name="Content-Disposition: form-data; name=\"\"\x0d\x0a"
|
||||
p_name="name=\""
|
||||
p_filename="filename=\""
|
||||
p_filename_star="filename*=UTF-8''f"
|
||||
p_boundary="boundary="
|
||||
p_boundary_q="boundary=\"b\""
|
||||
p_charset="charset=UTF-8"
|
||||
h_ct="Content-Type: "
|
||||
ct_text="Content-Type: text/plain\x0d\x0a"
|
||||
ct_octet="Content-Type: application/octet-stream\x0d\x0a"
|
||||
ct_nested="Content-Type: multipart/mixed; boundary=c\x0d\x0a"
|
||||
h_cte="Content-Transfer-Encoding: "
|
||||
cte_binary="Content-Transfer-Encoding: binary\x0d\x0a"
|
||||
cte_b64="Content-Transfer-Encoding: base64\x0d\x0a"
|
||||
cte_qp="Content-Transfer-Encoding: quoted-printable\x0d\x0a"
|
||||
quote="\""
|
||||
escaped_quote="\\\""
|
||||
semi_sp="; "
|
||||
@@ -0,0 +1,143 @@
|
||||
# libFuzzer dictionary for fuzz_request (GNU libmicrohttpd).
|
||||
#
|
||||
# Format: libFuzzer / AFL "name=\"value\"" tokens. Only \\, \" and \xAB are
|
||||
# valid escapes, so CR/LF are written as \x0d / \x0a.
|
||||
#
|
||||
# fuzz_request consumes a header of 4 configuration bytes followed by
|
||||
# length-prefixed send segments (see src/fuzz/README section 2.2), so a
|
||||
# dictionary token is useful mainly inside the segment payload - i.e. these
|
||||
# are the HTTP tokens that the request parser reacts to.
|
||||
|
||||
# --- line terminators and separators ------------------------------------
|
||||
crlf="\x0d\x0a"
|
||||
crlfcrlf="\x0d\x0a\x0d\x0a"
|
||||
bare_cr="\x0d"
|
||||
bare_lf="\x0a"
|
||||
fold="\x0d\x0a\x20"
|
||||
fold_tab="\x0d\x0a\x09"
|
||||
colon_sp=": "
|
||||
colon=":"
|
||||
semi=";"
|
||||
comma=","
|
||||
equals="="
|
||||
quote="\""
|
||||
|
||||
# --- request line -------------------------------------------------------
|
||||
m_get="GET "
|
||||
m_head="HEAD "
|
||||
m_post="POST "
|
||||
m_put="PUT "
|
||||
m_delete="DELETE "
|
||||
m_options="OPTIONS "
|
||||
m_connect="CONNECT "
|
||||
m_trace="TRACE "
|
||||
m_patch="PATCH "
|
||||
v_11=" HTTP/1.1"
|
||||
v_10=" HTTP/1.0"
|
||||
v_09=" HTTP/0.9"
|
||||
v_20=" HTTP/2.0"
|
||||
v_bad=" HTTP/1."
|
||||
tgt_root="/"
|
||||
tgt_star="*"
|
||||
tgt_abs="http://example.org/a"
|
||||
tgt_query="/a?b"
|
||||
tgt_query_noeq="/a?b&c"
|
||||
tgt_dotdot="/../"
|
||||
tgt_pct="%41"
|
||||
tgt_pct_bad="%zz"
|
||||
tgt_pct_trunc="%4"
|
||||
tgt_nul="%00"
|
||||
|
||||
# --- framing headers ----------------------------------------------------
|
||||
h_host="Host: "
|
||||
h_cl="Content-Length: "
|
||||
h_te="Transfer-Encoding: "
|
||||
h_te_chunked="Transfer-Encoding: chunked\x0d\x0a"
|
||||
h_te_ident="Transfer-Encoding: identity\x0d\x0a"
|
||||
h_te_double="Transfer-Encoding: chunked, chunked\x0d\x0a"
|
||||
h_conn="Connection: "
|
||||
h_conn_close="Connection: close\x0d\x0a"
|
||||
h_conn_ka="Connection: keep-alive\x0d\x0a"
|
||||
h_conn_upgrade="Connection: Upgrade\x0d\x0a"
|
||||
h_upgrade="Upgrade: "
|
||||
h_expect="Expect: 100-continue\x0d\x0a"
|
||||
h_ct="Content-Type: "
|
||||
h_cookie="Cookie: "
|
||||
h_trailer="Trailer: "
|
||||
h_te_hdr="TE: "
|
||||
v_chunked="chunked"
|
||||
v_identity="identity"
|
||||
v_gzip="gzip"
|
||||
v_100="100-continue"
|
||||
cl_zero="Content-Length: 0\x0d\x0a"
|
||||
cl_neg="Content-Length: -1\x0d\x0a"
|
||||
cl_huge="Content-Length: 18446744073709551615\x0d\x0a"
|
||||
cl_plus="Content-Length: +5\x0d\x0a"
|
||||
cl_hex="Content-Length: 0x5\x0d\x0a"
|
||||
cl_dup="Content-Length: 5\x0d\x0aContent-Length: 6\x0d\x0a"
|
||||
|
||||
# --- chunked transfer coding & chunk extensions -------------------------
|
||||
chunk_last="0\x0d\x0a\x0d\x0a"
|
||||
chunk_last_nl="0\x0d\x0a"
|
||||
chunk_5="5\x0d\x0a"
|
||||
chunk_hex="ff\x0d\x0a"
|
||||
chunk_big="7fffffffffffffff\x0d\x0a"
|
||||
chunk_ext=";ext"
|
||||
chunk_ext_val=";ext=val"
|
||||
chunk_ext_quoted=";ext=\"quoted\""
|
||||
chunk_ext_multi=";a=1;b=2;c"
|
||||
chunk_ext_unterminated=";ext=\"qqqqqqqqqqqqqqqq"
|
||||
chunk_ext_bs=";ext=\"a\\\"b\""
|
||||
chunk_ext_ws="; ext = val "
|
||||
trailer_line="X-Trailer: v\x0d\x0a"
|
||||
|
||||
# --- digest / basic authentication --------------------------------------
|
||||
h_authz="Authorization: "
|
||||
a_digest="Authorization: Digest "
|
||||
a_basic="Authorization: Basic "
|
||||
p_username="username=\""
|
||||
p_userhash="userhash="
|
||||
p_userhash_t="userhash=true"
|
||||
p_userhash_f="userhash=false"
|
||||
p_username_star="username*=UTF-8''a"
|
||||
p_realm="realm=\""
|
||||
p_realm_empty="realm=\"\""
|
||||
p_nonce="nonce=\""
|
||||
p_nonce_ph="%%NONCE%%"
|
||||
p_nonce_zero="nonce=\"0000\""
|
||||
p_uri="uri=\""
|
||||
p_response="response=\""
|
||||
p_cnonce="cnonce=\""
|
||||
p_nc="nc=00000001"
|
||||
p_opaque="opaque=\""
|
||||
p_qop="qop="
|
||||
p_algorithm="algorithm="
|
||||
a_md5="MD5"
|
||||
a_md5_sess="MD5-sess"
|
||||
a_sha256="SHA-256"
|
||||
a_sha256_sess="SHA-256-sess"
|
||||
a_sha512_256="SHA-512-256"
|
||||
a_sha512_256_sess="SHA-512-256-sess"
|
||||
a_unknown="SHA-1"
|
||||
q_auth="auth"
|
||||
q_auth_int="auth-int"
|
||||
q_auth_both="auth,auth-int"
|
||||
resp_hex="0123456789abcdef"
|
||||
resp_long="00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
|
||||
basic_b64="dXNlcjpwYXNz"
|
||||
scheme_bearer="Bearer "
|
||||
scheme_negotiate="Negotiate "
|
||||
|
||||
# --- post bodies (the handler may run MHD_post_process) -----------------
|
||||
ct_urlenc="Content-Type: application/x-www-form-urlencoded\x0d\x0a"
|
||||
ct_multipart="Content-Type: multipart/form-data; boundary=b\x0d\x0a"
|
||||
ct_mixed="Content-Type: multipart/mixed; boundary=b\x0d\x0a"
|
||||
mp_dashes="--"
|
||||
mp_boundary="--b\x0d\x0a"
|
||||
mp_end="--b--\x0d\x0a"
|
||||
cd_form="Content-Disposition: form-data; name=\"a\"\x0d\x0a"
|
||||
cd_file="Content-Disposition: form-data; name=\"a\"; filename=\"f\"\x0d\x0a"
|
||||
kv="a=b"
|
||||
kv_amp="&"
|
||||
kv_plus="+"
|
||||
kv_noeq="a"
|
||||
@@ -0,0 +1,95 @@
|
||||
# libFuzzer dictionary for fuzz_str (GNU libmicrohttpd).
|
||||
#
|
||||
# Byte 0 selects the target primitive in src/microhttpd/mhd_str.c
|
||||
# (hex_to_bin, bin_to_hex[_z], pct_decode strict/lenient, in-place pct
|
||||
# decode, unquote, quote, base64, str_to_uint64, token list handling,
|
||||
# equal_caseless), byte 1 is an auxiliary knob, byte 2.. is the payload
|
||||
# (truncated to 512 bytes by the harness). These tokens are payload
|
||||
# vocabulary.
|
||||
|
||||
# --- hex (MHD_hex_to_bin / MHD_bin_to_hex) ------------------------------
|
||||
hex_digits="0123456789abcdef"
|
||||
hex_upper="0123456789ABCDEF"
|
||||
hex_odd="abc"
|
||||
hex_32="0123456789abcdef0123456789abcdef"
|
||||
hex_64="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
hex_128="00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
|
||||
hex_bad="0g"
|
||||
hex_0x="0x41"
|
||||
|
||||
# --- percent-encoding ----------------------------------------------------
|
||||
pct="%"
|
||||
pct_a="%41"
|
||||
pct_lower="%61"
|
||||
pct_space="%20"
|
||||
pct_plus="%2B"
|
||||
pct_pct="%25"
|
||||
pct_nul="%00"
|
||||
pct_cr="%0D"
|
||||
pct_lf="%0A"
|
||||
pct_slash="%2F"
|
||||
pct_bad="%zz"
|
||||
pct_trunc="%4"
|
||||
pct_trunc2="%"
|
||||
pct_double="%2525"
|
||||
pct_utf8="%C3%A4"
|
||||
pct_overlong="%C0%80"
|
||||
pct_surrogate="%ED%A0%80"
|
||||
plus="+"
|
||||
|
||||
# --- quoted-string (MHD_str_unquote / MHD_str_quote) --------------------
|
||||
quote="\""
|
||||
empty_quoted="\"\""
|
||||
escaped_quote="\\\""
|
||||
escaped_bs="\\\\"
|
||||
bs="\\"
|
||||
bs_at_end="a\\"
|
||||
quoted_word="\"abc\""
|
||||
quoted_ws="\" \""
|
||||
quoted_ctl="\"\x01\""
|
||||
quoted_cr="\"\x0d\""
|
||||
|
||||
# --- base64 --------------------------------------------------------------
|
||||
b64_user_pass="dXNlcjpwYXNz"
|
||||
b64_pad1="YQ=="
|
||||
b64_pad2="YWI="
|
||||
b64_nopad="YWJj"
|
||||
b64_bad="!!!!"
|
||||
b64_ws="YWJj\x20"
|
||||
b64_lf="YWJj\x0a"
|
||||
b64_url="-_"
|
||||
b64_eq="="
|
||||
|
||||
# --- MHD_str_to_uint64 ---------------------------------------------------
|
||||
n_zero="0"
|
||||
n_one="1"
|
||||
n_max64="18446744073709551615"
|
||||
n_over64="18446744073709551616"
|
||||
n_neg="-1"
|
||||
n_plus="+1"
|
||||
n_lead0="000000000000000000001"
|
||||
n_hex="0x10"
|
||||
n_ws=" 1"
|
||||
n_dot="1.5"
|
||||
|
||||
# --- token lists (MHD_str_remove_token_caseless_ etc.) ------------------
|
||||
comma=","
|
||||
comma_sp=", "
|
||||
sp=" "
|
||||
tab="\x09"
|
||||
crlf="\x0d\x0a"
|
||||
semi=";"
|
||||
equals="="
|
||||
t_chunked="chunked"
|
||||
t_identity="identity"
|
||||
t_gzip="gzip"
|
||||
t_close="close"
|
||||
t_keepalive="keep-alive"
|
||||
t_upgrade="Upgrade"
|
||||
t_100="100-continue"
|
||||
t_md5="MD5"
|
||||
t_sha256="SHA-256"
|
||||
t_sha512_256="SHA-512-256"
|
||||
t_auth="auth"
|
||||
t_auth_int="auth-int"
|
||||
t_empty_item=",,"
|
||||
@@ -0,0 +1,3 @@
|
||||
[libfuzzer]
|
||||
dict = fuzz_auth_header.dict
|
||||
max_len = 4097
|
||||
@@ -0,0 +1,3 @@
|
||||
[libfuzzer]
|
||||
dict = fuzz_postprocessor.dict
|
||||
max_len = 8196
|
||||
@@ -0,0 +1,3 @@
|
||||
[libfuzzer]
|
||||
dict = fuzz_request.dict
|
||||
max_len = 8192
|
||||
@@ -0,0 +1,3 @@
|
||||
[libfuzzer]
|
||||
dict = fuzz_str.dict
|
||||
max_len = 514
|
||||
Executable
+85
@@ -0,0 +1,85 @@
|
||||
#!/bin/bash -eu
|
||||
#
|
||||
# Package src/fuzz/corpus/ into the $OUT/<fuzzer>_seed_corpus.zip files that
|
||||
# OSS-Fuzz/ClusterFuzz picks up automatically.
|
||||
#
|
||||
# This file is in the public domain.
|
||||
#
|
||||
# Usage: make_seed_corpus.sh [SRCDIR] [OUTDIR]
|
||||
#
|
||||
# SRCDIR top of the libmicrohttpd source tree (default: derived from $0)
|
||||
# OUTDIR where the zips are written (default: $OUT, else ./out)
|
||||
#
|
||||
# Corpus layout in src/fuzz/corpus/:
|
||||
#
|
||||
# fuzz_<harness>-NN.bin per-harness seeds; the file name prefix is the
|
||||
# harness the seed belongs to. Inputs are NOT
|
||||
# interchangeable between harnesses: byte 0 of
|
||||
# every harness input selects a different thing.
|
||||
# known-findings/K*.bin byte-exact reproducers for findings K1-K6
|
||||
# (src/fuzz/README section 6). All of them are
|
||||
# fuzz_request inputs, so they go into that
|
||||
# harness' seed corpus, where OSS-Fuzz will keep
|
||||
# re-running them forever - i.e. they become
|
||||
# permanent regression tests.
|
||||
# README documentation, not an input; excluded.
|
||||
#
|
||||
# The corpus itself is regenerated from the harnesses' built-in seeds with
|
||||
# make -C src/fuzz refresh-corpus
|
||||
# (which runs "./fuzz_<name> --write-corpus=corpus" for every harness).
|
||||
# known-findings/ is hand-maintained and is never touched by that.
|
||||
|
||||
SELF_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SRCDIR="${1:-$(cd "${SELF_DIR}/../.." && pwd)}"
|
||||
OUTDIR="${2:-${OUT:-$(pwd)/out}}"
|
||||
|
||||
CORPUS="${SRCDIR}/src/fuzz/corpus"
|
||||
FINDINGS="${CORPUS}/known-findings"
|
||||
|
||||
FUZZERS="fuzz_request fuzz_str fuzz_auth_header fuzz_postprocessor"
|
||||
|
||||
if [ ! -d "${CORPUS}" ]; then
|
||||
echo "ERROR: no corpus directory at ${CORPUS}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${OUTDIR}"
|
||||
|
||||
STAGE="$(mktemp -d "${TMPDIR:-/tmp}/mhd-seed-corpus.XXXXXX")"
|
||||
trap 'rm -rf "${STAGE}"' EXIT
|
||||
|
||||
for fuzzer in ${FUZZERS}; do
|
||||
dir="${STAGE}/${fuzzer}"
|
||||
mkdir -p "${dir}"
|
||||
|
||||
n=0
|
||||
for f in "${CORPUS}/${fuzzer}"-*.bin; do
|
||||
[ -f "${f}" ] || continue
|
||||
cp "${f}" "${dir}/$(basename "${f}")"
|
||||
n=$((n + 1))
|
||||
done
|
||||
|
||||
# The K1-K6 reproducers are fuzz_request inputs.
|
||||
if [ "${fuzzer}" = "fuzz_request" ] && [ -d "${FINDINGS}" ]; then
|
||||
for f in "${FINDINGS}"/*.bin; do
|
||||
[ -f "${f}" ] || continue
|
||||
cp "${f}" "${dir}/known-finding-$(basename "${f}")"
|
||||
n=$((n + 1))
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "${n}" -eq 0 ]; then
|
||||
echo "ERROR: no seeds found for ${fuzzer} in ${CORPUS}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
zip_path="${OUTDIR}/${fuzzer}_seed_corpus.zip"
|
||||
rm -f "${zip_path}"
|
||||
# -j: flat archive, which is what ClusterFuzz expects.
|
||||
# -X: no extra file attributes, so the zip stays reproducible.
|
||||
zip -q -j -X "${zip_path}" "${dir}"/* || {
|
||||
echo "ERROR: failed to create ${zip_path}" >&2
|
||||
exit 1
|
||||
}
|
||||
echo " ${zip_path}: ${n} seed(s)"
|
||||
done
|
||||
@@ -0,0 +1,54 @@
|
||||
#
|
||||
# OSS-Fuzz project configuration for GNU libmicrohttpd.
|
||||
#
|
||||
# This file should become projects/libmicrohttpd/project.yaml in the
|
||||
# google/oss-fuzz repository.
|
||||
#
|
||||
|
||||
homepage: "https://www.gnu.org/software/libmicrohttpd/"
|
||||
main_repo: "https://git.gnunet.org/libmicrohttpd.git"
|
||||
|
||||
language: c
|
||||
primary_contact: "grothoff@gmail.com"
|
||||
|
||||
# address: the primary target. The bug class that motivated this work
|
||||
# (commit 5a73c1ae, a stack buffer overflow in the digest-auth hex
|
||||
# decoder) is exactly what ASan catches, and the harnesses are
|
||||
# written for it: every output buffer in fuzz_str and
|
||||
# fuzz_auth_header is an exactly-sized heap allocation so that the
|
||||
# ASan redzone traps a single-byte overrun.
|
||||
# undefined: MHD's parsers do a lot of pointer arithmetic and integer width
|
||||
# juggling on attacker-controlled lengths (the read-buffer
|
||||
# shift-back underflow of 29eaa56b is of that family). UBSan is
|
||||
# cheap and catches those before they become memory errors.
|
||||
# memory: usable here only because build.sh configures --disable-https and
|
||||
# --disable-curl, which leaves libc and libpthread as the only
|
||||
# external dependencies; there is no uninstrumented third-party
|
||||
# library to poison the results. MSan finds the use of
|
||||
# uninitialised parser state that ASan cannot see.
|
||||
sanitizers:
|
||||
- address
|
||||
- undefined
|
||||
- memory
|
||||
|
||||
# Only x86_64 is claimed here because it is the only architecture this
|
||||
# configuration has been exercised on. i386 is interesting for MHD (the
|
||||
# severity of the MAX_DIGEST overflow depends on the word size, see
|
||||
# TESTING.md section P3) and can be added later, but OSS-Fuzz supports i386
|
||||
# only for the address sanitizer with libFuzzer, and the build has to be
|
||||
# verified first.
|
||||
architectures:
|
||||
- x86_64
|
||||
|
||||
# The harnesses are plain LLVMFuzzerTestOneInput() targets with no engine
|
||||
# specific code, so every in-process engine works. ("centipede" is
|
||||
# deliberately not listed: it is supported by OSS-Fuzz but has not been
|
||||
# tried against these targets.)
|
||||
fuzzing_engines:
|
||||
- libfuzzer
|
||||
- afl
|
||||
- honggfuzz
|
||||
|
||||
# Documentation a triager should read before filing/handling a report.
|
||||
help_url: "https://git.gnunet.org/libmicrohttpd.git/tree/src/fuzz/README"
|
||||
|
||||
Executable
+343
@@ -0,0 +1,343 @@
|
||||
#!/bin/sh
|
||||
# This file is in the public domain.
|
||||
#
|
||||
# Re-run already built libmicrohttpd test binaries across the daemon option
|
||||
# matrix defined in src/microhttpd/mhd_opt_matrix.c.
|
||||
#
|
||||
# See --help below for the full documentation.
|
||||
|
||||
set -u
|
||||
|
||||
me=`basename "$0"`
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defaults
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# The binaries that honour the matrix (they read the MHD_TEST_* environment
|
||||
# variables), plus a few libcurl based ones that at least get re-run in every
|
||||
# threading/polling mode of the daemon they start themselves.
|
||||
default_tests="src/microhttpd/test_option_matrix
|
||||
src/microhttpd/test_rq_shift_back
|
||||
src/microhttpd/test_chunked_ext
|
||||
src/microhttpd/test_dauth_malformed
|
||||
src/microhttpd/test_raw_requests
|
||||
src/testcurl/test_get
|
||||
src/testcurl/test_post
|
||||
src/testcurl/test_put_chunked
|
||||
src/testcurl/test_process_headers"
|
||||
|
||||
profiles_arg=""
|
||||
jobs=1
|
||||
builddir=""
|
||||
timeout_sec=600
|
||||
verbose=0
|
||||
list_only=0
|
||||
|
||||
usage ()
|
||||
{
|
||||
cat <<EOF
|
||||
Usage: $me [OPTION]... [TEST-BINARY]...
|
||||
|
||||
Re-runs libmicrohttpd test binaries across the daemon option matrix. The
|
||||
matrix is defined once, in src/microhttpd/mhd_opt_matrix.c, and is selected
|
||||
per run through the environment, so nothing has to be recompiled:
|
||||
|
||||
MHD_TEST_PROFILE the profile, by name or by index
|
||||
MHD_TEST_MEM_LIMIT override MHD_OPTION_CONNECTION_MEMORY_LIMIT
|
||||
MHD_TEST_DISCIPLINE override MHD_OPTION_CLIENT_DISCIPLINE_LVL
|
||||
MHD_TEST_STRICT_FOR_CLIENT override the same knob via the deprecated
|
||||
MHD_OPTION_STRICT_FOR_CLIENT
|
||||
MHD_TEST_THREADING external | internal | per-connection | pool
|
||||
MHD_TEST_POLL select | poll | epoll
|
||||
|
||||
This script only sets MHD_TEST_PROFILE; the other variables stay available
|
||||
for a manual run.
|
||||
|
||||
Options:
|
||||
--profiles=LIST comma separated profile names or indices to visit
|
||||
(default: every profile of the matrix)
|
||||
--jobs=N run up to N test processes in parallel (default: 1)
|
||||
--builddir=DIR the root of the built tree (default: the directory this
|
||||
script is called from, or the parent of contrib/)
|
||||
--timeout=SEC kill a single run after SEC seconds (default: $timeout_sec)
|
||||
--list-profiles print the profiles of the matrix and exit
|
||||
--verbose echo the output of every run
|
||||
--help print this help and exit
|
||||
|
||||
Arguments:
|
||||
TEST-BINARY... the binaries to run, relative to the build directory or
|
||||
absolute. The default set is:
|
||||
EOF
|
||||
echo "$default_tests" | sed 's/^/ /'
|
||||
cat <<EOF
|
||||
|
||||
Only the src/microhttpd binaries listed above read the
|
||||
MHD_TEST_* variables; the src/testcurl ones ignore them
|
||||
and are simply re-run, which is still useful as a
|
||||
stability check but does not vary the configuration.
|
||||
|
||||
Exit status:
|
||||
0 every run passed (or was skipped)
|
||||
1 at least one run failed
|
||||
2 usage error or nothing could be run
|
||||
|
||||
A run that exits with 77 is reported as SKIP and does not fail the script;
|
||||
99 is reported as ERROR and does fail it.
|
||||
EOF
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Argument parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
tests=""
|
||||
while [ $# -gt 0 ] ; do
|
||||
case "$1" in
|
||||
--help | -h)
|
||||
usage ; exit 0 ;;
|
||||
--profiles=*)
|
||||
profiles_arg=`expr "x$1" : 'x--profiles=\(.*\)'` ;;
|
||||
--jobs=*)
|
||||
jobs=`expr "x$1" : 'x--jobs=\(.*\)'` ;;
|
||||
-j)
|
||||
shift ; jobs="${1:-1}" ;;
|
||||
--builddir=*)
|
||||
builddir=`expr "x$1" : 'x--builddir=\(.*\)'` ;;
|
||||
--timeout=*)
|
||||
timeout_sec=`expr "x$1" : 'x--timeout=\(.*\)'` ;;
|
||||
--list-profiles)
|
||||
list_only=1 ;;
|
||||
--verbose | -v)
|
||||
verbose=1 ;;
|
||||
-*)
|
||||
echo "$me: unknown option '$1'; try '$me --help'." >&2 ; exit 2 ;;
|
||||
*)
|
||||
tests="$tests
|
||||
$1" ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
case "$jobs" in
|
||||
'' | *[!0-9]*) echo "$me: --jobs needs a number." >&2 ; exit 2 ;;
|
||||
esac
|
||||
[ "$jobs" -ge 1 ] || jobs=1
|
||||
case "$timeout_sec" in
|
||||
'' | *[!0-9]*) echo "$me: --timeout needs a number." >&2 ; exit 2 ;;
|
||||
esac
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Locate the built tree
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
if [ -z "$builddir" ] ; then
|
||||
if [ -x "src/microhttpd/test_option_matrix" ] ; then
|
||||
builddir="."
|
||||
else
|
||||
d=`dirname "$0"`/..
|
||||
if [ -x "$d/src/microhttpd/test_option_matrix" ] ; then
|
||||
builddir="$d"
|
||||
else
|
||||
builddir="."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
if [ ! -d "$builddir" ] ; then
|
||||
echo "$me: '$builddir' is not a directory." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
matrix_bin="$builddir/src/microhttpd/test_option_matrix"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The list of profiles
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# The matrix is defined in exactly one place: ask the test binary for it.
|
||||
# The fallback list is only used when the binary has not been built yet, so
|
||||
# that --list-profiles still says something useful.
|
||||
fallback_profiles="default
|
||||
mem-64
|
||||
mem-128
|
||||
mem-256
|
||||
mem-512
|
||||
mem-1024
|
||||
mem-2048
|
||||
mem-3072
|
||||
mem-4096
|
||||
strict-1
|
||||
strict-2
|
||||
strict-3
|
||||
lax-1
|
||||
lax-2
|
||||
lax-3
|
||||
legacy-strict
|
||||
legacy-lax"
|
||||
|
||||
if [ -x "$matrix_bin" ] ; then
|
||||
all_profiles=`"$matrix_bin" --list-profiles 2>/dev/null` || all_profiles=""
|
||||
fi
|
||||
if [ -z "${all_profiles:-}" ] ; then
|
||||
all_profiles="$fallback_profiles"
|
||||
fi
|
||||
|
||||
if [ -n "$profiles_arg" ] ; then
|
||||
profiles=`echo "$profiles_arg" | tr ',' '\n' | sed '/^$/d'`
|
||||
else
|
||||
profiles="$all_profiles"
|
||||
fi
|
||||
|
||||
if [ "$list_only" -eq 1 ] ; then
|
||||
echo "$all_profiles"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -z "$tests" ] ; then
|
||||
tests="$default_tests"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Run
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
workdir=`mktemp -d "${TMPDIR:-/tmp}/mhd-option-matrix.XXXXXX"` || {
|
||||
echo "$me: cannot create a temporary directory." >&2 ; exit 2 ; }
|
||||
trap 'rm -rf "$workdir"' 0
|
||||
trap 'rm -rf "$workdir" ; exit 130' 1 2 3 15
|
||||
|
||||
timeout_cmd=""
|
||||
if command -v timeout >/dev/null 2>&1 ; then
|
||||
timeout_cmd="timeout -k 5 $timeout_sec"
|
||||
fi
|
||||
|
||||
# Run one (profile, binary) pair and record the outcome.
|
||||
# $1 profile, $2 binary, $3 result file
|
||||
run_one ()
|
||||
{
|
||||
_prof="$1"
|
||||
_bin="$2"
|
||||
_out="$3"
|
||||
_path="$_bin"
|
||||
case "$_path" in
|
||||
/*) ;;
|
||||
*) _path="$builddir/$_bin" ;;
|
||||
esac
|
||||
if [ ! -x "$_path" ] ; then
|
||||
echo "MISSING $_prof $_bin" > "$_out"
|
||||
return 0
|
||||
fi
|
||||
MHD_TEST_PROFILE="$_prof" ; export MHD_TEST_PROFILE
|
||||
# shellcheck disable=SC2086
|
||||
$timeout_cmd "$_path" > "$_out.log" 2>&1
|
||||
_rc=$?
|
||||
case "$_rc" in
|
||||
0) echo "PASS $_prof $_bin" > "$_out" ;;
|
||||
77) echo "SKIP $_prof $_bin" > "$_out" ;;
|
||||
99) echo "ERROR $_prof $_bin (exit 99)" > "$_out" ;;
|
||||
124 | 137)
|
||||
echo "TIMEOUT $_prof $_bin (after ${timeout_sec}s)" > "$_out" ;;
|
||||
*) echo "FAIL $_prof $_bin (exit $_rc)" > "$_out" ;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
|
||||
n=0
|
||||
running=0
|
||||
for prof in $profiles ; do
|
||||
for t in $tests ; do
|
||||
n=`expr $n + 1`
|
||||
res="$workdir/r$n"
|
||||
echo "$prof" > "$res.prof"
|
||||
echo "$t" > "$res.bin"
|
||||
if [ "$jobs" -gt 1 ] ; then
|
||||
run_one "$prof" "$t" "$res" &
|
||||
running=`expr $running + 1`
|
||||
if [ "$running" -ge "$jobs" ] ; then
|
||||
wait
|
||||
running=0
|
||||
fi
|
||||
else
|
||||
run_one "$prof" "$t" "$res"
|
||||
if [ "$verbose" -eq 1 ] ; then
|
||||
cat "$res" 2>/dev/null
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
wait
|
||||
|
||||
if [ "$n" -eq 0 ] ; then
|
||||
echo "$me: nothing to run." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
total=0
|
||||
n_pass=0
|
||||
n_skip=0
|
||||
n_fail=0
|
||||
n_missing=0
|
||||
|
||||
echo ""
|
||||
echo "=========================================================================="
|
||||
echo " option matrix summary"
|
||||
echo "=========================================================================="
|
||||
|
||||
i=0
|
||||
for prof in $profiles ; do
|
||||
line=""
|
||||
bad=0
|
||||
base=$i
|
||||
for t in $tests ; do
|
||||
i=`expr $i + 1`
|
||||
res="$workdir/r$i"
|
||||
if [ -f "$res" ] ; then
|
||||
status=`cut -d' ' -f1 < "$res"`
|
||||
else
|
||||
status="ERROR"
|
||||
fi
|
||||
total=`expr $total + 1`
|
||||
case "$status" in
|
||||
PASS) n_pass=`expr $n_pass + 1` ;;
|
||||
SKIP) n_skip=`expr $n_skip + 1` ;;
|
||||
MISSING) n_missing=`expr $n_missing + 1` ;;
|
||||
*) n_fail=`expr $n_fail + 1` ; bad=1 ;;
|
||||
esac
|
||||
line="$line $status"
|
||||
done
|
||||
printf '%-16s%s\n' "$prof" "$line"
|
||||
if [ "$bad" -ne 0 ] || [ "$verbose" -eq 1 ] ; then
|
||||
j=$base
|
||||
for t in $tests ; do
|
||||
j=`expr $j + 1`
|
||||
res="$workdir/r$j"
|
||||
[ -f "$res" ] || continue
|
||||
status=`cut -d' ' -f1 < "$res"`
|
||||
case "$status" in
|
||||
PASS | SKIP | MISSING) continue ;;
|
||||
esac
|
||||
echo " --- $t: `cat "$res"`"
|
||||
if [ -f "$res.log" ] ; then
|
||||
sed 's/^/ /' < "$res.log" | tail -n 25
|
||||
fi
|
||||
done
|
||||
fi
|
||||
done
|
||||
|
||||
echo "--------------------------------------------------------------------------"
|
||||
echo "binaries, in the order of the columns above:"
|
||||
for t in $tests ; do
|
||||
echo " $t"
|
||||
done
|
||||
echo "--------------------------------------------------------------------------"
|
||||
echo "total $total, pass $n_pass, skip $n_skip, fail $n_fail, missing $n_missing"
|
||||
if [ "$n_fail" -ne 0 ] ; then
|
||||
echo "RESULT: FAILED"
|
||||
exit 1
|
||||
fi
|
||||
echo "RESULT: OK"
|
||||
exit 0
|
||||
@@ -9,6 +9,10 @@ SUBDIRS += testzzuf
|
||||
endif
|
||||
endif
|
||||
|
||||
if ENABLE_FUZZING
|
||||
SUBDIRS += fuzz
|
||||
endif
|
||||
|
||||
if BUILD_EXAMPLES
|
||||
SUBDIRS += examples
|
||||
endif
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# The built fuzzing harnesses.
|
||||
/fuzz_request
|
||||
/fuzz_str
|
||||
/fuzz_auth_header
|
||||
/fuzz_postprocessor
|
||||
|
||||
# Reproducers written for a failing input (MHD_FUZZ_CRASH_DIR).
|
||||
/crashes/
|
||||
|
||||
# Seed corpora packaged for OSS-Fuzz by contrib/oss-fuzz/make_seed_corpus.sh.
|
||||
/*_seed_corpus.zip
|
||||
@@ -0,0 +1,229 @@
|
||||
# Build integration for `src/fuzz/`
|
||||
|
||||
> **Status: applied.** The three changes described below are already
|
||||
> present in this tree — `configure.ac` carries the `--enable-fuzzing`
|
||||
> option and the `ENABLE_FUZZING` conditional, `AC_CONFIG_FILES` lists
|
||||
> `src/fuzz/Makefile`, and `src/Makefile.am` adds `fuzz` to `SUBDIRS`
|
||||
> under that conditional. Only §1.3 (the optional configuration-summary
|
||||
> line) is *not* applied. The sections below are kept as the record of
|
||||
> what was changed and as the recipe for porting `src/fuzz/` to another
|
||||
> branch.
|
||||
>
|
||||
> For running these same harnesses on OSS-Fuzz — a different build path
|
||||
> that does **not** go through `src/fuzz/Makefile.am` — see
|
||||
> `../../contrib/oss-fuzz/` and §6 at the end of this file.
|
||||
|
||||
`src/fuzz/Makefile.am` is complete. Three files outside `src/fuzz/`
|
||||
have to be touched. The snippets below are the *exact* text that was
|
||||
added; nothing else changes.
|
||||
|
||||
The harnesses are guarded by a new automake conditional
|
||||
`ENABLE_FUZZING` (`--enable-fuzzing`, default **no**), because they are
|
||||
only meaningful in a build with sanitizers and assertions, and because
|
||||
they need a static archive of the library (they call functions that are
|
||||
hidden in the shared object).
|
||||
|
||||
---
|
||||
|
||||
## 1. `configure.ac`
|
||||
|
||||
### 1.1 The `--enable-fuzzing` option and the conditional
|
||||
|
||||
Add this next to the other feature checks — a good spot is right after
|
||||
the `AM_CONDITIONAL([RUN_ZZUF_TESTS], ...)` /
|
||||
`AM_CONDITIONAL([FORCE_USE_ZZUF_SOCAT], ...)` pair (around line 5833 in
|
||||
the v1.0.7 tree):
|
||||
|
||||
```m4
|
||||
# Fuzzing harnesses (src/fuzz)
|
||||
AC_MSG_CHECKING([[whether to build the fuzzing harnesses]])
|
||||
AC_ARG_ENABLE([[fuzzing]],
|
||||
[AS_HELP_STRING([[--enable-fuzzing]],
|
||||
[build the in-process fuzzing harnesses in src/fuzz and run them ]
|
||||
[as part of "make check"; requires a static build of the library ]
|
||||
[and is only really useful together with --enable-asserts and ]
|
||||
[--enable-sanitizers=address,undefined [no]])],
|
||||
[], [enable_fuzzing="no"])
|
||||
AS_VAR_IF([enable_fuzzing], ["yes"],
|
||||
[
|
||||
AS_VAR_IF([enable_static], ["no"],
|
||||
[AC_MSG_RESULT([[no]])
|
||||
AC_MSG_ERROR([[--enable-fuzzing requires --enable-static]])],
|
||||
[AC_MSG_RESULT([[yes]])])
|
||||
],
|
||||
[enable_fuzzing="no"
|
||||
AC_MSG_RESULT([[no]])])
|
||||
AM_CONDITIONAL([ENABLE_FUZZING], [[test "x$enable_fuzzing" = "xyes"]])
|
||||
```
|
||||
|
||||
### 1.2 Register the new `Makefile`
|
||||
|
||||
In the final `AC_CONFIG_FILES([...])` list (around line 5931 of the
|
||||
v1.0.7 tree) add `src/fuzz/Makefile`, i.e. change
|
||||
|
||||
```
|
||||
src/microhttpd/Makefile
|
||||
...
|
||||
src/testzzuf/Makefile])
|
||||
```
|
||||
|
||||
to
|
||||
|
||||
```
|
||||
src/microhttpd/Makefile
|
||||
...
|
||||
src/testzzuf/Makefile
|
||||
src/fuzz/Makefile])
|
||||
```
|
||||
|
||||
### 1.3 (optional, NOT applied) mention it in the configuration summary
|
||||
|
||||
If you want the harness state to show up in the final summary, add a
|
||||
line next to the other `AC_MSG_NOTICE` entries:
|
||||
|
||||
```m4
|
||||
AC_MSG_NOTICE([[ Fuzzing harnesses: ${enable_fuzzing}]])
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. `src/Makefile.am`
|
||||
|
||||
Add the conditional `SUBDIRS` entry. The current file reads
|
||||
|
||||
```make
|
||||
SUBDIRS = include microhttpd .
|
||||
|
||||
if RUN_LIBCURL_TESTS
|
||||
SUBDIRS += testcurl
|
||||
if RUN_ZZUF_TESTS
|
||||
SUBDIRS += testzzuf
|
||||
endif
|
||||
endif
|
||||
```
|
||||
|
||||
Add immediately after that block:
|
||||
|
||||
```make
|
||||
if ENABLE_FUZZING
|
||||
SUBDIRS += fuzz
|
||||
endif
|
||||
```
|
||||
|
||||
Automake derives `DIST_SUBDIRS` from all branches of `SUBDIRS`
|
||||
automatically, so `make dist` keeps working and ships `src/fuzz/`
|
||||
regardless of the conditional.
|
||||
|
||||
---
|
||||
|
||||
## 3. `Makefile.am` (top level) — nothing to do
|
||||
|
||||
`src/fuzz/` is reached through `src/Makefile.am`; the top-level file
|
||||
needs no change.
|
||||
|
||||
---
|
||||
|
||||
## 4. Regenerate and build
|
||||
|
||||
```sh
|
||||
autoreconf -fi # or ./bootstrap
|
||||
./configure --enable-fuzzing \
|
||||
--enable-static \
|
||||
--enable-asserts \
|
||||
--enable-sanitizers=address,undefined \
|
||||
--disable-doc --disable-examples
|
||||
make
|
||||
make -C src/fuzz check
|
||||
```
|
||||
|
||||
A full fuzzing session:
|
||||
|
||||
```sh
|
||||
make -C src/fuzz check \
|
||||
MHD_FUZZ_ITERATIONS=5000000 \
|
||||
MHD_FUZZ_SEED=$RANDOM
|
||||
```
|
||||
|
||||
Replay the checked-in corpus (what CI should do after a fix):
|
||||
|
||||
```sh
|
||||
make -C src/fuzz check-corpus
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Notes / caveats
|
||||
|
||||
* **`--enable-static` is mandatory.** `fuzz_str` and
|
||||
`fuzz_auth_header` call `MHD_hex_to_bin()`, `MHD_str_quote()`,
|
||||
`MHD_pool_create()`, `MHD_get_rq_dauth_params_()` etc., which are
|
||||
built with `$(HIDDEN_VISIBILITY_CFLAGS)` and are therefore not
|
||||
exported from `libmicrohttpd.so`. The `-static` in the per-program
|
||||
`_LDFLAGS` makes libtool pick `.libs/libmicrohttpd.a`.
|
||||
|
||||
* `fuzz_auth_header` is only built when `HAVE_ANYAUTH` is true (Basic or
|
||||
Digest authentication enabled). Inside the harness the individual
|
||||
code paths are additionally guarded by `DAUTH_SUPPORT` /
|
||||
`BAUTH_SUPPORT`, and `fuzz_str` guards `MHD_str_unquote()`,
|
||||
`MHD_str_quote()` and `MHD_base64_to_bin_n()` the same way, so a
|
||||
`--disable-dauth --disable-bauth` build still compiles.
|
||||
|
||||
* `fuzz_postprocessor` needs `--enable-postprocessor` (the default); if
|
||||
you build with `--disable-postprocessor`, additionally guard it with
|
||||
the existing `HAVE_POSTPROCESSOR` conditional:
|
||||
|
||||
```make
|
||||
if HAVE_POSTPROCESSOR
|
||||
check_PROGRAMS += fuzz_postprocessor
|
||||
endif
|
||||
```
|
||||
|
||||
(it is currently listed unconditionally, matching the default build).
|
||||
|
||||
* The harnesses include internal headers (`internal.h`,
|
||||
`memorypool.h`, `gen_auth.h`, `mhd_str.h`), hence the
|
||||
`-I$(top_srcdir)/src/microhttpd` in `AM_CPPFLAGS`. `MHD_config.h` is
|
||||
found through automake's default `-I$(top_builddir)`.
|
||||
|
||||
* `make check` in `src/fuzz` runs the full discipline / memory-limit
|
||||
range (`MHD_FUZZ_MIN_DISCIPLINE=-3`, `MHD_FUZZ_MIN_MEM_LIMIT=0`) with
|
||||
50000 iterations per harness, roughly 3 seconds in total under
|
||||
ASAN+UBSAN. The findings K1-K6 of `README` section 6 are all fixed on
|
||||
master, so the full range is clean; their reproducers stay in
|
||||
`corpus/known-findings/` as regressions.
|
||||
|
||||
---
|
||||
|
||||
## 6. The OSS-Fuzz build path (does not use this Makefile.am)
|
||||
|
||||
`contrib/oss-fuzz/` builds the *same four harness sources* for
|
||||
libFuzzer/AFL++/honggfuzz without going through `src/fuzz/Makefile.am`
|
||||
at all. It configures the library out of tree, then compiles each
|
||||
harness by hand with `-DFUZZ_NO_MAIN` and links it against
|
||||
`$LIB_FUZZING_ENGINE`:
|
||||
|
||||
```sh
|
||||
$CC $CFLAGS -DFUZZ_NO_MAIN \
|
||||
-I$BUILD -I$SRCDIR -I$SRCDIR/src/include \
|
||||
-I$SRCDIR/src/microhttpd -I$SRCDIR/src/fuzz \
|
||||
-c $SRCDIR/src/fuzz/fuzz_request.c -o $WORK/fuzz_request.o
|
||||
$CXX $CXXFLAGS $WORK/fuzz_request.o -o $OUT/fuzz_request \
|
||||
$LIB_FUZZING_ENGINE $BUILD/src/microhttpd/.libs/libmicrohttpd.a -lpthread
|
||||
```
|
||||
|
||||
Two consequences for anyone editing `src/fuzz/`:
|
||||
|
||||
* **`LLVMFuzzerTestOneInput()` must stay unconditional.** Only
|
||||
`main()` may be `#ifndef FUZZ_NO_MAIN`; a harness whose fuzz target is
|
||||
itself conditional silently produces an empty OSS-Fuzz binary.
|
||||
* **Anything the fuzz target needs must not live inside the
|
||||
`#ifndef FUZZ_NO_MAIN` block of `fuzz_common.h`.** Today the split is
|
||||
right: the PRNG, the crash bookkeeping and `fuzz_report_finding()` are
|
||||
outside it; the generator loop, the mutator, the corpus walker and
|
||||
`main()` are inside.
|
||||
|
||||
`contrib/oss-fuzz/` also relies on two things this directory provides:
|
||||
`make -C src/fuzz refresh-corpus` (to regenerate `corpus/`) and the
|
||||
`corpus/known-findings/` reproducers, which it packages into
|
||||
`fuzz_request_seed_corpus.zip` so that K1–K6 become permanent
|
||||
regressions. It is deliberately **not** part of `contrib/ci/jobs/`.
|
||||
@@ -0,0 +1,124 @@
|
||||
# This Makefile.am is in the public domain
|
||||
|
||||
# In-process fuzzing harnesses. See README in this directory.
|
||||
#
|
||||
# The harnesses are built only with --enable-fuzzing (see
|
||||
# BUILD-INTEGRATION.md). They are ordinary check_PROGRAMS with a
|
||||
# built-in deterministic driver, so they need neither clang nor
|
||||
# libFuzzer nor AFL++; with those available the very same sources can be
|
||||
# compiled into libFuzzer/AFL++ targets (again, see the README).
|
||||
|
||||
SUBDIRS = .
|
||||
|
||||
# Number of generate/mutate iterations per harness during "make check".
|
||||
# Raise for a real fuzzing session, e.g.
|
||||
# make MHD_FUZZ_ITERATIONS=1000000 MHD_FUZZ_SEED=42 check
|
||||
# The whole suite takes about three seconds under ASAN+UBSAN at this value.
|
||||
MHD_FUZZ_ITERATIONS = 50000
|
||||
|
||||
# PRNG seed; every run is exactly reproducible from (harness, seed).
|
||||
MHD_FUZZ_SEED = 1
|
||||
|
||||
# Lower bound for MHD_OPTION_CLIENT_DISCIPLINE_LVL in fuzz_request.
|
||||
# -3 is the full range: it includes the deliberately non-conformant
|
||||
# parsing modes, which is where the most interesting inputs live.
|
||||
MHD_FUZZ_MIN_DISCIPLINE = -3
|
||||
|
||||
# Lower bound for MHD_OPTION_CONNECTION_MEMORY_LIMIT in fuzz_request.
|
||||
# 0 is the full range, including the tiny pools needed to reach the
|
||||
# read-buffer "shift back" code path.
|
||||
MHD_FUZZ_MIN_MEM_LIMIT = 0
|
||||
|
||||
# Where reproducers for failing inputs are written.
|
||||
MHD_FUZZ_CRASH_DIR = crashes
|
||||
|
||||
AM_CPPFLAGS = \
|
||||
-I$(top_srcdir)/src/include \
|
||||
-I$(top_srcdir)/src/microhttpd \
|
||||
$(CPPFLAGS_ac)
|
||||
|
||||
AM_CFLAGS = $(CFLAGS_ac)
|
||||
|
||||
AM_LDFLAGS = $(LDFLAGS_ac)
|
||||
|
||||
AM_TESTS_ENVIRONMENT = $(TESTS_ENVIRONMENT_ac) \
|
||||
MHD_FUZZ_ITERATIONS="$(MHD_FUZZ_ITERATIONS)" ; \
|
||||
export MHD_FUZZ_ITERATIONS ; \
|
||||
MHD_FUZZ_SEED="$(MHD_FUZZ_SEED)" ; export MHD_FUZZ_SEED ; \
|
||||
MHD_FUZZ_MIN_DISCIPLINE="$(MHD_FUZZ_MIN_DISCIPLINE)" ; \
|
||||
export MHD_FUZZ_MIN_DISCIPLINE ; \
|
||||
MHD_FUZZ_MIN_MEM_LIMIT="$(MHD_FUZZ_MIN_MEM_LIMIT)" ; \
|
||||
export MHD_FUZZ_MIN_MEM_LIMIT ; \
|
||||
MHD_FUZZ_CRASH_DIR="$(MHD_FUZZ_CRASH_DIR)" ; \
|
||||
export MHD_FUZZ_CRASH_DIR ;
|
||||
|
||||
if USE_COVERAGE
|
||||
AM_CFLAGS += -fprofile-arcs -ftest-coverage
|
||||
endif
|
||||
|
||||
# fuzz_str and fuzz_auth_header call functions that are internal to the
|
||||
# library (mhd_str.c, gen_auth.c, memorypool.c) and therefore hidden in
|
||||
# the shared object; link them against the static archive. This
|
||||
# requires a build with --enable-static (the default).
|
||||
LDADD = \
|
||||
$(top_builddir)/src/microhttpd/libmicrohttpd.la
|
||||
|
||||
$(top_builddir)/src/microhttpd/libmicrohttpd.la: $(top_builddir)/src/microhttpd/Makefile
|
||||
@echo ' cd $(top_builddir)/src/microhttpd && $(MAKE) $(AM_MAKEFLAGS) libmicrohttpd.la'; \
|
||||
$(am__cd) $(top_builddir)/src/microhttpd && $(MAKE) $(AM_MAKEFLAGS) libmicrohttpd.la
|
||||
|
||||
check_PROGRAMS = \
|
||||
fuzz_request \
|
||||
fuzz_str \
|
||||
fuzz_postprocessor
|
||||
|
||||
if HAVE_ANYAUTH
|
||||
check_PROGRAMS += \
|
||||
fuzz_auth_header
|
||||
endif
|
||||
|
||||
.NOTPARALLEL:
|
||||
|
||||
TESTS = $(check_PROGRAMS)
|
||||
|
||||
fuzz_request_SOURCES = \
|
||||
fuzz_request.c fuzz_common.h
|
||||
fuzz_request_LDFLAGS = $(AM_LDFLAGS) -static
|
||||
|
||||
fuzz_str_SOURCES = \
|
||||
fuzz_str.c fuzz_common.h
|
||||
fuzz_str_LDFLAGS = $(AM_LDFLAGS) -static
|
||||
|
||||
fuzz_auth_header_SOURCES = \
|
||||
fuzz_auth_header.c fuzz_common.h
|
||||
fuzz_auth_header_LDFLAGS = $(AM_LDFLAGS) -static
|
||||
|
||||
fuzz_postprocessor_SOURCES = \
|
||||
fuzz_postprocessor.c fuzz_common.h
|
||||
fuzz_postprocessor_LDFLAGS = $(AM_LDFLAGS) -static
|
||||
|
||||
# The whole corpus directory is distributed as-is.
|
||||
EXTRA_DIST = \
|
||||
README \
|
||||
BUILD-INTEGRATION.md \
|
||||
corpus
|
||||
|
||||
CLEANFILES = \
|
||||
$(MHD_FUZZ_CRASH_DIR)/*.bin
|
||||
|
||||
# Regenerate the on-disk seed corpus from the built-in one.
|
||||
.PHONY: refresh-corpus
|
||||
refresh-corpus: $(check_PROGRAMS)
|
||||
for p in $(check_PROGRAMS) ; do \
|
||||
./$$p --write-corpus=$(srcdir)/corpus || exit 1 ; \
|
||||
done
|
||||
|
||||
# Replay the whole on-disk corpus through every harness; this is what a
|
||||
# CI regression run should do after a crash has been fixed.
|
||||
.PHONY: check-corpus
|
||||
check-corpus: $(check_PROGRAMS)
|
||||
for p in $(check_PROGRAMS) ; do \
|
||||
MHD_FUZZ_MIN_DISCIPLINE="$(MHD_FUZZ_MIN_DISCIPLINE)" \
|
||||
MHD_FUZZ_MIN_MEM_LIMIT="$(MHD_FUZZ_MIN_MEM_LIMIT)" \
|
||||
./$$p --corpus-dir=$(srcdir)/corpus || exit 1 ; \
|
||||
done
|
||||
+560
@@ -0,0 +1,560 @@
|
||||
GNU libmicrohttpd -- in-process fuzzing harnesses
|
||||
=================================================
|
||||
|
||||
This directory contains four in-process fuzzing harnesses for MHD. All
|
||||
of them are *dual mode*:
|
||||
|
||||
* they export the libFuzzer entry point
|
||||
|
||||
int LLVMFuzzerTestOneInput (const uint8_t *data, size_t size);
|
||||
|
||||
so the very same source can be linked with clang/libFuzzer, AFL++ or
|
||||
OSS-Fuzz, and
|
||||
|
||||
* they ship a **built-in standalone driver** (`fuzz_common.h`) with a
|
||||
deterministic, seeded generator + mutator loop, so they are useful
|
||||
with nothing but gcc and `-fsanitize=address,undefined`.
|
||||
|
||||
The standalone driver is compiled unless `FUZZ_NO_MAIN` is defined.
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
1. The harnesses
|
||||
-------------------------------------------------------------------
|
||||
|
||||
fuzz_request.c the flagship. Feeds arbitrary bytes into a real
|
||||
`struct MHD_Daemon` through a `socketpair()`,
|
||||
using MHD_USE_NO_LISTEN_SOCKET +
|
||||
MHD_add_connection() and external polling
|
||||
(MHD_run()). Everything runs in one thread, so
|
||||
the harness is deterministic and fast (~20k
|
||||
requests/s under ASAN+UBSAN).
|
||||
|
||||
fuzz_str.c direct fuzzing of the string primitives in
|
||||
src/microhttpd/mhd_str.c. Every output buffer is
|
||||
malloc()ed at *exactly* the documented size so
|
||||
that ASAN's redzone catches a one-byte overrun.
|
||||
|
||||
fuzz_auth_header.c direct fuzzing of the "Authorization:" header
|
||||
parsers, MHD_get_rq_dauth_params_() and
|
||||
MHD_get_rq_bauth_params_() (gen_auth.c), through
|
||||
a minimal fabricated `struct MHD_Connection`.
|
||||
~250k execs/s.
|
||||
|
||||
fuzz_postprocessor.c fuzzing of MHD_post_process() with random
|
||||
Content-Type (urlencoded / multipart with random
|
||||
boundaries), random post-processor buffer sizes
|
||||
and random chunking of the POST data.
|
||||
|
||||
Shared code lives in `fuzz_common.h` (header-only, so every harness
|
||||
stays a single translation unit).
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
2. Design of fuzz_request
|
||||
-------------------------------------------------------------------
|
||||
|
||||
2.1 Why a socketpair
|
||||
--------------------
|
||||
|
||||
MHD_add_connection() accepts any already-connected socket, so a
|
||||
`socketpair(AF_UNIX, SOCK_STREAM)` is enough: no listen socket, no port,
|
||||
no TCP stack, no second thread. The daemon is started with
|
||||
MHD_USE_NO_LISTEN_SOCKET and *without* MHD_USE_INTERNAL_POLLING_THREAD,
|
||||
and the harness pumps it with MHD_run() between sends. A fake
|
||||
127.0.0.1 `struct sockaddr_in` is passed so that per-IP accounting and
|
||||
MHD_get_connection_info() see something sane.
|
||||
|
||||
2.2 Input format
|
||||
----------------
|
||||
|
||||
byte 0 connection memory limit selector
|
||||
(index into {default,128,192,256,320,384,512,768,1024,
|
||||
1400,1500,2048,4096,32768})
|
||||
byte 1 handler behaviour bitmask
|
||||
0x01 call MHD_digest_auth_check3() /
|
||||
MHD_queue_auth_required_response3()
|
||||
0x02 call MHD_basic_auth_get_username_password3()
|
||||
0x04 run the request body through MHD_post_process()
|
||||
0x08 iterate MHD_get_connection_values() over headers,
|
||||
GET arguments, cookies and footers
|
||||
0x20 reply with a larger, copied response body
|
||||
0x40 reply 403 instead of 200
|
||||
byte 2 low nibble: MHD_OPTION_CLIENT_DISCIPLINE_LVL selector
|
||||
(index into {-3,-2,-1,0,1,2});
|
||||
high nibble: reserved for MHD_OPTION_SERVER_INSANITY
|
||||
(MHD 1.0.7 only defines MHD_DSC_SANE, so the value is 0)
|
||||
byte 3 digest configuration: bits 0-1 select the algorithm of the
|
||||
401 challenge {SHA-256, MD5, SHA-512-256, SHA-256},
|
||||
bit 2 selects the QOP, bits 4-5 select MHD_OPTION_NONCE_NC_SIZE
|
||||
byte 4.. a sequence of *send segments*. Each segment starts with a
|
||||
little-endian 16 bit header:
|
||||
|
||||
(op << 14) | length length <= 0x3FFF
|
||||
|
||||
op 0 send `length` bytes on the current connection
|
||||
op 1 the payload is the *expected decoded request body*
|
||||
(ground truth for the body oracle, see 2.4); it is
|
||||
not sent
|
||||
op 2 send, then pump the daemon for extra rounds
|
||||
op 3 close the current connection, open a fresh one on
|
||||
the same daemon, then send
|
||||
|
||||
An explicit length encoding (rather than a magic delimiter) is used so
|
||||
that the fuzzer can move a split point without having to invent an
|
||||
escaping scheme. Splitting matters: MHD's parser is incremental and
|
||||
several bugs only appear for particular split points.
|
||||
|
||||
2.3 The %%NONCE%% placeholder
|
||||
-----------------------------
|
||||
|
||||
Interesting parts of digestauth.c are only reached *after* the client
|
||||
presents a nonce that MHD itself generated. A stateless fuzzer can
|
||||
never guess one. Therefore the harness rewrites the literal ASCII token
|
||||
|
||||
%%NONCE%%
|
||||
|
||||
inside a segment, at send time, into the most recent `nonce="..."` value
|
||||
seen in a response from the daemon. A generated (or hand-written) input
|
||||
can thus be:
|
||||
|
||||
request 1: GET /a -> handler calls MHD_digest_auth_check3(),
|
||||
gets MHD_DAUTH_WRONG_HEADER and replies
|
||||
401 + WWW-Authenticate: Digest ... nonce="X"
|
||||
op 3: new connection
|
||||
request 2: GET /a with Authorization: Digest ... nonce="%%NONCE%%"
|
||||
|
||||
which walks all the way into the 'response' comparison. Without this
|
||||
the over-long `response=` stack overflow (see 5.4) is unreachable.
|
||||
|
||||
2.4 Oracles
|
||||
-----------
|
||||
|
||||
Memory errors are caught by ASAN/UBSAN and aborts by the signal
|
||||
handlers. In addition fuzz_request installs two behavioural oracles:
|
||||
|
||||
a) MHD_set_panic_func() -- any MHD_PANIC() reached from network input
|
||||
is a finding (a remote abort), not a legitimate "API violation".
|
||||
|
||||
b) A request-body oracle. Framing bugs (chunked transfer coding,
|
||||
Content-Length) do not corrupt memory, they corrupt *data*, which
|
||||
is exactly what HTTP request smuggling exploits. The input can
|
||||
therefore declare the expected decoded body in an `op 1` segment.
|
||||
Every byte that MHD hands to the application must be the next
|
||||
expected byte, and when MHD completes the request the delivered
|
||||
body must be complete. MHD is free to reject the request at any
|
||||
point -- only what it *does* deliver is checked.
|
||||
|
||||
The declaration is honoured only for un-mutated inputs (the driver
|
||||
exposes this as `fuzz_pristine`), because a random mutation would
|
||||
of course invalidate the ground truth.
|
||||
|
||||
2.5 The generator
|
||||
-----------------
|
||||
|
||||
Purely random bytes essentially never form a valid HTTP request, so the
|
||||
standalone driver uses a small HTTP grammar (`fuzz_generate()`), and
|
||||
then optionally applies byte-level mutations on top. Shapes:
|
||||
|
||||
0 SHAPE_PLAIN random method/target/version + headers
|
||||
1 SHAPE_NOHDR_QARG *no header lines at all* plus a trailing
|
||||
query argument without '=' (this is the
|
||||
exact shape needed for the read-buffer
|
||||
shift-back bug; the generator also forces a
|
||||
small connection memory pool for it)
|
||||
2 SHAPE_CL_BODY Content-Length body + body oracle
|
||||
3 SHAPE_CHUNKED chunked body with chunk extensions
|
||||
(";ext", ";ext=val", ";ext=\"quoted\"",
|
||||
";a=1;b=2;c") + trailers + body oracle
|
||||
4 SHAPE_DIGEST_SIMPLE Authorization: Digest with a randomised
|
||||
parameter set, including unknown
|
||||
`algorithm=` tokens and `response=` values
|
||||
of every length up to 128 hex digits
|
||||
5 SHAPE_DIGEST_REPLAY the two-request nonce handshake of 2.3
|
||||
6 SHAPE_BASIC Authorization: Basic with random base64
|
||||
7 SHAPE_POST_FORM urlencoded / multipart POST bodies
|
||||
8 SHAPE_WEIRD folded headers, bare CR, bare LF,
|
||||
whitespace before the colon, percent
|
||||
encoding, absolute-form targets, ...
|
||||
|
||||
`MHD_FUZZ_SHAPE=<n>` restricts the generator to a single shape, which is
|
||||
very handy for triage and for regression-testing a specific past bug.
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
3. Running the harnesses
|
||||
-------------------------------------------------------------------
|
||||
|
||||
Build (gcc only, no clang required):
|
||||
|
||||
SRC=/path/to/libmicrohttpd # configured build tree
|
||||
gcc -g -O1 -Wall -Wextra \
|
||||
-fsanitize=address,undefined -fno-sanitize-recover=all \
|
||||
-I$SRC -I$SRC/src/include -I$SRC/src/microhttpd -I$SRC/src/fuzz \
|
||||
-o fuzz_request $SRC/src/fuzz/fuzz_request.c \
|
||||
$SRC/src/microhttpd/.libs/libmicrohttpd.a -lpthread
|
||||
|
||||
(the same command line for fuzz_str, fuzz_auth_header and
|
||||
fuzz_postprocessor; the static archive is required because fuzz_str and
|
||||
fuzz_auth_header use functions that are hidden in the shared object).
|
||||
|
||||
Options of the built-in driver (identical for all harnesses):
|
||||
|
||||
--iterations=N number of generate/mutate iterations [3000]
|
||||
--seed=N PRNG seed; (harness, seed) fully determines a run
|
||||
--corpus-dir=DIR replay every regular file in DIR and exit
|
||||
--file=PATH replay a single input and exit (crash repro)
|
||||
--crash-dir=DIR where reproducers are written [crashes]
|
||||
--timeout=SEC per-iteration watchdog, 0 disables [20]
|
||||
--write-corpus=DIR dump the built-in seed corpus to DIR
|
||||
--skip-seeds do not replay the built-in corpus first
|
||||
--verbose enable MHD's error log + print statistics
|
||||
--help
|
||||
|
||||
Environment variables (all optional):
|
||||
|
||||
MHD_FUZZ_ITERATIONS, MHD_FUZZ_SEED, MHD_FUZZ_TIMEOUT,
|
||||
MHD_FUZZ_CRASH_DIR, MHD_FUZZ_VERBOSE, MHD_FUZZ_SKIP_SEEDS
|
||||
|
||||
MHD_FUZZ_SHAPE=<n> (fuzz_request) restrict the generator
|
||||
to one grammar shape
|
||||
MHD_FUZZ_MIN_DISCIPLINE=<n> (fuzz_request) lower bound for
|
||||
MHD_OPTION_CLIENT_DISCIPLINE_LVL,
|
||||
default -3 (the full range)
|
||||
MHD_FUZZ_MIN_MEM_LIMIT=<n> (fuzz_request) lower bound for
|
||||
MHD_OPTION_CONNECTION_MEMORY_LIMIT,
|
||||
default 0 (the full range)
|
||||
MHD_FUZZ_MODEL_DIGEST_SINK=1 (fuzz_str) enable the modelled
|
||||
digest 'response' call site, see 5.4
|
||||
|
||||
Typical use:
|
||||
|
||||
# quick smoke test (a couple of seconds)
|
||||
./fuzz_request
|
||||
|
||||
# a real session
|
||||
./fuzz_request --iterations=5000000 --seed=$RANDOM
|
||||
|
||||
# regression: replay the whole checked-in corpus
|
||||
./fuzz_request --corpus-dir=corpus
|
||||
./fuzz_str --corpus-dir=corpus # ignores foreign files gracefully
|
||||
|
||||
# reproduce a crash
|
||||
./fuzz_request --file=crashes/crash-fuzz_request-seed3-iter55.bin
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
4. Reproducing a failure
|
||||
-------------------------------------------------------------------
|
||||
|
||||
Whenever the process dies -- ASAN error, UBSAN error, `mhd_assert()`,
|
||||
MHD_PANIC(), a body-oracle finding, or the watchdog -- the input of the
|
||||
running iteration is written to
|
||||
|
||||
$MHD_FUZZ_CRASH_DIR/crash-<harness>-seed<S>-iter<N>.bin
|
||||
|
||||
and a line is printed telling you the harness, the seed and the
|
||||
iteration. The dump is produced from
|
||||
|
||||
* `__sanitizer_set_death_callback()` (weakly linked; present whenever
|
||||
the binary is built with ASAN), and
|
||||
* SIGABRT/SIGSEGV/SIGBUS/SIGILL/SIGFPE/SIGALRM handlers,
|
||||
|
||||
using only async-signal-safe calls. Replay with `--file=...`; the run
|
||||
is fully deterministic, so `--seed=S --iterations=N+1` reproduces the
|
||||
whole session as well.
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
5. What these harnesses find (regression coverage)
|
||||
-------------------------------------------------------------------
|
||||
|
||||
The four vulnerabilities fixed in MHD 1.0.7+1 are all rediscovered from
|
||||
scratch. Each has a dedicated seed in `corpus/`, and the generator
|
||||
finds each of them on its own within a few thousand iterations.
|
||||
|
||||
5.1 digestauth.c: unknown `algorithm=` token -> MHD_PANIC()
|
||||
An `algorithm=` token MHD does not know parses to
|
||||
MHD_DIGEST_AUTH_ALGO3_INVALID, which is 0, so the allow-mask test
|
||||
`c_algo == (c_algo & malgo3)` passes for *any* mask; the code then
|
||||
calls digest_init_one_time() with an invalid algorithm and panics.
|
||||
Found by: SHAPE_DIGEST_SIMPLE / SHAPE_DIGEST_REPLAY, the panic hook,
|
||||
corpus seed `digest-unknown-algorithm`.
|
||||
|
||||
5.2 connection.c get_req_headers(): read-buffer shift-back underflow
|
||||
Needs, all at once: a small MHD_OPTION_CONNECTION_MEMORY_LIMIT
|
||||
(MHD_BUF_INC_SIZE (1500) > read_buffer_size), *no header lines*, and
|
||||
a trailing query argument without '=' (whose `value` is NULL).
|
||||
Found by: SHAPE_NOHDR_QARG, corpus seeds
|
||||
`small-pool-trailing-query-arg[-2]`.
|
||||
|
||||
5.3 connection.c process_request_body(): chunk-extension CRLF
|
||||
`chunk_size_line_len = i` instead of `i + 2` leaves the CRLF of the
|
||||
chunk-size line in the stream, so the following chunk data is
|
||||
shifted -- a body desync, i.e. a request-smuggling primitive. This
|
||||
corrupts no memory, so it is caught by the body oracle (2.4).
|
||||
Found by: SHAPE_CHUNKED, corpus seeds `chunked-with-extensions`,
|
||||
`chunked-split`.
|
||||
|
||||
5.4 digestauth.c: over-long `response=` -> stack buffer overflow
|
||||
`response` was accepted up to `digest_size * 4` characters (128 for
|
||||
SHA-256) and then decoded with MHD_hex_to_bin() into
|
||||
`uint8_t hash1_bin[MAX_DIGEST]` (32 bytes) -- up to 64 bytes
|
||||
written, 32 bytes of stack smashed. Reaching it requires a *valid*
|
||||
nonce, hence the %%NONCE%% mechanism of 2.3.
|
||||
Found by: SHAPE_DIGEST_REPLAY, corpus seed
|
||||
`digest-overlong-response`.
|
||||
|
||||
fuzz_str additionally reproduces the underlying primitive:
|
||||
MHD_hex_to_bin() has no output-size parameter and writes len/2
|
||||
bytes, so any caller with a fixed-size buffer must bound the input
|
||||
length itself. `MHD_FUZZ_MODEL_DIGEST_SINK=1` enables a target that
|
||||
replays exactly the pre-fix call site (32 byte heap buffer, input
|
||||
length bounded only by 4 * 32) and ASAN reports the overflow
|
||||
immediately. The target models a *caller*, not the library, so it
|
||||
is off by default.
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
6. Findings against MHD 1.0.7 - all fixed, kept as regressions
|
||||
-------------------------------------------------------------------
|
||||
|
||||
Running these harnesses against v1.0.7 built with `--enable-asserts`
|
||||
reported the following *additional* issues on top of the four
|
||||
vulnerabilities of section 5. All of them were `mhd_assert()`s reachable
|
||||
from network input, i.e. a remote abort in builds that keep assertions
|
||||
enabled, and all of them are fixed on master. They are documented here
|
||||
because the reproducers are kept as a regression corpus: a failure of one
|
||||
of them means the corresponding fix has been undone. Byte-exact reproducers are in `corpus/known-findings/`; replay
|
||||
one with
|
||||
|
||||
./fuzz_request --file=corpus/known-findings/K1-digest-empty-realm.bin
|
||||
|
||||
K1 digestauth.c:2467 is_param_equal():
|
||||
mhd_assert (0 != param->value.len) -> fixed in 300a2ab0
|
||||
Trigger (default daemon configuration!), one request:
|
||||
GET /a HTTP/1.1
|
||||
Host: x
|
||||
Authorization: Digest username="user", realm="", nonce="0000",
|
||||
uri="/a", response="00"
|
||||
digest_auth_check_all_inner() rejects a *missing* realm/username but
|
||||
not an *empty* one, so a zero-length parameter reaches
|
||||
is_param_equal(), whose documented precondition is a non-empty
|
||||
value. Requires only that the application calls
|
||||
MHD_digest_auth_check3(). Real defect (missing validation).
|
||||
Repro: corpus/known-findings/K1-digest-empty-realm.bin
|
||||
|
||||
K2 connection.c:3582 handle_recv_no_space():
|
||||
mhd_assert ((MHD_PROC_RECV_BODY_CHUNKED != stage) ||
|
||||
! c->rq.some_payload_processed) -> fixed in 68c83f22
|
||||
Trigger: small MHD_OPTION_CONNECTION_MEMORY_LIMIT (<= ~400 bytes)
|
||||
plus a chunked body whose *second* chunk-size line carries a chunk
|
||||
extension that does not fit into the remaining read buffer. The
|
||||
flag reflects the last application callback only and survives later
|
||||
reads, so the assertion is over-strong; the code below it already
|
||||
handles the situation. Stale assertion.
|
||||
Repro: corpus/known-findings/K2-chunkext-no-space.bin
|
||||
|
||||
K3 connection.c:2881 transmit_error_response_len():
|
||||
mhd_assert (! connection->stop_with_error) -> fixed in e04eb218
|
||||
Trigger: small connection memory pool plus an over-long, unterminated
|
||||
chunk extension:
|
||||
POST /a HTTP/1.1 / Transfer-Encoding: chunked
|
||||
d;ext="qqqqqqqq... (longer than the read buffer)
|
||||
handle_req_chunk_size_line_no_space() is missing a `return` after it
|
||||
has already queued the "chunk extension too big" response. Real
|
||||
defect: in a release build the second call forces the connection to
|
||||
MHD_CONNECTION_CLOSED and the 413 response is never sent.
|
||||
Repro: corpus/known-findings/K3-chunkext-stop-with-error.bin
|
||||
|
||||
K4 connection.c:6099 get_req_header():
|
||||
mhd_assert ((0 == c->rq.hdrs.hdr.value_start) ||
|
||||
(0 != c->rq.hdrs.hdr.name_len)) -> fixed in 0b750975
|
||||
Trigger a) MHD_OPTION_CLIENT_DISCIPLINE_LVL <= -1, first header line
|
||||
starting with whitespace:
|
||||
GET /a HTTP/1.1\r\n Host: x\r\n\r\n
|
||||
Trigger b) MHD_OPTION_CLIENT_DISCIPLINE_LVL <= -2, empty header
|
||||
name:
|
||||
GET /a HTTP/1.1\r\n: value\r\nHost: x\r\n\r\n
|
||||
Both shapes are explicitly allowed by those discipline levels.
|
||||
Stale assertion.
|
||||
Repro: corpus/known-findings/K4a-wsp-first-header.bin,
|
||||
corpus/known-findings/K4b-empty-header-name.bin
|
||||
|
||||
K5 connection.c:6394 get_req_header():
|
||||
mhd_assert ('\r' != chr) -> fixed in 6fcdfd43
|
||||
Trigger: MHD_OPTION_CLIENT_DISCIPLINE_LVL = -3, which sets
|
||||
`bare_cr_keep = true`; the branch that keeps a bare CR falls through
|
||||
into the "not a whitespace, not the end of the line" arm whose
|
||||
assertion predates that mode.
|
||||
GET /a HTTP/1.1\r\nHost: x\r\nX: y\r\r\n\r\n
|
||||
Stale assertion.
|
||||
Repro: corpus/known-findings/K5-bare-cr-keep.bin
|
||||
|
||||
K6 digestauth.c:860 check_nonce_nc():
|
||||
mhd_assert (0 == nn->nonce[noncelen])
|
||||
The nonce-nc slot array is indexed by a hash of the nonce, but the
|
||||
slot content is compared assuming the *stored* nonce has the same
|
||||
length as the presented one. A client can therefore make MHD read
|
||||
the terminator of a nonce at the wrong offset by presenting a nonce
|
||||
whose length belongs to a different digest algorithm.
|
||||
Note that this one is *timing dependent*: the nonce carries a
|
||||
millisecond timestamp and whether it counts as stale depends on the
|
||||
wall clock, so the same input reproduces only in a fraction of the
|
||||
replays (about 1 in 30 for the corpus file below). Replay it in a
|
||||
loop.
|
||||
Trigger (MHD_OPTION_NONCE_NC_SIZE = 1 makes every nonce land in slot
|
||||
0, which turns the collision into a certainty; larger arrays only
|
||||
need more attempts):
|
||||
request 1: GET /a -> 401 with a SHA-256 nonce
|
||||
(76 chars) stored in the slot
|
||||
request 2: Authorization: Digest username="user",
|
||||
realm="TestRealm", nonce="<44 zeros>", uri="/a",
|
||||
response="e"
|
||||
(no algorithm parameter -> MD5 -> nonce length 44,
|
||||
all-zero timestamp -> not stale)
|
||||
The generator needs ~150k iterations to hit it on its own.
|
||||
Repro: corpus/known-findings/K6-nonce-length-collision.bin
|
||||
|
||||
Status
|
||||
------
|
||||
|
||||
K1-K5 are fixed on master by commits 300a2ab0, 68c83f22, e04eb218,
|
||||
0b750975 and 6fcdfd43 respectively; K6 by f438804c. All seven
|
||||
reproducers in `corpus/known-findings/` therefore replay clean, and
|
||||
`make check-corpus` asserts exactly that.
|
||||
|
||||
The `make check` defaults in Makefile.am
|
||||
|
||||
MHD_FUZZ_ITERATIONS = 50000
|
||||
MHD_FUZZ_MIN_DISCIPLINE = -3 (full range)
|
||||
MHD_FUZZ_MIN_MEM_LIMIT = 0 (full range)
|
||||
|
||||
exercise the whole matrix and take about three seconds in total under
|
||||
ASAN+UBSAN. A real session:
|
||||
|
||||
MHD_FUZZ_MIN_DISCIPLINE=-3 MHD_FUZZ_MIN_MEM_LIMIT=0 \
|
||||
./fuzz_request --iterations=1000000 --seed=1
|
||||
|
||||
-------------------------------------------------------------------
|
||||
7. Building with clang/libFuzzer or AFL++
|
||||
-------------------------------------------------------------------
|
||||
|
||||
Both need `-DFUZZ_NO_MAIN` so that the driver's `main()` is left out.
|
||||
|
||||
7.1 libFuzzer
|
||||
-------------
|
||||
|
||||
# build the library itself with the same instrumentation
|
||||
./configure --enable-static --disable-shared --enable-asserts \
|
||||
CC=clang \
|
||||
CFLAGS="-g -O1 -fsanitize=fuzzer-no-link,address,undefined \
|
||||
-fno-sanitize-recover=all -fprofile-instr-generate \
|
||||
-fcoverage-mapping"
|
||||
make -C src/microhttpd
|
||||
|
||||
clang -g -O1 -DFUZZ_NO_MAIN \
|
||||
-fsanitize=fuzzer,address,undefined -fno-sanitize-recover=all \
|
||||
-I. -Isrc/include -Isrc/microhttpd -Isrc/fuzz \
|
||||
-o fuzz_request src/fuzz/fuzz_request.c \
|
||||
src/microhttpd/.libs/libmicrohttpd.a -lpthread
|
||||
|
||||
mkdir -p CORPUS && ./fuzz_request --help >/dev/null 2>&1 || true
|
||||
./fuzz_request CORPUS src/fuzz/corpus \
|
||||
-max_len=8192 -rss_limit_mb=4096 -timeout=20
|
||||
|
||||
# minimise a crash found by libFuzzer
|
||||
./fuzz_request -minimize_crash=1 -runs=100000 crash-<hash>
|
||||
|
||||
7.2 AFL++
|
||||
---------
|
||||
|
||||
export CC=afl-clang-lto AFL_USE_ASAN=1 AFL_USE_UBSAN=1
|
||||
./configure --enable-static --disable-shared --enable-asserts
|
||||
make -C src/microhttpd
|
||||
|
||||
afl-clang-lto -g -O1 -DFUZZ_NO_MAIN \
|
||||
-I. -Isrc/include -Isrc/microhttpd -Isrc/fuzz \
|
||||
-o fuzz_request src/fuzz/fuzz_request.c \
|
||||
$(afl-config --libdir 2>/dev/null || echo /usr/local/lib/afl)/afl-compiler-rt.o \
|
||||
/usr/local/lib/afl/libAFLDriver.a \
|
||||
src/microhttpd/.libs/libmicrohttpd.a -lpthread
|
||||
|
||||
afl-fuzz -i src/fuzz/corpus -o findings -- ./fuzz_request @@
|
||||
|
||||
(`libAFLDriver.a` provides a `main()` that reads the file named on the
|
||||
command line and calls LLVMFuzzerTestOneInput(); that is why
|
||||
`-DFUZZ_NO_MAIN` is required. With `AFL_LLVM_PERSISTENT` /
|
||||
`__AFL_LOOP` the same binary can be used in persistent mode.)
|
||||
|
||||
7.3 OSS-Fuzz
|
||||
------------
|
||||
|
||||
Ready to go: see `../../contrib/oss-fuzz/` and its README. That
|
||||
directory holds the OSS-Fuzz `build.sh`, `project.yaml` and `Dockerfile`,
|
||||
per-harness dictionaries (`dicts/fuzz_*.dict`), per-harness `.options`
|
||||
files (`max_len`, `dict`) and `make_seed_corpus.sh`, which packages
|
||||
`corpus/` — including `corpus/known-findings/`, so that K1-K6 become
|
||||
permanent regressions — into the `<fuzzer>_seed_corpus.zip` files
|
||||
OSS-Fuzz expects.
|
||||
|
||||
`build.sh` configures out of tree with
|
||||
|
||||
--enable-static --disable-shared --with-pic --enable-fuzzing
|
||||
--enable-asserts --disable-https --disable-curl --disable-doc
|
||||
--disable-examples --disable-tools --enable-build-type=neutral
|
||||
|
||||
honouring OSS-Fuzz's `$CFLAGS` (no `--enable-sanitizers`: OSS-Fuzz
|
||||
supplies the instrumentation), and then compiles each harness exactly as
|
||||
in 7.1 but against `$LIB_FUZZING_ENGINE`. HTTPS is off on purpose — the
|
||||
harnesses never speak TLS, and leaving GnuTLS out is what makes the
|
||||
MemorySanitizer build possible.
|
||||
|
||||
Two things to know before reading a report from there:
|
||||
|
||||
* the request-body oracle of 2.4 is **inactive** under libFuzzer. It
|
||||
is gated on `fuzz_pristine`, which nothing sets when `FUZZ_NO_MAIN`
|
||||
is defined — correctly so, since libFuzzer's mutations invalidate the
|
||||
declared ground truth. Pure framing/desync defects (5.3) therefore
|
||||
remain the job of the built-in driver, i.e. of `make check`;
|
||||
* `primary_contact` in `project.yaml` is a placeholder. OSS-Fuzz needs
|
||||
an address the maintainer controls; it has to be filled in before the
|
||||
project can be submitted.
|
||||
|
||||
The token list in `fuzz_common.h` (`fuzz_interesting_str`) is the
|
||||
generator's equivalent of those dictionaries; the two are intentionally
|
||||
similar but are not generated from each other.
|
||||
|
||||
OSS-Fuzz is deliberately not part of `contrib/ci/jobs/`; the bounded
|
||||
in-tree equivalents for CI are `make -C src/fuzz check` and
|
||||
`make -C src/fuzz check-corpus`.
|
||||
|
||||
|
||||
-------------------------------------------------------------------
|
||||
8. Adding a harness
|
||||
-------------------------------------------------------------------
|
||||
|
||||
Create `fuzz_<name>.c` with
|
||||
|
||||
#define FUZZ_HARNESS_NAME "fuzz_<name>"
|
||||
#include "fuzz_common.h"
|
||||
|
||||
and implement
|
||||
|
||||
int LLVMFuzzerTestOneInput (const uint8_t *, size_t);
|
||||
static size_t fuzz_generate (struct fuzz_rng *, uint8_t *, size_t);
|
||||
static size_t fuzz_seed_count (void);
|
||||
static const uint8_t *fuzz_seed_get (size_t, size_t *);
|
||||
|
||||
then add it to `check_PROGRAMS` in Makefile.am. Report non-crashing
|
||||
findings with `fuzz_report_finding("...")`, which dumps the reproducer
|
||||
and aborts.
|
||||
|
||||
Two rules learnt the hard way:
|
||||
|
||||
* do not violate documented *preconditions* of the function under
|
||||
test (e.g. MHD_str_remove_token_caseless_() asserts that the token
|
||||
contains no space, tab, comma or NUL) -- otherwise the harness only
|
||||
finds its own bugs;
|
||||
|
||||
* never evaluate a macro argument twice when it contains a PRNG call.
|
||||
@@ -0,0 +1,111 @@
|
||||
Seed corpus for the src/fuzz harnesses
|
||||
======================================
|
||||
|
||||
`<harness>-NN.bin` is the built-in seed corpus of that harness, dumped
|
||||
to disk. Regenerate at any time with
|
||||
|
||||
./fuzz_<harness> --write-corpus=<this directory>
|
||||
|
||||
or, from the build tree,
|
||||
|
||||
make -C src/fuzz refresh-corpus
|
||||
|
||||
Replay everything (this is what a CI regression run should do):
|
||||
|
||||
make -C src/fuzz check-corpus
|
||||
# or, per harness:
|
||||
./fuzz_request --corpus-dir=src/fuzz/corpus
|
||||
|
||||
Files belonging to another harness are simply uninteresting inputs for
|
||||
the harness that reads them, so pointing every harness at the whole
|
||||
directory is fine and gives some extra cross-pollination.
|
||||
|
||||
|
||||
fuzz_request seeds
|
||||
------------------
|
||||
|
||||
00 plain GET / with one header
|
||||
01 content-length-body body oracle, Content-Length
|
||||
02 chunked-with-extensions body oracle, chunk extensions
|
||||
-> chunk-extension CRLF bug
|
||||
03 chunked-split body oracle, chunk boundary split
|
||||
across two send() calls
|
||||
04 small-pool-trailing-query-arg 128..512 byte connection pool, no
|
||||
header lines, "?novalue"
|
||||
-> read-buffer shift-back bug
|
||||
05 small-pool-trailing-query-arg-2 same with "?a=1&b"
|
||||
06 digest-unknown-algorithm algorithm=BOGUS
|
||||
-> MHD_DIGEST_AUTH_ALGO3_INVALID
|
||||
MHD_PANIC()
|
||||
07 digest-overlong-response 401 challenge, then a replay on a
|
||||
fresh connection with the
|
||||
harvested nonce and a 128 hex
|
||||
digit response=
|
||||
-> stack buffer overflow
|
||||
08 digest-userhash userhash=true with a 128 char
|
||||
username
|
||||
09 basic-auth Authorization: Basic
|
||||
10 multipart-post chunked multipart/form-data
|
||||
11 urlencoded-post application/x-www-form-urlencoded
|
||||
12 folded-header obs-fold continuation line
|
||||
13 pipelined two requests in one segment
|
||||
|
||||
fuzz_str seeds
|
||||
--------------
|
||||
|
||||
Cover MHD_hex_to_bin (including the 128 character input that
|
||||
digestauth.c used to allow into a 32 byte buffer), MHD_bin_to_hex[_z],
|
||||
the percent-decoders (strict, lenient, in place), MHD_str_unquote,
|
||||
MHD_str_quote, MHD_base64_to_bin_n, MHD_str[x]_to_uint64_n_ and the
|
||||
token helpers.
|
||||
|
||||
fuzz_auth_header seeds
|
||||
----------------------
|
||||
|
||||
Well-formed and broken Digest parameter lists (unknown algorithm,
|
||||
extended `username*` notation, unterminated quoted strings, empty
|
||||
parameter list, over-long values) plus Basic `token68` variants.
|
||||
|
||||
fuzz_postprocessor seeds
|
||||
------------------------
|
||||
|
||||
urlencoded bodies with broken percent escapes, multipart bodies with
|
||||
ordinary, degenerate ("-") and quote-containing boundaries, a
|
||||
multipart Content-Type without a boundary, and a non-POST
|
||||
Content-Type.
|
||||
|
||||
|
||||
known-findings/
|
||||
---------------
|
||||
|
||||
Reproducers for the issues that this suite found in MHD 1.0.7 itself;
|
||||
see section 6 of ../README. K1-K5 are fixed by ../../../patches/B17,
|
||||
B19, B20, B21 and B22; K6 is handled separately and is still open. They
|
||||
are deliberately kept out of the main corpus directory so that
|
||||
`check-corpus` stays green on an unpatched tree. Replay one with
|
||||
|
||||
./fuzz_request --file=src/fuzz/corpus/known-findings/K1-digest-empty-realm.bin
|
||||
|
||||
K1-digest-empty-realm.bin digestauth.c is_param_equal()
|
||||
mhd_assert (0 != param->value.len)
|
||||
-> patches/B17.diff
|
||||
K2-chunkext-no-space.bin connection.c handle_recv_no_space()
|
||||
-> patches/B19.diff
|
||||
K3-chunkext-stop-with-error.bin connection.c
|
||||
transmit_error_response_len()
|
||||
-> patches/B20.diff
|
||||
K4a-wsp-first-header.bin connection.c get_req_header(),
|
||||
CLIENT_DISCIPLINE_LVL <= -1
|
||||
-> patches/B21.diff
|
||||
K4b-empty-header-name.bin connection.c get_req_header(),
|
||||
CLIENT_DISCIPLINE_LVL <= -2
|
||||
-> patches/B21.diff
|
||||
K5-bare-cr-keep.bin connection.c get_req_header(),
|
||||
CLIENT_DISCIPLINE_LVL == -3
|
||||
-> patches/B22.diff
|
||||
K6-nonce-length-collision.bin digestauth.c check_nonce_nc()
|
||||
mhd_assert (0 == nn->nonce[noncelen])
|
||||
-> still open
|
||||
|
||||
Once a patch has landed, move the corresponding file into the main
|
||||
corpus directory so that `check-corpus` keeps it as a regression seed.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
||||
Basic dXNlcjpwYXNz
|
||||
@@ -0,0 +1 @@
|
||||
Basic
|
||||
@@ -0,0 +1 @@
|
||||
Basic ====
|
||||
@@ -0,0 +1 @@
|
||||
Basic QQ==QQ==
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,5 @@
|
||||
XY
|
||||
--XY
|
||||
|
||||
noheaders
|
||||
--XY--
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user