From 03cb3b19966c5aec389a6e8568bd9ab366246491 Mon Sep 17 00:00:00 2001 From: Mark Atwood Date: Wed, 6 May 2026 13:51:53 -0700 Subject: [PATCH 01/10] docs: add AGENTS.md for PQC benchmark work --- AGENTS.md | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000000..839b9e07d7e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,178 @@ +# Agent Instructions — wolfSSL PQC Benchmark + +This work adds a self-contained PQC benchmark driver to wolfssl/wolfssl. + +Working branch: `feature/pqc-benchmark` +Epic: `wolfssl-jqr` + +--- + +## Goal + +Produce a reproducible, publishable PQC benchmark that wolfSSL owns and controls. +The deliverable is a driver script + documentation in `wolfcrypt/benchmark/` that +any researcher can clone and run to reproduce the numbers. + +**Not a PR into crt26/PQC-LEO.** The canonical benchmark lives here. + +--- + +## What Already Exists in benchmark.c + +`wolfcrypt/benchmark/benchmark.c` already covers PQC. Do not rewrite or duplicate it. + +| Algorithm | Benchmark function | Configure flag | +|---|---|---| +| ML-KEM-512/768/1024 | `bench_mlkem(WC_ML_KEM_512\|768\|1024)` | `--enable-mlkem` (default on) | +| ML-DSA-44/65/87 | `bench_dilithium(2\|3\|5)` | `--enable-dilithium` | +| SLH-DSA-SHAKE-128s/f…256s/f | `bench_slhdsa(SLHDSA_SHAKE128S…)` | `--enable-slhdsa` | +| Falcon-512/1024 | `bench_falconKeySign(1\|5)` | `--enable-falcon --with-liboqs` | + +**Relevant build-time defines:** +- `WOLFSSL_BENCHMARK_FIXED_CSV` — always emit CSV +- `GENERATE_MACHINE_PARSEABLE_REPORT` — prefix lines for grep-able parsing +- `WC_BENCH_HEAP_TRACKING` — add heap columns to output +- `WC_BENCH_STACK_TRACKING` — add stack columns to output +- Enabled automatically by: `--enable-memory --enable-trackmemory=verbose --enable-stacksize=verbose` + +**Relevant runtime flags:** +- `-csv` — CSV output +- `-kyber`, `-kyber512`, `-kyber768`, `-kyber1024` — ML-KEM variants +- `-slhdsa`, `-slhdsa-shake128s`, … — SLH-DSA variants +- `-dilithium` — all ML-DSA levels (see benchmark.c ~line 1402) +- `-falcon_level1`, `-falcon_level5` — Falcon (requires liboqs build) + +--- + +## Open Issues (epic wolfssl-jqr) + +| ID | Task | +|---|---| +| `wolfssl-9jt` | Audit benchmark.c PQC CSV output format precisely | +| `wolfssl-bkx` | Write `wolfcrypt/benchmark/pqc_bench.sh` driver script | +| `wolfssl-cdu` | Decide memory measurement approach (built-in vs. Valgrind) | +| `wolfssl-4e5` | Write `wolfcrypt/benchmark/README-pqc.md` | + +--- + +## Key Files + +``` +wolfcrypt/benchmark/ +├── benchmark.c # DO NOT REWRITE — extend only if genuinely needed +├── benchmark.h +├── README.md # existing general benchmark docs +├── pqc_bench.sh # NEW — our driver script +└── README-pqc.md # NEW — reproducibility docs for published numbers +``` + +--- + +## PQC Algorithm Reference + +**ML-KEM** (`wolfssl/wolfcrypt/wc_mlkem.h`) +- Type constants (enum): `WC_ML_KEM_512=0`, `WC_ML_KEM_768=1`, `WC_ML_KEM_1024=2` +- Key type: `MlKemKey` (aliased as `KyberKey`) +- Operations: keygen → `wc_MlKemKey_MakeKey()`, encaps → `wc_MlKemKey_Encapsulate()`, decaps → `wc_MlKemKey_Decapsulate()` + +**ML-DSA / Dilithium** (`wolfssl/wolfcrypt/dilithium.h`) +- Key type: `dilithium_key` +- Level set via: `wc_dilithium_set_level(key, 2|3|5)` → ML-DSA-44/65/87 +- Operations: `wc_dilithium_init()`, `wc_dilithium_make_key()`, `wc_dilithium_sign_msg()`, `wc_dilithium_verify_msg()` + +**SLH-DSA** (`wolfssl/wolfcrypt/wc_slhdsa.h`) +- Key type: `SlhDsaKey`; param: `enum SlhDsaParam` +- SHAKE variants (always available with `--enable-slhdsa`): + `SLHDSA_SHAKE128S=0`, `SLHDSA_SHAKE128F=1`, `SLHDSA_SHAKE192S=2`, + `SLHDSA_SHAKE192F=3`, `SLHDSA_SHAKE256S=4`, `SLHDSA_SHAKE256F=5` +- SHA2 variants (need `WOLFSSL_SLHDSA_SHA2`): `SLHDSA_SHA2_128S=6` … `SLHDSA_SHA2_256F=11` +- Operations: `wc_SlhDsaKey_Init()`, `wc_SlhDsaKey_MakeKey()`, `wc_SlhDsaKey_Sign()`, `wc_SlhDsaKey_Verify()` + +**Falcon** (`wolfssl/wolfcrypt/falcon.h`) +- Requires liboqs: `#error "HAVE_FALCON requires HAVE_LIBOQS."` at line 41 +- Configure: `--enable-falcon --with-liboqs=/path/to/liboqs` +- Falcon is **included** in the benchmark when liboqs is available, but the driver + must degrade gracefully when it is not (skip with a note, do not fail). + +--- + +## PQC-LEO Reference Repo + +The shallow clone at `~/WORK/PQC-LEO` (branch `upstream-main`, commit `9ea3d22`) is +kept as a **format reference only** — to verify that our CSV output columns are +compatible if someone later wants to feed our results into PQC-LEO's parsers. +Do not treat PQC-LEO as a dependency or deliverable target. + +PQC-LEO CSV column order for speed results: +``` +Algorithm | Operation | Operations | Seconds | ms/op | op/sec +``` +Operations for KEM: `keygen`, `encaps`, `decaps` +Operations for SIG: `keypair`, `sign`, `verify` + +--- + +## RustCrypto / Upstream PR Notes + +When this is ready for upstream: +- Remote `origin` = `git@github.com:MarkAtwood/wolfssl.git` (fork) +- Remote `upstream` = `github.com:wolfssl/wolfssl` +- Pre-push: `typos wolfcrypt/benchmark/`, clang-format on any new C, check that + existing benchmark tests still pass +- PR target: `wolfssl/wolfssl` master + +--- + +## Non-Interactive Shell Commands + +`cp`, `mv`, `rm` may be aliased to `-i` on this system. Always use: + +```bash +cp -f src dst +mv -f src dst +rm -f file +rm -rf dir +apt-get install -y pkg +``` + +--- + + +## Issue Tracking with bd (beads) + +**IMPORTANT**: This project uses **bd (beads)** for ALL issue tracking. Do NOT use markdown TODOs, task lists, or other tracking methods. + +### Quick Start + +```bash +bd ready --json # find available work +bd update --claim --json # claim atomically +bd close --reason "Done" --json # complete +bd create --title="..." --type=task --priority=2 --deps discovered-from: --json +``` + +### Issue Types +`bug` | `feature` | `task` | `epic` | `chore` + +### Priorities +`0`=critical, `1`=high, `2`=medium (default), `3`=low, `4`=backlog + +### Rules +- Use bd for ALL task tracking — no markdown TODOs +- Always `--json` for programmatic use +- Link discovered work with `--deps discovered-from:` +- `bd ready` before asking what to work on + +### Session Close Protocol + +Work is NOT complete until `git push` succeeds. + +```bash +bd close +bd dolt push +git pull --rebase +git push +git status # must show "up to date with origin" +``` + + From ad8927a42a8720a14a3c391103c0774d7b7737de Mon Sep 17 00:00:00 2001 From: Mark Atwood Date: Wed, 6 May 2026 13:55:38 -0700 Subject: [PATCH 02/10] docs: point issue tracking to PQC-LEO beads --- AGENTS.md | 41 ++++++----------------------------------- 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 839b9e07d7e..c7a2722c81e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -137,42 +137,13 @@ apt-get install -y pkg --- - -## Issue Tracking with bd (beads) +## Issue Tracking -**IMPORTANT**: This project uses **bd (beads)** for ALL issue tracking. Do NOT use markdown TODOs, task lists, or other tracking methods. - -### Quick Start - -```bash -bd ready --json # find available work -bd update --claim --json # claim atomically -bd close --reason "Done" --json # complete -bd create --title="..." --type=task --priority=2 --deps discovered-from: --json -``` - -### Issue Types -`bug` | `feature` | `task` | `epic` | `chore` - -### Priorities -`0`=critical, `1`=high, `2`=medium (default), `3`=low, `4`=backlog - -### Rules -- Use bd for ALL task tracking — no markdown TODOs -- Always `--json` for programmatic use -- Link discovered work with `--deps discovered-from:` -- `bd ready` before asking what to work on - -### Session Close Protocol - -Work is NOT complete until `git push` succeeds. +Issues for this work are tracked in the **PQC-LEO repo beads database** at +`~/WORK/PQC-LEO`. Run `bd` commands from there, not here. ```bash -bd close -bd dolt push -git pull --rebase -git push -git status # must show "up to date with origin" +cd ~/WORK/PQC-LEO +bd ready --json +bd show PQC-LEO-v6o # epic ``` - - From 56773ffdcb733ff5461c08ddc5cea227c43b0968 Mon Sep 17 00:00:00 2001 From: Mark Atwood Date: Wed, 6 May 2026 15:21:49 -0700 Subject: [PATCH 03/10] feat: add pqc_bench.sh PQC benchmark driver script Adds wolfcrypt/benchmark/pqc_bench.sh, a self-contained driver that: - Configures wolfSSL with --enable-mlkem --enable-dilithium --enable-slhdsa and memory/stack tracking options - Builds only the benchmark binary target (make wolfcrypt/benchmark/benchmark) - Runs all ML-KEM, ML-DSA, and SLH-DSA parameter sets with -csv output - Produces a clean single-header CSV file (strips wolfSSL output noise) - Handles both old (plain CSV) and new (GENERATE_MACHINE_PARSEABLE_REPORT) benchmark output formats - Detects aarch64 and adds -march=armv8-a for hardware cycle counters - POSIX sh compatible (verified with checkbashisms and dash) --- wolfcrypt/benchmark/pqc_bench.sh | 338 +++++++++++++++++++++++++++++++ 1 file changed, 338 insertions(+) create mode 100755 wolfcrypt/benchmark/pqc_bench.sh diff --git a/wolfcrypt/benchmark/pqc_bench.sh b/wolfcrypt/benchmark/pqc_bench.sh new file mode 100755 index 00000000000..49467b6c9a6 --- /dev/null +++ b/wolfcrypt/benchmark/pqc_bench.sh @@ -0,0 +1,338 @@ +#!/bin/sh +# pqc_bench.sh — wolfSSL PQC benchmark driver +# +# Builds wolfSSL with PQC algorithms enabled and runs the benchmark binary +# for each algorithm group, producing a single CSV output file. +# +# USAGE: +# ./pqc_bench.sh [OPTIONS] +# +# OPTIONS: +# --src-dir DIR wolfSSL source tree root (default: auto-detected from script location) +# --output FILE CSV output file (default: pqc_results.csv) +# --skip-build Skip configure+make; assume binary already built in --src-dir +# --help Print this usage and exit +# +# QUICK START: +# git clone https://github.com/wolfSSL/wolfssl.git +# cd wolfssl +# cp wolfcrypt/benchmark/pqc_bench.sh . # or run from the repo directly +# ./wolfcrypt/benchmark/pqc_bench.sh +# +# DEPENDENCIES: +# C compiler (gcc/clang), make, autoconf, automake, libtool +# No external PQC libraries required — wolfSSL implements all algorithms natively. +# +# EXIT CODES: +# 0 Success +# 1 Configuration error (bad args, missing tools) +# 2 Build failure (configure or make failed) +# 3 Benchmark run failure (one or more benchmark runs failed) +# +# NOTES: +# - On aarch64, -march=armv8-a is added to CFLAGS to enable hardware cycle counters. +# - wolfSSL benchmark binary uses '-ml-dsa' for ML-DSA (Dilithium) as of 5.9.1; +# the '-dilithium' alias is NOT recognized. +# - wolfSSL does not support autotools VPATH (out-of-tree) builds; configure and +# make run inside the source tree. Use git worktrees for parallel builds. + +set -eu + +# --------------------------------------------------------------------------- +# Defaults +# --------------------------------------------------------------------------- +OUTPUT_FILE="pqc_results.csv" +SKIP_BUILD=0 +SRC_DIR="" # auto-detected below after arg parsing + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +usage() { + awk '/^#!/{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "$0" + exit 0 +} + +while [ $# -gt 0 ]; do + case "$1" in + --src-dir) + SRC_DIR="$2"; shift 2 ;; + --output) + OUTPUT_FILE="$2"; shift 2 ;; + --skip-build) + SKIP_BUILD=1; shift ;; + --help|-h) + usage ;; + *) + echo "ERROR: Unknown option: $1" >&2 + echo "Run '$0 --help' for usage." >&2 + exit 1 ;; + esac +done + +# --------------------------------------------------------------------------- +# Locate wolfssl source root +# --------------------------------------------------------------------------- +# pqc_bench.sh lives at wolfcrypt/benchmark/pqc_bench.sh, so source root +# is two levels up from the script's directory. This is the canonical +# location — the script should always be run from inside the source tree. +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +AUTO_SRC="$(cd "$SCRIPT_DIR/../.." && pwd)" + +if [ -z "$SRC_DIR" ]; then + SRC_DIR="$AUTO_SRC" +fi + +# Normalize to absolute path +SRC_DIR="$(cd "$SRC_DIR" && pwd)" + +if [ ! -f "$SRC_DIR/configure.ac" ]; then + echo "ERROR: '$SRC_DIR' does not look like a wolfSSL source tree (no configure.ac)" >&2 + echo " Pass --src-dir /path/to/wolfssl or run from inside the source tree." >&2 + exit 1 +fi + +# --------------------------------------------------------------------------- +# Resolve OUTPUT_FILE to absolute path (before any cd) +# --------------------------------------------------------------------------- +case "$OUTPUT_FILE" in + /*) ;; + *) OUTPUT_FILE="$(pwd)/$OUTPUT_FILE" ;; +esac + +LOG_FILE="${OUTPUT_FILE%.csv}.log" + +echo "wolfSSL PQC Benchmark Driver" +echo " Source dir: $SRC_DIR" +echo " Output CSV: $OUTPUT_FILE" +echo " Log file: $LOG_FILE" +echo "" + +# --------------------------------------------------------------------------- +# Detect architecture for arch-specific CFLAGS +# --------------------------------------------------------------------------- +ARCH="$(uname -m)" +ARCH_CFLAGS="" +if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + # -march=armv8-a enables CNTVCT_EL0 hardware cycle counter access on Graviton. + ARCH_CFLAGS="-march=armv8-a" + echo "Detected aarch64: adding $ARCH_CFLAGS to CFLAGS" + echo "" +fi + +# --------------------------------------------------------------------------- +# Build phase +# --------------------------------------------------------------------------- +if [ "$SKIP_BUILD" -eq 0 ]; then + echo "=== Phase 1: Build ===" + + # wolfSSL uses autotools and does not support VPATH (out-of-tree) builds. + # configure and make run inside the source tree. This is intentional design. + cd "$SRC_DIR" + + # Ensure autogen has been run (configure script must exist in source root) + if [ ! -f "$SRC_DIR/configure" ]; then + echo "Running autogen.sh to generate configure script..." + ./autogen.sh || { + echo "ERROR: autogen.sh failed (exit $?)" >&2 + exit 2 + } + fi + + # Compile-time defines for machine-parseable output: + # GENERATE_MACHINE_PARSEABLE_REPORT: prefixes info lines with "###," and + # error lines with "!!!," so they are trivially filterable from CSV data. + # WOLFSSL_BENCHMARK_FIXED_CSV: forces CSV mode always (belt-and-suspenders + # alongside the -csv runtime flag). + PQC_CFLAGS="-DGENERATE_MACHINE_PARSEABLE_REPORT -DWOLFSSL_BENCHMARK_FIXED_CSV" + if [ -n "$ARCH_CFLAGS" ]; then + PQC_CFLAGS="$PQC_CFLAGS $ARCH_CFLAGS" + fi + + echo "Configuring wolfSSL with PQC flags..." + ./configure \ + --enable-mlkem \ + --enable-dilithium \ + --enable-slhdsa \ + --enable-memory \ + "--enable-trackmemory=verbose" \ + "--enable-stacksize=verbose" \ + "CFLAGS=$PQC_CFLAGS" || { + echo "ERROR: configure failed (exit $?)" >&2 + exit 2 + } + + echo "" + echo "Building benchmark binary..." + # Build only the benchmark binary target; no need to build the full library + # and all examples — 'wolfcrypt/benchmark/benchmark' is the specific target. + make -j"$(nproc)" wolfcrypt/benchmark/benchmark || { + echo "ERROR: make failed (exit $?)" >&2 + exit 2 + } + + echo "Build complete." + echo "" +fi + +# --------------------------------------------------------------------------- +# Locate benchmark binary +# --------------------------------------------------------------------------- +BENCH="$SRC_DIR/wolfcrypt/benchmark/benchmark" +if [ ! -x "$BENCH" ]; then + echo "ERROR: Benchmark binary not found or not executable: $BENCH" >&2 + if [ "$SKIP_BUILD" -eq 1 ]; then + echo " (--skip-build was set; run without --skip-build to build first)" >&2 + fi + exit 1 +fi +echo "Benchmark binary: $BENCH" +echo "" + +# --------------------------------------------------------------------------- +# Benchmark runs +# --------------------------------------------------------------------------- +# Each algorithm group is a separate invocation so one failure doesn't abort +# the entire run. Stderr (progress/verbose output) is tee'd to the log file. +# Stdout (CSV data) is appended to a temporary file for assembly. +# +# NOTE: '-dilithium' is NOT recognized by wolfSSL >= 5.9.1 benchmark binary. +# Use '-ml-dsa' (all security levels) or '-dilithium_level2/3/5' for +# specific levels. + +echo "=== Phase 2: Benchmark Runs ===" + +RAW_CSV="/tmp/pqc_bench_raw_$$.csv" +trap 'rm -f "$RAW_CSV"' EXIT + +BENCH_FAILURES=0 + +# run_bench LABEL FLAG [FLAG ...] +# Run benchmark for one group, appending CSV rows to RAW_CSV. +run_bench() { + _label="$1"; shift + echo " Benchmarking: $_label ..." + + # With GENERATE_MACHINE_PARSEABLE_REPORT compiled in, non-CSV lines get + # "###," or "!!!," prefixes. We redirect stderr to the log (it contains + # stack/heap summaries and verbose timing) and append stdout to RAW_CSV. + if ! "$BENCH" -csv "$@" >>"$RAW_CSV" 2>>"$LOG_FILE"; then + echo " WARNING: benchmark run for '$_label' exited non-zero" >&2 + BENCH_FAILURES=$((BENCH_FAILURES + 1)) + fi +} + +# ML-KEM (NIST FIPS 203, formerly CRYSTALS-Kyber) +run_bench "ML-KEM-512" -kyber512 +run_bench "ML-KEM-768" -kyber768 +run_bench "ML-KEM-1024" -kyber1024 + +# ML-DSA (NIST FIPS 204, formerly CRYSTALS-Dilithium) +# '-ml-dsa' benchmarks all three security levels (44/65/87) in one pass. +run_bench "ML-DSA (levels 44/65/87)" -ml-dsa + +# SLH-DSA (NIST FIPS 205, formerly SPHINCS+) — all parameter sets +run_bench "SLH-DSA-SHAKE-128s" -slhdsa-shake128s +run_bench "SLH-DSA-SHAKE-128f" -slhdsa-shake128f +run_bench "SLH-DSA-SHAKE-192s" -slhdsa-shake192s +run_bench "SLH-DSA-SHAKE-192f" -slhdsa-shake192f +run_bench "SLH-DSA-SHAKE-256s" -slhdsa-shake256s +run_bench "SLH-DSA-SHAKE-256f" -slhdsa-shake256f + +echo "" + +# --------------------------------------------------------------------------- +# Assemble output CSV: one header + all data rows, no noise +# --------------------------------------------------------------------------- +# Filtering rules (in priority order): +# 1. Lines prefixed "###," or "!!!," from GENERATE_MACHINE_PARSEABLE_REPORT +# are info/error annotations — discard. +# 2. Header line detection supports two formats produced by wolfSSL: +# New (with GENERATE_MACHINE_PARSEABLE_REPORT): +# "asym",Algorithm,key size,operation,... +# Legacy (without it): +# Algorithm,key size,operation,... +# Keep only the first occurrence; strip the leading type field and +# trailing comma if present. +# 3. Data rows in new format start with an unquoted type token ("asym,"). +# Legacy data rows have a numeric key size in field 2. +# Both cases: strip leading type field and trailing comma, emit once. +# 4. All other lines (banners, "Benchmark complete", memory/stack summaries, +# "This format allows...", section headings) are noise — drop silently. + +echo "=== Phase 3: Assembling CSV ===" + +awk ' +BEGIN { + header_printed = 0 +} +# GENERATE_MACHINE_PARSEABLE_REPORT annotation lines — always discard +/^###/ || /^!!!/ { next } + +# New format header: starts with quoted type field then "Algorithm," +# e.g.: "asym",Algorithm,key size,operation,... +/^"[a-z]*",Algorithm,/ { + if (!header_printed) { + # Strip leading quoted type field and its comma, then trailing comma + line = $0 + sub(/^"[^"]*",/, "", line) + sub(/,$/, "", line) + print line + header_printed = 1 + } + next +} + +# Legacy format header: starts directly with "Algorithm," +/^Algorithm,/ { + if (!header_printed) { + line = $0 + sub(/,$/, "", line) + print line + header_printed = 1 + } + next +} + +# New format data rows: start with unquoted type token (e.g. "asym,") +# followed by the algorithm name. The type token is a short lowercase word. +/^[a-z][a-z]*,[A-Z]/ { + # Strip the leading type field and emit the rest (stripped of trailing comma) + line = $0 + sub(/^[^,]*,/, "", line) + sub(/,$/, "", line) + print line + next +} + +# Legacy format data rows: at least 5 comma-separated fields, field 2 is +# a numeric key size (e.g. 128, 192, 256, 44, 65, 87). +{ + n = split($0, fields, ",") + if (n >= 5 && fields[2] ~ /^[[:space:]]*[0-9][0-9]*[[:space:]]*$/) { + line = $0 + sub(/,$/, "", line) + print line + } + # All other lines (banners, totals, section headings) are silently discarded +} +' "$RAW_CSV" > "$OUTPUT_FILE" + +DATA_ROWS="$(awk 'NR > 1' "$OUTPUT_FILE" | wc -l | tr -d ' ')" +echo " CSV rows written: $DATA_ROWS data rows + 1 header" +echo " Output: $OUTPUT_FILE" +echo " Log: $LOG_FILE" +echo "" + +# --------------------------------------------------------------------------- +# Final status +# --------------------------------------------------------------------------- +if [ "$BENCH_FAILURES" -gt 0 ]; then + echo "WARNING: $BENCH_FAILURES benchmark run(s) exited non-zero." >&2 + echo " Check $LOG_FILE for details." >&2 + echo " Partial CSV results written to $OUTPUT_FILE" >&2 + exit 3 +fi + +echo "All benchmarks complete. $DATA_ROWS algorithm/operation rows written to $OUTPUT_FILE" +exit 0 From 947111cbed65f69cb3e1b4fca46138115746a3f7 Mon Sep 17 00:00:00 2001 From: Mark Atwood Date: Wed, 6 May 2026 15:24:26 -0700 Subject: [PATCH 04/10] feat: add pqc_parse.py normalizer and document memory approach pqc_parse.py: - Normalises wolfSSL benchmark CSV to NIST algorithm names and canonical operation labels (keygen/encaps/decaps/sign/verify) - Supports both wolfssl CSV format (with Library column) and pqcleo pipe-delimited format for cross-library comparison - Handles 5, 8, and 12-column CSV variants (memory/cycles optional) - Accepts stdin with '-' for composable pipeline use pqc_bench.sh: - Adds MEMORY MEASUREMENT APPROACH comment block documenting the Option A (built-in wolfSSL tracking) vs Option B (Valgrind massif) decision: Option A implemented, Option B deferred --- wolfcrypt/benchmark/pqc_bench.sh | 16 ++ wolfcrypt/benchmark/pqc_parse.py | 378 +++++++++++++++++++++++++++++++ 2 files changed, 394 insertions(+) create mode 100755 wolfcrypt/benchmark/pqc_parse.py diff --git a/wolfcrypt/benchmark/pqc_bench.sh b/wolfcrypt/benchmark/pqc_bench.sh index 49467b6c9a6..439e6cf9fdf 100755 --- a/wolfcrypt/benchmark/pqc_bench.sh +++ b/wolfcrypt/benchmark/pqc_bench.sh @@ -35,6 +35,22 @@ # the '-dilithium' alias is NOT recognized. # - wolfSSL does not support autotools VPATH (out-of-tree) builds; configure and # make run inside the source tree. Use git worktrees for parallel builds. +# +# MEMORY MEASUREMENT APPROACH (Option A — wolfSSL built-in tracking): +# +# This script uses wolfSSL's built-in allocator instrumentation: +# --enable-memory --enable-trackmemory=verbose --enable-stacksize=verbose +# +# This produces heap_bytes, heap_allocs, and stack_bytes columns in the CSV. +# heap_bytes is the cumulative heap bytes allocated over the timed loop +# (not peak RSS per single operation). stack_bytes is peak stack depth +# measured via a canary-based thread stack probe. +# +# Alternative (Option B — Valgrind massif): +# True peak RSS per single operation, directly comparable to liboqs/PQC-LEO +# Valgrind numbers. Requires separate thin C wrapper programs per algorithm +# and ~20x runtime overhead. File a separate issue if cross-library memory +# comparison is required — Option A suffices for standalone wolfSSL reporting. set -eu diff --git a/wolfcrypt/benchmark/pqc_parse.py b/wolfcrypt/benchmark/pqc_parse.py new file mode 100755 index 00000000000..4ff29b2c982 --- /dev/null +++ b/wolfcrypt/benchmark/pqc_parse.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +""" +pqc_parse.py — Normalize wolfSSL PQC benchmark CSV for publication. + +Takes the CSV produced by pqc_bench.sh and emits clean, publication-ready +output in one of two formats: + + --format=wolfssl (default) + Pipe-aligned CSV with a Library column prepended. + Columns: Library, Algorithm, Operation, ops/sec, avg_ms, ops, secs + [, heap_bytes, heap_allocs, stack_bytes] + + --format=pqcleo + Pipe-delimited format matching PQC-LEO's parser expectations: + Algorithm | Operation | Operations | Seconds | ms/op | op/sec + (memory columns dropped — PQC-LEO memory is from Valgrind massif) + +USAGE: + python3 pqc_parse.py [OPTIONS] INPUT_CSV + +OPTIONS: + --format FORMAT Output format: wolfssl (default) or pqcleo + --library NAME Library name for wolfssl format (default: wolfSSL) + --output FILE Write to FILE instead of stdout + --help Print this help and exit + +EXAMPLES: + python3 pqc_parse.py pqc_results.csv + python3 pqc_parse.py --format=pqcleo --output=results.psv pqc_results.csv + +INPUT FORMAT: + The script accepts the CSV produced by pqc_bench.sh in both its supported + output variants: + - With GENERATE_MACHINE_PARSEABLE_REPORT + cycles columns (12 fields): + Algorithm,key size,operation,avg ms,ops/sec,ops,secs,cycles,cycles/op, + heap_bytes,heap_allocs,stack_bytes + - Without cycles (8 fields, older build): + Algorithm,key size,operation,avg ms,ops/sec,heap_bytes,heap_allocs, + stack_bytes + - Minimal (5 fields, no memory tracking): + Algorithm,key size,operation,avg ms,ops/sec + + Memory columns are optional; missing columns are reported as empty strings. +""" + +import argparse +import csv +import re +import sys +from typing import Optional + + +# --------------------------------------------------------------------------- +# Algorithm name normalisation +# --------------------------------------------------------------------------- +# wolfSSL benchmark output uses internal names that differ from NIST names. +# These maps translate what the benchmark binary actually emits. + +# ML-KEM: "ML-KEM 512 " (trailing space), key size in separate column. +# We reconstruct "ML-KEM-" from the algorithm + key-size fields. +_MLKEM_RE = re.compile(r"ML-KEM\s+(\d+)\s*$", re.IGNORECASE) + +# ML-DSA: algorithm field is "ML-DSA", key-size field is security level (44/65/87). +_MLDSA_RE = re.compile(r"ML-DSA\s*$", re.IGNORECASE) + +# SLH-DSA: algorithm field is "SLH-DSA-S" (small) or "SLH-DSA-F" (fast), +# key-size field is the security category (128/192/256). +# NIST name: SLH-DSA-SHAKE- +# S → s (small), F → f (fast) +_SLHDSA_RE = re.compile(r"SLH-DSA-([SF])\s*$", re.IGNORECASE) + +# Legacy Dilithium names (in case an older wolfSSL build is used): +# "DILITHIUM" with key-size field being the security level. +_DILITHIUM_RE = re.compile(r"DILITHIUM\s*$", re.IGNORECASE) + + +def normalise_algorithm(raw_algo: str, raw_keysize: str) -> str: + """ + Convert wolfSSL benchmark algorithm + key-size fields to a canonical + NIST/IETF algorithm name. + + Returns the normalised name as a string, or the original (stripped) name + if no normalisation rule applies (future-proofing for new algorithms). + """ + algo = raw_algo.strip() + keysize = raw_keysize.strip() + + m = _MLKEM_RE.match(algo) + if m: + # benchmark already encodes the security level in the name; + # the key-size column is the same value. Use the name field. + return f"ML-KEM-{m.group(1)}" + + if _MLDSA_RE.match(algo): + # key-size field is the security level (44, 65, 87) + return f"ML-DSA-{keysize}" + + m = _SLHDSA_RE.match(algo) + if m: + size_char = m.group(1).lower() # 's' or 'f' + return f"SLH-DSA-SHAKE-{keysize}{size_char}" + + if _DILITHIUM_RE.match(algo): + # Legacy name from older wolfSSL builds + return f"ML-DSA-{keysize}" + + # Unknown algorithm: return stripped name unchanged so we don't silently + # drop data from future wolfSSL versions that add new algorithms. + return algo + + +# --------------------------------------------------------------------------- +# Operation label normalisation +# --------------------------------------------------------------------------- +# wolfSSL uses slightly different operation names than the PQC-LEO convention. + +_OP_MAP = { + "key gen": "keygen", # KEM and SIG key generation + "gen": "keygen", # SLH-DSA uses 'gen' instead of 'key gen' + "encap": "encaps", # KEM encapsulation + "decap": "decaps", # KEM decapsulation + "sign": "sign", + "verify": "verify", + # SLH-DSA extended operations (pre-hash and message variants) + # Pass through with minor normalisation so the output remains valid. + "sign-msg": "sign-msg", + "vrfy-msg": "verify-msg", + "sign-pre": "sign-pre", + "vrfy-pre": "verify-pre", +} + + +def normalise_operation(raw_op: str) -> str: + """Normalise a wolfSSL operation label to canonical form.""" + op = raw_op.strip() + return _OP_MAP.get(op, op) # unknown ops pass through unchanged + + +# --------------------------------------------------------------------------- +# CSV column layout detection +# --------------------------------------------------------------------------- +# pqc_bench.sh can produce two column layouts depending on build flags: +# +# Layout A (GENERATE_MACHINE_PARSEABLE_REPORT, 12 fields): +# Algorithm, key size, operation, avg ms, ops/sec, +# ops, secs, cycles, cycles/op, heap_bytes, heap_allocs, stack_bytes +# +# Layout B (plain -csv, 8 fields): +# Algorithm, key size, operation, avg ms, ops/sec, +# heap_bytes, heap_allocs, stack_bytes +# +# Layout C (minimal, 5 fields, no memory tracking): +# Algorithm, key size, operation, avg ms, ops/sec + +def detect_layout(header: list[str]) -> dict: + """ + Given the parsed header row, return a dict mapping logical field names to + column indices. Returns None for fields not present in this layout. + + All header matching is case-insensitive and strips whitespace. + """ + h = [f.strip().lower() for f in header] + + def idx(name: str) -> Optional[int]: + try: + return h.index(name) + except ValueError: + return None + + layout = { + "algorithm": idx("algorithm"), + "key_size": idx("key size"), + "operation": idx("operation"), + "avg_ms": idx("avg ms"), + "ops_per_sec": idx("ops/sec"), + "ops": idx("ops"), + "secs": idx("secs"), + "cycles": idx("cycles"), + "cycles_per_op": idx("cycles/op"), + "heap_bytes": idx("heap_bytes"), + "heap_allocs": idx("heap_allocs"), + "stack_bytes": idx("stack_bytes"), + } + + # Validate that mandatory fields are present + mandatory = ("algorithm", "key_size", "operation", "avg_ms", "ops_per_sec") + for field in mandatory: + if layout[field] is None: + raise ValueError( + f"Input CSV is missing required column '{field}'. " + f"Got columns: {header}" + ) + + return layout + + +def get_field(row: list[str], layout: dict, field: str, default: str = "") -> str: + """Extract a field from a row using the pre-computed layout, or return default.""" + idx = layout.get(field) + if idx is None or idx >= len(row): + return default + return row[idx].strip() + + +# --------------------------------------------------------------------------- +# Output formatters +# --------------------------------------------------------------------------- + +def write_wolfssl_format(rows, layout: dict, library: str, out): + """ + Write normalised wolfssl CSV format. + + Columns: Library, Algorithm, Operation, ops/sec, avg_ms, ops, secs + [, heap_bytes, heap_allocs, stack_bytes] + + Memory columns are omitted if they were not present in the input. + """ + has_memory = layout["heap_bytes"] is not None + has_timing = layout["ops"] is not None + + header = ["Library", "Algorithm", "Operation", "ops/sec", "avg_ms"] + if has_timing: + header += ["ops", "secs"] + if has_memory: + header += ["heap_bytes", "heap_allocs", "stack_bytes"] + + writer = csv.writer(out, lineterminator="\n") + writer.writerow(header) + + for row in rows: + raw_algo = get_field(row, layout, "algorithm") + raw_keysize = get_field(row, layout, "key_size") + raw_op = get_field(row, layout, "operation") + + algorithm = normalise_algorithm(raw_algo, raw_keysize) + operation = normalise_operation(raw_op) + ops_per_sec = get_field(row, layout, "ops_per_sec") + avg_ms = get_field(row, layout, "avg_ms") + + out_row = [library, algorithm, operation, ops_per_sec, avg_ms] + + if has_timing: + out_row.append(get_field(row, layout, "ops")) + out_row.append(get_field(row, layout, "secs")) + + if has_memory: + out_row.append(get_field(row, layout, "heap_bytes")) + out_row.append(get_field(row, layout, "heap_allocs")) + out_row.append(get_field(row, layout, "stack_bytes")) + + writer.writerow(out_row) + + +def write_pqcleo_format(rows, layout: dict, out): + """ + Write PQC-LEO pipe-delimited format for cross-library comparison. + + PQC-LEO parser expects: + Algorithm | Operation | Operations | Seconds | ms/op | op/sec + + Memory columns are dropped (PQC-LEO memory comes from Valgrind massif, + which is incompatible with wolfSSL's inline allocation tracking). + """ + # PQC-LEO uses pipe delimiter with spaces: " | " + sep = " | " + + def write_row(fields): + out.write(sep.join(str(f) for f in fields) + "\n") + + write_row(["Algorithm", "Operation", "Operations", "Seconds", "ms/op", "op/sec"]) + + for row in rows: + raw_algo = get_field(row, layout, "algorithm") + raw_keysize = get_field(row, layout, "key_size") + raw_op = get_field(row, layout, "operation") + + algorithm = normalise_algorithm(raw_algo, raw_keysize) + operation = normalise_operation(raw_op) + ops_per_sec = get_field(row, layout, "ops_per_sec") + avg_ms = get_field(row, layout, "avg_ms") + ops = get_field(row, layout, "ops", default="") + secs = get_field(row, layout, "secs", default="") + + write_row([algorithm, operation, ops, secs, avg_ms, ops_per_sec]) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser( + description=__doc__.split("\n")[1].strip(), + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog="\n".join(__doc__.split("\n")[1:]), + ) + parser.add_argument( + "input", + metavar="INPUT_CSV", + help="CSV file produced by pqc_bench.sh (use '-' for stdin)", + ) + parser.add_argument( + "--format", + choices=("wolfssl", "pqcleo"), + default="wolfssl", + help="Output format: wolfssl (default) or pqcleo", + ) + parser.add_argument( + "--library", + default="wolfSSL", + help="Library name for wolfssl format (default: wolfSSL)", + ) + parser.add_argument( + "--output", + default="-", + help="Output file (default: stdout)", + ) + args = parser.parse_args() + + # Open input + if args.input == "-": + in_file = sys.stdin + else: + try: + in_file = open(args.input, newline="", encoding="utf-8") + except OSError as e: + print(f"ERROR: Cannot open input file: {e}", file=sys.stderr) + sys.exit(1) + + # Open output + if args.output == "-": + out_file = sys.stdout + else: + try: + out_file = open(args.output, "w", encoding="utf-8") + except OSError as e: + print(f"ERROR: Cannot open output file: {e}", file=sys.stderr) + sys.exit(1) + + try: + reader = csv.reader(in_file) + + # Read and parse header + try: + header = next(reader) + except StopIteration: + print("ERROR: Input CSV is empty (no header row)", file=sys.stderr) + sys.exit(1) + + try: + layout = detect_layout(header) + except ValueError as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + + # Read all data rows (skip blank lines) + data_rows = [row for row in reader if any(f.strip() for f in row)] + + if not data_rows: + print("WARNING: Input CSV has a header but no data rows", file=sys.stderr) + + # Emit in requested format + if args.format == "wolfssl": + write_wolfssl_format(data_rows, layout, args.library, out_file) + else: + write_pqcleo_format(data_rows, layout, out_file) + + finally: + if in_file is not sys.stdin: + in_file.close() + if out_file is not sys.stdout: + out_file.close() + + sys.exit(0) + + +if __name__ == "__main__": + main() From 4a07f0b2e3da9e444acad089bf2a23bf5121ac0d Mon Sep 17 00:00:00 2001 From: Mark Atwood Date: Wed, 6 May 2026 15:26:13 -0700 Subject: [PATCH 05/10] docs: add README-pqc.md for PQC benchmark suite Reproducibility documentation covering: - Prerequisites and tested environments (x86_64, aarch64) - Build and run instructions (copy-paste ready) - Output format reference for raw CSV and normalized formats - Memory measurement methodology and limitations - Timing methodology, frequency pinning, NUMA, thermal notes - Cross-library comparison with liboqs/PQC-LEO - Citation guidance --- wolfcrypt/benchmark/README-pqc.md | 297 ++++++++++++++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 wolfcrypt/benchmark/README-pqc.md diff --git a/wolfcrypt/benchmark/README-pqc.md b/wolfcrypt/benchmark/README-pqc.md new file mode 100644 index 00000000000..1b5f8c9f175 --- /dev/null +++ b/wolfcrypt/benchmark/README-pqc.md @@ -0,0 +1,297 @@ +# wolfSSL PQC Benchmark + +This directory contains a self-contained benchmark suite for wolfSSL's native +Post-Quantum Cryptography (PQC) algorithm implementations. + +**Algorithms covered:** + +| Algorithm Family | NIST Standard | Parameter Sets | +|---|---|---| +| ML-KEM (CRYSTALS-Kyber) | FIPS 203 | ML-KEM-512, ML-KEM-768, ML-KEM-1024 | +| ML-DSA (CRYSTALS-Dilithium) | FIPS 204 | ML-DSA-44, ML-DSA-65, ML-DSA-87 | +| SLH-DSA (SPHINCS+) | FIPS 205 | All 10 SHAKE parameter sets (128s/f, 192s/f, 256s/f, and SHA-2 variants) | + +All algorithms are implemented natively in wolfSSL — no liboqs dependency is +required for these results. + +**Tested on:** wolfSSL 5.9.1 + +--- + +## 1. Prerequisites + +### System packages + +**Debian/Ubuntu:** +```sh +apt-get install -y gcc make autoconf automake libtool git +``` + +**Amazon Linux 2023 / RHEL / Fedora:** +```sh +dnf install -y gcc gcc-c++ make autoconf automake libtool git +``` + +**macOS (with Homebrew):** +```sh +brew install autoconf automake libtool +``` + +**Python 3.6+** is required to run `pqc_parse.py`. It is available by default +on most modern systems; no third-party packages are needed. + +### Tested environments + +| Architecture | OS | Notes | +|---|---|---| +| x86_64 | Pop!_OS 24.04 LTS (Ubuntu-based) | Primary development platform | +| aarch64 | Amazon Linux 2023 (Graviton2, t4g.medium) | CI verification | + +--- + +## 2. How to Build and Run + +### Step 1: Clone wolfSSL + +```sh +git clone https://github.com/wolfSSL/wolfssl.git +cd wolfssl +``` + +### Step 2: Run the benchmark driver + +```sh +./wolfcrypt/benchmark/pqc_bench.sh +``` + +This single command: +1. Runs `autogen.sh` if needed to generate the `configure` script +2. Configures wolfSSL with PQC algorithms and memory/stack tracking enabled +3. Builds only the benchmark binary target (not the full library + tests) +4. Runs the benchmark for each algorithm group +5. Writes a clean CSV file to `pqc_results.csv` + +A run log is written alongside the CSV as `pqc_results.log`. + +### Step 3 (optional): Normalize the output + +```sh +python3 wolfcrypt/benchmark/pqc_parse.py pqc_results.csv +``` + +This normalises algorithm names to canonical NIST form and operation labels +to a consistent vocabulary (keygen/encaps/decaps/sign/verify). + +### Driver script reference + +``` +pqc_bench.sh — wolfSSL PQC benchmark driver + +USAGE: + ./pqc_bench.sh [OPTIONS] + +OPTIONS: + --src-dir DIR wolfSSL source tree root (default: auto-detected) + --output FILE CSV output file (default: pqc_results.csv) + --skip-build Skip configure+make; assume binary already built + --help Print this help and exit +``` + +### Normalizer reference + +``` +usage: pqc_parse.py [-h] [--format {wolfssl,pqcleo}] [--library LIBRARY] + [--output OUTPUT] INPUT_CSV + +OPTIONS: + --format wolfssl Normalized wolfSSL CSV with Library column (default) + --format pqcleo PQC-LEO pipe-delimited format for cross-library comparison + --library NAME Library name for wolfssl format (default: wolfSSL) + --output FILE Output file (default: stdout) +``` + +--- + +## 3. Output Format + +### Raw CSV (from `pqc_bench.sh`) + +The raw output file uses the format produced by wolfSSL's benchmark binary +with `GENERATE_MACHINE_PARSEABLE_REPORT` enabled. After noise stripping by +`pqc_bench.sh`, each data row has the following columns: + +| Column | Units | Description | +|---|---|---| +| `Algorithm` | — | wolfSSL internal algorithm name (e.g., `ML-KEM 512 `) | +| `key size` | bits | Security parameter / key size | +| `operation` | — | Operation name (e.g., `key gen`, `encap`, `sign`) | +| `avg ms` | milliseconds | Average time per operation | +| `ops/sec` | 1/s | Operations per second (averaged over the timed loop) | +| `ops` | count | Number of operations completed in the timed loop | +| `secs` | seconds | Actual elapsed time of the timed loop | +| `cycles` | CPU cycles | Total CPU cycles for the loop (when RDTSC available) | +| `cycles/op` | cycles | Average CPU cycles per operation | +| `heap_bytes` | bytes | Cumulative heap bytes allocated during the timed loop | +| `heap_allocs` | count | Number of heap allocations per operation | +| `stack_bytes` | bytes | Peak stack depth during the operation | + +**Note:** The `ops/sec` figure is the mean throughput over many repeated +operations (typically 1 second of work). It is not a single-shot latency +measurement. + +### Normalized CSV (`--format=wolfssl`) + +After normalization by `pqc_parse.py --format=wolfssl`: + +| Column | Description | +|---|---| +| `Library` | Always `wolfSSL` (overridable with `--library`) | +| `Algorithm` | Canonical NIST name (e.g., `ML-KEM-512`, `ML-DSA-44`, `SLH-DSA-SHAKE-128s`) | +| `Operation` | Canonical label: `keygen`, `encaps`, `decaps`, `sign`, `verify` | +| `ops/sec` | Same as raw | +| `avg_ms` | Same as raw | +| `ops`, `secs` | Loop iteration count and elapsed time (if present in input) | +| `heap_bytes`, `heap_allocs`, `stack_bytes` | Memory columns (if present in input) | + +### PQC-LEO format (`--format=pqcleo`) + +Pipe-delimited format matching the PQC-LEO parser: + +``` +Algorithm | Operation | Operations | Seconds | ms/op | op/sec +ML-KEM-512 | keygen | 69900 | 1.001389 | 0.014 | 69803.025 +... +``` + +Memory columns are omitted in this format — PQC-LEO memory numbers come from +Valgrind massif (see Section 5). + +--- + +## 4. Memory Measurement + +This benchmark uses wolfSSL's built-in allocator instrumentation +(`--enable-memory --enable-trackmemory=verbose --enable-stacksize=verbose`), +which produces the `heap_bytes`, `heap_allocs`, and `stack_bytes` columns. + +**What `heap_bytes` measures:** +Cumulative bytes allocated across all operations in the timed loop. This is +proportional to (but not equal to) the memory footprint of a single operation. +For example, if a KEM keygen allocates 3072 bytes once per call and the timed +loop runs 70,000 operations, `heap_bytes` will be `70,000 × 3072 = 214 MB`. +The per-operation heap cost is `heap_bytes / ops`. + +**What `heap_allocs` measures:** +Number of `malloc()`/`free()` calls per operation (as confirmed by inspection +of wolfSSL source). + +**What `stack_bytes` measures:** +Peak stack depth during the operation, measured by a canary-based probe. This +is the per-operation peak stack usage, not an aggregate. + +**Limitation:** The `heap_bytes` figure is cumulative, not a peak RSS +measurement. It cannot be directly compared with Valgrind massif results +(which measure peak resident set size for a single operation). If +apples-to-apples comparison with liboqs Valgrind numbers is required, a +separate Valgrind massif instrumentation pass is needed. + +--- + +## 5. Methodology Notes + +### Timing + +The benchmark binary uses a **1-second timed loop** per operation: it runs +the operation repeatedly until at least 1 second has elapsed, then reports +the mean throughput. This provides stable averages for fast operations +(KEM, ML-DSA) while remaining feasible for slow ones (SLH-DSA keygen/sign). + +### CPU frequency scaling + +For reproducible results, pin the CPU governor to performance mode before +running: + +```sh +# Linux (requires cpupower / linux-tools): +sudo cpupower frequency-set -g performance + +# macOS: not applicable (hardware-controlled) + +# Verify (Linux): +cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor +``` + +Revert after benchmarking: +```sh +sudo cpupower frequency-set -g powersave +``` + +### NUMA / CPU affinity + +On multi-socket machines, pin to a single NUMA node to avoid cross-socket +memory latency: + +```sh +taskset -c 0 ./wolfcrypt/benchmark/pqc_bench.sh +``` + +### Thermal throttling + +Results on laptops and mobile processors may vary significantly due to thermal +throttling. For publication-quality numbers: +- Run on desktop hardware or a bare-metal cloud instance (e.g., AWS m6i, c7g) +- Allow the system to reach thermal equilibrium before benchmarking +- Note the hardware platform in any published results + +### Cycle counts + +The `cycles` and `cycles/op` columns are populated via `RDTSC` (x86) or +`CNTVCT_EL0` (aarch64, enabled by `-march=armv8-a`). They are wall-clock +cycle counts, not retired instruction counts. Values may include OS +interrupt overhead. + +--- + +## 6. Comparison with liboqs / PQC-LEO + +wolfSSL implements ML-KEM, ML-DSA, and SLH-DSA natively without any +dependency on liboqs. Comparison against liboqs numbers is valid for +these algorithms. + +**Falcon** in wolfSSL requires liboqs (wolfSSL delegates Falcon to the +liboqs library via a thin shim). Falcon numbers, if present, use that path +and are not a native wolfSSL implementation measurement. + +### Generating PQC-LEO-compatible output + +```sh +# Run the benchmark +./wolfcrypt/benchmark/pqc_bench.sh --output pqc_results.csv + +# Convert to PQC-LEO pipe-delimited format +python3 wolfcrypt/benchmark/pqc_parse.py \ + --format=pqcleo \ + --output=pqc_results_pqcleo.psv \ + pqc_results.csv + +# Feed into PQC-LEO's parser (from the PQC-LEO repo): +python3 scripts/parsing_scripts/parse_results.py pqc_results_pqcleo.psv +``` + +Note that PQC-LEO memory numbers (`intits`, `peakBytes`, etc.) come from +Valgrind massif and are not included in wolfSSL's inline tracking output. +Cross-library memory comparison requires Option B instrumentation (see Section 4). + +--- + +## 7. Citation + +To cite wolfSSL in academic or technical work: + +> wolfSSL Inc., "wolfSSL Embedded SSL/TLS Library," https://www.wolfssl.com, 2024. + +For algorithm-specific citations, refer to the relevant NIST FIPS standards: + +- FIPS 203 (ML-KEM): https://doi.org/10.6028/NIST.FIPS.203 +- FIPS 204 (ML-DSA): https://doi.org/10.6028/NIST.FIPS.204 +- FIPS 205 (SLH-DSA): https://doi.org/10.6028/NIST.FIPS.205 From fd6fc8eda43d020e31847553001f9b75cb674bf0 Mon Sep 17 00:00:00 2001 From: Mark Atwood Date: Wed, 6 May 2026 17:30:53 -0700 Subject: [PATCH 06/10] feat: add copy() to SHA256 and SHA384 via wc_Sha256Copy/wc_Sha384Copy --- wrapper/rust/wolfssl-wolfcrypt/src/sha.rs | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/wrapper/rust/wolfssl-wolfcrypt/src/sha.rs b/wrapper/rust/wolfssl-wolfcrypt/src/sha.rs index 2de688b39d2..5698adccaa6 100644 --- a/wrapper/rust/wolfssl-wolfcrypt/src/sha.rs +++ b/wrapper/rust/wolfssl-wolfcrypt/src/sha.rs @@ -635,6 +635,24 @@ impl SHA256 { } Ok(()) } + + /// Copy the SHA-256 state into a new instance (O(1) clone via wc_Sha256Copy). + /// + /// Returns a new `SHA256` context with identical internal state, + /// allowing the same computation to be continued independently. + /// + /// # Returns + /// + /// Returns either `Ok(sha256)` containing the copied instance or `Err(e)` + /// containing the wolfSSL library error code value. + pub fn copy(&self) -> Result { + let mut dst: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); + let rc = unsafe { sys::wc_Sha256Copy(&self.wc_sha256 as *const _ as *mut _, dst.as_mut_ptr()) }; + if rc != 0 { + return Err(rc); + } + Ok(SHA256 { wc_sha256: unsafe { dst.assume_init() } }) + } } #[cfg(sha256)] @@ -846,6 +864,24 @@ impl SHA384 { } Ok(()) } + + /// Copy the SHA-384 state into a new instance (O(1) clone via wc_Sha384Copy). + /// + /// Returns a new `SHA384` context with identical internal state, + /// allowing the same computation to be continued independently. + /// + /// # Returns + /// + /// Returns either `Ok(sha384)` containing the copied instance or `Err(e)` + /// containing the wolfSSL library error code value. + pub fn copy(&self) -> Result { + let mut dst: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); + let rc = unsafe { sys::wc_Sha384Copy(&self.wc_sha384 as *const _ as *mut _, dst.as_mut_ptr()) }; + if rc != 0 { + return Err(rc); + } + Ok(SHA384 { wc_sha384: unsafe { dst.assume_init() } }) + } } #[cfg(sha384)] From 73847ed1fdd2f351b4f51e21ea9b4c4a679f051d Mon Sep 17 00:00:00 2001 From: Mark Atwood Date: Wed, 6 May 2026 18:47:13 -0700 Subject: [PATCH 07/10] feat: extend pqc_parse.py with liboqs, openssl, circl input parsers Add --input-format flag supporting wolfssl (existing), liboqs, openssl, and circl input formats, enabling a unified comparison CSV from any PQC library. All parsers normalise to the same canonical schema before the existing --format output stage. - liboqs: parses speed_kem/speed_sig fixed-width table output - openssl: parses 'openssl speed -mr' +R15..+R20 machine-readable lines - circl: parses 'go test -bench' ns/op output with GOMAXPROCS suffix stripping - wolfssl: existing parser retained unchanged, now selectable via --input-format --- wolfcrypt/benchmark/pqc_parse.py | 728 +++++++++++++++++++++---------- 1 file changed, 496 insertions(+), 232 deletions(-) diff --git a/wolfcrypt/benchmark/pqc_parse.py b/wolfcrypt/benchmark/pqc_parse.py index 4ff29b2c982..2360ee4aa0e 100755 --- a/wolfcrypt/benchmark/pqc_parse.py +++ b/wolfcrypt/benchmark/pqc_parse.py @@ -1,46 +1,67 @@ #!/usr/bin/env python3 """ -pqc_parse.py — Normalize wolfSSL PQC benchmark CSV for publication. +pqc_parse.py — Normalize PQC benchmark output for publication. -Takes the CSV produced by pqc_bench.sh and emits clean, publication-ready -output in one of two formats: +Accepts output from wolfSSL, liboqs, OpenSSL, or CIRCL and emits +clean, publication-ready output with a consistent schema. - --format=wolfssl (default) - Pipe-aligned CSV with a Library column prepended. - Columns: Library, Algorithm, Operation, ops/sec, avg_ms, ops, secs - [, heap_bytes, heap_allocs, stack_bytes] +OUTPUT FORMATS (--format): + wolfssl (default) + CSV with Library column. Columns: + Library, Algorithm, Operation, ops/sec, avg_ms + [, ops, secs] [, heap_bytes, heap_allocs, stack_bytes] - --format=pqcleo - Pipe-delimited format matching PQC-LEO's parser expectations: - Algorithm | Operation | Operations | Seconds | ms/op | op/sec - (memory columns dropped — PQC-LEO memory is from Valgrind massif) + pqcleo + Pipe-delimited format for PQC-LEO cross-library comparison: + Algorithm | Operation | Operations | Seconds | ms/op | op/sec + +INPUT FORMATS (--input-format): + wolfssl (default) + CSV produced by pqc_bench.sh. Supports 5, 8, and 12-column layouts. + + liboqs + Text output of liboqs speed_kem / speed_sig binaries. + Fixed-width table; algorithm name on a preceding header line. + Example: + ML-KEM-512 + keygen | 12345 | 3.000 | 243.2 | ... + encaps | 12345 | 3.000 | 243.2 | ... + decaps | 12345 | 3.000 | 243.2 | ... + + openssl + Machine-readable output of: openssl speed -mr -kem-algorithms + openssl speed -mr -signature-algorithms + Format: +R15::: (KEM keygen) + +R16::: (KEM encaps) + +R17::: (KEM decaps) + +R18::: (SIG keygen) + +R19::: (SIG sign) + +R20::: (SIG verify) + + circl + Output of: go test -bench=. -benchtime=5s ./kem/schemes/ ./sign/schemes/ + Format: BenchmarkGenerateKeyPair/ML-KEM-512 N 98765 ns/op + BenchmarkEncapsulate/ML-KEM-512 N 98765 ns/op + BenchmarkDecapsulate/ML-KEM-512 N 98765 ns/op + BenchmarkGenerateKeyPair/ML-DSA-44 N 98765 ns/op + BenchmarkSign/ML-DSA-44 N 98765 ns/op + BenchmarkVerify/ML-DSA-44 N 98765 ns/op USAGE: - python3 pqc_parse.py [OPTIONS] INPUT_CSV + python3 pqc_parse.py [OPTIONS] INPUT OPTIONS: - --format FORMAT Output format: wolfssl (default) or pqcleo - --library NAME Library name for wolfssl format (default: wolfSSL) - --output FILE Write to FILE instead of stdout - --help Print this help and exit + --input-format FMT Input format: wolfssl (default), liboqs, openssl, circl + --format FMT Output format: wolfssl (default) or pqcleo + --library NAME Library name override (default: auto-detected from input format) + --output FILE Write to FILE instead of stdout + --help Print this help and exit EXAMPLES: python3 pqc_parse.py pqc_results.csv - python3 pqc_parse.py --format=pqcleo --output=results.psv pqc_results.csv - -INPUT FORMAT: - The script accepts the CSV produced by pqc_bench.sh in both its supported - output variants: - - With GENERATE_MACHINE_PARSEABLE_REPORT + cycles columns (12 fields): - Algorithm,key size,operation,avg ms,ops/sec,ops,secs,cycles,cycles/op, - heap_bytes,heap_allocs,stack_bytes - - Without cycles (8 fields, older build): - Algorithm,key size,operation,avg ms,ops/sec,heap_bytes,heap_allocs, - stack_bytes - - Minimal (5 fields, no memory tracking): - Algorithm,key size,operation,avg ms,ops/sec - - Memory columns are optional; missing columns are reported as empty strings. + python3 pqc_parse.py --input-format=liboqs speed_kem.txt + python3 pqc_parse.py --input-format=openssl --library=OpenSSL-3.5 openssl_speed.txt + python3 pqc_parse.py --input-format=circl --format=pqcleo circl_bench.txt """ import argparse @@ -51,114 +72,126 @@ # --------------------------------------------------------------------------- -# Algorithm name normalisation +# Canonical data record # --------------------------------------------------------------------------- -# wolfSSL benchmark output uses internal names that differ from NIST names. -# These maps translate what the benchmark binary actually emits. - -# ML-KEM: "ML-KEM 512 " (trailing space), key size in separate column. -# We reconstruct "ML-KEM-" from the algorithm + key-size fields. -_MLKEM_RE = re.compile(r"ML-KEM\s+(\d+)\s*$", re.IGNORECASE) +# All input parsers produce a list of these dicts before the output +# formatters consume them. Fields not available from a given input source +# are left as empty strings. + +def make_record( + library: str, + algorithm: str, + operation: str, + ops_per_sec: str, + avg_ms: str, + ops: str = "", + secs: str = "", + heap_bytes: str = "", + heap_allocs: str = "", + stack_bytes: str = "", +) -> dict: + return { + "library": library, + "algorithm": algorithm, + "operation": operation, + "ops_per_sec": ops_per_sec, + "avg_ms": avg_ms, + "ops": ops, + "secs": secs, + "heap_bytes": heap_bytes, + "heap_allocs": heap_allocs, + "stack_bytes": stack_bytes, + } -# ML-DSA: algorithm field is "ML-DSA", key-size field is security level (44/65/87). -_MLDSA_RE = re.compile(r"ML-DSA\s*$", re.IGNORECASE) -# SLH-DSA: algorithm field is "SLH-DSA-S" (small) or "SLH-DSA-F" (fast), -# key-size field is the security category (128/192/256). -# NIST name: SLH-DSA-SHAKE- -# S → s (small), F → f (fast) -_SLHDSA_RE = re.compile(r"SLH-DSA-([SF])\s*$", re.IGNORECASE) +# --------------------------------------------------------------------------- +# Algorithm and operation name normalisation (shared across all parsers) +# --------------------------------------------------------------------------- -# Legacy Dilithium names (in case an older wolfSSL build is used): -# "DILITHIUM" with key-size field being the security level. -_DILITHIUM_RE = re.compile(r"DILITHIUM\s*$", re.IGNORECASE) +# wolfSSL emits "ML-KEM 512 " (space-separated, trailing space) +_WOLFSSL_MLKEM_RE = re.compile(r"ML-KEM\s+(\d+)\s*$", re.IGNORECASE) +_WOLFSSL_MLDSA_RE = re.compile(r"ML-DSA\s*$", re.IGNORECASE) +_WOLFSSL_SLHDSA_RE = re.compile(r"SLH-DSA-([SF])\s*$", re.IGNORECASE) +_WOLFSSL_DILITH_RE = re.compile(r"DILITHIUM\s*$", re.IGNORECASE) +# Canonical algorithm names already in NIST form (liboqs / OpenSSL / CIRCL) +_CANONICAL_RE = re.compile( + r"^(ML-KEM-(512|768|1024)|ML-DSA-(44|65|87)|SLH-DSA-(SHA2|SHAKE)-(128|192|256)[sf])$", + re.IGNORECASE, +) -def normalise_algorithm(raw_algo: str, raw_keysize: str) -> str: +def normalise_algorithm(raw_algo: str, raw_keysize: str = "") -> str: """ - Convert wolfSSL benchmark algorithm + key-size fields to a canonical - NIST/IETF algorithm name. - - Returns the normalised name as a string, or the original (stripped) name - if no normalisation rule applies (future-proofing for new algorithms). + Convert any benchmark tool's algorithm name to canonical NIST form. + raw_keysize is only needed for wolfSSL's two-column format. """ - algo = raw_algo.strip() + algo = raw_algo.strip() keysize = raw_keysize.strip() - m = _MLKEM_RE.match(algo) + # Already canonical (liboqs / OpenSSL / CIRCL emit NIST names) + if _CANONICAL_RE.match(algo): + return algo + + # wolfSSL internal names + m = _WOLFSSL_MLKEM_RE.match(algo) if m: - # benchmark already encodes the security level in the name; - # the key-size column is the same value. Use the name field. return f"ML-KEM-{m.group(1)}" - if _MLDSA_RE.match(algo): - # key-size field is the security level (44, 65, 87) + if _WOLFSSL_MLDSA_RE.match(algo): return f"ML-DSA-{keysize}" - m = _SLHDSA_RE.match(algo) + m = _WOLFSSL_SLHDSA_RE.match(algo) if m: - size_char = m.group(1).lower() # 's' or 'f' - return f"SLH-DSA-SHAKE-{keysize}{size_char}" + return f"SLH-DSA-SHAKE-{keysize}{m.group(1).lower()}" - if _DILITHIUM_RE.match(algo): - # Legacy name from older wolfSSL builds + if _WOLFSSL_DILITH_RE.match(algo): return f"ML-DSA-{keysize}" - # Unknown algorithm: return stripped name unchanged so we don't silently - # drop data from future wolfSSL versions that add new algorithms. + # Pass through unknown names unchanged return algo -# --------------------------------------------------------------------------- -# Operation label normalisation -# --------------------------------------------------------------------------- -# wolfSSL uses slightly different operation names than the PQC-LEO convention. - +# Operation label map — normalise all source tool labels to canonical form _OP_MAP = { - "key gen": "keygen", # KEM and SIG key generation - "gen": "keygen", # SLH-DSA uses 'gen' instead of 'key gen' - "encap": "encaps", # KEM encapsulation - "decap": "decaps", # KEM decapsulation - "sign": "sign", - "verify": "verify", - # SLH-DSA extended operations (pre-hash and message variants) - # Pass through with minor normalisation so the output remains valid. - "sign-msg": "sign-msg", - "vrfy-msg": "verify-msg", - "sign-pre": "sign-pre", - "vrfy-pre": "verify-pre", + # wolfSSL + "key gen": "keygen", + "gen": "keygen", + "encap": "encaps", + "decap": "decaps", + "sign": "sign", + "verify": "verify", + "sign-msg": "sign-msg", + "vrfy-msg": "verify-msg", + "sign-pre": "sign-pre", + "vrfy-pre": "verify-pre", + # liboqs + "keygen": "keygen", + "encaps": "encaps", + "decaps": "decaps", + "keypair": "keygen", # liboqs speed_sig uses "keypair" + # OpenSSL / CIRCL already use canonical names mostly + "generatekeypair": "keygen", + "encapsulate": "encaps", + "decapsulate": "decaps", } - def normalise_operation(raw_op: str) -> str: - """Normalise a wolfSSL operation label to canonical form.""" - op = raw_op.strip() - return _OP_MAP.get(op, op) # unknown ops pass through unchanged + op = raw_op.strip().lower() + return _OP_MAP.get(op, raw_op.strip()) + + +def _fmt_ops_per_sec(ops: float) -> str: + return f"{ops:.3f}" + +def _fmt_avg_ms(ms: float) -> str: + return f"{ms:.3f}" # --------------------------------------------------------------------------- -# CSV column layout detection +# wolfSSL input parser (existing CSV format) # --------------------------------------------------------------------------- -# pqc_bench.sh can produce two column layouts depending on build flags: -# -# Layout A (GENERATE_MACHINE_PARSEABLE_REPORT, 12 fields): -# Algorithm, key size, operation, avg ms, ops/sec, -# ops, secs, cycles, cycles/op, heap_bytes, heap_allocs, stack_bytes -# -# Layout B (plain -csv, 8 fields): -# Algorithm, key size, operation, avg ms, ops/sec, -# heap_bytes, heap_allocs, stack_bytes -# -# Layout C (minimal, 5 fields, no memory tracking): -# Algorithm, key size, operation, avg ms, ops/sec - -def detect_layout(header: list[str]) -> dict: - """ - Given the parsed header row, return a dict mapping logical field names to - column indices. Returns None for fields not present in this layout. - All header matching is case-insensitive and strips whitespace. - """ +def _detect_wolfssl_layout(header: list[str]) -> dict: h = [f.strip().lower() for f in header] def idx(name: str) -> Optional[int]: @@ -168,55 +201,312 @@ def idx(name: str) -> Optional[int]: return None layout = { - "algorithm": idx("algorithm"), - "key_size": idx("key size"), - "operation": idx("operation"), - "avg_ms": idx("avg ms"), - "ops_per_sec": idx("ops/sec"), - "ops": idx("ops"), - "secs": idx("secs"), - "cycles": idx("cycles"), - "cycles_per_op": idx("cycles/op"), - "heap_bytes": idx("heap_bytes"), - "heap_allocs": idx("heap_allocs"), - "stack_bytes": idx("stack_bytes"), + "algorithm": idx("algorithm"), + "key_size": idx("key size"), + "operation": idx("operation"), + "avg_ms": idx("avg ms"), + "ops_per_sec": idx("ops/sec"), + "ops": idx("ops"), + "secs": idx("secs"), + "heap_bytes": idx("heap_bytes"), + "heap_allocs": idx("heap_allocs"), + "stack_bytes": idx("stack_bytes"), } - # Validate that mandatory fields are present - mandatory = ("algorithm", "key_size", "operation", "avg_ms", "ops_per_sec") - for field in mandatory: + for field in ("algorithm", "key_size", "operation", "avg_ms", "ops_per_sec"): if layout[field] is None: raise ValueError( - f"Input CSV is missing required column '{field}'. " - f"Got columns: {header}" + f"wolfSSL CSV missing required column '{field}'. Got: {header}" ) - return layout -def get_field(row: list[str], layout: dict, field: str, default: str = "") -> str: - """Extract a field from a row using the pre-computed layout, or return default.""" - idx = layout.get(field) - if idx is None or idx >= len(row): - return default - return row[idx].strip() +def _get(row: list[str], layout: dict, field: str) -> str: + i = layout.get(field) + if i is None or i >= len(row): + return "" + return row[i].strip() + + +def parse_wolfssl(text: str, library: str) -> list[dict]: + records = [] + reader = csv.reader(text.splitlines()) + header = None + layout = None + + for row in reader: + if not any(f.strip() for f in row): + continue + + # New-format header: starts with quoted type + "Algorithm" + if row[0].strip().strip('"').lower() in ("asym", "sym") and \ + len(row) > 1 and row[1].strip().lower() == "algorithm": + # Strip leading type column + row = row[1:] + + if header is None: + if row[0].strip().lower() == "algorithm": + header = [f.strip() for f in row] + # Strip trailing comma artefact (empty last field) + if header and header[-1] == "": + header = header[:-1] + try: + layout = _detect_wolfssl_layout(header) + except ValueError as e: + print(f"WARNING: {e}", file=sys.stderr) + return records + continue + + # Data row: new format prefixes with type token, strip it + if row[0].strip().lower() in ("asym", "sym"): + row = row[1:] + + # Strip trailing empty field from wolfSSL's trailing-comma habit + if row and row[-1].strip() == "": + row = row[:-1] + + if len(row) < 5: + continue + + algo = normalise_algorithm(_get(row, layout, "algorithm"), + _get(row, layout, "key_size")) + op = normalise_operation(_get(row, layout, "operation")) + ops_sec = _get(row, layout, "ops_per_sec") + avg_ms = _get(row, layout, "avg_ms") + + records.append(make_record( + library=library, algorithm=algo, operation=op, + ops_per_sec=ops_sec, avg_ms=avg_ms, + ops=_get(row, layout, "ops"), + secs=_get(row, layout, "secs"), + heap_bytes=_get(row, layout, "heap_bytes"), + heap_allocs=_get(row, layout, "heap_allocs"), + stack_bytes=_get(row, layout, "stack_bytes"), + )) + return records + + +# --------------------------------------------------------------------------- +# liboqs input parser +# --------------------------------------------------------------------------- +# Output format from speed_kem / speed_sig: +# +# Started at ... +# Operation | Iterations | Total time (s) | Time (us): mean | pop. stdev | cycles/op | pop. stdev +# ------------------------------------ | ----------:| ---------------:| ---------------:| ----------:| ---------:| ----------: +# ML-KEM-512 +# keygen | 12345 | 3.000 | 243.18 | 1.23 | ... | ... +# encaps | 12345 | 3.000 | 243.18 | 1.23 | ... | ... +# decaps | 12345 | 3.000 | 243.18 | 1.23 | ... | ... +# +# Algorithm name appears on its own line immediately before its operations. +# The table header line contains "Iterations" (used for detection). +# Data lines have the operation name in column 0, iteration count in col 1, +# total time (s) in col 2, mean time (us) in col 3. + +_LIBOQS_HEADER_RE = re.compile(r"Iterations", re.IGNORECASE) +# A data row: starts with an operation name word, then pipe-separated numbers. +# Operation names: keygen, encaps, decaps, keypair, sign, verify, fullcycle... +_LIBOQS_DATA_RE = re.compile( + r"^\s*(\w[\w\-]*)\s*\|\s*(\d+)\s*\|\s*([\d.]+)\s*\|\s*([\d.]+)" +) +# Algorithm header line: a non-empty line that is NOT the column header, +# NOT a separator (---), and does NOT contain a pipe character. +_LIBOQS_ALG_RE = re.compile(r"^[A-Z][\w\-]+$") + + +def parse_liboqs(text: str, library: str) -> list[dict]: + records = [] + current_alg = None + in_table = False + + for raw_line in text.splitlines(): + line = raw_line.strip() + + if not line: + continue + + # Table header + if _LIBOQS_HEADER_RE.search(line): + in_table = True + continue + + # Separator line + if re.match(r"^[-| ]+$", line): + continue + + # Algorithm name line (no pipe, matches algo pattern) + if "|" not in line and _LIBOQS_ALG_RE.match(line): + current_alg = normalise_algorithm(line) + continue + + # Data row + if in_table and current_alg: + m = _LIBOQS_DATA_RE.match(line) + if m: + op_raw = m.group(1) + iterations = m.group(2) + total_secs = float(m.group(3)) + mean_us = float(m.group(4)) + avg_ms = mean_us / 1000.0 + ops_per_sec = 1_000_000.0 / mean_us if mean_us > 0 else 0.0 + + op = normalise_operation(op_raw) + # Skip fullcycle / fullcycletest rows — not a primitive operation + if "fullcycle" in op.lower(): + continue + + records.append(make_record( + library=library, + algorithm=current_alg, + operation=op, + ops_per_sec=_fmt_ops_per_sec(ops_per_sec), + avg_ms=_fmt_avg_ms(avg_ms), + ops=iterations, + secs=f"{total_secs:.3f}", + )) + + return records + + +# --------------------------------------------------------------------------- +# OpenSSL input parser (-mr machine-readable output) +# --------------------------------------------------------------------------- +# +R15::: — KEM keygen +# +R16::: — KEM encaps +# +R17::: — KEM decaps +# +R18::: — SIG keygen +# +R19::: — SIG sign +# +R20::: — SIG verify + +_OPENSSL_MR_RE = re.compile(r"^\+R(1[5-9]|20):(\d+):([^:]+):([\d.]+)") +_OPENSSL_OP_MAP = { + "15": "keygen", # KEM keygen + "16": "encaps", # KEM encaps + "17": "decaps", # KEM decaps + "18": "keygen", # SIG keygen + "19": "sign", # SIG sign + "20": "verify", # SIG verify +} + + +def parse_openssl(text: str, library: str) -> list[dict]: + records = [] + for line in text.splitlines(): + m = _OPENSSL_MR_RE.match(line.strip()) + if not m: + continue + rtype = m.group(1) + count = int(m.group(2)) + alg = m.group(3).strip() + secs = float(m.group(4)) + + op = _OPENSSL_OP_MAP.get(rtype, "unknown") + avg_ms = (secs / count * 1000.0) if count > 0 else 0.0 + ops_per_sec = count / secs if secs > 0 else 0.0 + algorithm = normalise_algorithm(alg) + + records.append(make_record( + library=library, + algorithm=algorithm, + operation=op, + ops_per_sec=_fmt_ops_per_sec(ops_per_sec), + avg_ms=_fmt_avg_ms(avg_ms), + ops=str(count), + secs=f"{secs:.3f}", + )) + return records + + +# --------------------------------------------------------------------------- +# CIRCL input parser (go test -bench output) +# --------------------------------------------------------------------------- +# Format: BenchmarkGenerateKeyPair/ML-KEM-512-8 1234 98765 ns/op +# The "-8" suffix is the GOMAXPROCS value; strip it. +# Benchmark function names map to operations: +# BenchmarkGenerateKeyPair -> keygen +# BenchmarkEncapsulate -> encaps +# BenchmarkDecapsulate -> decaps +# BenchmarkEncap -> encaps (alternate naming) +# BenchmarkDecap -> decaps +# BenchmarkSign -> sign +# BenchmarkVerify -> verify + +_CIRCL_LINE_RE = re.compile( + r"^Benchmark(\w+)/([^\s]+)\s+(\d+)\s+([\d.]+)\s+ns/op" +) +_CIRCL_OP_MAP = { + "GenerateKeyPair": "keygen", + "Encapsulate": "encaps", + "Decapsulate": "decaps", + "Encap": "encaps", + "Decap": "decaps", + "Sign": "sign", + "Verify": "verify", + "KeyGen": "keygen", + "KeyGenerate": "keygen", +} +# GOMAXPROCS suffix: "-N" at end of algorithm name +_CIRCL_GOMAXPROCS_RE = re.compile(r"-\d+$") + + +def parse_circl(text: str, library: str) -> list[dict]: + records = [] + for line in text.splitlines(): + m = _CIRCL_LINE_RE.match(line.strip()) + if not m: + continue + bench_fn = m.group(1) # e.g. "GenerateKeyPair" + alg_raw = m.group(2) # e.g. "ML-KEM-512-8" + iters = int(m.group(3)) + ns_per = float(m.group(4)) # nanoseconds per operation + + # Strip GOMAXPROCS suffix from algorithm name + alg_clean = _CIRCL_GOMAXPROCS_RE.sub("", alg_raw) + algorithm = normalise_algorithm(alg_clean) + + op = _CIRCL_OP_MAP.get(bench_fn) + if op is None: + # Unknown benchmark function — skip rather than emit garbage + continue + + avg_ms = ns_per / 1_000_000.0 + ops_per_sec = 1_000_000_000.0 / ns_per if ns_per > 0 else 0.0 + + records.append(make_record( + library=library, + algorithm=algorithm, + operation=op, + ops_per_sec=_fmt_ops_per_sec(ops_per_sec), + avg_ms=_fmt_avg_ms(avg_ms), + ops=str(iters), + secs=_fmt_avg_ms(iters * ns_per / 1e9), + )) + return records # --------------------------------------------------------------------------- # Output formatters # --------------------------------------------------------------------------- -def write_wolfssl_format(rows, layout: dict, library: str, out): - """ - Write normalised wolfssl CSV format. +# Algorithms we care about — filter out unrelated ones when present +_PQC_ALGO_RE = re.compile( + r"^(ML-KEM|ML-DSA|SLH-DSA)", re.IGNORECASE +) - Columns: Library, Algorithm, Operation, ops/sec, avg_ms, ops, secs - [, heap_bytes, heap_allocs, stack_bytes] +def _is_pqc(record: dict) -> bool: + return bool(_PQC_ALGO_RE.match(record["algorithm"])) - Memory columns are omitted if they were not present in the input. + +def write_wolfssl_format(records: list[dict], out) -> None: + """ + Normalised wolfssl CSV. + Columns: Library, Algorithm, Operation, ops/sec, avg_ms + [, ops, secs] [, heap_bytes, heap_allocs, stack_bytes] + Memory columns are included only if at least one record has them. """ - has_memory = layout["heap_bytes"] is not None - has_timing = layout["ops"] is not None + has_timing = any(r["ops"] for r in records) + has_memory = any(r["heap_bytes"] for r in records) header = ["Library", "Algorithm", "Operation", "ops/sec", "avg_ms"] if has_timing: @@ -227,149 +517,123 @@ def write_wolfssl_format(rows, layout: dict, library: str, out): writer = csv.writer(out, lineterminator="\n") writer.writerow(header) - for row in rows: - raw_algo = get_field(row, layout, "algorithm") - raw_keysize = get_field(row, layout, "key_size") - raw_op = get_field(row, layout, "operation") - - algorithm = normalise_algorithm(raw_algo, raw_keysize) - operation = normalise_operation(raw_op) - ops_per_sec = get_field(row, layout, "ops_per_sec") - avg_ms = get_field(row, layout, "avg_ms") - - out_row = [library, algorithm, operation, ops_per_sec, avg_ms] - + for r in records: + if not _is_pqc(r): + continue + row = [r["library"], r["algorithm"], r["operation"], + r["ops_per_sec"], r["avg_ms"]] if has_timing: - out_row.append(get_field(row, layout, "ops")) - out_row.append(get_field(row, layout, "secs")) - + row += [r["ops"], r["secs"]] if has_memory: - out_row.append(get_field(row, layout, "heap_bytes")) - out_row.append(get_field(row, layout, "heap_allocs")) - out_row.append(get_field(row, layout, "stack_bytes")) - - writer.writerow(out_row) + row += [r["heap_bytes"], r["heap_allocs"], r["stack_bytes"]] + writer.writerow(row) -def write_pqcleo_format(rows, layout: dict, out): - """ - Write PQC-LEO pipe-delimited format for cross-library comparison. - - PQC-LEO parser expects: - Algorithm | Operation | Operations | Seconds | ms/op | op/sec - - Memory columns are dropped (PQC-LEO memory comes from Valgrind massif, - which is incompatible with wolfSSL's inline allocation tracking). - """ - # PQC-LEO uses pipe delimiter with spaces: " | " +def write_pqcleo_format(records: list[dict], out) -> None: + """PQC-LEO pipe-delimited format.""" sep = " | " - def write_row(fields): + def wr(fields): out.write(sep.join(str(f) for f in fields) + "\n") - write_row(["Algorithm", "Operation", "Operations", "Seconds", "ms/op", "op/sec"]) - - for row in rows: - raw_algo = get_field(row, layout, "algorithm") - raw_keysize = get_field(row, layout, "key_size") - raw_op = get_field(row, layout, "operation") - - algorithm = normalise_algorithm(raw_algo, raw_keysize) - operation = normalise_operation(raw_op) - ops_per_sec = get_field(row, layout, "ops_per_sec") - avg_ms = get_field(row, layout, "avg_ms") - ops = get_field(row, layout, "ops", default="") - secs = get_field(row, layout, "secs", default="") - - write_row([algorithm, operation, ops, secs, avg_ms, ops_per_sec]) + wr(["Algorithm", "Operation", "Operations", "Seconds", "ms/op", "op/sec"]) + for r in records: + if not _is_pqc(r): + continue + wr([r["algorithm"], r["operation"], r["ops"], + r["secs"], r["avg_ms"], r["ops_per_sec"]]) # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- +_DEFAULT_LIBRARY = { + "wolfssl": "wolfSSL", + "liboqs": "liboqs", + "openssl": "OpenSSL", + "circl": "CIRCL", +} + + def main(): parser = argparse.ArgumentParser( - description=__doc__.split("\n")[1].strip(), + description="Normalize PQC benchmark output for publication.", formatter_class=argparse.RawDescriptionHelpFormatter, - epilog="\n".join(__doc__.split("\n")[1:]), + epilog=__doc__, ) parser.add_argument( - "input", - metavar="INPUT_CSV", - help="CSV file produced by pqc_bench.sh (use '-' for stdin)", + "input", metavar="INPUT", + help="Input file (use '-' for stdin)", + ) + parser.add_argument( + "--input-format", + choices=("wolfssl", "liboqs", "openssl", "circl"), + default="wolfssl", + help="Input format (default: wolfssl)", ) parser.add_argument( "--format", choices=("wolfssl", "pqcleo"), default="wolfssl", - help="Output format: wolfssl (default) or pqcleo", + help="Output format (default: wolfssl)", ) parser.add_argument( "--library", - default="wolfSSL", - help="Library name for wolfssl format (default: wolfSSL)", + default=None, + help="Library name override (default: auto from --input-format)", ) parser.add_argument( - "--output", - default="-", + "--output", default="-", help="Output file (default: stdout)", ) args = parser.parse_args() + library = args.library or _DEFAULT_LIBRARY[args.input_format] + # Open input if args.input == "-": - in_file = sys.stdin + text = sys.stdin.read() else: try: - in_file = open(args.input, newline="", encoding="utf-8") + with open(args.input, encoding="utf-8") as f: + text = f.read() except OSError as e: - print(f"ERROR: Cannot open input file: {e}", file=sys.stderr) + print(f"ERROR: Cannot open input: {e}", file=sys.stderr) sys.exit(1) + # Parse + parsers = { + "wolfssl": parse_wolfssl, + "liboqs": parse_liboqs, + "openssl": parse_openssl, + "circl": parse_circl, + } + records = parsers[args.input_format](text, library) + + if not records: + print("WARNING: no records parsed from input", file=sys.stderr) + # Open output if args.output == "-": - out_file = sys.stdout + out = sys.stdout + _write_and_close = False else: try: - out_file = open(args.output, "w", encoding="utf-8") + out = open(args.output, "w", encoding="utf-8") + _write_and_close = True except OSError as e: - print(f"ERROR: Cannot open output file: {e}", file=sys.stderr) + print(f"ERROR: Cannot open output: {e}", file=sys.stderr) sys.exit(1) try: - reader = csv.reader(in_file) - - # Read and parse header - try: - header = next(reader) - except StopIteration: - print("ERROR: Input CSV is empty (no header row)", file=sys.stderr) - sys.exit(1) - - try: - layout = detect_layout(header) - except ValueError as e: - print(f"ERROR: {e}", file=sys.stderr) - sys.exit(1) - - # Read all data rows (skip blank lines) - data_rows = [row for row in reader if any(f.strip() for f in row)] - - if not data_rows: - print("WARNING: Input CSV has a header but no data rows", file=sys.stderr) - - # Emit in requested format if args.format == "wolfssl": - write_wolfssl_format(data_rows, layout, args.library, out_file) + write_wolfssl_format(records, out) else: - write_pqcleo_format(data_rows, layout, out_file) - + write_pqcleo_format(records, out) finally: - if in_file is not sys.stdin: - in_file.close() - if out_file is not sys.stdout: - out_file.close() + if _write_and_close: + out.close() sys.exit(0) From 606f5f49bb7bd80ff577eec6a2919f28467029c6 Mon Sep 17 00:00:00 2001 From: Mark Atwood Date: Thu, 7 May 2026 13:07:56 -0700 Subject: [PATCH 08/10] fix: liboqs 0.15 format, openssl filename prefix, SLH-DSA names - liboqs 0.15 emits algorithm headers with trailing pipe fields; update regex to match and extract just the name token - liboqs 0.15 SLH-DSA names are SLH_DSA_PURE_SHA2_128S (underscores, PURE prefix); add normalisation rule to canonical SLH-DSA-SHA2-128s - OpenSSL grep-with-filename prefix stripped in parser (e.g. 'file:+R15') - liboqs _LIBOQS_ALG_RE now allows underscore in algorithm names --- wolfcrypt/benchmark/pqc_parse.py | 41 +++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/wolfcrypt/benchmark/pqc_parse.py b/wolfcrypt/benchmark/pqc_parse.py index 2360ee4aa0e..0981df57ffd 100755 --- a/wolfcrypt/benchmark/pqc_parse.py +++ b/wolfcrypt/benchmark/pqc_parse.py @@ -132,6 +132,18 @@ def normalise_algorithm(raw_algo: str, raw_keysize: str = "") -> str: if _CANONICAL_RE.match(algo): return algo + # liboqs 0.15+ SLH-DSA name format: SLH_DSA_PURE_SHA2_128S + # Map to canonical: SLH-DSA-SHA2-128s + _LIBOQS_SLHDSA_RE = re.compile( + r"^SLH_DSA_PURE_(SHA2|SHAKE)_(\d+)([SF])$", re.IGNORECASE + ) + m = _LIBOQS_SLHDSA_RE.match(algo) + if m: + hash_fn = m.group(1).upper() # SHA2 or SHAKE + bits = m.group(2) # 128, 192, 256 + size = m.group(3).lower() # s or f + return f"SLH-DSA-{hash_fn}-{bits}{size}" + # wolfSSL internal names m = _WOLFSSL_MLKEM_RE.match(algo) if m: @@ -310,9 +322,13 @@ def parse_wolfssl(text: str, library: str) -> list[dict]: _LIBOQS_DATA_RE = re.compile( r"^\s*(\w[\w\-]*)\s*\|\s*(\d+)\s*\|\s*([\d.]+)\s*\|\s*([\d.]+)" ) -# Algorithm header line: a non-empty line that is NOT the column header, -# NOT a separator (---), and does NOT contain a pipe character. -_LIBOQS_ALG_RE = re.compile(r"^[A-Z][\w\-]+$") +# Algorithm header line: starts with a NIST algorithm name, optionally +# followed by pipe-separated empty fields (liboqs 0.15+ format). +# Examples: +# "ML-KEM-512" (old format, bare name) +# "ML-KEM-512 | | ..." (new format, trailing pipes) +# "SLH_DSA_PURE_SHA2_128S" (liboqs 0.15 SLH-DSA name) +_LIBOQS_ALG_RE = re.compile(r"^([A-Z][A-Z0-9_\-]+)\s*(\|.*)?$") def parse_liboqs(text: str, library: str) -> list[dict]: @@ -335,10 +351,14 @@ def parse_liboqs(text: str, library: str) -> list[dict]: if re.match(r"^[-| ]+$", line): continue - # Algorithm name line (no pipe, matches algo pattern) - if "|" not in line and _LIBOQS_ALG_RE.match(line): - current_alg = normalise_algorithm(line) - continue + # Algorithm name line: starts with a NIST name, optional trailing pipes + m_alg = _LIBOQS_ALG_RE.match(line) + if m_alg and not _LIBOQS_DATA_RE.match(line): + candidate = m_alg.group(1).strip() + # Only treat as an algorithm header if it looks like a PQC name + if _PQC_ALGO_RE.match(candidate): + current_alg = normalise_algorithm(candidate) + continue # Data row if in_table and current_alg: @@ -393,7 +413,12 @@ def parse_liboqs(text: str, library: str) -> list[dict]: def parse_openssl(text: str, library: str) -> list[dict]: records = [] for line in text.splitlines(): - m = _OPENSSL_MR_RE.match(line.strip()) + # Strip any leading "filename:" prefix that grep -h omits but grep adds + # when given multiple files (e.g. "openssl_kem_mr.txt:+R15:...") + stripped = re.sub(r"^[^:+]+:", "", line.strip(), count=1) + # Only strip if what's left starts with +R (otherwise we'd strip algo names) + line = stripped if stripped.startswith("+R") else line.strip() + m = _OPENSSL_MR_RE.match(line) if not m: continue rtype = m.group(1) From 0879d0c0d382ec856f7c25900f256b5cebd5ad5c Mon Sep 17 00:00:00 2001 From: Mark Atwood Date: Thu, 7 May 2026 15:03:46 -0700 Subject: [PATCH 09/10] fix: normalise before PQC check in liboqs parser SLH_DSA_PURE_SHA2_128S doesn't match _PQC_ALGO_RE (^SLH-DSA) because the raw liboqs name uses underscores not hyphens. Normalise the candidate first, then test the normalised name. --- wolfcrypt/benchmark/pqc_parse.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/wolfcrypt/benchmark/pqc_parse.py b/wolfcrypt/benchmark/pqc_parse.py index 0981df57ffd..7445a244a99 100755 --- a/wolfcrypt/benchmark/pqc_parse.py +++ b/wolfcrypt/benchmark/pqc_parse.py @@ -355,9 +355,11 @@ def parse_liboqs(text: str, library: str) -> list[dict]: m_alg = _LIBOQS_ALG_RE.match(line) if m_alg and not _LIBOQS_DATA_RE.match(line): candidate = m_alg.group(1).strip() - # Only treat as an algorithm header if it looks like a PQC name - if _PQC_ALGO_RE.match(candidate): - current_alg = normalise_algorithm(candidate) + # Normalise first (converts SLH_DSA_PURE_SHA2_128S -> SLH-DSA-SHA2-128s), + # then check if the normalised name is a PQC algorithm we care about. + norm = normalise_algorithm(candidate) + if _PQC_ALGO_RE.match(norm): + current_alg = norm continue # Data row From fd67b8d8ea85840e2e065e55f4d69cef055dd4a4 Mon Sep 17 00:00:00 2001 From: Mark Atwood Date: Thu, 7 May 2026 16:38:24 -0700 Subject: [PATCH 10/10] feat: add SLH-DSA-SHA2 to bench driver and parser - pqc_bench.sh: enable-slhdsa=yes,sha2 in configure flags; add six SHA2 run_bench calls alongside the existing SHAKE set - pqc_parse.py: add _WOLFSSL_SLHDSA_SHA2_RE pattern to normalise wolfSSL SLH-DSA-SHA2-[SF] names to canonical SLH-DSA-SHA2-{bits}{sf} --- wolfcrypt/benchmark/pqc_bench.sh | 12 ++++++++++-- wolfcrypt/benchmark/pqc_parse.py | 13 +++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/wolfcrypt/benchmark/pqc_bench.sh b/wolfcrypt/benchmark/pqc_bench.sh index 439e6cf9fdf..d19ba1e2d63 100755 --- a/wolfcrypt/benchmark/pqc_bench.sh +++ b/wolfcrypt/benchmark/pqc_bench.sh @@ -169,7 +169,7 @@ if [ "$SKIP_BUILD" -eq 0 ]; then ./configure \ --enable-mlkem \ --enable-dilithium \ - --enable-slhdsa \ + "--enable-slhdsa=yes,sha2" \ --enable-memory \ "--enable-trackmemory=verbose" \ "--enable-stacksize=verbose" \ @@ -247,7 +247,7 @@ run_bench "ML-KEM-1024" -kyber1024 # '-ml-dsa' benchmarks all three security levels (44/65/87) in one pass. run_bench "ML-DSA (levels 44/65/87)" -ml-dsa -# SLH-DSA (NIST FIPS 205, formerly SPHINCS+) — all parameter sets +# SLH-DSA (NIST FIPS 205, formerly SPHINCS+) — SHAKE parameter sets run_bench "SLH-DSA-SHAKE-128s" -slhdsa-shake128s run_bench "SLH-DSA-SHAKE-128f" -slhdsa-shake128f run_bench "SLH-DSA-SHAKE-192s" -slhdsa-shake192s @@ -255,6 +255,14 @@ run_bench "SLH-DSA-SHAKE-192f" -slhdsa-shake192f run_bench "SLH-DSA-SHAKE-256s" -slhdsa-shake256s run_bench "SLH-DSA-SHAKE-256f" -slhdsa-shake256f +# SLH-DSA (NIST FIPS 205) — SHA2 parameter sets +run_bench "SLH-DSA-SHA2-128s" -slhdsa-sha2-128s +run_bench "SLH-DSA-SHA2-128f" -slhdsa-sha2-128f +run_bench "SLH-DSA-SHA2-192s" -slhdsa-sha2-192s +run_bench "SLH-DSA-SHA2-192f" -slhdsa-sha2-192f +run_bench "SLH-DSA-SHA2-256s" -slhdsa-sha2-256s +run_bench "SLH-DSA-SHA2-256f" -slhdsa-sha2-256f + echo "" # --------------------------------------------------------------------------- diff --git a/wolfcrypt/benchmark/pqc_parse.py b/wolfcrypt/benchmark/pqc_parse.py index 7445a244a99..a78db7c40fd 100755 --- a/wolfcrypt/benchmark/pqc_parse.py +++ b/wolfcrypt/benchmark/pqc_parse.py @@ -109,10 +109,11 @@ def make_record( # --------------------------------------------------------------------------- # wolfSSL emits "ML-KEM 512 " (space-separated, trailing space) -_WOLFSSL_MLKEM_RE = re.compile(r"ML-KEM\s+(\d+)\s*$", re.IGNORECASE) -_WOLFSSL_MLDSA_RE = re.compile(r"ML-DSA\s*$", re.IGNORECASE) -_WOLFSSL_SLHDSA_RE = re.compile(r"SLH-DSA-([SF])\s*$", re.IGNORECASE) -_WOLFSSL_DILITH_RE = re.compile(r"DILITHIUM\s*$", re.IGNORECASE) +_WOLFSSL_MLKEM_RE = re.compile(r"ML-KEM\s+(\d+)\s*$", re.IGNORECASE) +_WOLFSSL_MLDSA_RE = re.compile(r"ML-DSA\s*$", re.IGNORECASE) +_WOLFSSL_SLHDSA_SHA2_RE = re.compile(r"SLH-DSA-SHA2-([SF])\s*$", re.IGNORECASE) +_WOLFSSL_SLHDSA_RE = re.compile(r"SLH-DSA-([SF])\s*$", re.IGNORECASE) +_WOLFSSL_DILITH_RE = re.compile(r"DILITHIUM\s*$", re.IGNORECASE) # Canonical algorithm names already in NIST form (liboqs / OpenSSL / CIRCL) _CANONICAL_RE = re.compile( @@ -152,6 +153,10 @@ def normalise_algorithm(raw_algo: str, raw_keysize: str = "") -> str: if _WOLFSSL_MLDSA_RE.match(algo): return f"ML-DSA-{keysize}" + m = _WOLFSSL_SLHDSA_SHA2_RE.match(algo) + if m: + return f"SLH-DSA-SHA2-{keysize}{m.group(1).lower()}" + m = _WOLFSSL_SLHDSA_RE.match(algo) if m: return f"SLH-DSA-SHAKE-{keysize}{m.group(1).lower()}"