From 7909caa47c5bce9492d1e98669e3556097fb43f5 Mon Sep 17 00:00:00 2001 From: rustaceanrob Date: Tue, 26 May 2026 13:06:25 +0100 Subject: [PATCH] contrib: Add `compare-benches` tool When running benchmarks, we are often comparing how changes we've made will impact subsets of the program. `nanobench` offers an informative table, but does not compare across commits. This is a tool to compare `nanobench` results across to commits to catch regressions early or to prove statements on performance. I have been using this locally, and decided to generalize it to share. On approach, this is a Rust binary with 0 dependencies. Rather than using the `nanobench` JSON output, which would require a JSON parser, the results are parsed directly from `stdout`. The user can pass the number of runs they would like to perform for each build, and the minimum time of these runs is selected. The default directory to build `bench_bitcoin` is in the top level `build` folder, however the user may specify a different name. The user is also able to filter on bench targets using a substring match (i.e. MemPool). Example usage: `cargo run --release -- --filter SHA256` See the `README` for more details. --- contrib/compare-benches/Cargo.lock | 7 + contrib/compare-benches/Cargo.toml | 5 + contrib/compare-benches/README.md | 67 +++++ contrib/compare-benches/src/main.rs | 420 ++++++++++++++++++++++++++++ 4 files changed, 499 insertions(+) create mode 100644 contrib/compare-benches/Cargo.lock create mode 100644 contrib/compare-benches/Cargo.toml create mode 100644 contrib/compare-benches/README.md create mode 100644 contrib/compare-benches/src/main.rs diff --git a/contrib/compare-benches/Cargo.lock b/contrib/compare-benches/Cargo.lock new file mode 100644 index 000000000000..46654ac5ab3c --- /dev/null +++ b/contrib/compare-benches/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "compare-benches" +version = "0.1.0" diff --git a/contrib/compare-benches/Cargo.toml b/contrib/compare-benches/Cargo.toml new file mode 100644 index 000000000000..f25eb5035023 --- /dev/null +++ b/contrib/compare-benches/Cargo.toml @@ -0,0 +1,5 @@ +[package] +name = "compare-benches" +version = "0.1.0" +edition = "2021" +description = "Compare bench_bitcoin results between master and HEAD" diff --git a/contrib/compare-benches/README.md b/contrib/compare-benches/README.md new file mode 100644 index 000000000000..f128c192ffb6 --- /dev/null +++ b/contrib/compare-benches/README.md @@ -0,0 +1,67 @@ +# compare-benches + +Builds `bench` at a base git ref and at `HEAD`, runs the benchmarks, +and prints a side-by-side delta table. + +## Usage + +``` +cargo run --release -- [OPTIONS] +``` + +### Options + +| Flag | Description | Default | +|------|-------------|---------| +| `--filter ` | Substring match against benchmark names | all benchmarks | +| `--base ` | Base git ref (branch, tag, or SHA) | `master` | +| `--build ` | Build directory, relative to the repo root | `build` | +| `--runs ` | Runs per commit; keeps the minimum ns/op | `3` | +| `-h, --help` | Print help | | + +The filter is treated as a substring, so `--filter MemPool` matches +`MemPoolAddTransactions`, `MempoolEviction`, etc. + +### Examples + +```sh +# Compare all benchmarks against master +cargo run --release +``` + +```sh +# Compare only SHA256 benchmarks against master +cargo run --release -- --filter SHA256 +``` + +```sh +# Compare mempool benchmarks against a specific commit +cargo run --release -- --filter MemPool --base origin/master +``` + +## Output + +``` + BENCHMARK COMPARISON +========================================================================================================= + base : a1b2c3d4 commit subject of base + head : e5f6a7b8 commit subject of HEAD +========================================================================================================= + benchmark | base (ns) | head (ns) | Δ time% | base err% | head err% | Δ ins | Δ cyc + ───────────────────────────────────────────────────────────────────────────────────────────────────── + SHA256_32b_STANDARD | 4.710 | 4.760 | +1.06% | 0.40% | 0.30% | +0.050 | +0.030 + SHA256_32b_AVX2 | 3.840 | 3.820 | -0.52% | 0.20% | 0.20% | N/A | N/A +``` + +- **Δ time%** — percent change in ns/op; negative is faster +- **base/head err%** — nanobench measurement noise for each run +- **Δ ins / Δ cyc** — change in instructions and CPU cycles per op (requires + hardware performance counters; shows `N/A` otherwise) +- Benchmarks flagged as unstable by nanobench are excluded from the table + +## Notes + +The tool checks out each ref directly into the working tree and uses the +shared build directory (`build` by default, overridable with `--build`). +Your working tree must be clean before running (stash or commit any changes). +The original branch is restored automatically when the run completes. diff --git a/contrib/compare-benches/src/main.rs b/contrib/compare-benches/src/main.rs new file mode 100644 index 000000000000..2f46a77c4415 --- /dev/null +++ b/contrib/compare-benches/src/main.rs @@ -0,0 +1,420 @@ +use std::collections::HashMap; +use std::env; +use std::path::{Path, PathBuf}; +use std::process::Command; + +const ROW_WIDTH: usize = 129; +const RUNS: usize = 3; +const DEFAULT_BASE: &str = "master"; +const DEFAULT_BUILD: &str = "build"; +const DEFAULT_FILTER: &str = ".*"; + +struct CommitInfo { + short: String, + subject: String, +} + +#[derive(Clone)] +struct BenchResult { + name: String, + ns_per_op: f64, + err_pct: f64, + instructions: Option, + cycles: Option, + unstable: bool, +} + +fn print_help(prog: &str) { + println!( + "Usage: {prog} [OPTIONS] + +Build bench_bitcoin at and HEAD, run benchmarks, print a delta table. + +Options: + --filter Substring filter (default: {DEFAULT_FILTER}) + --base Base git ref (default: {DEFAULT_BASE}) + --build Build directory (repo-relative) (default: {DEFAULT_BUILD}) + --runs Runs per commit; keeps min (default: {RUNS}) + -h, --help Print this help" + ); +} + +fn default_jobs() -> String { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + .to_string() +} + +fn git_output(args: &[&str]) -> Result { + let out = Command::new("git") + .args(args) + .output() + .map_err(|e| format!("git: {e}"))?; + if !out.status.success() { + return Err(String::from_utf8_lossy(&out.stderr).trim().to_owned()); + } + Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned()) +} + +fn find_repo_root() -> Result { + git_output(&["rev-parse", "--show-toplevel"]).map(PathBuf::from) +} + +fn resolve_base(base: &str) -> Result { + if let Ok(sha) = git_output(&["rev-parse", "--verify", base]) { + return Ok(sha); + } + git_output(&["rev-parse", "--verify", &format!("origin/{base}")]) +} + +fn commit_subject(sha: &str) -> String { + git_output(&["log", "-1", "--format=%s", sha]).unwrap_or_default() +} + +fn short_sha(sha: &str) -> &str { + &sha[..8] +} + +fn cmake_configure(source: &Path, build: &Path) -> Result<(), String> { + let status = Command::new("cmake") + .arg("-B") + .arg(build) + .arg("-S") + .arg(source) + .arg("-DCMAKE_BUILD_TYPE=Release") + .arg("-DBUILD_BENCH=ON") + .arg("-DENABLE_WALLET=OFF") + .arg("-DBUILD_TESTS=OFF") + .arg("-DBUILD_UTIL=OFF") + .arg("-DBUILD_TX=OFF") + .arg("-DBUILD_DAEMON=OFF") + .arg("-DBUILD_CLI=OFF") + .arg("-DBUILD_GUI=OFF") + .status() + .map_err(|e| format!("cmake configure: {e}"))?; + if !status.success() { + return Err("cmake configure failed".to_owned()); + } + Ok(()) +} + +fn cmake_build(build: &Path) -> Result<(), String> { + let status = Command::new("cmake") + .arg("--build") + .arg(build) + .arg("-j") + .arg(default_jobs()) + .arg("--target") + .arg("bench_bitcoin") + .status() + .map_err(|e| format!("cmake build: {e}"))?; + if !status.success() { + return Err("cmake build failed".to_owned()); + } + Ok(()) +} + +fn parse_bench_stdout(output: &str) -> Vec { + let mut results = Vec::new(); + let mut err_col: usize = 3; + let mut ins_col: Option = None; + let mut cyc_col: Option = None; + + for line in output.lines() { + if !line.starts_with('|') { + continue; + } + let cols: Vec<&str> = line.split('|').map(str::trim).collect(); + if line.contains("---") { + continue; + } + if line.contains(" benchmark") { + err_col = cols.iter().position(|c| *c == "err%").unwrap_or(3); + ins_col = cols.iter().position(|c| c.starts_with("ins/")); + cyc_col = cols.iter().position(|c| c.starts_with("cyc/")); + continue; + } + let Some(ns_per_op) = cols + .get(1) + .and_then(|s| s.replace(',', "").trim().parse::().ok()) + else { + continue; + }; + let err_pct = cols + .get(err_col) + .and_then(|s| s.trim_end_matches('%').trim().parse::().ok()) + .unwrap_or(0.0); + let instructions = + ins_col.and_then(|i| cols.get(i)?.replace(',', "").trim().parse::().ok()); + let cycles = cyc_col.and_then(|i| cols.get(i)?.replace(',', "").trim().parse::().ok()); + let raw = match cols.iter().rev().find(|c| c.contains('`')) { + Some(s) => s.trim_matches('`').trim(), + None => continue, + }; + if raw.is_empty() { + continue; + } + let unstable = raw.contains(" (Unstable"); + let name = raw.trim().trim_matches('`').trim().to_owned(); + results.push(BenchResult { + name, + ns_per_op, + err_pct, + instructions, + cycles, + unstable, + }); + } + results +} + +fn run_bench(bin: &Path, filter: &str) -> Result, String> { + let pattern = format!(".*{filter}.*"); + let out = Command::new(bin) + .arg(format!("-filter={pattern}")) + .output() + .map_err(|e| format!("bench_bitcoin: {e}"))?; + if !out.status.success() { + return Err("bench_bitcoin exited with non-zero status".to_owned()); + } + Ok(parse_bench_stdout(&String::from_utf8_lossy(&out.stdout))) +} + +fn merge_runs(runs: Vec>) -> Vec { + let mut best: HashMap = HashMap::new(); + let mut order: Vec = Vec::new(); + for r in runs.into_iter().flatten() { + let entry = best.entry(r.name.clone()).or_insert_with(|| { + order.push(r.name.clone()); + r.clone() + }); + if r.ns_per_op < entry.ns_per_op { + *entry = r; + } + } + order.into_iter().filter_map(|n| best.remove(&n)).collect() +} + +fn fmt_val(ns: f64) -> String { + if ns.abs() >= 1_000_000.0 { + format!("{:>12.3}M", ns / 1_000_000.0) + } else if ns.abs() >= 1_000.0 { + format!("{:>12.3}K", ns / 1_000.0) + } else { + format!("{:>12.3} ", ns) + } +} + +fn fmt_delta(delta: f64) -> String { + let sign = if delta >= 0.0 { '+' } else { '-' }; + let a = delta.abs(); + let inner = if a >= 1_000_000.0 { + format!("{sign}{:.3}M", a / 1_000_000.0) + } else if a >= 1_000.0 { + format!("{sign}{:.3}K", a / 1_000.0) + } else { + format!("{sign}{:.3}", a) + }; + format!("{:>12}", inner) +} + +fn print_table( + base_info: &CommitInfo, + head_info: &CommitInfo, + base_results: &[BenchResult], + head_results: &[BenchResult], +) { + println!("\n{:^width$}", "BENCHMARK COMPARISON", width = ROW_WIDTH); + println!("{}", "=".repeat(ROW_WIDTH)); + println!(" base : {} {}", base_info.short, base_info.subject); + println!(" head : {} {}", head_info.short, head_info.subject); + println!("{}", "=".repeat(ROW_WIDTH)); + println!( + " {:<36} | {:>13} | {:>13} | {:>8} | {:>6} | {:>6} | {:>12} | {:>12}", + "benchmark", "base (ns)", "head (ns)", "Δ time%", "b.err%", "h.err%", "Δ ins", "Δ cyc" + ); + println!(" {}", "─".repeat(ROW_WIDTH - 2)); + + let head_map: HashMap<&str, &BenchResult> = + head_results.iter().map(|r| (r.name.as_str(), r)).collect(); + + for base_r in base_results { + if base_r.unstable { + continue; + } + let name = if base_r.name.len() > 36 { + format!( + "{}…{}", + &base_r.name[..17], + &base_r.name[base_r.name.len() - 18..] + ) + } else { + base_r.name.clone() + }; + let Some(head_r) = head_map.get(base_r.name.as_str()) else { + continue; + }; + let pct = if base_r.ns_per_op != 0.0 { + (head_r.ns_per_op - base_r.ns_per_op) / base_r.ns_per_op * 100.0 + } else { + 0.0 + }; + let ins_delta = base_r + .instructions + .zip(head_r.instructions) + .map(|(b, h)| h - b); + let cyc_delta = base_r.cycles.zip(head_r.cycles).map(|(b, h)| h - b); + let pct_s = format!("{:>8}", format!("{pct:+.2}%")); + let b_err = format!("{:>5.1}%", base_r.err_pct); + let h_err = format!("{:>5.1}%", head_r.err_pct); + let ins_s = ins_delta.map_or_else(|| format!("{:>12}", "N/A"), fmt_delta); + let cyc_s = cyc_delta.map_or_else(|| format!("{:>12}", "N/A"), fmt_delta); + println!( + " {:<36} | {} | {} | {} | {} | {} | {} | {}", + name, + fmt_val(base_r.ns_per_op), + fmt_val(head_r.ns_per_op), + pct_s, + b_err, + h_err, + ins_s, + cyc_s, + ); + } + + println!("{}", "=".repeat(ROW_WIDTH)); +} + +fn build_and_run( + repo: &Path, + build: &Path, + label: &str, + sha: &str, + filter: &str, + runs: usize, +) -> Result, String> { + println!("\n [{label}] checking out {}", short_sha(sha)); + + git_output(&["checkout", sha])?; + + cmake_configure(repo, build)?; + cmake_build(build)?; + + let bench_bin = build.join("bin").join("bench_bitcoin"); + if !bench_bin.exists() { + return Err(format!( + "bench_bitcoin not found at {}", + bench_bin.display() + )); + } + + let all: Result, _> = (1..=runs) + .map(|i| { + println!(" [{label}] run {i}/{runs}"); + run_bench(&bench_bin, filter) + }) + .collect(); + + Ok(merge_runs(all?)) +} + +#[allow(clippy::too_many_arguments)] +fn compare( + repo: &Path, + build: &Path, + base_sha: &str, + head_sha: &str, + base_info: &CommitInfo, + head_info: &CommitInfo, + filter: &str, + runs: usize, +) -> Result<(), String> { + let base_results = build_and_run(repo, build, "base", base_sha, filter, runs)?; + let head_results = build_and_run(repo, build, "head", head_sha, filter, runs)?; + print_table(base_info, head_info, &base_results, &head_results); + Ok(()) +} + +fn run() -> Result<(), String> { + let mut argv = env::args(); + let prog = argv.next().unwrap_or_default(); + + let mut filter = DEFAULT_FILTER.to_owned(); + let mut base = DEFAULT_BASE.to_owned(); + let mut build_dir = DEFAULT_BUILD.to_owned(); + let mut runs: usize = RUNS; + + while let Some(arg) = argv.next() { + match arg.as_str() { + "--filter" => { + filter = argv.next().ok_or("--filter requires a value")?; + } + "--base" => { + base = argv.next().ok_or("--base requires a value")?; + } + "--build" => { + build_dir = argv.next().ok_or("--build requires a value")?; + } + "--runs" => { + runs = argv + .next() + .ok_or("--runs requires a value")? + .parse::() + .map_err(|_| "--runs must be a positive integer")?; + if runs == 0 { + return Err("--runs must be at least 1".to_owned()); + } + } + "-h" | "--help" => { + print_help(&prog); + std::process::exit(0); + } + other => return Err(format!("unknown argument: {other}")), + } + } + + let repo = find_repo_root()?; + + let base_sha = resolve_base(&base)?; + let head_sha = git_output(&["rev-parse", "HEAD"])?; + + if base_sha == head_sha { + eprintln!( + "warning: base and HEAD resolve to the same commit ({})", + short_sha(&base_sha) + ); + } + + let base_info = CommitInfo { + short: short_sha(&base_sha).to_owned(), + subject: commit_subject(&base_sha), + }; + let head_info = CommitInfo { + short: short_sha(&head_sha).to_owned(), + subject: commit_subject(&head_sha), + }; + + let restore = git_output(&["symbolic-ref", "--short", "HEAD"])?; + + println!(" base : {} {}", base_info.short, base_info.subject); + println!(" head : {} {}", head_info.short, head_info.subject); + println!(" filter : {filter}"); + println!(" runs : {runs}"); + + let build = repo.join(&build_dir); + let result = compare( + &repo, &build, &base_sha, &head_sha, &base_info, &head_info, &filter, runs, + ); + + let _ = git_output(&["checkout", &restore]); + + result +} + +fn main() { + if let Err(e) = run() { + eprintln!("\nerror: {e}"); + std::process::exit(1); + } +}