diff --git a/.changeset/engine-parity-bench.md b/.changeset/engine-parity-bench.md new file mode 100644 index 000000000..54a148628 --- /dev/null +++ b/.changeset/engine-parity-bench.md @@ -0,0 +1,31 @@ +--- +"@smooai/smooth": minor +--- + +Restore the `smooth-bench` crate (Aider-Polyglot slice) and add an engine-parity axis. + +The mature bench crate was deleted in the microVM/daemon rewrite. This restores +the focused Aider-Polyglot slice — the `run_aider_polyglot` single-task runner, +the curated-task sweep, the WS chat driver, scoring, and auto-approve — against +current `main` (deps now resolve to the published `smooai-smooth-operator-core` +engine plus the in-tree `smooth-cast`/`smooth-code`/`smooth-pearls`). The +SWE-bench / replay / research / cleanup / TUI-driver scorers were left out. + +New engine-parity benchmark (pearl th-4c3e2d): `smooth-bench score` runs the +curated aider-polyglot suite through each of the five smooth-operator +`LocalServer` implementations — Rust, Go, TypeScript, Python, .NET — scoring per +engine (and per model). + +- `--engine ` (repeatable, default all) and + `--model ` (repeatable, default `deepseek-v4-flash`). +- Each engine is booted the way `scripts/operator-serve.sh` does (uniform + env contract: `SMOOAI_GATEWAY_URL/KEY`, `SMOOTH_PERSONA`, `SMOOAI_MODEL`, plus + the per-engine bind var), the sweep runs against it over the canonical WS + protocol, then it's torn down before the next cell. +- Per-engine×model results carry the engine + model dimensions and emit in the + JSON-lines + summary-table format. + +The matrix runner is parameterised on an `EngineBooter` trait, so the +engine×model aggregation and the engine→boot-command mapping are unit-tested +without a live LLM or real servers. A real scoring run needs `SMOOAI_GATEWAY_KEY` +on the runner. diff --git a/.changeset/th-7232fb-bench-canonical-driver.md b/.changeset/th-7232fb-bench-canonical-driver.md new file mode 100644 index 000000000..27465670e --- /dev/null +++ b/.changeset/th-7232fb-bench-canonical-driver.md @@ -0,0 +1,37 @@ +--- +'@smooai/smooth': patch +--- + +smooth-bench: rewire the live-drive path onto the canonical smooth-operator +`LocalServer` WebSocket protocol so the engine-parity sweep actually scores a +real turn. + +The old default driver (`chat_driver`) assumed the deleted microVM "create a +pearl + dispatch a teammate" model, and the legacy `SMOOTH_BENCH_LEGACY_DIRECT` +path spoke the retired bespoke `/ws` handshake (waited for a `Connected` event +the engines never send → "Timed out waiting for Connected"). Both are gone. + +- **New `canonical_driver`** speaks the schema-driven protocol every engine + now uses (`create_conversation_session` → `immediate_response.sessionId` → + `send_message` → drain to `eventual_response`), modeled on the daemon's + `OperatorTurnDriver::drive_once`. It parses `stream_chunk` tool-result events + into the `BenchResult`'s tool-call records and does a best-effort cost scan + (the polyglot servers don't surface cost → `$0`, noted). Fully unit-tested + (message construction + event classification, no live server needed). +- **Workspace-per-task boot**: the engine is now booted *per task* with its + workspace pointed at that task's scratch `work_dir` — rust via + `SMOOTH_WORKSPACE`; go/ts/python/dotnet (no workspace env) with + `cwd = work_dir`. Go is launched from a prebuilt binary (`go run` can't + launch from a foreign cwd), the rest via an absolute path / `--project`. +- Retired `chat_driver` + the `smooth-code` headless dependency. + +Verified live: the go engine boots per task, the canonical turn runs, the test +suite executes, and a real scored table is produced (nonzero wall-times). The +rust daemon engine additionally exercises real file edits in the task workspace +(`write_file` tool result parsed, file lands in `SMOOTH_WORKSPACE`). + +Known follow-up (out of scope for the driver rewire): the polyglot `cmd/serve` +binaries call `ServeLocal` **without `WithTools(...)`**, so their agents have no +file/bash tools — they can't edit the solution. Only the rust daemon ships a +coding toolset today. Giving the polyglot engines a coding toolset (via +`WithTools` or a bench-supplied extension) is a separate smooth-operator change. diff --git a/Cargo.lock b/Cargo.lock index 5759833e6..f83153d28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4862,6 +4862,31 @@ dependencies = [ "uuid", ] +[[package]] +name = "smooai-smooth-bench" +version = "0.22.0" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.8.8", + "chrono", + "clap", + "dirs-next", + "futures-util", + "reqwest", + "serde", + "serde_json", + "smooai-smooth-cast", + "smooai-smooth-operator-core", + "smooai-smooth-pearls", + "tempfile", + "tokio", + "tokio-tungstenite 0.26.2", + "toml", + "tracing", + "uuid", +] + [[package]] name = "smooai-smooth-cast" version = "0.22.0" diff --git a/crates/smooth-bench/Cargo.toml b/crates/smooth-bench/Cargo.toml new file mode 100644 index 000000000..ec36f46aa --- /dev/null +++ b/crates/smooth-bench/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "smooai-smooth-bench" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish = false +description = "Smooth benchmark harness — runs Aider Polyglot / SWE-bench / Terminal-Bench against the current routing config. Internal tool; not shipped in the `th` binary." + +[[bin]] +name = "smooth-bench" +path = "src/main.rs" + +[lib] +name = "smooth_bench" +path = "src/lib.rs" + +[dependencies] +smooth-cast.workspace = true +smooth-operator.workspace = true +smooth-pearls = { path = "../smooth-pearls", package = "smooai-smooth-pearls" } +anyhow.workspace = true +async-trait.workspace = true +chrono.workspace = true +clap.workspace = true +dirs-next.workspace = true +futures-util.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true +tokio-tungstenite.workspace = true +toml.workspace = true +uuid.workspace = true + +[dependencies.tracing] +workspace = true + +[dev-dependencies] +async-trait.workspace = true +tempfile.workspace = true +tokio.workspace = true +# Used by auto_approve tests to stand a fake Big Smooth. +axum.workspace = true + +[lints] +workspace = true diff --git a/crates/smooth-bench/curated-tasks.toml b/crates/smooth-bench/curated-tasks.toml new file mode 100644 index 000000000..bc1af331c --- /dev/null +++ b/crates/smooth-bench/curated-tasks.toml @@ -0,0 +1,164 @@ +# Curated Aider Polyglot task list — "The Line" +# +# This is the authoritative set of tasks for the `smooth-bench score` +# subcommand. The aggregate pass-rate across these tasks is the +# single number Smoo AI publishes with every release. +# +# Selection criteria (applied when the list was first curated): +# 1. Spread across difficulty — easy (e.g. `leap`, `hamming`), +# medium (e.g. `bowling`, `grade-school`), hard (e.g. `forth`, +# `poker`, `zebra-puzzle`). +# 2. Stable, deterministic tests — no network I/O, no time-based +# assertions, no randomness that isn't seeded. +# 3. Standard-library only — avoid tasks that require specific +# third-party packages (flaky npm/pip installs, version drift). +# 4. Small, self-contained corpora — the scratch work dir has to +# fit in a microVM's default disk budget (~1 GB). +# 5. Present in the upstream aider-polyglot corpus at the time of +# curation (2026-04-23). Tasks removed upstream will cause a +# `task not found` error at run time; re-curate when that +# happens. +# +# All six languages must have EXACTLY 20 tasks. The unit test +# `curated_list_has_exactly_20_per_language` enforces this. +# +# Upstream repo: https://github.com/Aider-AI/polyglot-benchmark +# Curated: 2026-04-23 + +python = [ + "affine-cipher", + "beer-song", + "book-store", + "bottle-song", + "bowling", + "connect", + "dominoes", + "food-chain", + "grade-school", + "grep", + "hangman", + "list-ops", + "phone-number", + "pig-latin", + "poker", + "proverb", + "rest-api", + "robot-name", + "scale-generator", + "wordy", +] + +rust = [ + "accumulate", + "acronym", + "alphametics", + "book-store", + "bowling", + "decimal", + "dot-dsl", + "forth", + "gigasecond", + "grade-school", + "grep", + "luhn-from", + "macros", + "pig-latin", + "poker", + "robot-name", + "say", + "scale-generator", + "simple-cipher", + "word-count", +] + +go = [ + "alphametics", + "beer-song", + "book-store", + "bottle-song", + "bowling", + "connect", + "counter", + "dominoes", + "food-chain", + "forth", + "hexadecimal", + "markdown", + "matrix", + "octal", + "pig-latin", + "poker", + "protein-translation", + "robot-simulator", + "scale-generator", + "trinary", +] + +javascript = [ + "affine-cipher", + "alphametics", + "beer-song", + "binary", + "book-store", + "bottle-song", + "bowling", + "complex-numbers", + "connect", + "food-chain", + "forth", + "grade-school", + "grep", + "list-ops", + "phone-number", + "pig-latin", + "poker", + "rest-api", + "robot-name", + "scale-generator", +] + +java = [ + "affine-cipher", + "all-your-base", + "alphametics", + "bank-account", + "book-store", + "bottle-song", + "bowling", + "change", + "circular-buffer", + "connect", + "food-chain", + "forth", + "hangman", + "kindergarten-garden", + "phone-number", + "pig-latin", + "poker", + "rest-api", + "simple-linked-list", + "transpose", +] + +cpp = [ + "all-your-base", + "allergies", + "bank-account", + "binary-search-tree", + "circular-buffer", + "clock", + "complex-numbers", + "crypto-square", + "diamond", + "dnd-character", + "gigasecond", + "grade-school", + "kindergarten-garden", + "linked-list", + "phone-number", + "queen-attack", + "robot-name", + "space-age", + "sublist", + "zebra-puzzle", +] diff --git a/crates/smooth-bench/src/auto_approve.rs b/crates/smooth-bench/src/auto_approve.rs new file mode 100644 index 000000000..6fc1603b4 --- /dev/null +++ b/crates/smooth-bench/src/auto_approve.rs @@ -0,0 +1,264 @@ +//! Unattended-run resolver for Safehouse Narc `Ask` verdicts. +//! +//! When a scenario or `th code --headless --auto-approve ` run +//! is in flight, no human is at the TUI to pick a scope on the +//! inline approval card. This module spawns a tokio task that polls +//! `/api/access/pending` and resolves each pending request per the +//! configured mode: +//! +//! - `deny` — resolves every Ask as Deny @ scope=once. The safe +//! default for unattended runs. +//! - `once` — Approve @ scope=once (re-asks next call). +//! - `session` — Approve @ scope=session (cached for the run's +//! lifetime). +//! - `project` / `user` — Approve at that scope. Side effects on +//! wonk-allow.toml; rarely what a bench scenario wants. +//! +//! Pearl th-400773. + +use std::sync::Arc; +use std::time::Duration; + +use serde::Serialize; + +use crate::scenarios::AutoApprove; + +/// Tokio task handle returned by [`spawn_resolver`]. Drop to stop +/// the resolver — the inner loop checks `Arc::strong_count` to know +/// when to exit (same shape as the TUI's SSE subscriber). +pub struct AutoApproveHandle { + /// Shared sentinel — when the only strong count is the + /// task's own, the loop exits. + _alive: Arc<()>, + /// Handle to abort if the caller wants a hard stop instead of + /// waiting for the sentinel. + pub task: tokio::task::JoinHandle<()>, +} + +#[derive(Serialize)] +struct ResolveBody<'a> { + id: &'a str, + scope: &'a str, +} + +/// Poll `/api/access/pending` every 100ms and resolve each entry +/// per `mode`. Returns a handle; drop it (or call `.task.abort()`) +/// to stop. +/// +/// `base_url` should NOT have a trailing slash. Typical value is +/// `http://127.0.0.1:4400` (default Big Smooth bind). +#[must_use] +pub fn spawn_resolver(base_url: String, mode: AutoApprove) -> AutoApproveHandle { + let sentinel = Arc::new(()); + let sentinel_for_task = Arc::clone(&sentinel); + let task = tokio::spawn(async move { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .unwrap_or_else(|_| reqwest::Client::new()); + let pending_url = format!("{base_url}/api/access/pending"); + let (verdict_path, scope_str): (&str, &str) = match mode { + AutoApprove::Deny => ("deny", "once"), + AutoApprove::Once => ("approve", "once"), + AutoApprove::Session => ("approve", "session"), + AutoApprove::Project => ("approve", "project"), + AutoApprove::User => ("approve", "user"), + }; + let resolve_url = format!("{base_url}/api/access/{verdict_path}"); + + loop { + // Exit when nothing outside this task still holds the + // sentinel — same shape as auto_mode's strong-count + // check, lets callers stop us by dropping the handle. + if Arc::strong_count(&sentinel_for_task) <= 1 { + tracing::debug!("auto-approve resolver: handle dropped, exiting"); + return; + } + + let pending: Vec = match client.get(&pending_url).send().await { + Ok(resp) if resp.status().is_success() => resp.json().await.unwrap_or_default(), + Ok(_) | Err(_) => Vec::new(), + }; + + for entry in pending { + let Some(id) = entry.get("id").and_then(|v| v.as_str()) else { + continue; + }; + let body = ResolveBody { id, scope: scope_str }; + let send = client.post(&resolve_url).json(&body).send().await; + if let Ok(resp) = send { + tracing::info!( + id, + scope = scope_str, + verdict = verdict_path, + status = %resp.status(), + "auto-approve resolver: resolved pending request" + ); + } + } + + tokio::time::sleep(Duration::from_millis(100)).await; + } + }); + + AutoApproveHandle { _alive: sentinel, task } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used)] +mod tests { + use super::*; + use std::net::SocketAddr; + use std::sync::Mutex; + + use axum::extract::State; + use axum::routing::{get, post}; + use axum::{Json, Router}; + + #[derive(Clone, Default)] + struct FakeBs { + // Pending requests we hand out; each call to /api/access/pending + // drains them so the resolver doesn't loop forever on the same id. + pending: Arc>>, + // (verdict_path, body) tuples the resolver POSTed. + resolved: Arc>>, + } + + #[derive(Debug, Clone, serde::Deserialize, PartialEq, Eq)] + struct ResolveBodyOwned { + id: String, + scope: String, + } + + async fn pending(State(s): State) -> Json> { + let drained = s.pending.lock().unwrap().drain(..).collect(); + Json(drained) + } + + async fn approve(State(s): State, Json(body): Json) -> axum::http::StatusCode { + s.resolved.lock().unwrap().push(("approve".into(), body)); + axum::http::StatusCode::OK + } + + async fn deny(State(s): State, Json(body): Json) -> axum::http::StatusCode { + s.resolved.lock().unwrap().push(("deny".into(), body)); + axum::http::StatusCode::OK + } + + async fn spawn_fake_bs() -> (String, FakeBs) { + let state = FakeBs::default(); + let app = Router::new() + .route("/api/access/pending", get(pending)) + .route("/api/access/approve", post(approve)) + .route("/api/access/deny", post(deny)) + .with_state(state.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr: SocketAddr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + tokio::time::sleep(Duration::from_millis(20)).await; + (format!("http://{addr}"), state) + } + + fn pending_request(id: &str) -> serde_json::Value { + serde_json::json!({ + "id": id, + "bead_id": "pearl", + "operator_id": "op", + "kind": "network", + "resource": "api.example.com", + "reason": "test", + "scope_options": ["once", "session", "pearl_project", "user"], + "created_at": "2026-05-14T00:00:00Z", + }) + } + + async fn wait_for_resolve(state: &FakeBs, want_verdict: &str, want_scope: &str, timeout: Duration) -> bool { + let deadline = std::time::Instant::now() + timeout; + while std::time::Instant::now() < deadline { + let found = { + let resolved = state.resolved.lock().unwrap(); + resolved.iter().any(|(v, b)| v == want_verdict && b.scope == want_scope) + }; + if found { + return true; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + false + } + + #[tokio::test] + async fn deny_mode_resolves_pending_as_deny_once() { + let (url, bs) = spawn_fake_bs().await; + bs.pending.lock().unwrap().push(pending_request("p1")); + + let _handle = spawn_resolver(url, AutoApprove::Deny); + let ok = wait_for_resolve(&bs, "deny", "once", Duration::from_secs(2)).await; + assert!(ok, "deny mode should resolve as deny @ once"); + } + + #[tokio::test] + async fn session_mode_resolves_pending_as_approve_session() { + let (url, bs) = spawn_fake_bs().await; + bs.pending.lock().unwrap().push(pending_request("p1")); + + let _handle = spawn_resolver(url, AutoApprove::Session); + let ok = wait_for_resolve(&bs, "approve", "session", Duration::from_secs(2)).await; + assert!(ok); + } + + #[tokio::test] + async fn once_mode_resolves_pending_as_approve_once() { + let (url, bs) = spawn_fake_bs().await; + bs.pending.lock().unwrap().push(pending_request("p1")); + + let _handle = spawn_resolver(url, AutoApprove::Once); + let ok = wait_for_resolve(&bs, "approve", "once", Duration::from_secs(2)).await; + assert!(ok); + } + + #[tokio::test] + async fn project_mode_resolves_pending_as_approve_project() { + let (url, bs) = spawn_fake_bs().await; + bs.pending.lock().unwrap().push(pending_request("p1")); + + let _handle = spawn_resolver(url, AutoApprove::Project); + let ok = wait_for_resolve(&bs, "approve", "project", Duration::from_secs(2)).await; + assert!(ok); + } + + #[tokio::test] + async fn dropping_handle_stops_the_loop() { + let (url, _bs) = spawn_fake_bs().await; + let handle = spawn_resolver(url, AutoApprove::Deny); + // Drop the handle; the task's Arc count drops to 1 (its + // own clone) and the next loop iteration exits. + drop(handle); + // No assertion needed — the test passes if it terminates + // cleanly. We just don't want a stuck tokio task at the + // end of the test runtime. + tokio::time::sleep(Duration::from_millis(200)).await; + } + + #[tokio::test] + async fn resolver_handles_multiple_pendings_in_one_poll() { + let (url, bs) = spawn_fake_bs().await; + bs.pending.lock().unwrap().push(pending_request("p1")); + bs.pending.lock().unwrap().push(pending_request("p2")); + bs.pending.lock().unwrap().push(pending_request("p3")); + + let _handle = spawn_resolver(url, AutoApprove::Session); + + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while std::time::Instant::now() < deadline { + let count = bs.resolved.lock().unwrap().len(); + if count == 3 { + return; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + panic!("expected 3 resolutions, got {}", bs.resolved.lock().unwrap().len()); + } +} diff --git a/crates/smooth-bench/src/canonical_driver.rs b/crates/smooth-bench/src/canonical_driver.rs new file mode 100644 index 000000000..e0b655589 --- /dev/null +++ b/crates/smooth-bench/src/canonical_driver.rs @@ -0,0 +1,315 @@ +//! Drive a benchmark task through the canonical smooth-operator +//! `LocalServer` WebSocket protocol — the schema-driven flow every +//! engine (rust/go/ts/python/dotnet) speaks. +//! +//! This replaces the retired `chat_driver` (which assumed the deleted +//! microVM "create a pearl + dispatch a teammate" model) and the even +//! older bespoke `/ws` handshake in `smooth_code::client`. The flow is +//! a straight copy of the daemon's own `OperatorTurnDriver::drive_once` +//! (`crates/smooth-daemon/src/scheduler.rs`): +//! +//! 1. connect `ws://host:port/ws?token=` (token omitted +//! for the anonymous polyglot servers; required for the rust daemon). +//! 2. send `create_conversation_session` → read until an +//! `immediate_response` carrying `data.sessionId`. +//! 3. send `send_message` with the task prompt. +//! 4. drain events until `eventual_response` (turn complete) or `error`, +//! collecting tool-call outcomes from `stream_chunk` events along the +//! way. +//! +//! Cost: the polyglot servers do not surface per-turn LLM cost/usage in +//! their protocol events, so `cost` is best-effort — we scan the terminal +//! `eventual_response` for a `cost`/`costUsd`/`totalCost` number and +//! otherwise leave it `0.0`. The bench's headline number is pass-rate; +//! cost is informational and will read `$0` for engines that don't emit it. + +use std::time::Duration; + +use anyhow::Result; +use futures_util::{SinkExt, StreamExt}; +use serde_json::{json, Value}; +use tokio_tungstenite::tungstenite::Message; + +/// What one canonical turn surfaced back to the bench. +#[derive(Debug, Clone, Default)] +pub struct CanonicalOutput { + /// Best-effort LLM spend for the turn. `0.0` when the engine doesn't + /// emit cost (all polyglot servers today). + pub cost: f64, + /// One record per tool *result* observed on the stream (success is + /// the negation of the engine's `isError` flag). + pub tool_calls: Vec, +} + +/// A single tool result observed during the turn. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CanonicalToolCall { + pub name: String, + pub success: bool, +} + +/// Run one canonical turn end-to-end against `url`, bounded by `deadline`. +/// +/// # Errors +/// Errors on WS connect failure, a server `error` event, or if the turn +/// doesn't complete within `deadline`. +pub async fn run_via_canonical(url: &str, prompt: &str, token: Option<&str>, deadline: Duration) -> Result { + tokio::time::timeout(deadline, drive_once(url, prompt, token)) + .await + .map_err(|_| anyhow::anyhow!("canonical turn timed out after {deadline:?}"))? +} + +/// One connect → create-session → send → drain cycle. +async fn drive_once(url: &str, prompt: &str, token: Option<&str>) -> Result { + let ws_url = ws_url(url, token); + let (stream, _) = tokio_tungstenite::connect_async(&ws_url) + .await + .map_err(|e| anyhow::anyhow!("operator WS connect failed ({ws_url}): {e}"))?; + let (mut sink, mut source) = stream.split(); + + // 1. Open a session. + sink.send(Message::Text(create_session_msg(&uuid::Uuid::new_v4().to_string()).into())).await?; + let mut session_id = None; + while let Some(Ok(msg)) = source.next().await { + let Message::Text(text) = msg else { continue }; + let Ok(v) = serde_json::from_str::(&text) else { continue }; + match classify(&v) { + Event::SessionCreated(sid) => { + session_id = Some(sid); + break; + } + Event::Error(m) => anyhow::bail!("operator error creating session: {m}"), + _ => {} + } + } + let sid = session_id.ok_or_else(|| anyhow::anyhow!("operator closed before a session was created"))?; + + // 2. Fire the task prompt. + sink.send(Message::Text(send_message_msg(&sid, prompt).into())).await?; + + // 3. Drain until the turn completes (or errors), collecting tool + // results + a best-effort cost off the terminal event. + let mut out = CanonicalOutput::default(); + while let Some(Ok(msg)) = source.next().await { + let text = match msg { + Message::Text(t) => t, + Message::Close(_) => break, + _ => continue, + }; + let Ok(v) = serde_json::from_str::(&text) else { continue }; + match classify(&v) { + Event::TurnComplete => { + if let Some(c) = find_cost(&v) { + out.cost = c; + } + break; + } + Event::Error(m) => anyhow::bail!("operator error on turn: {m}"), + Event::ToolResult { name, success } => out.tool_calls.push(CanonicalToolCall { name, success }), + _ => {} + } + } + let _ = sink.send(Message::Close(None)).await; + Ok(out) +} + +/// Classification of one inbound server event. +enum Event { + /// `immediate_response` carrying a `data.sessionId`. + SessionCreated(String), + /// `eventual_response` — the turn is done. + TurnComplete, + /// `error` — carries the human-readable message. + Error(String), + /// A `stream_chunk` carrying a tool result. + ToolResult { name: String, success: bool }, + /// Anything else (pongs, stream tokens, tool-call chunks, acks). + Other, +} + +/// Classify a parsed inbound frame. Pure — the whole reason the driver is +/// unit-testable without a live server. +fn classify(v: &Value) -> Event { + match v.get("type").and_then(Value::as_str) { + Some("immediate_response") => match v.pointer("/data/sessionId").and_then(Value::as_str) { + Some(sid) => Event::SessionCreated(sid.to_string()), + None => Event::Other, // send_message processing ack — no session id + }, + Some("eventual_response") => Event::TurnComplete, + Some("error") => Event::Error( + v.pointer("/data/message") + .and_then(Value::as_str) + .or_else(|| v.get("message").and_then(Value::as_str)) + .unwrap_or("unknown operator error") + .to_string(), + ), + Some("stream_chunk") => match v.pointer("/data/state/rawResponse/toolResult") { + Some(tr) => Event::ToolResult { + name: tr.get("name").and_then(Value::as_str).unwrap_or("unknown").to_string(), + success: !tr.get("isError").and_then(Value::as_bool).unwrap_or(false), + }, + None => Event::Other, + }, + _ => Event::Other, + } +} + +/// Build the `create_conversation_session` frame. +fn create_session_msg(agent_id: &str) -> String { + json!({ + "action": "create_conversation_session", + "requestId": "bench-cs", + "agentId": agent_id, + "userName": "bench", + }) + .to_string() +} + +/// Build the `send_message` frame. +fn send_message_msg(session_id: &str, prompt: &str) -> String { + json!({ + "action": "send_message", + "requestId": "bench-turn", + "sessionId": session_id, + "message": prompt, + }) + .to_string() +} + +/// Turn a base HTTP(S) URL into the operator `/ws` URL, appending a +/// url-encoded `?token=` only when a token is supplied. +fn ws_url(base: &str, token: Option<&str>) -> String { + let ws = base.replace("https://", "wss://").replace("http://", "ws://"); + let ws = ws.trim_end_matches('/'); + match token { + Some(t) => format!("{ws}/ws?token={}", urlencode(t)), + None => format!("{ws}/ws"), + } +} + +/// Best-effort recursive scan for a cost number under a `cost` / `costUsd` +/// / `totalCost` key anywhere in the terminal event. Returns the first hit. +fn find_cost(v: &Value) -> Option { + match v { + Value::Object(map) => { + for (k, val) in map { + if matches!(k.as_str(), "cost" | "costUsd" | "cost_usd" | "totalCost" | "total_cost") { + if let Some(n) = val.as_f64() { + return Some(n); + } + } + if let Some(found) = find_cost(val) { + return Some(found); + } + } + None + } + Value::Array(arr) => arr.iter().find_map(find_cost), + _ => None, + } +} + +/// Percent-encode a token for a `?token=` query param (RFC 3986 +/// unreserved set passes through; everything else is `%XX`). +fn urlencode(s: &str) -> String { + s.bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(), + _ => format!("%{b:02X}"), + }) + .collect() +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, reason = "unwrap is the idiom for test assertions")] +mod tests { + use super::*; + + #[test] + fn ws_url_omits_token_when_none_and_strips_trailing_slash() { + assert_eq!(ws_url("http://127.0.0.1:8792", None), "ws://127.0.0.1:8792/ws"); + assert_eq!(ws_url("http://127.0.0.1:8792/", None), "ws://127.0.0.1:8792/ws"); + assert_eq!(ws_url("https://host", None), "wss://host/ws"); + } + + #[test] + fn ws_url_appends_url_encoded_token() { + assert_eq!(ws_url("http://h:1", Some("abc123")), "ws://h:1/ws?token=abc123"); + // Non-unreserved bytes get escaped. + assert_eq!(ws_url("http://h:1", Some("a b/c")), "ws://h:1/ws?token=a%20b%2Fc"); + } + + #[test] + fn create_session_msg_shape() { + let v: Value = serde_json::from_str(&create_session_msg("agent-uuid")).unwrap(); + assert_eq!(v["action"], "create_conversation_session"); + assert_eq!(v["agentId"], "agent-uuid"); + assert_eq!(v["userName"], "bench"); + assert_eq!(v["requestId"], "bench-cs"); + } + + #[test] + fn send_message_msg_shape() { + let v: Value = serde_json::from_str(&send_message_msg("sess-1", "do the thing")).unwrap(); + assert_eq!(v["action"], "send_message"); + assert_eq!(v["sessionId"], "sess-1"); + assert_eq!(v["message"], "do the thing"); + assert_eq!(v["requestId"], "bench-turn"); + } + + #[test] + fn classify_session_created_from_immediate_response() { + let v = json!({"type":"immediate_response","data":{"sessionId":"s-9","conversationId":"c-9"}}); + assert!(matches!(classify(&v), Event::SessionCreated(sid) if sid == "s-9")); + } + + #[test] + fn classify_immediate_response_ack_without_session_is_other() { + // The send_message processing ack is an immediate_response with no sessionId. + let v = json!({"type":"immediate_response","data":{"status":200}}); + assert!(matches!(classify(&v), Event::Other)); + } + + #[test] + fn classify_turn_complete_from_eventual_response() { + let v = json!({"type":"eventual_response","data":{"messageId":"m1"}}); + assert!(matches!(classify(&v), Event::TurnComplete)); + } + + #[test] + fn classify_error_pulls_message() { + let v = json!({"type":"error","data":{"code":"BOOM","message":"it broke"}}); + assert!(matches!(classify(&v), Event::Error(m) if m == "it broke")); + } + + #[test] + fn classify_tool_result_success_and_failure() { + let ok = json!({"type":"stream_chunk","data":{"state":{"rawResponse":{"toolResult":{"name":"write_file","isError":false,"result":"ok"}}}}}); + assert!(matches!(classify(&ok), Event::ToolResult { name, success } if name == "write_file" && success)); + + let bad = json!({"type":"stream_chunk","data":{"state":{"rawResponse":{"toolResult":{"name":"run_bash","isError":true,"result":"Error: boom"}}}}}); + assert!(matches!(classify(&bad), Event::ToolResult { name, success } if name == "run_bash" && !success)); + } + + #[test] + fn classify_tool_call_chunk_is_other() { + // A toolCall chunk (no toolResult) shouldn't be counted as a result. + let v = json!({"type":"stream_chunk","data":{"state":{"rawResponse":{"toolCall":{"name":"read_file","arguments":{}}}}}}); + assert!(matches!(classify(&v), Event::Other)); + } + + #[test] + fn classify_stream_token_is_other() { + let v = json!({"type":"stream_token","data":{"token":"hello"}}); + assert!(matches!(classify(&v), Event::Other)); + } + + #[test] + fn find_cost_scans_nested_keys() { + let v = json!({"type":"eventual_response","data":{"response":{"usage":{"costUsd":0.0123}}}}); + assert_eq!(find_cost(&v), Some(0.0123)); + // Absent → None (the common case for polyglot servers). + let none = json!({"type":"eventual_response","data":{"messageId":"m"}}); + assert_eq!(find_cost(&none), None); + } +} diff --git a/crates/smooth-bench/src/curated.rs b/crates/smooth-bench/src/curated.rs new file mode 100644 index 000000000..11ffce2a1 --- /dev/null +++ b/crates/smooth-bench/src/curated.rs @@ -0,0 +1,293 @@ +//! Curated task list for `smooth-bench score`. +//! +//! The list lives in `curated-tasks.toml` in the crate root — edit +//! the TOML (no recompile) to change the sweep. `CuratedList::load` +//! reads and validates it; validation enforces the "exactly 20 +//! tasks per language, no duplicates" invariant. +//! +//! The TOML is embedded at build time via `include_str!` so a +//! `smooth-bench` binary run from anywhere still has a working +//! default list — edits to the source file only take effect on +//! rebuild. Operators who want to point at a different curated list +//! can use `CuratedList::from_toml_str` or +//! `CuratedList::from_toml_path` (future: `--tasks-from ` CLI +//! flag — out of scope for th-0465bb). + +use std::collections::BTreeMap; +use std::path::Path; + +use anyhow::{anyhow, Context}; +use serde::Deserialize; + +use crate::PolyglotLang; + +/// Required number of tasks per language. Anchored in a const so the +/// validation message and the curation target stay in sync. +pub const TASKS_PER_LANGUAGE: usize = 20; + +/// Raw TOML shape — one array per language, keyed by the lowercase +/// language name. Unknown keys are rejected so a typo in the TOML +/// (e.g. `javasript`) fails loud instead of silently producing a +/// 5-language sweep. +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +struct CuratedToml { + python: Vec, + rust: Vec, + go: Vec, + javascript: Vec, + java: Vec, + cpp: Vec, +} + +/// Parsed + validated curated task list. Indexed by `PolyglotLang`. +#[derive(Debug, Clone)] +pub struct CuratedList { + tasks: BTreeMap>, +} + +impl CuratedList { + /// The default list baked into the binary at build time. Falls + /// back to this when no path override is supplied. + /// + /// # Errors + /// Returns an error if the embedded TOML fails to parse or + /// validate. In practice this only fires when an engineer edits + /// `curated-tasks.toml` into an invalid state — the unit tests + /// catch that before merge. + pub fn default_embedded() -> anyhow::Result { + Self::from_toml_str(include_str!("../curated-tasks.toml")) + } + + /// Parse + validate a TOML string. Validation rules: + /// - Every language MUST have exactly `TASKS_PER_LANGUAGE` entries. + /// - No duplicates within a language. + /// - No empty task names. + /// + /// Duplicates across languages are allowed (e.g. `bowling` is + /// fine in both python and rust; the same exercise exists + /// independently in each language's corpus). + /// + /// # Errors + /// Returns an error if the TOML doesn't parse or any validation + /// rule fails. Error messages name the offending language so the + /// curator can find the line quickly. + pub fn from_toml_str(s: &str) -> anyhow::Result { + let raw: CuratedToml = toml::from_str(s).context("parsing curated tasks TOML")?; + let by_lang = [ + (PolyglotLang::Python, raw.python), + (PolyglotLang::Rust, raw.rust), + (PolyglotLang::Go, raw.go), + (PolyglotLang::Javascript, raw.javascript), + (PolyglotLang::Java, raw.java), + (PolyglotLang::Cpp, raw.cpp), + ]; + + let mut tasks = BTreeMap::new(); + for (lang, list) in by_lang { + validate_language_list(lang, &list)?; + tasks.insert(lang, list); + } + + Ok(Self { tasks }) + } + + /// Read a curated list from disk. Thin wrapper around + /// `from_toml_str` — kept separate so future `--tasks-from + /// ` CLI wiring has one call site. + /// + /// # Errors + /// Returns an error if the file can't be read or fails validation. + pub fn from_toml_path(path: &Path) -> anyhow::Result { + let body = std::fs::read_to_string(path).with_context(|| format!("reading curated tasks file {}", path.display()))?; + Self::from_toml_str(&body) + } + + /// Tasks for a language. Returns the slice directly so callers + /// can iterate without cloning. + #[must_use] + pub fn tasks_for(&self, lang: PolyglotLang) -> &[String] { + self.tasks.get(&lang).map_or(&[], Vec::as_slice) + } + + /// Every `(language, task)` pair in a stable ordering (language + /// alphabetical, task as curated). This is the iteration order + /// for a `--release` run — stable ordering means two runs on the + /// same code produce byte-identical JSON except for timings. + pub fn iter_all(&self) -> impl Iterator { + self.tasks.iter().flat_map(|(lang, tasks)| tasks.iter().map(move |t| (*lang, t.as_str()))) + } + + /// Total number of `(lang, task)` pairs. Convenience for + /// progress display. + #[must_use] + pub fn total(&self) -> usize { + self.tasks.values().map(Vec::len).sum() + } +} + +fn validate_language_list(lang: PolyglotLang, list: &[String]) -> anyhow::Result<()> { + let name = lang.dataset_dir(); + if list.len() != TASKS_PER_LANGUAGE { + return Err(anyhow!( + "curated list for `{name}` has {} tasks, expected exactly {TASKS_PER_LANGUAGE}", + list.len() + )); + } + let mut seen = std::collections::HashSet::new(); + for task in list { + if task.trim().is_empty() { + return Err(anyhow!("curated list for `{name}` contains an empty task name")); + } + if !seen.insert(task) { + return Err(anyhow!("curated list for `{name}` contains duplicate task `{task}`")); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn embedded_list_loads_and_validates() { + let list = CuratedList::default_embedded().expect("embedded list parses"); + for lang in [ + PolyglotLang::Python, + PolyglotLang::Rust, + PolyglotLang::Go, + PolyglotLang::Javascript, + PolyglotLang::Java, + PolyglotLang::Cpp, + ] { + assert_eq!( + list.tasks_for(lang).len(), + TASKS_PER_LANGUAGE, + "language {} should have exactly {TASKS_PER_LANGUAGE} tasks", + lang.dataset_dir() + ); + } + // 6 langs × 20 tasks = 120 pairs. + assert_eq!(list.total(), 120); + } + + #[test] + fn curated_list_has_exactly_20_per_language() { + // Explicit per-language check so test names make the + // invariant obvious when a future curator breaks it. + let list = CuratedList::default_embedded().unwrap(); + assert_eq!(list.tasks_for(PolyglotLang::Python).len(), 20); + assert_eq!(list.tasks_for(PolyglotLang::Rust).len(), 20); + assert_eq!(list.tasks_for(PolyglotLang::Go).len(), 20); + assert_eq!(list.tasks_for(PolyglotLang::Javascript).len(), 20); + assert_eq!(list.tasks_for(PolyglotLang::Java).len(), 20); + assert_eq!(list.tasks_for(PolyglotLang::Cpp).len(), 20); + } + + #[test] + fn curated_list_has_no_duplicates_within_a_language() { + let list = CuratedList::default_embedded().unwrap(); + for lang in [ + PolyglotLang::Python, + PolyglotLang::Rust, + PolyglotLang::Go, + PolyglotLang::Javascript, + PolyglotLang::Java, + PolyglotLang::Cpp, + ] { + let tasks = list.tasks_for(lang); + let unique: std::collections::HashSet<&String> = tasks.iter().collect(); + assert_eq!(tasks.len(), unique.len(), "language {} has duplicate tasks", lang.dataset_dir()); + } + } + + #[test] + fn rejects_wrong_task_count() { + let body = r#" +python = ["bowling"] +rust = ["bowling"] +go = ["bowling"] +javascript = ["bowling"] +java = ["bowling"] +cpp = ["bowling"] +"#; + let err = CuratedList::from_toml_str(body).unwrap_err(); + let msg = format!("{err:#}"); + assert!(msg.contains("expected exactly 20"), "got: {msg}"); + } + + #[test] + fn rejects_duplicate_task_within_language() { + let mut py = vec!["bowling".to_string(); 19]; + py.push("bowling".to_string()); // 20 entries, but duplicated + let body = format!( + "python = {py:?}\nrust = {r:?}\ngo = {g:?}\njavascript = {js:?}\njava = {jv:?}\ncpp = {c:?}\n", + r = twenty_of("r"), + g = twenty_of("g"), + js = twenty_of("js"), + jv = twenty_of("jv"), + c = twenty_of("c"), + ); + let err = CuratedList::from_toml_str(&body).unwrap_err(); + let msg = format!("{err:#}"); + assert!(msg.contains("duplicate task"), "got: {msg}"); + assert!(msg.contains("python"), "got: {msg}"); + } + + #[test] + fn rejects_empty_task_name() { + let mut py = twenty_of("py"); + py[3] = String::new(); + let body = format!( + "python = {py:?}\nrust = {r:?}\ngo = {g:?}\njavascript = {js:?}\njava = {jv:?}\ncpp = {c:?}\n", + r = twenty_of("r"), + g = twenty_of("g"), + js = twenty_of("js"), + jv = twenty_of("jv"), + c = twenty_of("c"), + ); + let err = CuratedList::from_toml_str(&body).unwrap_err(); + let msg = format!("{err:#}"); + assert!(msg.contains("empty task name"), "got: {msg}"); + } + + #[test] + fn rejects_unknown_top_level_key() { + // A typo like `javasript = [...]` must fail loud — otherwise + // we silently run a 5-language sweep. + let body = format!( + "python = {py:?}\nrust = {r:?}\ngo = {g:?}\njavasript = {js:?}\njava = {jv:?}\ncpp = {c:?}\n", + py = twenty_of("py"), + r = twenty_of("r"), + g = twenty_of("g"), + js = twenty_of("js"), + jv = twenty_of("jv"), + c = twenty_of("c"), + ); + let err = CuratedList::from_toml_str(&body).unwrap_err(); + let msg = format!("{err:#}"); + // `deny_unknown_fields` surfaces via toml's error message. + assert!(msg.to_lowercase().contains("unknown") || msg.contains("javasript"), "got: {msg}"); + } + + #[test] + fn iter_all_yields_every_pair_in_stable_order() { + let list = CuratedList::default_embedded().unwrap(); + let pairs: Vec<(PolyglotLang, &str)> = list.iter_all().collect(); + assert_eq!(pairs.len(), 120); + + // Stable order = language alphabetical (cpp, go, java, + // javascript, python, rust — BTreeMap + // orders by the enum's derived Ord, which follows variant + // declaration order: Python, Rust, Go, Javascript, Java, Cpp). + let first_langs: Vec<_> = pairs.iter().take(20).map(|(l, _)| *l).collect(); + let first = pairs[0].0; + assert!(first_langs.iter().all(|l| *l == first)); + } + + /// Helper: 20 distinct strings with the given prefix. + fn twenty_of(prefix: &str) -> Vec { + (0..20).map(|i| format!("{prefix}-{i}")).collect() + } +} diff --git a/crates/smooth-bench/src/engine.rs b/crates/smooth-bench/src/engine.rs new file mode 100644 index 000000000..0ece09d01 --- /dev/null +++ b/crates/smooth-bench/src/engine.rs @@ -0,0 +1,696 @@ +//! Engine-parity axis for the aider-polyglot sweep. +//! +//! There are five smooth-operator `LocalServer` implementations — +//! Rust, Go, TypeScript, Python, .NET — all speaking the same canonical +//! WebSocket protocol (the one [`crate::chat_driver`] drives). This +//! module boots each engine the way `scripts/operator-serve.sh` does +//! (pearl th-3f46fd), runs the curated aider-polyglot sweep against it, +//! tears it down, and tags the resulting [`Score`] with the engine + +//! model it was produced under. +//! +//! The matrix runner is parameterised on an [`EngineBooter`] trait so +//! the engine×model aggregation can be unit-tested with a fake booter + +//! canned [`TaskRunner`] — no live LLM, no real servers. The production +//! [`ProcessBooter`] spawns the real engine and waits for its port. + +use std::net::{SocketAddr, TcpStream}; +// Unix-only: used for `process_group` below. Windows has no equivalent and the +// bench only ever runs on unix (the engines need unix toolchains), but the crate +// must still COMPILE on windows for CI. (th-4c3e2d) +#[cfg(unix)] +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use serde::Serialize; + +use crate::curated::CuratedList; +use crate::score::Score; +use crate::sweep::{run_sweep, SweepConfig, SweepObserver, SweepRun, TaskOutcome, TaskRunner}; + +/// The five polyglot smooth-operator engine implementations. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Engine { + Rust, + Go, + Ts, + Python, + Dotnet, +} + +impl Engine { + /// All engines, in the canonical order `operator-serve.sh smoke` uses. + pub const ALL: [Self; 5] = [Self::Rust, Self::Go, Self::Ts, Self::Python, Self::Dotnet]; + + pub fn from_name(s: &str) -> Option { + match s.to_lowercase().as_str() { + "rust" | "rs" => Some(Self::Rust), + "go" | "golang" => Some(Self::Go), + "ts" | "typescript" | "node" => Some(Self::Ts), + "python" | "py" => Some(Self::Python), + "dotnet" | ".net" | "csharp" | "cs" => Some(Self::Dotnet), + _ => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Rust => "rust", + Self::Go => "go", + Self::Ts => "ts", + Self::Python => "python", + Self::Dotnet => "dotnet", + } + } + + /// Default bind port. Python's bind is hardcoded to 8787 upstream — + /// `operator-serve.sh` ignores the port argument for it — so we pin + /// the same value here. The others get distinct ports so a matrix + /// run doesn't collide when engines are booted back-to-back. + pub fn default_port(self) -> u16 { + match self { + Self::Rust => 8791, + Self::Go => 8792, + Self::Ts => 8793, + Self::Python => 8787, + Self::Dotnet => 8795, + } + } + + /// The spawn recipe for this engine's LocalServer at `port`, rooted + /// at the smooth-operator `repo`, with its workspace pointed at + /// `workspace` (the task's scratch dir). Pure data — mirrors the + /// `serve()` cases in `scripts/operator-serve.sh` so the mapping is + /// unit-testable without spawning anything. Gateway / persona / model + /// env (and the rust daemon's auth token) is layered on at spawn time + /// (see [`spawn_engine`]), not here. + /// + /// **Workspace wiring** — the agent must edit files in the task's + /// scratch dir, so the engine's file tools have to be rooted there: + /// - **rust** reads `SMOOTH_WORKSPACE`. + /// - **go / ts / python / dotnet** have no workspace env; their file + /// tools key off the process cwd, so we launch them with + /// `cwd = workspace`. Because that means the process is no longer + /// started from its own project dir, the launcher args carry an + /// **absolute** path back to the server (the go package, the ts + /// bundle, `--project` for python/dotnet) so the toolchain still + /// resolves the module while the child runs in `workspace`. + pub fn boot_command(self, repo: &Path, port: u16, workspace: &Path) -> BootCommand { + let bind = format!("127.0.0.1:{port}"); + let ws = workspace.to_path_buf(); + match self { + // The one runnable Rust LocalServer is the daemon itself; it + // confines its fs/shell tools to SMOOTH_WORKSPACE. + Self::Rust => BootCommand { + program: "th".into(), + args: vec!["daemon".into()], + cwd: None, + env: vec![("SMOOTH_ADDR".into(), bind), ("SMOOTH_WORKSPACE".into(), ws.display().to_string())], + }, + Self::Go => BootCommand { + // `go run` resolves modules from the *cwd*, not the package + // path, so it can't launch from `workspace`. Instead we run a + // binary prebuilt by `prepare_engine` (see [`go_serve_bin`]) + // with cwd = workspace so the file tools root there. + program: go_serve_bin().display().to_string(), + args: vec![], + cwd: Some(ws), + env: vec![("SMOOTH_OPERATOR_BIND".into(), bind)], + }, + Self::Ts => BootCommand { + program: "node".into(), + // node resolves imports/node_modules from the bundle's own + // dir, so an absolute path to dist/main.js runs fine with + // cwd = workspace. + args: vec![repo.join("typescript").join("server").join("dist").join("main.js").display().to_string()], + cwd: Some(ws), + env: vec![ + ("SMOOTH_OPERATOR_HOST".into(), "127.0.0.1".into()), + ("SMOOTH_OPERATOR_PORT".into(), port.to_string()), + ], + }, + Self::Python => BootCommand { + program: "uv".into(), + // --project pins the server's pyproject/venv; cwd = workspace + // roots the file tools. Bind is hardcoded 127.0.0.1:8787. + args: vec![ + "run".into(), + "--project".into(), + repo.join("python").join("server").display().to_string(), + "python".into(), + "-m".into(), + "smooth_operator_server".into(), + ], + cwd: Some(ws), + env: vec![], + }, + Self::Dotnet => BootCommand { + program: "dotnet".into(), + // --project builds/runs the host project from anywhere; cwd = + // workspace roots the file tools. + args: vec![ + "run".into(), + "--project".into(), + repo.join("dotnet").join("server").join("host").display().to_string(), + ], + cwd: Some(ws), + env: vec![("ASPNETCORE_URLS".into(), format!("http://{bind}"))], + }, + } + } +} + +/// Where `prepare_engine` builds the Go server binary. Outside both repos +/// (a stable temp path) so building it never dirties the smooth-operator +/// checkout; deterministic within a process so [`Engine::boot_command`] +/// and the unit test agree. +fn go_serve_bin() -> PathBuf { + std::env::temp_dir().join("smooth-bench-go-operator-serve") +} + +/// A fully-resolved spawn recipe. `program` + `args` run in `cwd` with +/// `env` (bind + workspace, plus the shared gateway/persona/model env and +/// the rust daemon's auth token layered on at spawn time) in the +/// environment. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BootCommand { + pub program: String, + pub args: Vec, + pub cwd: Option, + pub env: Vec<(String, String)>, +} + +/// The LLM-gateway + persona env every engine reads, per the uniform +/// contract in `operator-serve.sh`. `SMOOAI_MODEL` is passed per-boot +/// (it varies across the matrix), not here. +#[derive(Debug, Clone, Default)] +pub struct EngineEnv { + pub gateway_url: Option, + pub gateway_key: Option, + pub persona: Option, +} + +/// A booted engine: a [`TaskRunner`] pointed at it, its base URL, and a +/// teardown guard that kills the process (and its children) on drop. +pub struct BootedEngine { + pub runner: Box, + pub url: String, + // Dropped after the sweep → tears the engine down. `()` for fakes. + _guard: Box, +} + +/// Injection point for booting an engine. Production ([`ProcessBooter`]) +/// spawns the real server; tests provide a fake that returns a canned +/// runner so the matrix aggregation is exercised without a live LLM. +#[async_trait] +pub trait EngineBooter: Send + Sync { + async fn boot(&self, engine: Engine, model: &str) -> Result; +} + +/// Kills a spawned engine (and its child processes) when dropped. +struct KillOnDrop(Child); + +impl Drop for KillOnDrop { + fn drop(&mut self) { + // `go run` / `dotnet run` spawn a child that holds the socket — + // kill the whole tree, matching `operator-serve.sh`'s cleanup. + let pid = self.0.id(); + let _ = Command::new("pkill").arg("-P").arg(pid.to_string()).status(); + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +/// Production booter: spawns the real engine LocalServer and waits for +/// its port to accept TCP (a listening socket = booted; a strict-auth +/// 401 on GET still counts as listening). +pub struct ProcessBooter { + pub repo: PathBuf, + pub env: EngineEnv, + pub ready_timeout: Duration, +} + +impl ProcessBooter { + pub fn new(repo: PathBuf, env: EngineEnv) -> Self { + Self { + repo, + env, + ready_timeout: Duration::from_secs(300), + } + } +} + +#[async_trait] +impl EngineBooter for ProcessBooter { + async fn boot(&self, engine: Engine, model: &str) -> Result { + // The OS process is booted PER TASK (with its workspace pointed at + // that task's scratch dir) inside the returned runner — not here. + // `boot` just hands back a runner carrying the boot config. + Ok(BootedEngine { + runner: Box::new(EngineTaskRunner { + engine, + model: model.to_string(), + repo: self.repo.clone(), + env: self.env.clone(), + ready_timeout: self.ready_timeout, + }), + url: String::new(), + _guard: Box::new(()), + }) + } +} + +/// The production per-task runner: for each task it prepares the scratch +/// dir, boots the engine with `workspace = work_dir`, drives one canonical +/// turn, scores, and tears the engine down before the next task. +pub struct EngineTaskRunner { + pub engine: Engine, + pub model: String, + pub repo: PathBuf, + pub env: EngineEnv, + pub ready_timeout: Duration, +} + +#[async_trait] +impl TaskRunner for EngineTaskRunner { + async fn run_one(&self, lang: crate::PolyglotLang, task: &str, opts: &crate::BenchOpts) -> Result { + let setup = crate::prepare_task(lang, task)?; + let port = self.engine.default_port(); + let (url, token, _guard) = spawn_engine(self.engine, &self.model, &setup.work_dir, &self.repo, &self.env, self.ready_timeout, port)?; + let result = crate::run_prepared(lang, task, &setup, &url, token.as_deref(), opts).await?; + // `_guard` drops here → the engine (and its children) are reaped + // before the next task boots on the same port. + Ok(crate::sweep::outcome_from_result(&result)) + } +} + +/// Boot one engine LocalServer with its workspace pointed at `workspace`, +/// waiting for its port to accept TCP. Returns the base URL, the auth +/// token (the rust daemon runs strict-auth; `None` for the anonymous +/// polyglot servers), and a teardown guard that kills the process (and +/// its children) on drop. +/// +/// # Errors +/// Errors on prep failure, spawn failure, or if the port never listens +/// within `ready_timeout`. +#[allow(clippy::too_many_arguments)] +fn spawn_engine( + engine: Engine, + model: &str, + workspace: &Path, + repo: &Path, + env: &EngineEnv, + ready_timeout: Duration, + port: u16, +) -> Result<(String, Option, KillOnDrop)> { + prepare_engine(engine, repo)?; + let cmd = engine.boot_command(repo, port, workspace); + + let mut command = Command::new(&cmd.program); + command.args(&cmd.args); + if let Some(cwd) = &cmd.cwd { + command.current_dir(cwd); + } + for (k, v) in &cmd.env { + command.env(k, v); + } + command.env("SMOOAI_MODEL", model); + if let Some(u) = &env.gateway_url { + command.env("SMOOAI_GATEWAY_URL", u); + } + if let Some(k) = &env.gateway_key { + command.env("SMOOAI_GATEWAY_KEY", k); + } + if let Some(p) = &env.persona { + command.env("SMOOTH_PERSONA", p); + } + // The rust daemon runs strict-auth; hand it a known token so the driver + // can authenticate. The polyglot servers are anonymous (token = None). + let token = if engine == Engine::Rust { + let t = uuid::Uuid::new_v4().simple().to_string(); + command.env("SMOOTH_LOCAL_TOKEN", &t); + Some(t) + } else { + None + }; + // New process group so we can reap `go run` / `dotnet run` children. + // Unix-only; windows has no equivalent (see the cfg'd import above). + #[cfg(unix)] + command.process_group(0); + command.stdout(Stdio::null()).stderr(Stdio::null()); + + let child = command + .spawn() + .with_context(|| format!("spawning {} engine ({} {:?})", engine.as_str(), cmd.program, cmd.args))?; + let guard = KillOnDrop(child); + + let addr: SocketAddr = format!("127.0.0.1:{port}").parse().context("engine bind addr")?; + wait_for_port(addr, ready_timeout).with_context(|| format!("{} engine never opened {addr}", engine.as_str()))?; + + Ok((format!("http://127.0.0.1:{port}"), token, guard)) +} + +/// Best-effort pre-spawn prep the shell script does inline: the TS +/// server needs a `dist/` build, the Python server a synced venv. Keyed +/// off the engine's project dir under `repo` (independent of where the +/// server is later launched from). +fn prepare_engine(engine: Engine, repo: &Path) -> Result<()> { + match engine { + Engine::Go => { + // Build the server binary once (incremental after the first — + // Go's build cache keys off content, not cwd) so it can be + // launched from the task workspace. `go run` can't, because it + // resolves the module from cwd. + let dir = repo.join("go").join("server"); + let bin = go_serve_bin(); + run_prep(&dir, "go", &["build", "-o", &bin.display().to_string(), "./cmd/serve"])?; + } + Engine::Ts => { + let dir = repo.join("typescript").join("server"); + if !dir.join("dist").join("main.js").exists() { + run_prep(&dir, "pnpm", &["install", "--silent"])?; + run_prep(&dir, "pnpm", &["build"])?; + } + } + Engine::Python => { + run_prep(&repo.join("python").join("server"), "uv", &["sync", "--quiet"])?; + } + _ => {} + } + Ok(()) +} + +fn run_prep(cwd: &Path, program: &str, args: &[&str]) -> Result<()> { + let status = Command::new(program) + .args(args) + .current_dir(cwd) + .status() + .with_context(|| format!("running prep `{program} {}`", args.join(" ")))?; + anyhow::ensure!(status.success(), "prep `{program} {}` failed", args.join(" ")); + Ok(()) +} + +/// Poll a TCP connect until it succeeds or `timeout` elapses. +fn wait_for_port(addr: SocketAddr, timeout: Duration) -> Result<()> { + let start = Instant::now(); + while start.elapsed() < timeout { + if TcpStream::connect_timeout(&addr, Duration::from_secs(2)).is_ok() { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(500)); + } + anyhow::bail!("timed out after {timeout:?} waiting for {addr}") +} + +/// One engine×model sweep result — the [`Score`] flattened onto its +/// engine + model provenance. Serialised as one JSON-lines record. +#[derive(Debug, Clone, Serialize)] +pub struct EngineScore { + pub engine: String, + pub model: String, + #[serde(flatten)] + pub score: Score, +} + +/// The full engine×model matrix run. +#[derive(Debug, Clone, Serialize)] +pub struct EngineMatrixRun { + pub results: Vec, +} + +impl EngineMatrixRun { + /// One JSON object per line (JSON-lines) — the streaming record format. + /// + /// # Errors + /// Propagates a `serde_json` failure if a record can't be serialised. + pub fn to_jsonl(&self) -> Result { + let mut out = String::new(); + for r in &self.results { + out.push_str(&serde_json::to_string(r).context("serialising EngineScore")?); + out.push('\n'); + } + Ok(out) + } + + /// Human summary: one row per engine×model with the headline numbers. + pub fn render_summary(&self) -> String { + let mut out = String::from("engine model pass green/att $cost\n"); + for r in &self.results { + out.push_str(&format!( + "{engine:<8} {model:<20} {rate:>5.1}% {green:>3}/{att:<3} ${cost:.4}\n", + engine = r.engine, + model = r.model, + rate = r.score.overall_pass_rate * 100.0, + green = r.score.tasks_green, + att = r.score.tasks_attempted, + cost = r.score.cost_usd, + )); + } + out + } +} + +/// Run the curated aider-polyglot sweep against every `engine` × `model` +/// combination. Each cell boots the engine via `booter`, points the +/// sweep's `task_opts.big_smooth_url` (and model) at it, runs the sweep, +/// and tears the engine down before the next cell. A boot failure for +/// one cell is recorded and skipped — one dead engine doesn't abort the +/// matrix. +/// +/// # Errors +/// Propagates only if the observer or aggregation itself fails; per-cell +/// boot/transport failures are surfaced via `eprintln!` and skipped. +pub async fn run_engine_matrix( + curated: &CuratedList, + booter: &B, + engines: &[Engine], + models: &[String], + sweep_cfg: &SweepConfig, + observer: &mut O, +) -> Result +where + B: EngineBooter, + O: SweepObserver, +{ + let mut results = Vec::with_capacity(engines.len() * models.len()); + for &engine in engines { + for model in models { + let booted = match booter.boot(engine, model).await { + Ok(b) => b, + Err(e) => { + eprintln!("engine {} model {model}: boot failed, skipping: {e:#}", engine.as_str()); + continue; + } + }; + let mut cfg = sweep_cfg.clone(); + cfg.task_opts.big_smooth_url = booted.url.clone(); + cfg.task_opts.model = Some(model.clone()); + let SweepRun { score, per_task: _ } = run_sweep(curated, booted.runner.as_ref(), &cfg, observer).await?; + results.push(EngineScore { + engine: engine.as_str().to_string(), + model: model.clone(), + score, + }); + // `booted` drops here → engine torn down before the next cell. + } + } + Ok(EngineMatrixRun { results }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + use crate::sweep::SweepGate; + use crate::{BenchOpts, PolyglotLang}; + + #[test] + fn engine_from_name_and_as_str_round_trip() { + for e in Engine::ALL { + assert_eq!(Engine::from_name(e.as_str()), Some(e)); + } + assert_eq!(Engine::from_name("golang"), Some(Engine::Go)); + assert_eq!(Engine::from_name("typescript"), Some(Engine::Ts)); + assert_eq!(Engine::from_name(".net"), Some(Engine::Dotnet)); + assert_eq!(Engine::from_name("nonsense"), None); + } + + // Unix-only: asserts exact unix-style path strings (`repo.join(..)` yields + // backslashes on windows). The bench only ever RUNS on unix — the engines + // need unix toolchains — but the crate must still compile+test on windows + // for CI, so skip this mapping assertion there. (th-4c3e2d) + #[cfg(unix)] + #[test] + fn boot_command_mapping_matches_operator_serve() { + let repo = Path::new("/repo"); + let ws = Path::new("/scratch/task-work"); + + // rust: workspace via env, no cwd override (daemon confines to SMOOTH_WORKSPACE). + let rust = Engine::Rust.boot_command(repo, 8791, ws); + assert_eq!(rust.program, "th"); + assert_eq!(rust.args, ["daemon"]); + assert_eq!(rust.cwd, None); + assert_eq!( + rust.env, + [ + ("SMOOTH_ADDR".to_string(), "127.0.0.1:8791".to_string()), + ("SMOOTH_WORKSPACE".to_string(), "/scratch/task-work".to_string()), + ] + ); + + // go/ts/python/dotnet: cwd = workspace. Go runs a prebuilt binary + // (go run can't launch from a foreign cwd); the rest use an absolute + // path back to the server. + let go = Engine::Go.boot_command(repo, 8792, ws); + assert_eq!(go.program, go_serve_bin().display().to_string()); + assert!(go.args.is_empty()); + assert_eq!(go.cwd, Some(PathBuf::from("/scratch/task-work"))); + assert_eq!(go.env, [("SMOOTH_OPERATOR_BIND".to_string(), "127.0.0.1:8792".to_string())]); + + let ts = Engine::Ts.boot_command(repo, 8793, ws); + assert_eq!(ts.program, "node"); + assert_eq!(ts.args, ["/repo/typescript/server/dist/main.js"]); + assert_eq!(ts.cwd, Some(PathBuf::from("/scratch/task-work"))); + assert_eq!( + ts.env, + [ + ("SMOOTH_OPERATOR_HOST".to_string(), "127.0.0.1".to_string()), + ("SMOOTH_OPERATOR_PORT".to_string(), "8793".to_string()), + ] + ); + + let py = Engine::Python.boot_command(repo, 9999, ws); + assert_eq!(py.program, "uv"); + assert_eq!(py.args, ["run", "--project", "/repo/python/server", "python", "-m", "smooth_operator_server"]); + assert_eq!(py.cwd, Some(PathBuf::from("/scratch/task-work"))); + assert!(py.env.is_empty(), "python bind is hardcoded upstream"); + // Python's port is fixed regardless of what's requested. + assert_eq!(Engine::Python.default_port(), 8787); + + let net = Engine::Dotnet.boot_command(repo, 8795, ws); + assert_eq!(net.program, "dotnet"); + assert_eq!(net.args, ["run", "--project", "/repo/dotnet/server/host"]); + assert_eq!(net.cwd, Some(PathBuf::from("/scratch/task-work"))); + assert_eq!(net.env, [("ASPNETCORE_URLS".to_string(), "http://127.0.0.1:8795".to_string())]); + } + + #[test] + fn default_ports_are_distinct_except_pinned_python() { + let mut ports: Vec = Engine::ALL.iter().map(|e| e.default_port()).collect(); + ports.sort_unstable(); + ports.dedup(); + assert_eq!(ports.len(), 5, "each engine gets its own port"); + } + + /// A canned runner: solved/cost/duration keyed by language, so the + /// same fake yields deterministic per-engine scores without a live + /// LLM. Every call for a given language returns the same outcome. + struct CannedRunner { + solved: bool, + cost: f64, + } + + #[async_trait] + impl TaskRunner for CannedRunner { + async fn run_one(&self, _lang: PolyglotLang, _task: &str, _opts: &BenchOpts) -> Result { + Ok(TaskOutcome { + solved: self.solved, + cost_usd: self.cost, + duration_ms: 1000, + inconclusive: false, + }) + } + } + + /// Fake booter: never spawns a process. Returns a canned runner whose + /// pass/fail is a function of the engine, so the matrix has variety. + struct FakeBooter { + booted: Mutex>, + } + + #[async_trait] + impl EngineBooter for FakeBooter { + async fn boot(&self, engine: Engine, model: &str) -> Result { + self.booted.lock().unwrap().push((engine, model.to_string())); + // Rust + Go "solve"; the rest "fail" — arbitrary but deterministic. + let solved = matches!(engine, Engine::Rust | Engine::Go); + Ok(BootedEngine { + runner: Box::new(CannedRunner { solved, cost: 0.05 }), + url: format!("http://127.0.0.1:{}", engine.default_port()), + _guard: Box::new(()), + }) + } + } + + fn pr_one_per_lang() -> SweepConfig { + SweepConfig { + gate: SweepGate::Pr { tasks_per_language: 1 }, // 6 tasks per cell + budget_usd_cap: 100.0, + smooth_version: "0.0.0-test".to_string(), + commit_sha: "deadbeef".to_string(), + task_opts: BenchOpts::default(), + } + } + + #[tokio::test] + async fn matrix_covers_every_engine_and_carries_dimensions() { + let curated = CuratedList::default_embedded().unwrap(); + let booter = FakeBooter { + booted: Mutex::new(Vec::new()), + }; + let engines = [Engine::Rust, Engine::Go, Engine::Ts]; + let models = vec!["deepseek-v4-flash".to_string()]; + let cfg = pr_one_per_lang(); + let mut obs = crate::sweep::StdoutObserver; + + let run = run_engine_matrix(&curated, &booter, &engines, &models, &cfg, &mut obs).await.unwrap(); + + // One result per engine×model cell, each tagged. + assert_eq!(run.results.len(), 3); + let tags: Vec<(&str, &str)> = run.results.iter().map(|r| (r.engine.as_str(), r.model.as_str())).collect(); + assert_eq!(tags, [("rust", "deepseek-v4-flash"), ("go", "deepseek-v4-flash"), ("ts", "deepseek-v4-flash")]); + + // Rust + Go solved all 6; ts solved 0 — proves per-cell scoring. + let by_engine = |name: &str| run.results.iter().find(|r| r.engine == name).unwrap(); + assert_eq!(by_engine("rust").score.tasks_green, 6); + assert_eq!(by_engine("go").score.tasks_green, 6); + assert_eq!(by_engine("ts").score.tasks_green, 0); + assert!((by_engine("rust").score.overall_pass_rate - 1.0).abs() < 1e-9); + assert!((by_engine("ts").score.overall_pass_rate).abs() < 1e-9); + + // Every cell was actually booted. + assert_eq!(booter.booted.lock().unwrap().len(), 3); + } + + #[tokio::test] + async fn matrix_multiplies_engines_by_models() { + let curated = CuratedList::default_embedded().unwrap(); + let booter = FakeBooter { + booted: Mutex::new(Vec::new()), + }; + let engines = [Engine::Rust, Engine::Python]; + let models = vec!["deepseek-v4-flash".to_string(), "groq-gpt-oss-120b".to_string()]; + let cfg = pr_one_per_lang(); + let mut obs = crate::sweep::StdoutObserver; + + let run = run_engine_matrix(&curated, &booter, &engines, &models, &cfg, &mut obs).await.unwrap(); + + // 2 engines × 2 models = 4 cells. + assert_eq!(run.results.len(), 4); + // JSON-lines: one line per cell, each carrying engine + model. + let jsonl = run.to_jsonl().unwrap(); + assert_eq!(jsonl.lines().count(), 4); + for line in jsonl.lines() { + let v: serde_json::Value = serde_json::from_str(line).unwrap(); + assert!(v.get("engine").is_some(), "record carries engine"); + assert!(v.get("model").is_some(), "record carries model"); + assert!(v.get("overall_pass_rate").is_some(), "flattened Score present"); + } + // Summary renders one row per cell (+ header). + assert_eq!(run.render_summary().lines().count(), 5); + } +} diff --git a/crates/smooth-bench/src/lang_detect.rs b/crates/smooth-bench/src/lang_detect.rs new file mode 100644 index 000000000..35872b7e6 --- /dev/null +++ b/crates/smooth-bench/src/lang_detect.rs @@ -0,0 +1,213 @@ +//! Workspace language + test-command detection for `score-replay`. +//! +//! The existing aider-polyglot path can hard-code the language (the +//! dataset directory tells us). When we replay a real-world PR we +//! don't have that luxury — the harvested repo could be any of +//! Cargo, Pytest, Npm/Jest, or a polyglot mix (e.g. a TS monorepo +//! that ships a Python tool alongside it). This module makes a single +//! cheap pass over the workspace root and reports what it found. +//! +//! Detection is **filesystem-only** — we never execute build tools. +//! That keeps the function side-effect-free and safe to call before +//! we've decided whether the workdir is trusted. +//! +//! The picker stays here (`test_command`) so the same module owns +//! both "what is this?" and "how do I test it?" — when we add a new +//! language we touch one file. + +use std::path::{Path, PathBuf}; + +/// What we detected at the root of a workspace. +/// +/// `Mixed` means we found markers for more than one language at the +/// same root. Callers can choose to run every detected test command +/// (default) or pick the first one — see [`test_command`] for the +/// fan-out. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Workspace { + Cargo, + Pytest, + Npm, + Mixed(Vec), + Unknown, +} + +/// Inspect `root` for build-system markers and report what we find. +/// +/// Pure filesystem; no subprocesses. Looks one level deep — markers +/// nested inside a subdirectory of a polyglot repo are not surfaced +/// here. That's deliberate: the caller of `score-replay` checks out +/// the repo at the PR's base SHA and runs the test command from the +/// root, so the root-level marker is what governs the canonical test +/// command. +#[must_use] +pub fn detect(root: &Path) -> Workspace { + let mut found = Vec::new(); + if root.join("Cargo.toml").is_file() { + found.push(Workspace::Cargo); + } + if root.join("pyproject.toml").is_file() || root.join("setup.py").is_file() || root.join("pytest.ini").is_file() || root.join("tox.ini").is_file() { + found.push(Workspace::Pytest); + } + if root.join("package.json").is_file() { + found.push(Workspace::Npm); + } + if found.is_empty() { + return Workspace::Unknown; + } + if found.len() == 1 { + // Single entry — unwrap the Vec without a panicking expect. + // We just length-checked, so `into_iter().next()` always yields. + return found.into_iter().next().unwrap_or(Workspace::Unknown); + } + Workspace::Mixed(found) +} + +/// Resolve the test command(s) to run for `ws`. +/// +/// Returns a `Vec>` — each inner vector is one argv. The +/// outer vector lets `Mixed` repos return multiple commands the +/// caller runs in sequence; `Unknown` returns an empty vector so +/// callers can detect "nothing to do" without an Option dance. +/// +/// `focused_files` is the set of source files the human PR touched. +/// For most workspaces we just run the full test suite (passing +/// focused paths to e.g. `pytest`/`jest` is technically possible but +/// brittle — many repos require running the suite as a unit for +/// fixtures, conftest.py discovery, etc.). The parameter is reserved +/// for future per-file targeting (pearl follow-up); for v1 we ignore +/// it. Surfacing it in the signature now keeps the call sites stable +/// when we do plumb it through. +#[must_use] +pub fn test_command(ws: &Workspace, focused_files: &[PathBuf]) -> Vec> { + // `focused_files` is reserved for future per-file targeting; v1 + // ignores it but accepts it in the signature so the call sites + // remain stable when we plumb it through. The `_ = …` discard is + // there to keep the parameter live without tripping clippy's + // `only_used_in_recursion` (we genuinely just don't use it today). + let _ = focused_files; + match ws { + Workspace::Cargo => vec![vec!["cargo".into(), "test".into(), "--".into(), "--include-ignored".into()]], + Workspace::Pytest => vec![vec!["python3".into(), "-m".into(), "pytest".into(), "-q".into()]], + Workspace::Npm => vec![vec!["sh".into(), "-c".into(), "npm install --silent --no-audit --no-fund && npm test".into()]], + Workspace::Mixed(inner) => { + // Fan out to every detected sub-workspace, preserving + // detection order. Skips Unknown / nested Mixed (we + // only ever construct one level deep in `detect`, but + // the recursive shape keeps the function total). We + // pass an empty `focused_files` slice in the recursive + // call — recursion would otherwise be the only use, and + // clippy flags that as a smell. + let mut out = Vec::new(); + for w in inner { + out.extend(test_command(w, &[])); + } + out + } + Workspace::Unknown => Vec::new(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn cargo_root_detects_cargo() { + let dir = tempdir().unwrap(); + std::fs::write(dir.path().join("Cargo.toml"), "[package]\nname=\"x\"\nversion=\"0.1.0\"\nedition=\"2021\"\n").unwrap(); + assert_eq!(detect(dir.path()), Workspace::Cargo); + } + + #[test] + fn pyproject_detects_pytest() { + let dir = tempdir().unwrap(); + std::fs::write(dir.path().join("pyproject.toml"), "[project]\nname='x'\n").unwrap(); + assert_eq!(detect(dir.path()), Workspace::Pytest); + } + + #[test] + fn setup_py_detects_pytest() { + let dir = tempdir().unwrap(); + std::fs::write(dir.path().join("setup.py"), "from setuptools import setup\nsetup(name='x')\n").unwrap(); + assert_eq!(detect(dir.path()), Workspace::Pytest); + } + + #[test] + fn package_json_detects_npm() { + let dir = tempdir().unwrap(); + std::fs::write(dir.path().join("package.json"), "{\"name\":\"x\"}").unwrap(); + assert_eq!(detect(dir.path()), Workspace::Npm); + } + + #[test] + fn empty_root_returns_unknown() { + let dir = tempdir().unwrap(); + assert_eq!(detect(dir.path()), Workspace::Unknown); + } + + #[test] + fn polyglot_returns_mixed() { + let dir = tempdir().unwrap(); + std::fs::write(dir.path().join("Cargo.toml"), "[package]\nname=\"x\"\nversion=\"0.1.0\"\nedition=\"2021\"\n").unwrap(); + std::fs::write(dir.path().join("pyproject.toml"), "[project]\nname='x'\n").unwrap(); + let detected = detect(dir.path()); + match detected { + Workspace::Mixed(inner) => { + assert!(inner.contains(&Workspace::Cargo)); + assert!(inner.contains(&Workspace::Pytest)); + } + other => panic!("expected Mixed, got {other:?}"), + } + } + + #[test] + fn test_command_for_cargo_uses_include_ignored() { + let argv = test_command(&Workspace::Cargo, &[]); + assert_eq!(argv.len(), 1); + assert_eq!(argv[0][0], "cargo"); + assert!(argv[0].contains(&"--include-ignored".to_string())); + } + + #[test] + fn test_command_for_pytest_uses_quiet() { + let argv = test_command(&Workspace::Pytest, &[]); + assert_eq!(argv.len(), 1); + assert_eq!(argv[0][0], "python3"); + assert!(argv[0].contains(&"-q".to_string())); + } + + #[test] + fn test_command_for_npm_installs_first() { + let argv = test_command(&Workspace::Npm, &[]); + assert_eq!(argv.len(), 1); + assert!(argv[0].iter().any(|s| s.contains("npm install"))); + assert!(argv[0].iter().any(|s| s.contains("npm test"))); + } + + #[test] + fn test_command_for_unknown_is_empty() { + let argv = test_command(&Workspace::Unknown, &[]); + assert!(argv.is_empty()); + } + + #[test] + fn test_command_for_mixed_fans_out() { + let mixed = Workspace::Mixed(vec![Workspace::Cargo, Workspace::Pytest]); + let argv = test_command(&mixed, &[]); + assert_eq!(argv.len(), 2); + assert_eq!(argv[0][0], "cargo"); + assert_eq!(argv[1][0], "python3"); + } + + #[test] + fn test_command_ignores_focused_files_in_v1() { + // Documenting the current behavior: focused_files is reserved + // but not yet wired. If we change this, update the doc on + // `test_command` accordingly. + let with_files = test_command(&Workspace::Pytest, &[PathBuf::from("tests/test_foo.py")]); + let without_files = test_command(&Workspace::Pytest, &[]); + assert_eq!(with_files, without_files); + } +} diff --git a/crates/smooth-bench/src/lib.rs b/crates/smooth-bench/src/lib.rs new file mode 100644 index 000000000..17a8704fb --- /dev/null +++ b/crates/smooth-bench/src/lib.rs @@ -0,0 +1,1763 @@ +//! Smooth benchmark harness. +//! +//! Internal tool — not part of the user-facing `th` binary. Run via +//! `cargo run -p smooai-smooth-bench --` or the top-level +//! `scripts/bench.sh` wrapper. +//! +//! Phase-1 MVP: **Aider Polyglot** single-task runs. +//! +//! Flow: +//! 1. Ensure the upstream dataset is cached at `~/.smooth/bench-cache/polyglot-benchmark/`. +//! (Cloned on first use; reused after — the benchmark repo is static.) +//! 2. Copy the task's source + test + instruction files into a fresh +//! scratch run dir at `~/.smooth/bench-runs//work/`. +//! 3. Boot a smooth-operator `LocalServer` engine with its workspace +//! pointed at the scratch dir and drive one canonical-protocol turn +//! ([`canonical_driver`]), capturing tool results (+ best-effort cost). +//! 4. Run the language's test command in the scratch dir, count +//! pass/fail. +//! 5. Write `result.json` and print a one-line summary. +//! +//! Not yet wired: batch mode (`--all`), parallelism, the web scoreboard, +//! SWE-bench, Terminal-Bench. Those are separate pearls. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Instant; + +use anyhow::{anyhow, Context}; +use serde::{Deserialize, Serialize}; + +pub mod auto_approve; +pub mod canonical_driver; +pub mod curated; +pub mod engine; +pub mod lang_detect; +pub mod scenarios; +pub mod score; +pub mod sweep; + +/// Where we cache the cloned polyglot-benchmark repo. +pub fn cache_root() -> anyhow::Result { + let home = dirs_next::home_dir().ok_or_else(|| anyhow!("no home dir"))?; + Ok(home.join(".smooth").join("bench-cache")) +} + +/// Where we put per-run scratch dirs + result.json. +pub fn runs_root() -> anyhow::Result { + let home = dirs_next::home_dir().ok_or_else(|| anyhow!("no home dir"))?; + Ok(home.join(".smooth").join("bench-runs")) +} + +const POLYGLOT_REPO: &str = "https://github.com/Aider-AI/polyglot-benchmark.git"; +const POLYGLOT_DIR: &str = "polyglot-benchmark"; + +/// Supported Aider Polyglot languages. MVP handles Python + Rust; +/// the rest have known test-command shapes but aren't exercised yet. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum PolyglotLang { + Python, + Rust, + Go, + Javascript, + Java, + Cpp, +} + +impl PolyglotLang { + pub fn from_name(s: &str) -> Option { + match s.to_lowercase().as_str() { + "python" | "py" => Some(Self::Python), + "rust" | "rs" => Some(Self::Rust), + "go" => Some(Self::Go), + "javascript" | "js" | "node" => Some(Self::Javascript), + "java" => Some(Self::Java), + "cpp" | "c++" | "cplusplus" => Some(Self::Cpp), + _ => None, + } + } + + pub fn dataset_dir(self) -> &'static str { + match self { + Self::Python => "python", + Self::Rust => "rust", + Self::Go => "go", + Self::Javascript => "javascript", + Self::Java => "java", + Self::Cpp => "cpp", + } + } + + /// Shell command used to run the task's tests inside the + /// scratch work dir. Split on whitespace; first element is the + /// program. + pub fn test_command(self) -> &'static [&'static str] { + // Prefer per-test-case output over terse summaries — the + // judge can count `PASS/FAIL` lines, but gets fooled by a + // single `ok ` summary (undercounts a 31-case + // suite as 1). Keep the commands verbose enough that the + // summary names individual cases. + match self { + Self::Python => &["python3", "-m", "pytest", "-q"], + // `--include-ignored` runs tests marked `#[ignore]` — the + // polyglot Rust tasks ship with most tests `#[ignore]`d + // (bowling: 30 of 31). Without this, we'd score "solved" + // off a single trivial case and miss the real evaluation. + // `--` separates cargo flags from test-runner flags. + Self::Rust => &["cargo", "test", "--", "--include-ignored"], + // `-v` gives `--- PASS: TestName` / `--- FAIL:` per subtest, + // instead of only the terminal `ok ` line. + Self::Go => &["go", "test", "-v", "./..."], + // `npm install` first — devDependencies (jest, babel, etc.) aren't + // in the task's scratch dir by default; only the package.json is. + // `--silent --no-audit --no-fund` keep install output terse so the + // judge still has a short tail with jest's actual summary. + Self::Javascript => &["sh", "-c", "npm install --silent --no-audit --no-fund && npm test"], + // Use the task's bundled Gradle wrapper (`gradlew`) so we don't + // depend on a system-wide gradle of a specific version. + // `--no-daemon` avoids leaking a background daemon per task. + Self::Java => &["sh", "-c", "./gradlew test --no-daemon"], + // -DEXERCISM_RUN_ALL_TESTS=1 unblocks every test case past + // the first — polyglot C++ files wrap all but the first + // test in `#if defined(EXERCISM_RUN_ALL_TESTS)` gates, so + // without the define the runner reports "1 assertion in + // 1 test case" no matter what the agent did. Bench + // previously hit this as the build-step blocker AFTER + // cmake was on PATH and after the work_dir rename. + Self::Cpp => &[ + "sh", + "-c", + "mkdir -p build && cd build && cmake -DEXERCISM_RUN_ALL_TESTS=1 .. && make && ctest --output-on-failure", + ], + } + } +} + +/// Counts parsed out of a test runner's output. +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct TestCounts { + pub passed: u32, + pub failed: u32, + pub total: u32, +} + +impl TestCounts { + pub fn solved(&self) -> bool { + self.total > 0 && self.failed == 0 && self.passed == self.total + } +} + +/// Options for running a single Aider Polyglot task. +#[derive(Debug, Clone)] +pub struct BenchOpts { + pub big_smooth_url: String, + pub budget_usd: Option, + pub model: Option, +} + +impl Default for BenchOpts { + fn default() -> Self { + Self { + big_smooth_url: "http://localhost:4400".into(), + // Bench tasks need enough headroom for the workflow to + // push through plateaus — budget is the real limiter + // and should not fire on a run that was going to converge. + budget_usd: Some(5.00), + model: None, + } + } +} + +/// Final result written to `/result.json`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BenchResult { + pub run_id: String, + pub run_dir: PathBuf, + pub benchmark: String, + pub task: String, + pub lang: String, + pub timestamp: String, + pub model: Option, + pub budget_usd: Option, + pub counts: TestCounts, + pub solved: bool, + pub duration_s: f64, + pub cost_usd: f64, + pub tool_calls: Vec, + pub llm_error: Option, + pub test_stdout: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolCallRecord { + pub name: String, + pub success: bool, +} + +/// Set-up artefacts produced by `prepare_task` — shared between +/// `run_aider_polyglot` (WebSocket path) and the score-tui flow so +/// both exercise the same scratch dir, prompt, and file snapshot. +#[derive(Debug, Clone)] +pub struct TaskSetup { + pub run_dir: PathBuf, + pub work_dir: PathBuf, + pub prompt: String, + /// Snapshot of files present BEFORE the agent runs — used by + /// `strip_agent_added_tests` after the agent finishes. + pub original_files: std::collections::HashSet, +} + +/// Prepare a scratch run dir for an aider-polyglot task: clone the +/// dataset if needed, copy the task files, enable skipped tests, +/// snapshot the original file set, write `PROMPT.txt`. Returns the +/// `TaskSetup` for the caller to drive however they like. +/// +/// Extracted so the WebSocket path (`run_aider_polyglot`) and the +/// TUI path (`tui_score::run_polyglot_task_via_tui`) share identical +/// setup logic — drift here would silently shift the benchmark. +/// +/// # Errors +/// Errors when the upstream dataset can't be cloned, the task +/// directory doesn't exist, or the scratch dir can't be created. +pub fn prepare_task(lang: PolyglotLang, task: &str) -> anyhow::Result { + ensure_dataset()?; + let task_src = locate_task(lang, task)?; + let run = new_run_dir()?; + // Name the work dir after the task slug instead of a generic + // "work". C++ aider-polyglot tasks derive filenames from the + // directory name (`get_filename_component(exercise … NAME)` + // then `${file}_test.cpp` etc.), so a generic dir made CMake + // look for `work_test.cpp` which doesn't exist and every C++ + // task FAILed at the build step. Other languages don't depend + // on the directory name and aren't affected by the rename. + let work_dir = run.join(task); + std::fs::create_dir_all(&work_dir).with_context(|| format!("mkdir {}", work_dir.display()))?; + + copy_task_files(&task_src, &work_dir)?; + enable_skipped_tests(lang, &work_dir)?; + // Snapshot the original file set BEFORE the agent touches + // anything. Used after the agent finishes to delete any + // test-file-pattern files it added — the polyglot scorer runs + // the test command over the whole work dir, so agent-written + // `test_*.py` / `*_test.go` / `*.spec.ts` / `*Test.java` / etc. + // would get counted and tilt the score. Benchmark invariant: + // only the provided tests count. + let original_files = snapshot_files(&work_dir)?; + let prompt = build_prompt(task, lang, &work_dir)?; + std::fs::write(run.join("PROMPT.txt"), &prompt)?; + + Ok(TaskSetup { + run_dir: run, + work_dir, + prompt, + original_files, + }) +} + +/// Strip agent-added test files + score the work dir's test output. +/// Mirrors the post-dispatch tail of `run_aider_polyglot` so the +/// score-tui path can call it once the human-loop driver finishes. +/// +/// Returns `(test_stdout, counts)` exactly like `score_work_dir`. +/// +/// # Errors +/// Errors only when the test command can't be spawned. LLM-judge +/// failures degrade to zero counts rather than propagate. +pub async fn finalize_and_score(lang: PolyglotLang, setup: &TaskSetup) -> anyhow::Result<(String, TestCounts)> { + let stripped_files = strip_agent_added_tests(lang, &setup.work_dir, &setup.original_files)?; + if !stripped_files.is_empty() { + eprintln!( + "bench: stripped {} agent-added test file(s) before scoring: {:?}", + stripped_files.len(), + stripped_files + ); + } + score_work_dir(lang, &setup.work_dir).await +} + +/// Run one Aider Polyglot task end-to-end. +/// +/// # Errors +/// Returns an error for setup failures (dataset clone, scratch dir +/// creation, task not found). LLM-side errors are captured in the +/// `llm_error` field of the result rather than propagated, so a +/// 504-interrupted run still produces a scored result. +pub async fn run_aider_polyglot(lang: PolyglotLang, task: &str, opts: &BenchOpts) -> anyhow::Result { + let setup = prepare_task(lang, task)?; + // Drive against the engine already listening at `opts.big_smooth_url` + // (the single-task CLI path — the caller booted / is running the engine + // itself). The sweep path boots an engine per task with its workspace + // pointed at `setup.work_dir` and calls `run_prepared` directly. + run_prepared(lang, task, &setup, &opts.big_smooth_url, None, opts).await +} + +/// Read the per-turn deadline from `SMOOTH_BENCH_DEADLINE_S` (default 1800s). +#[must_use] +pub fn turn_deadline() -> std::time::Duration { + std::time::Duration::from_secs(std::env::var("SMOOTH_BENCH_DEADLINE_S").ok().and_then(|v| v.parse().ok()).unwrap_or(1800)) +} + +/// Drive one prepared task through the canonical LocalServer protocol at +/// `url`, then strip agent-added tests, run the test command, and write +/// `result.json`. Shared by the single-task CLI path and the per-task +/// engine-booting sweep runner so both score identically. +/// +/// `token` is the operator auth token (the rust daemon runs strict-auth); +/// omit for the anonymous polyglot servers. +/// +/// # Errors +/// Setup-adjacent failures (scoring, forensic writes) propagate; a driver +/// error is captured in `llm_error` so an interrupted turn still scores. +pub async fn run_prepared(lang: PolyglotLang, task: &str, setup: &TaskSetup, url: &str, token: Option<&str>, opts: &BenchOpts) -> anyhow::Result { + let run = setup.run_dir.clone(); + let prompt = setup.prompt.clone(); + + let t0 = Instant::now(); + let (cost_usd, tool_calls, llm_error) = match crate::canonical_driver::run_via_canonical(url, &prompt, token, turn_deadline()).await { + Ok(out) => ( + out.cost, + out.tool_calls + .into_iter() + .map(|t| ToolCallRecord { + name: t.name, + success: t.success, + }) + .collect(), + None, + ), + Err(e) => (0.0, Vec::new(), Some(e.to_string())), + }; + let duration_s = t0.elapsed().as_secs_f64(); + + // Delete agent-added test files and run the test command. Shared + // tail with the score-tui path so both surface identical scores. + let (test_stdout, counts) = finalize_and_score(lang, setup).await?; + + let result = BenchResult { + run_id: run.file_name().and_then(|s| s.to_str()).unwrap_or("unknown").to_string(), + run_dir: run.clone(), + benchmark: "aider-polyglot".into(), + task: task.into(), + lang: lang.dataset_dir().into(), + timestamp: chrono::Utc::now().to_rfc3339(), + model: opts.model.clone(), + budget_usd: opts.budget_usd, + counts, + solved: counts.solved(), + duration_s, + cost_usd, + tool_calls, + llm_error, + test_stdout, + }; + + let json = serde_json::to_string_pretty(&result)?; + std::fs::write(run.join("result.json"), json)?; + + Ok(result) +} + +fn ensure_dataset() -> anyhow::Result<()> { + let root = cache_root()?; + let repo = root.join(POLYGLOT_DIR); + if repo.join(".git").is_dir() { + return Ok(()); + } + std::fs::create_dir_all(&root)?; + let status = Command::new("git") + .arg("clone") + .arg("--depth=1") + .arg(POLYGLOT_REPO) + .arg(&repo) + .status() + .with_context(|| "spawning git clone")?; + if !status.success() { + return Err(anyhow!("git clone {POLYGLOT_REPO} failed")); + } + Ok(()) +} + +fn locate_task(lang: PolyglotLang, task: &str) -> anyhow::Result { + let repo = cache_root()?.join(POLYGLOT_DIR); + let candidate = repo.join(lang.dataset_dir()).join("exercises").join("practice").join(task); + if !candidate.is_dir() { + return Err(anyhow!( + "task '{task}' not found for language '{}' at {}", + lang.dataset_dir(), + candidate.display() + )); + } + Ok(candidate) +} + +fn new_run_dir() -> anyhow::Result { + let runs = runs_root()?; + std::fs::create_dir_all(&runs)?; + let id = uuid::Uuid::new_v4().simple().to_string(); + let short: String = id.chars().take(8).collect(); + let dir = runs.join(short); + std::fs::create_dir_all(&dir)?; + Ok(dir) +} + +/// Copy the task's source files into `dst`. Skips `.meta/` (which +/// contains the reference solution) so the agent never sees it. +/// Copies `.docs/instructions.md` (+ append) to `INSTRUCTIONS.md` +/// at the work dir root for easy discovery. +fn copy_task_files(src: &Path, dst: &Path) -> anyhow::Result<()> { + let mut any_source = false; + for entry in std::fs::read_dir(src)? { + let entry = entry?; + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + + if name_str == ".meta" { + continue; + } + if name_str == ".docs" { + continue; // handled below + } + if entry.file_type()?.is_dir() { + copy_dir_recursive(&entry.path(), &dst.join(&*name_str))?; + } else { + std::fs::copy(entry.path(), dst.join(&*name_str))?; + any_source = true; + } + } + if !any_source { + return Err(anyhow!("no source files found in {}", src.display())); + } + + // Concatenate .docs/instructions.md and .docs/instructions.append.md + let docs = src.join(".docs"); + let mut instructions = String::new(); + for doc in ["introduction.md", "instructions.md", "instructions.append.md"] { + let p = docs.join(doc); + if let Ok(body) = std::fs::read_to_string(&p) { + if !instructions.is_empty() { + instructions.push_str("\n\n"); + } + instructions.push_str(&body); + } + } + if !instructions.is_empty() { + std::fs::write(dst.join("INSTRUCTIONS.md"), instructions)?; + } + + Ok(()) +} + +fn copy_dir_recursive(src: &Path, dst: &Path) -> anyhow::Result<()> { + std::fs::create_dir_all(dst)?; + for entry in std::fs::read_dir(src)? { + let entry = entry?; + let name = entry.file_name(); + if entry.file_type()?.is_dir() { + copy_dir_recursive(&entry.path(), &dst.join(&name))?; + } else { + std::fs::copy(entry.path(), dst.join(&name))?; + } + } + Ok(()) +} + +/// Strip per-language "skip this test" markers so every case in the +/// upstream dataset actually runs. Aider Polyglot intentionally ships +/// most of its tests disabled (so the stub compiles); unless we flip +/// them on the bench scores only the one trivial remaining case. +/// +/// Handled here rather than as a blanket command-line flag because +/// some languages (JS) don't have a "run-skipped" flag — the markers +/// are literal in the test file and have to be rewritten. +/// +/// Safe to call on any lang: it's a no-op when the language has no +/// skip markers or the target files aren't present. +fn enable_skipped_tests(lang: PolyglotLang, work_dir: &Path) -> anyhow::Result<()> { + match lang { + PolyglotLang::Rust => { + // Nothing to do on disk — Rust runs all tests via the + // `cargo test -- --include-ignored` command-line flag. + Ok(()) + } + PolyglotLang::Javascript => { + // jest skip shapes: + // xtest( / xit( — the whole case is skipped + // test.skip( / it.skip( — same thing, alternate syntax + // Rewrite all four variants in every *.spec.js (or similar) + // file under the work dir. + rewrite_jest_skips(work_dir) + } + PolyglotLang::Java => { + // JUnit 5 / JUnit 4 skip shapes: + // @Disabled — JUnit 5 + // @Disabled("reason") — JUnit 5 with reason + // @Ignore — JUnit 4 + // @Ignore("reason") — JUnit 4 with reason + // Strip the annotations from every *.java under the test + // tree so the underlying `@Test` runs. + rewrite_junit_skips(work_dir) + } + PolyglotLang::Python | PolyglotLang::Go | PolyglotLang::Cpp => { + // No-op: polyglot Python/Go tasks don't ship skipped + // tests; C++ is future work once it's exercised. + Ok(()) + } + } +} + +/// Walk the work dir and remove every JUnit `@Disabled`/`@Ignore` +/// annotation — whether bare (`@Disabled`) or with a reason string +/// (`@Disabled("not yet implemented")`). Leaves the underlying +/// `@Test` intact so the case actually runs. +/// +/// Only touches `.java` files under `src/test/…`; leaves production +/// code alone even if it happens to include a doc-comment mentioning +/// the annotation. +fn rewrite_junit_skips(dir: &Path) -> anyhow::Result<()> { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + let name = entry.file_name().to_string_lossy().into_owned(); + if entry.file_type()?.is_dir() && !name.starts_with('.') && name != "build" && name != ".gradle" { + rewrite_junit_skips(&path)?; + continue; + } + if !name.ends_with(".java") { + continue; + } + // Only touch test files — avoid stripping a @Disabled the + // project uses legitimately in prod code. + let s = path.to_string_lossy(); + if !s.contains("/test/") && !s.ends_with("Test.java") { + continue; + } + let Ok(body) = std::fs::read_to_string(&path) else { continue }; + let rewritten = strip_junit_skip_annotations(&body); + if rewritten != body { + std::fs::write(&path, rewritten)?; + } + } + Ok(()) +} + +/// Remove JUnit skip annotation lines from a Java source. A line is +/// dropped when its only non-whitespace content is `@Disabled` or +/// `@Ignore`, optionally followed by a parenthesized argument. Keeps +/// surrounding whitespace layout tidy. +fn strip_junit_skip_annotations(body: &str) -> String { + let mut out = String::with_capacity(body.len()); + for line in body.split_inclusive('\n') { + let trimmed = line.trim_start(); + let is_skip = trimmed.starts_with("@Disabled") || trimmed.starts_with("@Ignore"); + if is_skip { + // Verify the NEXT non-blank thing is the annotation (not + // a word like `@DisabledByDefault`) — require it to end + // the identifier cleanly before `(` / whitespace / EOL. + let rest = &trimmed[1..]; // past the @ + let (name, _) = rest.split_once(|c: char| !c.is_ascii_alphanumeric()).unwrap_or((rest, "")); + if name == "Disabled" || name == "Ignore" { + continue; // skip the whole line + } + } + out.push_str(line); + } + out +} + +fn rewrite_jest_skips(dir: &Path) -> anyhow::Result<()> { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + let name = entry.file_name().to_string_lossy().into_owned(); + if entry.file_type()?.is_dir() && !name.starts_with('.') && name != "node_modules" { + rewrite_jest_skips(&path)?; + continue; + } + if !(name.ends_with(".spec.js") || name.ends_with(".test.js") || name.ends_with(".spec.ts") || name.ends_with(".test.ts")) { + continue; + } + let Ok(body) = std::fs::read_to_string(&path) else { continue }; + let rewritten = body + .replace("xtest(", "test(") + .replace("xit(", "it(") + .replace("test.skip(", "test(") + .replace("it.skip(", "it(") + .replace("describe.skip(", "describe(") + .replace("xdescribe(", "describe("); + if rewritten != body { + std::fs::write(&path, rewritten)?; + } + } + Ok(()) +} + +/// Recursively collect the set of relative paths under `root`. +/// Directories aren't included — just files — because the strip +/// step only cares about file-level additions. Skips `.git` + +/// `.smooth` (transient/metadata dirs that never appear in the +/// dataset). +fn snapshot_files(root: &Path) -> anyhow::Result> { + let mut out = std::collections::HashSet::new(); + walk(root, root, &mut out)?; + return Ok(out); + + fn walk(base: &Path, dir: &Path, out: &mut std::collections::HashSet) -> anyhow::Result<()> { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str == ".git" || name_str == ".smooth" || name_str == "node_modules" || name_str == "target" || name_str == "build" || name_str == ".gradle" + { + continue; + } + let path = entry.path(); + if entry.file_type()?.is_dir() { + walk(base, &path, out)?; + } else if let Ok(rel) = path.strip_prefix(base) { + out.insert(rel.to_path_buf()); + } + } + Ok(()) + } +} + +/// Delete any test-file-pattern files the agent added that weren't +/// in the original task snapshot. Non-test files the agent created +/// (helpers, new modules) are left alone — the polyglot scorer +/// only looks at test output, so a new module only matters if a +/// test imports it, and imports from the agent's own code are +/// fine. +/// +/// Returns the list of relative paths that got deleted so callers +/// can log them. +fn strip_agent_added_tests(lang: PolyglotLang, work_dir: &Path, original: &std::collections::HashSet) -> anyhow::Result> { + let current = snapshot_files(work_dir)?; + let mut stripped = Vec::new(); + for rel in current.difference(original) { + if is_test_file(lang, rel) { + let full = work_dir.join(rel); + if std::fs::remove_file(&full).is_ok() { + stripped.push(rel.clone()); + } + } + } + Ok(stripped) +} + +/// Public re-export of [`is_test_file`] under a more descriptive name +/// for callers in `tui_score` who use it to scope the hash-based +/// no-edit guard (pearl th-a5ca18 Bug 3) to non-test files. +#[must_use] +pub fn is_test_file_for_hash(lang: PolyglotLang, rel: &Path) -> bool { + is_test_file(lang, rel) +} + +/// Per-language test-file naming conventions. Only matches files +/// the agent ADDED; originals stay in place (they're excluded at a +/// higher level via the snapshot diff). +fn is_test_file(lang: PolyglotLang, rel: &Path) -> bool { + let name = match rel.file_name().and_then(|n| n.to_str()) { + Some(n) => n, + None => return false, + }; + let rel_str = rel.to_string_lossy(); + match lang { + PolyglotLang::Python => { + // pytest discovery patterns: `test_*.py`, `*_test.py`, and + // anything under a top-level `tests/` dir. + name.starts_with("test_") && name.ends_with(".py") || name.ends_with("_test.py") || rel_str.starts_with("tests/") && name.ends_with(".py") + } + PolyglotLang::Rust => { + // Rust integration tests live under `tests/`. Unit tests + // are inline `#[cfg(test)]` blocks — we can't strip those + // without parsing, so leave them. They're inside source + // files anyway; agent additions to lib.rs don't count as + // new files. + rel_str.starts_with("tests/") && name.ends_with(".rs") + } + PolyglotLang::Go => name.ends_with("_test.go"), + PolyglotLang::Javascript => name.ends_with(".spec.js") || name.ends_with(".test.js") || name.ends_with(".spec.ts") || name.ends_with(".test.ts"), + PolyglotLang::Java => { + // JUnit discovery: classes whose name ends in Test / + // Tests, or files under `src/test/…`. + rel_str.contains("/test/") && name.ends_with(".java") || name.ends_with("Test.java") || name.ends_with("Tests.java") + } + PolyglotLang::Cpp => { + name.ends_with("_test.cpp") || name.starts_with("test_") && name.ends_with(".cpp") || rel_str.contains("/tests/") && name.ends_with(".cpp") + } + } +} + +fn build_prompt(task: &str, lang: PolyglotLang, work_dir: &Path) -> anyhow::Result { + let files = list_non_hidden_files(work_dir)?; + let files_joined = files.join(", "); + let cmd = lang.test_command().join(" "); + + // Deliberately a SINGLE LINE (no embedded `\n`s). The bench + // drives the `smooth-code` TUI via tmux paste-buffer; that TUI's + // input handler treats every newline as Enter (submit). Without + // this flattening, a multi-line prompt arrives as N separate + // `You:` submissions — see pearl th-01c714 and the regression + // log at `~/.smooth/bench-runs/80c092b0/python-affine-cipher.pane.log` + // where a 13-line prompt produced 13 `You:` blocks. Bracketed + // paste (`paste-buffer -p`) would let a bracketed-paste-aware + // TUI keep multi-line content intact, but we do not depend on + // that — flattening guarantees one submission regardless of + // receiver behaviour. + Ok(format!( + "You are solving Aider Polyglot task `{task}` ({lang}). \ +Working directory: the current directory. \ +Files present: {files_joined}. \ +Your job: read INSTRUCTIONS.md and the EXISTING test file to understand the requirements; \ +edit the source file(s) so `{cmd}` passes every test in the existing test file; \ +do not modify, replace, or create test files (any new test file you add will be DELETED \ +before scoring — only the original test file determines pass/fail); \ +run `{cmd}` exactly (no other test command or subset) to verify; \ +stop once `{cmd}` exits successfully — do not keep iterating. \ +Constraints: use only the standard library for the language; \ +keep the implementation idiomatic and concise.", + lang = lang.dataset_dir(), + )) +} + +fn list_non_hidden_files(dir: &Path) -> anyhow::Result> { + let mut out = Vec::new(); + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with('.') { + continue; + } + out.push(name); + } + out.sort(); + Ok(out) +} + +/// Run the language's test command, then ask `smooth-judge` to +/// extract counts from its stdout. Agentic scoring — no per-language +/// regex parsers, so new languages and test runner format drift +/// don't require harness changes. +/// +/// CARGO isolation (pearl th-a5ca18 Bug 3): we forcibly point +/// `CARGO_TARGET_DIR` at a per-task `/target` so the +/// machine-wide `~/.cargo/shared-target` (set in user +/// `~/.cargo/config.toml`) can't cache a previously-compiled test +/// binary across bench runs of the same package name. Without this, +/// `cargo test` returned "ok" on un-edited workspaces because the +/// cached test binary still held a previously-edited run's compiled +/// implementation — confirmed by hand-running `cargo test` in the +/// bench-run scratch dir with vs. without `CARGO_TARGET_DIR` +/// override (un-edited: "10 passed" via shared cache; "10 failed +/// (todo!() panic)" with isolated target dir). The override has no +/// effect on non-Rust languages; setting it unconditionally is the +/// simplest defence. +async fn score_work_dir(lang: PolyglotLang, work_dir: &Path) -> anyhow::Result<(String, TestCounts)> { + let argv = lang.test_command(); + let program = argv[0]; + let args = &argv[1..]; + let isolated_target = work_dir.join("target"); + let output = Command::new(program) + .args(args) + .current_dir(work_dir) + .env("CARGO_TARGET_DIR", &isolated_target) + .output() + .with_context(|| format!("spawning `{}`", argv.join(" ")))?; + let exit_code = output.status.code(); + let mut combined = String::new(); + combined.push_str(&String::from_utf8_lossy(&output.stdout)); + combined.push_str(&String::from_utf8_lossy(&output.stderr)); + + // Pearl th-086f0f: prefer deterministic regex parse of the test + // runner's own summary line (cargo's "test result:", pytest's + // `N passed, N failed`, etc.) over the LLM judge. The judge gets + // 4 KB of trimmed output and routinely returns 0/0/0 when the + // canonical summary line is in the trimmed-out middle — which + // marks the task FAIL even when every test passed. We saw this + // on rust-acronym across all 4 models in the 4-model matrix: + // saved src/lib.rs passed 10/10 against `cargo test` on the + // host, but bench scored FAIL because the judge couldn't see + // the summary in the trimmed window. + // + // Regex first → judge fallback → forensic dump (always). + let (counts, source) = match parse_native_test_summary(lang, &combined) { + Some(c) => (c, "native_regex"), + None => (judge_test_output(&combined).await.unwrap_or_default(), "judge_llm"), + }; + + if let Err(e) = write_score_forensic(work_dir, lang, argv, exit_code, &combined, source, counts) { + // Don't fail the score on a forensic-write error; just log. + eprintln!("bench: score forensic write failed for {}: {e:#}", work_dir.display()); + } + + Ok((combined, counts)) +} + +/// Deterministic regex parse of the test-runner's own summary line. +/// Returns `None` when no summary line is found — caller falls back +/// to the LLM judge. Returning `None` is a *signal* that the output +/// doesn't have the canonical shape; `Some(passed: 0, failed: 0)` +/// would be a real "the runner found nothing" result. +#[must_use] +pub fn parse_native_test_summary(lang: PolyglotLang, combined: &str) -> Option { + match lang { + PolyglotLang::Rust => parse_cargo_summary(combined), + PolyglotLang::Python => parse_pytest_summary(combined), + PolyglotLang::Javascript => parse_jest_summary(combined), + // Go, Java, Cpp: TODO once we hit a case where the judge LLM + // miscounts them. The judge handles them OK today. + _ => None, + } +} + +/// `cargo test` prints one or more `test result: ok|FAILED. N passed; N failed; …` +/// lines (one per test binary, plus one for doc-tests). Sum across all +/// of them — a task with 1 unit test + 9 integration tests reports two +/// summary lines and we want the union. +#[must_use] +pub fn parse_cargo_summary(combined: &str) -> Option { + let mut total_passed = 0u32; + let mut total_failed = 0u32; + let mut found = false; + for line in combined.lines() { + let trimmed = line.trim(); + if !trimmed.starts_with("test result:") { + continue; + } + found = true; + if let Some(p) = extract_count_before(trimmed, " passed") { + total_passed = total_passed.saturating_add(p); + } + if let Some(f) = extract_count_before(trimmed, " failed") { + total_failed = total_failed.saturating_add(f); + } + } + if !found { + return None; + } + Some(TestCounts { + passed: total_passed, + failed: total_failed, + total: total_passed.saturating_add(total_failed), + }) +} + +/// pytest summary line. All of these are valid shapes: +/// - `===== 10 passed, 2 failed, 1 skipped in 1.23s =====` +/// - `========== 16 passed in 0.01s ==========` +/// - `16 passed in 0.01s` (bars omitted when pytest can't detect terminal +/// width — e.g. output piped to a file, which is exactly our capture path) +/// - `1 failed in 0.02s` / `1 error in 0.01s` / `no tests ran in 0.00s` +/// +/// Take the LAST recognized summary line — pytest can print interim +/// per-file summaries; the final one is the suite total. Returns +/// `Some(0, 0, 0)` for `no tests ran` (not a pass — `solved()` +/// requires `total > 0`) so we still skip the LLM judge (pearl th-19ab7c). +#[must_use] +pub fn parse_pytest_summary(combined: &str) -> Option { + let mut found = false; + let mut passed = 0u32; + let mut failed = 0u32; + for line in combined.lines() { + // Strip the optional `====` decoration so the same logic handles + // both the bar-wrapped and bare forms. + let trimmed = line.trim().trim_matches('=').trim(); + if !is_pytest_summary(trimmed) { + continue; + } + found = true; + // Each summary line is self-contained; the last one wins. + passed = extract_count_before(trimmed, " passed").unwrap_or(0); + failed = extract_count_before(trimmed, " failed").unwrap_or(0); + // pytest's "error"/"errors" counts (collection errors) as failures. + if let Some(e) = extract_count_before(trimmed, " error") { + failed = failed.saturating_add(e); + } + } + if !found { + return None; + } + Some(TestCounts { + passed, + failed, + total: passed.saturating_add(failed), + }) +} + +/// True when `line` (already stripped of `=` decoration) is a pytest +/// result summary. Discriminator: it reports at least one status keyword +/// and ends with a pytest duration (`… in …s`). The duration tail +/// is what separates a real summary from arbitrary test stdout that +/// happens to contain the word "passed". +fn is_pytest_summary(line: &str) -> bool { + const KEYWORDS: &[&str] = &["passed", "failed", "error", "skipped", "xfailed", "xpassed", "no tests ran"]; + if !KEYWORDS.iter().any(|k| line.contains(k)) { + return false; + } + // Must end in `… in ` where duration starts with a digit and + // ends with `s` (`0.01s`, `1m 2.34s`, `12s`). + matches!(line.rsplit_once(" in "), Some((_, dur)) if dur.ends_with('s') && dur.starts_with(|c: char| c.is_ascii_digit())) +} + +/// jest summary line: `Tests: 1 failed, 10 passed, 11 total`. +/// Take the last occurrence (jest can print per-file then aggregate). +#[must_use] +pub fn parse_jest_summary(combined: &str) -> Option { + let mut last: Option<(u32, u32, u32)> = None; + for line in combined.lines() { + let trimmed = line.trim(); + if !trimmed.starts_with("Tests:") { + continue; + } + let passed = extract_count_before(trimmed, " passed").unwrap_or(0); + let failed = extract_count_before(trimmed, " failed").unwrap_or(0); + let total = extract_count_before(trimmed, " total").unwrap_or_else(|| passed.saturating_add(failed)); + last = Some((passed, failed, total)); + } + last.map(|(passed, failed, total)| TestCounts { passed, failed, total }) +} + +/// Find the integer that immediately precedes `suffix` in `line`. +/// e.g. `extract_count_before("test result: ok. 10 passed; 0 failed", " passed")` +/// returns `Some(10)`. Returns `None` when the suffix isn't present or +/// no digits precede it. +fn extract_count_before(line: &str, suffix: &str) -> Option { + let idx = line.find(suffix)?; + let prefix = &line[..idx]; + // Walk back from the end of prefix collecting digits. + let bytes = prefix.as_bytes(); + let mut end = bytes.len(); + // Skip a trailing whitespace, if any (cargo prints "10 passed", + // pytest prints " 10 passed" with leading space we don't need). + while end > 0 && bytes[end - 1] == b' ' { + end -= 1; + } + let mut start = end; + while start > 0 && bytes[start - 1].is_ascii_digit() { + start -= 1; + } + if start == end { + return None; + } + std::str::from_utf8(&bytes[start..end]).ok()?.parse().ok() +} + +/// Forensic dump for `cargo test` runs: writes a JSON sidecar + +/// the raw combined stdout/stderr to `work_dir/.smooth-score-forensic/` +/// so any future FAIL is debuggable in 30 seconds instead of "where +/// did the bench score this from?". +fn write_score_forensic( + work_dir: &Path, + lang: PolyglotLang, + argv: &[&str], + exit_code: Option, + combined: &str, + source: &str, + counts: TestCounts, +) -> anyhow::Result<()> { + let dir = work_dir.join(".smooth-score-forensic"); + std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?; + + // Full raw output — the only authoritative record. Capped at 1 MiB + // so a runaway loop doesn't fill disk. + let raw = if combined.len() > 1_048_576 { + let truncated = &combined[..1_048_576]; + format!("{truncated}\n\n[... truncated, original was {} bytes ...]\n", combined.len()) + } else { + combined.to_string() + }; + std::fs::write(dir.join("combined.txt"), raw).with_context(|| format!("write {}", dir.join("combined.txt").display()))?; + + let summary = serde_json::json!({ + "timestamp": chrono::Utc::now().to_rfc3339(), + "lang": lang.dataset_dir(), + "command": argv, + "exit_code": exit_code, + "parsed_via": source, + "counts": { + "passed": counts.passed, + "failed": counts.failed, + "total": counts.total, + "solved": counts.solved(), + }, + "combined_bytes": combined.len(), + }); + std::fs::write(dir.join("summary.json"), serde_json::to_string_pretty(&summary)?) + .with_context(|| format!("write {}", dir.join("summary.json").display()))?; + Ok(()) +} + +/// Ask the `smooth-judge` slot to extract pass/fail/total counts +/// from a test runner's combined stdout+stderr. Returns the default +/// `TestCounts` on any failure — we'd rather under-report (marks the +/// run as unsolved) than fabricate success. +/// +/// # Errors +/// Returns an error only when the registry can't be loaded at all. +/// LLM failures are converted into zero counts, not propagated. +pub async fn judge_test_output(combined_stdout: &str) -> anyhow::Result { + use smooth_cast::provider_migration::load_providers_with_migration; + use smooth_operator::conversation::Message; + use smooth_operator::llm::LlmClient; + use smooth_operator::providers::Activity; + + let providers_path = dirs_next::home_dir() + .map(|h| h.join(".smooth/providers.json")) + .ok_or_else(|| anyhow!("no home dir"))?; + let registry = load_providers_with_migration(&providers_path).context("loading providers.json")?; + let config = registry.llm_config_for(Activity::Judge).context("no `judge` routing slot configured")?; + let llm = LlmClient::new(config); + + // Keep the input modest — judge doesn't need 2MB of verbose + // cargo test output, just enough to see the summary lines. Keep + // both ends since some runners print the tally at the top + // (e.g. `go test -v`) and some at the bottom (pytest, cargo). + let trimmed = trim_for_judge(combined_stdout, 4000); + + let system = Message::system( + "You extract test-result counts from the output of a test \ + runner (pytest, cargo test, go test, jest, etc.). Respond \ + with a SINGLE line of JSON only: \ + {\"passed\": N, \"failed\": N, \"total\": N}. \ + No code fences, no prose, no preamble.\n\n\ + Scoring rules:\n\ + - Prefer per-case counts when the runner prints them \ + (pytest's `N passed, N failed`, cargo's `test result: ok. \ + N passed; N failed`, go's `--- PASS:` / `--- FAIL:` lines, \ + jest's `Tests: N passed, N failed, N total`).\n\ + - When a suite-level runner only prints `ok ` or \ + `FAIL ` with no per-case breakdown (e.g. `go test \ + ./...` in non-verbose mode, or `cargo test --quiet`), treat \ + that as a single test: `ok` ⇒ passed=1 failed=0 total=1, \ + `FAIL` ⇒ passed=0 failed=1 total=1. DO NOT return all \ + zeros — the suite is definitive, the counts just aren't.\n\ + - Build/compile errors that prevent the tests from running \ + count as failed=1 total=1.\n\ + - Only return all zeros when the output is truly empty or \ + gives no signal about whether tests ran.", + ); + let user = Message::user(format!("Test runner output:\n\n{trimmed}")); + let response = llm.chat(&[&system, &user], &[]).await.context("smooth-judge call failed")?; + + Ok(parse_judge_response(&response.content)) +} + +/// Parse the judge's JSON response into `TestCounts`. Lenient: +/// strips code fences, finds the first `{...}` block, accepts +/// partial totals (infers total when the model only gives passed + +/// failed). Unit-tested without a live LLM. +#[allow(clippy::cast_possible_truncation)] +pub fn parse_judge_response(raw: &str) -> TestCounts { + let body = strip_code_fence(raw.trim()); + let Some(json_slice) = extract_first_object(body) else { + return TestCounts::default(); + }; + let Ok(value) = serde_json::from_str::(json_slice) else { + return TestCounts::default(); + }; + + let passed = value.get("passed").and_then(serde_json::Value::as_u64).unwrap_or(0) as u32; + let failed = value.get("failed").and_then(serde_json::Value::as_u64).unwrap_or(0) as u32; + let total = value + .get("total") + .and_then(serde_json::Value::as_u64) + .map_or_else(|| passed.saturating_add(failed), |n| n as u32); + + TestCounts { passed, failed, total } +} + +fn strip_code_fence(s: &str) -> &str { + let s = s.trim(); + s.strip_prefix("```json") + .or_else(|| s.strip_prefix("```")) + .map_or(s, |rest| rest.trim_end_matches("```").trim()) +} + +fn extract_first_object(s: &str) -> Option<&str> { + let start = s.find('{')?; + let end = s.rfind('}')?; + if end <= start { + return None; + } + Some(&s[start..=end]) +} + +fn trim_for_judge(s: &str, max_bytes: usize) -> String { + if s.len() <= max_bytes { + return s.to_string(); + } + // Keep the head (setup errors) and the tail (summary). Those are + // the two spots test runners print their counts. + let head_bytes = max_bytes / 3; + let tail_bytes = max_bytes - head_bytes - 64; + + // Careful with UTF-8 — step back to a char boundary. + let head_end = head_bytes.min(s.len()); + let head_end = (0..=head_end).rev().find(|&i| s.is_char_boundary(i)).unwrap_or(0); + + let tail_start_raw = s.len().saturating_sub(tail_bytes); + let tail_start = (tail_start_raw..s.len()).find(|&i| s.is_char_boundary(i)).unwrap_or(tail_start_raw); + + format!( + "{}\n\n[... {} bytes elided ...]\n\n{}", + &s[..head_end], + s.len() - head_end - (s.len() - tail_start), + &s[tail_start..] + ) +} + +// --------------------------------------------------------------------------- +// CLI entry point +// --------------------------------------------------------------------------- + +/// Pretty-print a run summary to stdout. +pub fn print_summary(r: &BenchResult) { + let status = if r.solved { "SOLVED" } else { "UNSOLVED" }; + println!(); + println!("Benchmark: aider-polyglot/{}/{}", r.lang, r.task); + println!( + "Result: {} ({}/{} passed, {} failed)", + status, r.counts.passed, r.counts.total, r.counts.failed + ); + println!("Duration: {:.1}s", r.duration_s); + println!("Cost: ${:.6}", r.cost_usd); + if let Some(m) = &r.model { + println!("Model: {m}"); + } + if let Some(err) = &r.llm_error { + println!("LLM note: {err}"); + } + println!("Results: {}", r.run_dir.join("result.json").display()); +} + +#[cfg(test)] +mod tests { + use super::*; + + // ── Pearl th-086f0f: native-summary parsers ─────────────────── + + /// Regression for the rust-acronym scoring bug: glm-5.1's saved + /// src/lib.rs passes 10/10 when run with `cargo test + /// -- --include-ignored`, but bench scored FAIL because the + /// judge LLM couldn't see the summary line in the 4 KB trim + /// window. The native parser must extract 10 passed / 0 failed + /// from this exact stdout. + #[test] + fn parse_cargo_summary_extracts_passed_failed_from_real_output() { + let real = "\nrunning 10 tests\ntest basic ... ok\ntest lowercase_words ... ok\ntest consecutive_delimiters ... ok\n\ + test punctuation ... ok\ntest punctuation_without_whitespace ... ok\n\ + test underscore_emphasis ... ok\ntest very_long_abbreviation ... ok\n\ + test all_caps_word ... ok\ntest two_letter_abbreviation ... ok\n\ + test consecutive_delimiters_in_camel_case ... ok\n\n\ + test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s\n"; + let counts = parse_cargo_summary(real).expect("regex must match"); + assert_eq!(counts.passed, 10); + assert_eq!(counts.failed, 0); + assert_eq!(counts.total, 10); + assert!(counts.solved()); + } + + #[test] + fn parse_cargo_summary_handles_failures() { + let out = "running 5 tests\ntest result: FAILED. 2 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out\n"; + let counts = parse_cargo_summary(out).expect("matches"); + assert_eq!(counts.passed, 2); + assert_eq!(counts.failed, 3); + assert_eq!(counts.total, 5); + assert!(!counts.solved()); + } + + #[test] + fn parse_cargo_summary_sums_multiple_test_binaries_and_doctests() { + // A typical task with unit tests, integration tests, AND + // doctests prints three "test result:" lines. + let out = "test result: ok. 5 passed; 0 failed; 0 ignored\n\ + test result: ok. 3 passed; 1 failed; 0 ignored\n\ + test result: ok. 2 passed; 0 failed; 0 ignored\n"; + let counts = parse_cargo_summary(out).expect("matches"); + assert_eq!(counts.passed, 10); + assert_eq!(counts.failed, 1); + assert_eq!(counts.total, 11); + } + + #[test] + fn parse_cargo_summary_returns_none_when_no_summary_present() { + // Compilation failure → no "test result:" lines. Caller falls + // back to the LLM judge, which knows to count build failures + // as failed=1. + let out = "error[E0599]: no method named `foo` found\nerror: could not compile `acronym`\n"; + assert!(parse_cargo_summary(out).is_none()); + } + + #[test] + fn parse_pytest_summary_extracts_passed_failed_from_real_output() { + // Real pytest output format. + let out = "============== 10 passed, 2 failed in 1.23s ==============\n"; + let counts = parse_pytest_summary(out).expect("matches"); + assert_eq!(counts.passed, 10); + assert_eq!(counts.failed, 2); + assert_eq!(counts.total, 12); + } + + #[test] + fn parse_pytest_summary_handles_passed_only() { + let out = "============== 12 passed in 0.01s ==============\n"; + let counts = parse_pytest_summary(out).expect("matches"); + assert_eq!(counts.passed, 12); + assert_eq!(counts.failed, 0); + assert_eq!(counts.total, 12); + } + + #[test] + fn parse_pytest_summary_treats_collection_errors_as_failed() { + // pytest's "error" count covers collection errors (broken + // imports, syntax errors). Score those as failures. + let out = "============== 5 passed, 2 errors in 0.01s ==============\n"; + let counts = parse_pytest_summary(out).expect("matches"); + assert_eq!(counts.passed, 5); + assert_eq!(counts.failed, 2); + } + + #[test] + fn parse_pytest_summary_takes_last_summary_line() { + // Interim "X passed" can appear earlier; the final aggregate + // line is the truth. + let out = "============== 1 passed in 0.5s ==============\n\ + ============== 10 passed, 2 failed in 1.2s ==============\n"; + let counts = parse_pytest_summary(out).expect("matches"); + assert_eq!(counts.passed, 10); + assert_eq!(counts.failed, 2); + } + + #[test] + fn parse_pytest_summary_handles_bare_undecorated_lines() { + // pearl th-19ab7c: when pytest can't detect terminal width (output + // piped to a file — our capture path) it drops the `====` bars. + // The regression case is the exact combined.txt from smoke run + // 115e2ee1: a progress line + a bare `16 passed in 0.01s`. + let cases: &[(&str, u32, u32, u32)] = &[ + // (combined output, expected passed, failed, total) + ("................ [100%]\n16 passed in 0.01s\n", 16, 0, 16), + ("1 passed in 0.00s", 1, 0, 1), + ("3 failed, 12 passed in 0.42s", 12, 3, 15), + ("1 failed in 0.02s", 0, 1, 1), + ("1 error in 0.01s", 0, 1, 1), + ("2 passed, 1 skipped, 3 warnings in 0.10s", 2, 0, 2), + ("10 passed, 2 failed, 1 skipped in 1.23s", 10, 2, 12), + // Long-form duration. + ("5 passed in 1m 2.34s", 5, 0, 5), + ]; + for (out, p, f, t) in cases { + let counts = parse_pytest_summary(out).unwrap_or_else(|| panic!("should parse: {out:?}")); + assert_eq!(counts.passed, *p, "passed for {out:?}"); + assert_eq!(counts.failed, *f, "failed for {out:?}"); + assert_eq!(counts.total, *t, "total for {out:?}"); + } + } + + #[test] + fn parse_pytest_summary_no_tests_ran_is_zero_not_judge() { + // `no tests ran` must be recognized (so we don't pay the judge + // tax) but score as 0/0/0 — not a pass, since all_passed() needs + // total > 0. + let counts = parse_pytest_summary("no tests ran in 0.00s").expect("recognized"); + assert_eq!((counts.passed, counts.failed, counts.total), (0, 0, 0)); + assert!(!counts.solved()); + } + + #[test] + fn parse_pytest_summary_ignores_non_summary_stdout() { + // A test that prints "passed" in its own output must NOT be + // mistaken for a summary — the duration tail is the discriminator. + assert!(parse_pytest_summary("assert result == 'passed'\n").is_none()); + assert!(parse_pytest_summary("the check passed in review\n").is_none()); + } + + #[test] + fn parse_jest_summary_extracts_passed_failed_total() { + let out = "Tests: 1 failed, 10 passed, 11 total\nTest Suites: 1 failed, 1 total\n"; + let counts = parse_jest_summary(out).expect("matches"); + assert_eq!(counts.passed, 10); + assert_eq!(counts.failed, 1); + assert_eq!(counts.total, 11); + } + + #[test] + fn parse_native_dispatches_per_language() { + let cargo = "test result: ok. 3 passed; 0 failed"; + assert_eq!(parse_native_test_summary(PolyglotLang::Rust, cargo).unwrap().passed, 3); + assert!(parse_native_test_summary(PolyglotLang::Python, cargo).is_none()); + assert!(parse_native_test_summary(PolyglotLang::Go, cargo).is_none()); // not yet wired + } + + #[test] + fn extract_count_before_handles_leading_and_trailing_whitespace() { + assert_eq!(extract_count_before("test result: ok. 10 passed; 0 failed", " passed"), Some(10)); + assert_eq!(extract_count_before("Tests: 1 failed, 10 passed, 11 total", " passed"), Some(10)); + assert_eq!(extract_count_before("===== 12 passed in 0.01s =====", " passed"), Some(12)); + assert_eq!(extract_count_before("no numeric prefix passed", " passed"), None); + assert_eq!(extract_count_before("missing suffix", " passed"), None); + } + + #[test] + fn write_score_forensic_creates_summary_and_combined() { + let tmp = tempfile::tempdir().expect("tmp"); + let work = tmp.path(); + let out = "test result: ok. 10 passed; 0 failed\n"; + let counts = TestCounts { + passed: 10, + failed: 0, + total: 10, + }; + write_score_forensic(work, PolyglotLang::Rust, &["cargo", "test"], Some(0), out, "native_regex", counts).expect("write"); + let forensic_dir = work.join(".smooth-score-forensic"); + assert!(forensic_dir.join("combined.txt").exists()); + assert!(forensic_dir.join("summary.json").exists()); + let summary: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(forensic_dir.join("summary.json")).unwrap()).unwrap(); + assert_eq!(summary["parsed_via"], "native_regex"); + assert_eq!(summary["counts"]["passed"], 10); + assert_eq!(summary["counts"]["solved"], true); + } + + // ────────────────────────────────────────────────────────────── + + #[test] + fn polyglot_lang_parses_common_names() { + assert_eq!(PolyglotLang::from_name("python"), Some(PolyglotLang::Python)); + assert_eq!(PolyglotLang::from_name("py"), Some(PolyglotLang::Python)); + assert_eq!(PolyglotLang::from_name("Rust"), Some(PolyglotLang::Rust)); + assert_eq!(PolyglotLang::from_name("rs"), Some(PolyglotLang::Rust)); + assert_eq!(PolyglotLang::from_name("JS"), Some(PolyglotLang::Javascript)); + assert_eq!(PolyglotLang::from_name("fortran"), None); + } + + #[test] + fn test_counts_solved_semantics() { + assert!(TestCounts { + passed: 20, + failed: 0, + total: 20 + } + .solved()); + assert!(!TestCounts { + passed: 0, + failed: 0, + total: 0 + } + .solved()); + assert!(!TestCounts { + passed: 19, + failed: 1, + total: 20 + } + .solved()); + assert!(!TestCounts { + passed: 20, + failed: 0, + total: 21 + } + .solved()); + } + + #[test] + fn judge_response_plain_json() { + let c = parse_judge_response(r#"{"passed": 20, "failed": 0, "total": 20}"#); + assert_eq!( + c, + TestCounts { + passed: 20, + failed: 0, + total: 20 + } + ); + } + + #[test] + fn judge_response_strips_json_code_fence() { + let c = parse_judge_response("```json\n{\"passed\": 5, \"failed\": 2, \"total\": 7}\n```"); + assert_eq!( + c, + TestCounts { + passed: 5, + failed: 2, + total: 7 + } + ); + } + + #[test] + fn judge_response_strips_bare_code_fence() { + let c = parse_judge_response("```\n{\"passed\": 1, \"failed\": 0, \"total\": 1}\n```"); + assert_eq!( + c, + TestCounts { + passed: 1, + failed: 0, + total: 1 + } + ); + } + + #[test] + fn judge_response_infers_total_from_passed_plus_failed() { + let c = parse_judge_response(r#"{"passed": 3, "failed": 2}"#); + assert_eq!( + c, + TestCounts { + passed: 3, + failed: 2, + total: 5 + } + ); + } + + #[test] + fn judge_response_tolerates_prose_around_object() { + let c = parse_judge_response("Sure! Here you go: {\"passed\": 10, \"failed\": 0, \"total\": 10} — hope that helps."); + assert_eq!( + c, + TestCounts { + passed: 10, + failed: 0, + total: 10 + } + ); + } + + #[test] + fn judge_response_malformed_returns_zero() { + assert_eq!(parse_judge_response("I don't know."), TestCounts::default()); + assert_eq!(parse_judge_response("{not json}"), TestCounts::default()); + assert_eq!(parse_judge_response(""), TestCounts::default()); + } + + #[test] + fn trim_for_judge_keeps_head_and_tail_under_budget() { + let big = "head-line\n".repeat(500) + &"tail-line\n".repeat(500); + let out = trim_for_judge(&big, 500); + assert!(out.len() <= 800, "trimmed output should be ≲ budget + elision note: got {}", out.len()); + assert!(out.starts_with("head-line")); + assert!(out.contains("[... ")); + assert!(out.trim_end().ends_with("tail-line")); + } + + #[test] + fn trim_for_judge_below_budget_is_unchanged() { + let s = "short output"; + assert_eq!(trim_for_judge(s, 1000), s); + } + + #[test] + fn copy_task_files_excludes_meta() { + let src = tempfile::tempdir().expect("src"); + let dst = tempfile::tempdir().expect("dst"); + + std::fs::write(src.path().join("main.py"), b"pass\n").unwrap(); + std::fs::write(src.path().join("main_test.py"), b"def test_x(): pass\n").unwrap(); + std::fs::create_dir(src.path().join(".meta")).unwrap(); + std::fs::write(src.path().join(".meta/example.py"), b"class A: pass\n").unwrap(); + std::fs::create_dir(src.path().join(".docs")).unwrap(); + std::fs::write(src.path().join(".docs/instructions.md"), b"do the thing").unwrap(); + + copy_task_files(src.path(), dst.path()).expect("copy ok"); + + assert!(dst.path().join("main.py").exists()); + assert!(dst.path().join("main_test.py").exists()); + assert!(dst.path().join("INSTRUCTIONS.md").exists()); + assert!(!dst.path().join(".meta").exists(), ".meta must not leak to the agent"); + assert!(!dst.path().join("example.py").exists()); + } + + #[test] + fn build_prompt_lists_files_and_test_command() { + let tmp = tempfile::tempdir().expect("tmpdir"); + std::fs::write(tmp.path().join("grade_school.py"), b"").unwrap(); + std::fs::write(tmp.path().join("grade_school_test.py"), b"").unwrap(); + std::fs::write(tmp.path().join("INSTRUCTIONS.md"), b"stuff").unwrap(); + + let prompt = build_prompt("grade-school", PolyglotLang::Python, tmp.path()).expect("prompt"); + assert!(prompt.contains("grade-school")); + assert!(prompt.contains("grade_school.py")); + assert!(prompt.contains("grade_school_test.py")); + assert!(prompt.contains("INSTRUCTIONS.md")); + assert!(prompt.contains("python3 -m pytest")); + } + + /// Pearl th-71e1fa regression: the agent claimed "6/6 PASSED" but the + /// scorer reported FAIL. Root cause was the agent creating a tiny fake + /// test file that passed (e.g. `test_affine_simple.py`), claiming + /// success, then `strip_agent_added_tests` removed it before scoring + /// ran the REAL test file. The prompt now explicitly forbids creating + /// new test files AND tells the agent the original test command is + /// the only one that counts. Guards both phrasings so future tweaks + /// don't silently lose either. + #[test] + fn build_prompt_forbids_creating_test_files() { + let tmp = tempfile::tempdir().expect("tmpdir"); + std::fs::write(tmp.path().join("affine_cipher.py"), b"").unwrap(); + std::fs::write(tmp.path().join("affine_cipher_test.py"), b"").unwrap(); + std::fs::write(tmp.path().join("INSTRUCTIONS.md"), b"stuff").unwrap(); + + let prompt = build_prompt("affine-cipher", PolyglotLang::Python, tmp.path()).expect("prompt"); + let lower = prompt.to_lowercase(); + assert!( + lower.contains("do not modify") && lower.contains("create test files"), + "prompt must forbid both modifying AND creating test files. Prompt was:\n{prompt}" + ); + assert!( + lower.contains("deleted before scoring"), + "prompt must warn that agent-added test files get stripped before scoring. Prompt was:\n{prompt}" + ); + assert!( + lower.contains("(no other test command or subset)"), + "prompt must tell the agent to run the exact `{{cmd}}`, not a subset or fake. Prompt was:\n{prompt}" + ); + } + + /// Regression for pearl th-01c714: the bench-driven `smooth-code` + /// TUI treats every newline in pasted input as Enter (submit), so + /// a multi-line prompt arrived as N separate `You:` submissions + /// instead of one cohesive task. The fix flattens `build_prompt` + /// to a single line. This test guards that no `\n` ever sneaks + /// back in — neither from the template itself nor from a file + /// name with embedded whitespace. + #[test] + fn build_prompt_is_single_line() { + let tmp = tempfile::tempdir().expect("tmpdir"); + std::fs::write(tmp.path().join("affine_cipher.py"), b"").unwrap(); + std::fs::write(tmp.path().join("affine_cipher_test.py"), b"").unwrap(); + std::fs::write(tmp.path().join("INSTRUCTIONS.md"), b"stuff").unwrap(); + + let prompt = build_prompt("affine-cipher", PolyglotLang::Python, tmp.path()).expect("prompt"); + assert!( + !prompt.contains('\n'), + "build_prompt produced a multi-line prompt; the TUI would split it into multiple You: submissions. Prompt was:\n{prompt}" + ); + assert!( + !prompt.contains('\r'), + "build_prompt produced a carriage return; same hazard as `\\n`. Prompt was:\n{prompt}" + ); + } + + #[test] + fn rewrite_jest_skips_flips_every_variant() { + let tmp = tempfile::tempdir().expect("tmp"); + let spec = tmp.path().join("foo.spec.js"); + std::fs::write( + &spec, + r#" +describe("bowling", () => { + test("one", () => { expect(1).toBe(1); }); + xtest("two", () => {}); + test.skip("three", () => {}); + it.skip("four", () => {}); + xit("five", () => {}); + xdescribe("nested", () => { + test("six", () => {}); + }); + describe.skip("also nested", () => { + test("seven", () => {}); + }); +}); +"#, + ) + .unwrap(); + + rewrite_jest_skips(tmp.path()).expect("rewrite"); + + let body = std::fs::read_to_string(&spec).unwrap(); + assert!(!body.contains("xtest("), "xtest not rewritten: {body}"); + assert!(!body.contains("xit("), "xit not rewritten: {body}"); + assert!(!body.contains("test.skip("), "test.skip not rewritten: {body}"); + assert!(!body.contains("it.skip("), "it.skip not rewritten: {body}"); + assert!(!body.contains("xdescribe("), "xdescribe not rewritten: {body}"); + assert!(!body.contains("describe.skip("), "describe.skip not rewritten: {body}"); + // Every case becomes an active test/it/describe call. + assert_eq!(body.matches("test(").count(), 5); + } + + #[test] + fn rewrite_jest_skips_recurses_into_subdirs() { + let tmp = tempfile::tempdir().expect("tmp"); + let nested = tmp.path().join("a/b"); + std::fs::create_dir_all(&nested).unwrap(); + std::fs::write(nested.join("deep.spec.js"), "xtest(\"x\", () => {});").unwrap(); + + rewrite_jest_skips(tmp.path()).expect("rewrite"); + + let body = std::fs::read_to_string(nested.join("deep.spec.js")).unwrap(); + assert_eq!(body, "test(\"x\", () => {});"); + } + + #[test] + fn rewrite_jest_skips_skips_node_modules() { + let tmp = tempfile::tempdir().expect("tmp"); + std::fs::create_dir_all(tmp.path().join("node_modules")).unwrap(); + std::fs::write(tmp.path().join("node_modules/foo.spec.js"), "xtest(\"x\", () => {});").unwrap(); + + rewrite_jest_skips(tmp.path()).expect("rewrite"); + + // node_modules content untouched — we don't rewrite third-party code. + let body = std::fs::read_to_string(tmp.path().join("node_modules/foo.spec.js")).unwrap(); + assert!(body.contains("xtest(")); + } + + #[test] + fn enable_skipped_tests_is_noop_for_python_rust_go() { + let tmp = tempfile::tempdir().expect("tmp"); + for lang in [PolyglotLang::Python, PolyglotLang::Rust, PolyglotLang::Go] { + enable_skipped_tests(lang, tmp.path()).expect("no-op"); + } + } + + #[test] + fn strip_junit_skip_annotations_removes_bare_disabled() { + let src = r#"import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Disabled; + +class BowlingTest { + @Test + void passing() {} + + @Disabled + @Test + void skipped() {} + + @Disabled("not yet implemented") + @Test + void skippedWithReason() {} +} +"#; + let out = strip_junit_skip_annotations(src); + assert!(!out.contains("@Disabled"), "all @Disabled lines should be gone: {out}"); + // @Test for each method must remain (3 of them). + assert_eq!(out.matches("@Test").count(), 3); + // The method bodies remain. + assert!(out.contains("void skipped()")); + assert!(out.contains("void skippedWithReason()")); + } + + #[test] + fn strip_junit_skip_annotations_handles_junit4_ignore() { + let src = "@Ignore\n@Test public void x() {}\n@Ignore(\"meh\")\n@Test public void y() {}\n"; + let out = strip_junit_skip_annotations(src); + assert!(!out.contains("@Ignore")); + assert!(out.contains("@Test public void x()")); + assert!(out.contains("@Test public void y()")); + } + + #[test] + fn strip_junit_skip_annotations_preserves_unrelated_annotations() { + let src = "@DisabledInNativeImage\n@Test void z() {}\n"; + let out = strip_junit_skip_annotations(src); + // `@DisabledInNativeImage` is NOT `@Disabled` — must survive. + assert!(out.contains("@DisabledInNativeImage")); + } + + #[test] + fn rewrite_junit_skips_recurses_into_test_dir() { + let tmp = tempfile::tempdir().expect("tmp"); + let test_dir = tmp.path().join("src/test/java"); + std::fs::create_dir_all(&test_dir).unwrap(); + let file = test_dir.join("BowlingTest.java"); + std::fs::write(&file, "@Disabled\n@Test void a() {}\n").unwrap(); + + rewrite_junit_skips(tmp.path()).expect("rewrite"); + + let body = std::fs::read_to_string(&file).unwrap(); + assert!(!body.contains("@Disabled")); + } + + #[test] + fn is_test_file_matches_per_language_conventions() { + // Python — pytest discovery + assert!(is_test_file(PolyglotLang::Python, Path::new("test_bowling.py"))); + assert!(is_test_file(PolyglotLang::Python, Path::new("bowling_test.py"))); + assert!(is_test_file(PolyglotLang::Python, Path::new("tests/extra.py"))); + assert!(!is_test_file(PolyglotLang::Python, Path::new("bowling.py"))); + assert!(!is_test_file(PolyglotLang::Python, Path::new("helper.py"))); + + // Rust — integration-test files under tests/ only + assert!(is_test_file(PolyglotLang::Rust, Path::new("tests/edge_cases.rs"))); + assert!(!is_test_file(PolyglotLang::Rust, Path::new("src/lib.rs"))); + assert!(!is_test_file(PolyglotLang::Rust, Path::new("src/tests.rs"))); + + // Go + assert!(is_test_file(PolyglotLang::Go, Path::new("bowling_test.go"))); + assert!(is_test_file(PolyglotLang::Go, Path::new("extra_test.go"))); + assert!(!is_test_file(PolyglotLang::Go, Path::new("bowling.go"))); + + // JavaScript + assert!(is_test_file(PolyglotLang::Javascript, Path::new("bowling.spec.js"))); + assert!(is_test_file(PolyglotLang::Javascript, Path::new("extra.test.ts"))); + assert!(!is_test_file(PolyglotLang::Javascript, Path::new("bowling.js"))); + + // Java + assert!(is_test_file(PolyglotLang::Java, Path::new("src/test/java/BowlingTest.java"))); + assert!(is_test_file(PolyglotLang::Java, Path::new("ExtraTests.java"))); + assert!(!is_test_file(PolyglotLang::Java, Path::new("src/main/java/BowlingGame.java"))); + } + + #[test] + fn strip_agent_added_tests_removes_only_new_test_files() { + use std::fs; + let tmp = tempfile::tempdir().expect("tmp"); + let root = tmp.path(); + + // Original task files + fs::write(root.join("bowling.py"), "class BowlingGame: pass").unwrap(); + fs::write(root.join("bowling_test.py"), "def test_x(): pass").unwrap(); + + let original = snapshot_files(root).unwrap(); + assert_eq!(original.len(), 2); + + // Agent adds: one new test file (should be stripped), one + // new helper module (should be kept). + fs::write(root.join("test_extra.py"), "def test_edge(): pass").unwrap(); + fs::write(root.join("helper.py"), "def helper(): pass").unwrap(); + // Agent modifies an existing file — should stay put. + fs::write(root.join("bowling.py"), "class BowlingGame:\n def score(self): return 0").unwrap(); + + let stripped = strip_agent_added_tests(PolyglotLang::Python, root, &original).unwrap(); + + // test_extra.py is gone + assert_eq!(stripped.len(), 1); + assert_eq!(stripped[0], Path::new("test_extra.py")); + assert!(!root.join("test_extra.py").exists()); + // helper.py and bowling.py (modified) survive + assert!(root.join("helper.py").exists()); + assert!(root.join("bowling.py").exists()); + // Original bowling_test.py untouched + assert!(root.join("bowling_test.py").exists()); + } + + #[test] + fn strip_agent_added_tests_ignores_untouched_originals() { + use std::fs; + let tmp = tempfile::tempdir().expect("tmp"); + let root = tmp.path(); + fs::write(root.join("bowling_test.py"), "def test_x(): pass").unwrap(); + let original = snapshot_files(root).unwrap(); + // Agent didn't add any new test files. + let stripped = strip_agent_added_tests(PolyglotLang::Python, root, &original).unwrap(); + assert!(stripped.is_empty()); + assert!(root.join("bowling_test.py").exists()); + } + + #[test] + fn rewrite_junit_skips_leaves_production_code_alone() { + let tmp = tempfile::tempdir().expect("tmp"); + let main_dir = tmp.path().join("src/main/java"); + std::fs::create_dir_all(&main_dir).unwrap(); + // A production-code file that happens to reference + // `@Disabled` via a doc comment or something — don't + // rewrite it just because the annotation name appears. + let prod = main_dir.join("BowlingGame.java"); + std::fs::write(&prod, "class BowlingGame {\n // See @Disabled tests in BowlingTest\n}\n").unwrap(); + + rewrite_junit_skips(tmp.path()).expect("rewrite"); + + let body = std::fs::read_to_string(&prod).unwrap(); + assert!(body.contains("@Disabled")); + } +} diff --git a/crates/smooth-bench/src/main.rs b/crates/smooth-bench/src/main.rs new file mode 100644 index 000000000..150008c87 --- /dev/null +++ b/crates/smooth-bench/src/main.rs @@ -0,0 +1,230 @@ +//! `smooth-bench` — internal benchmark harness binary. +//! +//! Not shipped in the `th` CLI. Run via: +//! +//! # single task against an already-running engine at --url +//! cargo run -p smooai-smooth-bench -- aider-polyglot --task grade-school +//! +//! # engine-parity sweep: boot each engine, score it, tear it down +//! cargo run -p smooai-smooth-bench -- score --engine rust --engine go +//! +//! The `score` command is the engine-parity benchmark (pearl th-4c3e2d): +//! it runs the curated aider-polyglot suite through each of the five +//! smooth-operator LocalServer implementations and emits per-engine +//! (and per-model) scores. + +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::{Context, Result}; +use clap::{Parser, Subcommand}; +use smooth_bench::curated::CuratedList; +use smooth_bench::engine::{run_engine_matrix, Engine, EngineEnv, EngineMatrixRun, ProcessBooter}; +use smooth_bench::sweep::{current_commit_sha, StdoutObserver, SweepConfig, SweepGate}; +use smooth_bench::{print_summary, run_aider_polyglot, BenchOpts, PolyglotLang}; + +#[derive(Parser)] +#[command(name = "smooth-bench", version, about = "Smooth engine-parity benchmark harness (internal)", long_about = None)] +struct Cli { + #[command(subcommand)] + cmd: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Run a single Aider Polyglot task against an already-running + /// engine at `--url`. + AiderPolyglot { + /// Task name (e.g. `grade-school`, `leap`, `forth`). + #[arg(long)] + task: String, + /// Language subset. Default: python. + #[arg(long, default_value = "python")] + lang: String, + /// Budget limit in USD for the LLM calls. Default: $5.00. + #[arg(long, default_value_t = 5.00)] + budget: f64, + /// Override the routing model (passed through to the engine). + #[arg(long)] + model: Option, + /// Engine URL. Defaults to http://localhost:4400. + #[arg(long, default_value = "http://localhost:4400")] + url: String, + }, + + /// Engine-parity sweep: run the curated aider-polyglot suite through + /// each selected smooth-operator engine, scoring per engine (and per + /// model). Boots each engine's LocalServer the way + /// `scripts/operator-serve.sh` does, runs the tasks, tears it down. + /// Pearl th-4c3e2d. + Score(ScoreArgs), +} + +#[derive(Parser, Debug)] +struct ScoreArgs { + /// Which engine(s) to benchmark. Repeatable. Default: all five + /// (rust, go, ts, python, dotnet). + #[arg(long = "engine", value_parser = parse_engine)] + engines: Vec, + + /// Which model(s) to run each engine under. Repeatable. Default: + /// deepseek-v4-flash. + #[arg(long = "model")] + models: Vec, + + /// Authoritative sample: every curated task × language. Mutually + /// exclusive with `--pr`. + #[arg(long, conflicts_with = "pr")] + release: bool, + + /// CI-gate sample: `--tasks-per-language` tasks × 6 languages. + /// Default when neither `--release` nor `--pr` is given. + #[arg(long)] + pr: bool, + + /// Tasks per language in the PR gate. Default: 3. + #[arg(long, default_value_t = 3)] + tasks_per_language: usize, + + /// Hard USD cap per engine×model cell. When the running cost total + /// exceeds this, that cell's sweep aborts and emits a partial Score + /// with `budget_usd_hit: true`. + #[arg(long, default_value_t = 10.0)] + budget_usd: f64, + + /// smooth-operator repo root (where the polyglot engine servers + /// live). Default: $SMOOTH_OPERATOR_REPO or ~/dev/smooai/smooth-operator. + #[arg(long)] + repo: Option, + + /// Per-engine boot timeout in seconds (wait for the port to listen). + #[arg(long, default_value_t = 300)] + boot_timeout_s: u64, + + /// Output path. If given, JSON-lines records are written there and + /// the summary table still prints to stdout; otherwise both go to + /// stdout. + #[arg(long)] + output: Option, +} + +fn parse_engine(s: &str) -> Result { + Engine::from_name(s).ok_or_else(|| format!("unknown engine {s:?} (valid: rust, go, ts, python, dotnet)")) +} + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + match cli.cmd { + Commands::AiderPolyglot { + task, + lang, + budget, + model, + url, + } => { + let lang_enum = + PolyglotLang::from_name(&lang).ok_or_else(|| anyhow::anyhow!("unknown language: {lang} (try python, rust, go, javascript, java, cpp)"))?; + let opts = BenchOpts { + big_smooth_url: url, + budget_usd: Some(budget), + model, + }; + println!("Running aider-polyglot/{}/{task} …", lang_enum.dataset_dir()); + let result = run_aider_polyglot(lang_enum, &task, &opts).await?; + print_summary(&result); + if result.solved { + Ok(()) + } else { + std::process::exit(1); + } + } + Commands::Score(args) => run_score(args).await, + } +} + +async fn run_score(args: ScoreArgs) -> Result<()> { + let engines = if args.engines.is_empty() { + Engine::ALL.to_vec() + } else { + args.engines.clone() + }; + let models = if args.models.is_empty() { + vec!["deepseek-v4-flash".to_string()] + } else { + args.models.clone() + }; + + let gate = if args.release { + SweepGate::Release + } else { + if !args.pr { + eprintln!( + "neither --release nor --pr given; defaulting to --pr ({} tasks × 6 langs per engine)", + args.tasks_per_language + ); + } + SweepGate::Pr { + tasks_per_language: args.tasks_per_language, + } + }; + + let repo = args + .repo + .or_else(|| std::env::var_os("SMOOTH_OPERATOR_REPO").map(PathBuf::from)) + .or_else(|| dirs_next::home_dir().map(|h| h.join("dev").join("smooai").join("smooth-operator"))) + .context("could not resolve smooth-operator repo root")?; + + let env = EngineEnv { + gateway_url: std::env::var("SMOOAI_GATEWAY_URL").ok(), + gateway_key: std::env::var("SMOOAI_GATEWAY_KEY").ok(), + persona: std::env::var("SMOOTH_PERSONA").ok(), + }; + if env.gateway_key.is_none() { + eprintln!("warning: SMOOAI_GATEWAY_KEY is unset — engines will boot but turns will error; scores will be all-FAIL"); + } + + let curated = CuratedList::default_embedded().context("loading embedded curated task list")?; + let cfg = SweepConfig { + gate, + budget_usd_cap: args.budget_usd, + smooth_version: env!("CARGO_PKG_VERSION").to_string(), + commit_sha: current_commit_sha(), + task_opts: BenchOpts { + big_smooth_url: String::new(), // filled per engine by the matrix runner + budget_usd: Some(args.budget_usd), + model: None, + }, + }; + + let mut booter = ProcessBooter::new(repo, env); + booter.ready_timeout = Duration::from_secs(args.boot_timeout_s); + + let mut observer = StdoutObserver; + let run = run_engine_matrix(&curated, &booter, &engines, &models, &cfg, &mut observer).await?; + + emit(&run, args.output.as_deref())?; + + // Non-zero exit if any cell hit its budget cap — CI will notice. + if run.results.iter().any(|r| r.score.budget_usd_hit) { + std::process::exit(2); + } + Ok(()) +} + +fn emit(run: &EngineMatrixRun, output: Option<&std::path::Path>) -> Result<()> { + let jsonl = run.to_jsonl()?; + match output { + Some(path) => { + std::fs::write(path, &jsonl).with_context(|| format!("writing JSON-lines to {}", path.display()))?; + eprintln!("wrote {}", path.display()); + print!("{}", run.render_summary()); + } + None => { + print!("{jsonl}"); + println!(); + print!("{}", run.render_summary()); + } + } + Ok(()) +} diff --git a/crates/smooth-bench/src/scenarios.rs b/crates/smooth-bench/src/scenarios.rs new file mode 100644 index 000000000..cd9df1e44 --- /dev/null +++ b/crates/smooth-bench/src/scenarios.rs @@ -0,0 +1,600 @@ +//! TUI-driven bench scenarios. +//! +//! Pearl: th-139b02 +//! +//! A scenario is a synthetic user session against the `th` TUI: a +//! fixture repo (a real `.git` directory + source files), a list of +//! user inputs, and assertions about the chat the user *would have +//! seen*. +//! +//! This module covers the schema + TOML parser. The pty-driven +//! runner that actually spawns `th` and captures the rendered chat +//! lives in [`runner`](super::runner) (next subtask of th-139b02). +//! +//! ## Layout on disk +//! +//! ```text +//! crates/smooth-bench/scenarios/ +//! ├── repo-overview/ +//! │ ├── scenario.toml # this file's schema +//! │ └── fixture/ # checked-in synthetic repo +//! ├── stack-discovery/ +//! ├── edit-readme/ +//! └── commit-to-main/ # negative test — agent proposes +//! # command, must NOT auto-commit +//! ``` +//! +//! ## scenario.toml schema (v1) +//! +//! ```toml +//! [meta] +//! title = "User asks for a repo overview" +//! description = "First-turn factual Q routes to oracle, agent gives terse answer." +//! agent = "auto" # or "oracle" / "fixer" / etc. to pin +//! +//! [[turns]] +//! input = "what is this project" +//! +//! [[turns.assert]] +//! kind = "intent_role" +//! expected = "oracle" +//! +//! [[turns.assert]] +//! kind = "tool_called" +//! name = "project_inspect" +//! +//! [[turns.assert]] +//! kind = "response_contains_any" +//! strings = ["budgeting", "next.js", "drizzle"] +//! +//! [[turns.assert]] +//! kind = "response_does_not_contain" +//! strings = ["postgres"] +//! ``` + +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Context, Result}; +use serde::{Deserialize, Serialize}; + +/// Top-level scenario, one per `scenario.toml`. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Scenario { + pub meta: ScenarioMeta, + /// Ordered user turns. Each turn drives a single TUI input and + /// runs its assertions against the captured chat for that turn + /// only — earlier turns' chat is left alone (the runner keeps + /// the TUI session open across turns to exercise in-session + /// memory, since pearl th-422b93 made that a real feature). + #[serde(default)] + pub turns: Vec, +} + +/// Free-form description so failure reports + the LLM judge have +/// human-language context. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ScenarioMeta { + pub title: String, + pub description: String, + /// Lead role to dispatch under. `"auto"` means "let the intent + /// classifier pick" (the normal user path). Any role from + /// `Cast::builtin()` (`oracle`, `fixer`, `mapper`, `heckler`, + /// `runner`, `scout`) pins the role for the whole scenario. + #[serde(default = "default_agent")] + pub agent: String, + /// Per-turn timeout in seconds — past this the runner kills the + /// turn and records a timeout failure. Default 120s, generous + /// for sandboxed LLM dispatches but bounded so a wedged run + /// doesn't hang the whole bench loop. + #[serde(default = "default_turn_timeout_s")] + pub turn_timeout_s: u64, + /// What to do when an Ask fires during the scenario run. + /// Defaults to `deny` — bench runs are unattended and the safe + /// default is to refuse anything the agent didn't have a + /// pre-approved grant for. Override per-scenario when the test + /// *wants* to exercise the auto-approve path. Pearl th-400773. + #[serde(default)] + pub auto_approve: AutoApprove, +} + +/// How a bench scenario resolves Asks raised by Safehouse Narc +/// during the run. Mirrors the CLI's `--auto-approve` flag. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum AutoApprove { + /// Deny every Ask. The safe default for unattended runs. + #[default] + Deny, + /// Approve at scope=once. The narrowest possible auto-approve + /// — re-asks on the next request. + Once, + /// Approve at scope=session. Subsequent identical asks within + /// the same scenario run skip the prompt. + Session, + /// Approve at scope=project. Writes the grant to the project's + /// wonk-allow.toml — most bench scenarios should NOT pick this + /// because it pollutes the project's persistent state. + Project, + /// Approve at scope=user. Even more invasive than project; left + /// here for completeness but bench scenarios almost never want + /// it. + User, +} + +impl AutoApprove { + /// Parse from the CLI flag form (`once`, `session`, `project`, + /// `user`, `deny`). Case-insensitive. Returns `None` for unknown + /// values so the caller can render a clear error. + #[must_use] + pub fn parse(s: &str) -> Option { + match s.to_ascii_lowercase().as_str() { + "deny" => Some(Self::Deny), + "once" => Some(Self::Once), + "session" => Some(Self::Session), + "project" | "pearl_project" | "pearl-project" => Some(Self::Project), + "user" => Some(Self::User), + _ => None, + } + } + + /// Stable string form for logs and the CLI flag. + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Deny => "deny", + Self::Once => "once", + Self::Session => "session", + Self::Project => "project", + Self::User => "user", + } + } + + /// True if this mode should deny pending Asks rather than + /// approve them. + #[must_use] + pub fn is_deny(self) -> bool { + matches!(self, Self::Deny) + } +} + +fn default_agent() -> String { + "auto".to_string() +} + +fn default_turn_timeout_s() -> u64 { + 120 +} + +/// One user input + its assertions. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Turn { + /// What the user types into the input box this turn. + pub input: String, + /// Assertions evaluated against the captured chat for this + /// turn's response. All must pass for the turn to be green. + #[serde(default, rename = "assert")] + pub assertions: Vec, +} + +/// Single assertion kind. Tagged on `kind` so TOML is +/// `[[turns.assert]]\nkind = "tool_called"\nname = "grep"`. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Assertion { + /// The TUI's status bar showed a specific role at any point + /// during the turn. Used to verify the intent classifier + /// routed the way we expect (e.g. `commit` keywords go to + /// `oracle`, not `fixer`). + IntentRole { expected: String }, + /// A tool call with this name appeared in the captured chat. + /// Order-independent. + ToolCalled { name: String }, + /// The final assistant response contains *any* of these + /// strings (case-insensitive substring match). + ResponseContainsAny { strings: Vec }, + /// The final assistant response contains *all* of these + /// strings (case-insensitive substring match). + ResponseContainsAll { strings: Vec }, + /// The final assistant response contains *none* of these + /// strings (case-insensitive). Used for negative facts — + /// "the agent must not say 'postgres' when the repo uses + /// SQLite". + ResponseDoesNotContain { strings: Vec }, + /// The response includes a fenced code block — used by the + /// `commit-to-main` scenario to verify the agent proposed a + /// `git ...` command instead of pretending to run it. + /// `language` filters to a specific fence label (`bash`, + /// `sh`, `git`); `None` accepts any fence. + CommandProposed { + #[serde(default)] + language: Option, + contains_any: Vec, + }, + /// No tool with this name was called. Catches the + /// hallucinated-fix loop — `write_file` should not appear in + /// a `commit-to-main` scenario response. + ToolNotCalled { name: String }, + /// An Ask was filed against the AccessStore during this turn. + /// The TUI / bench harness should observe the request for + /// `ask_for` (typically a hostname or tool name) and resolve + /// it at `resolve_with` scope (`once`/`session`/`project`/ + /// `user`/`deny`). When `must_fire` is true, the absence of a + /// matching pending request fails the assertion — proves the + /// gating layer actually triggered. When false, the assertion + /// passes if the request was either resolved correctly OR + /// never fired (e.g. because a persistent grant covered it). + /// Pearl th-400773. + Permission { + /// Resource the Ask should mention — host for network, + /// tool name for tool, command for cli. Substring match + /// case-insensitive. + ask_for: String, + /// Resolution to apply: `once` / `session` / `project` / + /// `user` / `deny`. Same vocabulary as the CLI flag. + resolve_with: String, + /// When true (default), the assertion fails if no matching + /// Ask appeared. When false, only the resolution shape is + /// checked. + #[serde(default = "default_must_fire")] + must_fire: bool, + }, +} + +fn default_must_fire() -> bool { + true +} + +/// Read and parse a scenario from `/scenario.toml`. The +/// returned scenario's paths are relative to `dir` — the runner +/// resolves them when copying the fixture into a scratch dir. +pub fn load_scenario(dir: &Path) -> Result { + let path = dir.join("scenario.toml"); + let raw = std::fs::read_to_string(&path).with_context(|| format!("reading scenario {}", path.display()))?; + parse_scenario(&raw).with_context(|| format!("parsing scenario {}", path.display())) +} + +/// Parse a scenario from a raw TOML string. Public for tests + +/// for callers that want to validate without hitting the disk. +pub fn parse_scenario(raw: &str) -> Result { + let scenario: Scenario = toml::from_str(raw).map_err(|e| anyhow!("invalid scenario.toml: {e}"))?; + validate_scenario(&scenario)?; + Ok(scenario) +} + +fn validate_scenario(s: &Scenario) -> Result<()> { + if s.meta.title.trim().is_empty() { + return Err(anyhow!("scenario.meta.title must be non-empty")); + } + if s.turns.is_empty() { + return Err(anyhow!("scenario must have at least one turn")); + } + for (i, turn) in s.turns.iter().enumerate() { + if turn.input.trim().is_empty() { + return Err(anyhow!("turn {}: input must be non-empty", i + 1)); + } + } + Ok(()) +} + +/// Discover every `scenarios//scenario.toml` under the bench +/// crate's checkout. Returns `(name, scenario_dir, parsed)` triples +/// in stable lexical order so test runs are deterministic. +pub fn discover_scenarios(scenarios_root: &Path) -> Result> { + if !scenarios_root.is_dir() { + return Err(anyhow!("scenarios root not a directory: {}", scenarios_root.display())); + } + let mut entries: Vec<(String, PathBuf, Scenario)> = Vec::new(); + let mut dirs: Vec<_> = std::fs::read_dir(scenarios_root) + .with_context(|| format!("reading {}", scenarios_root.display()))? + .filter_map(std::result::Result::ok) + .filter(|e| e.path().is_dir()) + .collect(); + dirs.sort_by_key(std::fs::DirEntry::file_name); + for entry in dirs { + let dir = entry.path(); + let name = dir + .file_name() + .and_then(|s| s.to_str()) + .ok_or_else(|| anyhow!("scenario dir name not utf-8: {}", dir.display()))? + .to_string(); + let scenario_path = dir.join("scenario.toml"); + if !scenario_path.is_file() { + // Skip directories that aren't scenarios (e.g. a + // shared `_lib/` helpers dir). Not an error. + continue; + } + let scenario = load_scenario(&dir).with_context(|| format!("loading scenario {name}"))?; + entries.push((name, dir, scenario)); + } + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn minimal_toml() -> &'static str { + r#" +[meta] +title = "Repo overview" +description = "User asks what the project does." + +[[turns]] +input = "what is this project" + +[[turns.assert]] +kind = "tool_called" +name = "project_inspect" + +[[turns.assert]] +kind = "response_contains_any" +strings = ["budgeting", "drizzle"] +"# + } + + #[test] + fn parse_minimal_scenario() { + let s = parse_scenario(minimal_toml()).expect("parse"); + assert_eq!(s.meta.title, "Repo overview"); + assert_eq!(s.meta.agent, "auto"); // default + assert_eq!(s.meta.turn_timeout_s, 120); + assert_eq!(s.turns.len(), 1); + assert_eq!(s.turns[0].input, "what is this project"); + assert_eq!(s.turns[0].assertions.len(), 2); + match &s.turns[0].assertions[0] { + Assertion::ToolCalled { name } => assert_eq!(name, "project_inspect"), + other => panic!("unexpected variant {other:?}"), + } + match &s.turns[0].assertions[1] { + Assertion::ResponseContainsAny { strings } => { + assert_eq!(strings.len(), 2); + assert!(strings.contains(&"budgeting".to_string())); + } + other => panic!("unexpected variant {other:?}"), + } + } + + #[test] + fn empty_title_rejected() { + let raw = r#" +[meta] +title = "" +description = "x" + +[[turns]] +input = "hi" +"#; + let err = parse_scenario(raw).expect_err("must reject empty title"); + assert!(err.to_string().contains("title")); + } + + #[test] + fn empty_input_rejected() { + let raw = r#" +[meta] +title = "x" +description = "y" + +[[turns]] +input = "" +"#; + let err = parse_scenario(raw).expect_err("must reject empty turn input"); + assert!(err.to_string().contains("input")); + } + + #[test] + fn no_turns_rejected() { + let raw = r#" +[meta] +title = "x" +description = "y" +"#; + let err = parse_scenario(raw).expect_err("must reject zero turns"); + assert!(err.to_string().contains("at least one")); + } + + #[test] + fn assertion_variants_roundtrip() { + // Pin the wire shape so a future serde_with_change doesn't + // silently rename a kind and break authored scenario files. + let raw = r#" +[meta] +title = "all kinds" +description = "y" + +[[turns]] +input = "x" + +[[turns.assert]] +kind = "intent_role" +expected = "oracle" + +[[turns.assert]] +kind = "tool_called" +name = "grep" + +[[turns.assert]] +kind = "tool_not_called" +name = "write_file" + +[[turns.assert]] +kind = "response_contains_any" +strings = ["a"] + +[[turns.assert]] +kind = "response_contains_all" +strings = ["a", "b"] + +[[turns.assert]] +kind = "response_does_not_contain" +strings = ["postgres"] + +[[turns.assert]] +kind = "command_proposed" +language = "bash" +contains_any = ["git commit", "git add"] + +[[turns.assert]] +kind = "permission" +ask_for = "api.openai.com" +resolve_with = "session" +"#; + let s = parse_scenario(raw).expect("parse"); + let kinds: Vec<&Assertion> = s.turns[0].assertions.iter().collect(); + assert_eq!(kinds.len(), 8); + // Spot-check each variant landed in the right shape. + assert!(matches!(kinds[0], Assertion::IntentRole { .. })); + assert!(matches!(kinds[1], Assertion::ToolCalled { .. })); + assert!(matches!(kinds[2], Assertion::ToolNotCalled { .. })); + assert!(matches!(kinds[3], Assertion::ResponseContainsAny { .. })); + assert!(matches!(kinds[4], Assertion::ResponseContainsAll { .. })); + assert!(matches!(kinds[5], Assertion::ResponseDoesNotContain { .. })); + assert!(matches!(kinds[6], Assertion::CommandProposed { .. })); + match kinds[7] { + Assertion::Permission { + ask_for, + resolve_with, + must_fire, + } => { + assert_eq!(ask_for, "api.openai.com"); + assert_eq!(resolve_with, "session"); + assert!(must_fire, "must_fire defaults to true"); + } + other => panic!("expected Permission, got {other:?}"), + } + } + + #[test] + fn permission_assertion_must_fire_can_be_overridden() { + let raw = r#" +[meta] +title = "T" +description = "D" + +[[turns]] +input = "x" + +[[turns.assert]] +kind = "permission" +ask_for = "api.example.com" +resolve_with = "deny" +must_fire = false +"#; + let s = parse_scenario(raw).expect("parse"); + match &s.turns[0].assertions[0] { + Assertion::Permission { must_fire, .. } => assert!(!*must_fire), + other => panic!("unexpected variant {other:?}"), + } + } + + #[test] + fn auto_approve_defaults_to_deny() { + let s = parse_scenario(minimal_toml()).expect("parse"); + assert_eq!(s.meta.auto_approve, AutoApprove::Deny); + assert!(s.meta.auto_approve.is_deny()); + } + + #[test] + fn auto_approve_can_be_set_in_meta() { + let raw = r#" +[meta] +title = "T" +description = "D" +auto_approve = "session" + +[[turns]] +input = "x" +"#; + let s = parse_scenario(raw).expect("parse"); + assert_eq!(s.meta.auto_approve, AutoApprove::Session); + assert!(!s.meta.auto_approve.is_deny()); + } + + #[test] + fn auto_approve_parses_canonical_forms() { + assert_eq!(AutoApprove::parse("deny"), Some(AutoApprove::Deny)); + assert_eq!(AutoApprove::parse("once"), Some(AutoApprove::Once)); + assert_eq!(AutoApprove::parse("session"), Some(AutoApprove::Session)); + assert_eq!(AutoApprove::parse("project"), Some(AutoApprove::Project)); + assert_eq!(AutoApprove::parse("user"), Some(AutoApprove::User)); + // Aliases. + assert_eq!(AutoApprove::parse("pearl_project"), Some(AutoApprove::Project)); + assert_eq!(AutoApprove::parse("Pearl-Project"), Some(AutoApprove::Project)); + // Case-insensitive. + assert_eq!(AutoApprove::parse("SESSION"), Some(AutoApprove::Session)); + // Unknown returns None so the caller can render a clear error. + assert_eq!(AutoApprove::parse("forever"), None); + assert_eq!(AutoApprove::parse(""), None); + } + + #[test] + fn auto_approve_round_trips_through_as_str_and_parse() { + for mode in [ + AutoApprove::Deny, + AutoApprove::Once, + AutoApprove::Session, + AutoApprove::Project, + AutoApprove::User, + ] { + let s = mode.as_str(); + assert_eq!(AutoApprove::parse(s), Some(mode), "round-trip {s}"); + } + } + + #[test] + fn auto_approve_serde_uses_snake_case() { + // The TOML schema treats `pearl_project` as the canonical + // wire form for Project — match smooth_narc::judge::Scope + // exactly. + assert_eq!(serde_json::to_string(&AutoApprove::Project).unwrap(), "\"project\""); + let m: AutoApprove = serde_json::from_str("\"session\"").unwrap(); + assert_eq!(m, AutoApprove::Session); + } + + #[test] + fn discover_scenarios_returns_sorted_list() { + let tmp = tempfile::tempdir().expect("tmp"); + let root = tmp.path(); + for name in ["zebra", "alpha", "middle"] { + let dir = root.join(name); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join("scenario.toml"), + format!( + r#" +[meta] +title = "{name}" +description = "x" + +[[turns]] +input = "hi" +"# + ), + ) + .unwrap(); + } + // A non-scenario directory (no scenario.toml) must be + // silently skipped, not error out. + std::fs::create_dir_all(root.join("_helpers")).unwrap(); + + let found = discover_scenarios(root).expect("discover"); + let names: Vec<&str> = found.iter().map(|(n, _, _)| n.as_str()).collect(); + assert_eq!(names, vec!["alpha", "middle", "zebra"]); + } + + #[test] + fn unknown_assertion_kind_rejected() { + let raw = r#" +[meta] +title = "x" +description = "y" + +[[turns]] +input = "hi" + +[[turns.assert]] +kind = "do_a_barrel_roll" +"#; + assert!(parse_scenario(raw).is_err()); + } +} diff --git a/crates/smooth-bench/src/score.rs b/crates/smooth-bench/src/score.rs new file mode 100644 index 000000000..92a58f296 --- /dev/null +++ b/crates/smooth-bench/src/score.rs @@ -0,0 +1,321 @@ +//! `Score` — aggregated result of a curated aider-polyglot sweep. +//! +//! This is the JSON artifact "The Line" publishes with every Smooth +//! release. Single number across 6 languages × 20 tasks, plus +//! per-language breakdown, cost, duration, and the budget-cap flag. +//! +//! A `Score` is *also* what gets emitted when a `--pr` CI-gate run +//! cuts short — the fewer-tasks sample has the same shape as the +//! authoritative `--release` sample, so downstream tooling (README +//! badge, release notes, PR bot) doesn't care which gate produced it. +//! +//! Serde round-trips losslessly; see `score_serde_roundtrip` in +//! tests for the invariant. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +/// Per-language pass breakdown inside a `Score`. +/// +/// `pass_rate` is `tasks_green / tasks_attempted`, with 0/0 returning +/// 0.0 (never NaN — downstream consumers serialise and compare these +/// numbers, and NaN breaks both). +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct LanguageScore { + pub pass_rate: f64, + pub tasks_attempted: u32, + pub tasks_green: u32, +} + +impl LanguageScore { + /// Build a `LanguageScore` from raw counts. Handles the 0/0 case + /// by returning `pass_rate = 0.0` rather than producing NaN. + #[must_use] + pub fn from_counts(tasks_attempted: u32, tasks_green: u32) -> Self { + let pass_rate = if tasks_attempted == 0 { + 0.0 + } else { + f64::from(tasks_green) / f64::from(tasks_attempted) + }; + Self { + pass_rate, + tasks_attempted, + tasks_green, + } + } +} + +impl Score { + /// Render a human-readable summary of the Score. Shared between + /// `smooth-bench score` (no `--output` → stdout) and `th bench + /// score` (the baked-in Line the shipped binary carries) so both + /// surfaces print identically. + #[must_use] + pub fn render_table(&self) -> String { + use std::fmt::Write; + let mut out = String::new(); + let _ = writeln!(out, "The Line — smooth-bench score"); + let _ = writeln!(out, " smooth version: {}", self.smooth_version); + let _ = writeln!(out, " commit: {}", self.commit_sha); + let _ = writeln!(out, " ran at: {}", self.ran_at.to_rfc3339()); + let _ = writeln!( + out, + " overall pass rate: {:.1}% ({}/{} tasks green)", + self.overall_pass_rate * 100.0, + self.tasks_green, + self.tasks_attempted + ); + if self.tasks_inconclusive > 0 { + let real_attempts = self.tasks_attempted.saturating_sub(self.tasks_inconclusive); + let real_pass_rate = if real_attempts == 0 { + 0.0 + } else { + f64::from(self.tasks_green.saturating_sub(self.tasks_inconclusive)) / f64::from(real_attempts) + }; + let _ = writeln!( + out, + " inconclusive: {} (HTTP-timeout starter passes; not counted as real)", + self.tasks_inconclusive + ); + let _ = writeln!( + out, + " real-attempt rate: {:.1}% ({}/{} excluding inconclusive)", + real_pass_rate * 100.0, + self.tasks_green.saturating_sub(self.tasks_inconclusive), + real_attempts + ); + } + let _ = writeln!(out, " cost: ${:.4} (cap ${:.2})", self.cost_usd, self.budget_usd_cap); + if self.budget_usd_hit { + let _ = writeln!(out, " BUDGET CAP HIT — score is partial"); + } + let _ = writeln!(out, " median task time: {} ms", self.median_task_ms); + let _ = writeln!(out); + let _ = writeln!(out, " by language:"); + for (lang, ls) in &self.by_language { + let _ = writeln!(out, " {lang:<12} {:.1}% ({}/{})", ls.pass_rate * 100.0, ls.tasks_green, ls.tasks_attempted); + } + out + } +} + +/// Aggregate score emitted by `smooth-bench score`. +/// +/// Written to stdout (or `--output `) as pretty-printed JSON +/// when the output path ends in `.json`. Otherwise a human table is +/// rendered; the JSON can still be recovered from +/// `~/.smooth/bench-runs//score.json`. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Score { + pub smooth_version: String, + pub commit_sha: String, + pub ran_at: chrono::DateTime, + pub overall_pass_rate: f64, + pub by_language: BTreeMap, + pub tasks_attempted: u32, + pub tasks_green: u32, + /// Tasks where the harness can't tell if a real attempt was made — + /// the chat-agent dispatch hit `SMOOTH_BENCH_CHAT_HTTP_TIMEOUT_S` + /// before returning a pearl id, no `[METRICS]` comment landed, and + /// the workspace wasn't mutated. Polyglot starter code happens to + /// satisfy the test suite for some tasks (e.g. rust/accumulate), + /// which would otherwise score as PASS without any model work being + /// done. Counted separately so the headline pass rate reflects + /// real attempts. + #[serde(default)] + pub tasks_inconclusive: u32, + pub cost_usd: f64, + pub median_task_ms: u64, + pub budget_usd_cap: f64, + pub budget_usd_hit: bool, +} + +/// Compute the median of `values` in milliseconds. Empty input +/// returns 0 (there's no meaningful "median of nothing", and 0 is +/// the value that makes the downstream display harmless). +/// +/// Even-length inputs average the two middle values and truncate +/// toward zero — we're milliseconds, a half-millisecond delta is noise. +#[must_use] +pub fn median_ms(values: &[u64]) -> u64 { + if values.is_empty() { + return 0; + } + let mut sorted: Vec = values.to_vec(); + sorted.sort_unstable(); + let n = sorted.len(); + if n.is_multiple_of(2) { + // average of the two middle elements + let lo = sorted[n / 2 - 1]; + let hi = sorted[n / 2]; + // `u64 + u64 / 2` — neither value will come close to u64::MAX + // in any realistic run, but be safe and use wrapping-averaging + // via `((a ^ b) >> 1) + (a & b)` so we don't overflow. + ((lo ^ hi) >> 1).wrapping_add(lo & hi) + } else { + sorted[n / 2] + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + #[test] + fn language_score_pass_rate_handles_zero_zero() { + let s = LanguageScore::from_counts(0, 0); + assert_eq!(s.pass_rate, 0.0); + assert!(!s.pass_rate.is_nan()); + } + + #[test] + fn language_score_pass_rate_basic() { + let s = LanguageScore::from_counts(20, 17); + assert!((s.pass_rate - 0.85).abs() < 1e-9); + } + + #[test] + fn language_score_pass_rate_all_green() { + let s = LanguageScore::from_counts(20, 20); + assert!((s.pass_rate - 1.0).abs() < 1e-9); + } + + #[test] + fn language_score_pass_rate_all_red() { + let s = LanguageScore::from_counts(20, 0); + assert_eq!(s.pass_rate, 0.0); + } + + #[test] + fn median_empty_returns_zero() { + assert_eq!(median_ms(&[]), 0); + } + + #[test] + fn median_single_entry() { + assert_eq!(median_ms(&[42]), 42); + } + + #[test] + fn median_odd_count_takes_middle() { + assert_eq!(median_ms(&[3, 1, 2]), 2); + assert_eq!(median_ms(&[10, 50, 20, 30, 40]), 30); + } + + #[test] + fn median_even_count_averages_middle_two() { + assert_eq!(median_ms(&[1, 2, 3, 4]), 2); // (2+3)/2 = 2 (trunc) + assert_eq!(median_ms(&[10, 20]), 15); + assert_eq!(median_ms(&[100, 200, 300, 400]), 250); + } + + #[test] + fn median_unsorted_input_still_correct() { + // Should sort before computing — don't trust input order. + assert_eq!(median_ms(&[300, 100, 200]), 200); + } + + #[test] + fn score_serde_roundtrip() { + let mut by_language = BTreeMap::new(); + by_language.insert("python".to_string(), LanguageScore::from_counts(20, 17)); + by_language.insert("rust".to_string(), LanguageScore::from_counts(20, 15)); + + let original = Score { + smooth_version: "0.42.1".to_string(), + commit_sha: "abc123def456".to_string(), + ran_at: chrono::Utc.with_ymd_and_hms(2026, 4, 23, 12, 34, 56).unwrap(), + overall_pass_rate: 0.8, + by_language, + tasks_attempted: 40, + tasks_green: 32, + tasks_inconclusive: 0, + cost_usd: 4.23, + median_task_ms: 15_000, + budget_usd_cap: 10.0, + budget_usd_hit: false, + }; + + let json = serde_json::to_string(&original).expect("serialize"); + let decoded: Score = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(decoded, original); + } + + #[test] + fn score_serde_roundtrip_with_budget_hit() { + // The partial-result case — budget hit mid-run. + let mut by_language = BTreeMap::new(); + by_language.insert("python".to_string(), LanguageScore::from_counts(5, 3)); + + let original = Score { + smooth_version: "0.42.1".to_string(), + commit_sha: "abc123".to_string(), + ran_at: chrono::Utc.with_ymd_and_hms(2026, 4, 23, 0, 0, 0).unwrap(), + overall_pass_rate: 0.6, + by_language, + tasks_attempted: 5, + tasks_green: 3, + tasks_inconclusive: 0, + cost_usd: 10.07, + median_task_ms: 8_000, + budget_usd_cap: 10.0, + budget_usd_hit: true, + }; + + let json = serde_json::to_string_pretty(&original).expect("serialize"); + let decoded: Score = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(decoded, original); + assert!(decoded.budget_usd_hit); + } + + #[test] + fn render_table_separates_real_from_inconclusive() { + // 18 attempts, 9 PASS, 5 of those are HTTP-timeout starter + // passes. Real-attempt pass rate must drop accordingly. + let s = Score { + smooth_version: "test".into(), + commit_sha: "abc".into(), + ran_at: chrono::Utc.with_ymd_and_hms(2026, 5, 3, 0, 0, 0).unwrap(), + overall_pass_rate: 9.0 / 18.0, + by_language: BTreeMap::new(), + tasks_attempted: 18, + tasks_green: 9, + tasks_inconclusive: 5, + cost_usd: 0.42, + median_task_ms: 30_000, + budget_usd_cap: 10.0, + budget_usd_hit: false, + }; + let table = s.render_table(); + // Headline still reports raw pass rate. + assert!(table.contains("9/18"), "{table}"); + // Inconclusive line surfaces the suspect count. + assert!(table.contains("inconclusive: 5"), "{table}"); + // Real-attempt rate excludes inconclusive: (9-5) / (18-5) = 4/13. + assert!(table.contains("real-attempt rate"), "{table}"); + assert!(table.contains("(4/13"), "{table}"); + } + + #[test] + fn render_table_omits_inconclusive_block_when_zero() { + let s = Score { + smooth_version: "test".into(), + commit_sha: "abc".into(), + ran_at: chrono::Utc.with_ymd_and_hms(2026, 5, 3, 0, 0, 0).unwrap(), + overall_pass_rate: 0.5, + by_language: BTreeMap::new(), + tasks_attempted: 4, + tasks_green: 2, + tasks_inconclusive: 0, + cost_usd: 0.0, + median_task_ms: 0, + budget_usd_cap: 10.0, + budget_usd_hit: false, + }; + let table = s.render_table(); + assert!(!table.contains("inconclusive"), "{table}"); + assert!(!table.contains("real-attempt rate"), "{table}"); + } +} diff --git a/crates/smooth-bench/src/sweep.rs b/crates/smooth-bench/src/sweep.rs new file mode 100644 index 000000000..977dd5e76 --- /dev/null +++ b/crates/smooth-bench/src/sweep.rs @@ -0,0 +1,590 @@ +//! Multi-task sweep runner for `smooth-bench score`. +//! +//! Wraps the single-task `run_aider_polyglot` runner in a loop over +//! the curated task list, aggregates per-task `BenchResult`s into an +//! aggregate `Score`, and honours the `--budget-usd` hard cap. +//! +//! Streams per-task results to stdout as they complete — the final +//! aggregate `Score` is emitted at the end so operators see progress +//! during the (potentially multi-hour) authoritative run. +//! +//! The runner is parameterised on a `TaskRunner` trait so unit tests +//! can exercise aggregation + budget-cap logic without an LLM. + +use std::collections::BTreeMap; +use std::time::Instant; + +use async_trait::async_trait; + +use crate::curated::CuratedList; +use crate::score::{median_ms, LanguageScore, Score}; +use crate::{BenchOpts, BenchResult, PolyglotLang}; + +/// Single-run result needed by the sweep. A thin projection of +/// `BenchResult` so unit tests don't have to fabricate every field +/// of the full struct. +#[derive(Debug, Clone)] +pub struct TaskOutcome { + pub solved: bool, + pub cost_usd: f64, + pub duration_ms: u64, + /// True when the harness can't tell whether a real attempt was made + /// (starter code that happens to satisfy the suite with no model + /// work). The canonical driver runs the turn to completion before + /// scoring, so this is always `false` today — the field is retained + /// for the aggregate `Score` shape and future re-detection. + pub inconclusive: bool, +} + +/// Injection point for the per-task runner. Production implementation +/// (`engine::EngineTaskRunner`) boots an engine with its workspace at the +/// task's scratch dir and drives one canonical turn; unit tests provide a +/// canned-response implementation to exercise aggregation + budget-cap +/// logic without hitting the network. +#[async_trait] +pub trait TaskRunner: Send + Sync { + async fn run_one(&self, lang: PolyglotLang, task: &str, opts: &BenchOpts) -> anyhow::Result; +} + +/// Map a raw single-task result into a `TaskOutcome`. Helpful for +/// callers who already ran the task and want to feed the result into +/// the aggregator directly. +#[must_use] +pub fn outcome_from_result(r: &BenchResult) -> TaskOutcome { + TaskOutcome { + solved: r.solved, + cost_usd: r.cost_usd, + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss, clippy::cast_precision_loss)] + duration_ms: (r.duration_s * 1000.0).max(0.0) as u64, + inconclusive: false, + } +} + +/// Which `Score` "gate" a sweep corresponds to. `Release` is the +/// authoritative 20×6=120 run. `Pr` is the small CI-gate sample. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SweepGate { + Release, + /// PR gate: `tasks_per_language` is the fixed per-lang sample + /// size (typically 3). + Pr { + tasks_per_language: usize, + }, +} + +/// Configuration for a sweep run. `budget_usd_cap` is a HARD cap — +/// the sweep aborts as soon as the running total exceeds it, with +/// `budget_usd_hit: true` in the emitted `Score`. +#[derive(Debug, Clone)] +pub struct SweepConfig { + pub gate: SweepGate, + pub budget_usd_cap: f64, + pub smooth_version: String, + pub commit_sha: String, + /// Per-task options forwarded to the single-task runner. The + /// sweep overrides `budget_usd` per task based on remaining + /// headroom under the sweep-level cap. + pub task_opts: BenchOpts, +} + +/// Result of a sweep: the aggregate `Score` plus the raw per-task +/// outcomes (useful for detailed reporting or debugging — the +/// top-level CLI emits only the `Score`). +#[derive(Debug, Clone)] +pub struct SweepRun { + pub score: Score, + pub per_task: Vec<(PolyglotLang, String, TaskOutcome)>, +} + +/// Streaming hook: called once per task after it completes, before +/// the next one starts. Default implementation prints a one-line +/// summary to stdout. Kept as a trait so tests can capture events. +pub trait SweepObserver: Send { + fn on_task_complete(&mut self, idx: usize, total: usize, lang: PolyglotLang, task: &str, outcome: &TaskOutcome, cumulative_cost: f64); + fn on_budget_hit(&mut self, cumulative_cost: f64, cap: f64); +} + +/// Default observer: prints to stdout. +pub struct StdoutObserver; + +impl SweepObserver for StdoutObserver { + fn on_task_complete(&mut self, idx: usize, total: usize, lang: PolyglotLang, task: &str, outcome: &TaskOutcome, cumulative_cost: f64) { + let tag = if outcome.solved { "PASS" } else { "FAIL" }; + println!( + "[{idx:>3}/{total:>3}] {tag} {lang:<10} {task:<28} {ms:>6}ms ${cost:.4} (total ${cum:.4})", + lang = lang.dataset_dir(), + task = task, + ms = outcome.duration_ms, + cost = outcome.cost_usd, + cum = cumulative_cost, + ); + } + + fn on_budget_hit(&mut self, cumulative_cost: f64, cap: f64) { + eprintln!("budget cap reached: cumulative ${cumulative_cost:.4} exceeds ${cap:.4} — aborting sweep early"); + } +} + +/// Run a curated sweep end-to-end. Emits streaming task results to +/// the observer; returns the aggregate `Score` + per-task outcomes +/// at the end. +/// +/// Budget semantics: before kicking off task N, we compute +/// `remaining = cap - cumulative_cost`. If `remaining <= 0` we abort +/// *without* running task N and emit a partial `Score` with +/// `budget_usd_hit = true`. Tasks that finish and put us over the +/// cap cause the NEXT task to be skipped — we never interrupt a +/// running task mid-flight. +/// +/// # Errors +/// Setup errors (e.g. missing providers config on the very first +/// call) propagate. Per-task LLM failures are captured in the +/// outcome's `solved = false` and DO NOT abort the sweep. +pub async fn run_sweep(curated: &CuratedList, runner: &R, cfg: &SweepConfig, observer: &mut O) -> anyhow::Result +where + R: TaskRunner + ?Sized, + O: SweepObserver, +{ + let pairs = curated_pairs(curated, cfg.gate); + let total = pairs.len(); + let mut per_task: Vec<(PolyglotLang, String, TaskOutcome)> = Vec::with_capacity(total); + let mut durations_ms: Vec = Vec::with_capacity(total); + let mut cumulative_cost = 0.0_f64; + let mut budget_hit = false; + + let sweep_started = Instant::now(); + + for (idx, (lang, task)) in pairs.iter().enumerate() { + if cumulative_cost >= cfg.budget_usd_cap { + budget_hit = true; + observer.on_budget_hit(cumulative_cost, cfg.budget_usd_cap); + break; + } + + // Scale the per-task budget so a single task can't blow the + // whole remaining headroom. This is best-effort — the + // single-task runner's own budget is the authoritative stop. + let remaining = cfg.budget_usd_cap - cumulative_cost; + let mut task_opts = cfg.task_opts.clone(); + task_opts.budget_usd = Some(task_opts.budget_usd.map_or(remaining, |b| b.min(remaining))); + + let outcome = match runner.run_one(*lang, task, &task_opts).await { + Ok(o) => o, + Err(e) => { + // Treat setup / transport failures as unsolved-with-zero-cost + // so one dead task doesn't kill the whole sweep. Log the + // error via the observer path (stdout for the default + // observer, stderr for the error). + eprintln!("task {}/{} ({}/{}): runner error: {e:#}", idx + 1, total, lang.dataset_dir(), task); + TaskOutcome { + solved: false, + cost_usd: 0.0, + duration_ms: 0, + inconclusive: false, + } + } + }; + + cumulative_cost += outcome.cost_usd; + durations_ms.push(outcome.duration_ms); + observer.on_task_complete(idx + 1, total, *lang, task, &outcome, cumulative_cost); + per_task.push((*lang, task.clone(), outcome)); + } + + let score = aggregate( + &per_task, + AggregateInputs { + smooth_version: cfg.smooth_version.clone(), + commit_sha: cfg.commit_sha.clone(), + cost_usd: cumulative_cost, + durations_ms: &durations_ms, + budget_usd_cap: cfg.budget_usd_cap, + budget_usd_hit: budget_hit, + }, + ); + + // Touch the sweep-wall-clock so clippy doesn't warn about the + // unused Instant — it's kept for future reporting. + let _ = sweep_started.elapsed(); + + Ok(SweepRun { score, per_task }) +} + +/// Pick out the `(lang, task)` pairs to run based on the gate. +/// +/// For `Release` we run every pair. For `Pr` we take the first +/// `tasks_per_language` tasks per language in the order they were +/// curated — stable and cheap to reason about. +fn curated_pairs(curated: &CuratedList, gate: SweepGate) -> Vec<(PolyglotLang, String)> { + match gate { + SweepGate::Release => curated.iter_all().map(|(l, t)| (l, t.to_string())).collect(), + SweepGate::Pr { tasks_per_language } => { + let mut out = Vec::new(); + for lang in [ + PolyglotLang::Python, + PolyglotLang::Rust, + PolyglotLang::Go, + PolyglotLang::Javascript, + PolyglotLang::Java, + PolyglotLang::Cpp, + ] { + for task in curated.tasks_for(lang).iter().take(tasks_per_language) { + out.push((lang, task.clone())); + } + } + out + } + } +} + +struct AggregateInputs<'a> { + smooth_version: String, + commit_sha: String, + cost_usd: f64, + durations_ms: &'a [u64], + budget_usd_cap: f64, + budget_usd_hit: bool, +} + +fn aggregate(per_task: &[(PolyglotLang, String, TaskOutcome)], inputs: AggregateInputs<'_>) -> Score { + let mut by_lang_counts: BTreeMap = BTreeMap::new(); + let mut tasks_attempted: u32 = 0; + let mut tasks_green: u32 = 0; + let mut tasks_inconclusive: u32 = 0; + for (lang, _task, outcome) in per_task { + let entry = by_lang_counts.entry(*lang).or_insert((0, 0)); + entry.0 += 1; + tasks_attempted += 1; + if outcome.solved { + entry.1 += 1; + tasks_green += 1; + } + if outcome.inconclusive { + tasks_inconclusive += 1; + } + } + + let overall_pass_rate = if tasks_attempted == 0 { + 0.0 + } else { + f64::from(tasks_green) / f64::from(tasks_attempted) + }; + + let by_language: BTreeMap = by_lang_counts + .into_iter() + .map(|(lang, (attempted, green))| (lang.dataset_dir().to_string(), LanguageScore::from_counts(attempted, green))) + .collect(); + + Score { + smooth_version: inputs.smooth_version, + commit_sha: inputs.commit_sha, + ran_at: chrono::Utc::now(), + overall_pass_rate, + by_language, + tasks_attempted, + tasks_green, + tasks_inconclusive, + cost_usd: inputs.cost_usd, + median_task_ms: median_ms(inputs.durations_ms), + budget_usd_cap: inputs.budget_usd_cap, + budget_usd_hit: inputs.budget_usd_hit, + } +} + +/// Resolve the current commit SHA via `git rev-parse HEAD`. Returns +/// `"unknown"` if git fails (e.g. not a git checkout) — we'd rather +/// publish a Score tagged `unknown` than abort the release. +#[must_use] +pub fn current_commit_sha() -> String { + // Clear inherited git env. Under the git pre-push hook (and some CI + // checkouts) GIT_DIR / GIT_INDEX_FILE / GIT_WORK_TREE / GIT_PREFIX / + // GIT_COMMON_DIR point at the hook's context, which makes a child + // `git rev-parse HEAD` print nothing (exit 0, empty stdout) instead + // of the real sha — failing the non-empty assertion below and, more + // importantly, tagging release Scores with "". Stripping them lets + // git rediscover the repo from cwd. Pearl th-e2cbc9. + let mut cmd = std::process::Command::new("git"); + cmd.args(["rev-parse", "HEAD"]); + for var in ["GIT_DIR", "GIT_INDEX_FILE", "GIT_WORK_TREE", "GIT_PREFIX", "GIT_COMMON_DIR"] { + cmd.env_remove(var); + } + let Ok(out) = cmd.output() else { + return "unknown".to_string(); + }; + if !out.status.success() { + return "unknown".to_string(); + } + let sha = String::from_utf8_lossy(&out.stdout).trim().to_string(); + // Never return empty — callers and the release pipeline expect a + // tag, and "" silently corrupts Score provenance. Fall back to the + // same sentinel used for the git-failure path. + if sha.is_empty() { + return "unknown".to_string(); + } + sha +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + /// A canned runner: answers from a pre-programmed queue of + /// `(solved, cost, duration_ms)` tuples. If the queue is + /// exhausted it returns an unsolved $0 task so tests don't panic + /// mid-sweep. + struct CannedRunner { + queue: Mutex>, + } + + impl CannedRunner { + fn new(queue: Vec<(bool, f64, u64)>) -> Self { + Self { queue: Mutex::new(queue) } + } + } + + #[async_trait] + impl TaskRunner for CannedRunner { + async fn run_one(&self, _lang: PolyglotLang, _task: &str, _opts: &BenchOpts) -> anyhow::Result { + let mut q = self.queue.lock().unwrap(); + if q.is_empty() { + return Ok(TaskOutcome { + solved: false, + cost_usd: 0.0, + duration_ms: 0, + inconclusive: false, + }); + } + let (solved, cost_usd, duration_ms) = q.remove(0); + Ok(TaskOutcome { + solved, + cost_usd, + duration_ms, + inconclusive: false, + }) + } + } + + struct CapturingObserver { + events: Vec, + budget_hit: Option<(f64, f64)>, + } + + impl CapturingObserver { + fn new() -> Self { + Self { + events: Vec::new(), + budget_hit: None, + } + } + } + + impl SweepObserver for CapturingObserver { + fn on_task_complete(&mut self, idx: usize, total: usize, lang: PolyglotLang, task: &str, outcome: &TaskOutcome, cumulative_cost: f64) { + self.events.push(format!( + "{idx}/{total} {}/{task} solved={} cost={} cum={cumulative_cost:.4}", + lang.dataset_dir(), + outcome.solved, + outcome.cost_usd, + )); + } + + fn on_budget_hit(&mut self, cumulative_cost: f64, cap: f64) { + self.budget_hit = Some((cumulative_cost, cap)); + } + } + + fn tiny_curated_pr_pairs() -> (CuratedList, SweepConfig) { + let list = CuratedList::default_embedded().unwrap(); + let cfg = SweepConfig { + // PR gate with 1 task per language = exactly 6 tasks. + gate: SweepGate::Pr { tasks_per_language: 1 }, + budget_usd_cap: 10.0, + smooth_version: "0.0.0-test".to_string(), + commit_sha: "deadbeef".to_string(), + task_opts: BenchOpts::default(), + }; + (list, cfg) + } + + #[tokio::test] + async fn sweep_aggregates_per_language_correctly() { + // 6 tasks in PR gate (1 per lang × 6 langs). Python solved, + // everyone else fails. Expected: overall_pass_rate = 1/6. + let (list, cfg) = tiny_curated_pr_pairs(); + let runner = CannedRunner::new(vec![ + (true, 0.1, 1000), // python + (false, 0.2, 2000), // rust + (false, 0.15, 1500), // go + (false, 0.2, 2500), // javascript + (false, 0.3, 3000), // java + (false, 0.05, 500), // cpp + ]); + let mut obs = CapturingObserver::new(); + let run = run_sweep(&list, &runner, &cfg, &mut obs).await.unwrap(); + + assert_eq!(run.score.tasks_attempted, 6); + assert_eq!(run.score.tasks_green, 1); + assert!((run.score.overall_pass_rate - 1.0 / 6.0).abs() < 1e-9); + assert!(!run.score.budget_usd_hit); + + // Per-language shape: exactly 6 entries, each with + // attempted=1. Only python is green. + assert_eq!(run.score.by_language.len(), 6); + assert_eq!(run.score.by_language["python"].tasks_green, 1); + assert_eq!(run.score.by_language["rust"].tasks_green, 0); + assert_eq!(run.score.by_language["cpp"].tasks_green, 0); + + // Streaming observer saw every task. + assert_eq!(obs.events.len(), 6); + assert!(obs.budget_hit.is_none()); + } + + #[tokio::test] + async fn sweep_budget_cap_aborts_on_third_task() { + // 3 tasks queued with costs [3.0, 4.0, 4.0]. Budget = 5.0. + // + // Timeline: + // Before task 1: cum=0.0 < 5.0, run → cum=3.0 + // Before task 2: cum=3.0 < 5.0, run → cum=7.0 (OVER — but task 2 finishes) + // Before task 3: cum=7.0 >= 5.0 → abort, budget_usd_hit=true + // + // Expected: 2 tasks recorded, budget_usd_hit=true, Score still emitted. + let (list, _) = tiny_curated_pr_pairs(); + let cfg = SweepConfig { + gate: SweepGate::Pr { tasks_per_language: 1 }, // 6 pairs, but we'll cap short + budget_usd_cap: 5.0, + smooth_version: "0.0.0-test".to_string(), + commit_sha: "deadbeef".to_string(), + task_opts: BenchOpts::default(), + }; + let runner = CannedRunner::new(vec![ + (true, 3.0, 1000), + (true, 4.0, 2000), + (true, 4.0, 3000), // must never run + ]); + let mut obs = CapturingObserver::new(); + let run = run_sweep(&list, &runner, &cfg, &mut obs).await.unwrap(); + + assert_eq!(run.score.tasks_attempted, 2, "third task must not run"); + assert_eq!(run.score.tasks_green, 2); + assert!(run.score.budget_usd_hit, "budget cap flag must be set"); + assert!((run.score.cost_usd - 7.0).abs() < 1e-9); + assert_eq!(obs.events.len(), 2); + let (cum, cap) = obs.budget_hit.expect("on_budget_hit fires"); + assert!((cum - 7.0).abs() < 1e-9); + assert!((cap - 5.0).abs() < 1e-9); + } + + #[tokio::test] + async fn sweep_partial_results_shape_matches_full_run() { + // Partial-run Score shape must JSON-round-trip just like a + // complete run. Use a queue whose first task already blows + // the cap — the sweep then aborts before task 2. + let (list, _) = tiny_curated_pr_pairs(); + let cfg = SweepConfig { + gate: SweepGate::Pr { tasks_per_language: 1 }, + budget_usd_cap: 1.0, + smooth_version: "0.0.0-test".to_string(), + commit_sha: "deadbeef".to_string(), + task_opts: BenchOpts::default(), + }; + // Task 1 costs $2.50 (already over the $1 cap). Task 2 must + // never run. + let runner = CannedRunner::new(vec![(true, 2.5, 1500), (true, 0.1, 500)]); + let mut obs = CapturingObserver::new(); + let run = run_sweep(&list, &runner, &cfg, &mut obs).await.unwrap(); + + assert_eq!(run.score.tasks_attempted, 1, "second task must not run"); + assert!(run.score.budget_usd_hit); + assert!((run.score.cost_usd - 2.5).abs() < 1e-9); + assert_eq!(run.score.median_task_ms, 1500); + + // JSON round-trip preserves the partial-result shape. + let json = serde_json::to_string(&run.score).unwrap(); + let decoded: Score = serde_json::from_str(&json).unwrap(); + assert_eq!(decoded, run.score); + } + + #[tokio::test] + async fn sweep_zero_budget_runs_no_tasks() { + // Edge case: cap=0 → abort before task 1. Score has zero + // tasks attempted and `budget_usd_hit: true`. + let (list, _) = tiny_curated_pr_pairs(); + let cfg = SweepConfig { + gate: SweepGate::Pr { tasks_per_language: 1 }, + budget_usd_cap: 0.0, + smooth_version: "0.0.0-test".to_string(), + commit_sha: "deadbeef".to_string(), + task_opts: BenchOpts::default(), + }; + let runner = CannedRunner::new(vec![(true, 0.1, 500)]); + let mut obs = CapturingObserver::new(); + let run = run_sweep(&list, &runner, &cfg, &mut obs).await.unwrap(); + + assert_eq!(run.score.tasks_attempted, 0); + assert!(run.score.budget_usd_hit); + assert_eq!(run.score.overall_pass_rate, 0.0); + assert_eq!(run.score.median_task_ms, 0); + } + + #[tokio::test] + async fn sweep_gate_pr_limits_total_tasks() { + // PR gate with 2 per language → exactly 12 tasks (not 120). + let list = CuratedList::default_embedded().unwrap(); + let cfg = SweepConfig { + gate: SweepGate::Pr { tasks_per_language: 2 }, + budget_usd_cap: 100.0, + smooth_version: "0".to_string(), + commit_sha: "x".to_string(), + task_opts: BenchOpts::default(), + }; + let runner = CannedRunner::new(vec![(false, 0.0, 1); 20]); // plenty + let mut obs = CapturingObserver::new(); + let run = run_sweep(&list, &runner, &cfg, &mut obs).await.unwrap(); + assert_eq!(run.score.tasks_attempted, 12); + } + + #[tokio::test] + async fn sweep_runner_error_is_recorded_as_failure_not_abort() { + struct FailFirstRunner { + called: Mutex, + } + #[async_trait] + impl TaskRunner for FailFirstRunner { + async fn run_one(&self, _l: PolyglotLang, _t: &str, _o: &BenchOpts) -> anyhow::Result { + let mut n = self.called.lock().unwrap(); + *n += 1; + if *n == 1 { + Err(anyhow::anyhow!("pretend network blew up")) + } else { + Ok(TaskOutcome { + solved: true, + cost_usd: 0.1, + duration_ms: 500, + inconclusive: false, + }) + } + } + } + let (list, mut cfg) = tiny_curated_pr_pairs(); + cfg.budget_usd_cap = 100.0; + let runner = FailFirstRunner { called: Mutex::new(0) }; + let mut obs = CapturingObserver::new(); + let run = run_sweep(&list, &runner, &cfg, &mut obs).await.unwrap(); + + assert_eq!(run.score.tasks_attempted, 6); + // First task counted as failed-with-no-cost. + assert_eq!(run.score.tasks_green, 5); + assert!(!run.score.budget_usd_hit); + } + + #[test] + fn current_commit_sha_returns_something_non_empty() { + // In a git checkout it should be a 40-char hex string; in a + // tarball checkout it's "unknown". Either is fine; just + // assert it's non-empty and doesn't panic. + let sha = current_commit_sha(); + assert!(!sha.is_empty()); + } +} diff --git a/crates/smooth-code/src/client.rs b/crates/smooth-code/src/client.rs index 7d45268ac..fcd135f07 100644 --- a/crates/smooth-code/src/client.rs +++ b/crates/smooth-code/src/client.rs @@ -430,6 +430,26 @@ impl BigSmoothClient { /// Ensure Big Smooth is running, starting it if needed. async fn ensure_server(&self) -> anyhow::Result<()> { + // A live socket at the target address IS the readiness signal for an + // externally-managed engine. The polyglot smooth-operator LocalServers + // (go/ts/python/dotnet) serve `/ws` but no `/health`, so the HTTP probe + // below would wrongly conclude "not started" and try to `th up` a Rust + // daemon — never reaching the real WS connect. Short-circuit on a live + // socket first; the `/health`+autostart path still runs for `th code` + // when nothing is listening yet. (bench engine axis, th-4c3e2d) + { + use std::net::ToSocketAddrs; + if let Some(addr) = self.url.split("://").nth(1).and_then(|s| s.split('/').next()) { + if let Ok(mut it) = addr.to_socket_addrs() { + if let Some(sa) = it.next() { + if std::net::TcpStream::connect_timeout(&sa, Duration::from_secs(2)).is_ok() { + return Ok(()); + } + } + } + } + } + let health_url = format!("{}/health", self.url); let client = reqwest::Client::builder().timeout(Duration::from_secs(2)).build()?;