th-4c3e2d: Smooth Operator engine-parity benchmark (engine × model axis)

.changeset/engine-parity-bench.md

@@ -0,0 +1,31 @@
+---
+"@smooai/smooth": minor
+---
+
+Restore the `smooth-bench` crate (Aider-Polyglot slice) and add an engine-parity axis.
+
+The mature bench crate was deleted in the microVM/daemon rewrite. This restores
+the focused Aider-Polyglot slice — the `run_aider_polyglot` single-task runner,
+the curated-task sweep, the WS chat driver, scoring, and auto-approve — against
+current `main` (deps now resolve to the published `smooai-smooth-operator-core`
+engine plus the in-tree `smooth-cast`/`smooth-code`/`smooth-pearls`). The
+SWE-bench / replay / research / cleanup / TUI-driver scorers were left out.
+
+New engine-parity benchmark (pearl th-4c3e2d): `smooth-bench score` runs the
+curated aider-polyglot suite through each of the five smooth-operator
+`LocalServer` implementations — Rust, Go, TypeScript, Python, .NET — scoring per
+engine (and per model).
+
+- `--engine <rust|go|ts|python|dotnet>` (repeatable, default all) and
+  `--model <id>` (repeatable, default `deepseek-v4-flash`).
+- Each engine is booted the way `scripts/operator-serve.sh` does (uniform
+  env contract: `SMOOAI_GATEWAY_URL/KEY`, `SMOOTH_PERSONA`, `SMOOAI_MODEL`, plus
+  the per-engine bind var), the sweep runs against it over the canonical WS
+  protocol, then it's torn down before the next cell.
+- Per-engine×model results carry the engine + model dimensions and emit in the
+  JSON-lines + summary-table format.
+
+The matrix runner is parameterised on an `EngineBooter` trait, so the
+engine×model aggregation and the engine→boot-command mapping are unit-tested
+without a live LLM or real servers. A real scoring run needs `SMOOAI_GATEWAY_KEY`
+on the runner.

.changeset/th-7232fb-bench-canonical-driver.md

@@ -0,0 +1,37 @@
+---
+'@smooai/smooth': patch
+---
+
+smooth-bench: rewire the live-drive path onto the canonical smooth-operator
+`LocalServer` WebSocket protocol so the engine-parity sweep actually scores a
+real turn.
+
+The old default driver (`chat_driver`) assumed the deleted microVM "create a
+pearl + dispatch a teammate" model, and the legacy `SMOOTH_BENCH_LEGACY_DIRECT`
+path spoke the retired bespoke `/ws` handshake (waited for a `Connected` event
+the engines never send → "Timed out waiting for Connected"). Both are gone.
+
+- **New `canonical_driver`** speaks the schema-driven protocol every engine
+  now uses (`create_conversation_session` → `immediate_response.sessionId` →
+  `send_message` → drain to `eventual_response`), modeled on the daemon's
+  `OperatorTurnDriver::drive_once`. It parses `stream_chunk` tool-result events
+  into the `BenchResult`'s tool-call records and does a best-effort cost scan
+  (the polyglot servers don't surface cost → `$0`, noted). Fully unit-tested
+  (message construction + event classification, no live server needed).
+- **Workspace-per-task boot**: the engine is now booted *per task* with its
+  workspace pointed at that task's scratch `work_dir` — rust via
+  `SMOOTH_WORKSPACE`; go/ts/python/dotnet (no workspace env) with
+  `cwd = work_dir`. Go is launched from a prebuilt binary (`go run` can't
+  launch from a foreign cwd), the rest via an absolute path / `--project`.
+- Retired `chat_driver` + the `smooth-code` headless dependency.
+
+Verified live: the go engine boots per task, the canonical turn runs, the test
+suite executes, and a real scored table is produced (nonzero wall-times). The
+rust daemon engine additionally exercises real file edits in the task workspace
+(`write_file` tool result parsed, file lands in `SMOOTH_WORKSPACE`).
+
+Known follow-up (out of scope for the driver rewire): the polyglot `cmd/serve`
+binaries call `ServeLocal` **without `WithTools(...)`**, so their agents have no
+file/bash tools — they can't edit the solution. Only the rust daemon ships a
+coding toolset today. Giving the polyglot engines a coding toolset (via
+`WithTools` or a bench-supplied extension) is a separate smooth-operator change.

crates/smooth-bench/Cargo.toml

@@ -0,0 +1,47 @@
+[package]
+name = "smooai-smooth-bench"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+publish = false
+description = "Smooth benchmark harness — runs Aider Polyglot / SWE-bench / Terminal-Bench against the current routing config. Internal tool; not shipped in the `th` binary."
+
+[[bin]]
+name = "smooth-bench"
+path = "src/main.rs"
+
+[lib]
+name = "smooth_bench"
+path = "src/lib.rs"
+
+[dependencies]
+smooth-cast.workspace = true
+smooth-operator.workspace = true
+smooth-pearls = { path = "../smooth-pearls", package = "smooai-smooth-pearls" }
+anyhow.workspace = true
+async-trait.workspace = true
+chrono.workspace = true
+clap.workspace = true
+dirs-next.workspace = true
+futures-util.workspace = true
+reqwest.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+tokio.workspace = true
+tokio-tungstenite.workspace = true
+toml.workspace = true
+uuid.workspace = true
+
+[dependencies.tracing]
+workspace = true
+
+[dev-dependencies]
+async-trait.workspace = true
+tempfile.workspace = true
+tokio.workspace = true
+# Used by auto_approve tests to stand a fake Big Smooth.
+axum.workspace = true
+
+[lints]
+workspace = true

crates/smooth-bench/curated-tasks.toml

@@ -0,0 +1,164 @@
+# Curated Aider Polyglot task list — "The Line"
+#
+# This is the authoritative set of tasks for the `smooth-bench score`
+# subcommand. The aggregate pass-rate across these tasks is the
+# single number Smoo AI publishes with every release.
+#
+# Selection criteria (applied when the list was first curated):
+#   1. Spread across difficulty — easy (e.g. `leap`, `hamming`),
+#      medium (e.g. `bowling`, `grade-school`), hard (e.g. `forth`,
+#      `poker`, `zebra-puzzle`).
+#   2. Stable, deterministic tests — no network I/O, no time-based
+#      assertions, no randomness that isn't seeded.
+#   3. Standard-library only — avoid tasks that require specific
+#      third-party packages (flaky npm/pip installs, version drift).
+#   4. Small, self-contained corpora — the scratch work dir has to
+#      fit in a microVM's default disk budget (~1 GB).
+#   5. Present in the upstream aider-polyglot corpus at the time of
+#      curation (2026-04-23). Tasks removed upstream will cause a
+#      `task not found` error at run time; re-curate when that
+#      happens.
+#
+# All six languages must have EXACTLY 20 tasks. The unit test
+# `curated_list_has_exactly_20_per_language` enforces this.
+#
+# Upstream repo: https://github.com/Aider-AI/polyglot-benchmark
+# Curated: 2026-04-23
+
+python = [
+    "affine-cipher",
+    "beer-song",
+    "book-store",
+    "bottle-song",
+    "bowling",
+    "connect",
+    "dominoes",
+    "food-chain",
+    "grade-school",
+    "grep",
+    "hangman",
+    "list-ops",
+    "phone-number",
+    "pig-latin",
+    "poker",
+    "proverb",
+    "rest-api",
+    "robot-name",
+    "scale-generator",
+    "wordy",
+]
+
+rust = [
+    "accumulate",
+    "acronym",
+    "alphametics",
+    "book-store",
+    "bowling",
+    "decimal",
+    "dot-dsl",
+    "forth",
+    "gigasecond",
+    "grade-school",
+    "grep",
+    "luhn-from",
+    "macros",
+    "pig-latin",
+    "poker",
+    "robot-name",
+    "say",
+    "scale-generator",
+    "simple-cipher",
+    "word-count",
+]
+
+go = [
+    "alphametics",
+    "beer-song",
+    "book-store",
+    "bottle-song",
+    "bowling",
+    "connect",
+    "counter",
+    "dominoes",
+    "food-chain",
+    "forth",
+    "hexadecimal",
+    "markdown",
+    "matrix",
+    "octal",
+    "pig-latin",
+    "poker",
+    "protein-translation",
+    "robot-simulator",
+    "scale-generator",
+    "trinary",
+]
+
+javascript = [
+    "affine-cipher",
+    "alphametics",
+    "beer-song",
+    "binary",
+    "book-store",
+    "bottle-song",
+    "bowling",
+    "complex-numbers",
+    "connect",
+    "food-chain",
+    "forth",
+    "grade-school",
+    "grep",
+    "list-ops",
+    "phone-number",
+    "pig-latin",
+    "poker",
+    "rest-api",
+    "robot-name",
+    "scale-generator",
+]
+
+java = [
+    "affine-cipher",
+    "all-your-base",
+    "alphametics",
+    "bank-account",
+    "book-store",
+    "bottle-song",
+    "bowling",
+    "change",
+    "circular-buffer",
+    "connect",
+    "food-chain",
+    "forth",
+    "hangman",
+    "kindergarten-garden",
+    "phone-number",
+    "pig-latin",
+    "poker",
+    "rest-api",
+    "simple-linked-list",
+    "transpose",
+]
+
+cpp = [
+    "all-your-base",
+    "allergies",
+    "bank-account",
+    "binary-search-tree",
+    "circular-buffer",
+    "clock",
+    "complex-numbers",
+    "crypto-square",
+    "diamond",
+    "dnd-character",
+    "gigasecond",
+    "grade-school",
+    "kindergarten-garden",
+    "linked-list",
+    "phone-number",
+    "queen-attack",
+    "robot-name",
+    "space-age",
+    "sublist",
+    "zebra-puzzle",
+]