diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 0000000..765c922 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,40 @@ +name: CodSpeed + +on: + push: + branches: [main] + pull_request: + branches: [main] + # Lets CodSpeed trigger backtest runs to build the initial baseline. + workflow_dispatch: + +permissions: + contents: read + id-token: write # OpenID Connect authentication with CodSpeed + +env: + CARGO_TERM_COLOR: always + +jobs: + benchmarks: + name: Run benchmarks + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v6 + + - name: Setup rust toolchain, cache and cargo-codspeed binary + uses: moonrepo/setup-rust@v1 + with: + channel: stable + cache-target: release + bins: cargo-codspeed + + - name: Build the benchmarks + run: cargo codspeed build --features bench + + - name: Run the benchmarks + uses: CodSpeedHQ/action@v5 + with: + mode: simulation + run: cargo codspeed run diff --git a/AGENTS.md b/AGENTS.md index c06d097..9c1d91d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,6 +22,7 @@ make build # debug build make run # launch TUI make sync # cargo run -- sync (FORCE=1 reprocesses all) make search Q="query" # CLI search +make bench # CodSpeed benchmarks (needs the codspeed CLI) cargo test # run a single test by name filter cargo test integration::regression # regression suite cargo test integration::eval_harness # eval harness @@ -81,6 +82,12 @@ Data flow: source adapters → sync → SQLite → search → CLI/TUI. initializes tracing and calls them. Internal modules are `pub(crate)` by default — do not widen module, type, or function visibility unless a current in-repo caller requires it. +- The optional `bench` feature is the one exception to that rule: it compiles + `src/bench_api.rs` as `pub mod bench_api` so `benches/recall.rs` can drive + internal hot paths and build deterministic fixtures. Default builds and the + shipped binary are unaffected. Add new benchmark fixtures inside `bench_api` + instead of widening visibility further, and keep them side-effect free + (temp dirs, in-memory SQLite) so they never touch a real `recall.db`. - `publish = false` in Cargo.toml is intentional: Recall ships binaries and Homebrew assets, not a crates.io package. Do not remove it or add public Rust API for external consumers unless the release strategy changes. diff --git a/Cargo.lock b/Cargo.lock index 7305143..7eee09d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,6 +102,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -364,6 +373,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "chrono" version = "0.4.45" @@ -397,6 +412,7 @@ dependencies = [ "anstyle", "clap_lex", "strsim", + "terminal_size", ] [[package]] @@ -426,12 +442,81 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "codspeed" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7083f253260bcb4aaa3b4aa4c52973703dabc1a85c2f193997e2689aafa8a919" +dependencies = [ + "anyhow", + "cc", + "colored", + "getrandom 0.4.2", + "glob", + "libc", + "nix", + "serde", + "serde_json", + "statrs", +] + +[[package]] +name = "codspeed-divan-compat" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1065d507e1cbab731a7976db4cef7e47e49b87b4dbc0a925df07d343558420" +dependencies = [ + "clap", + "codspeed", + "codspeed-divan-compat-macros", + "codspeed-divan-compat-walltime", + "regex", +] + +[[package]] +name = "codspeed-divan-compat-macros" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd05482a95823ffe421e8a9ba24fa22a6a30d594e2c60455cbb43a41bf2d8fa" +dependencies = [ + "divan-macros", + "itertools 0.14.0", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "codspeed-divan-compat-walltime" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f8eae75b8fa85357020a404899c4280d590c9fd1c47640b3ce53106c62d2ee" +dependencies = [ + "cfg-if", + "clap", + "codspeed", + "condtype", + "divan-macros", + "libc", + "regex-lite", +] + [[package]] name = "colorchoice" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "compact_str" version = "0.8.2" @@ -461,6 +546,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "condtype" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" + [[package]] name = "console" version = "0.16.3" @@ -843,6 +934,17 @@ dependencies = [ "syn", ] +[[package]] +name = "divan-macros" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc51d98e636f5e3b0759a39257458b22619cac7e96d932da6eeb052891bb67c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "document-features" version = "0.2.12" @@ -2204,6 +2306,18 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -2555,6 +2669,15 @@ dependencies = [ "syn", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -2758,6 +2881,7 @@ dependencies = [ "chrono", "clap", "clap_complete", + "codspeed-divan-compat", "crossterm", "dirs", "flate2", @@ -2841,6 +2965,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.11" @@ -3264,6 +3394,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3388,6 +3528,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix 1.1.4", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -3570,6 +3720,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + [[package]] name = "tower" version = "0.5.3" @@ -4298,6 +4478,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "winsafe" version = "0.0.19" diff --git a/Cargo.toml b/Cargo.toml index bcf149b..190438b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,12 +55,28 @@ candle-transformers = "0.10" [features] default = [] cuda = ["candle-core/cuda", "candle-nn/cuda", "candle-transformers/cuda"] +# Exposes `recall::bench_api`, the shim the `benches/` targets use to reach +# internal hot paths. Never enabled for the shipped binary. +bench = [] [profile.release] opt-level = 3 lto = true strip = true +# Benchmarks are compiled with the release profile; thin LTO keeps build times +# reasonable in CI without giving up cross-crate inlining. +[profile.bench] +lto = "thin" + +[dev-dependencies] +divan = { version = "5.0.1", package = "codspeed-divan-compat" } + +[[bench]] +name = "recall" +harness = false +required-features = ["bench"] + [package.metadata.release] publish = false verify = false diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index fa14a0f..7160cf8 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -186,6 +186,39 @@ The former standalone targets `cargo test --test regression` and `cargo test --t Always run `make check` before pushing. If it passes locally, CI will pass. +## Benchmarks + +Performance is tracked on every push and pull request by +[CodSpeed](https://app.codspeed.io/samzong/Recall) via +`.github/workflows/codspeed.yml`, using CPU simulation so results do not depend +on runner noise. + +`benches/recall.rs` is a single [divan](https://github.com/nvzqz/divan) target +grouped by pipeline stage: + +| Group | Covers | +| ----------- | ------------------------------------------------------------- | +| `parsing` | Claude Code and Codex JSONL transcript parsing | +| `indexing` | session/message writes, including FTS5 index maintenance | +| `search` | FTS5 keyword search, hybrid FTS + sqlite-vec search, export | +| `analytics` | usage dashboard aggregation, embedding text, remote URL parse | +| `rendering` | plain-text transcript and shareable HTML rendering | + +Benchmarks sit behind the `bench` feature, which exposes `recall::bench_api` — +the shim that lets the bench binary reach crate-private hot paths and build +deterministic fixtures. Nothing in it is compiled into the shipped binary, and +every fixture uses a temporary directory or an in-memory SQLite database, so +running benchmarks never touches your `recall.db`. + +```bash +make bench # build + run through the codspeed CLI +cargo codspeed build --features bench # build only +``` + +Add a benchmark by extending `src/bench_api.rs` with the fixture (it can reach +any internal module) and wiring it into the matching group in +`benches/recall.rs`. + ## Releases Releases are driven by `cargo-release`, which bumps `Cargo.toml`, updates diff --git a/Makefile b/Makefile index 322a7a5..4960ba3 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,14 @@ lint: ## Run clippy fmt: ## Format code $(CARGO) fmt --all +# ── Benchmarks ─────────────────────────────────────────────────────────────── + +.PHONY: bench + +bench: ## Build and run the CodSpeed benchmarks (needs the codspeed CLI) + $(CARGO) codspeed build --features bench + codspeed run --mode simulation -- $(CARGO) codspeed run + # ── Documentation ──────────────────────────────────────────────────────────── .PHONY: doc diff --git a/README.md b/README.md index 69ef81c..1c298a1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ # Recall +[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://app.codspeed.io/samzong/Recall?utm_source=badge) + > Local-first search across every AI coding session on your machine. [![Recall](docs/recall.png)](https://asciinema.org/a/909453) diff --git a/README.zh-CN.md b/README.zh-CN.md index 0e07789..dcb3a05 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -2,6 +2,8 @@ # Recall +[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://app.codspeed.io/samzong/Recall?utm_source=badge) + > 本地优先，搜索你机器上所有 AI 编程会话。 [![Recall](docs/recall.png)](https://asciinema.org/a/909453) diff --git a/benches/recall.rs b/benches/recall.rs new file mode 100644 index 0000000..3daa9b9 --- /dev/null +++ b/benches/recall.rs @@ -0,0 +1,144 @@ +//! CodSpeed benchmarks for Recall's hot paths. +//! +//! The groups follow the pipeline a session goes through: transcripts are +//! parsed (`parsing`), written to the SQLite index (`indexing`), read back for +//! search and export (`search`), aggregated for the usage dashboard +//! (`analytics`) and finally rendered (`rendering`). +//! +//! Run locally with: +//! +//! ```sh +//! cargo codspeed build --features bench +//! cargo codspeed run +//! ``` + +use recall::bench_api::{ + BenchStore, IndexWorkload, RenderWorkload, SearchIndex, Transcript, UsageWorkload, + build_embedding_text, normalize_remote_url, +}; + +fn main() { + divan::main(); +} + +/// Turning raw agent transcripts into sessions, messages, usage and events. +mod parsing { + use super::*; + + /// Turn counts spanning a short session up to a long working session. + const TURNS: &[usize] = &[8, 64, 256]; + + #[divan::bench(args = TURNS)] + fn claude_code(bencher: divan::Bencher, turns: usize) { + let transcript = Transcript::claude(turns); + bencher.bench(|| divan::black_box(transcript.parse_claude(true))); + } + + /// Same transcript without the tool-call event stream, the path taken when + /// only search and usage data is refreshed. + #[divan::bench] + fn claude_code_messages_only(bencher: divan::Bencher) { + let transcript = Transcript::claude(64); + bencher.bench(|| divan::black_box(transcript.parse_claude(false))); + } + + #[divan::bench(args = TURNS)] + fn codex(bencher: divan::Bencher, turns: usize) { + let transcript = Transcript::codex(turns); + bencher.bench(|| divan::black_box(transcript.parse_codex(true))); + } +} + +/// Writing sessions into SQLite, including FTS5 index maintenance. +mod indexing { + use super::*; + + #[divan::bench(args = [8, 64])] + fn persist_sessions(bencher: divan::Bencher, sessions: usize) { + let workload = IndexWorkload::generate(sessions, 24); + bencher + .with_inputs(BenchStore::empty) + .bench_local_refs(|store| divan::black_box(workload.persist(store))); + } + + /// One long session: message insert throughput rather than session upserts. + #[divan::bench] + fn persist_long_session(bencher: divan::Bencher) { + let workload = IndexWorkload::generate(1, 512); + bencher + .with_inputs(BenchStore::empty) + .bench_local_refs(|store| divan::black_box(workload.persist(store))); + } +} + +/// Reading the index back: search and export. +mod search { + use super::*; + + #[divan::bench] + fn keyword_single_term(bencher: divan::Bencher) { + let index = SearchIndex::build(64, 24, false); + bencher.bench_local(|| divan::black_box(index.keyword_search("embedding"))); + } + + #[divan::bench] + fn keyword_multi_term(bencher: divan::Bencher) { + let index = SearchIndex::build(64, 24, false); + bencher.bench_local(|| divan::black_box(index.keyword_search("sqlite vector index"))); + } + + #[divan::bench] + fn hybrid_fts_and_vector(bencher: divan::Bencher) { + let index = SearchIndex::build(64, 24, true); + bencher.bench_local(|| divan::black_box(index.hybrid_search("sqlite vector index"))); + } + + #[divan::bench] + fn export_jsonl(bencher: divan::Bencher) { + let index = SearchIndex::build(32, 24, false); + bencher.bench_local(|| divan::black_box(index.export_jsonl())); + } +} + +/// Usage dashboard aggregation and the per-message helpers around it. +mod analytics { + use super::*; + + #[divan::bench(args = [1_000, 10_000])] + fn aggregate_usage(bencher: divan::Bencher, events: usize) { + let workload = UsageWorkload::generate(events); + bencher.bench(|| divan::black_box(workload.aggregate())); + } + + #[divan::bench] + fn embedding_text() { + let content = "\ + Recall keeps every local coding session in one SQLite index, then \ + layers FTS5 keyword search and sqlite-vec similarity on top of it."; + divan::black_box(build_embedding_text(divan::black_box("Index a long session"), content)); + } + + #[divan::bench] + fn remote_url_normalization() { + divan::black_box(normalize_remote_url(divan::black_box( + "git@github.com:samzong/Recall.git", + ))); + } +} + +/// Rendering a session for the CLI and for the shareable HTML page. +mod rendering { + use super::*; + + #[divan::bench(args = [32, 256])] + fn plain_transcript(bencher: divan::Bencher, messages: usize) { + let workload = RenderWorkload::generate(messages); + bencher.bench(|| divan::black_box(workload.render_plain())); + } + + #[divan::bench(args = [32, 256])] + fn share_html(bencher: divan::Bencher, messages: usize) { + let workload = RenderWorkload::generate(messages); + bencher.bench(|| divan::black_box(workload.render_html())); + } +} diff --git a/src/adapters/claude_code.rs b/src/adapters/claude_code.rs index aca8c08..77d13fa 100644 --- a/src/adapters/claude_code.rs +++ b/src/adapters/claude_code.rs @@ -341,10 +341,10 @@ fn parse_claude_session_file( })) } -struct ParsedConversation { - messages: Vec, - usage_events: Vec, - events: Vec, +pub(crate) struct ParsedConversation { + pub(crate) messages: Vec, + pub(crate) usage_events: Vec, + pub(crate) events: Vec, cwd: Option, custom_title: Option, summary: Option, @@ -353,7 +353,7 @@ struct ParsedConversation { session_id: Option, } -fn parse_conversation_jsonl( +pub(crate) fn parse_conversation_jsonl( path: &Path, fallback_timestamp: i64, include_events: bool, diff --git a/src/adapters/codex.rs b/src/adapters/codex.rs index eacbcfb..a352432 100644 --- a/src/adapters/codex.rs +++ b/src/adapters/codex.rs @@ -253,7 +253,7 @@ fn parse_codex_session(path: &Path) -> anyhow::Result> { parse_codex_session_with_options(path, true) } -fn parse_codex_session_with_options( +pub(crate) fn parse_codex_session_with_options( path: &Path, include_events: bool, ) -> anyhow::Result> { diff --git a/src/bench_api.rs b/src/bench_api.rs new file mode 100644 index 0000000..5443146 --- /dev/null +++ b/src/bench_api.rs @@ -0,0 +1,635 @@ +//! Benchmark surface for the targets in `benches/`. +//! +//! Every module of this crate is `pub(crate)`, so the benchmark binaries — which +//! link the library as an external crate — cannot reach the hot paths directly. +//! This module is compiled only with the `bench` feature and exposes fixtures +//! that build deterministic workloads and drive the real production code. +//! +//! Everything here is intentionally side-effect free: transcripts are written to +//! a temporary directory and the index lives in an in-memory SQLite database, so +//! benchmarks never touch the user's `recall.db`. + +use std::io::Write as _; +use std::path::{Path, PathBuf}; + +use serde_json::json; +use tempfile::TempDir; + +use crate::adapters::{claude_code, codex}; +use crate::db::search::{SearchEngine, SearchFilters, TimeRange}; +use crate::db::store::{SessionTopologyWrite, Store}; +use crate::export::{ExportIncludes, ExportOptions}; +use crate::share::meta::SessionDisplayMeta; +use crate::share::render; +use crate::types::{Message, Role, Session, UsageEventRecord}; +use crate::{db, export, repo_identity, semantic, transcript, usage}; + +/// Width of the `message_vec` virtual table. +pub const EMBEDDING_DIM: usize = 384; + +/// Deterministic timestamp base (2025-01-01T00:00:00Z) so fixtures never drift. +const BASE_TIMESTAMP_MS: i64 = 1_735_689_600_000; + +const MODELS: &[&str] = &["claude-sonnet-4-5", "claude-opus-4-1", "gpt-5-codex", "gemini-2.5-pro"]; +const TOOLS: &[&str] = &["Read", "Edit", "Bash", "Grep", "WebFetch"]; +const WORDS: &[&str] = &[ + "session", + "transcript", + "adapter", + "sqlite", + "embedding", + "vector", + "index", + "query", + "snippet", + "token", + "usage", + "cache", + "provider", + "workspace", + "directory", + "repository", + "commit", + "rollout", + "summary", + "message", + "assistant", + "prompt", + "context", + "window", + "latency", + "throughput", + "regression", + "baseline", + "migration", + "schema", + "parser", + "handoff", + "semantic", + "keyword", + "hybrid", + "dedupe", + "aggregate", + "timeline", + "sidechain", + "subagent", +]; + +/// Small xorshift generator: no extra dependency and identical data every run. +struct Rng(u64); + +impl Rng { + fn new(seed: u64) -> Self { + Self(seed | 1) + } + + fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + fn below(&mut self, bound: usize) -> usize { + (self.next_u64() % bound as u64) as usize + } + + fn pick(&mut self, items: &'static [&'static str]) -> &'static str { + items[self.below(items.len())] + } +} + +fn words(rng: &mut Rng, count: usize) -> String { + let mut out = String::with_capacity(count * 8); + for index in 0..count { + if index > 0 { + out.push(' '); + } + out.push_str(rng.pick(WORDS)); + } + out +} + +/// A user turn: a couple of sentences, the shape most prompts have. +fn user_prompt(rng: &mut Rng) -> String { + format!("{}?\n\n{}.", words(rng, 12), words(rng, 24)) +} + +/// An assistant turn: prose, a bullet list and a fenced code block, which is +/// what the markdown renderer has to deal with in practice. +fn assistant_markdown(rng: &mut Rng) -> String { + format!( + "## {}\n\n{}.\n\n- {}\n- {}\n- {}\n\n```rust\nfn {}() -> usize {{\n // {}\n {}\n}}\n```\n\n{}.", + words(rng, 4), + words(rng, 40), + words(rng, 8), + words(rng, 8), + words(rng, 8), + rng.pick(WORDS), + words(rng, 10), + rng.below(4096), + words(rng, 30), + ) +} + +fn rfc3339(millis: i64) -> String { + chrono::DateTime::from_timestamp_millis(millis) + .unwrap_or_default() + .to_rfc3339_opts(chrono::SecondsFormat::Millis, true) +} + +/// What a transcript parse produced, so benchmarks can black-box a real result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ParsedCounts { + pub messages: usize, + pub usage_events: usize, + pub events: usize, +} + +/// A synthetic transcript on disk, in the JSONL dialect of a given source. +pub struct Transcript { + _dir: TempDir, + path: PathBuf, +} + +impl Transcript { + pub fn path(&self) -> &Path { + &self.path + } + + fn write(name: &str, lines: &[String]) -> Self { + let dir = TempDir::new().expect("temp dir"); + let path = dir.path().join(name); + let mut file = std::fs::File::create(&path).expect("create transcript"); + for line in lines { + writeln!(file, "{line}").expect("write transcript"); + } + file.sync_all().expect("flush transcript"); + Self { _dir: dir, path } + } + + /// Claude Code transcript: `{user,assistant}` records with content blocks, + /// tool calls and per-message `usage` payloads. + pub fn claude(turns: usize) -> Self { + let mut rng = Rng::new(0x5eed_1234); + let session_id = "6f1c3ba4-1f42-4c1a-9c2f-0f9a51d2b7e1"; + let mut lines = Vec::with_capacity(turns * 2 + 2); + lines.push( + json!({ + "type": "summary", + "summary": words(&mut rng, 10), + "leafUuid": session_id, + }) + .to_string(), + ); + for turn in 0..turns { + let ts = BASE_TIMESTAMP_MS + turn as i64 * 45_000; + lines.push( + json!({ + "type": "user", + "sessionId": session_id, + "cwd": "/home/dev/projects/recall", + "timestamp": rfc3339(ts), + "message": { "role": "user", "content": user_prompt(&mut rng) }, + }) + .to_string(), + ); + let tool = rng.pick(TOOLS).to_string(); + lines.push( + json!({ + "type": "assistant", + "sessionId": session_id, + "requestId": format!("req_{turn}"), + "timestamp": rfc3339(ts + 12_000), + "message": { + "id": format!("msg_{turn}"), + "role": "assistant", + "model": rng.pick(MODELS), + "content": [ + { "type": "text", "text": assistant_markdown(&mut rng) }, + { + "type": "tool_use", + "id": format!("toolu_{turn}"), + "name": tool, + "input": { "file_path": "/home/dev/projects/recall/src/db/search.rs", "pattern": words(&mut rng, 3) }, + }, + ], + "usage": { + "input_tokens": 1_200 + rng.below(800), + "output_tokens": 400 + rng.below(600), + "cache_read_input_tokens": 20_000 + rng.below(10_000), + "cache_creation_input_tokens": rng.below(4_000), + }, + }, + }) + .to_string(), + ); + lines.push( + json!({ + "type": "user", + "sessionId": session_id, + "timestamp": rfc3339(ts + 14_000), + "message": { + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": format!("toolu_{turn}"), + "content": [{ "type": "text", "text": words(&mut rng, 60) }], + }], + }, + }) + .to_string(), + ); + } + Self::write(&format!("{session_id}.jsonl"), &lines) + } + + /// Codex rollout: `session_meta`, `event_msg` and `response_item` records + /// with cumulative `token_count` events. + pub fn codex(turns: usize) -> Self { + let mut rng = Rng::new(0xc0de_9876); + let session_id = "019e6d8d-588b-7fd2-a326-c525469ed120"; + let mut lines = Vec::with_capacity(turns * 4 + 1); + lines.push( + json!({ + "timestamp": rfc3339(BASE_TIMESTAMP_MS), + "type": "session_meta", + "payload": { + "id": session_id, + "timestamp": rfc3339(BASE_TIMESTAMP_MS), + "cwd": "/home/dev/projects/recall", + "model": "gpt-5-codex", + "model_provider": "openai", + }, + }) + .to_string(), + ); + let mut input_total = 0i64; + let mut output_total = 0i64; + let mut cached_total = 0i64; + let mut reasoning_total = 0i64; + for turn in 0..turns { + let ts = BASE_TIMESTAMP_MS + turn as i64 * 45_000; + lines.push( + json!({ + "timestamp": rfc3339(ts), + "type": "event_msg", + "payload": { "type": "user_message", "message": user_prompt(&mut rng) }, + }) + .to_string(), + ); + lines.push( + json!({ + "timestamp": rfc3339(ts + 9_000), + "type": "response_item", + "payload": { + "type": "message", + "role": "assistant", + "content": [{ "type": "output_text", "text": assistant_markdown(&mut rng) }], + }, + }) + .to_string(), + ); + lines.push( + json!({ + "timestamp": rfc3339(ts + 11_000), + "type": "response_item", + "payload": { + "type": "function_call", + "name": "shell", + "arguments": json!({ "command": ["bash", "-lc", words(&mut rng, 6)] }).to_string(), + "call_id": format!("call_{turn}"), + }, + }) + .to_string(), + ); + lines.push( + json!({ + "timestamp": rfc3339(ts + 13_000), + "type": "response_item", + "payload": { + "type": "function_call_output", + "call_id": format!("call_{turn}"), + "output": words(&mut rng, 40), + }, + }) + .to_string(), + ); + input_total += 900 + rng.below(600) as i64; + output_total += 300 + rng.below(400) as i64; + cached_total += 5_000 + rng.below(2_000) as i64; + reasoning_total += rng.below(500) as i64; + lines.push( + json!({ + "timestamp": rfc3339(ts + 14_000), + "type": "event_msg", + "payload": { + "type": "token_count", + "info": { + "total_token_usage": { + "input_tokens": input_total, + "cached_input_tokens": cached_total, + "output_tokens": output_total, + "reasoning_output_tokens": reasoning_total, + "total_tokens": input_total + output_total + cached_total, + }, + "last_token_usage": { + "input_tokens": 900, + "cached_input_tokens": 5_000, + "output_tokens": 300, + "reasoning_output_tokens": 100, + "total_tokens": 6_300, + }, + "model_context_window": 272_000, + }, + }, + }) + .to_string(), + ); + } + Self::write(&format!("rollout-2025-01-01T00-00-00-{session_id}.jsonl"), &lines) + } + + /// Full Claude Code transcript parse, the `recall sync` hot path. + pub fn parse_claude(&self, include_events: bool) -> ParsedCounts { + let parsed = + claude_code::parse_conversation_jsonl(&self.path, BASE_TIMESTAMP_MS, include_events) + .expect("parse claude transcript"); + ParsedCounts { + messages: parsed.messages.len(), + usage_events: parsed.usage_events.len(), + events: parsed.events.len(), + } + } + + /// Full Codex rollout parse, the `recall sync` hot path. + pub fn parse_codex(&self, include_events: bool) -> ParsedCounts { + let parsed = codex::parse_codex_session_with_options(&self.path, include_events) + .expect("parse codex rollout") + .expect("non-empty codex rollout"); + ParsedCounts { + messages: parsed.messages.len(), + usage_events: parsed.usage_events.len(), + events: parsed.events.len(), + } + } +} + +fn synthetic_session(index: usize, message_count: usize) -> Session { + let mut rng = Rng::new(0xa11ce ^ index as u64); + Session { + id: format!("claude-code:session-{index}"), + source: "claude-code".to_string(), + source_id: format!("session-{index}"), + title: words(&mut rng, 6), + directory: Some(format!("/home/dev/projects/project-{}", index % 16)), + repo_remote: Some("github.com/samzong/recall".to_string()), + repo_slug: Some("samzong/recall".to_string()), + repo_name: Some("recall".to_string()), + started_at: BASE_TIMESTAMP_MS - (index as i64 * 3_600_000), + updated_at: Some(BASE_TIMESTAMP_MS - (index as i64 * 3_600_000) + 900_000), + message_count: message_count as u32, + entrypoint: Some("cli".to_string()), + custom_title: None, + summary: Some(words(&mut rng, 12)), + duration_minutes: Some(15), + source_file_path: Some(format!("/home/dev/.claude/projects/p/session-{index}.jsonl")), + is_import: false, + } +} + +fn synthetic_messages(session: &Session, count: usize) -> Vec { + let mut rng = Rng::new(0xb0b ^ session.started_at as u64); + (0..count) + .map(|seq| { + let role = if seq % 2 == 0 { Role::User } else { Role::Assistant }; + let content = match role { + Role::User => user_prompt(&mut rng), + Role::Assistant => assistant_markdown(&mut rng), + }; + Message { + session_id: session.id.clone(), + role, + content, + timestamp: Some(session.started_at + seq as i64 * 30_000), + seq: seq as u32, + } + }) + .collect() +} + +fn embedding(seed: u64) -> Vec { + let mut rng = Rng::new(seed); + let mut values: Vec = + (0..EMBEDDING_DIM).map(|_| rng.below(2_000) as f32 / 1_000.0 - 1.0).collect(); + let norm = values.iter().map(|v| v * v).sum::().sqrt().max(f32::EPSILON); + for value in &mut values { + *value /= norm; + } + values +} + +/// An empty in-memory index, schema already migrated. +pub struct BenchStore { + store: Store, +} + +impl BenchStore { + pub fn empty() -> Self { + db::schema::register_sqlite_vec(); + Self { store: Store::open_in_memory().expect("in-memory store") } + } +} + +/// Sessions and messages waiting to be written to the index, the write half of +/// `recall sync` (session upsert, message insert, FTS5 index maintenance). +pub struct IndexWorkload { + sessions: Vec<(Session, Vec)>, +} + +impl IndexWorkload { + pub fn generate(sessions: usize, messages_per_session: usize) -> Self { + Self { + sessions: (0..sessions) + .map(|index| { + let session = synthetic_session(index, messages_per_session); + let messages = synthetic_messages(&session, messages_per_session); + (session, messages) + }) + .collect(), + } + } + + pub fn persist(&self, target: &BenchStore) -> usize { + let mut written = 0; + for (session, messages) in &self.sessions { + target + .store + .persist_session_with_usage_and_events_with_topology( + session, + messages, + &[], + None, + &[], + None, + &SessionTopologyWrite::none(), + ) + .expect("persist session"); + written += messages.len(); + } + written + } +} + +/// A populated index used to benchmark the read paths: keyword search, hybrid +/// (FTS + vector) search and JSONL export. +pub struct SearchIndex { + store: Store, + query_embedding: Vec, +} + +impl SearchIndex { + pub fn build(sessions: usize, messages_per_session: usize, with_vectors: bool) -> Self { + let target = BenchStore::empty(); + IndexWorkload::generate(sessions, messages_per_session).persist(&target); + let store = target.store; + if with_vectors { + for index in 0..sessions { + let session_id = format!("claude-code:session-{index}"); + let messages = store.embeddable_messages(&session_id).expect("embeddable messages"); + let vectors: Vec<(i64, Vec)> = + messages.iter().map(|(id, _)| (*id, embedding(*id as u64 + 7))).collect(); + let items: Vec<(i64, &[f32])> = + vectors.iter().map(|(id, vector)| (*id, vector.as_slice())).collect(); + store.upsert_embeddings(&items).expect("upsert embeddings"); + } + } + Self { store, query_embedding: embedding(42) } + } + + fn filters() -> SearchFilters { + SearchFilters { + sources: None, + time_range: TimeRange::All, + directory: None, + repo: None, + thread_role: None, + } + } + + /// FTS5-only search, what `recall search` runs without a local model. + pub fn keyword_search(&self, query: &str) -> usize { + SearchEngine::new(&self.store.conn) + .hybrid_search(query, None, &Self::filters(), 20, 3) + .expect("keyword search") + .len() + } + + /// FTS5 + sqlite-vec reciprocal-rank fusion, the default `recall search`. + pub fn hybrid_search(&self, query: &str) -> usize { + SearchEngine::new(&self.store.conn) + .hybrid_search(query, Some(&self.query_embedding), &Self::filters(), 20, 3) + .expect("hybrid search") + .len() + } + + /// `recall export`: read every session back out and serialize it as JSONL. + pub fn export_jsonl(&self) -> usize { + let options = ExportOptions { + session_ids: Vec::new(), + sources: None, + time_range: TimeRange::All, + project: None, + repo: None, + thread_role: None, + limit: None, + includes: ExportIncludes::full(), + }; + let mut buffer = Vec::with_capacity(1 << 16); + export::write_jsonl(&self.store, &options, &mut buffer).expect("export jsonl"); + buffer.len() + } +} + +/// Usage events feeding the `recall usage` dashboard aggregation. +pub struct UsageWorkload { + events: Vec, +} + +impl UsageWorkload { + pub fn generate(events: usize) -> Self { + let mut rng = Rng::new(0xfeed_beef); + Self { + events: (0..events) + .map(|index| { + let session = index / 24; + UsageEventRecord { + session_id: format!("claude-code:session-{session}"), + source: if index % 3 == 0 { "codex" } else { "claude-code" }.to_string(), + source_id: format!("session-{session}"), + event_key: format!("assistant:req_{index}:msg_{index}"), + timestamp: BASE_TIMESTAMP_MS - index as i64 * 600_000, + model: rng.pick(MODELS).to_string(), + provider: if index % 3 == 0 { "openai" } else { "anthropic" }.to_string(), + input_tokens: 900 + rng.below(900) as i64, + output_tokens: 300 + rng.below(500) as i64, + cache_read_tokens: 12_000 + rng.below(8_000) as i64, + cache_write_tokens: rng.below(3_000) as i64, + reasoning_tokens: rng.below(700) as i64, + token_source: if index % 3 == 0 { "derived" } else { "observed" } + .to_string(), + } + }) + .collect(), + } + } + + /// Dedupe + group by source, model, day, week and month. + pub fn aggregate(&self) -> i64 { + usage::aggregate_usage_events(&self.events).summary.tokens.total_tokens + } +} + +/// A session ready to be rendered as plain text or as a shareable HTML page. +pub struct RenderWorkload { + session: Session, + messages: Vec, + display_meta: SessionDisplayMeta, +} + +impl RenderWorkload { + pub fn generate(messages: usize) -> Self { + let session = synthetic_session(0, messages); + let messages = synthetic_messages(&session, messages); + Self { + session, + messages, + display_meta: SessionDisplayMeta { + models: vec!["claude-sonnet-4-5".to_string()], + thinking_depths: vec!["high".to_string()], + }, + } + } + + /// `recall session show`: plain-text transcript. + pub fn render_plain(&self) -> usize { + transcript::render_plain(&self.session, &self.messages).len() + } + + /// `recall session share`: markdown to a self-contained HTML page. + pub fn render_html(&self) -> usize { + render::render_session_html(&self.session, &self.messages, &self.display_meta).len() + } +} + +/// Prompt-to-embedding-input normalization, called once per indexed message. +pub fn build_embedding_text(title: &str, content: &str) -> String { + semantic::build_embedding_text(title, content) +} + +/// Git remote normalization, called once per session directory during sync. +pub fn normalize_remote_url(url: &str) -> Option { + repo_identity::normalize_remote_url(url).map(|identity| identity.slug) +} diff --git a/src/db/store.rs b/src/db/store.rs index 7336f07..eeb4eb9 100644 --- a/src/db/store.rs +++ b/src/db/store.rs @@ -81,7 +81,7 @@ pub(crate) struct SessionTopologyWrite<'a> { } impl SessionTopologyWrite<'_> { - #[cfg(test)] + #[cfg(any(test, feature = "bench"))] pub(crate) fn none() -> SessionTopologyWrite<'static> { SessionTopologyWrite { thread_role: None, parents: &[], parser_version: None } } @@ -115,7 +115,7 @@ impl Store { Ok(Store { conn }) } - #[cfg(test)] + #[cfg(any(test, feature = "bench"))] pub(crate) fn open_in_memory() -> Result { let conn = Connection::open_in_memory()?; conn.execute_batch("PRAGMA busy_timeout=5000; PRAGMA foreign_keys=ON;")?; diff --git a/src/lib.rs b/src/lib.rs index 5e968b4..40742a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,7 @@ pub(crate) mod adapters; pub(crate) mod bench; +#[cfg(feature = "bench")] +pub mod bench_api; pub(crate) mod cli; pub(crate) mod config; pub(crate) mod db; diff --git a/src/share/mod.rs b/src/share/mod.rs index da62ac5..df1d2e7 100644 --- a/src/share/mod.rs +++ b/src/share/mod.rs @@ -1,7 +1,7 @@ mod assets; -mod meta; +pub(crate) mod meta; mod publish; -mod render; +pub(crate) mod render; pub(crate) use publish::{ default_project_name, default_publish_dir, expand_path, init_cloudflare_pages,