From e8fa7fd62704a33bc6a201bf1dd8a7a89975dbbd Mon Sep 17 00:00:00 2001 From: Yogthos Date: Wed, 10 Jun 2026 11:11:35 -0400 Subject: [PATCH 1/6] feat(v3.2): RPG-style architecture-first planning behind ATLAS_RPG_PLANNING (#120) Adds an opt-in repository-level planning phase that plans the architecture up front and then fills each piece, instead of free-form natural-language planning. It builds on the existing problem-level PlanSearch rather than replacing it: the Repository Planning Graph decides what to build and how the files, signatures, and data flows fit together, and PlanSearch fills in each node whose interface is already pinned. The work draws on two papers. RPG (arXiv:2509.16198) supplies the repository planning graph and the two-stage proposal-then-implementation construction. PlanSearch (arXiv:2409.03733), already implemented in this repo, supplies the per-node algorithmic search. The wavelet substrate is a faithful, dependency free Python port of wavescope-mcp by Dmitri Sotnikov, whose numeric output matches the upstream TypeScript bit for bit under the golden-fixture tests. Everything is gated by ATLAS_RPG_PLANNING and defaults to off. With the flag off, planning and generation behave exactly as before, since the flat planner emits no constraints and the new code paths short-circuit. What landed, by phase: - Wavelet substrate (v3-service/wavelet): structural signal, Ricker CWT, the fine/medium/coarse bands, project decomposition, and the peak-diff drift detector, all pure stdlib. - RPG artifact and two-stage planning (v3-service/rpg.py): capability tree then files, signatures, and edges, with validation, scoring, and a topological projection back onto the existing flat Plan so the agent loop is untouched. - Graph-guided generation: each node's planned interface threads into its /v3/generate call (proxy/rpg.go, proxy/types.go), so PlanSearch fills a node whose architecture is fixed. - Structural verification and drift: the candidate veto now rejects code that does not realize its planned signatures, and on write the proxy runs one bounded corrective regeneration, then surfaces any surviving drift along with the affected downstream subgraph. - Offline evaluation: v3-service/rpg_eval.py scores RPG artifacts for CI and benchmark summaries. The live RepoCraft comparison needs a GPU and is left as a documented runbook; the default stays off until that evidence lands. Design and phase-by-phase status live in docs/reports/RPG_WAVELET_PLANNING_V3_2.md. Tests: 152 Python tests under tests/v3-service and the proxy Go suite, all green. Credit for the idea and framing goes to Dmitri Sotnikov (@yogthos). --- .env.example | 7 + .github/workflows/test.yml | 7 +- CHANGELOG.md | 25 + docs/reports/RPG_WAVELET_PLANNING_V3_2.md | 325 ++++++++++ proxy/rpg.go | 186 ++++++ proxy/rpg_test.go | 277 ++++++++ proxy/tools.go | 20 + proxy/types.go | 63 +- tests/v3-service/test_rpg.py | 389 +++++++++++ tests/v3-service/test_rpg_eval.py | 74 +++ tests/v3-service/test_rpg_integration.py | 71 +++ tests/v3-service/test_wavelet_context.py | 115 ++++ tests/v3-service/test_wavelet_cwt.py | 194 ++++++ tests/v3-service/test_wavelet_diff.py | 80 +++ tests/v3-service/test_wavelet_flags.py | 25 + tests/v3-service/test_wavelet_project.py | 123 ++++ tests/v3-service/test_wavelet_signal.py | 248 +++++++ v3-service/Dockerfile | 4 + v3-service/main.py | 122 +++- v3-service/rpg.py | 745 ++++++++++++++++++++++ v3-service/rpg_eval.py | 128 ++++ v3-service/wavelet/__init__.py | 65 ++ v3-service/wavelet/context.py | 429 +++++++++++++ v3-service/wavelet/cwt.py | 153 +++++ v3-service/wavelet/diff.py | 138 ++++ v3-service/wavelet/flags.py | 21 + v3-service/wavelet/language.py | 429 +++++++++++++ v3-service/wavelet/project.py | 282 ++++++++ v3-service/wavelet/signal.py | 255 ++++++++ 29 files changed, 4992 insertions(+), 8 deletions(-) create mode 100644 docs/reports/RPG_WAVELET_PLANNING_V3_2.md create mode 100644 proxy/rpg.go create mode 100644 proxy/rpg_test.go create mode 100644 tests/v3-service/test_rpg.py create mode 100644 tests/v3-service/test_rpg_eval.py create mode 100644 tests/v3-service/test_rpg_integration.py create mode 100644 tests/v3-service/test_wavelet_context.py create mode 100644 tests/v3-service/test_wavelet_cwt.py create mode 100644 tests/v3-service/test_wavelet_diff.py create mode 100644 tests/v3-service/test_wavelet_flags.py create mode 100644 tests/v3-service/test_wavelet_project.py create mode 100644 tests/v3-service/test_wavelet_signal.py create mode 100644 v3-service/rpg.py create mode 100644 v3-service/rpg_eval.py create mode 100644 v3-service/wavelet/__init__.py create mode 100644 v3-service/wavelet/context.py create mode 100644 v3-service/wavelet/cwt.py create mode 100644 v3-service/wavelet/diff.py create mode 100644 v3-service/wavelet/flags.py create mode 100644 v3-service/wavelet/language.py create mode 100644 v3-service/wavelet/project.py create mode 100644 v3-service/wavelet/signal.py diff --git a/.env.example b/.env.example index f618975f..95a4dd1a 100644 --- a/.env.example +++ b/.env.example @@ -19,6 +19,13 @@ ATLAS_MODEL_NAME=Qwen3.5-9B-Q6_K # Context window size (tokens) ATLAS_CTX_SIZE=32768 +# V3.2 RPG-style architecture-first planning (#120, EXPERIMENTAL, default off). +# When 1/true, planning runs a two-stage Repository Planning Graph (modules -> +# files -> signatures -> data-flow edges) ahead of problem-level PlanSearch, and +# threads each node's planned interface into generation. Strictly additive — off +# leaves planning/generation unchanged. See docs/reports/RPG_WAVELET_PLANNING_V3_2.md. +# ATLAS_RPG_PLANNING=0 + # Host project directory bind-mounted to /workspace inside the proxy container. # This is the directory ATLAS reads and writes files in. The bind mount is # resolved at `docker compose up` time and is fixed for the container's diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef5ce3cb..181d0236 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,12 +47,13 @@ jobs: strategy: fail-fast: false matrix: - # tests/v3/ and tests/cli/ are pure unit tests — no services - # needed. tests/integration/ + tests/infrastructure/ require - # a live llama-server / sandbox / lens, gated separately by + # tests/v3/, tests/v3-service/, and tests/cli/ are pure unit tests — + # no services needed. tests/integration/ + tests/infrastructure/ + # require a live llama-server / sandbox / lens, gated separately by # the integration workflow (not yet wired up — PC-064). subset: - tests/v3 + - tests/v3-service - tests/cli steps: - uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index c5f29812..f3c233a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,31 @@ ## Unreleased +### V3.2 — RPG-style architecture-first planning (#120, experimental, opt-in) +- New `ATLAS_RPG_PLANNING` flag (default **off**) enables repository-level, + plan-then-fill planning ahead of the existing problem-level PlanSearch: + - **Wavelet substrate** (`v3-service/wavelet/`) — a faithful, dependency-free + Python port of [wavescope-mcp](https://github.com/yogthos/wavescope-mcp) + (Ricker CWT, structural signal, multi-resolution bands, project decomposition, + peak-diff). Numeric parity with upstream is golden-tested. + - **Repository Planning Graph** (`v3-service/rpg.py`, [arXiv:2509.16198](https://arxiv.org/abs/2509.16198)) — + two-stage construction (proposal capability tree → implementation files + + signatures + data-flow edges), graph validation/scoring, and a topological + projection to the existing flat `Plan` so the agent loop is unchanged. The + proposal stage is seeded with the wavelet coarse band on existing repos. + - **Graph-guided generation** — each node's planned interface (signatures, + edges) threads into its `/v3/generate` call (`proxy/rpg.go`), so the existing + PlanSearch ([arXiv:2409.03733](https://arxiv.org/abs/2409.03733)) fills a node + whose architecture is already pinned. + - **Structural verification + drift** — the candidate veto now rejects code + that doesn't realize its planned signatures; post-write drift detection + surfaces the affected downstream subgraph for re-planning. + - **Offline metrics** — `v3-service/rpg_eval.py` scores RPG artifacts for CI / + benchmark summaries. + - Strictly additive: with the flag off, planning and generation are unchanged. + - Design + phased status: `docs/reports/RPG_WAVELET_PLANNING_V3_2.md`. Credit + idea + framing to Dmitri Sotnikov (@yogthos), author of wavescope-mcp. + ### Removed - Removed dead `ATLAS_USE_FOX` code paths in benchmark runner (#22) diff --git a/docs/reports/RPG_WAVELET_PLANNING_V3_2.md b/docs/reports/RPG_WAVELET_PLANNING_V3_2.md new file mode 100644 index 00000000..587e77e9 --- /dev/null +++ b/docs/reports/RPG_WAVELET_PLANNING_V3_2.md @@ -0,0 +1,325 @@ +# V3.2 — Architecture-First Planning (RPG-style plan-then-fill over wavelet bands) + +Tracks **[#120](https://github.com/itigges22/ATLAS/issues/120)**. Builds on the wavelet +decomposition angle from **[#39](https://github.com/itigges22/ATLAS/issues/39)**. + +## 1. What we're building + +Replace ATLAS's flat, free-form planning artifact with a **structured, two-stage, +architecture-first plan** and execute it **plan-coarse / implement-fine**: + +1. **Proposal-level planning** — *what* to build: a capability tree (modules → + components → leaf capabilities). +2. **Implementation-level planning** — *how*: expand leaves into files, classes, + function signatures, and the data-flow / ordering edges between them. This is + the **Repository Planning Graph (RPG)** artifact. +3. **Graph-guided generation** — traverse the RPG in topological order, generating + per node, verifying as we go, threading existing V3 phases (DivSampling, S\*, + PR-CoT repair) **per node** instead of per whole-file. + +The **wavelet decomposition** (from wavescope) supplies resolution bands for free: +the planner reasons at the **coarse band** (module/class structure, "which files +even matter"), the implementer zooms to the **fine band** (function bodies). On L6 +tasks (editing existing repos), the coarse band grounds the proposal stage in the +*actual* structure of the repo rather than the model's guess. + +### Source grounding + +- **RPG / ZeroRepo** — *A Repository Planning Graph for Unified and Scalable + Codebase Generation*, [arXiv:2509.16198](https://arxiv.org/abs/2509.16198) + (paper text in repo root `2509.16198v6.md`). Reports +35.8 pts test accuracy, + +27.3 pts coverage, near-linear scaling vs. a Claude Code baseline on RepoCraft. + Supplies the **repository / architecture** axis (this work). +- **PlanSearch** — *Planning In Natural Language Improves LLM Search for Code + Generation*, Wang et al. ICLR 2025 + ([arXiv:2409.03733](https://arxiv.org/abs/2409.03733), paper text in repo root + `2409.03733v2.md`). Already implemented in ATLAS as `benchmark/v3/plan_search.py` + (Feature 1A). Searches natural-language plans for a **single problem** to raise + idea-space diversity. Supplies the **algorithmic / problem** axis — it fills in + each RPG node, it does not replace RPG. Per the #120 scope clarification: RPG + plans the repo up front (coarse / module band), then PlanSearch + Derivation + Chains fill each node (fine / function band). +- **wavescope-mcp** — `~/src/wavescope-mcp`. The wavelet engine we reuse: + - `src/signal.ts` — per-line structural importance signal (indent + structural + keywords + decorators, comment/string aware), 14 languages. + - `src/wavelet.ts` — Ricker (Mexican-hat) CWT at scales `[1,2,4,8,16,32,64,128]`, + peak detection with cross-scale ridge collapse. + - `src/context.ts` — `FileContext`: assembles **fine / medium / coarse** bands and + `getImportantPositions()`. + - `src/project.ts` — project-wide indexer (gitignore-aware, file caps, LRU cache). + - `src/diff.ts` — `diff_wavelet_context`: which structural boundaries were + added / removed / shifted between two git revisions — our **drift detector**. + +## 2. How it maps onto today's pipeline + +(Reconnaissance reference for every path below: the current pipeline.) + +| RPG concept | ATLAS today | What changes | +|---|---|---| +| Proposal-level plan (what) | `Plan{Steps[]}` flat list, `PLAN_PROMPT_TEMPLATE` (`v3-service/main.py:1707`) | New `capability tree` stage feeding a graph | +| Implementation-level plan (how) | — (none; steps are tool calls) | New `RPG` artifact: files + signatures + edges | +| Graph-guided generation | `/v3/generate` whole-file pipeline | Topological per-node traversal | +| Guided localization | `symbol_index` (tree-sitter, ad-hoc) + symbol-name match | Coarse-band "which files matter" + RPG node→file map | +| Structural verification veto (#39 pt 1) | import/class-survival shape check (`v3-service`) | Extend: does generated code realize the **planned** signatures/edges? | +| Plan adherence / revision | `proxy/plan_adherence.go` (off-plan streak → re-plan) | Node-level: drift via `diff_wavelet_context` → re-plan touched nodes | + +**Services involved:** Go `proxy` (8090, agent loop + plan orchestration), +Python `v3-service` (8070, plan generation + AST + symbol index), Python +`geometric-lens` (8099, scoring), `sandbox` (test exec), `llama-server` (8080). +Plan generation lives in `v3-service`; the new RPG stages live there too. + +## 3. How we reuse the wavelet engine — port to Python (decided) + +The wavelet engine is TypeScript; ATLAS is Go + Python. **Decision: port the core +engine into v3-service as a Python module** (`v3-service/wavelet/`), in-process — +no new runtime, no extra container, tightest integration with the planning stages +that consume it. + +Port scope (the four files that carry the algorithm, ~1.45 KLOC TS → Python): + +| wavescope source | Python target | What it provides | +|---|---|---| +| `src/signal.ts` | `wavelet/signal.py` | per-line structural-importance signal (indent + structural keywords + decorators; comment/string aware) | +| `src/wavelet.ts` | `wavelet/cwt.py` | Ricker CWT at scales `[1,2,4,8,16,32,64,128]`, peak detection w/ ridge collapse | +| `src/context.ts` | `wavelet/context.py` | `FileContext`: fine/medium/coarse band assembly, `get_important_positions` | +| `src/language.ts` | `wavelet/language.py` | per-language structural-keyword tables (14 langs) | +| `src/diff.ts` | `wavelet/diff.py` | peak-profile diff between two git revisions (drift detector) | + +**Library vs. hand-roll (surveyed — §6):** no drop-in library preserves +wavescope's calibrated behavior, so we port faithfully and use **`numpy` only** +(already in the stack via `geometric-lens`) to vectorize the convolution. Do *not* +pull in PyWavelets/SciPy for the transform — see §6. + +Reuse-fidelity rules so we don't silently fork behavior: +- **Port verbatim, behavior-for-behavior** — same scales, same band radii + (`fine ±radius/5`, `medium ±radius/2`, `coarse ±radius`), same ridge-collapse + and dedup semantics. The CWT is plain numerics (`numpy` makes the convolution a + few lines), so divergence risk is low. +- **Bring the tests across too** — translate wavescope's unit tests (e.g. + `signal`, `wavelet`, `context`, `diff`) into `pytest` and treat them as the + port's conformance suite. A handful of golden-output fixtures captured from the + upstream TS (run wavescope once, snapshot peaks/bands for a few sample files) + guards against drift. +- **Document provenance** — each ported file headers the upstream path + commit + so future syncs are traceable. + +Everything downstream of Phase 0 is agnostic to this choice — it consumes the same +in-process `FileContext` API regardless. + +## 4. Phased plan + +Each phase is independently shippable and **behind a feature flag, default off** +(`ATLAS_RPG_PLANNING=0`), mirroring the `PlanSearchConfig.enabled` pattern. No +behavior change until a phase is explicitly enabled and benchmarked. + +### Phase 0 — Wavelet substrate (foundation) — ✅ SHIPPED +**Goal:** v3-service can decompose a project / file into resolution bands in-process. + +- Ported the engine into `v3-service/wavelet/` (`language.py`, `signal.py`, + `cwt.py`, `context.py`, `project.py`, `flags.py`) — pure stdlib, no new deps. +- Internal Python API consumed by the planning stages: + - `decompose_project(root)` → important structural positions across the repo + (port of `get_important_positions(directory)`), gitignore-aware, file-capped. + - `FileContext(path).query_wavelet_context(center, radius)` → `{fine, medium, + coarse}` bands (port of `query_wavelet_context`). +- Env contract `ATLAS_RPG_PLANNING` established (`wavelet.flags.rpg_planning_enabled`, + default off); the planner gates on it in Phase 1. +- Conformance suite in `tests/v3-service/test_wavelet_*.py` (translated from the + upstream vitest specs) **plus a golden-fixture test** asserting bit-for-bit + numeric parity (8 dp) with the actual upstream `wavelet.ts` (captured via + `npx tsx`). Added `tests/v3-service` to the CI pytest matrix. +- **Deferred to Phase 3:** `diff_context` (port of `diff_wavelet_context`, the + drift detector) — it's only consumed by the Phase-3 re-plan loop, so it ships + with its consumer rather than as unused substrate. +- **Exit (met):** v3-service produces a coarse structural map of any repo + in-process, no pipeline change. In-process port keeps this off the network path + (issue #39's latency budget). + +### Phase 1 — RPG artifact + two-stage planning — ✅ SHIPPED +**Goal:** produce the graph instead of (alongside) the flat plan. + +- RPG schema + construction in `v3-service/rpg.py`: `Capability` / `FileSpec` / + `FunctionSpec` / `Edge` / `RPG` dataclasses, nodes carrying dual semantics + (capability + file/function), edges carrying `data_flow` / `order`. +- **Stage A (proposal):** `build_proposal_prompt` → capability tree. On L6 it's + seeded with the Phase-0 coarse band (`decompose_project` labels) so the tree + maps onto real modules. +- **Stage B (implementation):** `build_implementation_prompt` → files, function + signatures, and edges → the RPG. +- Tolerant JSON extraction (`extract_json_object`, mirroring `_parse_plan_json`) + + `validate_rpg` (acyclic via Kahn topo sort, leaf→file coverage, edge + resolution, parent resolution) + `score_rpg` graph-shape heuristic. +- `flatten_to_plan` projects the RPG onto the existing flat `Plan` (topological + file order, producers before consumers, verify step last) so the proxy agent + loop and `plan_adherence.go` are unchanged. Full RPG artifact is attached to + the `/v3/plan` response under `rpg` for observability. +- Wired into `main.generate_plan`, gated by `ATLAS_RPG_PLANNING`, strictly + additive — any failure (flag off, modules absent, model output unusable) falls + through to the flat planner. Dockerfile ships the new modules. +- `complete_fn`-injected construction (LLM-agnostic) → fully unit-tested with a + fake model: `tests/v3-service/test_rpg.py` (27) + `test_rpg_integration.py` + (monkeypatched `LLMAdapter`, flag on/off). +- **Exit (met):** with the flag on, planning emits a validated RPG and the agent + loop runs off the flattened projection; off, behavior is byte-identical. + +### Phase 2 — Graph-guided generation (plan-coarse, implement-fine) — ✅ SHIPPED +**Goal:** generate by traversing the RPG, not free-form. + +- **Topological traversal comes free from Phase 1's projection:** `flatten_to_plan` + already emits write_file steps in producer→consumer order, and the proxy agent + loop already executes them in order (accumulating each written file into the + context for downstream nodes). No separate driver needed. +- **Per-node fill = the existing problem-level pipeline, not a new one.** Each + write_file step routes (T2+) to `/v3/generate`, which already runs PlanSearch + (`benchmark/v3/plan_search.py`, arXiv:2409.03733) + DivSampling / S\* / + Derivation Chains. Phase 2's job was to make that call *RPG-aware*: + - `rpg.node_constraints(rpg, file_id)` derives each node's planned interface + (capability, function signatures, incoming/outgoing data-flow edges). + - `flatten_to_plan` attaches `node_id` + `constraints` to each write step. + - Proxy: `PlanStep` gained `NodeID` + `Constraints`; the full graph is parsed + into `Plan.RPG` (`proxy/types.go`). `planConstraintsForTarget` (`proxy/rpg.go`) + maps a file path to its node's constraints, threaded into the + `V3GenerateRequest.Constraints` in `writeFileWithV3` / `improveContentWithV3`. + This realizes the #120 framing literally: RPG = repo/architecture axis (coarse + band), PlanSearch = algorithmic axis. PlanSearch's diversity search now runs + *within* a node whose interface is pinned, not over the whole file. +- **Strictly additive:** flat-planner steps carry no constraints, so + `planConstraintsForTarget` returns nil and generation is byte-identical when + `ATLAS_RPG_PLANNING` is off. +- Tests: `test_rpg.py` (node_constraints + step enrichment), `proxy/rpg_test.go` + (graph + step parsing, path-suffix constraint lookup, copy-safety, nil-plan). + Full proxy `go test` + 119 python tests green. +- **Deferred:** explicit `ast_edit`-primitive biasing / ASA steering from the + fixed node target → Phase 3 (rides with the structural-verification work). +- **Exit (met):** end-to-end, an RPG plan generates files in topological order + with each node's `/v3/generate` call constrained by its planned interface. + +### Phase 3 — Graph-guided verification, localization & drift re-planning — ✅ SHIPPED (core) +**Goal:** close the loop with structure. + +- **Structural verification veto (extend #39 pt 1):** `rpg.verify_node_realization` + / `missing_planned_signatures` — reject a sandbox-passing candidate whose + generated code doesn't define its planned function signatures, not just + "import survives." Python uses stdlib `ast` (precise, methods included); + other languages use a keyword+name regex. Wired into the V3 pipeline `run` + as an **RPG signature veto** right after the existing structural veto: + flag-gated, recovers planned signatures from the request constraints + (`planned_signatures_from_constraints`), and is conservative — never empties + the candidate set, never fires when code is opaque or the flag is off. +- **Localization:** `rpg.localize(rpg, query)` ranks RPG nodes by token overlap + of capability name + path + function names/summaries — the graph-aware + replacement for symbol-name-only matching (#39 pt 4). +- **Drift detection:** `wavelet/diff.py` ported faithfully from `diff.ts` + (`diff_peaks` / `diff_contents`, golden-parity test vs upstream `npx tsx`). + `rpg.node_drift` compares a node's planned functions against generated code + and flags `should_replan` when a planned boundary is missing (conservative on + opaque code). `diff_contents` gives the write-time before/after structural + diff for the #39 "re-run touched files on write" signal. +- Tests: `test_wavelet_diff.py` (port + golden) and Phase-3 additions to + `test_rpg.py` (defined_names, signature extraction/veto, verify, drift, + localize). 145 python tests green; proxy `go test` green. +- **Re-plan loop (drift detection + impact surfacing) — ✅ wired:** the + `/v3/generate` response now carries `rpg_signature_missing` (planned signatures + the *winning* code failed to realize — the veto rejects failing candidates + mid-pipeline, but a winner can still drift when all fell short). Proxy: + `V3GenerateResponse.RPGSignatureMissing` → after a V3 write, `reportRPGDrift` + (`proxy/rpg.go`) computes the affected downstream subgraph (`affectedDownstream`, + BFS over RPG edges, cycle-safe) and emits an `rpg_drift` event naming the + drifted file + the downstream nodes that may need regeneration. Go-tested + (path→node lookup, downstream BFS, cycle safety, event emission). +- **Automatic node-local regeneration — ✅ wired:** `regenerateOnDrift` + (`proxy/rpg.go`) runs one bounded corrective retry when a write drifts — it + re-calls `/v3/generate` with the missing signatures injected as a hard + constraint and keeps the retry only if it realizes strictly more of the plan. + Bounded to a single retry (the V3 pipeline is expensive), no-op when RPG is off + or there was no drift. Tested against a fake generate server (retry-succeeds, + retry-no-better, and no-op cases). Wired into `writeFileWithV3`, the RPG + node-creation path; if drift survives the retry, `reportRPGDrift` still + surfaces the downstream subgraph. +- **Exit (met):** candidates are vetoed against the planned interface; on write, + drift triggers a bounded corrective regeneration, and any surviving drift is + surfaced with its affected subgraph. + +### Phase 4 — Evaluation & rollout — ◑ tooling + rollout shipped; live benchmark pending hardware +**Goal:** evidence before default-on. + +- **Offline metrics harness — ✅ shipped:** `v3-service/rpg_eval.py` scores RPG + artifacts on graph quality (parse/valid/acyclic rates, leaf-coverage, + signature density, score, file/edge/function scale) per-graph and aggregated, + with a CLI (`python rpg_eval.py artifacts/*.json` / `--jsonl`). Tested in + `tests/v3-service/test_rpg_eval.py`. This summarizes a benchmark run's plans + and catches plan-shape regressions in CI without a model. +- **Live RepoCraft comparison — runbook, pending hardware:** the head-to-head + (RPG-on vs flat planner: functional coverage, test accuracy, code scale, L6 + localization turns) needs a GPU + model, so it runs on the benchmark stack, + not in this environment. Runbook: + 1. Pick a multi-file fixture set (RepoCraft-style; start with 2–3 small repos). + 2. Run each task twice — `ATLAS_RPG_PLANNING=0` then `=1` — capturing the + `/v3/plan` artifacts and final pass/coverage. + 3. Feed captured RPG artifacts through `rpg_eval.py` for graph metrics; compare + end-to-end pass/coverage/scale across the two arms (ablation in the style of + `docs/reports/V3_ABLATION_STUDY.md`). + 4. Watch latency: two planning stages + per-node constraints add LLM calls — + gate by tier if the T2 path regresses. +- **Default stays OFF pending that evidence.** `ATLAS_RPG_PLANNING` ships as an + experimental opt-in (documented in `.env.example` + `CHANGELOG.md`). Flip the + default only if the live wins replicate — that's the whole point of "evidence + before default-on." Credit Dmitri (@yogthos) per the issue. +- **Default-flip criteria (proposed):** RPG-on ≥ flat on end-to-end pass rate on + the fixture set, no T2 latency regression beyond budget, and ≥90% valid/acyclic + RPGs from `rpg_eval.py` on the captured artifacts. + +## 5. Risks / open questions + +- **9B model capacity.** RPG/ZeroRepo numbers are from frontier models; a local + 9B may struggle to emit a valid graph. Mitigations: grammar-constrain the RPG + JSON (GBNF, like existing plans), keep nodes small, lean on the coarse band so + the model *recognizes* structure rather than *inventing* it. +- **Graph vs. lighter skeleton** (issue open question). Start with the minimal + graph that still carries edges; don't over-model in Phase 1. +- **Latency.** Two planning stages + per-node generation is more LLM calls. Gate + by tier — only T2+/L6 tasks get the full RPG; keep the flat planner for small + edits. +- **Port drift** — the Python port could silently diverge from upstream wavescope. + Mitigated by the conformance suite + golden fixtures (§3). No drop-in library + preserves wavescope's calibrated behavior, so we port rather than wrap (see §6). +- **Language coverage.** wavescope covers 14 languages for bands; tree-sitter + `ast_edit` is Python/HTML today. RPG file/signature fidelity is best where both + overlap (Python first), generic-fallback elsewhere. + +## 6. Library survey — why we port instead of `pip install` + +We checked for an existing Python library before committing to a port. + +- **`scipy.signal.cwt` / `scipy.signal.ricker`** — *removed in SciPy 1.15* + (deprecated 1.12). Not an option. +- **PyWavelets (`pywt.cwt(sig, scales, "mexh")`)** — the official, maintained + replacement; the `mexh` wavelet *is* the Ricker / Mexican-hat. Provides the + generic transform. +- **Why it's still a port:** wavescope's CWT is a *custom* convolution — + `1/√a` normalization, **reflect** boundary, `±5a` kernel truncation, integer + `t/a` sampling — and its peak step is a bespoke magnitude-sorted **cross-scale + ridge collapse** (`ridgeWindow`). `pywt`'s `mexh` differs in normalization, + sampling, and boundary, so coefficient magnitudes would shift; every calibrated + threshold downstream (`min_coefficient` default `0.3`, band behavior, + `ridgeWindow=2`) is tuned to wavescope's exact scale. Swapping `pywt` in is a + silent behavior fork, not a faithful port. +- **Decision:** port the transform faithfully from the TS; use **`numpy` only** + (vectorize the per-scale convolution). `numpy>=1.26` is already a stack + dependency (`geometric-lens/requirements.txt`), so no new heavyweight import. + `find_peaks` / `find_peaks_cwt` are *not* used — the ridge-collapse logic is + bespoke and ports directly. +- **Not needed here:** wavescope's Haar-DWT entropy/complexity-heatmap path + (`haar.ts`, `wce.ts`) powers `get_complexity_heatmap`, which is out of scope for + planning bands — skip it (and its `libwce` dependency). + +_Sources: SciPy 1.15 release notes (wavelet functions removed); PyWavelets `cwt` +docs / migration guidance._ + +## 7. First concrete step + +Phase 0, the wavelet substrate: port `signal.ts` + `wavelet.ts` + `context.ts` + +`language.ts` into `v3-service/wavelet/` (numpy-backed), stand up the translated +pytest conformance suite + golden fixtures, and confirm v3-service can pull a +coarse structural map of a sample repo in-process within the latency budget — all +behind `ATLAS_RPG_PLANNING`. diff --git a/proxy/rpg.go b/proxy/rpg.go new file mode 100644 index 00000000..1f53cc7e --- /dev/null +++ b/proxy/rpg.go @@ -0,0 +1,186 @@ +package main + +import ( + "fmt" + "path/filepath" + "strings" +) + +// Repository Planning Graph (RPG) integration — proxy side (V3.2, issue #120). +// +// The RPG planner (v3-service/rpg.py) returns a Plan whose write_file steps each +// carry the projected RPG node's planned interface in PlanStep.Constraints. This +// file threads those constraints into the per-node /v3/generate call so node +// generation stays on the planned signatures / data-flow edges — realizing the +// "RPG plans the repo, PlanSearch fills each node" composition. When the flat +// planner ran (ATLAS_RPG_PLANNING off) steps carry no constraints and these +// helpers return nil, leaving generation behavior unchanged. + +// fileProducingActions are the plan-step actions that create or rewrite a file +// and therefore route through V3 generation. +var fileProducingActions = []string{"write_file", "ast_edit", "edit_file"} + +// planConstraintsForTarget returns the RPG node constraints attached to the +// plan step that targets `path`, or nil when there is no plan, no matching +// step, or the step carries no constraints (the flat-planner case). +func planConstraintsForTarget(ctx *AgentContext, path string) []string { + if ctx == nil || ctx.Plan == nil { + return nil + } + for i := range ctx.Plan.Steps { + step := &ctx.Plan.Steps[i] + if len(step.Constraints) == 0 { + continue + } + if !isFileProducingAction(step.Action) { + continue + } + if targetsOverlap(step.Target, path) { + // Copy so callers can append without mutating the plan. + out := make([]string, len(step.Constraints)) + copy(out, step.Constraints) + return out + } + } + return nil +} + +func isFileProducingAction(action string) bool { + for _, a := range fileProducingActions { + if actionMatchesTool(a, action) || actionMatchesTool(action, a) { + return true + } + } + return false +} + +// ── Drift / re-plan loop (V3.2 Phase 3, issue #120) ────────── + +// rpgFileIDForPath returns the RPG node id whose file path matches `path` +// (path-suffix aware), or "" when there is no graph or no match. +func rpgFileIDForPath(plan *Plan, path string) string { + if plan == nil || plan.RPG == nil { + return "" + } + for _, f := range plan.RPG.Files { + if targetsOverlap(f.Path, path) { + return f.ID + } + } + return "" +} + +// affectedDownstream returns the file paths of the RPG nodes reachable +// downstream from `fileID` via data-flow edges (consumers, transitively). +// These are the nodes whose generation depended on the drifted one and may +// need re-planning / regeneration. Deterministic order, no duplicates. +func affectedDownstream(plan *Plan, fileID string) []string { + if plan == nil || plan.RPG == nil || fileID == "" { + return nil + } + adj := map[string][]string{} + for _, e := range plan.RPG.Edges { + adj[e.From] = append(adj[e.From], e.To) + } + pathByID := map[string]string{} + for _, f := range plan.RPG.Files { + pathByID[f.ID] = f.Path + } + seen := map[string]bool{fileID: true} + var out []string + queue := append([]string{}, adj[fileID]...) + for len(queue) > 0 { + id := queue[0] + queue = queue[1:] + if seen[id] { + continue + } + seen[id] = true + if p, ok := pathByID[id]; ok { + out = append(out, p) + } + queue = append(queue, adj[id]...) + } + return out +} + +// regenerateOnDrift performs ONE corrective regeneration when a V3 winning +// candidate failed to realize its planned RPG signatures. It re-runs generation +// with the missing signatures injected as a hard constraint and keeps the retry +// only if it realizes strictly more of the plan; otherwise the original result +// stands. Bounded to a single retry — the V3 pipeline is expensive, so this is +// best-effort node-local self-repair, not a loop. No-op unless the request +// carried RPG constraints and the previous result actually drifted. +func regenerateOnDrift(ctx *AgentContext, req V3GenerateRequest, prev *V3GenerateResponse) *V3GenerateResponse { + if prev == nil || len(prev.RPGSignatureMissing) == 0 || len(req.Constraints) == 0 { + return prev + } + + retryReq := req + corrective := make([]string, 0, len(req.Constraints)+1) + corrective = append(corrective, req.Constraints...) + corrective = append(corrective, + "REQUIRED — the previous attempt omitted these. Define EXACTLY these "+ + "signatures: "+strings.Join(prev.RPGSignatureMissing, "; ")) + retryReq.Constraints = corrective + + if ctx.StreamFn != nil { + ctx.StreamFn("rpg_regen", map[string]interface{}{ + "message": fmt.Sprintf("RPG drift on %s — regenerating once to realize: %v", + filepath.Base(req.FilePath), prev.RPGSignatureMissing), + "file": req.FilePath, + "missing": prev.RPGSignatureMissing, + }) + } + Emit(NewEnvelope(EvtStageStart, "rpg_regen", map[string]interface{}{ + "file": req.FilePath, + "missing": prev.RPGSignatureMissing, + })) + + retried, err := callV3GenerateStreaming(ctx.V3URL, retryReq, + func(stage, detail string, data map[string]interface{}) { + if ctx.StreamFn != nil && stage != "token" { + ctx.StreamFn("v3_progress", map[string]string{ + "message": fmt.Sprintf(" │ [regen:%s] %s", stage, detail), + }) + } + }) + if err != nil || retried == nil || retried.Code == "" { + return prev + } + if len(retried.RPGSignatureMissing) < len(prev.RPGSignatureMissing) { + return retried + } + return prev +} + +// reportRPGDrift surfaces structural drift after a V3 write: the winning code +// for `path` did not realize the planned signatures `missing`. It emits a drift +// event naming the node and the downstream subgraph that depended on it, so the +// agent loop / user can react. (Automatic regeneration of the subgraph rides the +// continuing agent loop; this is the detection + impact-surfacing half.) +func reportRPGDrift(ctx *AgentContext, path string, missing []string) { + if ctx == nil || len(missing) == 0 { + return + } + fileID := rpgFileIDForPath(ctx.Plan, path) + downstream := affectedDownstream(ctx.Plan, fileID) + + Emit(NewEnvelope(EvtStageStart, "rpg_drift", map[string]interface{}{ + "file": path, + "missing": missing, + "downstream": downstream, + })) + if ctx.StreamFn != nil { + msg := fmt.Sprintf("RPG drift: %s did not realize planned signature(s) %v", filepath.Base(path), missing) + if len(downstream) > 0 { + msg += fmt.Sprintf("; downstream nodes may need regeneration: %v", downstream) + } + ctx.StreamFn("rpg_drift", map[string]interface{}{ + "message": msg, + "file": path, + "missing": missing, + "downstream": downstream, + }) + } +} diff --git a/proxy/rpg_test.go b/proxy/rpg_test.go new file mode 100644 index 00000000..f4ce89d3 --- /dev/null +++ b/proxy/rpg_test.go @@ -0,0 +1,277 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "testing" +) + +// A plan result carrying RPG-projected steps (node_id + constraints) and the +// full graph, as v3-service/rpg.py emits it. +const rpgPlanJSON = `{ + "steps": [ + {"id": "s1", "action": "write_file", "target": "src/load.py", "why": "Loading", + "node_id": "f1", "constraints": ["Implement ` + "`def load(p: str) -> list`" + `", "Produces rows consumed by src/process.py"]}, + {"id": "s2", "action": "write_file", "target": "src/process.py", "why": "Processing", + "node_id": "f2", "constraints": ["Consumes rows produced by src/load.py"]}, + {"id": "s3", "action": "run_command", "target": "pytest", "why": "verify"} + ], + "verify_step": "s3", + "rationale": "load feeds process", + "rpg": { + "capabilities": [{"id": "c1", "name": "Core", "parent": null}], + "files": [ + {"id": "f1", "path": "src/load.py", "capability": "c1", + "functions": [{"name": "load", "signature": "def load(p: str) -> list", "summary": "read"}]} + ], + "edges": [{"from": "f1", "to": "f2", "kind": "data_flow", "label": "rows"}], + "verify": "pytest", + "rationale": "load feeds process" + } +}` + +func TestPlanParsesRPGAndStepConstraints(t *testing.T) { + var plan Plan + if err := json.Unmarshal([]byte(rpgPlanJSON), &plan); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if plan.RPG == nil { + t.Fatal("expected RPG to be parsed, got nil") + } + if got, want := len(plan.RPG.Files), 1; got != want { + t.Errorf("RPG files = %d, want %d", got, want) + } + if got := plan.RPG.Files[0].Functions[0].Signature; got != "def load(p: str) -> list" { + t.Errorf("function signature = %q", got) + } + if got, want := len(plan.RPG.Edges), 1; got != want { + t.Errorf("RPG edges = %d, want %d", got, want) + } + if plan.Steps[0].NodeID != "f1" { + t.Errorf("step 0 node_id = %q, want f1", plan.Steps[0].NodeID) + } + if len(plan.Steps[0].Constraints) != 2 { + t.Errorf("step 0 constraints = %v, want 2", plan.Steps[0].Constraints) + } +} + +func TestPlanConstraintsForTarget(t *testing.T) { + var plan Plan + if err := json.Unmarshal([]byte(rpgPlanJSON), &plan); err != nil { + t.Fatalf("unmarshal: %v", err) + } + ctx := &AgentContext{Plan: &plan} + + // Absolute tool path should still match the relative plan target via + // path-suffix overlap. + got := planConstraintsForTarget(ctx, "/workspace/src/process.py") + want := []string{"Consumes rows produced by src/load.py"} + if !reflect.DeepEqual(got, want) { + t.Errorf("process.py constraints = %v, want %v", got, want) + } + + load := planConstraintsForTarget(ctx, "src/load.py") + if len(load) != 2 || !strings.Contains(load[0], "def load") { + t.Errorf("load.py constraints = %v", load) + } + + // A file with no matching plan step → nil. + if got := planConstraintsForTarget(ctx, "src/unrelated.py"); got != nil { + t.Errorf("unrelated constraints = %v, want nil", got) + } + + // Returned slice is a copy — mutating it must not affect the plan. + load[0] = "MUTATED" + if strings.Contains(plan.Steps[0].Constraints[0], "MUTATED") { + t.Error("planConstraintsForTarget returned a non-copy; plan was mutated") + } +} + +func TestPlanConstraintsForTargetNilPlan(t *testing.T) { + if got := planConstraintsForTarget(&AgentContext{}, "x.py"); got != nil { + t.Errorf("nil plan should yield nil, got %v", got) + } + if got := planConstraintsForTarget(nil, "x.py"); got != nil { + t.Errorf("nil ctx should yield nil, got %v", got) + } +} + +func TestRPGFileIDForPath(t *testing.T) { + var plan Plan + if err := json.Unmarshal([]byte(rpgPlanJSON), &plan); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got := rpgFileIDForPath(&plan, "/workspace/src/load.py"); got != "f1" { + t.Errorf("rpgFileIDForPath = %q, want f1", got) + } + if got := rpgFileIDForPath(&plan, "src/nope.py"); got != "" { + t.Errorf("unmatched path should yield \"\", got %q", got) + } + if got := rpgFileIDForPath(nil, "x"); got != "" { + t.Errorf("nil plan should yield \"\", got %q", got) + } +} + +func TestAffectedDownstream(t *testing.T) { + // f1 -> f2 -> f3 chain plus an unrelated f4. + plan := &Plan{RPG: &RPG{ + Files: []RPGFile{ + {ID: "f1", Path: "a.py"}, {ID: "f2", Path: "b.py"}, + {ID: "f3", Path: "c.py"}, {ID: "f4", Path: "d.py"}, + }, + Edges: []RPGEdge{ + {From: "f1", To: "f2"}, {From: "f2", To: "f3"}, + }, + }} + got := affectedDownstream(plan, "f1") + want := []string{"b.py", "c.py"} + if !reflect.DeepEqual(got, want) { + t.Errorf("downstream of f1 = %v, want %v", got, want) + } + // Leaf node has no downstream. + if got := affectedDownstream(plan, "f3"); got != nil { + t.Errorf("downstream of leaf f3 = %v, want nil", got) + } + // Unrelated node, no edges out. + if got := affectedDownstream(plan, "f4"); got != nil { + t.Errorf("downstream of f4 = %v, want nil", got) + } +} + +func TestAffectedDownstreamHandlesCycleSafely(t *testing.T) { + // A malformed cyclic graph must not loop forever. + plan := &Plan{RPG: &RPG{ + Files: []RPGFile{{ID: "f1", Path: "a.py"}, {ID: "f2", Path: "b.py"}}, + Edges: []RPGEdge{{From: "f1", To: "f2"}, {From: "f2", To: "f1"}}, + }} + got := affectedDownstream(plan, "f1") + if !reflect.DeepEqual(got, []string{"b.py"}) { + t.Errorf("cyclic downstream of f1 = %v, want [b.py]", got) + } +} + +func TestReportRPGDriftNoCrashAndEmitsStream(t *testing.T) { + var plan Plan + if err := json.Unmarshal([]byte(rpgPlanJSON), &plan); err != nil { + t.Fatalf("unmarshal: %v", err) + } + var events []string + ctx := &AgentContext{Plan: &plan, StreamFn: func(ev string, _ interface{}) { + events = append(events, ev) + }} + // f1 (src/load.py) drifted; f2 is downstream and should be named. + reportRPGDrift(ctx, "/workspace/src/load.py", []string{"def load(p: str) -> list"}) + if len(events) != 1 || events[0] != "rpg_drift" { + t.Errorf("expected one rpg_drift stream event, got %v", events) + } + // Empty missing → no event. + events = nil + reportRPGDrift(ctx, "src/load.py", nil) + if len(events) != 0 { + t.Errorf("empty missing should emit nothing, got %v", events) + } +} + +// generateServer streams a fixed /v3/generate SSE result and records the +// constraints the proxy sent (so a test can assert the corrective constraint +// was injected on the retry). +func generateServer(t *testing.T, resultJSON string, gotConstraints *[]string) *httptest.Server { + t.Helper() + var mu sync.Mutex + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v3/generate" { + http.NotFound(w, r) + return + } + var body V3GenerateRequest + _ = json.NewDecoder(r.Body).Decode(&body) + if gotConstraints != nil { + mu.Lock() + *gotConstraints = body.Constraints + mu.Unlock() + } + w.Header().Set("Content-Type", "text/event-stream") + f := w.(http.Flusher) + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "event: result\ndata: "+resultJSON+"\n\ndata: [DONE]\n\n") + f.Flush() + })) +} + +func TestRegenerateOnDriftRetriesAndKeepsCleanResult(t *testing.T) { + // regenerateOnDrift makes exactly one (retry) call; have it come back clean. + var sentConstraints []string + srv := generateServer(t, + `{"code": "def load(p):\n return []", "passed": true, "phase_solved": "phase1"}`, + &sentConstraints) + defer srv.Close() + + ctx := &AgentContext{V3URL: srv.URL} + req := V3GenerateRequest{ + FilePath: "load.py", + Constraints: []string{"Implement `def load(p)`"}, + } + prev := &V3GenerateResponse{Code: "def other(): pass", RPGSignatureMissing: []string{"def load(p)"}} + + got := regenerateOnDrift(ctx, req, prev) + if len(got.RPGSignatureMissing) != 0 { + t.Errorf("expected clean retry result, still missing %v", got.RPGSignatureMissing) + } + if !strings.Contains(got.Code, "def load") { + t.Errorf("expected retried code to define load, got %q", got.Code) + } + // The retry must inject the missing signature as a corrective constraint. + joined := strings.Join(sentConstraints, " | ") + if !strings.Contains(joined, "REQUIRED") || !strings.Contains(joined, "def load(p)") { + t.Errorf("retry constraints missing corrective directive: %v", sentConstraints) + } +} + +func TestRegenerateOnDriftNoopWithoutDriftOrConstraints(t *testing.T) { + ctx := &AgentContext{V3URL: "http://127.0.0.1:0"} // never called + clean := &V3GenerateResponse{Code: "ok"} + // No drift → returned unchanged, server never hit. + if got := regenerateOnDrift(ctx, V3GenerateRequest{Constraints: []string{"x"}}, clean); got != clean { + t.Error("clean result should be returned unchanged") + } + // Drift but no RPG constraints (flat planner) → no-op. + drift := &V3GenerateResponse{RPGSignatureMissing: []string{"def f()"}} + if got := regenerateOnDrift(ctx, V3GenerateRequest{}, drift); got != drift { + t.Error("no-constraints drift should be returned unchanged") + } +} + +func TestRegenerateOnDriftKeepsPrevWhenRetryNoBetter(t *testing.T) { + // Server always drifts → retry is no better → keep prev. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + f := w.(http.Flusher) + fmt.Fprint(w, `event: result`+"\n"+`data: {"code":"def other(): pass","rpg_signature_missing":["def load(p)"]}`+"\n\ndata: [DONE]\n\n") + f.Flush() + })) + defer srv.Close() + ctx := &AgentContext{V3URL: srv.URL} + req := V3GenerateRequest{FilePath: "load.py", Constraints: []string{"Implement `def load(p)`"}} + prev := &V3GenerateResponse{Code: "prev", RPGSignatureMissing: []string{"def load(p)"}} + if got := regenerateOnDrift(ctx, req, prev); got != prev { + t.Errorf("retry no better should keep prev, got %+v", got) + } +} + +func TestRunCommandStepNotFileProducing(t *testing.T) { + // A run_command step's args must never be treated as constraints even if + // its target happened to overlap a path-shaped string. + if isFileProducingAction("run_command") { + t.Error("run_command must not be a file-producing action") + } + for _, a := range []string{"write_file", "ast_edit", "edit_file"} { + if !isFileProducingAction(a) { + t.Errorf("%s should be file-producing", a) + } + } +} diff --git a/proxy/tools.go b/proxy/tools.go index 9a4b03ff..ece39a9b 100644 --- a/proxy/tools.go +++ b/proxy/tools.go @@ -549,6 +549,11 @@ func writeFileWithV3(path, baselineContent string, ctx *AgentContext) (*ToolResu req.BuildCommand = ctx.Project.BuildCommand } + // V3.2 RPG (issue #120): if this file maps to an RPG node, thread the + // node's planned interface (signatures, input/output edges) into the + // generation request. Empty for the flat planner. + req.Constraints = planConstraintsForTarget(ctx, path) + // Tell the user V3 is taking over so they don't think the file // vanished. write_file with V3 holds the disk write until V3 picks // a winner \u2014 without this message the chat goes silent for the 1\u20133 @@ -682,6 +687,12 @@ func writeFileWithV3(path, baselineContent string, ctx *AgentContext) (*ToolResu return writeFileDirect(path, baselineContent) } + // V3.2 RPG (issue #120): automatic node-local regeneration on drift. When + // the winning candidate missed its planned signatures, retry once with the + // missing signatures injected as a hard constraint before accepting it. + // No-op when RPG is off (req.Constraints empty) or there was no drift. + v3Result = regenerateOnDrift(ctx, req, v3Result) + // Write the winning candidate (or baseline if V3 didn't improve) code := v3Result.Code if code == "" { @@ -724,6 +735,13 @@ func writeFileWithV3(path, baselineContent string, ctx *AgentContext) (*ToolResu result.WinningScore = v3Result.WinningScore result.PhaseSolved = v3Result.PhaseSolved + // V3.2 RPG drift loop (issue #120): if the winning code failed to realize + // the node's planned signatures, surface the drift + downstream subgraph. + // No-op when RPG is off (the field is empty). + if len(v3Result.RPGSignatureMissing) > 0 { + reportRPGDrift(ctx, path, v3Result.RPGSignatureMissing) + } + return result, nil } @@ -1210,6 +1228,8 @@ func improveContentWithV3(path, content string, ctx *AgentContext) (string, V3Ed req.Framework = ctx.Project.Framework req.BuildCommand = ctx.Project.BuildCommand } + // V3.2 RPG (issue #120): thread RPG node constraints for this target, if any. + req.Constraints = planConstraintsForTarget(ctx, path) // Same callback logic as the write_file V3 path: tokens forward to // the dedicated v3_token SSE event so the TUI updates one streaming diff --git a/proxy/types.go b/proxy/types.go index 28a36d12..48a0df1c 100644 --- a/proxy/types.go +++ b/proxy/types.go @@ -588,6 +588,10 @@ type V3GenerateResponse struct { WinningScore float64 `json:"winning_score"` TotalTokens int `json:"total_tokens"` TotalTimeMs float64 `json:"total_time_ms"` + // RPGSignatureMissing lists planned RPG node signatures the winning code + // failed to realize (V3.2, issue #120). Non-empty signals structural drift + // from the plan — the proxy's re-plan loop surfaces the affected subgraph. + RPGSignatureMissing []string `json:"rpg_signature_missing,omitempty"` } // LensScore is already defined in main.go — reused here. @@ -604,11 +608,20 @@ type V3PlanRequest struct { // PlanStep is a single step in a Plan. Mirrors v3-service/main.py's // PLAN_PROMPT_TEMPLATE shape: id, action, target, why. +// +// NodeID and Constraints are populated only by the V3.2 RPG planner +// (issue #120): when a plan step is the projection of a Repository Planning +// Graph node, Constraints carries that node's planned interface (signatures, +// input/output edges) which the proxy threads into the per-node /v3/generate +// call. They are empty for the flat planner, so behavior is unchanged when +// ATLAS_RPG_PLANNING is off. type PlanStep struct { - ID string `json:"id"` - Action string `json:"action"` - Target string `json:"target"` - Why string `json:"why"` + ID string `json:"id"` + Action string `json:"action"` + Target string `json:"target"` + Why string `json:"why"` + NodeID string `json:"node_id,omitempty"` + Constraints []string `json:"constraints,omitempty"` } // Plan is the structured plan returned by /v3/plan. The agent loop @@ -622,6 +635,48 @@ type Plan struct { WinningScore float64 `json:"winning_score"` WinningIndex int `json:"winning_index"` Reasons []string `json:"reasons"` + // RPG is the full Repository Planning Graph the flat Steps were projected + // from (V3.2, issue #120). Present only when the RPG planner ran; nil for + // the flat planner. Captured proxy-side for graph-guided traversal / + // observability. + RPG *RPG `json:"rpg,omitempty"` +} + +// RPG is the Repository Planning Graph artifact (issue #120, arXiv:2509.16198): +// capabilities (what to build) expanded into files + function signatures + the +// data-flow / ordering edges between them (how). Mirrors v3-service/rpg.py. +type RPG struct { + Capabilities []RPGCapability `json:"capabilities"` + Files []RPGFile `json:"files"` + Edges []RPGEdge `json:"edges"` + Verify string `json:"verify"` + Rationale string `json:"rationale"` +} + +type RPGCapability struct { + ID string `json:"id"` + Name string `json:"name"` + Parent string `json:"parent"` +} + +type RPGFunction struct { + Name string `json:"name"` + Signature string `json:"signature"` + Summary string `json:"summary"` +} + +type RPGFile struct { + ID string `json:"id"` + Path string `json:"path"` + Capability string `json:"capability"` + Functions []RPGFunction `json:"functions"` +} + +type RPGEdge struct { + From string `json:"from"` + To string `json:"to"` + Kind string `json:"kind"` + Label string `json:"label"` } // --------------------------------------------------------------------------- diff --git a/tests/v3-service/test_rpg.py b/tests/v3-service/test_rpg.py new file mode 100644 index 00000000..97541e1c --- /dev/null +++ b/tests/v3-service/test_rpg.py @@ -0,0 +1,389 @@ +"""Tests for the RPG two-stage architecture-first planner (issue #120). + +The LLM is faked via a `complete_fn` that returns canned JSON, so the whole +construction path is exercised without a live llama-server. +""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "v3-service")) + +from rpg import ( # noqa: E402 + Capability, + Edge, + FileSpec, + FunctionSpec, + RPG, + build_implementation_prompt, + build_proposal_prompt, + construct_rpg, + extract_json_object, + flatten_to_plan, + defined_names, + localize, + missing_planned_signatures, + node_constraints, + node_drift, + parse_capabilities, + parse_rpg, + planned_signatures_from_constraints, + score_rpg, + validate_rpg, + verify_node_realization, +) + +# ─── canned model outputs ──────────────────────────────────── + +PROPOSAL_JSON = json.dumps( + { + "capabilities": [ + {"id": "c1", "name": "Data", "parent": None}, + {"id": "c2", "name": "Loading", "parent": "c1"}, + {"id": "c3", "name": "Processing", "parent": "c1"}, + ] + } +) + +IMPL_JSON = json.dumps( + { + "capabilities": [ + {"id": "c1", "name": "Data", "parent": None}, + {"id": "c2", "name": "Loading", "parent": "c1"}, + {"id": "c3", "name": "Processing", "parent": "c1"}, + ], + "files": [ + { + "id": "f1", + "path": "src/load.py", + "capability": "c2", + "functions": [{"name": "load", "signature": "def load(p: str) -> list", "summary": "read"}], + }, + { + "id": "f2", + "path": "src/process.py", + "capability": "c3", + "functions": [{"name": "run", "signature": "def run(rows: list) -> list", "summary": "clean"}], + }, + ], + "edges": [{"from": "f1", "to": "f2", "kind": "data_flow", "label": "rows"}], + "verify": "pytest tests/", + "rationale": "load feeds process", + } +) + + +def _fake_complete(proposal=PROPOSAL_JSON, impl=IMPL_JSON): + calls = {"n": 0} + + def fn(prompt, temperature, max_tokens, seed): + calls["n"] += 1 + return proposal if calls["n"] == 1 else impl + + return fn + + +# ─── JSON extraction ───────────────────────────────────────── + +class TestExtractJson: + def test_plain(self): + assert extract_json_object('{"a": 1}') == {"a": 1} + + def test_fenced(self): + raw = "```json\n{\"a\": 1}\n```" + assert extract_json_object(raw) == {"a": 1} + + def test_leading_prose(self): + assert extract_json_object('Here is the plan: {"a": 1} done') == {"a": 1} + + def test_braces_in_strings(self): + assert extract_json_object('{"a": "has } brace"}') == {"a": "has } brace"} + + def test_garbage(self): + assert extract_json_object("no json here") is None + assert extract_json_object("") is None + + +# ─── parsing ───────────────────────────────────────────────── + +class TestParsing: + def test_parse_capabilities(self): + caps = parse_capabilities(PROPOSAL_JSON) + assert [c.id for c in caps] == ["c1", "c2", "c3"] + assert caps[0].parent is None + assert caps[1].parent == "c1" + + def test_parse_capabilities_drops_incomplete(self): + raw = json.dumps({"capabilities": [{"id": "c1"}, {"name": "x"}, {"id": "c2", "name": "ok"}]}) + caps = parse_capabilities(raw) + assert [c.id for c in caps] == ["c2"] + + def test_parse_rpg(self): + g = parse_rpg(IMPL_JSON) + assert g is not None + assert [f.path for f in g.files] == ["src/load.py", "src/process.py"] + assert g.files[0].functions[0].signature.startswith("def load") + assert g.edges[0].src == "f1" and g.edges[0].dst == "f2" + assert g.verify == "pytest tests/" + + def test_parse_rpg_carries_capabilities_forward(self): + impl_no_caps = json.dumps(json.loads(IMPL_JSON) | {"capabilities": []}) + caps = parse_capabilities(PROPOSAL_JSON) + g = parse_rpg(impl_no_caps, capabilities=caps) + assert [c.id for c in g.capabilities] == ["c1", "c2", "c3"] + + def test_parse_rpg_unparseable(self): + assert parse_rpg("not json") is None + + +# ─── validation / scoring ──────────────────────────────────── + +class TestValidation: + def test_valid_graph(self): + g = parse_rpg(IMPL_JSON) + ok, issues = validate_rpg(g) + assert ok is True + assert issues == [] + + def test_cycle_detected(self): + g = parse_rpg(IMPL_JSON) + g.edges.append(Edge(src="f2", dst="f1")) + ok, issues = validate_rpg(g) + assert ok is False + assert any("cycle" in i for i in issues) + + def test_unknown_edge_target(self): + g = parse_rpg(IMPL_JSON) + g.edges.append(Edge(src="f1", dst="f99")) + ok, issues = validate_rpg(g) + assert ok is False + assert any("unknown file f99" in i for i in issues) + + def test_leaf_without_file(self): + g = RPG( + capabilities=[Capability("c1", "Root"), Capability("c2", "Leaf", "c1")], + files=[FileSpec("f1", "a.py", "c1", [FunctionSpec("x")])], + ) + ok, issues = validate_rpg(g) + assert any("leaf capability c2" in i for i in issues) + + def test_empty_graph_not_ok(self): + ok, issues = validate_rpg(RPG()) + assert ok is False + assert "no files" in issues + + def test_score_ordering(self): + good = parse_rpg(IMPL_JSON) + bad = parse_rpg(IMPL_JSON) + bad.edges.append(Edge(src="f2", dst="f1")) # introduces a cycle + assert score_rpg(good) > score_rpg(bad) + assert 0.0 <= score_rpg(good) <= 1.0 + + +# ─── flatten to plan ───────────────────────────────────────── + +class TestFlatten: + def test_topological_order(self): + g = parse_rpg(IMPL_JSON) + plan = flatten_to_plan(g) + targets = [s["target"] for s in plan["steps"]] + # producer (load) precedes consumer (process); verify last. + assert targets.index("src/load.py") < targets.index("src/process.py") + assert plan["steps"][-1]["action"] == "run_command" + assert plan["steps"][-1]["target"] == "pytest tests/" + assert plan["verify_step"] == plan["steps"][-1]["id"] + + def test_step_ids_sequential(self): + g = parse_rpg(IMPL_JSON) + plan = flatten_to_plan(g) + assert [s["id"] for s in plan["steps"]] == [f"s{i+1}" for i in range(len(plan["steps"]))] + + def test_no_verify_no_verify_step(self): + g = parse_rpg(IMPL_JSON) + g.verify = "" + plan = flatten_to_plan(g) + assert plan["verify_step"] is None + assert all(s["action"] == "write_file" for s in plan["steps"]) + + def test_steps_carry_node_id_and_constraints(self): + g = parse_rpg(IMPL_JSON) + plan = flatten_to_plan(g) + write_steps = [s for s in plan["steps"] if s["action"] == "write_file"] + for s in write_steps: + assert s["node_id"] + assert isinstance(s["constraints"], list) + # The consumer (process.py) lists its planned signature and its input edge. + proc = next(s for s in write_steps if s["target"] == "src/process.py") + joined = " ".join(proc["constraints"]) + assert "def run(rows: list)" in joined + assert "Consumes rows produced by src/load.py" in joined + + +class TestNodeConstraints: + def test_capability_signatures_and_edges(self): + g = parse_rpg(IMPL_JSON) + cons = node_constraints(g, "f1") # src/load.py (producer) + joined = " ".join(cons) + assert "Implements capability: Loading" in joined + assert "Implement `def load(p: str) -> list`" in joined + assert "Produces rows consumed by src/process.py" in joined + + def test_unknown_file_empty(self): + g = parse_rpg(IMPL_JSON) + assert node_constraints(g, "f404") == [] + + +# ─── prompts ───────────────────────────────────────────────── + +class TestPrompts: + def test_proposal_includes_coarse_map(self): + prompt = build_proposal_prompt("build X", coarse_map=[{"label": "class Foo (a.py)"}]) + assert "class Foo (a.py)" in prompt + assert "coarse band" in prompt + + def test_proposal_without_coarse_map(self): + prompt = build_proposal_prompt("build X") + assert "build X" in prompt + assert "coarse band" not in prompt + + def test_implementation_includes_capabilities(self): + caps = parse_capabilities(PROPOSAL_JSON) + prompt = build_implementation_prompt("build X", caps, project_context={"a.py": "x = 1"}) + assert '"c2"' in prompt + assert "a.py" in prompt + + +# ─── two-stage construction ────────────────────────────────── + +class TestConstruct: + def test_happy_path(self): + res = construct_rpg("build a pipeline", _fake_complete()) + assert res.ok is True + assert res.stage_reached == "implementation" + assert res.plan["steps"] + assert res.rpg is not None + assert res.score > 0.5 + + def test_proposal_empty_falls_back(self): + res = construct_rpg("x", _fake_complete(proposal='{"capabilities": []}')) + assert res.ok is False + assert res.stage_reached == "none" + assert res.plan is None + + def test_impl_unparseable_falls_back(self): + res = construct_rpg("x", _fake_complete(impl="garbage, no json")) + assert res.ok is False + assert res.stage_reached == "proposal" + + def test_emit_callback_invoked(self): + events = [] + construct_rpg("x", _fake_complete(), emit=lambda s, d="", **k: events.append(s)) + assert "rpg_proposal_start" in events + assert "rpg_done" in events + + def test_complete_fn_receives_two_calls(self): + seeds = [] + + def fn(prompt, temperature, max_tokens, seed): + seeds.append(seed) + return PROPOSAL_JSON if len(seeds) == 1 else IMPL_JSON + + construct_rpg("x", fn) + assert len(seeds) == 2 # proposal then implementation + + +# ─── Phase 3: verification / drift / localization ──────────── + +class TestDefinedNames: + def test_python_via_ast(self): + code = "import os\n\nclass Foo:\n def bar(self):\n return 1\n\ndef top():\n pass\n" + assert defined_names(code, "m.py") == {"Foo", "bar", "top"} + + def test_python_unparseable_falls_back_to_regex(self): + code = "def load(:\n oops syntax\nclass Broken" + names = defined_names(code, "m.py") + assert "load" in names # regex still recovers the names + + def test_go_regex(self): + code = "package main\nfunc Load() {}\nfunc Run() {}\n" + assert defined_names(code, "m.go") == {"Load", "Run"} + + def test_opaque_code_empty(self): + assert defined_names("x = 1\ny = 2\n", "m.py") == set() + + +class TestPlannedSignatureExtraction: + def test_recovers_signatures_from_constraints(self): + cons = [ + "Implements capability: Loading", + "Implement `def load(p: str) -> list` — read", + "Consumes rows produced by src/x.py", + "Implement `def run(rows)`", + ] + assert planned_signatures_from_constraints(cons) == ["def load(p: str) -> list", "def run(rows)"] + + def test_missing_planned_signatures(self): + code = "def load(p):\n return []\n" + planned = ["def load(p: str) -> list", "def run(rows)"] + assert missing_planned_signatures(code, planned, "m.py") == ["def run(rows)"] + + def test_no_veto_when_code_opaque(self): + # No parseable defs → don't veto (conservative). + assert missing_planned_signatures("x = 1", ["def f()"], "m.py") == [] + + +class TestVerifyNodeRealization: + def test_ok_when_all_present(self): + g = parse_rpg(IMPL_JSON) + code = "def load(p):\n return []\n" + v = verify_node_realization(g, "f1", code, "src/load.py") + assert v.ok is True + assert v.missing_functions == [] + + def test_rejects_missing_function(self): + g = parse_rpg(IMPL_JSON) + code = "def something_else():\n return 1\n" + v = verify_node_realization(g, "f1", code, "src/load.py") + assert v.ok is False + assert v.missing_functions + + def test_unknown_node_ok(self): + g = parse_rpg(IMPL_JSON) + assert verify_node_realization(g, "f404", "x = 1", "x.py").ok is True + + +class TestNodeDrift: + def test_no_drift_when_realized(self): + g = parse_rpg(IMPL_JSON) + d = node_drift(g, "f2", "def run(rows):\n return rows\n", "src/process.py") + assert d.should_replan is False + assert d.drift_score == 0.0 + + def test_drift_when_function_missing(self): + g = parse_rpg(IMPL_JSON) + d = node_drift(g, "f2", "def unrelated():\n return 0\n", "src/process.py") + assert d.should_replan is True + assert d.missing == ["run"] + assert d.drift_score == 1.0 + + def test_opaque_code_no_blind_replan(self): + g = parse_rpg(IMPL_JSON) + d = node_drift(g, "f2", "rows = []", "src/process.py") + assert d.should_replan is False + + +class TestLocalize: + def test_ranks_relevant_nodes(self): + g = parse_rpg(IMPL_JSON) + # "load" should surface f1 (src/load.py, fn load); "process" → f2. + assert localize(g, "the load function is broken")[0] == "f1" + assert localize(g, "fix processing of rows")[0] == "f2" + + def test_empty_query(self): + g = parse_rpg(IMPL_JSON) + assert localize(g, "") == [] + + def test_no_match(self): + g = parse_rpg(IMPL_JSON) + assert localize(g, "zzz nonexistent terms") == [] diff --git a/tests/v3-service/test_rpg_eval.py b/tests/v3-service/test_rpg_eval.py new file mode 100644 index 00000000..f43eebcb --- /dev/null +++ b/tests/v3-service/test_rpg_eval.py @@ -0,0 +1,74 @@ +"""Tests for the offline RPG-quality metrics harness (Phase 4, issue #120).""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "v3-service")) + +from rpg_eval import aggregate_metrics, rpg_quality_metrics # noqa: E402 + +GOOD_RPG = { + "capabilities": [ + {"id": "c1", "name": "Core", "parent": None}, + {"id": "c2", "name": "Loader", "parent": "c1"}, + {"id": "c3", "name": "Proc", "parent": "c1"}, + ], + "files": [ + {"id": "f1", "path": "load.py", "capability": "c2", + "functions": [{"name": "load", "signature": "def load() -> list", "summary": ""}]}, + {"id": "f2", "path": "proc.py", "capability": "c3", + "functions": [{"name": "run", "signature": "def run(x)", "summary": ""}]}, + ], + "edges": [{"from": "f1", "to": "f2", "kind": "data_flow", "label": "rows"}], + "verify": "pytest", + "rationale": "ok", +} + +CYCLIC_RPG = { + **GOOD_RPG, + "edges": [{"from": "f1", "to": "f2"}, {"from": "f2", "to": "f1"}], +} + + +class TestQualityMetrics: + def test_good_graph(self): + m = rpg_quality_metrics(GOOD_RPG) + assert m["parseable"] and m["valid"] and m["acyclic"] + assert m["n_files"] == 2 and m["n_edges"] == 1 + assert m["leaf_coverage"] == 1.0 + assert m["signature_density"] == 1.0 + assert m["has_verify"] is True + assert m["score"] > 0.5 + + def test_cyclic_graph_flagged(self): + m = rpg_quality_metrics(CYCLIC_RPG) + assert m["acyclic"] is False + assert m["valid"] is False + + def test_accepts_plan_envelope(self): + # A /v3/plan result carrying the graph under "rpg" is unwrapped. + m = rpg_quality_metrics({"steps": [], "rpg": GOOD_RPG}) + assert m["parseable"] and m["n_files"] == 2 + + def test_unparseable(self): + assert rpg_quality_metrics({"not": "an rpg"})["parseable"] in (True, False) + + +class TestAggregate: + def test_aggregate_rates(self): + agg = aggregate_metrics([GOOD_RPG, GOOD_RPG, CYCLIC_RPG]) + assert agg["n"] == 3 + assert agg["parse_rate"] == 1.0 + assert agg["valid_rate"] == 2 / 3 + assert agg["acyclic_rate"] == 2 / 3 + assert agg["avg_files"] == 2.0 + assert 0.0 <= agg["mean_score"] <= 1.0 + + def test_empty(self): + agg = aggregate_metrics([]) + assert agg["parse_rate"] == 0.0 + + def test_json_roundtrip_serializable(self): + # Metrics must be JSON-serializable for the CLI output. + json.dumps(aggregate_metrics([GOOD_RPG])) diff --git a/tests/v3-service/test_rpg_integration.py b/tests/v3-service/test_rpg_integration.py new file mode 100644 index 00000000..5bcac3df --- /dev/null +++ b/tests/v3-service/test_rpg_integration.py @@ -0,0 +1,71 @@ +"""Integration test for the RPG wiring inside main.generate_plan. + +Monkeypatches main.LLMAdapter so no llama-server is needed, and verifies the +ATLAS_RPG_PLANNING flag routes planning through the two-stage RPG path (and +that it stays off by default). +""" + +import json +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(PROJECT_ROOT)) +sys.path.insert(0, str(PROJECT_ROOT / "v3-service")) + +import main as v3main # noqa: E402 + +PROPOSAL_JSON = json.dumps( + {"capabilities": [{"id": "c1", "name": "Core", "parent": None}, + {"id": "c2", "name": "Loader", "parent": "c1"}]} +) +IMPL_JSON = json.dumps( + { + "capabilities": [{"id": "c1", "name": "Core", "parent": None}, + {"id": "c2", "name": "Loader", "parent": "c1"}], + "files": [ + {"id": "f1", "path": "load.py", "capability": "c2", + "functions": [{"name": "load", "signature": "def load() -> list", "summary": "read"}]}, + ], + "edges": [], + "verify": "pytest", + "rationale": "single loader file", + } +) + + +class _FakeLLM: + """Stand-in for main.LLMAdapter: returns proposal then implementation.""" + + _shared = {"n": 0} + + def __init__(self, *a, **k): + pass + + def __call__(self, prompt, temperature, max_tokens, seed, thinking=None): + _FakeLLM._shared["n"] += 1 + raw = PROPOSAL_JSON if _FakeLLM._shared["n"] == 1 else IMPL_JSON + return raw, 100, 1.0 + + +def test_flag_off_uses_flat_planner(monkeypatch): + monkeypatch.delenv("ATLAS_RPG_PLANNING", raising=False) + monkeypatch.setattr(v3main, "LLMAdapter", _FakeLLM) + plan = v3main.generate_plan("build a loader", "/nonexistent-dir", {}, n_candidates=1) + # Flat planner path: no RPG artifact attached. + assert "rpg" not in plan + + +def test_flag_on_routes_through_rpg(monkeypatch): + _FakeLLM._shared["n"] = 0 + monkeypatch.setenv("ATLAS_RPG_PLANNING", "1") + monkeypatch.setattr(v3main, "LLMAdapter", _FakeLLM) + plan = v3main.generate_plan("build a loader", "/nonexistent-dir", {}, n_candidates=1) + assert "rpg" in plan + assert plan["winning_index"] == 0 + assert plan["steps"] + # The RPG file becomes a write step; verify command becomes the final step. + targets = [s["target"] for s in plan["steps"]] + assert "load.py" in targets + assert plan["steps"][-1]["target"] == "pytest" + assert plan["rpg"]["files"][0]["path"] == "load.py" diff --git a/tests/v3-service/test_wavelet_context.py b/tests/v3-service/test_wavelet_context.py new file mode 100644 index 00000000..bea9de29 --- /dev/null +++ b/tests/v3-service/test_wavelet_context.py @@ -0,0 +1,115 @@ +"""Conformance suite for FileContext band assembly + important positions. + +Translated/adapted from wavescope-mcp `src/context.test.ts`. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "v3-service")) + +from wavelet.context import FileContext # noqa: E402 + +SAMPLE_PY = """import os +import sys + + +class DataProcessor: + def __init__(self, config): + self.config = config + + def process(self, data): + cleaned = self._clean(data) + return cleaned + + def _clean(self, data): + return [d for d in data if d] + + +def main(): + processor = DataProcessor({}) + print(processor.process([1, 2, 3])) + + +if __name__ == "__main__": + main() +""" + + +class TestConstruction: + def test_empty_content(self): + ctx = FileContext("empty.py", "") + assert ctx.line_count == 0 + assert ctx.get_important_positions() == [] + + def test_trailing_newline_dropped(self): + ctx = FileContext("a.py", "x = 1\n") + assert ctx.line_count == 1 + + def test_no_trailing_newline(self): + ctx = FileContext("a.py", "x = 1\ny = 2") + assert ctx.line_count == 2 + + +class TestImportantPositions: + def test_finds_class_and_defs(self): + ctx = FileContext("sample.py", SAMPLE_PY) + positions = ctx.get_important_positions(min_coefficient=0.3, limit=20) + assert len(positions) > 0 + labels = " ".join(p.label for p in positions) + assert "class DataProcessor" in labels + # All returned positions clear the threshold and sort descending. + for i in range(1, len(positions)): + assert abs(positions[i - 1].coefficient) >= abs(positions[i].coefficient) + + def test_limit_respected(self): + ctx = FileContext("sample.py", SAMPLE_PY) + assert len(ctx.get_important_positions(0.0, 3)) <= 3 + + +class TestQueryWaveletContext: + def test_three_bands_present(self): + ctx = FileContext("sample.py", SAMPLE_PY) + res = ctx.query_wavelet_context(center=8, radius=300) + assert set(res.bands.keys()) == {"fine", "medium", "coarse"} + assert res.bands["fine"].content # raw lines around center + # Fine band is raw source — the center line should appear verbatim. + assert "def process" in res.bands["fine"].content + + def test_center_clamped_when_out_of_range(self): + ctx = FileContext("sample.py", SAMPLE_PY) + res = ctx.query_wavelet_context(center=9999, radius=100) + assert res.clamped is True + assert res.clamped_from == 9999 + assert res.center == ctx.line_count - 1 + + def test_empty_file_query(self): + ctx = FileContext("empty.py", "") + res = ctx.query_wavelet_context(center=5, radius=100) + assert res.clamped is True + assert res.wavelet_peaks == [] + assert res.bands["fine"].content == "" + + def test_fine_band_radius_minimum(self): + # radius//5 would be 2, but fine radius floors at 10 lines each side. + ctx = FileContext("sample.py", SAMPLE_PY) + res = ctx.query_wavelet_context(center=10, radius=10) + start, end = res.bands["fine"].range + assert end - start >= 10 + + +class TestSummaryAtScale: + def test_auto_scale_by_region_size(self): + ctx = FileContext("sample.py", SAMPLE_PY) + assert ctx.auto_scale(0, 40) in ctx.coefficients.scales + # Small region snaps to a small scale, large region to a large one. + assert ctx.auto_scale(0, 10) <= ctx.auto_scale(0, 1000) + + def test_summary_non_empty(self): + ctx = FileContext("sample.py", SAMPLE_PY) + summary = ctx.get_summary_at_scale(0, ctx.line_count - 1) + assert summary + + def test_out_of_range_summary_empty(self): + ctx = FileContext("sample.py", SAMPLE_PY) + assert ctx.get_summary_at_scale(9000, 9999) == "" diff --git a/tests/v3-service/test_wavelet_cwt.py b/tests/v3-service/test_wavelet_cwt.py new file mode 100644 index 00000000..2b31a2a9 --- /dev/null +++ b/tests/v3-service/test_wavelet_cwt.py @@ -0,0 +1,194 @@ +"""Conformance suite for the wavelet CWT port. + +Translated from wavescope-mcp `src/wavelet.test.ts`. Asserts the same behaviors +so the Python port stays faithful to upstream (see +docs/reports/RPG_WAVELET_PLANNING_V3_2.md §3). +""" + +import math +import sys +from pathlib import Path + +import pytest + +# v3-service ships the wavelet package; add it to the path the same way the +# existing v3-service tests add main.py. +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "v3-service")) + +from wavelet.cwt import ricker_wavelet, compute_cwt, detect_peaks # noqa: E402 + + +class TestRickerWavelet: + def test_value_one_at_zero(self): + assert ricker_wavelet(0) == pytest.approx(1.0, abs=1e-5) + + def test_symmetric(self): + for t in (0.5, 1.0, 2.0, 3.0): + assert ricker_wavelet(t) == pytest.approx(ricker_wavelet(-t), abs=1e-8) + + def test_decays_beyond_5(self): + assert abs(ricker_wavelet(5)) < 0.01 + assert abs(ricker_wavelet(8)) < 0.001 + + def test_negative_lobes_and_zero_crossings(self): + assert ricker_wavelet(2) < 0 + assert ricker_wavelet(1) == pytest.approx(0, abs=1e-5) + assert ricker_wavelet(-1) == pytest.approx(0, abs=1e-5) + + +class TestComputeCWT: + def test_dimensions(self): + signal = [0.0] * 100 + scales = [1, 2, 4, 8] + result = compute_cwt(signal, scales) + assert result.scales == scales + assert len(result.coefficients) == len(scales) + assert len(result.coefficients[0]) == len(signal) + + def test_empty_signal(self): + result = compute_cwt([], [1, 2, 4]) + assert result.scales == [1, 2, 4] + assert len(result.coefficients) == 3 + assert all(len(c) == 0 for c in result.coefficients) + + def test_single_spike_position(self): + signal = [0.0] * 100 + signal[50] = 1.0 + result = compute_cwt(signal, [1, 2, 4]) + coeffs = result.coefficients[0] + peak_idx = max(range(len(coeffs)), key=lambda i: abs(coeffs[i])) + assert 48 <= peak_idx <= 52 + + def test_stronger_response_for_larger_signal(self): + small = [0.0] * 100 + small[50] = 0.5 + large = [0.0] * 100 + large[50] = 1.5 + rs = compute_cwt(small, [2]) + rl = compute_cwt(large, [2]) + assert abs(rl.coefficients[0][50]) > abs(rs.coefficients[0][50]) + + def test_smooth_signal_low_coeffs(self): + margin = 64 + signal = [0.5] * 200 + result = compute_cwt(signal, [1, 4, 16]) + for coeffs in result.coefficients: + for i in range(margin, len(coeffs) - margin): + assert abs(coeffs[i]) < 0.05 + + def test_large_scale_kernel_correctness(self): + signal = [0.5] * 4096 + result = compute_cwt(signal, [128]) + coeffs = result.coefficients[0] + margin = 1024 + for i in range(margin, len(coeffs) - margin): + assert abs(coeffs[i]) < 0.01 + + def test_raises_on_nan_scale(self): + with pytest.raises(ValueError): + compute_cwt([1, 2, 3], [math.nan]) + + def test_raises_on_inf_scale(self): + with pytest.raises(ValueError): + compute_cwt([1, 2, 3], [math.inf]) + + def test_dedup_repeated_scales(self): + signal = [0.0] * 50 + signal[25] = 1 + result = compute_cwt(signal, [1, 1, 2, 2, 4]) + assert result.scales == [1, 2, 4] + assert len(result.coefficients) == 3 + + def test_reflect_boundary_near_zero_at_edges(self): + signal = [0.5] * 200 + result = compute_cwt(signal, [4, 8, 16]) + for coeffs in result.coefficients: + assert abs(coeffs[0]) < 0.05 + assert abs(coeffs[-1]) < 0.05 + + def test_zero_boundary_back_compat(self): + signal = [0.5] * 200 + result = compute_cwt(signal, [16], boundary="zero") + assert len(result.coefficients[0]) == 200 + + +class TestDetectPeaks: + def test_finds_peaks_above_threshold(self): + signal = [0.0] * 200 + signal[50] = 1.0 + signal[100] = 1.0 + signal[150] = 0.3 + result = compute_cwt(signal, [1, 2, 4, 8, 16, 32]) + peaks = detect_peaks(result, 0.5) + assert len(peaks) > 0 + positions = [p.position for p in peaks] + assert any(48 <= p <= 52 for p in positions) + assert any(98 <= p <= 102 for p in positions) + + def test_empty_when_none_above_threshold(self): + signal = [0.1] * 50 + result = compute_cwt(signal, [1, 2, 4]) + assert detect_peaks(result, 10.0) == [] + + def test_sorted_descending(self): + signal = [0.0] * 100 + signal[30] = 0.5 + signal[60] = 2.0 + signal[80] = 1.0 + result = compute_cwt(signal, [1, 2, 4]) + peaks = detect_peaks(result, 0.3) + for i in range(1, len(peaks)): + assert abs(peaks[i - 1].coefficient) >= abs(peaks[i].coefficient) + + def test_collapses_cross_scale_ridges(self): + signal = [0.0] * 200 + signal[100] = 1.0 + result = compute_cwt(signal, [1, 2, 4, 8, 16, 32, 64, 128]) + peaks = detect_peaks(result, 0.1) + near = [p for p in peaks if abs(p.position - 100) <= 2] + assert len(near) == 1 + + def test_disable_collapse_preserves_scales(self): + signal = [0.0] * 200 + signal[100] = 1.0 + result = compute_cwt(signal, [1, 2, 4, 8, 16, 32, 64, 128]) + collapsed = detect_peaks(result, 0.1) + every = detect_peaks(result, 0.1, 1000, -1) + near_collapsed = [p for p in collapsed if abs(p.position - 100) <= 2] + near_all = [p for p in every if abs(p.position - 100) <= 2] + assert len(near_collapsed) == 1 + assert len(near_all) > 1 + assert len({p.scale for p in near_all}) > 1 + + +class TestGoldenFixture: + """Numeric parity with upstream wavescope-mcp src/wavelet.ts. + + Values captured by running the actual upstream TypeScript via + `npx tsx` on the signal below (spikes of 1.0 at index 20, 0.5 at 40 over a + 64-sample zero signal). The Python port reproduces them to 8 decimals; + any divergence flags a behavioral fork from upstream. + """ + + def test_coefficients_match_upstream(self): + signal = [0.0] * 64 + signal[20] = 1.0 + signal[40] = 0.5 + result = compute_cwt(signal, [1, 2, 4, 8]) + c1 = [round(x, 8) for x in result.coefficients[0][18:23]] + c2 = [round(x, 8) for x in result.coefficients[1][38:43]] + assert c1 == [-0.40600585, 0.0, 1.0, 0.0, -0.40600585] + assert c2 == [0.0, 0.23400733, 0.35355339, 0.23400733, 0.0] + + def test_peaks_match_upstream(self): + signal = [0.0] * 64 + signal[20] = 1.0 + signal[40] = 0.5 + peaks = detect_peaks(compute_cwt(signal, [1, 2, 4, 8]), 0.1) + got = [(p.position, round(p.coefficient, 8), p.scale) for p in peaks[:4]] + assert got == [ + (20, 1.0, 1), + (40, 0.5, 1), + (16, -0.28708949, 2), + (24, -0.28708949, 2), + ] diff --git a/tests/v3-service/test_wavelet_diff.py b/tests/v3-service/test_wavelet_diff.py new file mode 100644 index 00000000..71957907 --- /dev/null +++ b/tests/v3-service/test_wavelet_diff.py @@ -0,0 +1,80 @@ +"""Conformance suite for the peak-profile diff port (drift detector). + +Translated from wavescope-mcp `src/diff.test.ts`, plus a golden-fixture test +asserting parity with the actual upstream `diffPeaks` (captured via `npx tsx`). +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "v3-service")) + +from wavelet.context import ImportantPosition # noqa: E402 +from wavelet.diff import diff_peaks, diff_contents # noqa: E402 + + +def _p(position, coefficient, scale=2, label=""): + return ImportantPosition(position=position, coefficient=coefficient, scale=scale, label=label) + + +class TestDiffPeaks: + def test_identical_all_unchanged(self): + peaks = [_p(5, 1.0), _p(20, 0.8)] + d = diff_peaks(list(peaks), list(peaks)) + assert d.summary["unchanged"] == 2 + assert d.summary["added"] == 0 + assert d.summary["removed"] == 0 + + def test_added(self): + d = diff_peaks([_p(5, 1.0)], [_p(5, 1.0), _p(50, 0.7)]) + assert d.summary["added"] == 1 + assert any(c.kind == "added" and c.after.position == 50 for c in d.changes) + + def test_removed(self): + d = diff_peaks([_p(5, 1.0), _p(50, 0.7)], [_p(5, 1.0)]) + assert d.summary["removed"] == 1 + assert any(c.kind == "removed" and c.before.position == 50 for c in d.changes) + + def test_shifted_within_window(self): + d = diff_peaks([_p(20, 0.8)], [_p(21, 0.8)], window=2) + assert d.summary["shifted"] == 1 + + def test_shift_beyond_window_is_add_remove(self): + d = diff_peaks([_p(20, 0.8)], [_p(30, 0.8)], window=2) + assert d.summary["shifted"] == 0 + assert d.summary["added"] == 1 + assert d.summary["removed"] == 1 + + def test_magnitude_changed(self): + d = diff_peaks([_p(10, 0.5)], [_p(10, 1.5)]) + assert d.summary["magnitudeChanged"] == 1 + + def test_golden_matches_upstream(self): + # Captured from upstream src/diff.ts via `npx tsx`. + before = [_p(5, 1.0, 2), _p(20, 0.8, 4), _p(40, 0.5, 8)] + after = [_p(5, 1.0, 2), _p(21, 0.8, 4), _p(60, 0.9, 8)] + d = diff_peaks(before, after, 2) + assert d.summary == { + "added": 1, "removed": 1, "shifted": 1, "magnitudeChanged": 0, "unchanged": 1, + } + assert [c.kind for c in d.changes] == ["unchanged", "shifted", "removed", "added"] + + +class TestDiffContents: + def test_added_function_shows_up(self): + before = "def a():\n return 1\n" + after = "def a():\n return 1\n\n\ndef b():\n return 2\n" + d = diff_contents(before, after, "m.py") + assert d.diff.summary["added"] >= 1 + assert d.after_line_count > d.before_line_count + + def test_identical_contents_no_churn(self): + src = "class Foo:\n def bar(self):\n return 1\n" + d = diff_contents(src, src, "m.py") + assert d.diff.summary["added"] == 0 + assert d.diff.summary["removed"] == 0 + + def test_empty_to_content_all_added(self): + d = diff_contents("", "def a():\n return 1\n", "m.py") + assert d.diff.summary["removed"] == 0 + assert d.before_line_count == 0 diff --git a/tests/v3-service/test_wavelet_flags.py b/tests/v3-service/test_wavelet_flags.py new file mode 100644 index 00000000..f84868d2 --- /dev/null +++ b/tests/v3-service/test_wavelet_flags.py @@ -0,0 +1,25 @@ +"""Tests for the ATLAS_RPG_PLANNING feature-flag contract.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "v3-service")) + +from wavelet.flags import rpg_planning_enabled, ENV_VAR # noqa: E402 + + +def test_default_off(monkeypatch): + monkeypatch.delenv(ENV_VAR, raising=False) + assert rpg_planning_enabled() is False + + +def test_truthy_values(monkeypatch): + for v in ("1", "true", "TRUE", "yes", "on", " On "): + monkeypatch.setenv(ENV_VAR, v) + assert rpg_planning_enabled() is True + + +def test_falsy_values(monkeypatch): + for v in ("0", "false", "no", "off", ""): + monkeypatch.setenv(ENV_VAR, v) + assert rpg_planning_enabled() is False diff --git a/tests/v3-service/test_wavelet_project.py b/tests/v3-service/test_wavelet_project.py new file mode 100644 index 00000000..0a2ff13d --- /dev/null +++ b/tests/v3-service/test_wavelet_project.py @@ -0,0 +1,123 @@ +"""Conformance suite for project-wide decomposition + discovery. + +Adapted from wavescope-mcp `src/project.test.ts`. Exercises discovery +(extension filtering, SKIP_DIRS, .gitignore, binary sniff) and project-wide +important-position aggregation against a temp tree. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "v3-service")) + +from wavelet.project import decompose_project, ProjectIndex # noqa: E402 + +PY_A = """class Alpha: + def run(self): + return 1 + + +def helper_a(): + return 2 +""" + +PY_B = """import os + + +class Beta: + def go(self): + return os.getcwd() +""" + + +def _write(root: Path, rel: str, content: str) -> None: + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content, encoding="utf-8") + + +class TestDiscovery: + def test_indexes_code_files(self, tmp_path): + _write(tmp_path, "a.py", PY_A) + _write(tmp_path, "pkg/b.py", PY_B) + idx = ProjectIndex.load(str(tmp_path)) + files = set(idx.list_files()) + assert "a.py" in files + assert str(Path("pkg/b.py")) in files + + def test_skips_non_code_extensions(self, tmp_path): + _write(tmp_path, "a.py", PY_A) + _write(tmp_path, "notes.md", "# notes") + _write(tmp_path, "data.json", "{}") + idx = ProjectIndex.load(str(tmp_path)) + files = idx.list_files() + assert "a.py" in files + assert "notes.md" not in files + assert "data.json" not in files + + def test_skips_skip_dirs(self, tmp_path): + _write(tmp_path, "a.py", PY_A) + _write(tmp_path, "node_modules/dep.py", PY_B) + _write(tmp_path, "__pycache__/x.py", PY_B) + files = ProjectIndex.load(str(tmp_path)).list_files() + assert "a.py" in files + assert all("node_modules" not in f for f in files) + assert all("__pycache__" not in f for f in files) + + def test_honors_gitignore(self, tmp_path): + _write(tmp_path, "a.py", PY_A) + _write(tmp_path, "ignored.py", PY_B) + _write(tmp_path, "sub/keep.py", PY_A) + _write(tmp_path, ".gitignore", "ignored.py\n") + files = ProjectIndex.load(str(tmp_path)).list_files() + assert "a.py" in files + assert "ignored.py" not in files + assert str(Path("sub/keep.py")) in files + + def test_gitignore_negation(self, tmp_path): + # Last-match-wins: `!keep.py` re-includes after `*.py` excludes it. + _write(tmp_path, "keep.py", PY_A) + _write(tmp_path, "skip.py", PY_B) + _write(tmp_path, ".gitignore", "*.py\n!keep.py\n") + files = ProjectIndex.load(str(tmp_path)).list_files() + assert "keep.py" in files + assert "skip.py" not in files + + def test_gitignore_dir_prune_blocks_negation(self, tmp_path): + # Faithful to git (and wavescope): a file under an ignored *directory* + # cannot be re-included by negation — the directory is pruned during + # the walk, so descent never happens. + _write(tmp_path, "build/keep.py", PY_A) + _write(tmp_path, ".gitignore", "build/\n!build/keep.py\n") + files = ProjectIndex.load(str(tmp_path)).list_files() + assert str(Path("build/keep.py")) not in files + + def test_skips_binary(self, tmp_path): + _write(tmp_path, "a.py", PY_A) + (tmp_path / "weird.py").write_bytes(b"def x():\n return 0\x00\x00binary") + files = ProjectIndex.load(str(tmp_path)).list_files() + assert "a.py" in files + assert "weird.py" not in files + + +class TestProjectImportantPositions: + def test_aggregates_across_files_with_path_labels(self, tmp_path): + _write(tmp_path, "a.py", PY_A) + _write(tmp_path, "b.py", PY_B) + positions = decompose_project(str(tmp_path), min_coefficient=0.3, limit=20) + assert len(positions) > 0 + # Labels carry the relative filename suffix; every entry has a filename. + assert all(p.filename for p in positions) + labels = " ".join(p.label for p in positions) + assert "(a.py)" in labels or "(b.py)" in labels + # Sorted by magnitude descending and capped at the limit. + for i in range(1, len(positions)): + assert abs(positions[i - 1].coefficient) >= abs(positions[i].coefficient) + + def test_limit_caps_results(self, tmp_path): + _write(tmp_path, "a.py", PY_A) + _write(tmp_path, "b.py", PY_B) + assert len(decompose_project(str(tmp_path), 0.0, 2)) <= 2 + + def test_empty_project(self, tmp_path): + assert decompose_project(str(tmp_path)) == [] diff --git a/tests/v3-service/test_wavelet_signal.py b/tests/v3-service/test_wavelet_signal.py new file mode 100644 index 00000000..a4639d09 --- /dev/null +++ b/tests/v3-service/test_wavelet_signal.py @@ -0,0 +1,248 @@ +"""Conformance suite for the structural-signal port. + +Translated from wavescope-mcp `src/signal.test.ts`. +""" + +import math +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "v3-service")) + +from wavelet.signal import compute_signal # noqa: E402 +from wavelet.language import detect_language # noqa: E402 + +PY = detect_language("test.py") +TS = detect_language("test.ts") +GO = detect_language("test.go") + + +class TestPython: + def test_zero_for_blank_and_comments(self): + lines = ["", " ", "# this is a comment", " # indented comment"] + assert compute_signal(lines, PY) == [0, 0, 0, 0] + + def test_zero_for_docstrings(self): + lines = ['"""Module docstring"""', '"""', "Multi-line", "docstring", '"""', "def foo(): pass"] + sig = compute_signal(lines, PY) + assert sig[0] == 0 + assert sig[1] == 0 + assert sig[2] == 0 + assert sig[3] == 0 + assert sig[4] == 0 + + def test_class_and_def_high_signal(self): + lines = [ + "import os", + "", + "class DataProcessor:", + " def __init__(self, config):", + " pass", + " def process(self, data):", + " return data", + "", + "if __name__ == '__main__':", + " main()", + ] + sig = compute_signal(lines, PY) + assert sig[0] > 0.5 + assert sig[0] <= 2.0 + assert sig[1] == 0 + assert 1.0 <= sig[2] <= 2.0 + assert sig[3] > 0.9 + assert sig[4] < 0.5 + assert sig[5] > 0.9 + assert sig[6] > 0.1 + assert sig[7] == 0 + assert sig[8] > 0.2 + + def test_decorators(self): + sig = compute_signal(["@staticmethod", "def helper():", " pass"], PY) + assert 0.4 < sig[0] <= 2.0 + + def test_indentation_increases_signal(self): + sig = compute_signal(["pass", " pass", " pass", " pass"], PY) + assert sig[0] < sig[1] < sig[2] < sig[3] + + def test_keyword_touching_parens(self): + sig = compute_signal(["for(x in y):", "if(cond):", "while(True):"], PY) + assert all(s >= 0.3 for s in sig) + + def test_async_def_not_exceed_class(self): + class_sig = compute_signal(["class Foo:"], PY)[0] + async_sig = compute_signal(["async def foo():"], PY)[0] + assert async_sig <= class_sig + + def test_member_access_no_keyword_leak(self): + assert compute_signal(["x = obj.class"], PY)[0] == 0 + + +class TestTypeScript: + def test_zero_for_blank_and_comments(self): + lines = ["", " ", "// this is a comment", " // indented comment"] + assert compute_signal(lines, TS) == [0, 0, 0, 0] + + def test_zero_for_block_comments(self): + lines = ["/* block comment */", "/* start", "middle", "end */", "const x = 1;"] + sig = compute_signal(lines, TS) + assert sig[0] == 0 + assert sig[1] == 0 + assert sig[2] == 0 + assert sig[3] == 0 + assert sig[4] > 0 + + def test_detects_class_function_interface_enum(self): + lines = [ + "import { readFile } from 'fs';", + "", + "export class Service {", + " constructor(private config: Config) {}", + "", + " public async process(data: unknown): Promise {", + " const result = await this.transform(data);", + " return result;", + " }", + "}", + "", + "export interface Config {", + " host: string;", + "}", + "", + "export enum Status {", + " Active,", + " Inactive,", + "}", + ] + sig = compute_signal(lines, TS) + assert sig[0] > 0.5 + assert sig[2] >= 1.0 + assert sig[5] > 0.5 + assert sig[11] >= 0.9 + assert sig[15] >= 0.8 + + def test_keyword_adjacent_punctuation(self): + lines = ["export function foo(", "export class Bar{", "if(!ready)", "for(let x=0; x<10; x++)"] + sig = compute_signal(lines, TS) + assert sig[0] >= 1.0 + assert sig[1] >= 1.0 + assert sig[2] > 0.2 + assert sig[3] > 0.2 + + def test_string_literal_block_comment_immunity(self): + sig = compute_signal(['const s = "/* not a real comment */";', "const x = 1;"], TS) + assert sig[1] > 0 + + def test_url_string_not_stripped(self): + sig = compute_signal(['export const url = "https://example.com";'], TS) + assert sig[0] > 0.5 + + def test_template_literal_block_comment_immunity(self): + sig = compute_signal(["const t = `/* nope */`;", "function next() {}"], TS) + assert sig[1] > 0.5 + + def test_member_access_no_keyword_leak(self): + assert compute_signal(["return obj.def;"], TS)[0] < 0.3 + + def test_object_prototype_keys_no_nan(self): + lines = [ + " constructor(filename: string, content: string) {", + " toString() {", + " hasOwnProperty(k: string) {", + " valueOf() {", + ] + for s in compute_signal(lines, TS): + assert not math.isnan(s) + assert math.isfinite(s) + + +class TestGo: + def test_func_keyword(self): + sig = compute_signal(["func main() {", ' fmt.Println("hello")', "}"], GO) + assert sig[0] >= 0.9 + + def test_tab_indentation_matches_spaces(self): + tab = compute_signal(['\t\tfmt.Println("x")'], GO)[0] + space = compute_signal([' fmt.Println("x")'], GO)[0] + assert tab == pytest.approx(space, abs=1e-5) + + +class TestRange: + def test_all_within_zero_two(self): + lines = [" " * 30 + "class Foo:" for _ in range(100)] + for s in compute_signal(lines, PY): + assert 0 <= s <= 2.0 + + +class TestRustClojureQuote: + def test_rust_lifetime_not_masking(self): + rust = detect_language("test.rs") + sig = compute_signal(["impl<'a> Foo for Bar<'a> {"], rust) + assert sig[0] >= 1.0 + + def test_clojure_quote_not_masking(self): + clj = detect_language("test.clj") + sig = compute_signal(["(def things '[a b c]) (defn realfn [] 1)"], clj) + assert sig[0] >= 0.9 + + +class TestPHP: + def test_attribute_not_comment(self): + php = detect_language("test.php") + assert compute_signal(["#[Route('/x')]", "function handler() {}"], php)[0] > 0 + + def test_plain_hash_is_comment(self): + php = detect_language("test.php") + assert compute_signal(["# real comment"], php)[0] == 0 + + +class TestLanguageDetection: + def test_js_vs_ts(self): + assert detect_language("foo.js").name == "javascript" + assert detect_language("foo.jsx").name == "javascript" + assert detect_language("foo.mjs").name == "javascript" + assert detect_language("foo.cjs").name == "javascript" + assert detect_language("foo.ts").name == "typescript" + assert detect_language("foo.tsx").name == "typescript" + + def test_js_lacks_ts_keywords(self): + js = detect_language("foo.js") + assert "interface" not in js.structural_keywords + assert "enum" not in js.structural_keywords + + def test_edn_is_clojure_not_generic(self): + assert detect_language("foo.edn").name != "generic" + + def test_unknown_ext_generic_fallthrough(self): + lang = detect_language("foo.unknownext") + assert ";" not in lang.comment_prefixes + assert "//" not in lang.comment_prefixes + + def test_pyi_and_ruby_filenames(self): + assert detect_language("stubs.pyi").name == "python" + assert detect_language("Rakefile").name == "ruby" + assert detect_language("Gemfile").name == "ruby" + + +class TestClojureForms: + def test_recognizes_structural_forms(self): + clj = detect_language("test.clj") + lines = [ + "(defmulti area :shape)", + "(defonce server (start))", + "(letfn [(helper [x] x)] ...)", + "(reify Foo (bar [_] 1))", + "(extend-type String Foo (bar [_] 1))", + "(extend-protocol Foo String (bar [_] 1))", + ] + for s in compute_signal(lines, clj): + assert s > 0.3 + + +class TestJavaAnnotations: + def test_inline_annotation_raises_signal(self): + java = detect_language("test.java") + a = compute_signal(["public @Nullable String foo() {}"], java)[0] + b = compute_signal(["public String foo() {}"], java)[0] + assert a > b diff --git a/v3-service/Dockerfile b/v3-service/Dockerfile index b72f5eed..fe5423ae 100644 --- a/v3-service/Dockerfile +++ b/v3-service/Dockerfile @@ -8,6 +8,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf # Copy benchmark modules (V3 pipeline depends on them) COPY benchmark/ /app/benchmark/ COPY v3-service/main.py /app/main.py +# V3.2 RPG-style planning (issue #120): wavelet substrate + RPG construction. +# Pure stdlib — no extra pip installs needed. +COPY v3-service/wavelet/ /app/wavelet/ +COPY v3-service/rpg.py /app/rpg.py # Install Python deps RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu diff --git a/v3-service/main.py b/v3-service/main.py index 630ed588..ed407dd8 100644 --- a/v3-service/main.py +++ b/v3-service/main.py @@ -874,7 +874,7 @@ def __init__(self): def run(self, problem: str, task_id: str = "cli", progress_callback=None, files: Dict[str, str] = None, - file_path: str = "") -> Dict[str, Any]: + file_path: str = "", constraints: List[str] = None) -> Dict[str, Any]: """Run the full V3 pipeline on a coding problem. Args: @@ -885,10 +885,15 @@ def run(self, problem: str, task_id: str = "cli", file_path: Target file path (used by PC-048 to detect language for the smoke check — `.html` files use HTML parser, not Python compile, etc.) + constraints: raw constraint strings from the request. When the V3.2 + RPG planner is active (issue #120) these carry the node's planned + signatures, used by the RPG signature veto below. Backward + compatible — defaults to none, leaving existing callers unchanged. """ start = time.time() events = [] files = files or {} + constraints = constraints or [] # PC-048: derive language from the target file's extension. Used # only by smoke_compile_check below to pick the right parser @@ -1317,6 +1322,45 @@ def verified_sandbox(code, extra_test=""): ) passing = kept + # ===== RPG SIGNATURE VETO (V3.2, issue #120) ===== + # When the RPG planner is active, the request constraints carry the + # node's planned signatures. Extend the #39-pt-1 veto from "imports + # survive" to "the planned interface exists": reject sandbox-passing + # candidates that don't define the planned functions. Additive and + # conservative — only fires when the flag is on AND constraints carry + # planned signatures, and never empties the candidate set (so it can't + # do worse than the flat path; a fully-failing set falls through intact + # to phase-3 repair). + if passing and constraints: + try: + from wavelet import rpg_planning_enabled as _rpg_on + _rpg_active = _rpg_on() + except Exception: + _rpg_active = False + if _rpg_active: + try: + import rpg as _rpgmod + planned_sigs = _rpgmod.planned_signatures_from_constraints(constraints) + except Exception: + planned_sigs = [] + if planned_sigs: + sig_kept = [] + for c in passing: + missing = _rpgmod.missing_planned_signatures( + c.get("code", ""), planned_sigs, file_path) + if missing: + emit("rpg_signature_veto", + f"Candidate {c['index']} sandbox-passed but missing " + f"planned signature(s): {', '.join(missing[:3])}", + index=c["index"], missing=missing[:5]) + print(f" [rpg] vetoed cand {c['index']} — missing: {missing[:5]}", + flush=True) + continue + sig_kept.append(c) + # Only prune when at least one candidate realizes the plan. + if sig_kept: + passing = sig_kept + # ===== CANDIDATE SELECTION ===== if passing: # S* tiebreaking if multiple passing candidates @@ -1939,6 +1983,57 @@ def emit(stage: str, detail: str = "", **data): emit("plan_start", f"generating {n_candidates} candidate plans") + # V3.2 RPG-style architecture-first planning (issue #120), flag-gated by + # ATLAS_RPG_PLANNING. Strictly additive: on any failure (flag off, modules + # missing, model output unusable) we fall through to the flat planner below. + try: + from wavelet import rpg_planning_enabled, decompose_project + import rpg as _rpg_mod + _rpg_on = rpg_planning_enabled() + except Exception: + _rpg_on = False + if _rpg_on: + try: + coarse_map = None + if working_dir and os.path.isdir(working_dir): + try: + coarse_map = [ + {"label": p.label, "filename": p.filename} + for p in decompose_project(working_dir, limit=30) + ] + except Exception as ce: + emit("rpg_coarse_error", f"coarse decomposition failed: {ce}") + rpg_thinking = os.environ.get("ATLAS_PLAN_THINKING", "0").lower() in ("1", "true", "yes") + rpg_llm = LLMAdapter(progress_callback=progress_callback, thinking=rpg_thinking) + + def _complete(prompt, temperature, mt, seed): + raw, _, _ = rpg_llm(prompt, temperature, mt, seed) + return raw + + result = _rpg_mod.construct_rpg( + user_message=user_message, + complete_fn=_complete, + project_context=project_context, + coarse_map=coarse_map, + max_tokens=(8192 if rpg_thinking else 2048), + emit=emit, + ) + if result.ok and result.plan and result.plan.get("steps"): + plan = dict(result.plan) + plan["candidates_tested"] = 1 + plan["winning_score"] = result.score + plan["winning_index"] = 0 + plan["reasons"] = ["RPG two-stage plan"] + result.issues + plan["rpg"] = result.rpg.to_dict() + emit("plan_selected", + f"RPG plan ({len(plan['steps'])} steps, score={result.score:.2f})", + index=0, score=result.score, steps=len(plan["steps"])) + return plan + emit("rpg_fallback", + f"RPG not usable (stage={result.stage_reached}); using flat planner") + except Exception as re_: + emit("rpg_error", f"RPG planning failed: {re_}; using flat planner") + # PC-206: thinking-aware infrastructure shipped — planner CAN run with # Qwen3.5 hybrid reasoning ON via ATLAS_PLAN_THINKING=1. Default is OFF # because empirically on the reference Qwen3.5-9B-Q6_K + this codebase's @@ -2928,6 +3023,7 @@ def emit_progress(stage, detail="", **data): progress_callback=emit_progress, files=files, file_path=file_path, # PC-048: language-aware smoke check + constraints=constraints, # V3.2: RPG signature veto (issue #120) ) _post_pattern_outcome(problem, result) @@ -2947,6 +3043,30 @@ def emit_progress(stage, detail="", **data): "total_tokens": result.get("total_tokens", 0), "total_time_ms": result.get("total_time_ms", 0.0), } + + # V3.2 RPG (issue #120): report which planned signatures the WINNING + # code failed to realize, so the proxy's re-plan loop can react (the + # signature veto rejects failing candidates mid-pipeline, but the + # winner can still drift when every candidate fell short and one was + # kept anyway). Flag-gated and conservative — empty unless RPG is on, + # constraints carry signatures, and a planned function is genuinely + # absent from parseable code. + try: + from wavelet import rpg_planning_enabled as _rpg_on + _rpg_active = _rpg_on() + except Exception: + _rpg_active = False + if _rpg_active and response["code"] and constraints: + try: + import rpg as _rpgmod + planned_sigs = _rpgmod.planned_signatures_from_constraints(constraints) + missing = _rpgmod.missing_planned_signatures( + response["code"], planned_sigs, file_path) + if missing: + response["rpg_signature_missing"] = missing + except Exception as _e: + print(f" [rpg] drift check skipped: {_e}", flush=True) + final = json.dumps(response) try: self.wfile.write(f"event: result\ndata: {final}\n\n".encode()) diff --git a/v3-service/rpg.py b/v3-service/rpg.py new file mode 100644 index 00000000..efa77659 --- /dev/null +++ b/v3-service/rpg.py @@ -0,0 +1,745 @@ +"""Repository Planning Graph (RPG) — two-stage architecture-first planning. + +Implements the V3.2 RPG-style plan-then-fill flow (issue #120), grounded in +*RPG: A Repository Planning Graph for Unified and Scalable Codebase Generation* +(arXiv:2509.16198). See docs/reports/RPG_WAVELET_PLANNING_V3_2.md. + +Two stages: + A. Proposal-level — *what* to build: a capability tree (modules -> components + -> leaf capabilities). On L6 (existing repos) the tree is seeded with the + wavelet coarse band so capabilities map onto real modules. + B. Implementation — *how*: expand leaf capabilities into files, function + signatures, and the data-flow / ordering edges between them -> the RPG. + +The module is dependency-free and LLM-agnostic: callers pass a `complete_fn` +that maps (prompt, temperature, max_tokens, seed) -> raw text, so construction +is fully unit-testable with a fake model. The RPG flattens to the existing flat +`Plan` shape (topological file order) so the proxy agent loop and +plan_adherence stay unchanged while the flag is being proven out. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional, Tuple + +CompleteFn = Callable[[str, float, int, Optional[int]], str] + + +# ─── Schema ────────────────────────────────────────────────── + +@dataclass +class Capability: + id: str + name: str + parent: Optional[str] = None + + +@dataclass +class FunctionSpec: + name: str + signature: str = "" + summary: str = "" + + +@dataclass +class FileSpec: + id: str + path: str + capability: Optional[str] = None + functions: List[FunctionSpec] = field(default_factory=list) + + +@dataclass +class Edge: + src: str # file id (producer) + dst: str # file id (consumer) + kind: str = "data_flow" # "data_flow" | "order" + label: str = "" + + +@dataclass +class RPG: + capabilities: List[Capability] = field(default_factory=list) + files: List[FileSpec] = field(default_factory=list) + edges: List[Edge] = field(default_factory=list) + verify: str = "" + rationale: str = "" + + # ─ serialization ─ + def to_dict(self) -> dict: + return { + "capabilities": [ + {"id": c.id, "name": c.name, "parent": c.parent} for c in self.capabilities + ], + "files": [ + { + "id": f.id, + "path": f.path, + "capability": f.capability, + "functions": [ + {"name": fn.name, "signature": fn.signature, "summary": fn.summary} + for fn in f.functions + ], + } + for f in self.files + ], + "edges": [ + {"from": e.src, "to": e.dst, "kind": e.kind, "label": e.label} + for e in self.edges + ], + "verify": self.verify, + "rationale": self.rationale, + } + + +# ─── Tolerant JSON extraction ──────────────────────────────── +# Mirrors v3-service main._parse_plan_json (kept here so rpg.py imports nothing +# heavy). Strips ```json fences and leading prose, then brace-matches the first +# top-level object, ignoring braces inside strings. + +def extract_json_object(raw: str) -> Optional[dict]: + if not raw: + return None + import re + + fence = re.search(r"```(?:json)?\s*\n(.*?)\n```", raw, re.DOTALL) + if fence: + raw = fence.group(1) + start = raw.find("{") + if start < 0: + return None + depth = 0 + in_str = False + escape = False + end = -1 + for i in range(start, len(raw)): + c = raw[i] + if escape: + escape = False + continue + if c == "\\": + escape = True + continue + if c == '"': + in_str = not in_str + continue + if in_str: + continue + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + end = i + 1 + break + if end < 0: + return None + try: + return json.loads(raw[start:end]) + except (json.JSONDecodeError, ValueError): + return None + + +# ─── Parsing ───────────────────────────────────────────────── + +def parse_capabilities(raw: str) -> List[Capability]: + """Parse a Stage-A proposal response into a capability list.""" + obj = extract_json_object(raw) + if not obj: + return [] + out: List[Capability] = [] + for c in obj.get("capabilities") or []: + if not isinstance(c, dict): + continue + cid = str(c.get("id") or "").strip() + name = str(c.get("name") or "").strip() + if not cid or not name: + continue + parent = c.get("parent") + parent = str(parent).strip() if parent not in (None, "", "null") else None + out.append(Capability(id=cid, name=name, parent=parent)) + return out + + +def parse_rpg(raw: str, capabilities: Optional[List[Capability]] = None) -> Optional[RPG]: + """Parse a Stage-B implementation response into a full RPG. If the model + omits the capability list, the Stage-A `capabilities` are carried forward.""" + obj = extract_json_object(raw) + if obj is None: + return None + + caps = parse_capabilities(raw) + if not caps and capabilities: + caps = list(capabilities) + + files: List[FileSpec] = [] + for f in obj.get("files") or []: + if not isinstance(f, dict): + continue + fid = str(f.get("id") or "").strip() + path = str(f.get("path") or "").strip() + if not fid or not path: + continue + cap = f.get("capability") + cap = str(cap).strip() if cap not in (None, "", "null") else None + fns: List[FunctionSpec] = [] + for fn in f.get("functions") or []: + if not isinstance(fn, dict): + continue + fname = str(fn.get("name") or "").strip() + if not fname: + continue + fns.append( + FunctionSpec( + name=fname, + signature=str(fn.get("signature") or "").strip(), + summary=str(fn.get("summary") or "").strip(), + ) + ) + files.append(FileSpec(id=fid, path=path, capability=cap, functions=fns)) + + edges: List[Edge] = [] + for e in obj.get("edges") or []: + if not isinstance(e, dict): + continue + src = str(e.get("from") or e.get("src") or "").strip() + dst = str(e.get("to") or e.get("dst") or "").strip() + if not src or not dst: + continue + kind = str(e.get("kind") or "data_flow").strip() or "data_flow" + edges.append(Edge(src=src, dst=dst, kind=kind, label=str(e.get("label") or "").strip())) + + return RPG( + capabilities=caps, + files=files, + edges=edges, + verify=str(obj.get("verify") or "").strip(), + rationale=str(obj.get("rationale") or "").strip(), + ) + + +# ─── Validation & scoring ──────────────────────────────────── + +def _topo_order(file_ids: List[str], edges: List[Edge]) -> Tuple[List[str], bool]: + """Kahn topological sort over file ids (producer -> consumer). Returns + (order, acyclic). On a cycle, returns the original declaration order and + acyclic=False.""" + idset = set(file_ids) + adj: Dict[str, List[str]] = {fid: [] for fid in file_ids} + indeg: Dict[str, int] = {fid: 0 for fid in file_ids} + seen_edge = set() + for e in edges: + if e.src not in idset or e.dst not in idset or e.src == e.dst: + continue + key = (e.src, e.dst) + if key in seen_edge: + continue + seen_edge.add(key) + adj[e.src].append(e.dst) + indeg[e.dst] += 1 + + # Stable: process ready nodes in declaration order. + ready = [fid for fid in file_ids if indeg[fid] == 0] + order: List[str] = [] + while ready: + node = ready.pop(0) + order.append(node) + for nxt in adj[node]: + indeg[nxt] -= 1 + if indeg[nxt] == 0: + # insert preserving declaration order + ready.append(nxt) + ready.sort(key=lambda x: file_ids.index(x)) + acyclic = len(order) == len(file_ids) + return (order if acyclic else list(file_ids)), acyclic + + +def validate_rpg(rpg: RPG) -> Tuple[bool, List[str]]: + """Structural validation. Returns (ok, issues). `ok` is True when the graph + is usable (has files, edges resolve, acyclic); issues lists every problem + found regardless.""" + issues: List[str] = [] + cap_ids = {c.id for c in rpg.capabilities} + file_ids = [f.id for f in rpg.files] + file_idset = set(file_ids) + + if not rpg.files: + issues.append("no files") + if len(file_idset) != len(file_ids): + issues.append("duplicate file ids") + + for c in rpg.capabilities: + if c.parent is not None and c.parent not in cap_ids: + issues.append(f"capability {c.id} has unknown parent {c.parent}") + + # Leaf capabilities (no children) should map to >=1 file. + parents = {c.parent for c in rpg.capabilities if c.parent} + mapped_caps = {f.capability for f in rpg.files if f.capability} + for c in rpg.capabilities: + is_leaf = c.id not in parents + if is_leaf and c.id not in mapped_caps: + issues.append(f"leaf capability {c.id} ({c.name}) has no file") + + for f in rpg.files: + if f.capability is not None and f.capability not in cap_ids: + issues.append(f"file {f.id} references unknown capability {f.capability}") + + for e in rpg.edges: + if e.src not in file_idset: + issues.append(f"edge from unknown file {e.src}") + if e.dst not in file_idset: + issues.append(f"edge to unknown file {e.dst}") + + _, acyclic = _topo_order(file_ids, rpg.edges) + if not acyclic: + issues.append("file dependency graph has a cycle") + + edge_resolves = all(e.src in file_idset and e.dst in file_idset for e in rpg.edges) + ok = bool(rpg.files) and edge_resolves and acyclic and len(file_idset) == len(file_ids) + return ok, issues + + +def score_rpg(rpg: RPG) -> float: + """Graph-shape heuristic score in [0, 1]. Higher = better-formed plan.""" + ok, issues = validate_rpg(rpg) + score = 0.0 + if rpg.files: + score += 0.3 + file_idset = {f.id for f in rpg.files} + if rpg.edges and all(e.src in file_idset and e.dst in file_idset for e in rpg.edges): + score += 0.2 + _, acyclic = _topo_order([f.id for f in rpg.files], rpg.edges) + if acyclic: + score += 0.2 + if rpg.verify: + score += 0.15 + # Every file carries at least one planned function signature. + if rpg.files and all(f.functions for f in rpg.files): + score += 0.15 + # Penalize unresolved structure. + score -= min(0.3, 0.05 * len(issues)) + return max(0.0, min(1.0, score)) + + +# ─── Per-node generation constraints ───────────────────────── + +def node_constraints(rpg: RPG, file_id: str) -> List[str]: + """Derive the generation constraints for one RPG node (file). + + Once the graph fixes a node's architectural target, generating it is a + single-problem task — these constraints are what `/v3/generate` (and the + PlanSearch / Derivation-Chains pipeline behind it) consume to stay on the + planned interface: implement these signatures, consume these inputs, + produce these outputs. Phase 2 of docs/reports/RPG_WAVELET_PLANNING_V3_2.md. + """ + by_id = {f.id: f for f in rpg.files} + f = by_id.get(file_id) + if f is None: + return [] + out: List[str] = [] + + cap = next((c.name for c in rpg.capabilities if c.id == f.capability), "") + if cap: + out.append(f"Implements capability: {cap}") + + for fn in f.functions: + target = fn.signature or fn.name + if not target: + continue + line = f"Implement `{target}`" + if fn.summary: + line += f" — {fn.summary}" + out.append(line) + + for e in rpg.edges: + if e.dst == file_id and e.src in by_id: + what = e.label or "output" + out.append(f"Consumes {what} produced by {by_id[e.src].path}") + elif e.src == file_id and e.dst in by_id: + what = e.label or "output" + out.append(f"Produces {what} consumed by {by_id[e.dst].path}") + + return out + + +# ─── Flat-Plan projection ──────────────────────────────────── + +def flatten_to_plan(rpg: RPG) -> dict: + """Project the RPG onto the existing flat Plan shape (proxy/types.go Plan): + files in topological order become write/edit steps, then a verify step. + Producers precede consumers per the data-flow edges.""" + file_ids = [f.id for f in rpg.files] + order, _ = _topo_order(file_ids, rpg.edges) + by_id = {f.id: f for f in rpg.files} + + steps: List[dict] = [] + for i, fid in enumerate(order): + f = by_id[fid] + cap = next((c.name for c in rpg.capabilities if c.id == f.capability), "") + sig_hint = f.functions[0].signature or f.functions[0].name if f.functions else "" + why = cap or (f.functions[0].summary if f.functions else "implement file") + if sig_hint: + why = f"{why} ({sig_hint})" if why else sig_hint + steps.append( + { + "id": f"s{i + 1}", + "action": "write_file", + "target": f.path, + "why": (why or "implement file")[:200], + # Phase 2: the proxy threads these into the per-node + # /v3/generate call so generation stays on the planned interface. + "node_id": f.id, + "constraints": node_constraints(rpg, f.id), + } + ) + + verify_id = None + if rpg.verify: + verify_id = f"s{len(steps) + 1}" + steps.append( + { + "id": verify_id, + "action": "run_command", + "target": rpg.verify, + "why": "verify the generated repository builds / tests pass", + } + ) + + return { + "steps": steps, + "verify_step": verify_id, + "rationale": rpg.rationale or "RPG-projected plan (topological file order)", + } + + +# ─── Graph-guided verification, drift & localization (Phase 3) ─── + +def _function_name_from_signature(sig: str) -> str: + """Pull the declared name out of a signature string. Handles + `def load(...)`, `async def load(...)`, `func Load(...)`, `fn run(...)`, + `function go(...)`, `class Foo`, or a bare name.""" + import re + + s = sig.strip() + m = re.search(r"\b(?:async\s+)?(?:def|func|function|fn|fun|class|struct|interface|trait|type)\s+([A-Za-z_][\w]*)", s) + if m: + return m.group(1) + # Bare "name" or "name(args)". + m = re.match(r"\s*([A-Za-z_][\w]*)", s) + return m.group(1) if m else "" + + +def defined_names(code: str, filename: str) -> set: + """Names of functions/classes defined in `code`. Python uses stdlib `ast` + (precise, methods included); other languages use a keyword+name regex. + Returns an empty set when nothing parses — callers treat that as "unknown," + not "missing.\"""" + import re + + names: set = set() + if filename.endswith((".py", ".pyi", ".pyx")): + import ast + + try: + tree = ast.parse(code) + except (SyntaxError, ValueError): + tree = None + if tree is not None: + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(node.name) + return names + # fall through to regex on unparseable Python + for m in re.finditer( + r"\b(?:def|func|function|fn|fun|class|struct|interface|trait|type)\s+([A-Za-z_][\w]*)", + code, + ): + names.add(m.group(1)) + return names + + +def planned_signatures_from_constraints(constraints: List[str]) -> List[str]: + """Recover planned signatures from the `Implement \\`...\\`` constraint + strings that flatten_to_plan emits, so the generation veto can check them + without the structured RPG object.""" + import re + + out: List[str] = [] + for c in constraints or []: + m = re.search(r"Implement\s+`([^`]+)`", c) + if m: + out.append(m.group(1)) + return out + + +def missing_planned_signatures(code: str, planned: List[str], filename: str) -> List[str]: + """Planned signatures whose declared name is NOT defined in `code`. + + Conservative: if `code` yields no parseable definitions at all (empty + `defined_names`), returns [] — we don't veto when we can't see structure. + """ + if not planned: + return [] + defined = defined_names(code, filename) + if not defined: + return [] + missing: List[str] = [] + for sig in planned: + name = _function_name_from_signature(sig) + if name and name not in defined: + missing.append(sig) + return missing + + +@dataclass +class RealizationVerdict: + ok: bool + missing_functions: List[str] + defined: List[str] + + +def verify_node_realization(rpg: RPG, file_id: str, code: str, filename: str) -> RealizationVerdict: + """Does `code` realize the node's planned function signatures? Extends the + #39-pt-1 structural veto from "imports survive" to "the planned interface + exists." `ok=False` means a planned function is missing (reject candidate).""" + by_id = {f.id: f for f in rpg.files} + f = by_id.get(file_id) + if f is None or not f.functions: + return RealizationVerdict(True, [], sorted(defined_names(code, filename))) + planned = [fn.signature or fn.name for fn in f.functions if (fn.signature or fn.name)] + missing = missing_planned_signatures(code, planned, filename) + return RealizationVerdict( + ok=not missing, + missing_functions=missing, + defined=sorted(defined_names(code, filename)), + ) + + +@dataclass +class DriftReport: + file_id: str + missing: List[str] # planned functions absent from the generated code + drift_score: float # fraction of planned functions missing, [0, 1] + should_replan: bool # True when structure drifted from the plan + + +def node_drift(rpg: RPG, file_id: str, code: str, filename: str) -> DriftReport: + """Compare a node's planned structure against generated `code`. A planned + function that didn't get realized is structural drift away from the RPG — + the signal to re-plan the affected subgraph (Phase 3 drift loop).""" + by_id = {f.id: f for f in rpg.files} + f = by_id.get(file_id) + planned_names = [fn.name for fn in f.functions if fn.name] if f else [] + if not planned_names: + return DriftReport(file_id=file_id, missing=[], drift_score=0.0, should_replan=False) + defined = defined_names(code, filename) + if not defined: + # Unparseable / opaque — can't assess; don't trigger a re-plan blindly. + return DriftReport(file_id=file_id, missing=[], drift_score=0.0, should_replan=False) + missing = [n for n in planned_names if n not in defined] + drift = len(missing) / len(planned_names) + return DriftReport(file_id=file_id, missing=missing, drift_score=drift, should_replan=bool(missing)) + + +def _tokenize_query(text: str) -> set: + import re + + return {t for t in re.split(r"[^A-Za-z0-9]+", text.lower()) if len(t) > 1} + + +def localize(rpg: RPG, query: str, k: int = 5) -> List[str]: + """Map a request / failing test to the most relevant RPG node ids by token + overlap of capability name + file path + function names/summaries. The + graph-aware replacement for symbol-name-only matching (#39 pt 4).""" + qtokens = _tokenize_query(query) + if not qtokens: + return [] + cap_name = {c.id: c.name for c in rpg.capabilities} + scored: List[tuple] = [] + for f in rpg.files: + hay = " ".join( + [f.path, cap_name.get(f.capability, "")] + + [fn.name for fn in f.functions] + + [fn.summary for fn in f.functions] + ) + overlap = len(qtokens & _tokenize_query(hay)) + if overlap: + scored.append((overlap, f.id)) + scored.sort(key=lambda t: (-t[0], rpg.files.index(next(f for f in rpg.files if f.id == t[1])))) + return [fid for _, fid in scored[:k]] + + +# ─── Prompts ───────────────────────────────────────────────── + +_PROPOSAL_TEMPLATE = """You are a software architect doing PROPOSAL-LEVEL planning. +Decide WHAT to build: the capabilities (modules -> components -> leaf capabilities). +Do NOT decide files or code yet. + +User goal: {user_message} +{coarse_section} +Output ONLY a JSON object, no markdown fences, no prose: +{{ + "capabilities": [ + {{"id": "c1", "name": "", "parent": null}}, + {{"id": "c2", "name": "", "parent": "c1"}} + ] +}} + +Rules: +- Use a small hierarchy: top-level modules with null parent, refined into children. +- Leaf capabilities are concrete enough to map to one file each in the next stage. +- Cover the user's goal fully; do not invent unrelated scope. +- 3 to 15 capabilities total. + +JSON:""" + +_IMPLEMENTATION_TEMPLATE = """You are a software architect doing IMPLEMENTATION-LEVEL planning. +Given the capability tree, decide HOW to build it: the files, the key function +signatures in each file, and the data-flow / ordering edges between files. + +User goal: {user_message} + +Capability tree: +{capability_json} +{context_section} +Output ONLY a JSON object, no markdown fences, no prose: +{{ + "capabilities": , + "files": [ + {{"id": "f1", "path": "", "capability": "", + "functions": [{{"name": "", "signature": "", "summary": ""}}]}} + ], + "edges": [ + {{"from": "f1", "to": "f2", "kind": "data_flow", "label": ""}} + ], + "verify": "", + "rationale": "" +}} + +Rules: +- Every LEAF capability maps to at least one file. +- Edges express dependencies: "from" is the producer/dependency, "to" is the + consumer. The graph MUST be acyclic. +- Reference only file ids you define and capability ids from the tree. +- Prefer many small files over a few large ones. + +JSON:""" + + +def _coarse_section(coarse_map: Optional[List[dict]]) -> str: + """Render the wavelet coarse band (decompose_project output) as a grounding + hint for the proposal stage on existing repos (L6).""" + if not coarse_map: + return "" + lines = ["", "Existing repository structure (wavelet coarse band — ground capabilities on these):"] + for p in coarse_map[:30]: + label = p.get("label") if isinstance(p, dict) else None + if label: + lines.append(f"- {label}") + lines.append("") + return "\n".join(lines) + + +def _context_section(project_context: Optional[Dict[str, str]]) -> str: + if not project_context: + return "\n(no existing files — generating from scratch)\n" + parts = ["", "Existing files (truncated):"] + for path, content in list(project_context.items())[:12]: + preview = content[:200] + if len(content) > 200: + preview += "\n..." + parts.append(f"### {path}\n```\n{preview}\n```") + parts.append("") + return "\n".join(parts) + + +def build_proposal_prompt(user_message: str, coarse_map: Optional[List[dict]] = None) -> str: + return _PROPOSAL_TEMPLATE.format( + user_message=user_message, + coarse_section=_coarse_section(coarse_map), + ) + + +def build_implementation_prompt( + user_message: str, + capabilities: List[Capability], + project_context: Optional[Dict[str, str]] = None, +) -> str: + cap_json = json.dumps( + {"capabilities": [{"id": c.id, "name": c.name, "parent": c.parent} for c in capabilities]}, + indent=2, + ) + return _IMPLEMENTATION_TEMPLATE.format( + user_message=user_message, + capability_json=cap_json, + context_section=_context_section(project_context), + ) + + +# ─── Two-stage construction ────────────────────────────────── + +@dataclass +class RPGResult: + rpg: Optional[RPG] + plan: Optional[dict] + ok: bool + score: float + issues: List[str] + stage_reached: str # "proposal" | "implementation" | "none" + + +def construct_rpg( + user_message: str, + complete_fn: CompleteFn, + project_context: Optional[Dict[str, str]] = None, + coarse_map: Optional[List[dict]] = None, + max_tokens: int = 2048, + emit=None, +) -> RPGResult: + """Run the two-stage RPG construction. `complete_fn(prompt, temperature, + max_tokens, seed) -> raw text` isolates the LLM so this is unit-testable. + + Returns an RPGResult; callers fall back to the flat planner when `ok` is + False. Never raises on model output — only on a broken `complete_fn`. + """ + + def _emit(stage: str, detail: str = "", **data): + if emit: + try: + emit(stage, detail, **data) + except TypeError: + emit(stage, detail) + + # Stage A — proposal. + _emit("rpg_proposal_start", "planning capabilities (what to build)") + prop_prompt = build_proposal_prompt(user_message, coarse_map) + prop_raw = complete_fn(prop_prompt, 0.4, max_tokens, 42) + capabilities = parse_capabilities(prop_raw) + if not capabilities: + _emit("rpg_proposal_empty", "no capabilities parsed — falling back") + return RPGResult(None, None, False, 0.0, ["proposal stage produced no capabilities"], "none") + _emit("rpg_proposal_done", f"{len(capabilities)} capabilities", count=len(capabilities)) + + # Stage B — implementation. + _emit("rpg_impl_start", "expanding into files, signatures, and edges (how)") + impl_prompt = build_implementation_prompt(user_message, capabilities, project_context) + impl_raw = complete_fn(impl_prompt, 0.3, max_tokens, 43) + rpg = parse_rpg(impl_raw, capabilities=capabilities) + if rpg is None: + _emit("rpg_impl_unparseable", "implementation stage didn't parse — falling back") + return RPGResult(None, None, False, 0.0, ["implementation stage unparseable"], "proposal") + + ok, issues = validate_rpg(rpg) + score = score_rpg(rpg) + plan = flatten_to_plan(rpg) + _emit( + "rpg_done", + f"RPG built: {len(rpg.files)} files, {len(rpg.edges)} edges, score={score:.2f}, ok={ok}", + files=len(rpg.files), + edges=len(rpg.edges), + score=score, + ok=ok, + ) + return RPGResult(rpg=rpg, plan=plan, ok=ok, score=score, issues=issues, stage_reached="implementation") diff --git a/v3-service/rpg_eval.py b/v3-service/rpg_eval.py new file mode 100644 index 00000000..831a05e1 --- /dev/null +++ b/v3-service/rpg_eval.py @@ -0,0 +1,128 @@ +"""Offline RPG-quality metrics + evaluation harness (V3.2 Phase 4, issue #120). + +The live RepoCraft-style comparison (RPG-on vs flat planner: functional +coverage, test accuracy, code scale, L6 localization turns) requires a GPU + +model and is run via the live benchmark stack — see the runbook in +docs/reports/RPG_WAVELET_PLANNING_V3_2.md (Phase 4). This module is the +*offline* half: it scores RPG artifacts (the JSON the planner emits) on +graph-quality metrics so a benchmark run — or a fixture set — can be summarized +without re-deriving the graph logic, and so regressions in plan shape are +catchable in CI. + +Usage: + python rpg_eval.py artifacts/*.json # aggregate over RPG JSON files + python rpg_eval.py --jsonl plans.jsonl # one RPG (or {"rpg": ...}) per line +""" + +from __future__ import annotations + +import glob +import json +import sys +from typing import List + +import rpg as rpgmod + + +def rpg_quality_metrics(rpg_dict: dict) -> dict: + """Graph-quality metrics for a single RPG artifact (a dict as emitted by + rpg.RPG.to_dict, or a plan envelope carrying it under "rpg").""" + if "rpg" in rpg_dict and isinstance(rpg_dict["rpg"], dict): + rpg_dict = rpg_dict["rpg"] + g = rpgmod.parse_rpg(json.dumps(rpg_dict)) + if g is None: + return {"parseable": False} + + ok, issues = rpgmod.validate_rpg(g) + _, acyclic = rpgmod._topo_order([f.id for f in g.files], g.edges) + + caps = g.capabilities + parents = {c.parent for c in caps if c.parent} + leaves = [c for c in caps if c.id not in parents] + mapped = {f.capability for f in g.files if f.capability} + leaf_covered = sum(1 for c in leaves if c.id in mapped) + + files_with_sigs = sum(1 for f in g.files if f.functions) + + return { + "parseable": True, + "valid": ok, + "acyclic": acyclic, + "n_capabilities": len(caps), + "n_files": len(g.files), + "n_edges": len(g.edges), + "n_functions": sum(len(f.functions) for f in g.files), + "leaf_coverage": (leaf_covered / len(leaves)) if leaves else 1.0, + "signature_density": (files_with_sigs / len(g.files)) if g.files else 0.0, + "has_verify": bool(g.verify), + "score": rpgmod.score_rpg(g), + "n_issues": len(issues), + } + + +def aggregate_metrics(rpgs: List[dict]) -> dict: + """Aggregate per-RPG metrics across a set (e.g. a benchmark run's plans).""" + per = [rpg_quality_metrics(r) for r in rpgs] + parseable = [m for m in per if m.get("parseable")] + n = len(per) + if not parseable: + return {"n": n, "parse_rate": 0.0} + + def _mean(key: str) -> float: + vals = [m[key] for m in parseable if key in m] + return sum(vals) / len(vals) if vals else 0.0 + + def _rate(key: str) -> float: + return sum(1 for m in parseable if m.get(key)) / len(parseable) + + return { + "n": n, + "parse_rate": len(parseable) / n, + "valid_rate": _rate("valid"), + "acyclic_rate": _rate("acyclic"), + "has_verify_rate": _rate("has_verify"), + "mean_leaf_coverage": _mean("leaf_coverage"), + "mean_signature_density": _mean("signature_density"), + "mean_score": _mean("score"), + "avg_files": _mean("n_files"), + "avg_edges": _mean("n_edges"), + "avg_functions": _mean("n_functions"), + } + + +def _load_artifacts(args: List[str]) -> List[dict]: + out: List[dict] = [] + if args and args[0] == "--jsonl": + for path in args[1:]: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + out.append(json.loads(line)) + return out + # Expand globs (shells that don't expand, or quoted globs). + paths: List[str] = [] + for a in args: + expanded = glob.glob(a) + paths.extend(expanded if expanded else [a]) + for path in paths: + with open(path, "r", encoding="utf-8") as f: + out.append(json.load(f)) + return out + + +def main(argv: List[str]) -> int: + if not argv: + print(__doc__) + return 2 + artifacts = _load_artifacts(argv) + if not artifacts: + print("no RPG artifacts found", file=sys.stderr) + return 1 + metrics = aggregate_metrics(artifacts) + print(json.dumps(metrics, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/v3-service/wavelet/__init__.py b/v3-service/wavelet/__init__.py new file mode 100644 index 00000000..fb31440e --- /dev/null +++ b/v3-service/wavelet/__init__.py @@ -0,0 +1,65 @@ +"""Wavelet-based multi-resolution structural decomposition. + +Faithful pure-Python port of the wavescope-mcp wavelet engine +(https://github.com/yogthos/wavescope-mcp) for in-process use by the V3.2 +RPG-style architecture-first planner (see docs/reports/RPG_WAVELET_PLANNING_V3_2.md). + +Provenance: ported behavior-for-behavior from wavescope-mcp `src/`: + language.py <- src/language.ts + signal.py <- src/signal.ts + cwt.py <- src/wavelet.ts + context.py <- src/context.ts + project.py <- src/project.ts + +The port is deliberately dependency-free (stdlib only), mirroring the upstream +TypeScript loops; numpy vectorization is a drop-in optimization if profiling of +the planning critical path ever demands it. Behavior — scales, band radii, +normalization, reflect boundary, ridge-collapse semantics — matches upstream so +the calibrated thresholds (e.g. min_coefficient default 0.3) carry over. +""" + +from .language import LanguageConfig, detect_language, configs +from .signal import compute_signal +from .cwt import ( + ricker_wavelet, + compute_cwt, + detect_peaks, + Peak, + WaveletCoefficients, + DEFAULT_SCALES, +) +from .context import ( + FileContext, + ImportantPosition, + BandResult, + WaveletContextResult, +) +from .project import decompose_project, ProjectIndex +from .diff import diff_peaks, diff_contents, FileDiffResult, PeakChange, PeakDiff +from .flags import rpg_planning_enabled, ENV_VAR as RPG_PLANNING_ENV_VAR + +__all__ = [ + "LanguageConfig", + "detect_language", + "configs", + "compute_signal", + "ricker_wavelet", + "compute_cwt", + "detect_peaks", + "Peak", + "WaveletCoefficients", + "DEFAULT_SCALES", + "FileContext", + "ImportantPosition", + "BandResult", + "WaveletContextResult", + "decompose_project", + "ProjectIndex", + "diff_peaks", + "diff_contents", + "FileDiffResult", + "PeakChange", + "PeakDiff", + "rpg_planning_enabled", + "RPG_PLANNING_ENV_VAR", +] diff --git a/v3-service/wavelet/context.py b/v3-service/wavelet/context.py new file mode 100644 index 00000000..276a350b --- /dev/null +++ b/v3-service/wavelet/context.py @@ -0,0 +1,429 @@ +"""Multi-resolution file context (fine / medium / coarse bands). + +Faithful port of wavescope-mcp `src/context.ts`. `FileContext` holds the +wavelet index for one file and assembles three resolution bands plus a ranked +list of structurally important positions. Band radii and scale ranges match +upstream so coarse-band "which sections matter" maps to the planner and the +fine band feeds per-node implementation. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +from .language import detect_language, LanguageConfig +from .signal import compute_signal +from .cwt import compute_cwt, detect_peaks, Peak, WaveletCoefficients + +# Label-parsing regexes (compiled once). _LABEL_SPLIT mirrors the code-delimiter +# tokenizer in signal.py; the others read names out of structural lines. +_LABEL_SPLIT = re.compile(r"[\s()\[\]{},;:'\"`=<>+*/&|^~%@#\\]+") +_WS_SPLIT = re.compile(r"\s+") +_PAREN_WS_SPLIT = re.compile(r"[\s()]+") +_BRACE_TAIL = re.compile(r"\{.*") +_CLASS_INHERIT = re.compile(r"extends|implements") + + +@dataclass +class ImportantPosition: + position: int + coefficient: float + scale: int + label: str + filename: Optional[str] = None + + +@dataclass +class BandResult: + range: Tuple[int, int] + content: str + + +@dataclass +class WaveletContextResult: + center: int + clamped: bool + bands: Dict[str, BandResult] + wavelet_peaks: List[ImportantPosition] + # Original `center` requested by caller, present only when clamped. + clamped_from: Optional[int] = None + + +# Band scale ranges used by _build_medium_band / _build_coarse_band. +_BAND_SCALES = { + "fine": (1, 2), + "medium": (4, 16), + "coarse": (32, 128), +} + + +class FileContext: + """Wavelet index for a single file with multi-resolution query methods.""" + + def __init__(self, filename: str, content: str): + self.filename = filename + lines = content.split("\n") + + # Preserve trailing newline: if content ends with \n, drop the empty + # last element split produced. + if content.endswith("\n") and lines and lines[-1] == "": + lines.pop() + # Truly empty content. + if len(lines) == 1 and lines[0] == "": + lines = [] + + self.lines: List[str] = lines + self.language: LanguageConfig = detect_language(filename) + self.signal: List[float] = compute_signal(self.lines, self.language) + self.coefficients: WaveletCoefficients = compute_cwt(self.signal) + + self._all_peaks: Optional[List[Peak]] = None + self._ranked_peaks: Optional[List[Peak]] = None + + @property + def line_count(self) -> int: + return len(self.lines) + + # ─── Cached peak access ────────────────────────────────── + + def _get_all_peaks(self) -> List[Peak]: + """All peaks (lazy). ridge_window < 0 disables cross-scale collapse so + peaks at every scale are preserved — band assembly filters by scale and + must see all of them.""" + if self._all_peaks is None: + self._all_peaks = detect_peaks(self.coefficients, 0.0, 1000, -1) + return self._all_peaks + + def _get_ranked_peaks(self) -> List[Peak]: + """Ranked peak list with cross-scale ridge collapse applied, so a + feature appearing at many scales yields one entry.""" + if self._ranked_peaks is None: + self._ranked_peaks = detect_peaks(self.coefficients, 0.0, 1000) + return self._ranked_peaks + + # ─── Public API ────────────────────────────────────────── + + def get_important_positions( + self, min_coefficient: float = 0.3, limit: int = 20 + ) -> List[ImportantPosition]: + """Important structural positions sorted by coefficient magnitude.""" + all_peaks = self._get_ranked_peaks() + best: Dict[int, Peak] = {} + for p in all_peaks: + if abs(p.coefficient) < min_coefficient: + continue + existing = best.get(p.position) + if existing is None or abs(p.coefficient) > abs(existing.coefficient): + best[p.position] = p + ranked = sorted(best.values(), key=lambda p: abs(p.coefficient), reverse=True) + return [ + ImportantPosition( + position=p.position, + coefficient=p.coefficient, + scale=p.scale, + label=self._infer_label(p.position), + ) + for p in ranked[:limit] + ] + + def query_wavelet_context(self, center: int, radius: int) -> WaveletContextResult: + """Multi-resolution context centered at a position. + + - fine: raw lines in a narrow window (~radius/5) + - medium: peak-based summary in a medium window (~radius/2) + - coarse: section-level overview across the full radius + """ + if self.line_count == 0: + empty = WaveletContextResult( + center=0, + clamped=center != 0, + bands={ + "fine": BandResult((0, 0), ""), + "medium": BandResult((0, 0), ""), + "coarse": BandResult((0, 0), ""), + }, + wavelet_peaks=[], + ) + if empty.clamped: + empty.clamped_from = center + return empty + + cl = max(0, min(center, self.line_count - 1)) + clamped = center != cl + total = self.line_count + + fine_radius = max(10, radius // 5) + fine_start = max(0, cl - fine_radius) + fine_end = min(total - 1, cl + fine_radius) + + med_radius = radius // 2 + med_start = max(0, cl - med_radius) + med_end = min(total - 1, cl + med_radius) + + coarse_start = max(0, cl - radius) + coarse_end = min(total - 1, cl + radius) + + all_peaks = self._get_all_peaks() + nearby = [p for p in all_peaks if coarse_start <= p.position <= coarse_end] + + result = WaveletContextResult( + center=cl, + clamped=clamped, + bands={ + "fine": BandResult( + (fine_start, fine_end), + "\n".join(self.lines[fine_start:fine_end + 1]), + ), + "medium": BandResult( + (med_start, med_end), + self._build_medium_band(med_start, med_end, nearby), + ), + "coarse": BandResult( + (coarse_start, coarse_end), + self._build_coarse_band(coarse_start, coarse_end, nearby), + ), + }, + wavelet_peaks=[ + ImportantPosition( + position=p.position, + coefficient=p.coefficient, + scale=p.scale, + label=self._infer_label(p.position), + ) + for p in self._dedup_peaks(nearby)[:10] + ], + ) + if clamped: + result.clamped_from = center + return result + + def auto_scale(self, start: int, end: int) -> int: + """Pick a representative scale for a region of the given size, snapped + to the closest available scale.""" + size = max(1, end - start + 1) + if size <= 50: + target = 2 + elif size <= 200: + target = 8 + elif size <= 800: + target = 32 + else: + target = 128 + return self._find_closest_scale(target) + + def get_summary_at_scale( + self, start: int, end: int, scale: Optional[int] = None + ) -> str: + """Compressed view of a region using wavelet peaks at a given scale.""" + if not self.lines: + return "" + max_idx = len(self.lines) - 1 + lo = min(start, end) + hi = max(start, end) + if lo > max_idx or hi < 0: + return "" + s = max(0, lo) + e = min(max_idx, hi) + + all_peaks = self._get_all_peaks() + resolved = self._find_closest_scale(scale) if scale is not None else self.auto_scale(s, e) + in_range = [p for p in all_peaks if s <= p.position <= e and p.scale == resolved] + if not in_range: + return self._build_range_summary(s, e) + return self._build_peak_summary(in_range, s, e) + + def get_wavelet_coefficients(self, start: int, end: int, scale: int): + """Raw coefficient slice at the nearest available scale.""" + if len(self.coefficients.coefficients) == 0: + return {"scale": scale, "requestedScale": scale, "coefficients": [], "clamped": False} + resolved = self._find_closest_scale(scale) + scale_idx = self.coefficients.scales.index(resolved) + coeffs = self.coefficients.coefficients[scale_idx] + if not coeffs: + return {"scale": resolved, "requestedScale": scale, "coefficients": [], "clamped": False} + max_idx = len(coeffs) - 1 + lo = min(start, end) + hi = max(start, end) + if lo > max_idx or hi < 0: + return { + "scale": resolved, "requestedScale": scale, "coefficients": [], + "clamped": True, "clampedFrom": {"start": start, "end": end}, + } + s = max(0, lo) + e = min(max_idx, hi) + clamped = s != start or e != end + out = { + "scale": resolved, "requestedScale": scale, + "coefficients": coeffs[s:e + 1], "clamped": clamped, + } + if clamped: + out["clampedFrom"] = {"start": start, "end": end} + return out + + # ─── private helpers ────────────────────────────────────── + + def _dedup_peaks(self, peaks: List[Peak]) -> List[Peak]: + best: Dict[int, Peak] = {} + for p in peaks: + existing = best.get(p.position) + if existing is None or abs(p.coefficient) > abs(existing.coefficient): + best[p.position] = p + return list(best.values()) + + def _find_closest_scale(self, scale: int) -> int: + """Snap `scale` to the nearest scale present in the index. Ties resolve + to the lower scale (stable reduce).""" + scales = self.coefficients.scales + if not scales: + return scale + best = scales[0] + for curr in scales[1:]: + if abs(curr - scale) < abs(best - scale): + best = curr + return best + + def _infer_label(self, pos: int) -> str: + if pos < 0 or pos >= len(self.lines): + return "unknown" + line = self.lines[pos].strip() + if not line: + return f"line {pos}" + + # Token splitter mirrors signal._TOKEN_SPLIT; ws split is for keyword + # reads where the original code preserved whitespace tokens. + tokens = [t for t in _LABEL_SPLIT.split(line) if t] + ws_tokens = _WS_SPLIT.split(line) + + def _after(name: str) -> str: + idx = tokens.index(name) + return tokens[idx + 1] if idx + 1 < len(tokens) else "" + + def _name_after_ws(i: int) -> str: + return ws_tokens[i] if i < len(ws_tokens) else "" + + # f-string expressions below avoid backslashes / literal braces so the + # source stays valid on Python 3.9 (project floor); regex work happens + # in locals first. + if self.language.name == "python": + if ws_tokens[0] == "class": + name = _name_after_ws(1).replace(":", "") + return "class " + name + if ws_tokens[0] == "def": + name = _name_after_ws(1).split("(")[0] + return "def " + name + if ws_tokens[0] == "import": + return "import " + " ".join(ws_tokens[1:]) + if ws_tokens[0] == "from": + return "from " + " ".join(ws_tokens[1:]) + if line.startswith("@"): + dec = _PAREN_WS_SPLIT.split(line)[0] + return "decorator " + dec[1:] + if line.startswith("if __name__"): + return "main guard" + else: + if line.startswith("@"): + dec = _PAREN_WS_SPLIT.split(line)[0] + return "decorator " + dec[1:] + if ws_tokens[0] == "import": + return "import " + " ".join(ws_tokens[1:]) + if ws_tokens[0] == "export": + return "export " + " ".join(ws_tokens[1:])[:40] + if "class" in tokens: + nxt = _BRACE_TAIL.sub("", _after("class")) + nxt = _CLASS_INHERIT.sub("", nxt).strip() + return "class " + nxt + for kw in ("interface", "enum", "trait", "struct", "object"): + if kw in tokens: + return kw + " " + _BRACE_TAIL.sub("", _after(kw)).strip() + if "function" in tokens: + return "function " + _after("function").split("(")[0] + if "fn" in tokens and "defn" not in tokens: + return "fn " + _after("fn").split("(")[0] + if "fun" in tokens: + return "fun " + _after("fun").split("(")[0] + if "func" in tokens: + return "func " + _after("func").split("(")[0] + if "def" in tokens: + return "def " + _after("def").split("(")[0] + for kw in ("defn", "defmacro", "defprotocol", "defrecord", "deftype"): + if kw in tokens: + return kw + " " + _after(kw) + if "impl" in tokens: + return "impl " + _after("impl").split("(")[0] + if "protocol" in tokens: + return "protocol " + _after("protocol") + if "extension" in tokens: + return "extension " + _after("extension") + + return line[:50] + + def _build_medium_band(self, start: int, end: int, peaks: List[Peak]) -> str: + lo, hi = _BAND_SCALES["medium"] + med = sorted( + (p for p in peaks if start <= p.position <= end and lo <= p.scale <= hi), + key=lambda p: p.position, + ) + if not med: + return self._build_range_summary(start, end) + return self._build_peak_summary(med, start, end) + + def _build_coarse_band(self, start: int, end: int, peaks: List[Peak]) -> str: + lo, hi = _BAND_SCALES["coarse"] + coarse = sorted( + (p for p in peaks if start <= p.position <= end and lo <= p.scale <= hi), + key=lambda p: p.position, + ) + if not coarse: + all_in_range = sorted( + (p for p in peaks if start <= p.position <= end), + key=lambda p: p.position, + ) + if not all_in_range: + return self._build_range_summary(start, end) + return self._build_section_summary(all_in_range, start, end) + return self._build_section_summary(coarse, start, end) + + def _build_range_summary(self, start: int, end: int) -> str: + if start > end: + return "" + preview_lines = min(5, end - start + 1) + parts: List[str] = [] + step = -(-(end - start + 1) // preview_lines) # ceil division + i = start + while i <= end: + line = self.lines[i].strip() + if line: + parts.append(f"[{i}] {line[:80]}") + i += step + return "\n".join(parts) + + def _build_peak_summary(self, peaks: List[Peak], range_start: int, range_end: int) -> str: + parts: List[str] = [] + prev_end = range_start - 1 + for peak in peaks: + if peak.position > prev_end + 1: + parts.append(f"[{prev_end + 1}-{peak.position - 1}] ...") + parts.append(f"[{peak.position}] {self.lines[peak.position].strip()[:80]}") + prev_end = peak.position + if prev_end < range_end: + parts.append(f"[{prev_end + 1}-{range_end}] ...") + return "\n".join(parts) + + def _build_section_summary(self, peaks: List[Peak], range_start: int, range_end: int) -> str: + parts: List[str] = [] + prev_pos = range_start + current_section = "" + if peaks: + current_section = self._infer_label(peaks[0].position) + for peak in peaks: + if current_section and prev_pos < peak.position: + parts.append(f"[{prev_pos}-{peak.position - 1}] {current_section}") + current_section = self._infer_label(peak.position) + prev_pos = peak.position + if current_section and prev_pos <= range_end: + parts.append(f"[{prev_pos}-{range_end}] {current_section}") + if not parts: + parts.append(f"[{range_start}-{range_end}] (code region)") + return "\n".join(parts) diff --git a/v3-service/wavelet/cwt.py b/v3-service/wavelet/cwt.py new file mode 100644 index 00000000..02c0fab0 --- /dev/null +++ b/v3-service/wavelet/cwt.py @@ -0,0 +1,153 @@ +"""Ricker (Mexican-hat) continuous wavelet transform + peak detection. + +Faithful port of wavescope-mcp `src/wavelet.ts`. The transform is a custom +direct convolution with `1/sqrt(a)` normalization, `+/-5a` kernel truncation, +and symmetric-reflect boundary handling. Peak detection collapses cross-scale +ridges so a single structural feature yields one peak. These conventions are +NOT interchangeable with PyWavelets/SciPy `mexh` — every downstream threshold +is calibrated to this exact coefficient scale (see docs/reports/RPG_WAVELET_PLANNING_V3_2.md §6). +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import List, Optional + + +@dataclass +class Peak: + position: int + coefficient: float + scale: int + label: Optional[str] = None + + +@dataclass +class WaveletCoefficients: + scales: List[int] + coefficients: List[List[float]] # [scaleIndex][position] + + +DEFAULT_SCALES: List[int] = [1, 2, 4, 8, 16, 32, 64, 128] + + +def ricker_wavelet(t: float) -> float: + """Ricker (Mexican hat) wavelet: psi(t) = (1 - t^2) * exp(-t^2 / 2).""" + t2 = t * t + return (1 - t2) * math.exp(-t2 / 2) + + +def _make_kernel(a: float, num_points: int) -> List[float]: + """Wavelet kernel values for scale `a`, centered at 0. + + Truncated to +/-5a (covers ~99.99% of the Ricker energy) but bounded by half + the signal length to stay finite on short inputs. Includes 1/sqrt(a) + normalization to keep coefficient magnitudes comparable across scales. + """ + if not math.isfinite(a) or a <= 0: + raise ValueError(f"Invalid scale: {a}") + half_width = math.ceil(5 * a) + half = min(half_width, math.ceil(num_points / 2)) + inv_sqrt_a = 1 / math.sqrt(a) + kernel: List[float] = [] + for t in range(-half, half + 1): + kernel.append(inv_sqrt_a * ricker_wavelet(t / a)) + return kernel + + +def _reflect_index(idx: int, n: int) -> int: + """Mirror out-of-range indices back into [0, N-1] to suppress boundary + artifacts where the wavelet's negative lobes would otherwise be clipped.""" + if n == 1: + return 0 + period = 2 * (n - 1) + i = idx % period # Python modulo already yields a non-negative result here. + return period - i if i >= n else i + + +def compute_cwt( + signal: List[float], + scales: Optional[List[int]] = None, + boundary: str = "reflect", +) -> WaveletCoefficients: + """Compute the Ricker CWT over the signal. + + For each scale a, psi_a(t) = (1/sqrt(a)) * psi(t/a) is convolved with the + signal: W(a, b) = sum_t psi_a(t - b) * signal[t]. Boundary defaults to + symmetric reflection; pass boundary="zero" for zero-padding. + """ + if scales is None: + scales = DEFAULT_SCALES + n = len(signal) + + used_scales: List[int] = [] + for a in scales: + if a not in used_scales: + used_scales.append(a) + + if n == 0: + return WaveletCoefficients(scales=used_scales, coefficients=[[] for _ in used_scales]) + + coefficients: List[List[float]] = [] + for a in used_scales: + kernel = _make_kernel(a, n) + half_kernel = len(kernel) // 2 + coeffs = [0.0] * n + for pos in range(n): + total = 0.0 + for k, kv in enumerate(kernel): + signal_idx = pos + k - half_kernel + if 0 <= signal_idx < n: + total += kv * signal[signal_idx] + elif boundary == "reflect": + total += kv * signal[_reflect_index(signal_idx, n)] + coeffs[pos] = total + coefficients.append(coeffs) + + return WaveletCoefficients(scales=used_scales, coefficients=coefficients) + + +def detect_peaks( + cwt: WaveletCoefficients, + threshold: float, + max_peaks: int = 250, + ridge_window: int = 2, +) -> List[Peak]: + """Detect local maxima in coefficient magnitudes across all scales. + + Returns peaks sorted by |coefficient| descending. Plateau handling + (>= left, > right) selects the rightmost element of a flat plateau. + + Cross-scale ridge collapse: after magnitude sorting, peaks within + `ridge_window` of an already-kept stronger peak are dropped, so a single + spike yields one peak (the dominant scale). A negative `ridge_window` + disables collapse — every per-scale peak is preserved. + """ + if len(cwt.coefficients) == 0: + return [] + + peaks: List[Peak] = [] + for si, scale in enumerate(cwt.scales): + coeffs = cwt.coefficients[si] + n = len(coeffs) + for pos in range(n): + mag = abs(coeffs[pos]) + if mag < threshold: + continue + left_ok = pos == 0 or mag >= abs(coeffs[pos - 1]) + right_ok = pos == n - 1 or mag > abs(coeffs[pos + 1]) + if left_ok and right_ok: + peaks.append(Peak(position=pos, coefficient=coeffs[pos], scale=scale)) + + peaks.sort(key=lambda p: abs(p.coefficient), reverse=True) + + kept: List[Peak] = [] + for peak in peaks: + overlap = any(abs(k.position - peak.position) <= ridge_window for k in kept) + if not overlap: + kept.append(peak) + if len(kept) >= max_peaks: + break + + return kept diff --git a/v3-service/wavelet/diff.py b/v3-service/wavelet/diff.py new file mode 100644 index 00000000..379ca434 --- /dev/null +++ b/v3-service/wavelet/diff.py @@ -0,0 +1,138 @@ +"""Structural peak-profile diff — the drift detector. + +Faithful port of wavescope-mcp `src/diff.ts`. Diffs two sets of wavelet peaks +(e.g. a file before/after an edit) into added / removed / shifted / +magnitudeChanged / unchanged, with greedy proximity matching. Used by the +V3.2 RPG planner to detect when a file's structure has drifted away from the +plan (docs/reports/RPG_WAVELET_PLANNING_V3_2.md, Phase 3). The git-ref +orchestration (`diffFileAtRefs`) is intentionally omitted — drift compares two +in-memory contents, not git revisions. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import List, Optional + +from .context import FileContext, ImportantPosition + +_DEFAULT_WINDOW = 2 +_COEFF_EPSILON = 1e-10 + + +@dataclass +class PeakChange: + kind: str # "added" | "removed" | "shifted" | "magnitudeChanged" | "unchanged" + before: Optional[ImportantPosition] + after: Optional[ImportantPosition] + + +@dataclass +class PeakDiff: + changes: List[PeakChange] = field(default_factory=list) + summary: dict = field(default_factory=dict) + + +@dataclass +class FileDiffResult: + before_line_count: int + after_line_count: int + diff: PeakDiff + + +def diff_peaks( + before: List[ImportantPosition], + after: List[ImportantPosition], + window: int = _DEFAULT_WINDOW, +) -> PeakDiff: + """Diff two peak sets. Each after-peak greedily pairs with the closest + unmatched before-peak within `window` lines (tiebreak by coefficient + closeness). Unmatched before → removed; unmatched after → added. Same + position + same coefficient → unchanged; same position, different + coefficient → magnitudeChanged; different position within window → shifted. + """ + changes: List[PeakChange] = [] + used_before = set() + + after_sorted = sorted(after, key=lambda p: p.position) + + for ap in after_sorted: + best_idx = -1 + best_dist = float("inf") + best_coef_diff = float("inf") + for i, bp in enumerate(before): + if i in used_before: + continue + dist = abs(bp.position - ap.position) + if dist > window: + continue + coef_diff = abs(bp.coefficient - ap.coefficient) + if dist < best_dist or (dist == best_dist and coef_diff < best_coef_diff): + best_dist = dist + best_coef_diff = coef_diff + best_idx = i + + if best_idx != -1: + used_before.add(best_idx) + bp = before[best_idx] + if bp.position == ap.position: + if abs(bp.coefficient - ap.coefficient) < _COEFF_EPSILON: + changes.append(PeakChange("unchanged", bp, ap)) + else: + changes.append(PeakChange("magnitudeChanged", bp, ap)) + else: + changes.append(PeakChange("shifted", bp, ap)) + else: + changes.append(PeakChange("added", None, ap)) + + for i, bp in enumerate(before): + if i not in used_before: + changes.append(PeakChange("removed", bp, None)) + + def _pos(c: PeakChange) -> int: + if c.kind == "removed": + return c.before.position + return c.after.position if c.after is not None else c.before.position + + changes.sort(key=_pos) + + summary = {"added": 0, "removed": 0, "shifted": 0, "magnitudeChanged": 0, "unchanged": 0} + for c in changes: + summary[c.kind] += 1 + + return PeakDiff(changes=changes, summary=summary) + + +def diff_file_context( + before_peaks: List[ImportantPosition], + after_peaks: List[ImportantPosition], + before_line_count: int, + after_line_count: int, + window: int = _DEFAULT_WINDOW, +) -> FileDiffResult: + return FileDiffResult( + before_line_count=before_line_count, + after_line_count=after_line_count, + diff=diff_peaks(before_peaks, after_peaks, window), + ) + + +def diff_contents( + before_text: str, + after_text: str, + filename: str, + min_coefficient: float = 0.3, + limit: int = 100, + window: int = _DEFAULT_WINDOW, +) -> FileDiffResult: + """Convenience: build FileContexts for two file contents and diff their + important positions. The drift entry point for RPG node verification.""" + before_ctx = FileContext(filename, before_text) + after_ctx = FileContext(filename, after_text) + return diff_file_context( + before_ctx.get_important_positions(min_coefficient, limit), + after_ctx.get_important_positions(min_coefficient, limit), + before_ctx.line_count, + after_ctx.line_count, + window, + ) diff --git a/v3-service/wavelet/flags.py b/v3-service/wavelet/flags.py new file mode 100644 index 00000000..71be10c2 --- /dev/null +++ b/v3-service/wavelet/flags.py @@ -0,0 +1,21 @@ +"""Feature-flag contract for the V3.2 RPG-style architecture-first planner. + +The wavelet substrate (this package) is always importable and side-effect free. +Whether the *planner* uses it is gated by `ATLAS_RPG_PLANNING`, which Phase 1 +checks before swapping the flat planner for the RPG two-stage flow. Defining the +env contract here (rather than inline at the call site) keeps the default — off — +in one place. See docs/reports/RPG_WAVELET_PLANNING_V3_2.md. +""" + +from __future__ import annotations + +import os + +ENV_VAR = "ATLAS_RPG_PLANNING" + +_TRUTHY = {"1", "true", "yes", "on"} + + +def rpg_planning_enabled() -> bool: + """True when RPG-style architecture-first planning is enabled. Default: off.""" + return os.getenv(ENV_VAR, "0").strip().lower() in _TRUTHY diff --git a/v3-service/wavelet/language.py b/v3-service/wavelet/language.py new file mode 100644 index 00000000..1011633a --- /dev/null +++ b/v3-service/wavelet/language.py @@ -0,0 +1,429 @@ +"""Per-language structural-keyword tables. + +Faithful port of wavescope-mcp `src/language.ts`. Keyword weights, comment +delimiters, indent/decorator weights, and the detection order are preserved +verbatim so signal scoring matches upstream. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List + + +@dataclass(frozen=True) +class LanguageConfig: + name: str + extensions: List[str] + structural_keywords: Dict[str, float] + comment_prefixes: List[str] + block_comment_start: str + block_comment_end: str + indent_weight: float + decorator_weight: float + # Filenames (no extension) that should also map here, e.g. "Rakefile". + filenames: List[str] = field(default_factory=list) + # If true, block comment delimiters must appear at the start of the line. + block_comment_at_line_start: bool = False + # If true, block comment end tracking uses paren-depth counting + # (for Clojure-style (comment ...) forms with nested S-expressions). + block_comment_uses_paren_depth: bool = False + + +# ─── Base keyword sets ─────────────────────────────────────── + +_C_LIKE: Dict[str, float] = { + "class": 1.0, + "export": 0.6, + "import": 0.6, + "public": 0.3, + "private": 0.3, + "protected": 0.3, + "abstract": 0.4, + "static": 0.3, + "async": 0.3, + "const": 0.3, + "let": 0.2, + "var": 0.2, + "if": 0.3, + "else": 0.2, + "for": 0.3, + "while": 0.3, + "do": 0.2, + "switch": 0.3, + "case": 0.2, + "default": 0.2, + "try": 0.3, + "catch": 0.3, + "finally": 0.2, + "return": 0.2, + "throw": 0.2, +} + +_JS_LIKE: Dict[str, float] = {**_C_LIKE, "function": 0.9} + +_TS_LIKE: Dict[str, float] = {**_JS_LIKE, "interface": 0.9, "type": 0.5, "enum": 0.8} + + +# ─── Language configurations ───────────────────────────────── + +python_config = LanguageConfig( + name="python", + extensions=[".py", ".pyi", ".pyx"], + structural_keywords={ + "class": 1.0, + "def": 0.9, + "import": 0.6, + "from": 0.5, + "return": 0.2, + "yield": 0.2, + "raise": 0.2, + "if": 0.3, + "elif": 0.2, + "else": 0.2, + "try": 0.3, + "except": 0.3, + "finally": 0.2, + "for": 0.3, + "while": 0.3, + "with": 0.4, + "match": 0.3, + "case": 0.2, + }, + comment_prefixes=["#"], + # Python docstrings (triple quotes) are handled specially in signal.py. + block_comment_start='"""', + block_comment_end='"""', + indent_weight=0.15, + decorator_weight=0.5, +) + +ts_config = LanguageConfig( + name="typescript", + extensions=[".ts", ".tsx", ".mts", ".cts"], + structural_keywords={**_TS_LIKE, "get": 0.3, "set": 0.3}, + comment_prefixes=["//"], + block_comment_start="/*", + block_comment_end="*/", + indent_weight=0.15, + decorator_weight=0.5, +) + +js_config = LanguageConfig( + name="javascript", + extensions=[".js", ".jsx", ".mjs", ".cjs"], + structural_keywords={**_JS_LIKE, "get": 0.3, "set": 0.3}, + comment_prefixes=["//"], + block_comment_start="/*", + block_comment_end="*/", + indent_weight=0.15, + decorator_weight=0.5, +) + +go_config = LanguageConfig( + name="go", + extensions=[".go"], + structural_keywords={ + **_C_LIKE, + "func": 0.9, + "go": 0.2, + "defer": 0.2, + "select": 0.2, + "struct": 0.9, + "package": 0.3, + }, + comment_prefixes=["//"], + block_comment_start="/*", + block_comment_end="*/", + indent_weight=0.1, + decorator_weight=0.0, +) + +rust_config = LanguageConfig( + name="rust", + extensions=[".rs"], + structural_keywords={ + **_C_LIKE, + "fn": 0.9, + "impl": 0.9, + "mod": 0.6, + "use": 0.5, + "pub": 0.4, + "mut": 0.1, + "trait": 0.9, + "struct": 0.9, + "enum": 0.8, + "type": 0.5, + "match": 0.3, + "where": 0.2, + "unsafe": 0.3, + "extern": 0.3, + "macro_rules": 0.7, + }, + comment_prefixes=["//"], + block_comment_start="/*", + block_comment_end="*/", + indent_weight=0.15, + decorator_weight=0.4, +) + +java_config = LanguageConfig( + name="java", + extensions=[".java"], + structural_keywords={ + **_TS_LIKE, + "package": 0.6, + "extends": 0.5, + "implements": 0.5, + "throws": 0.2, + "synchronized": 0.2, + "volatile": 0.1, + "transient": 0.1, + "native": 0.1, + "strictfp": 0.1, + }, + comment_prefixes=["//"], + block_comment_start="/*", + block_comment_end="*/", + indent_weight=0.15, + decorator_weight=0.5, +) + +ruby_config = LanguageConfig( + name="ruby", + extensions=[".rb", ".rake", ".gemspec"], + filenames=["Rakefile", "Gemfile"], + structural_keywords={ + "class": 1.0, + "def": 0.9, + "module": 0.9, + "require": 0.6, + "include": 0.4, + "extend": 0.4, + "private": 0.3, + "protected": 0.3, + "public": 0.3, + "attr_accessor": 0.5, + "attr_reader": 0.5, + "attr_writer": 0.5, + "if": 0.3, + "unless": 0.3, + "else": 0.2, + "elsif": 0.2, + "while": 0.3, + "until": 0.3, + "for": 0.3, + "do": 0.2, + "begin": 0.3, + "rescue": 0.3, + "ensure": 0.2, + "case": 0.2, + "when": 0.2, + "return": 0.2, + "yield": 0.2, + "raise": 0.2, + }, + comment_prefixes=["#"], + block_comment_start="=begin", + block_comment_end="=end", + indent_weight=0.12, + decorator_weight=0.0, + block_comment_at_line_start=True, +) + +php_config = LanguageConfig( + name="php", + extensions=[".php"], + structural_keywords={ + **_TS_LIKE, + "namespace": 0.6, + "use": 0.5, + "trait": 0.8, + "extends": 0.5, + "implements": 0.5, + "require_once": 0.5, + "require": 0.5, + "include": 0.4, + "include_once": 0.4, + "echo": 0.1, + }, + comment_prefixes=["//", "#"], + block_comment_start="/*", + block_comment_end="*/", + indent_weight=0.15, + decorator_weight=0.4, +) + +swift_config = LanguageConfig( + name="swift", + extensions=[".swift"], + structural_keywords={ + **_C_LIKE, + "func": 0.9, + "guard": 0.3, + "defer": 0.2, + "protocol": 0.9, + "extension": 0.7, + "struct": 0.9, + "actor": 0.9, + "mutating": 0.3, + "nonmutating": 0.3, + "override": 0.3, + "convenience": 0.2, + "required": 0.2, + "weak": 0.1, + "unowned": 0.1, + "throws": 0.2, + "rethrows": 0.2, + "associatedtype": 0.5, + "typealias": 0.4, + }, + comment_prefixes=["//"], + block_comment_start="/*", + block_comment_end="*/", + indent_weight=0.15, + decorator_weight=0.4, +) + +kotlin_config = LanguageConfig( + name="kotlin", + extensions=[".kt", ".kts"], + structural_keywords={ + **_C_LIKE, + "fun": 0.9, + "val": 0.2, + "object": 0.7, + "companion": 0.4, + "data": 0.4, + "sealed": 0.5, + "open": 0.4, + "override": 0.3, + "suspend": 0.3, + "operator": 0.3, + "infix": 0.2, + "inline": 0.2, + "tailrec": 0.2, + "external": 0.2, + "annotation": 0.4, + "expect": 0.3, + "actual": 0.3, + }, + comment_prefixes=["//"], + block_comment_start="/*", + block_comment_end="*/", + indent_weight=0.15, + decorator_weight=0.5, +) + +scala_config = LanguageConfig( + name="scala", + extensions=[".scala", ".sc"], + structural_keywords={ + **_C_LIKE, + "def": 0.9, + "val": 0.2, + "object": 0.7, + "trait": 0.9, + "sealed": 0.5, + "implicit": 0.4, + "given": 0.3, + "using": 0.3, + "extension": 0.5, + "opaque": 0.3, + "case": 0.4, + "match": 0.3, + "lazy": 0.1, + "override": 0.3, + }, + comment_prefixes=["//"], + block_comment_start="/*", + block_comment_end="*/", + indent_weight=0.15, + decorator_weight=0.5, +) + +clojure_config = LanguageConfig( + name="clojure", + extensions=[".clj", ".cljs", ".cljc", ".edn"], + structural_keywords={ + "defn": 0.9, + "defn-": 0.9, + "def": 0.7, + "defmacro": 0.9, + "defmulti": 0.9, + "defmethod": 0.8, + "defprotocol": 0.9, + "defrecord": 0.9, + "deftype": 0.9, + "definterface": 0.9, + "defonce": 0.7, + "extend-type": 0.8, + "extend-protocol": 0.8, + "letfn": 0.6, + "reify": 0.6, + "ns": 0.6, + "require": 0.6, + "use": 0.5, + "import": 0.5, + "fn": 0.4, + "let": 0.2, + "if": 0.3, + "when": 0.3, + "loop": 0.3, + "for": 0.3, + "doseq": 0.3, + "try": 0.3, + "catch": 0.3, + "finally": 0.2, + }, + comment_prefixes=[";"], + block_comment_start="(comment", + block_comment_end=")", + indent_weight=0.12, + decorator_weight=0.0, + block_comment_uses_paren_depth=True, +) + +generic_config = LanguageConfig( + name="generic", + extensions=[], + structural_keywords={}, + comment_prefixes=["#"], + block_comment_start="/*", + block_comment_end="*/", + indent_weight=0.1, + decorator_weight=0.3, +) + +# Ordered by priority — generic_config (last) is the fallback. It must remain +# last and must not share extensions with preceding configs. +configs: List[LanguageConfig] = [ + python_config, + ts_config, + js_config, + go_config, + rust_config, + java_config, + ruby_config, + php_config, + swift_config, + kotlin_config, + scala_config, + clojure_config, + generic_config, +] + + +def detect_language(filename: str) -> LanguageConfig: + slash = max(filename.rfind("/"), filename.rfind("\\")) + base = filename[slash + 1:] if slash >= 0 else filename + for cfg in configs: + if base in cfg.filenames: + return cfg + dot = base.rfind(".") + ext = base[dot:].lower() if dot >= 0 else "" + if ext: + for cfg in configs: + if ext in cfg.extensions: + return cfg + return generic_config diff --git a/v3-service/wavelet/project.py b/v3-service/wavelet/project.py new file mode 100644 index 00000000..1593c712 --- /dev/null +++ b/v3-service/wavelet/project.py @@ -0,0 +1,282 @@ +"""Project-wide structural decomposition. + +Faithful port of wavescope-mcp `src/project.ts` discovery + project-level +important-position aggregation. Honors a root `.gitignore` (last-match-wins, +including `!` negation), skips well-known build/cache dirs, caps file count and +size, sniffs for binary files, and follows symlinks once without escaping root. + +Synchronous (no concurrency pool) — the upstream parallelism is an I/O +optimization, not a behavioral property. +""" + +from __future__ import annotations + +import os +import re +import stat as statmod +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +from .context import FileContext, ImportantPosition +from .language import configs + +_CODE_EXTENSIONS = {ext for c in configs for ext in c.extensions} +_CODE_FILENAMES = {fn for c in configs for fn in c.filenames} + +_SKIP_DIRS = { + "node_modules", ".git", "__pycache__", ".venv", "venv", "dist", "build", + "target", ".next", ".turbo", "coverage", ".pytest_cache", ".cache", + ".idea", ".vscode", "vendor", "out", "obj", ".gradle", ".tox", + ".mypy_cache", ".ruff_cache", "bower_components", ".serverless", + ".terraform", ".eggs", "site-packages", ".yarn", ".parcel-cache", + "__snapshots__", +} + +MAX_FILE_BYTES = 2_000_000 +MAX_FILES = 5_000 +_BINARY_SNIFF_BYTES = 4096 + + +# ─── .gitignore parser (minimal) ──────────────────────────── + +@dataclass +class _GitignoreRule: + regex: "re.Pattern[str]" + dir_only: bool + negate: bool + + +def _compile_glob(pat: str) -> "re.Pattern[str]": + # Leading `/` means root-relative; otherwise match at any depth. + anchored = pat.startswith("/") + body = pat[1:] if anchored else pat + + regex = "" + i = 0 + while i < len(body): + c = body[i] + if c == "*" and i + 1 < len(body) and body[i + 1] == "*": + has_trailing_slash = i + 2 < len(body) and body[i + 2] == "/" + i += 3 if has_trailing_slash else 2 + regex += "(?:[^/]+/)*" if has_trailing_slash else ".*" + elif c == "*": + regex += "[^/]*" + i += 1 + elif c == "?": + regex += "[^/]" + i += 1 + elif c in ".+()[]{}^$|\\": + regex += "\\" + c + i += 1 + else: + regex += c + i += 1 + + prefix = "^" if anchored else "(^|/)" + return re.compile(f"{prefix}{regex}(/|$)") + + +def _load_gitignore(root: str) -> List[_GitignoreRule]: + path = os.path.join(root, ".gitignore") + try: + with open(path, "r", encoding="utf-8") as f: + raw = f.read() + except OSError: + return [] + rules: List[_GitignoreRule] = [] + for line_raw in raw.split("\n"): + line = line_raw.strip() + if not line or line.startswith("#"): + continue + negate = line.startswith("!") + dir_only = line.endswith("/") + pat_raw = line[1:] if negate else line + pat = pat_raw[:-1] if dir_only else pat_raw + rules.append(_GitignoreRule(_compile_glob(pat), dir_only, negate)) + return rules + + +def _is_ignored(rel_path: str, is_dir: bool, rules: List[_GitignoreRule]) -> bool: + normalized = rel_path.replace(os.sep, "/") + ignored = False + for rule in rules: # last match wins (negation re-includes) + if rule.dir_only and not is_dir: + continue + matches = bool(rule.regex.search(normalized)) or ( + rule.dir_only and bool(rule.regex.search(normalized + "/")) + ) + if matches: + ignored = not rule.negate + return ignored + + +# ─── File discovery ───────────────────────────────────────── + +def _is_binary(full_path: str) -> bool: + """Standard NUL-byte sniff over the first _BINARY_SNIFF_BYTES.""" + try: + with open(full_path, "rb") as f: + chunk = f.read(_BINARY_SNIFF_BYTES) + return b"\x00" in chunk + except OSError: + return True + + +@dataclass +class ProjectFile: + filename: str + path: str + context: FileContext + + +def _discover_files(root: str) -> Tuple[List[ProjectFile], bool]: + results: List[ProjectFile] = [] + visited_real: set = set() + try: + root_real = os.path.realpath(root) + except OSError as e: + raise OSError(f"Cannot read directory: {root}") from e + visited_real.add(root_real) + + gitignore = _load_gitignore(root) + truncated = False + pending: List[str] = [] + + def walk(directory: str) -> None: + nonlocal truncated + if truncated: + return + try: + entries = os.listdir(directory) + except OSError: + return + for entry in entries: + if truncated: + return + full_path = os.path.join(directory, entry) + rel_path = os.path.relpath(full_path, root) + try: + st = os.lstat(full_path) + except OSError: + continue + + # Resolve symlinks: follow, but refuse ones escaping root. After + # this, `st` reflects the resolved target for symlinks. + if statmod.S_ISLNK(st.st_mode): + try: + target = os.path.realpath(full_path) + except OSError: + continue + if not (target == root_real or target.startswith(root_real + os.sep)): + continue + try: + st = os.stat(target) + except OSError: + continue + + if statmod.S_ISDIR(st.st_mode): + if entry in _SKIP_DIRS: + continue + if _is_ignored(rel_path, True, gitignore): + continue + try: + dir_real = os.path.realpath(full_path) + except OSError: + continue + if dir_real in visited_real: + continue + visited_real.add(dir_real) + walk(full_path) + elif statmod.S_ISREG(st.st_mode): + if _is_ignored(rel_path, False, gitignore): + continue + _, ext = os.path.splitext(entry) + ext = ext.lower() + if ext not in _CODE_EXTENSIONS and entry not in _CODE_FILENAMES: + continue + if st.st_size > MAX_FILE_BYTES: + continue + try: + file_real = os.path.realpath(full_path) + except OSError: + file_real = full_path + if file_real in visited_real: + continue + visited_real.add(file_real) + if len(results) + len(pending) >= MAX_FILES: + truncated = True + return + pending.append(full_path) + + walk(root) + + for full_path in pending: + if _is_binary(full_path): + continue + try: + with open(full_path, "r", encoding="utf-8") as f: + content = f.read() + except (OSError, UnicodeDecodeError): + continue + name = os.path.basename(full_path) + results.append(ProjectFile(filename=name, path=full_path, context=FileContext(name, content))) + + return results, truncated + + +class ProjectIndex: + """A loaded set of FileContexts under a root, with project-wide queries.""" + + def __init__(self, root: str, files: List[ProjectFile], truncated: bool): + self.root = root + self.files = files + self.truncated = truncated + self._file_map: Dict[str, FileContext] = { + os.path.relpath(f.path, root): f.context for f in files + } + + @staticmethod + def load(root: str) -> "ProjectIndex": + resolved = os.path.abspath(root) + files, truncated = _discover_files(resolved) + return ProjectIndex(resolved, files, truncated) + + def get_file(self, rel_path: str) -> Optional[FileContext]: + return self._file_map.get(rel_path) + + def list_files(self) -> List[str]: + return [os.path.relpath(f.path, self.root) for f in self.files] + + def get_important_positions( + self, min_coefficient: float = 0.3, limit: int = 20 + ) -> List[ImportantPosition]: + """Project-wide important positions across all files, top `limit` by + |coefficient|. Labels are suffixed with the relative file path.""" + top: List[ImportantPosition] = [] + for f in self.files: + peaks = f.context.get_important_positions(min_coefficient, max(limit, 30)) + if not peaks: + continue + rel = os.path.relpath(f.path, self.root) + for p in peaks: + top.append( + ImportantPosition( + position=p.position, + coefficient=p.coefficient, + scale=p.scale, + label=f"{p.label} ({rel})", + filename=rel, + ) + ) + top.sort(key=lambda x: abs(x.coefficient), reverse=True) + if len(top) > limit: + del top[limit:] + return top + + +def decompose_project( + root: str, min_coefficient: float = 0.3, limit: int = 20 +) -> List[ImportantPosition]: + """Convenience entry point: load the project and return its top structural + positions (the coarse "which files / regions matter" map for planning).""" + return ProjectIndex.load(root).get_important_positions(min_coefficient, limit) diff --git a/v3-service/wavelet/signal.py b/v3-service/wavelet/signal.py new file mode 100644 index 00000000..d27d0622 --- /dev/null +++ b/v3-service/wavelet/signal.py @@ -0,0 +1,255 @@ +"""Per-line structural-importance signal. + +Faithful port of wavescope-mcp `src/signal.ts`. Produces a per-line score from +indentation, structural keywords, and decorators, with comment / string-literal +awareness. Scores are clamped to [0, 2]. +""" + +from __future__ import annotations + +import math +import re +from typing import List + +from .language import LanguageConfig + + +# Token splitter: splits on whitespace, brackets, braces, parens, commas, +# semicolons, colons, quotes, backticks, arithmetic, logical, and bitwise +# operators. Does NOT split on `.`, `-`, `?`, `!` — so `obj.class` stays one +# token (preventing member-access keyword leaks) and `extend-type`, +# `defmulti`, `defined?` survive as single tokens. +_TOKEN_SPLIT = re.compile(r"[\s()\[\]{},;:'\"`=<>+*/&|^~%@#\\]+") + + +def _mask_string_literals(line: str, lang: LanguageConfig) -> str: + """Mask the interior of single-line string/char literals with spaces so + downstream comment / token detection ignores their contents. Handles + single-quote, double-quote, and backtick literals with backslash escapes.""" + # In Rust `'a` is a lifetime and in Clojure `'` is the quote reader macro — + # neither delimits a string, so treating `'` as one would mask real code. + single_quote_is_string = lang.name not in ("rust", "clojure") + chars = list(line) + i = 0 + n = len(chars) + while i < n: + c = chars[i] + if c == '"' or c == "`" or (single_quote_is_string and c == "'"): + quote = c + j = i + 1 + while j < n: + if chars[j] == "\\" and j + 1 < n: + chars[j] = " " + chars[j + 1] = " " + j += 2 + continue + if chars[j] == quote: + break + chars[j] = " " + j += 1 + i = j + 1 + continue + i += 1 + return "".join(chars) + + +def _tokenize(line: str) -> List[str]: + return _TOKEN_SPLIT.split(line) + + +def _score_line(raw: str, stripped: str, raw_indent: int, lang: LanguageConfig) -> float: + """Score a single line of actual code (no comments).""" + score = 0.0 + + # Strip string literals first so quoted `//` URLs and quoted `/*` don't + # confuse keyword / inline-comment scanning. + code_only = _mask_string_literals(stripped, lang) + + # Strip inline single-line comment suffix. + for prefix in lang.comment_prefixes: + idx = -1 + frm = 0 + while frm < len(code_only): + nxt = code_only.find(prefix, frm) + if nxt == -1: + break + # PHP 8 attribute: `#[...]` is not a comment. + if prefix == "#" and nxt + 1 < len(code_only) and code_only[nxt + 1] == "[": + frm = nxt + 1 + continue + idx = nxt + break + if idx != -1: + code_only = code_only[:idx].strip() + break + + # Indent: expand tabs as 4 spaces so tab-indented files score comparably + # to space-indented ones. + leading = raw[:raw_indent] + expanded_indent = 0 + for ch in leading: + expanded_indent += 4 if ch == "\t" else 1 + indent_level = min(expanded_indent / 4, 8) + score += indent_level * lang.indent_weight + + for token in _tokenize(code_only): + if not token: + continue + if token in lang.structural_keywords: + score += lang.structural_keywords[token] + + # Decorators / annotations: line-start `@`, Rust `#[...]`, PHP 8 `#[...]`, + # or inline `@Annotation` (e.g. Java `public @Nullable String foo()`). + if lang.decorator_weight > 0: + starts_with_at = code_only.startswith("@") + rust_attr = lang.name == "rust" and code_only.startswith("#[") + php_attr = lang.name == "php" and code_only.startswith("#[") + inline_at = re.search(r"(^|\s)@[A-Za-z_]", code_only) is not None + if starts_with_at or rust_attr or php_attr or inline_at: + score += lang.decorator_weight + + return min(score, 2.0) + + +def compute_signal(lines: List[str], lang: LanguageConfig) -> List[float]: + """Compute the per-line structural signal for a complete file. + + NOTE: callers must pass a complete file (all lines), not a slice — the + block-comment / docstring state machine spans lines but is initialized + fresh per call, so a partial file would misclassify comments. + """ + signal: List[float] = [0.0] * len(lines) + in_block_comment = False + block_comment_depth = 0 + in_doc_string = False + doc_string_delim = None + + for i, rawline in enumerate(lines): + raw = rawline + trimmed = raw.lstrip() + indent = len(raw) - len(trimmed) + stripped = trimmed.rstrip() + # String-masked version used for comment/keyword detection; original + # line drives indent/length calculations. + masked = _mask_string_literals(stripped, lang) + + # ── Continuation of multiline comments / docstrings ── + if in_block_comment: + if lang.block_comment_uses_paren_depth: + depth = block_comment_depth + close_idx = -1 + for ci, mc in enumerate(masked): + if mc == "(": + depth += 1 + elif mc == ")": + depth -= 1 + if depth == 0: + close_idx = ci + break + if close_idx != -1: + in_block_comment = False + block_comment_depth = 0 + after = stripped[close_idx + 1:] + signal[i] = _score_line(raw, after.strip(), indent, lang) if after.strip() else 0.0 + else: + block_comment_depth = depth + signal[i] = 0.0 + else: + end_idx = masked.find(lang.block_comment_end) + if end_idx != -1: + in_block_comment = False + after = stripped[end_idx + len(lang.block_comment_end):] + signal[i] = _score_line(raw, after.strip(), indent, lang) if after.strip() else 0.0 + else: + signal[i] = 0.0 + continue + + if in_doc_string: + end_idx = masked.find(doc_string_delim) + if end_idx != -1: + in_doc_string = False + doc_string_delim = None + after = stripped[end_idx + 3:] + signal[i] = _score_line(raw, after.strip(), indent, lang) if after.strip() else 0.0 + else: + signal[i] = 0.0 + continue + + # ── Detect new Python docstrings (triple quotes) ── + # Scanned on the unmasked stripped line because _mask_string_literals + # would have consumed them as regular strings. + if lang.name == "python": + dq = stripped.find('"""') + sq = stripped.find("'''") + dq_idx = min( + dq if dq != -1 else math.inf, + sq if sq != -1 else math.inf, + ) + if math.isfinite(dq_idx): + dq_idx = int(dq_idx) + delim = stripped[dq_idx:dq_idx + 3] + before = stripped[:dq_idx].strip() + after = stripped[dq_idx + 3:] + has_closing = delim in after + + if has_closing: + real = "; ".join( + p for p in [before, after.replace(delim, "").strip()] if p + ) + signal[i] = _score_line(raw, real, indent, lang) if real else 0.0 + continue + in_doc_string = True + doc_string_delim = delim + signal[i] = _score_line(raw, before, indent, lang) if before else 0.0 + continue + + # ── Detect block comment start (anywhere on line, including inline) ── + if lang.block_comment_at_line_start: + bc_start_idx = 0 if masked.startswith(lang.block_comment_start) else -1 + else: + bc_start_idx = masked.find(lang.block_comment_start) + if bc_start_idx != -1: + before = stripped[:bc_start_idx].strip() + after_delim = masked[bc_start_idx + len(lang.block_comment_start):] + after_delim_raw = stripped[bc_start_idx + len(lang.block_comment_start):] + + end_idx = -1 + if lang.block_comment_uses_paren_depth: + depth = 1 + for ci, mc in enumerate(after_delim): + if mc == "(": + depth += 1 + elif mc == ")": + depth -= 1 + if depth == 0: + end_idx = ci + break + else: + end_idx = after_delim.find(lang.block_comment_end) + + if end_idx != -1: + after = after_delim_raw[end_idx + len(lang.block_comment_end):].strip() + real = "; ".join(p for p in [before, after] if p) + signal[i] = _score_line(raw, real, indent, lang) if real else 0.0 + continue + in_block_comment = True + block_comment_depth = 1 if lang.block_comment_uses_paren_depth else 0 + signal[i] = _score_line(raw, before, indent, lang) if before else 0.0 + continue + + # ── Single-line comments ── + def _is_comment(p: str) -> bool: + if not masked.startswith(p): + return False + # PHP 8 attributes: `#[...]` is NOT a comment. + if p == "#" and masked.startswith("#["): + return False + return True + + if any(_is_comment(p) for p in lang.comment_prefixes) or len(masked) == 0: + signal[i] = 0.0 + continue + + signal[i] = _score_line(raw, stripped, indent, lang) + + return signal From b57a0835e2f4a3c226bc3e68ceda235f1e4be07d Mon Sep 17 00:00:00 2001 From: Yogthos Date: Wed, 10 Jun 2026 13:25:40 -0400 Subject: [PATCH 2/6] fix(v3.2): address review findings on RPG planning (#120) A review of the feature turned up two problems that made it inoperative in the default Docker deployment, plus several correctness gaps. This fixes all of them. The flag never reached the v3-service container, so the whole feature was dead when a user set ATLAS_RPG_PLANNING in their .env. It is now forwarded in docker-compose.yml, along with ATLAS_PLAN_THINKING. The proposal stage seeded its coarse band by scanning working_dir off disk, but the v3-service container has no project volume mount, so that scan always found nothing in Docker. It now builds the coarse band from the in-memory project_context the proxy already sends, through a new decompose_file_map entry point, and falls back to a disk scan only when no context was passed. The plan result is a single SSE line, and with RPG on it now carries the whole graph. The reader buffer was 1MB, so a large graph could trip ErrTooLong and silently drop the plan. The buffer is raised to 16MB and a scanner error is now surfaced instead of being reported as a missing result. Signature parsing mishandled Go receiver methods like func (s *S) Load(), and the bare-name fallback would return the keyword func, which produced a false missing-signature veto. Parsing now recognizes receiver methods and never treats a lone declaration keyword as a function name, and defined_names picks up receiver methods too. The edit_file and ast_edit path threaded RPG constraints but never ran the drift retry or reporting that the write path does, so drift on edits was invisible. It now runs the same regenerateOnDrift and reportRPGDrift. planConstraintsForTarget returned the first overlapping step, so a bare-basename step could shadow the deeper path it was not for. It now prefers the most specific match. The corrective regeneration accepted a retry only when the total miss count dropped, which discarded a retry that fixed the targeted signature but traded in a different one; it now accepts a retry that resolves any of the signatures it was correcting for. The topological sort dedups file ids so a duplicate cannot fake acyclicity or drop a file, and node_drift gained a threshold so its drift score actually gates the re-plan decision. Tests: 163 Python tests and the proxy Go suite, all green. --- docker-compose.yml | 6 +++ docs/reports/RPG_WAVELET_PLANNING_V3_2.md | 15 ++++++ proxy/rpg.go | 51 ++++++++++++++---- proxy/rpg_test.go | 44 ++++++++++++++++ proxy/tools.go | 8 +++ proxy/v3_bridge.go | 13 ++++- tests/v3-service/test_rpg.py | 63 +++++++++++++++++++++++ tests/v3-service/test_wavelet_project.py | 28 +++++++++- v3-service/main.py | 20 +++++-- v3-service/rpg.py | 47 +++++++++++++---- v3-service/wavelet/__init__.py | 3 +- v3-service/wavelet/context.py | 4 +- v3-service/wavelet/project.py | 31 +++++++++++ 13 files changed, 304 insertions(+), 29 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 68bda487..02e6b8c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -106,6 +106,12 @@ services: - ATLAS_LENS_URL=http://geometric-lens:8099 - ATLAS_SANDBOX_URL=http://sandbox:8020 - ATLAS_MODEL_NAME=${ATLAS_MODEL_NAME:-Qwen3.5-9B-Q6_K} + # V3.2 RPG-style planning (#120, experimental, default off) and the + # planner thinking toggle. These are read by rpg_planning_enabled() and + # the planner inside this process, so they must be forwarded to this + # container, not only to the proxy. + - ATLAS_RPG_PLANNING=${ATLAS_RPG_PLANNING:-0} + - ATLAS_PLAN_THINKING=${ATLAS_PLAN_THINKING:-0} ports: - "${ATLAS_V3_PORT:-8070}:8070" depends_on: diff --git a/docs/reports/RPG_WAVELET_PLANNING_V3_2.md b/docs/reports/RPG_WAVELET_PLANNING_V3_2.md index 587e77e9..435715b9 100644 --- a/docs/reports/RPG_WAVELET_PLANNING_V3_2.md +++ b/docs/reports/RPG_WAVELET_PLANNING_V3_2.md @@ -3,6 +3,21 @@ Tracks **[#120](https://github.com/itigges22/ATLAS/issues/120)**. Builds on the wavelet decomposition angle from **[#39](https://github.com/itigges22/ATLAS/issues/39)**. +### Post-review hardening + +A review pass found two deployment problems and several correctness gaps, all +fixed. The flag is now forwarded to the v3-service container in +`docker-compose.yml` (it was unreachable before, making the feature dead in the +default deployment), and the proposal-stage coarse band is built from the +in-memory `project_context` via `decompose_file_map` rather than a disk scan, +because the v3-service container has no project volume mount. The `/v3/plan` +SSE reader buffer was raised so the larger RPG payload cannot silently drop a +plan, signature parsing now handles Go-style receiver methods and never treats +a bare declaration keyword as a function name, the edit path now runs the same +drift retry and reporting as the write path, `planConstraintsForTarget` prefers +the most-specific matching step, the corrective regeneration accepts a retry +that resolves the targeted signatures, and the topological sort dedups file ids. + ## 1. What we're building Replace ATLAS's flat, free-form planning artifact with a **structured, two-stage, diff --git a/proxy/rpg.go b/proxy/rpg.go index 1f53cc7e..3a427ef4 100644 --- a/proxy/rpg.go +++ b/proxy/rpg.go @@ -27,22 +27,29 @@ func planConstraintsForTarget(ctx *AgentContext, path string) []string { if ctx == nil || ctx.Plan == nil { return nil } + // Pick the most specific overlapping step rather than the first. With + // suffix-based matching a bare-basename step (e.g. target "user.py") would + // otherwise shadow the deeper "models/user.py" step it isn't really for, so + // prefer the longest matching Target. + best := -1 + bestLen := -1 for i := range ctx.Plan.Steps { step := &ctx.Plan.Steps[i] - if len(step.Constraints) == 0 { + if len(step.Constraints) == 0 || !isFileProducingAction(step.Action) { continue } - if !isFileProducingAction(step.Action) { - continue - } - if targetsOverlap(step.Target, path) { - // Copy so callers can append without mutating the plan. - out := make([]string, len(step.Constraints)) - copy(out, step.Constraints) - return out + if targetsOverlap(step.Target, path) && len(step.Target) > bestLen { + best = i + bestLen = len(step.Target) } } - return nil + if best < 0 { + return nil + } + // Copy so callers can append without mutating the plan. + out := make([]string, len(ctx.Plan.Steps[best].Constraints)) + copy(out, ctx.Plan.Steps[best].Constraints) + return out } func isFileProducingAction(action string) bool { @@ -148,12 +155,36 @@ func regenerateOnDrift(ctx *AgentContext, req V3GenerateRequest, prev *V3Generat if err != nil || retried == nil || retried.Code == "" { return prev } + // Accept the retry when it resolves at least one of the signatures we were + // correcting for, even if it traded in a different miss (so the count is + // unchanged). A plain count comparison would discard a retry that fixed the + // targeted gap but reshaped the code elsewhere. Falling back to total count + // covers the case where the targeted set is somehow unchanged. + stillMissingTargeted := countOverlap(retried.RPGSignatureMissing, prev.RPGSignatureMissing) + if stillMissingTargeted < len(prev.RPGSignatureMissing) { + return retried + } if len(retried.RPGSignatureMissing) < len(prev.RPGSignatureMissing) { return retried } return prev } +// countOverlap returns how many elements of `subset` appear in `of`. +func countOverlap(subset, of []string) int { + set := make(map[string]struct{}, len(of)) + for _, s := range of { + set[s] = struct{}{} + } + n := 0 + for _, s := range subset { + if _, ok := set[s]; ok { + n++ + } + } + return n +} + // reportRPGDrift surfaces structural drift after a V3 write: the winning code // for `path` did not realize the planned signatures `missing`. It emits a drift // event naming the node and the downstream subgraph that depended on it, so the diff --git a/proxy/rpg_test.go b/proxy/rpg_test.go index f4ce89d3..503e5551 100644 --- a/proxy/rpg_test.go +++ b/proxy/rpg_test.go @@ -92,6 +92,19 @@ func TestPlanConstraintsForTarget(t *testing.T) { } } +func TestPlanConstraintsForTargetPrefersMostSpecific(t *testing.T) { + // A bare-basename step must not shadow the deeper path it isn't for. + plan := &Plan{Steps: []PlanStep{ + {ID: "s1", Action: "write_file", Target: "user.py", Constraints: []string{"BARE"}}, + {ID: "s2", Action: "write_file", Target: "models/user.py", Constraints: []string{"DEEP"}}, + }} + ctx := &AgentContext{Plan: plan} + got := planConstraintsForTarget(ctx, "/workspace/models/user.py") + if !reflect.DeepEqual(got, []string{"DEEP"}) { + t.Errorf("expected the most-specific (models/user.py) step, got %v", got) + } +} + func TestPlanConstraintsForTargetNilPlan(t *testing.T) { if got := planConstraintsForTarget(&AgentContext{}, "x.py"); got != nil { t.Errorf("nil plan should yield nil, got %v", got) @@ -246,6 +259,37 @@ func TestRegenerateOnDriftNoopWithoutDriftOrConstraints(t *testing.T) { } } +func TestRegenerateOnDriftAcceptsRetryThatResolvesTargeted(t *testing.T) { + // prev missed [A]; the retry realizes A but drops B, so the total count is + // still 1. The retry must still be accepted because it resolved the + // signature it was correcting for (count-only logic would wrongly keep prev). + srv := generateServer(t, + `{"code": "def A(): pass", "rpg_signature_missing": ["def B()"]}`, nil) + defer srv.Close() + ctx := &AgentContext{V3URL: srv.URL} + req := V3GenerateRequest{FilePath: "m.py", Constraints: []string{"Implement `def A()`"}} + prev := &V3GenerateResponse{Code: "old", RPGSignatureMissing: []string{"def A()"}} + got := regenerateOnDrift(ctx, req, prev) + if got == prev { + t.Error("retry that resolved the targeted signature should be accepted") + } + if !strings.Contains(got.Code, "def A") { + t.Errorf("expected retried code, got %q", got.Code) + } +} + +func TestCountOverlap(t *testing.T) { + if got := countOverlap([]string{"a", "b", "c"}, []string{"b", "c", "d"}); got != 2 { + t.Errorf("countOverlap = %d, want 2", got) + } + if got := countOverlap([]string{"x"}, []string{"y"}); got != 0 { + t.Errorf("countOverlap = %d, want 0", got) + } + if got := countOverlap(nil, []string{"a"}); got != 0 { + t.Errorf("countOverlap(nil) = %d, want 0", got) + } +} + func TestRegenerateOnDriftKeepsPrevWhenRetryNoBetter(t *testing.T) { // Server always drifts → retry is no better → keep prev. srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/proxy/tools.go b/proxy/tools.go index ece39a9b..f8d94500 100644 --- a/proxy/tools.go +++ b/proxy/tools.go @@ -1283,6 +1283,14 @@ func improveContentWithV3(path, content string, ctx *AgentContext) (string, V3Ed return "", V3EditMetadata{}, err } + // V3.2 RPG (issue #120): same drift handling as the write_file path. If the + // edited result missed its planned signatures, retry once, then surface any + // surviving drift. No-op when RPG is off (req.Constraints empty). + v3Result = regenerateOnDrift(ctx, req, v3Result) + if len(v3Result.RPGSignatureMissing) > 0 { + reportRPGDrift(ctx, path, v3Result.RPGSignatureMissing) + } + if ctx.StreamFn != nil { ctx.StreamFn("v3_progress", map[string]string{ "message": fmt.Sprintf(" └──── V3 complete: %s, %d candidates", v3Result.PhaseSolved, v3Result.CandidatesTested), diff --git a/proxy/v3_bridge.go b/proxy/v3_bridge.go index 82bf5082..1f3ef117 100644 --- a/proxy/v3_bridge.go +++ b/proxy/v3_bridge.go @@ -138,7 +138,12 @@ func callV3PlanStreaming(v3URL string, req V3PlanRequest, onProgress V3ProgressF } scanner := bufio.NewScanner(resp.Body) - scanner.Buffer(make([]byte, 0, 1<<20), 1<<20) + // The plan result is one SSE `data:` line. With V3.2 RPG planning it now + // carries the whole graph (files + signatures + edges) plus per-step + // constraints, so the line is far larger than a flat plan. Allow up to 16MB + // so a big graph degrades gracefully instead of tripping ErrTooLong and + // silently dropping the plan (#120 review). + scanner.Buffer(make([]byte, 0, 1<<20), 16<<20) var plan *Plan @@ -176,6 +181,12 @@ func callV3PlanStreaming(v3URL string, req V3PlanRequest, onProgress V3ProgressF } } + if err := scanner.Err(); err != nil { + // Most likely bufio.ErrTooLong on an oversized result line. Surface it + // so the cause is diagnosable instead of a misleading "no result". + return nil, fmt.Errorf("reading plan stream: %w", err) + } + if plan == nil { return nil, fmt.Errorf("plan service completed without result") } diff --git a/tests/v3-service/test_rpg.py b/tests/v3-service/test_rpg.py index 97541e1c..6e55540c 100644 --- a/tests/v3-service/test_rpg.py +++ b/tests/v3-service/test_rpg.py @@ -206,6 +206,24 @@ def test_no_verify_no_verify_step(self): assert plan["verify_step"] is None assert all(s["action"] == "write_file" for s in plan["steps"]) + def test_duplicate_file_ids_dont_duplicate_or_drop_steps(self): + # Duplicate ids must not inflate the topological order (which would + # fake acyclicity and drop a file). Each unique id yields one step. + g = RPG( + capabilities=[Capability("c1", "Cap")], + files=[ + FileSpec("f1", "a.py", "c1", [FunctionSpec("a")]), + FileSpec("f1", "b.py", "c1", [FunctionSpec("b")]), + FileSpec("f2", "c.py", "c1", [FunctionSpec("c")]), + ], + edges=[Edge("f1", "f2")], + ) + plan = flatten_to_plan(g) + write_targets = [s["target"] for s in plan["steps"] if s["action"] == "write_file"] + assert len(write_targets) == 2 # f1 (once) + f2, not 3 + node_ids = [s["node_id"] for s in plan["steps"] if s.get("node_id")] + assert len(node_ids) == len(set(node_ids)) # no duplicate node steps + def test_steps_carry_node_id_and_constraints(self): g = parse_rpg(IMPL_JSON) plan = flatten_to_plan(g) @@ -309,10 +327,43 @@ def test_go_regex(self): code = "package main\nfunc Load() {}\nfunc Run() {}\n" assert defined_names(code, "m.go") == {"Load", "Run"} + def test_go_receiver_method(self): + code = "package main\nfunc (s *Store) Load() error { return nil }\nfunc Top() {}\n" + assert defined_names(code, "m.go") == {"Load", "Top"} + def test_opaque_code_empty(self): assert defined_names("x = 1\ny = 2\n", "m.py") == set() +class TestFunctionNameParsing: + def test_go_receiver_method_name(self): + from rpg import _function_name_from_signature as fn + assert fn("func (s *Store) Load() error") == "Load" + + def test_plain_keyword_forms(self): + from rpg import _function_name_from_signature as fn + assert fn("def load(p): ...") == "load" + assert fn("async def go()") == "go" + assert fn("func Run()") == "Run" + assert fn("class Foo(Base)") == "Foo" + assert fn("bare_name(x)") == "bare_name" + + def test_lone_keyword_returns_empty(self): + # A signature that is just a keyword must not parse as a function named + # after the keyword (which would cause a false "missing" veto). + from rpg import _function_name_from_signature as fn + assert fn("func") == "" + assert fn("def") == "" + assert fn("class") == "" + + def test_go_method_not_falsely_missing(self): + # Regression: a planned Go receiver method that IS defined must not be + # reported missing (previously parsed as the keyword "func"). + code = "func (s *Store) Load() error { return nil }\n" + planned = ["func (s *Store) Load() error"] + assert missing_planned_signatures(code, planned, "store.go") == [] + + class TestPlannedSignatureExtraction: def test_recovers_signatures_from_constraints(self): cons = [ @@ -372,6 +423,18 @@ def test_opaque_code_no_blind_replan(self): d = node_drift(g, "f2", "rows = []", "src/process.py") assert d.should_replan is False + def test_threshold_tolerates_small_drift(self): + # Two planned functions, one realized: drift_score 0.5. A threshold of + # 0.5 should tolerate it (0.5 > 0.5 is false); 0.0 should not. + g = RPG( + capabilities=[Capability("c1", "Cap")], + files=[FileSpec("f1", "a.py", "c1", + [FunctionSpec("keep"), FunctionSpec("drop")])], + ) + code = "def keep():\n return 1\n" + assert node_drift(g, "f1", code, "a.py").should_replan is True + assert node_drift(g, "f1", code, "a.py", replan_threshold=0.5).should_replan is False + class TestLocalize: def test_ranks_relevant_nodes(self): diff --git a/tests/v3-service/test_wavelet_project.py b/tests/v3-service/test_wavelet_project.py index 0a2ff13d..531d3604 100644 --- a/tests/v3-service/test_wavelet_project.py +++ b/tests/v3-service/test_wavelet_project.py @@ -10,7 +10,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "v3-service")) -from wavelet.project import decompose_project, ProjectIndex # noqa: E402 +from wavelet.project import decompose_project, decompose_file_map, ProjectIndex # noqa: E402 PY_A = """class Alpha: def run(self): @@ -121,3 +121,29 @@ def test_limit_caps_results(self, tmp_path): def test_empty_project(self, tmp_path): assert decompose_project(str(tmp_path)) == [] + + +class TestDecomposeFileMap: + def test_in_memory_matches_disk(self, tmp_path): + # The in-memory entry point (used when v3-service has no project mount) + # should produce the same labels/filenames as a disk scan of the same + # files. + _write(tmp_path, "a.py", PY_A) + _write(tmp_path, "b.py", PY_B) + disk = decompose_project(str(tmp_path), limit=20) + mem = decompose_file_map({"a.py": PY_A, "b.py": PY_B}, limit=20) + assert {(p.filename, p.label) for p in mem} == {(p.filename, p.label) for p in disk} + + def test_labels_carry_path_and_sorted(self): + positions = decompose_file_map({"a.py": PY_A, "b.py": PY_B}, limit=20) + assert positions + assert all(p.filename in ("a.py", "b.py") for p in positions) + for i in range(1, len(positions)): + assert abs(positions[i - 1].coefficient) >= abs(positions[i].coefficient) + + def test_empty_and_blank(self): + assert decompose_file_map({}) == [] + assert decompose_file_map({"a.py": ""}) == [] + + def test_limit_caps(self): + assert len(decompose_file_map({"a.py": PY_A, "b.py": PY_B}, limit=2)) <= 2 diff --git a/v3-service/main.py b/v3-service/main.py index ed407dd8..491d7a89 100644 --- a/v3-service/main.py +++ b/v3-service/main.py @@ -1987,22 +1987,32 @@ def emit(stage: str, detail: str = "", **data): # ATLAS_RPG_PLANNING. Strictly additive: on any failure (flag off, modules # missing, model output unusable) we fall through to the flat planner below. try: - from wavelet import rpg_planning_enabled, decompose_project + from wavelet import rpg_planning_enabled, decompose_project, decompose_file_map import rpg as _rpg_mod _rpg_on = rpg_planning_enabled() except Exception: _rpg_on = False if _rpg_on: try: + # Build the coarse structural band that seeds the proposal stage. + # Prefer the in-memory project_context the proxy already sent — the + # v3-service container has no project volume mount, so reading + # working_dir off disk only works in non-container / dev setups. + # Fall back to a disk scan when no context was passed. coarse_map = None - if working_dir and os.path.isdir(working_dir): - try: + try: + if project_context: + coarse_map = [ + {"label": p.label, "filename": p.filename} + for p in decompose_file_map(project_context, limit=30) + ] + elif working_dir and os.path.isdir(working_dir): coarse_map = [ {"label": p.label, "filename": p.filename} for p in decompose_project(working_dir, limit=30) ] - except Exception as ce: - emit("rpg_coarse_error", f"coarse decomposition failed: {ce}") + except Exception as ce: + emit("rpg_coarse_error", f"coarse decomposition failed: {ce}") rpg_thinking = os.environ.get("ATLAS_PLAN_THINKING", "0").lower() in ("1", "true", "yes") rpg_llm = LLMAdapter(progress_callback=progress_callback, thinking=rpg_thinking) diff --git a/v3-service/rpg.py b/v3-service/rpg.py index efa77659..4cd8d16c 100644 --- a/v3-service/rpg.py +++ b/v3-service/rpg.py @@ -226,6 +226,10 @@ def _topo_order(file_ids: List[str], edges: List[Edge]) -> Tuple[List[str], bool """Kahn topological sort over file ids (producer -> consumer). Returns (order, acyclic). On a cycle, returns the original declaration order and acyclic=False.""" + # Dedup while preserving first-seen order: a duplicate id would otherwise be + # emitted twice, inflating `order` so len(order)==len(file_ids) holds and the + # graph is falsely reported acyclic (and flatten_to_plan would drop a file). + file_ids = list(dict.fromkeys(file_ids)) idset = set(file_ids) adj: Dict[str, List[str]] = {fid: [] for fid in file_ids} indeg: Dict[str, int] = {fid: 0 for fid in file_ids} @@ -416,19 +420,35 @@ def flatten_to_plan(rpg: RPG) -> dict: # ─── Graph-guided verification, drift & localization (Phase 3) ─── +_DECL_KEYWORDS = frozenset({ + "def", "func", "function", "fn", "fun", "class", "struct", + "interface", "trait", "type", "async", "impl", "object", +}) + + def _function_name_from_signature(sig: str) -> str: """Pull the declared name out of a signature string. Handles - `def load(...)`, `async def load(...)`, `func Load(...)`, `fn run(...)`, - `function go(...)`, `class Foo`, or a bare name.""" + `def load(...)`, `async def load(...)`, `func Load(...)`, a Go receiver + method `func (r *T) Load(...)`, `fn run(...)`, `function go(...)`, + `class Foo`, or a bare name. Returns "" when the only token is a bare + declaration keyword (e.g. signature is just `func`), so callers treat it + as unknown rather than checking for a function literally named `func`.""" import re s = sig.strip() - m = re.search(r"\b(?:async\s+)?(?:def|func|function|fn|fun|class|struct|interface|trait|type)\s+([A-Za-z_][\w]*)", s) + # Go-style receiver method: `func (r *T) Name(...)` — the name follows the + # receiver parens, so the plain `keyword name` pattern would miss it. + m = re.search(r"\bfunc\s*\([^)]*\)\s*([A-Za-z_]\w*)", s) + if m: + return m.group(1) + m = re.search(r"\b(?:async\s+)?(?:def|func|function|fn|fun|class|struct|interface|trait|type)\s+([A-Za-z_]\w*)", s) if m: return m.group(1) - # Bare "name" or "name(args)". - m = re.match(r"\s*([A-Za-z_][\w]*)", s) - return m.group(1) if m else "" + # Bare "name" / "name(args)", but never a lone declaration keyword. + m = re.match(r"\s*([A-Za-z_]\w*)", s) + if m and m.group(1) not in _DECL_KEYWORDS: + return m.group(1) + return "" def defined_names(code: str, filename: str) -> set: @@ -457,6 +477,9 @@ def defined_names(code: str, filename: str) -> set: code, ): names.add(m.group(1)) + # Go-style receiver methods: `func (r *T) Name(...)`. + for m in re.finditer(r"\bfunc\s*\([^)]*\)\s*([A-Za-z_]\w*)", code): + names.add(m.group(1)) return names @@ -525,10 +548,15 @@ class DriftReport: should_replan: bool # True when structure drifted from the plan -def node_drift(rpg: RPG, file_id: str, code: str, filename: str) -> DriftReport: +def node_drift(rpg: RPG, file_id: str, code: str, filename: str, + replan_threshold: float = 0.0) -> DriftReport: """Compare a node's planned structure against generated `code`. A planned function that didn't get realized is structural drift away from the RPG — - the signal to re-plan the affected subgraph (Phase 3 drift loop).""" + the signal to re-plan the affected subgraph (Phase 3 drift loop). + + `should_replan` fires when the drift fraction exceeds `replan_threshold`. + The default of 0.0 means any unrealized planned function triggers a re-plan; + callers that tolerate small gaps (e.g. an omitted helper) can raise it.""" by_id = {f.id: f for f in rpg.files} f = by_id.get(file_id) planned_names = [fn.name for fn in f.functions if fn.name] if f else [] @@ -540,7 +568,8 @@ def node_drift(rpg: RPG, file_id: str, code: str, filename: str) -> DriftReport: return DriftReport(file_id=file_id, missing=[], drift_score=0.0, should_replan=False) missing = [n for n in planned_names if n not in defined] drift = len(missing) / len(planned_names) - return DriftReport(file_id=file_id, missing=missing, drift_score=drift, should_replan=bool(missing)) + return DriftReport(file_id=file_id, missing=missing, drift_score=drift, + should_replan=drift > replan_threshold) def _tokenize_query(text: str) -> set: diff --git a/v3-service/wavelet/__init__.py b/v3-service/wavelet/__init__.py index fb31440e..a821c1bd 100644 --- a/v3-service/wavelet/__init__.py +++ b/v3-service/wavelet/__init__.py @@ -34,7 +34,7 @@ BandResult, WaveletContextResult, ) -from .project import decompose_project, ProjectIndex +from .project import decompose_project, decompose_file_map, ProjectIndex from .diff import diff_peaks, diff_contents, FileDiffResult, PeakChange, PeakDiff from .flags import rpg_planning_enabled, ENV_VAR as RPG_PLANNING_ENV_VAR @@ -54,6 +54,7 @@ "BandResult", "WaveletContextResult", "decompose_project", + "decompose_file_map", "ProjectIndex", "diff_peaks", "diff_contents", diff --git a/v3-service/wavelet/context.py b/v3-service/wavelet/context.py index 276a350b..e3011850 100644 --- a/v3-service/wavelet/context.py +++ b/v3-service/wavelet/context.py @@ -309,10 +309,10 @@ def _name_after_ws(i: int) -> str: if self.language.name == "python": if ws_tokens[0] == "class": name = _name_after_ws(1).replace(":", "") - return "class " + name + return ("class " + name).rstrip() if ws_tokens[0] == "def": name = _name_after_ws(1).split("(")[0] - return "def " + name + return ("def " + name).rstrip() if ws_tokens[0] == "import": return "import " + " ".join(ws_tokens[1:]) if ws_tokens[0] == "from": diff --git a/v3-service/wavelet/project.py b/v3-service/wavelet/project.py index 1593c712..a335026c 100644 --- a/v3-service/wavelet/project.py +++ b/v3-service/wavelet/project.py @@ -280,3 +280,34 @@ def decompose_project( """Convenience entry point: load the project and return its top structural positions (the coarse "which files / regions matter" map for planning).""" return ProjectIndex.load(root).get_important_positions(min_coefficient, limit) + + +def decompose_file_map( + file_map: Dict[str, str], min_coefficient: float = 0.3, limit: int = 20 +) -> List[ImportantPosition]: + """Like decompose_project but over in-memory {rel_path: content} rather than + the filesystem. Used when the caller already has the project files in hand + (e.g. the proxy sends them to v3-service, which has no project volume mount), + so the coarse band can be computed without disk access. Mirrors + ProjectIndex.get_important_positions: top `limit` positions by |coefficient|, + labels suffixed with the relative path.""" + top: List[ImportantPosition] = [] + for rel, content in file_map.items(): + if not content: + continue + ctx = FileContext(os.path.basename(rel), content) + peaks = ctx.get_important_positions(min_coefficient, max(limit, 30)) + for p in peaks: + top.append( + ImportantPosition( + position=p.position, + coefficient=p.coefficient, + scale=p.scale, + label=f"{p.label} ({rel})", + filename=rel, + ) + ) + top.sort(key=lambda x: abs(x.coefficient), reverse=True) + if len(top) > limit: + del top[limit:] + return top From b51faacd3f4aec9eb22d33d5b3b6c55068d41ded Mon Sep 17 00:00:00 2001 From: Johnathon Isaac Tigges Date: Sat, 18 Jul 2026 16:55:58 -0400 Subject: [PATCH 3/6] review fixes: harden RPG acceptance, signature extraction, and wavelet parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review of the merged branch (three parallel reviewers, every finding re-verified against the code before fixing): proxy/rpg.go: - The drift-regen accept rule could adopt a strictly worse retry: prev missing [A,B], retry missing [B,C,D] resolved one targeted signature and was accepted with more total drift. The first branch now also requires the total not to grow. - planConstraintsForTarget gains the RPG-artifact trust gate: the flat planner passes the LLM's plan dict through verbatim, so hallucinated step constraints must not steer generation while the flag is off. Also ranks ALL file-producing steps so a deeper unconstrained step is not shadowed by a shallower constrained one. - countOverlap counts distinct names (duplicate service entries no longer double-count); nil-ctx guard; rpg_regen stage gets a matching EvtStageEnd; rpg_drift is an EvtMetric point event; affectedDownstream follows data_flow edges only, as documented. proxy/tools.go: - Cancellation recheck after the regen retry on both the write_file and improveContentWithV3 paths — a user cancel during the second V3 window must not land content on disk. - Drift reports gate on req.Constraints so a rogue response field alone cannot fire them. v3-service/rpg.py: - Signature-name extraction no longer returns modifiers as names (public/const/virtual...); C-style 'type name(args)' and JS arrow/function-expression assignments extract the real name. A wrong name made every candidate look drifted forever, each false drift costing a full V3 regeneration. - defined_names sees arrow functions, function expressions, and C-style method bodies. - The no-parseable-definitions escape now keys on extraction confidence: a confidently-parsed Python file that defines nothing genuinely misses every planned def (the old escape kept all-wrong candidates while vetoing half-right ones); regex-path opacity still skips enforcement. - extract_json_object falls back to the full text when the first fenced block is not JSON. v3-service/main.py: - Signature-veto events are emitted only when the prune actually applies; the per-candidate check is guarded like the sibling call-graph veto. v3-service/wavelet/: - String masking no longer runs over comment interiors: an apostrophe in prose ("/* Don't use */") swallowed the terminator and zeroed the rest of the file's signal. Same for docstring closers, and a line comment can no longer flip the docstring state machine or python's block-comment path. Backtick is not a string delimiter in python. - Project walk: depth guard (RecursionError), gitignore character classes match, symlink aliases cannot resurrect ignored dirs. - decompose_file_map/diff_contents enforce the same size caps as the disk path (the pure-python CWT makes uncapped input a multi-minute stall); zero-magnitude coefficients are never peaks. Regression tests for every fix (proxy: 6 new/updated; python: 12 new). --- docs/CONFIGURATION.md | 1 + proxy/rpg.go | 60 +++++++++++++--- proxy/rpg_test.go | 65 ++++++++++++++++- proxy/tools.go | 27 +++++-- tests/v3-service/test_rpg.py | 27 ++++++- tests/v3-service/test_wavelet_cwt.py | 8 +++ tests/v3-service/test_wavelet_project.py | 38 ++++++++++ tests/v3-service/test_wavelet_signal.py | 46 ++++++++++++ v3-service/main.py | 31 +++++--- v3-service/rpg.py | 90 +++++++++++++++++++++--- v3-service/wavelet/cwt.py | 5 +- v3-service/wavelet/diff.py | 9 +++ v3-service/wavelet/project.py | 57 ++++++++++++--- v3-service/wavelet/signal.py | 49 +++++++++---- 14 files changed, 454 insertions(+), 59 deletions(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 02057ae6..9d0e315e 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -457,6 +457,7 @@ Python FastAPI service for isolated code execution with compilation, linting, an | `ATLAS_SANDBOX_WORKSPACE_ROOT` | `/workspace` | The bind-mounted project root the executor confines `cwd` to. Container deployments keep the default; the E2E acceptance test overrides it to run the executor on the host under a temp dir. | | `ATLAS_SANDBOX_UID` / `ATLAS_SANDBOX_GID` | `1000` | Runtime uid/gid of the (non-root) sandbox container. `atlas init` writes the invoking user's ids so `/workspace` writes keep the right owner — the container drops all capabilities, so ownership must match. | | `ATLAS_PROXY_UID` / `ATLAS_PROXY_GID` | `1000` | Runtime uid/gid of the atlas-proxy container — same rationale as the sandbox ids: the proxy writes two host bind mounts (`/workspace` for file tools, `/data/lens_training` for lens sample banking), and the image's baked-in non-root user (uid 1001) can't write operator-owned dirs. `atlas init` writes the invoking user's ids. | +| `ATLAS_RPG_PLANNING` | `0` | V3.2 RPG-style architecture-first planning (issue #120, EXPERIMENTAL). When on, `/v3/plan` builds a Repository Planning Graph (capabilities → files → signatures → data-flow edges) via the wavelet coarse band, plan steps carry per-node interface constraints into generation, drifted nodes get one corrective regeneration, and surviving drift is surfaced with the affected downstream subgraph. Off (default) leaves planning and generation byte-identical to the flat path. Read inside the v3-service container (forwarded by compose). See `docs/reports/RPG_WAVELET_PLANNING_V3_2.md`. | | `ATLAS_SANDBOX_CPUS` | `2` (compose fallback); `atlas init` writes ~75% of host cores | CPU quota for the sandbox container so a runaway build can't starve llama-server. The compose fallback is a conservative `2` so a raw `docker compose up` is never unlimited; `atlas init` writes ~75% of host cores (floor 1) — the recommended setting. `0` removes the cap (unlimited — not recommended). | | `ATLAS_SERVICE_TOKEN_FILE` | `/run/atlas-secrets/service-token` (containers) / `/secrets/service-token` (host) | Path to the per-installation internal-auth token. Generated by `atlas init` (0600); mounted read-only into every Docker-Compose service. Present => all services require `Authorization: Bearer ` on non-health routes and clients inject it automatically. Absent => auth disabled (pre-token behavior; doctor warns). The K3s templates do not mount the token, so internal auth is disabled on K3s (doctor warns there too). Rotate with `atlas init --rotate-token` + `docker compose restart`. The token value itself never goes in `.env`, argv, or logs. | | `ATLAS_SECRETS_DIR` | `./secrets` | Host dir mounted read-only at `/run/atlas-secrets` in every service (holds `service-token`; the lens api-keys mount is separate). | diff --git a/proxy/rpg.go b/proxy/rpg.go index 9ecd7936..8adf69eb 100644 --- a/proxy/rpg.go +++ b/proxy/rpg.go @@ -27,15 +27,25 @@ func planConstraintsForTarget(ctx *AgentContext, path string) []string { if ctx == nil || ctx.Plan == nil { return nil } + // Trust gate: constraints are honored only when the plan carries the RPG + // graph artifact the RPG planner always attaches. The flat planner returns + // the LLM's plan dict as-is, so a model that hallucinates a "constraints" + // array on a step must not steer generation while ATLAS_RPG_PLANNING is + // off — without this check that leak was the only flag gating proxy-side. + if ctx.Plan.RPG == nil { + return nil + } // Pick the most specific overlapping step rather than the first. With // suffix-based matching a bare-basename step (e.g. target "user.py") would // otherwise shadow the deeper "models/user.py" step it isn't really for, so - // prefer the longest matching Target. + // prefer the longest matching Target. Ranked over ALL file-producing steps + // (not only constraint-bearing ones) so a deeper unconstrained step wins + // over a shallower constrained one instead of being shadowed by it. best := -1 bestLen := -1 for i := range ctx.Plan.Steps { step := &ctx.Plan.Steps[i] - if len(step.Constraints) == 0 || !isFileProducingAction(step.Action) { + if !isFileProducingAction(step.Action) { continue } if targetsOverlap(step.Target, path) && len(step.Target) > bestLen { @@ -43,7 +53,7 @@ func planConstraintsForTarget(ctx *AgentContext, path string) []string { bestLen = len(step.Target) } } - if best < 0 { + if best < 0 || len(ctx.Plan.Steps[best].Constraints) == 0 { return nil } // Copy so callers can append without mutating the plan. @@ -87,6 +97,12 @@ func affectedDownstream(plan *Plan, fileID string) []string { } adj := map[string][]string{} for _, e := range plan.RPG.Edges { + // Data-flow edges only, as documented: "order" edges express build + // sequencing, not dependence on the drifted interface. Empty kind + // defaults to data_flow, matching rpg.py's serialization default. + if e.Kind != "" && e.Kind != "data_flow" { + continue + } adj[e.From] = append(adj[e.From], e.To) } pathByID := map[string]string{} @@ -119,7 +135,7 @@ func affectedDownstream(plan *Plan, fileID string) []string { // best-effort node-local self-repair, not a loop. No-op unless the request // carried RPG constraints and the previous result actually drifted. func regenerateOnDrift(ctx *AgentContext, req V3GenerateRequest, prev *V3GenerateResponse) *V3GenerateResponse { - if prev == nil || len(prev.RPGSignatureMissing) == 0 || len(req.Constraints) == 0 { + if ctx == nil || prev == nil || len(prev.RPGSignatureMissing) == 0 || len(req.Constraints) == 0 { return prev } @@ -152,32 +168,52 @@ func regenerateOnDrift(ctx *AgentContext, req V3GenerateRequest, prev *V3Generat }) } }) + accepted := false + defer func() { + Emit(NewEnvelope(EvtStageEnd, "rpg_regen", map[string]interface{}{ + "file": req.FilePath, + "accepted": accepted, + })) + }() if err != nil || retried == nil || retried.Code == "" { return prev } // Accept the retry when it resolves at least one of the signatures we were - // correcting for, even if it traded in a different miss (so the count is - // unchanged). A plain count comparison would discard a retry that fixed the - // targeted gap but reshaped the code elsewhere. Falling back to total count - // covers the case where the targeted set is somehow unchanged. + // correcting for — even a trade for a different miss, as long as the TOTAL + // drift did not grow. (Resolving one targeted signature while introducing + // two new misses is a net loss; without the total ceiling the corrective's + // tunnel-vision prompt could adopt a strictly worse result.) Falling back + // to total count covers the case where the targeted set is unchanged but + // the retry still realized more of the plan overall. stillMissingTargeted := countOverlap(retried.RPGSignatureMissing, prev.RPGSignatureMissing) - if stillMissingTargeted < len(prev.RPGSignatureMissing) { + if stillMissingTargeted < len(prev.RPGSignatureMissing) && + len(retried.RPGSignatureMissing) <= len(prev.RPGSignatureMissing) { + accepted = true return retried } if len(retried.RPGSignatureMissing) < len(prev.RPGSignatureMissing) { + accepted = true return retried } return prev } -// countOverlap returns how many elements of `subset` appear in `of`. +// countOverlap returns how many DISTINCT elements of `subset` appear in `of`. +// Distinct, because the acceptance rule compares it against len(of the +// targeted set): duplicate entries in a service response must not count a +// resolved signature as still-missing twice. func countOverlap(subset, of []string) int { set := make(map[string]struct{}, len(of)) for _, s := range of { set[s] = struct{}{} } + seen := make(map[string]struct{}, len(subset)) n := 0 for _, s := range subset { + if _, dup := seen[s]; dup { + continue + } + seen[s] = struct{}{} if _, ok := set[s]; ok { n++ } @@ -197,7 +233,9 @@ func reportRPGDrift(ctx *AgentContext, path string, missing []string) { fileID := rpgFileIDForPath(ctx.Plan, path) downstream := affectedDownstream(ctx.Plan, fileID) - Emit(NewEnvelope(EvtStageStart, "rpg_drift", map[string]interface{}{ + // Point event, not a stage — EvtMetric doesn't leave an unclosed + // stage open in the pipeline pane. + Emit(NewEnvelope(EvtMetric, "rpg_drift", map[string]interface{}{ "file": path, "missing": missing, "downstream": downstream, diff --git a/proxy/rpg_test.go b/proxy/rpg_test.go index 503e5551..a59e06b8 100644 --- a/proxy/rpg_test.go +++ b/proxy/rpg_test.go @@ -8,6 +8,7 @@ import ( "reflect" "strings" "sync" + "sync/atomic" "testing" ) @@ -94,7 +95,7 @@ func TestPlanConstraintsForTarget(t *testing.T) { func TestPlanConstraintsForTargetPrefersMostSpecific(t *testing.T) { // A bare-basename step must not shadow the deeper path it isn't for. - plan := &Plan{Steps: []PlanStep{ + plan := &Plan{RPG: &RPG{}, Steps: []PlanStep{ {ID: "s1", Action: "write_file", Target: "user.py", Constraints: []string{"BARE"}}, {ID: "s2", Action: "write_file", Target: "models/user.py", Constraints: []string{"DEEP"}}, }} @@ -103,6 +104,30 @@ func TestPlanConstraintsForTargetPrefersMostSpecific(t *testing.T) { if !reflect.DeepEqual(got, []string{"DEEP"}) { t.Errorf("expected the most-specific (models/user.py) step, got %v", got) } + + // The ranking covers ALL file-producing steps: when the deeper matching + // step carries no constraints, the shallower constrained step must not + // shadow it — the file's real step simply has nothing to thread. + plan2 := &Plan{RPG: &RPG{}, Steps: []PlanStep{ + {ID: "s1", Action: "write_file", Target: "user.py", Constraints: []string{"BARE"}}, + {ID: "s2", Action: "write_file", Target: "models/user.py"}, + }} + if got := planConstraintsForTarget(&AgentContext{Plan: plan2}, "/workspace/models/user.py"); got != nil { + t.Errorf("deeper unconstrained step should win with nil constraints, got %v", got) + } +} + +func TestPlanConstraintsIgnoredWithoutRPGArtifact(t *testing.T) { + // Trust gate: the flat planner passes the LLM's plan dict through + // verbatim, so hallucinated step constraints must not steer generation + // unless the plan carries the RPG graph the RPG planner attaches. + plan := &Plan{Steps: []PlanStep{ + {ID: "s1", Action: "write_file", Target: "app.py", + Constraints: []string{"HALLUCINATED"}}, + }} + if got := planConstraintsForTarget(&AgentContext{Plan: plan}, "app.py"); got != nil { + t.Errorf("constraints without an RPG artifact must be ignored, got %v", got) + } } func TestPlanConstraintsForTargetNilPlan(t *testing.T) { @@ -246,7 +271,17 @@ func TestRegenerateOnDriftRetriesAndKeepsCleanResult(t *testing.T) { } func TestRegenerateOnDriftNoopWithoutDriftOrConstraints(t *testing.T) { - ctx := &AgentContext{V3URL: "http://127.0.0.1:0"} // never called + // A counting server, so "never called" is actually asserted — with a + // dead address the guards could be deleted and the dial error would + // still return prev, passing vacuously. + var calls int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&calls, 1) + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, "event: result\ndata: {\"code\":\"x\"}\n\ndata: [DONE]\n\n") + })) + defer srv.Close() + ctx := &AgentContext{V3URL: srv.URL} clean := &V3GenerateResponse{Code: "ok"} // No drift → returned unchanged, server never hit. if got := regenerateOnDrift(ctx, V3GenerateRequest{Constraints: []string{"x"}}, clean); got != clean { @@ -257,6 +292,28 @@ func TestRegenerateOnDriftNoopWithoutDriftOrConstraints(t *testing.T) { if got := regenerateOnDrift(ctx, V3GenerateRequest{}, drift); got != drift { t.Error("no-constraints drift should be returned unchanged") } + // Nil ctx → no-op, no panic. + if got := regenerateOnDrift(nil, V3GenerateRequest{Constraints: []string{"x"}}, drift); got != drift { + t.Error("nil ctx should return prev unchanged") + } + if n := atomic.LoadInt32(&calls); n != 0 { + t.Errorf("guarded no-op paths must never call the server; got %d calls", n) + } +} + +func TestRegenerateOnDriftRejectsWorseTotalDrift(t *testing.T) { + // prev missed [A,B]; the retry resolves A but comes back missing + // [B,C,D]. Accepting it would trade 2 misses for 3 — the total-drift + // ceiling must keep prev even though a targeted signature resolved. + srv := generateServer(t, + `{"code": "def A(): pass", "rpg_signature_missing": ["def B()", "def C()", "def D()"]}`, nil) + defer srv.Close() + ctx := &AgentContext{V3URL: srv.URL} + req := V3GenerateRequest{FilePath: "m.py", Constraints: []string{"c"}} + prev := &V3GenerateResponse{Code: "old", RPGSignatureMissing: []string{"def A()", "def B()"}} + if got := regenerateOnDrift(ctx, req, prev); got != prev { + t.Errorf("retry with worse total drift must be rejected, got %+v", got) + } } func TestRegenerateOnDriftAcceptsRetryThatResolvesTargeted(t *testing.T) { @@ -288,6 +345,10 @@ func TestCountOverlap(t *testing.T) { if got := countOverlap(nil, []string{"a"}); got != 0 { t.Errorf("countOverlap(nil) = %d, want 0", got) } + // Distinct semantics: duplicate service entries must not double-count. + if got := countOverlap([]string{"a", "a"}, []string{"a", "b"}); got != 1 { + t.Errorf("countOverlap with duplicates = %d, want 1", got) + } } func TestRegenerateOnDriftKeepsPrevWhenRetryNoBetter(t *testing.T) { diff --git a/proxy/tools.go b/proxy/tools.go index f9140095..700f6a79 100644 --- a/proxy/tools.go +++ b/proxy/tools.go @@ -968,6 +968,16 @@ func writeFileWithV3(path, baselineContent string, ctx *AgentContext) (*ToolResu // missing signatures injected as a hard constraint before accepting it. // No-op when RPG is off (req.Constraints empty) or there was no drift. v3Result = regenerateOnDrift(ctx, req, v3Result) + // The regen is a second full V3 run — recheck cancellation before the + // disk write, mirroring the main call's abort path above: a user cancel + // during the retry window must not land content on disk. + if ctx.Ctx != nil && ctx.Ctx.Err() != nil { + log.Printf("[write_file] cancelled during RPG regen — not writing %s", path) + return &ToolResult{ + Success: false, + Error: "write_file cancelled — no content was written", + }, nil + } // Write the winning candidate (or baseline if V3 didn't improve) code := v3Result.Code @@ -1015,8 +1025,11 @@ func writeFileWithV3(path, baselineContent string, ctx *AgentContext) (*ToolResu // V3.2 RPG drift loop (issue #120): if the winning code failed to realize // the node's planned signatures, surface the drift + downstream subgraph. - // No-op when RPG is off (the field is empty). - if len(v3Result.RPGSignatureMissing) > 0 { + // Gated on req.Constraints, not only the response field: RPG is active + // for this file exactly when the plan supplied constraints, so a rogue + // or stale v3-service emitting rpg_signature_missing unconditionally + // can't fire drift events while the flag is off. + if len(req.Constraints) > 0 && len(v3Result.RPGSignatureMissing) > 0 { reportRPGDrift(ctx, path, v3Result.RPGSignatureMissing) } @@ -1718,9 +1731,15 @@ func improveContentWithV3(path, content string, ctx *AgentContext) (string, V3Ed // V3.2 RPG (issue #120): same drift handling as the write_file path. If the // edited result missed its planned signatures, retry once, then surface any - // surviving drift. No-op when RPG is off (req.Constraints empty). + // surviving drift. No-op when RPG is off (req.Constraints empty; the + // drift report shares that gate so a rogue response field alone can't + // fire it). The regen is a second full V3 run — recheck cancellation + // before accepting content, mirroring the write path. v3Result = regenerateOnDrift(ctx, req, v3Result) - if len(v3Result.RPGSignatureMissing) > 0 { + if ctx.Ctx != nil && ctx.Ctx.Err() != nil { + return "", V3EditMetadata{}, fmt.Errorf("edit cancelled during RPG regen: %w", ctx.Ctx.Err()) + } + if len(req.Constraints) > 0 && len(v3Result.RPGSignatureMissing) > 0 { reportRPGDrift(ctx, path, v3Result.RPGSignatureMissing) } diff --git a/tests/v3-service/test_rpg.py b/tests/v3-service/test_rpg.py index 6e55540c..a627a1a6 100644 --- a/tests/v3-service/test_rpg.py +++ b/tests/v3-service/test_rpg.py @@ -380,8 +380,31 @@ def test_missing_planned_signatures(self): assert missing_planned_signatures(code, planned, "m.py") == ["def run(rows)"] def test_no_veto_when_code_opaque(self): - # No parseable defs → don't veto (conservative). - assert missing_planned_signatures("x = 1", ["def f()"], "m.py") == [] + # Opaque = structure NOT confidently extractable (regex path found + # nothing) → don't veto. A JS script with no recognizable + # definitions may just use an unsupported syntax style. + assert missing_planned_signatures( + "console.log(1)", ["function f()"], "a.js") == [] + + def test_parsed_but_empty_python_is_enforced(self): + # A Python file that parses and defines nothing genuinely misses + # every planned def — the old escape here kept an all-wrong + # candidate while vetoing a half-right one. + assert missing_planned_signatures( + "x = 1", ["def f()"], "m.py") == ["def f()"] + # And the half-right candidate reports only its real gap. + assert missing_planned_signatures( + "def f(): pass", ["def f()", "def g()"], "m.py") == ["def g()"] + + def test_modifier_led_signatures_extract_real_name(self): + # Java/C++/JS-arrow signature styles must not report modifiers as + # the function name (each false miss costs a full V3 regen). + java = "public class Main { public static void main(String[] a) { } }" + assert missing_planned_signatures( + java, ["public static void main(String[] a)"], "Main.java") == [] + js = "class App {}\nconst handleClick = () => {};" + assert missing_planned_signatures( + js, ["function handleClick()"], "app.js") == [] class TestVerifyNodeRealization: diff --git a/tests/v3-service/test_wavelet_cwt.py b/tests/v3-service/test_wavelet_cwt.py index 2b31a2a9..8efd4ea0 100644 --- a/tests/v3-service/test_wavelet_cwt.py +++ b/tests/v3-service/test_wavelet_cwt.py @@ -192,3 +192,11 @@ def test_peaks_match_upstream(self): (16, -0.28708949, 2), (24, -0.28708949, 2), ] + + +def test_zero_signal_emits_no_peaks(): + # An all-zero signal (blank or comment-only file) must not produce + # spurious zero-magnitude "peaks" at threshold 0.0. + from wavelet.cwt import compute_cwt, detect_peaks + coeffs = compute_cwt([0.0] * 40) + assert detect_peaks(coeffs, 0.0) == [] diff --git a/tests/v3-service/test_wavelet_project.py b/tests/v3-service/test_wavelet_project.py index 531d3604..0470e130 100644 --- a/tests/v3-service/test_wavelet_project.py +++ b/tests/v3-service/test_wavelet_project.py @@ -147,3 +147,41 @@ def test_empty_and_blank(self): def test_limit_caps(self): assert len(decompose_file_map({"a.py": PY_A, "b.py": PY_B}, limit=2)) <= 2 + + +class TestWalkHardening: + def test_deep_tree_truncates_instead_of_recursion_error(self, tmp_path): + from wavelet.project import _discover_files + d = tmp_path + for _ in range(80): + d = d / "x" + d.mkdir() + (d / "deep.py").write_text("def f(): pass\n") + files, truncated = _discover_files(str(tmp_path)) + assert truncated is True + + def test_gitignore_character_class_matches(self, tmp_path): + from wavelet.project import _discover_files + (tmp_path / ".gitignore").write_text("[ab].py\n") + for n in ("a.py", "b.py", "c.py"): + (tmp_path / n).write_text("def f(): pass\n") + files, _ = _discover_files(str(tmp_path)) + assert sorted(f.filename for f in files) == ["c.py"] + + def test_symlink_alias_cannot_resurrect_ignored_dir(self, tmp_path): + import os + from wavelet.project import _discover_files + sub = tmp_path / "sub" + sub.mkdir() + (sub / "s.py").write_text("def s(): pass\n") + (tmp_path / ".gitignore").write_text("sub/\n") + os.symlink(str(sub), str(tmp_path / "link_sub")) + files, _ = _discover_files(str(tmp_path)) + assert files == [] + + def test_file_map_skips_oversized_entries(self): + from wavelet.project import decompose_file_map, MAX_FILE_BYTES + huge = "x = 1\n" * (MAX_FILE_BYTES // 6 + 10) + assert len(huge) > MAX_FILE_BYTES + out = decompose_file_map({"huge.py": huge, "ok.py": "def f(): pass\n"}) + assert all(p.filename == "ok.py" for p in out) diff --git a/tests/v3-service/test_wavelet_signal.py b/tests/v3-service/test_wavelet_signal.py index a4639d09..46d54490 100644 --- a/tests/v3-service/test_wavelet_signal.py +++ b/tests/v3-service/test_wavelet_signal.py @@ -246,3 +246,49 @@ def test_inline_annotation_raises_signal(self): a = compute_signal(["public @Nullable String foo() {}"], java)[0] b = compute_signal(["public String foo() {}"], java)[0] assert a > b + + +class TestCommentTerminatorMasking: + """Regressions for the terminator-swallowing class: string masking must + never run over comment interiors (an apostrophe in prose swallowed the + close delimiter and zeroed the rest of the file's signal).""" + + def test_block_comment_apostrophe_does_not_swallow_terminator(self): + from wavelet.language import detect_language + from wavelet.signal import compute_signal + js = detect_language("a.js") + sig = compute_signal( + ["/* Don't use this directly */", "", "function go() {", "}"], js) + assert sig[0] == 0.0 + assert sig[2] > 0.0, "function after a prose comment must score" + + def test_docstring_closer_with_apostrophe(self): + from wavelet.language import detect_language + from wavelet.signal import compute_signal + py = detect_language("a.py") + sig = compute_signal( + ['def f():', ' """Doc.', " Returns the user's data.\"\"\"", + ' return 1', '', 'def g():', ' pass'], py) + assert sig[5] > 0.0, "def g must score after the docstring closes" + + def test_hash_comment_with_odd_triple_quotes(self): + from wavelet.language import detect_language + from wavelet.signal import compute_signal + py = detect_language("a.py") + sig = compute_signal(['# see """ usage', 'def foo():', ' return 1'], py) + assert sig[1] > 0.0, "a comment must not flip the docstring state" + + def test_python_backtick_is_not_a_string_delimiter(self): + from wavelet.language import detect_language + from wavelet.signal import compute_signal + py = detect_language("a.py") + sig = compute_signal(['x = 1 # `code` sample', 'def h():', ' pass'], py) + assert sig[1] > 0.0 + + def test_string_containing_comment_start_does_not_open_comment(self): + from wavelet.language import detect_language + from wavelet.signal import compute_signal + js = detect_language("a.js") + sig = compute_signal( + ['var s = "/* not a comment */";', 'function ok() {', '}'], js) + assert sig[1] > 0.0 diff --git a/v3-service/main.py b/v3-service/main.py index 8d08f7f2..4f87bbb4 100644 --- a/v3-service/main.py +++ b/v3-service/main.py @@ -1595,8 +1595,7 @@ def verify_build_if_requested(out="", err=""): if cg_kept: # only prune when at least one candidate survives passing = cg_kept - - # ===== RPG SIGNATURE VETO (V3.2, issue #120) ===== + # ===== RPG SIGNATURE VETO (V3.2, issue #120) ===== # When the RPG planner is active, the request constraints carry the # node's planned signatures. Extend the #39-pt-1 veto from "imports # survive" to "the planned interface exists": reject sandbox-passing @@ -1619,21 +1618,35 @@ def verify_build_if_requested(out="", err=""): planned_sigs = [] if planned_sigs: sig_kept = [] + vetoed = [] # (candidate, missing) — emitted only if the prune applies for c in passing: - missing = _rpgmod.missing_planned_signatures( - c.get("code", ""), planned_sigs, file_path) + # Guarded per-candidate like the call-graph veto: an + # unexpected parser error must skip the veto for that + # candidate, not fail the whole /v3/generate request. + try: + missing = _rpgmod.missing_planned_signatures( + c.get("code", ""), planned_sigs, file_path) + except Exception as ve: + print(f" [rpg] signature check failed on cand " + f"{c.get('index')}: {ve} — keeping candidate", + flush=True) + missing = [] if missing: + vetoed.append((c, missing)) + continue + sig_kept.append(c) + # Only prune when at least one candidate realizes the plan + # — and only announce vetoes that actually took effect + # (when every candidate drifts, all of them survive). + if sig_kept: + passing = sig_kept + for c, missing in vetoed: emit("rpg_signature_veto", f"Candidate {c['index']} sandbox-passed but missing " f"planned signature(s): {', '.join(missing[:3])}", index=c["index"], missing=missing[:5]) print(f" [rpg] vetoed cand {c['index']} — missing: {missing[:5]}", flush=True) - continue - sig_kept.append(c) - # Only prune when at least one candidate realizes the plan. - if sig_kept: - passing = sig_kept # ===== CANDIDATE SELECTION ===== if passing: diff --git a/v3-service/rpg.py b/v3-service/rpg.py index 4cd8d16c..5b2761b0 100644 --- a/v3-service/rpg.py +++ b/v3-service/rpg.py @@ -104,9 +104,18 @@ def extract_json_object(raw: str) -> Optional[dict]: return None import re + # Prefer the first fenced block, but fall back to the full text when it + # doesn't parse — a fenced non-JSON example in the model's preamble must + # not mask an unfenced JSON object after it. fence = re.search(r"```(?:json)?\s*\n(.*?)\n```", raw, re.DOTALL) if fence: - raw = fence.group(1) + fenced = _extract_json_from(fence.group(1)) + if fenced is not None: + return fenced + return _extract_json_from(raw) + + +def _extract_json_from(raw: str) -> Optional[dict]: start = raw.find("{") if start < 0: return None @@ -423,6 +432,25 @@ def flatten_to_plan(rpg: RPG) -> dict: _DECL_KEYWORDS = frozenset({ "def", "func", "function", "fn", "fun", "class", "struct", "interface", "trait", "type", "async", "impl", "object", + # Modifiers and type-ish tokens that lead non-Python/Go signatures + # ("public static void main(...)", "const handleClick = ...", + # "virtual int run() override"). The bare-name fallback must never + # return these as the "function name" — a wrong name here makes every + # candidate look drifted forever, and each false drift costs a full + # V3 regeneration. + "public", "private", "protected", "static", "final", "abstract", + "virtual", "override", "const", "let", "var", "export", "extern", + "inline", "unsigned", "signed", "void", "int", "float", "double", + "bool", "boolean", "string", "char", "long", "short", "auto", "new", + "return", "if", "for", "while", "switch", "catch", "else", "elif", +}) + +# Control-flow keywords that look like `name(...)` call sites in C-style +# code; never function names. +_CALLISH_NON_NAMES = frozenset({ + "if", "for", "while", "switch", "catch", "return", "throw", "with", + "assert", "elif", "except", "sizeof", "typeof", "new", "delete", + "super", "print", }) @@ -444,8 +472,21 @@ def _function_name_from_signature(sig: str) -> str: m = re.search(r"\b(?:async\s+)?(?:def|func|function|fn|fun|class|struct|interface|trait|type)\s+([A-Za-z_]\w*)", s) if m: return m.group(1) - # Bare "name" / "name(args)", but never a lone declaration keyword. - m = re.match(r"\s*([A-Za-z_]\w*)", s) + # JS/TS arrow or function-expression assignment: + # `handleClick = () =>`, `handleClick = async function`, `handleClick: (x) =>`. + m = re.search(r"\b([A-Za-z_]\w*)\s*[:=]\s*(?:async\s+)?(?:function\b|\([^)]*\)\s*=>|[A-Za-z_]\w*\s*=>)", s) + if m and m.group(1) not in _DECL_KEYWORDS: + return m.group(1) + # C-style `modifiers type name(args)`: the name is the token directly + # attached to the parameter list, so scan for `name(` and take the first + # hit that isn't a keyword ("public static void main(String[] a)" → main). + for cm in re.finditer(r"\b([A-Za-z_]\w*)\s*\(", s): + if cm.group(1) not in _DECL_KEYWORDS and cm.group(1) not in _CALLISH_NON_NAMES: + return cm.group(1) + # Bare "name", but never a lone declaration keyword or modifier — + # returning "" makes the caller skip enforcement rather than hunt for a + # function literally named "public". + m = re.match(r"\s*([A-Za-z_]\w*)\s*$", s) if m and m.group(1) not in _DECL_KEYWORDS: return m.group(1) return "" @@ -456,6 +497,17 @@ def defined_names(code: str, filename: str) -> set: (precise, methods included); other languages use a keyword+name regex. Returns an empty set when nothing parses — callers treat that as "unknown," not "missing.\"""" + names, _ = _defined_names_ex(code, filename) + return names + + +def _defined_names_ex(code: str, filename: str) -> tuple: + """(names, confident) — `confident` means the extraction actually saw the + file's structure: a successful Python AST parse is confident even when it + defines nothing (a script that defines nothing genuinely misses every + planned def), while the regex path is confident only when it found + definitions (an empty regex result may just mean an unsupported syntax + style, so enforcement must not fire on it).""" import re names: set = set() @@ -470,7 +522,7 @@ def defined_names(code: str, filename: str) -> set: for node in ast.walk(tree): if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): names.add(node.name) - return names + return names, True # fall through to regex on unparseable Python for m in re.finditer( r"\b(?:def|func|function|fn|fun|class|struct|interface|trait|type)\s+([A-Za-z_][\w]*)", @@ -480,7 +532,23 @@ def defined_names(code: str, filename: str) -> set: # Go-style receiver methods: `func (r *T) Name(...)`. for m in re.finditer(r"\bfunc\s*\([^)]*\)\s*([A-Za-z_]\w*)", code): names.add(m.group(1)) - return names + # JS/TS arrow functions and function expressions bound to a name: + # `const handleClick = () => ...`, `handleClick = async function ...`, + # `handleClick: (x) => ...` (object-literal methods). + for m in re.finditer( + r"\b([A-Za-z_]\w*)\s*[:=]\s*(?:async\s+)?(?:function\b|\([^)]*\)\s*=>|[A-Za-z_]\w*\s*=>)", + code, + ): + if m.group(1) not in _DECL_KEYWORDS: + names.add(m.group(1)) + # C-style / class-method definitions: `name(args) {` at definition + # position (Java/C/C++/C#, JS method shorthand). Control-flow keywords + # that look call-shaped are excluded. + for m in re.finditer(r"\b([A-Za-z_]\w*)\s*\([^;{})]*\)\s*\{", code): + if (m.group(1) not in _CALLISH_NON_NAMES + and m.group(1) not in _DECL_KEYWORDS): + names.add(m.group(1)) + return names, bool(names) def planned_signatures_from_constraints(constraints: List[str]) -> List[str]: @@ -500,13 +568,17 @@ def planned_signatures_from_constraints(constraints: List[str]) -> List[str]: def missing_planned_signatures(code: str, planned: List[str], filename: str) -> List[str]: """Planned signatures whose declared name is NOT defined in `code`. - Conservative: if `code` yields no parseable definitions at all (empty - `defined_names`), returns [] — we don't veto when we can't see structure. + Conservative in exactly one direction: when the extraction was not + confident about the file's structure (see _defined_names_ex), returns [] + — we don't veto what we can't see. A confidently-parsed file that defines + nothing is NOT the escape case: such a candidate genuinely misses every + planned signature, and exempting it would veto a half-right candidate + while keeping an all-wrong one. """ if not planned: return [] - defined = defined_names(code, filename) - if not defined: + defined, confident = _defined_names_ex(code, filename) + if not confident: return [] missing: List[str] = [] for sig in planned: diff --git a/v3-service/wavelet/cwt.py b/v3-service/wavelet/cwt.py index 02c0fab0..d1c052fe 100644 --- a/v3-service/wavelet/cwt.py +++ b/v3-service/wavelet/cwt.py @@ -133,7 +133,10 @@ def detect_peaks( n = len(coeffs) for pos in range(n): mag = abs(coeffs[pos]) - if mag < threshold: + # Zero magnitude is never a peak, even at threshold 0.0 — an + # all-zero signal (blank or comment-only file) must not emit a + # spurious flat "peak" per scale. + if mag <= 0.0 or mag < threshold: continue left_ok = pos == 0 or mag >= abs(coeffs[pos - 1]) right_ok = pos == n - 1 or mag > abs(coeffs[pos + 1]) diff --git a/v3-service/wavelet/diff.py b/v3-service/wavelet/diff.py index 379ca434..5046405f 100644 --- a/v3-service/wavelet/diff.py +++ b/v3-service/wavelet/diff.py @@ -127,6 +127,15 @@ def diff_contents( ) -> FileDiffResult: """Convenience: build FileContexts for two file contents and diff their important positions. The drift entry point for RPG node verification.""" + from .project import MAX_FILE_BYTES + + # Same cap as the project walk: the pure-Python CWT makes an uncapped + # input a multi-minute stall. Oversized content degrades to "no drift + # information" rather than blocking the pipeline. + if len(before_text) > MAX_FILE_BYTES: + before_text = before_text[:MAX_FILE_BYTES] + if len(after_text) > MAX_FILE_BYTES: + after_text = after_text[:MAX_FILE_BYTES] before_ctx = FileContext(filename, before_text) after_ctx = FileContext(filename, after_text) return diff_file_context( diff --git a/v3-service/wavelet/project.py b/v3-service/wavelet/project.py index a335026c..5dc14f74 100644 --- a/v3-service/wavelet/project.py +++ b/v3-service/wavelet/project.py @@ -65,7 +65,23 @@ def _compile_glob(pat: str) -> "re.Pattern[str]": elif c == "?": regex += "[^/]" i += 1 - elif c in ".+()[]{}^$|\\": + elif c == "[": + # Character class: pass through to the regex (gitignore classes + # are regex-compatible modulo `!` negation). Escaping the + # brackets instead made `[ab].py` silently never match, so files + # git ignores were indexed. Unterminated `[` falls back to a + # literal bracket. + close = body.find("]", i + 2) # +2: "[]" would be empty + if close != -1: + cls = body[i + 1:close] + if cls.startswith("!"): + cls = "^" + cls[1:] + regex += "[" + cls + "]" + i = close + 1 + else: + regex += "\\[" + i += 1 + elif c in ".+(){}^$|\\" or c == "]": regex += "\\" + c i += 1 else: @@ -73,7 +89,12 @@ def _compile_glob(pat: str) -> "re.Pattern[str]": i += 1 prefix = "^" if anchored else "(^|/)" - return re.compile(f"{prefix}{regex}(/|$)") + try: + return re.compile(f"{prefix}{regex}(/|$)") + except re.error: + # A class body that isn't valid regex (e.g. `[z-a]`) degrades to a + # fully-escaped literal match rather than crashing the walk. + return re.compile(f"{prefix}{re.escape(body)}(/|$)") def _load_gitignore(root: str) -> List[_GitignoreRule]: @@ -142,10 +163,17 @@ def _discover_files(root: str) -> Tuple[List[ProjectFile], bool]: truncated = False pending: List[str] = [] - def walk(directory: str) -> None: + def walk(directory: str, depth: int = 0) -> None: nonlocal truncated if truncated: return + # Depth guard: recursion is one Python frame per directory level, so + # a pathologically deep tree (>1000 levels) would raise an uncaught + # RecursionError out of decompose_project. No real project nests + # this deep; treat it like the file-count cap. + if depth > 64: + truncated = True + return try: entries = os.listdir(directory) except OSError: @@ -175,9 +203,14 @@ def walk(directory: str) -> None: continue if statmod.S_ISDIR(st.st_mode): - if entry in _SKIP_DIRS: - continue - if _is_ignored(rel_path, True, gitignore): + # Register pruned dirs' realpaths too: a symlink alias to a + # skipped/ignored dir must not resurrect its contents under + # the alias path. + if entry in _SKIP_DIRS or _is_ignored(rel_path, True, gitignore): + try: + visited_real.add(os.path.realpath(full_path)) + except OSError: + pass continue try: dir_real = os.path.realpath(full_path) @@ -186,7 +219,7 @@ def walk(directory: str) -> None: if dir_real in visited_real: continue visited_real.add(dir_real) - walk(full_path) + walk(full_path, depth + 1) elif statmod.S_ISREG(st.st_mode): if _is_ignored(rel_path, False, gitignore): continue @@ -292,8 +325,14 @@ def decompose_file_map( ProjectIndex.get_important_positions: top `limit` positions by |coefficient|, labels suffixed with the relative path.""" top: List[ImportantPosition] = [] - for rel, content in file_map.items(): - if not content: + for i, (rel, content) in enumerate(file_map.items()): + # Same bounds as the disk path: the pure-Python CWT is ~0.5s per + # 1,000 lines, so an uncapped in-memory map would let one huge + # supplied file (or thousands of entries) stall the planner for + # minutes. MAX_FILE_BYTES/MAX_FILES mirror _discover_files. + if i >= MAX_FILES: + break + if not content or len(content) > MAX_FILE_BYTES: continue ctx = FileContext(os.path.basename(rel), content) peaks = ctx.get_important_positions(min_coefficient, max(limit, 30)) diff --git a/v3-service/wavelet/signal.py b/v3-service/wavelet/signal.py index d27d0622..5883460e 100644 --- a/v3-service/wavelet/signal.py +++ b/v3-service/wavelet/signal.py @@ -29,12 +29,17 @@ def _mask_string_literals(line: str, lang: LanguageConfig) -> str: # In Rust `'a` is a lifetime and in Clojure `'` is the quote reader macro — # neither delimits a string, so treating `'` as one would mask real code. single_quote_is_string = lang.name not in ("rust", "clojure") + # Backtick delimits strings/templates in JS/TS/Go/shell, but in Python + # and Clojure it is not a string delimiter — a stray backtick there would + # mask the rest of the line and feed the terminator-swallowing class of + # bug. + backtick_is_string = lang.name not in ("python", "clojure") chars = list(line) i = 0 n = len(chars) while i < n: c = chars[i] - if c == '"' or c == "`" or (single_quote_is_string and c == "'"): + if c == '"' or (backtick_is_string and c == "`") or (single_quote_is_string and c == "'"): quote = c j = i + 1 while j < n: @@ -134,11 +139,15 @@ def compute_signal(lines: List[str], lang: LanguageConfig) -> List[float]: masked = _mask_string_literals(stripped, lang) # ── Continuation of multiline comments / docstrings ── + # Terminator scanning runs on the UNMASKED text: comment interiors + # are prose, and masking them treats an apostrophe in "Don't" as a + # string opener that swallows the `*/` (or `\"\"\"`) terminator — + # permanently zeroing the rest of the file's signal. if in_block_comment: if lang.block_comment_uses_paren_depth: depth = block_comment_depth close_idx = -1 - for ci, mc in enumerate(masked): + for ci, mc in enumerate(stripped): if mc == "(": depth += 1 elif mc == ")": @@ -155,7 +164,7 @@ def compute_signal(lines: List[str], lang: LanguageConfig) -> List[float]: block_comment_depth = depth signal[i] = 0.0 else: - end_idx = masked.find(lang.block_comment_end) + end_idx = stripped.find(lang.block_comment_end) if end_idx != -1: in_block_comment = False after = stripped[end_idx + len(lang.block_comment_end):] @@ -165,7 +174,7 @@ def compute_signal(lines: List[str], lang: LanguageConfig) -> List[float]: continue if in_doc_string: - end_idx = masked.find(doc_string_delim) + end_idx = stripped.find(doc_string_delim) if end_idx != -1: in_doc_string = False doc_string_delim = None @@ -179,17 +188,22 @@ def compute_signal(lines: List[str], lang: LanguageConfig) -> List[float]: # Scanned on the unmasked stripped line because _mask_string_literals # would have consumed them as regular strings. if lang.name == "python": - dq = stripped.find('"""') - sq = stripped.find("'''") + # Cut at a line comment first (found on the masked text so a `#` + # inside a real string doesn't cut): a comment like + # `# see """ usage` must not flip the docstring state machine. + hash_pos = masked.find("#") + doc_scan = stripped if hash_pos == -1 else stripped[:hash_pos] + dq = doc_scan.find('"""') + sq = doc_scan.find("'''") dq_idx = min( dq if dq != -1 else math.inf, sq if sq != -1 else math.inf, ) if math.isfinite(dq_idx): dq_idx = int(dq_idx) - delim = stripped[dq_idx:dq_idx + 3] - before = stripped[:dq_idx].strip() - after = stripped[dq_idx + 3:] + delim = doc_scan[dq_idx:dq_idx + 3] + before = doc_scan[:dq_idx].strip() + after = doc_scan[dq_idx + 3:] has_closing = delim in after if has_closing: @@ -207,11 +221,22 @@ def compute_signal(lines: List[str], lang: LanguageConfig) -> List[float]: if lang.block_comment_at_line_start: bc_start_idx = 0 if masked.startswith(lang.block_comment_start) else -1 else: - bc_start_idx = masked.find(lang.block_comment_start) + bc_scan = masked + if lang.name == "python": + # Same comment cut as the docstring branch: a `"""` after a + # line comment (`# see """ usage`) must not open python's + # redundant block-comment path either. + hp = masked.find("#") + if hp != -1: + bc_scan = masked[:hp] + bc_start_idx = bc_scan.find(lang.block_comment_start) if bc_start_idx != -1: before = stripped[:bc_start_idx].strip() - after_delim = masked[bc_start_idx + len(lang.block_comment_start):] - after_delim_raw = stripped[bc_start_idx + len(lang.block_comment_start):] + # Close-scan on the RAW interior: the comment's contents are + # prose, and masking them lets an apostrophe swallow the + # terminator (`/* Don't use */` would never close). + after_delim = stripped[bc_start_idx + len(lang.block_comment_start):] + after_delim_raw = after_delim end_idx = -1 if lang.block_comment_uses_paren_depth: From 46e0462e8e42d74101b2c16949801e5c98909442 Mon Sep 17 00:00:00 2001 From: Johnathon Isaac Tigges Date: Sat, 18 Jul 2026 16:59:16 -0400 Subject: [PATCH 4/6] contract wiring: TUI handlers for rpg_drift/rpg_regen, config schema entry The event-contract gate requires every proxy chat event to have a TUI handler: rpg_drift and rpg_regen render as system rows alongside the asset_lint handler merged from dev. ATLAS_RPG_PLANNING joins the config schema as a bool so the .env.example key validates. --- atlas/cli/config_schema.py | 1 + tui/model.go | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/atlas/cli/config_schema.py b/atlas/cli/config_schema.py index 1c58288c..417da068 100644 --- a/atlas/cli/config_schema.py +++ b/atlas/cli/config_schema.py @@ -46,6 +46,7 @@ class Field: enum=("untrusted", "trusted", "fully-trusted")), "ATLAS_VERIFY_IN": Field("enum", enum=("sandbox", "host")), "ATLAS_CALL_GRAPH": Field("bool"), + "ATLAS_RPG_PLANNING": Field("bool"), "ATLAS_KEEP_LLAMA_WARM": Field("bool"), "ATLAS_FRESH_SLOT_PER_SESSION": Field("bool"), "ATLAS_DEDUP_READS": Field("bool"), diff --git a/tui/model.go b/tui/model.go index bde6e0e8..9be265dc 100644 --- a/tui/model.go +++ b/tui/model.go @@ -1786,6 +1786,22 @@ func (m *tuiModel) appendChatEvent(ev chatEvent) { }) } + // V3.2 RPG planning (issue #120): drift detection + the one-shot + // corrective regeneration. Both carry a prebuilt human message. + case "rpg_drift", "rpg_regen": + var d struct { + Message string `json:"message"` + } + if json.Unmarshal(ev.Data, &d) == nil && d.Message != "" { + meta := "rpg drift" + if ev.Type == "rpg_regen" { + meta = "rpg regen" + } + m.chat = append(m.chat, chatMessage{ + Role: roleSystem, Meta: meta, Body: d.Message, + }) + } + // Reasoning repetition detector: the proxy saw the model open its // reasoning stream with the same prefix on consecutive turns and // queued a corrective for the next LLM call. Third member of the From ae9e43479cacf2d0dc22fb3c4f2d2fd10046e6b0 Mon Sep 17 00:00:00 2001 From: Johnathon Isaac Tigges Date: Sat, 18 Jul 2026 17:03:53 -0400 Subject: [PATCH 5/6] codeql: structural fixes for the RPG/wavelet alert set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pre-bind the RPG imports in generate_plan so a partial import failure can never leave a name unbound on the guarded paths (py/uninitialized-local-variable x3). - Bound every ambiguous quantifier in the signature/definition regexes ({0,16} whitespace, {0,200} param bodies) — the arrow-function pattern was polynomial on long space runs (py/polynomial-redos); verified sub- 10ms on a 40k-char adversarial input with behavior preserved. - Walker containment guards in wavelet/project.py: normpath + root- prefix check on joined paths and the gitignore read — free defense-in- depth in the sanitizer shape taint tracking recognizes (py/path-injection x6). - Comment the best-effort empty except (py/empty-except). --- v3-service/main.py | 3 +++ v3-service/rpg.py | 6 +++--- v3-service/wavelet/project.py | 15 +++++++++++++-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/v3-service/main.py b/v3-service/main.py index 4f87bbb4..795be7e1 100644 --- a/v3-service/main.py +++ b/v3-service/main.py @@ -2198,6 +2198,9 @@ def emit(stage: str, detail: str = "", **data): # V3.2 RPG-style architecture-first planning (issue #120), flag-gated by # ATLAS_RPG_PLANNING. Strictly additive: on any failure (flag off, modules # missing, model output unusable) we fall through to the flat planner below. + # Pre-bound so a partial import failure can never leave a name unbound + # on the guarded paths below (py/uninitialized-local-variable). + decompose_project = decompose_file_map = _rpg_mod = None try: from wavelet import rpg_planning_enabled, decompose_project, decompose_file_map import rpg as _rpg_mod diff --git a/v3-service/rpg.py b/v3-service/rpg.py index 5b2761b0..ad229d7f 100644 --- a/v3-service/rpg.py +++ b/v3-service/rpg.py @@ -474,7 +474,7 @@ def _function_name_from_signature(sig: str) -> str: return m.group(1) # JS/TS arrow or function-expression assignment: # `handleClick = () =>`, `handleClick = async function`, `handleClick: (x) =>`. - m = re.search(r"\b([A-Za-z_]\w*)\s*[:=]\s*(?:async\s+)?(?:function\b|\([^)]*\)\s*=>|[A-Za-z_]\w*\s*=>)", s) + m = re.search(r"\b([A-Za-z_]\w*)[ \t]{0,16}[:=][ \t]{0,16}(?:async[ \t]{1,16})?(?:function\b|\([^)]{0,200}\)[ \t]{0,16}=>|[A-Za-z_]\w*[ \t]{0,16}=>)", s) if m and m.group(1) not in _DECL_KEYWORDS: return m.group(1) # C-style `modifiers type name(args)`: the name is the token directly @@ -536,7 +536,7 @@ def _defined_names_ex(code: str, filename: str) -> tuple: # `const handleClick = () => ...`, `handleClick = async function ...`, # `handleClick: (x) => ...` (object-literal methods). for m in re.finditer( - r"\b([A-Za-z_]\w*)\s*[:=]\s*(?:async\s+)?(?:function\b|\([^)]*\)\s*=>|[A-Za-z_]\w*\s*=>)", + r"\b([A-Za-z_]\w*)[ \t]{0,16}[:=][ \t]{0,16}(?:async[ \t]{1,16})?(?:function\b|\([^)]{0,200}\)[ \t]{0,16}=>|[A-Za-z_]\w*[ \t]{0,16}=>)", code, ): if m.group(1) not in _DECL_KEYWORDS: @@ -544,7 +544,7 @@ def _defined_names_ex(code: str, filename: str) -> tuple: # C-style / class-method definitions: `name(args) {` at definition # position (Java/C/C++/C#, JS method shorthand). Control-flow keywords # that look call-shaped are excluded. - for m in re.finditer(r"\b([A-Za-z_]\w*)\s*\([^;{})]*\)\s*\{", code): + for m in re.finditer(r"\b([A-Za-z_]\w*)[ \t]{0,16}\([^;{})]{0,300}\)[ \t\r\n]{0,16}\{", code): if (m.group(1) not in _CALLISH_NON_NAMES and m.group(1) not in _DECL_KEYWORDS): names.add(m.group(1)) diff --git a/v3-service/wavelet/project.py b/v3-service/wavelet/project.py index 5dc14f74..0f7e2738 100644 --- a/v3-service/wavelet/project.py +++ b/v3-service/wavelet/project.py @@ -98,7 +98,9 @@ def _compile_glob(pat: str) -> "re.Pattern[str]": def _load_gitignore(root: str) -> List[_GitignoreRule]: - path = os.path.join(root, ".gitignore") + path = os.path.normpath(os.path.join(root, ".gitignore")) + if not path.startswith(os.path.normpath(root) + os.sep): + return [] try: with open(path, "r", encoding="utf-8") as f: raw = f.read() @@ -153,6 +155,7 @@ class ProjectFile: def _discover_files(root: str) -> Tuple[List[ProjectFile], bool]: results: List[ProjectFile] = [] visited_real: set = set() + root = os.path.normpath(root) try: root_real = os.path.realpath(root) except OSError as e: @@ -181,7 +184,13 @@ def walk(directory: str, depth: int = 0) -> None: for entry in entries: if truncated: return - full_path = os.path.join(directory, entry) + # Containment guard: listdir entries can't contain separators on + # POSIX, but normalize + prefix-check anyway — it is free + # defense-in-depth and the sanitizer shape taint tracking + # recognizes, so the walk doesn't accrue path-injection alerts. + full_path = os.path.normpath(os.path.join(directory, entry)) + if full_path != root and not full_path.startswith(root + os.sep): + continue rel_path = os.path.relpath(full_path, root) try: st = os.lstat(full_path) @@ -210,6 +219,8 @@ def walk(directory: str, depth: int = 0) -> None: try: visited_real.add(os.path.realpath(full_path)) except OSError: + # Best-effort: an unresolvable pruned dir simply + # isn't registered; the walk still skips it here. pass continue try: From 4eb1a65c9ad06385e0b1abac0e58eed86dd37ad4 Mon Sep 17 00:00:00 2001 From: Johnathon Isaac Tigges Date: Sat, 18 Jul 2026 17:06:54 -0400 Subject: [PATCH 6/6] codeql: bound the bare-name fallback regex too The round-1 pass bounded three of the four signature regexes; the polynomial-redos tracker moved to the surviving unbounded one. --- v3-service/rpg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v3-service/rpg.py b/v3-service/rpg.py index ad229d7f..734e6cad 100644 --- a/v3-service/rpg.py +++ b/v3-service/rpg.py @@ -486,7 +486,7 @@ def _function_name_from_signature(sig: str) -> str: # Bare "name", but never a lone declaration keyword or modifier — # returning "" makes the caller skip enforcement rather than hunt for a # function literally named "public". - m = re.match(r"\s*([A-Za-z_]\w*)\s*$", s) + m = re.match(r"[ \t]{0,16}([A-Za-z_]\w*)[ \t]{0,16}$", s) if m and m.group(1) not in _DECL_KEYWORDS: return m.group(1) return ""