diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e52a25..7d0fce1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,15 @@ jobs: with: key: libkrun # distinct from the no-default-features `test` job's artifacts - name: Install libkrun - run: brew tap libkrun/krun && brew install libkrun libkrunfw + # Homebrew now gates third-party taps behind HOMEBREW_REQUIRE_TAP_TRUST + # (set on the runner image) — `brew install` refuses an untapped-trust + # formula ("Refusing to load formula … from untrusted tap"). `brew trust` + # the tap between tap+install; non-interactive + forward-compatible for + # when tap-trust becomes the Homebrew default. + run: | + brew tap libkrun/krun + brew trust libkrun/krun + brew install libkrun libkrunfw - name: Clippy (libkrun) run: cargo clippy --all-targets --features libkrun -- -D warnings - name: Test (libkrun) diff --git a/.gitignore b/.gitignore index 91a2ba0..ee22d87 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,12 @@ __pycache__/ *.pyc .venv/ scripts/eval/toolz-tasks/ + +# Run logs + ghost scratch outputs — noisy, regenerable, not evidence (all just +# {task,cond,trial,score,cost} scalars + progress lines; no transcripts/memory). +# The curated σ̂ result jsonls under scripts/eval/segmentation/results/ ARE tracked +# (the evidence archive — see its README); only the .log transcripts there are ignored. +scripts/eval/segmentation/results/*.log +scripts/ghost/*.jsonl +scripts/ghost/*.json +scripts/ghost/*.log diff --git a/scripts/bench/README.md b/scripts/bench/README.md new file mode 100644 index 0000000..c8d1930 --- /dev/null +++ b/scripts/bench/README.md @@ -0,0 +1,45 @@ +# Startup benchmark + +`startup.py` runs pillbox startup cases, reads host-emitted +`session.started` lifecycle events, and summarizes `startup_ms` plus per-stage +timings. + +## Examples + +```sh +# One Docker PTY case, five measured runs after one warmup. +scripts/bench/startup.py --case docker-claude --warmup 1 --repeat 5 + +# Server-mode opencode on Docker and libkrun, with a fixed model. +# Defaults to the same opencode image used by the smoke/eval scripts: +# PILLBOX_RUNNER_IMAGE=pillbox-runner:dev. +scripts/bench/startup.py \ + --case docker-opencode \ + --case libkrun-opencode \ + --model zai-coding-plan/glm-4.5-air \ + --warmup 1 \ + --repeat 5 + +# Show the commands without starting agents. +scripts/bench/startup.py --all-docker --dry-run + +# Emit raw measured runs plus aggregate summaries. +scripts/bench/startup.py --all-docker --json +``` + +## Notes + +- Cases require the corresponding agent auth and backend support. +- opencode cases default to `pillbox-runner:dev`; `libkrun-codex-serve` defaults + to `pillbox-runner:dev`, matching `scripts/smoke/`. Override with + `--runner-image IMAGE` or `PILLBOX_RUNNER_IMAGE`. +- Every run uses `--detach --json --ttl` and is removed with + `pillbox session rm` after its event is captured. +- By default, the benchmark creates one temporary empty workspace and reuses it + for every run. Use `--workspace PATH` to benchmark a real tree. +- Commands time out after 300 seconds by default. Override with + `--timeout SECONDS` for slow cold-start environments. +- Use `--json` for raw run records and aggregate summaries, or `--csv` for one + row per measured startup stage. +- `PILLBOX` can point at a specific binary; otherwise the script uses + `target/debug/pillbox` when present, then falls back to `pillbox` on `PATH`. diff --git a/scripts/bench/startup.py b/scripts/bench/startup.py new file mode 100755 index 0000000..030c88a --- /dev/null +++ b/scripts/bench/startup.py @@ -0,0 +1,594 @@ +#!/usr/bin/env python3 +"""Benchmark pillbox sandbox startup timings. + +The script consumes the lifecycle fields emitted by host-side +`session.started` events: + + - startup_ms + - startup_stages: [{name, duration_ms}, ...] + +It runs configured cases, captures each run's session id from `pillbox run +--json`, reads `pillbox session events --json`, and summarizes totals plus +per-stage p50/p95. The harness deliberately stays outside pillbox proper so the +measurement surface can be used before and after each optimization PR. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import os +import shutil +import subprocess +import sys +import tempfile +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional, Union + + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_PILLBOX = ROOT / "target" / "debug" / "pillbox" + + +@dataclass(frozen=True) +class Case: + name: str + backend: str + agent: str + description: str + needs_model: bool = False + runner_image: Optional[str] = None + + +CASES: dict[str, Case] = { + "docker-claude": Case( + "docker-claude", + "docker", + "claude", + "Docker backend, Claude PTY session", + ), + "docker-codex": Case( + "docker-codex", + "docker", + "codex", + "Docker backend, Codex PTY session", + ), + "docker-opencode": Case( + "docker-opencode", + "docker", + "opencode", + "Docker backend, opencode server session", + needs_model=True, + runner_image="pillbox-runner:dev", + ), + "libkrun-claude": Case( + "libkrun-claude", + "libkrun", + "claude", + "libkrun backend, Claude PTY session", + ), + "libkrun-codex": Case( + "libkrun-codex", + "libkrun", + "codex", + "libkrun backend, Codex PTY session", + ), + "libkrun-opencode": Case( + "libkrun-opencode", + "libkrun", + "opencode", + "libkrun backend, opencode server session", + needs_model=True, + runner_image="pillbox-runner:dev", + ), + "libkrun-codex-serve": Case( + "libkrun-codex-serve", + "libkrun", + "codex-serve", + "libkrun backend, codex app-server session", + needs_model=True, + runner_image="pillbox-runner:dev", + ), +} + + +def main() -> int: + args = parse_args() + if args.list_cases: + list_cases() + return 0 + + case_names = selected_cases(args) + if not case_names: + print( + "startup-bench: select at least one --case, --all-docker, --all-libkrun, or --all", + file=sys.stderr, + ) + return 2 + + pillbox = resolve_pillbox(args.pillbox) + workspace_guard = None + workspace = Path(args.workspace).resolve() if args.workspace else None + if workspace is None: + workspace_guard = tempfile.TemporaryDirectory(prefix="pillbox-startup-bench-") + workspace = Path(workspace_guard.name) + (workspace / "README.md").write_text("startup benchmark workspace\n", encoding="utf-8") + + base_cmd = [str(pillbox)] + if args.pillbox_name: + base_cmd.extend(["--pillbox", args.pillbox_name]) + + runs: list[dict[str, Any]] = [] + failures: list[dict[str, str]] = [] + try: + for name in case_names: + case = CASES[name] + total = args.warmup + args.repeat + for index in range(total): + measured = index >= args.warmup + label = f"startup-bench-{case.name}-{uuid.uuid4().hex[:8]}" + run = run_case( + args=args, + case=case, + base_cmd=base_cmd, + workspace=workspace, + label=label, + measured=measured, + ) + if run.get("ok"): + if measured: + runs.append(run) + else: + failures.append( + { + "case": case.name, + "label": label, + "reason": str(run.get("reason", "unknown failure")), + } + ) + finally: + if workspace_guard is not None: + workspace_guard.cleanup() + + if args.dry_run: + return 0 + if args.output_json: + print( + json.dumps( + {"runs": runs, "summary": summarize(runs), "failures": failures}, + indent=2, + sort_keys=True, + ) + ) + elif args.output_csv: + print_csv(runs) + else: + print_human(runs, failures) + return 1 if failures else 0 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run pillbox startup cases and summarize host session.started startup timings.", + ) + parser.add_argument( + "--case", + action="append", + choices=sorted(CASES), + default=[], + help="Case to run. Repeatable.", + ) + parser.add_argument("--all-docker", action="store_true", help="Run every Docker case.") + parser.add_argument("--all-libkrun", action="store_true", help="Run every libkrun case.") + parser.add_argument("--all", action="store_true", help="Run every known case.") + parser.add_argument("--list-cases", action="store_true", help="List cases and exit.") + parser.add_argument( + "--repeat", + type=positive_int, + default=5, + help="Measured runs per case. Default: 5.", + ) + parser.add_argument( + "--warmup", + type=nonnegative_int, + default=1, + help="Unmeasured warmup runs per case. Default: 1.", + ) + parser.add_argument("--workspace", help="Workspace to mount. Default: temporary empty workspace.") + parser.add_argument( + "--workspace-name", + default="startup-bench", + help="Guest /workspace mount name. Default: startup-bench.", + ) + parser.add_argument("--ttl", default="30m", help="Detached session TTL. Default: 30m.") + parser.add_argument("--model", default=os.environ.get("MODEL"), help="Model for server-mode agents.") + parser.add_argument( + "--runner-image", + default=os.environ.get("PILLBOX_RUNNER_IMAGE"), + help="Override PILLBOX_RUNNER_IMAGE for every case.", + ) + parser.add_argument( + "--pillbox", + default=os.environ.get("PILLBOX"), + help="Pillbox binary. Default: target/debug/pillbox, then PATH.", + ) + parser.add_argument("--pillbox-name", help="Pass --pillbox NAME to the CLI.") + parser.add_argument( + "--cwd", + default=os.getcwd(), + help="Directory where pillbox commands run. Default: current directory.", + ) + parser.add_argument( + "--timeout", + type=positive_int, + default=300, + help="Per pillbox command timeout in seconds. Default: 300.", + ) + parser.add_argument("--dry-run", action="store_true", help="Print commands without running them.") + parser.add_argument("--json", dest="output_json", action="store_true", help="Print JSON results.") + parser.add_argument("--csv", dest="output_csv", action="store_true", help="Print CSV rows for measured runs.") + return parser.parse_args() + + +def positive_int(raw: str) -> int: + value = int(raw) + if value < 1: + raise argparse.ArgumentTypeError("must be >= 1") + return value + + +def nonnegative_int(raw: str) -> int: + value = int(raw) + if value < 0: + raise argparse.ArgumentTypeError("must be >= 0") + return value + + +def resolve_pillbox(raw: Optional[str]) -> Union[Path, str]: + if raw: + return Path(raw).resolve() if "/" in raw else raw + if DEFAULT_PILLBOX.exists(): + return DEFAULT_PILLBOX + found = shutil.which("pillbox") + return found if found else "pillbox" + + +def selected_cases(args: argparse.Namespace) -> list[str]: + names: list[str] = [] + if args.all: + names.extend(CASES) + if args.all_docker: + names.extend(name for name, case in CASES.items() if case.backend == "docker") + if args.all_libkrun: + names.extend(name for name, case in CASES.items() if case.backend == "libkrun") + names.extend(args.case) + out: list[str] = [] + seen: set[str] = set() + for name in names: + if name not in seen: + seen.add(name) + out.append(name) + return out + + +def list_cases() -> None: + width = max(len(name) for name in CASES) + for name, case in CASES.items(): + image = f" image={case.runner_image}" if case.runner_image else "" + print(f"{name:<{width}} backend={case.backend:<7} agent={case.agent:<11}{image} {case.description}") + + +def run_case( + *, + args: argparse.Namespace, + case: Case, + base_cmd: list[str], + workspace: Path, + label: str, + measured: bool, +) -> dict[str, Any]: + cmd = build_run_cmd(args, case, base_cmd, workspace, label) + env = os.environ.copy() + if case.backend == "libkrun": + env["PILLBOX_BACKEND"] = "libkrun" + else: + env.pop("PILLBOX_BACKEND", None) + runner_image = args.runner_image or case.runner_image + if runner_image: + env["PILLBOX_RUNNER_IMAGE"] = runner_image + + prefix = "measure" if measured else "warmup" + if args.dry_run: + print(f"[{prefix}] {case.name}: {env_prefix(case, runner_image)}{shell_join(cmd)}") + return {"ok": True, "case": case.name, "dry_run": True} + + started = time.time() + try: + proc = subprocess.run( + cmd, + cwd=args.cwd, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=args.timeout, + ) + except subprocess.TimeoutExpired as exc: + elapsed_ms = int((time.time() - started) * 1000) + output = tail_timeout_output(exc) + return { + "ok": False, + "case": case.name, + "reason": f"run timed out after {args.timeout}s ({elapsed_ms}ms elapsed): {output}", + } + elapsed_ms = int((time.time() - started) * 1000) + if proc.returncode != 0: + return { + "ok": False, + "case": case.name, + "reason": tail(proc.stderr or proc.stdout), + } + + session_id = parse_session_id(proc.stdout) + if not session_id: + return { + "ok": False, + "case": case.name, + "reason": f"no session id in stdout: {tail(proc.stdout)}", + } + + try: + event = find_started_event(base_cmd, args.cwd, env, session_id, args.timeout) + if event is None: + return {"ok": False, "case": case.name, "reason": f"no host session.started event for {session_id}"} + stages = event.get("startup_stages") or [] + return { + "ok": True, + "case": case.name, + "backend": event.get("backend") or case.backend, + "agent_id": event.get("agent_id") or case.agent, + "session_id": session_id, + "startup_ms": event.get("startup_ms"), + "startup_stages": stages, + "run_elapsed_ms": elapsed_ms, + "measured": measured, + } + finally: + cleanup_session(base_cmd, args.cwd, env, session_id, args.timeout) + + +def build_run_cmd( + args: argparse.Namespace, + case: Case, + base_cmd: list[str], + workspace: Path, + label: str, +) -> list[str]: + cmd = [ + *base_cmd, + "run", + "--agent", + case.agent, + "--workspace", + str(workspace), + "--name", + args.workspace_name, + "--detach", + "--label", + label, + "--ttl", + args.ttl, + "--json", + ] + if case.needs_model and args.model: + cmd.extend(["--model", args.model]) + return cmd + + +def env_prefix(case: Case, runner_image: Optional[str]) -> str: + parts = [] + if case.backend == "libkrun": + parts.append("PILLBOX_BACKEND=libkrun") + if runner_image: + parts.append(f"PILLBOX_RUNNER_IMAGE={shell_quote(runner_image)}") + return "".join(f"{part} " for part in parts) + + +def parse_session_id(stdout: str) -> Optional[str]: + try: + payload = json.loads(stdout) + except json.JSONDecodeError: + return None + session = payload.get("session") + if isinstance(session, dict): + sid = session.get("id") + return sid if isinstance(sid, str) and sid else None + return None + + +def find_started_event( + base_cmd: list[str], + cwd: str, + env: dict[str, str], + session_id: str, + timeout: int, +) -> Optional[dict[str, Any]]: + try: + proc = subprocess.run( + [*base_cmd, "session", "events", "--json"], + cwd=cwd, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return None + if proc.returncode != 0: + return None + found: Optional[dict[str, Any]] = None + for line in proc.stdout.splitlines(): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if ( + event.get("session_id") == session_id + and event.get("event") == "session.started" + and event.get("emitter") == "host" + ): + found = event + return found + + +def cleanup_session( + base_cmd: list[str], + cwd: str, + env: dict[str, str], + session_id: str, + timeout: int, +) -> None: + try: + subprocess.run( + [*base_cmd, "session", "rm", session_id], + cwd=cwd, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + text=True, + timeout=min(timeout, 30), + ) + except subprocess.TimeoutExpired: + pass + + +def summarize(runs: list[dict[str, Any]]) -> dict[str, Any]: + out: dict[str, Any] = {} + for case in sorted({run["case"] for run in runs}): + case_runs = [run for run in runs if run["case"] == case] + totals = [int(run["startup_ms"]) for run in case_runs if isinstance(run.get("startup_ms"), int)] + stages: dict[str, list[int]] = {} + for run in case_runs: + for stage in run.get("startup_stages") or []: + name = stage.get("name") + dur = stage.get("duration_ms") + if isinstance(name, str) and isinstance(dur, int): + stages.setdefault(name, []).append(dur) + out[case] = { + "n": len(case_runs), + "startup_ms": stats(totals), + "stages": {name: stats(values) for name, values in sorted(stages.items())}, + } + return out + + +def stats(values: list[int]) -> dict[str, Optional[int]]: + if not values: + return {"min": None, "p50": None, "p95": None, "max": None} + ordered = sorted(values) + return { + "min": ordered[0], + "p50": percentile(ordered, 50), + "p95": percentile(ordered, 95), + "max": ordered[-1], + } + + +def percentile(ordered: list[int], pct: int) -> int: + if not ordered: + raise ValueError("percentile needs at least one value") + index = max( + 0, + min(len(ordered) - 1, math.ceil((pct / 100.0) * len(ordered)) - 1), + ) + return ordered[index] + + +def print_human(runs: list[dict[str, Any]], failures: list[dict[str, str]]) -> None: + summary = summarize(runs) + if not summary: + print("(no measured startup events captured)") + for case, data in summary.items(): + total = data["startup_ms"] + print( + f"{case}: n={data['n']} startup_ms " + f"min={total['min']} p50={total['p50']} p95={total['p95']} max={total['max']}" + ) + for stage, st in data["stages"].items(): + print(f" {stage:<24} min={st['min']} p50={st['p50']} p95={st['p95']} max={st['max']}") + if failures: + print("\nfailures:", file=sys.stderr) + for fail in failures: + print(f" {fail['case']} [{fail['label']}]: {fail['reason']}", file=sys.stderr) + + +def print_csv(runs: list[dict[str, Any]]) -> None: + writer = csv.writer(sys.stdout) + writer.writerow( + ["case", "session_id", "backend", "agent_id", "startup_ms", "stage_name", "stage_duration_ms"] + ) + for run in runs: + stages = run.get("startup_stages") or [] + if not stages: + writer.writerow( + [ + run["case"], + run["session_id"], + run["backend"], + run["agent_id"], + run.get("startup_ms"), + "", + "", + ] + ) + continue + for stage in stages: + writer.writerow( + [ + run["case"], + run["session_id"], + run["backend"], + run["agent_id"], + run.get("startup_ms"), + stage.get("name"), + stage.get("duration_ms"), + ] + ) + + +def tail(text: str, limit: int = 1200) -> str: + text = text.strip() + return text[-limit:] if len(text) > limit else text + + +def tail_timeout_output(exc: subprocess.TimeoutExpired, limit: int = 1200) -> str: + parts = [] + for value in (exc.stderr, exc.stdout): + if isinstance(value, bytes): + value = value.decode("utf-8", "replace") + if isinstance(value, str) and value.strip(): + parts.append(value) + return tail("\n".join(parts), limit) if parts else "no output" + + +def shell_join(cmd: list[str]) -> str: + return " ".join(shell_quote(part) for part in cmd) + + +def shell_quote(part: str) -> str: + if not part: + return "''" + safe = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_+-=.,:/") + if all(ch in safe for ch in part): + return part + return "'" + part.replace("'", "'\\''") + "'" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/eval/segmentation/results/README.md b/scripts/eval/segmentation/results/README.md index 98b74f3..a4bc097 100644 --- a/scripts/eval/segmentation/results/README.md +++ b/scripts/eval/segmentation/results/README.md @@ -12,6 +12,16 @@ consumes them lives in [`docs/optimization-gate.md`](../../../../docs/optimizati | `h2-segretries0-glm51-n10.jsonl` | H2 retry-isolation, 2026-06-15 | 10 | zai-coding-plan/glm-5.1 | same 3 tasks, `SEG_RETRIES=0`; σ̂ 0.251 → 0.037, lift +0.25 [0.13, 0.33] still excludes zero → retry not the driver | | `h4-horizon-reset-glm51-n10.jsonl` | H4 reset-vs-scope, 2026-06-15 | 10 | zai-coding-plan/glm-5.1 | 3 arms (monolithic/chained/segmented); σ̂ 0.198/0.055/0.052; scope alone +0.52 [0.23,0.69], horizon-reset-on-top −0.025 [−0.075,0.0] → **scope is the mechanism, session reset adds nothing** | | `enum-control-3task-glm51-n10.jsonl` | enumerated-monolithic control (`ENUM_MONO=1`), 2026-06-19 | 10 | zai-coding-plan/glm-5.1 | 4 arms; σ̂ mono 0.305 / enum 0.109 / chained 0.045 / seg 0.057; prompt-decomp (enum−mono) +0.17, Δσ̂ −0.196; checkpoint-gating (chained−enum) +0.18, Δσ̂ −0.064, pass 11→19/30 → **gating is a real, separable lever — "just a better prompt" refuted** | +| `control-serial-glm52-pov-n10.jsonl` | H5-inversion refutation, **serial** driver, 2026-06-19 | 10 | zai-coding-plan/glm-5.2 | 3 arms, ap_pov; σ̂ mono 0.292 / chained 0.000 / seg 0.000; mean 0.13 → 1.00. **Refutes the H5 "inversion"**: run serially, glm-5.2 chained σ̂ = 0.000 (== H4) — the cross-model campaign's chained σ̂ 0.317 was a **parallel-driver artifact** ($1.92) | +| `h5-opencode-go_deepseek-v4-pro-n10.jsonl` | H5 cross-model ⚠️ parallel driver, 2026-06-19 | 10 | deepseek-v4-pro (opencode-go) | 3 tasks × 3 arms; mono 0.50/σ̂0.29 → seg 0.91/σ̂0.15; mean lift **+0.41** ($10.32) | +| `h5-opencode-go_kimi-k2.7-code-n10.jsonl` | H5 cross-model ⚠️ parallel driver, 2026-06-19 | 10 | kimi-k2.7-code (opencode-go) | 3 tasks × 3 arms; mono 0.80/σ̂0.28 → seg 0.90/σ̂0.17; mean lift **+0.11** (high mono baseline) ($13.79) | +| `h5-zai-coding-plan_glm-5.2-n10.jsonl` | H5 cross-model ⚠️ parallel driver, 2026-06-19 | 10 | zai-coding-plan/glm-5.2 | 3 tasks × 3 arms; mono 0.47/σ̂0.38 → seg 0.88/σ̂0.19; mean lift **+0.41** ($7.12) | + +> ⚠️ **The `h5-*` cross-model files came from the 5-models-concurrent driver.** The +> concurrency inflated σ̂ via VM boot/drive contention, so treat their per-model **σ̂ +> as an upper bound** — the clean σ̂ is the serial `control-serial-*` run (chained +> 0.000). The cross-model **mean lift is robust** to contention and holds across all +> three models (mono → seg: +0.41 / +0.11 / +0.41). Run σ̂ campaigns **serially**. Re-derive the stats from any file: diff --git a/scripts/eval/segmentation/results/control-serial-glm52-pov-n10.jsonl b/scripts/eval/segmentation/results/control-serial-glm52-pov-n10.jsonl new file mode 100644 index 0000000..d5b3d4f --- /dev/null +++ b/scripts/eval/segmentation/results/control-serial-glm52-pov-n10.jsonl @@ -0,0 +1,30 @@ +{"task": "ap_pov", "cond": "monolithic", "trial": 1, "score": 0.0, "cost": 0.032649} +{"task": "ap_pov", "cond": "chained", "trial": 1, "score": 1.0, "cost": 0.091435} +{"task": "ap_pov", "cond": "segmented", "trial": 1, "score": 1.0, "cost": 0.051192999999999995} +{"task": "ap_pov", "cond": "monolithic", "trial": 2, "score": 0.0, "cost": 0.013125} +{"task": "ap_pov", "cond": "chained", "trial": 2, "score": 1.0, "cost": 0.072287} +{"task": "ap_pov", "cond": "segmented", "trial": 2, "score": 1.0, "cost": 0.087895} +{"task": "ap_pov", "cond": "monolithic", "trial": 3, "score": 0.933, "cost": 0.090671} +{"task": "ap_pov", "cond": "chained", "trial": 3, "score": 1.0, "cost": 0.111443} +{"task": "ap_pov", "cond": "segmented", "trial": 3, "score": 1.0, "cost": 0.067679} +{"task": "ap_pov", "cond": "monolithic", "trial": 4, "score": 0.0, "cost": 0.005309} +{"task": "ap_pov", "cond": "chained", "trial": 4, "score": 1.0, "cost": 0.125006} +{"task": "ap_pov", "cond": "segmented", "trial": 4, "score": 1.0, "cost": 0.06986} +{"task": "ap_pov", "cond": "monolithic", "trial": 5, "score": 0.0, "cost": 0.013431} +{"task": "ap_pov", "cond": "chained", "trial": 5, "score": 1.0, "cost": 0.093992} +{"task": "ap_pov", "cond": "segmented", "trial": 5, "score": 1.0, "cost": 0.066916} +{"task": "ap_pov", "cond": "monolithic", "trial": 6, "score": 0.0, "cost": 0.018271} +{"task": "ap_pov", "cond": "chained", "trial": 6, "score": 1.0, "cost": 0.09103} +{"task": "ap_pov", "cond": "segmented", "trial": 6, "score": 1.0, "cost": 0.117451} +{"task": "ap_pov", "cond": "monolithic", "trial": 7, "score": 0.0, "cost": 0.012999} +{"task": "ap_pov", "cond": "chained", "trial": 7, "score": 1.0, "cost": 0.0} +{"task": "ap_pov", "cond": "segmented", "trial": 7, "score": 1.0, "cost": 0.085784} +{"task": "ap_pov", "cond": "monolithic", "trial": 8, "score": 0.0, "cost": 0.013296} +{"task": "ap_pov", "cond": "chained", "trial": 8, "score": 1.0, "cost": 0.07658} +{"task": "ap_pov", "cond": "segmented", "trial": 8, "score": 1.0, "cost": 0.080613} +{"task": "ap_pov", "cond": "monolithic", "trial": 9, "score": 0.4, "cost": 0.128307} +{"task": "ap_pov", "cond": "chained", "trial": 9, "score": 1.0, "cost": 0.105255} +{"task": "ap_pov", "cond": "segmented", "trial": 9, "score": 1.0, "cost": 0.068423} +{"task": "ap_pov", "cond": "monolithic", "trial": 10, "score": 0.0, "cost": 0.012558} +{"task": "ap_pov", "cond": "chained", "trial": 10, "score": 1.0, "cost": 0.056784} +{"task": "ap_pov", "cond": "segmented", "trial": 10, "score": 1.0, "cost": 0.059369} diff --git a/scripts/eval/segmentation/results/h5-opencode-go_deepseek-v4-pro-n10.jsonl b/scripts/eval/segmentation/results/h5-opencode-go_deepseek-v4-pro-n10.jsonl new file mode 100644 index 0000000..5aed3f1 --- /dev/null +++ b/scripts/eval/segmentation/results/h5-opencode-go_deepseek-v4-pro-n10.jsonl @@ -0,0 +1,90 @@ +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 1, "score": 0.0, "cost": 0.032344} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 1, "score": 0.917, "cost": 0.141349} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 1, "score": 0.667, "cost": 0.147177} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 2, "score": 0.0, "cost": 0.03633} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 2, "score": 0.667, "cost": 0.146986} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 2, "score": 0.667, "cost": 0.16428} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 3, "score": 0.583, "cost": 0.086237} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 3, "score": 0.667, "cost": 0.125122} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 3, "score": 0.667, "cost": 0.154767} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 4, "score": 0.5, "cost": 0.093657} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 4, "score": 0.917, "cost": 0.159142} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 4, "score": 0.667, "cost": 0.142023} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 5, "score": 0.583, "cost": 0.124264} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 5, "score": 0.667, "cost": 0.161917} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 5, "score": 0.5, "cost": 0.135798} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 6, "score": 0.583, "cost": 0.072098} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 6, "score": 0.917, "cost": 0.161697} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 6, "score": 0.917, "cost": 0.150611} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 7, "score": 0.083, "cost": 0.247415} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 7, "score": 0.917, "cost": 0.164992} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 7, "score": 0.667, "cost": 0.216306} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 8, "score": 0.583, "cost": 0.086125} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 8, "score": 0.167, "cost": 0.0} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 8, "score": 0.917, "cost": 0.162578} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 9, "score": 0.083, "cost": 0.102048} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 9, "score": 0.667, "cost": 0.213085} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 9, "score": 0.917, "cost": 0.154304} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 10, "score": 0.583, "cost": 0.091732} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 10, "score": 0.667, "cost": 0.154709} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 10, "score": 0.75, "cost": 0.168482} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 1, "score": 0.75, "cost": 0.111367} +{"task": "ap_grade_school", "cond": "chained", "trial": 1, "score": 1.0, "cost": 0.110381} +{"task": "ap_grade_school", "cond": "segmented", "trial": 1, "score": 1.0, "cost": 0.099389} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 2, "score": 0.5, "cost": 0.060658} +{"task": "ap_grade_school", "cond": "chained", "trial": 2, "score": 1.0, "cost": 0.091303} +{"task": "ap_grade_school", "cond": "segmented", "trial": 2, "score": 1.0, "cost": 0.119384} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 3, "score": 0.5, "cost": 0.06832} +{"task": "ap_grade_school", "cond": "chained", "trial": 3, "score": 1.0, "cost": 0.098258} +{"task": "ap_grade_school", "cond": "segmented", "trial": 3, "score": 1.0, "cost": 0.11217099999999999} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 4, "score": 0.75, "cost": 0.070945} +{"task": "ap_grade_school", "cond": "chained", "trial": 4, "score": 1.0, "cost": 0.083209} +{"task": "ap_grade_school", "cond": "segmented", "trial": 4, "score": 1.0, "cost": 0.114643} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 5, "score": 0.5, "cost": 0.072215} +{"task": "ap_grade_school", "cond": "chained", "trial": 5, "score": 1.0, "cost": 0.072289} +{"task": "ap_grade_school", "cond": "segmented", "trial": 5, "score": 1.0, "cost": 0.11388499999999999} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 6, "score": 0.5, "cost": 0.049899} +{"task": "ap_grade_school", "cond": "chained", "trial": 6, "score": 1.0, "cost": 0.095387} +{"task": "ap_grade_school", "cond": "segmented", "trial": 6, "score": 1.0, "cost": 0.120114} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 7, "score": 0.5, "cost": 0.082006} +{"task": "ap_grade_school", "cond": "chained", "trial": 7, "score": 1.0, "cost": 0.112602} +{"task": "ap_grade_school", "cond": "segmented", "trial": 7, "score": 1.0, "cost": 0.115014} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 8, "score": 0.75, "cost": 0.069136} +{"task": "ap_grade_school", "cond": "chained", "trial": 8, "score": 1.0, "cost": 0.123614} +{"task": "ap_grade_school", "cond": "segmented", "trial": 8, "score": 1.0, "cost": 0.116203} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 9, "score": 0.5, "cost": 0.096931} +{"task": "ap_grade_school", "cond": "chained", "trial": 9, "score": 1.0, "cost": 0.100424} +{"task": "ap_grade_school", "cond": "segmented", "trial": 9, "score": 1.0, "cost": 0.13287100000000002} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 10, "score": 0.5, "cost": 0.050032} +{"task": "ap_grade_school", "cond": "chained", "trial": 10, "score": 1.0, "cost": 0.085625} +{"task": "ap_grade_school", "cond": "segmented", "trial": 10, "score": 1.0, "cost": 0.113237} +{"task": "ap_pov", "cond": "monolithic", "trial": 1, "score": 0.733, "cost": 0.122212} +{"task": "ap_pov", "cond": "chained", "trial": 1, "score": 1.0, "cost": 0.131951} +{"task": "ap_pov", "cond": "segmented", "trial": 1, "score": 0.867, "cost": 0.14813900000000002} +{"task": "ap_pov", "cond": "monolithic", "trial": 2, "score": 0.933, "cost": 0.050135} +{"task": "ap_pov", "cond": "chained", "trial": 2, "score": 0.533, "cost": 0.09224} +{"task": "ap_pov", "cond": "segmented", "trial": 2, "score": 1.0, "cost": 0.19605100000000003} +{"task": "ap_pov", "cond": "monolithic", "trial": 3, "score": 0.0, "cost": 0.035189} +{"task": "ap_pov", "cond": "chained", "trial": 3, "score": 0.4, "cost": 0.065955} +{"task": "ap_pov", "cond": "segmented", "trial": 3, "score": 1.0, "cost": 0.153869} +{"task": "ap_pov", "cond": "monolithic", "trial": 4, "score": 0.0, "cost": 0.035865} +{"task": "ap_pov", "cond": "chained", "trial": 4, "score": 0.4, "cost": 0.074975} +{"task": "ap_pov", "cond": "segmented", "trial": 4, "score": 1.0, "cost": 0.15659499999999998} +{"task": "ap_pov", "cond": "monolithic", "trial": 5, "score": 0.0, "cost": 0.031009} +{"task": "ap_pov", "cond": "chained", "trial": 5, "score": 1.0, "cost": 0.141648} +{"task": "ap_pov", "cond": "segmented", "trial": 5, "score": 1.0, "cost": 0.128922} +{"task": "ap_pov", "cond": "monolithic", "trial": 6, "score": 0.733, "cost": 0.15045} +{"task": "ap_pov", "cond": "chained", "trial": 6, "score": 1.0, "cost": 0.155336} +{"task": "ap_pov", "cond": "segmented", "trial": 6, "score": 1.0, "cost": 0.19031599999999999} +{"task": "ap_pov", "cond": "monolithic", "trial": 7, "score": 0.733, "cost": 0.112585} +{"task": "ap_pov", "cond": "chained", "trial": 7, "score": 0.533, "cost": 0.082022} +{"task": "ap_pov", "cond": "segmented", "trial": 7, "score": 1.0, "cost": 0.15221} +{"task": "ap_pov", "cond": "monolithic", "trial": 8, "score": 0.733, "cost": 0.087855} +{"task": "ap_pov", "cond": "chained", "trial": 8, "score": 1.0, "cost": 0.146665} +{"task": "ap_pov", "cond": "segmented", "trial": 8, "score": 1.0, "cost": 0.135879} +{"task": "ap_pov", "cond": "monolithic", "trial": 9, "score": 0.933, "cost": 0.106641} +{"task": "ap_pov", "cond": "chained", "trial": 9, "score": 0.4, "cost": 0.114157} +{"task": "ap_pov", "cond": "segmented", "trial": 9, "score": 1.0, "cost": 0.133425} +{"task": "ap_pov", "cond": "monolithic", "trial": 10, "score": 0.733, "cost": 0.113005} +{"task": "ap_pov", "cond": "chained", "trial": 10, "score": 0.4, "cost": 0.06951} +{"task": "ap_pov", "cond": "segmented", "trial": 10, "score": 1.0, "cost": 0.149174} diff --git a/scripts/eval/segmentation/results/h5-opencode-go_kimi-k2.7-code-n10.jsonl b/scripts/eval/segmentation/results/h5-opencode-go_kimi-k2.7-code-n10.jsonl new file mode 100644 index 0000000..4be70bb --- /dev/null +++ b/scripts/eval/segmentation/results/h5-opencode-go_kimi-k2.7-code-n10.jsonl @@ -0,0 +1,90 @@ +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 1, "score": 0.583, "cost": 0.263714} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 1, "score": 0.667, "cost": 0.239858} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 1, "score": 0.667, "cost": 0.43297800000000003} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 2, "score": 0.583, "cost": 0.134228} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 2, "score": 0.5, "cost": 0.127307} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 2, "score": 0.917, "cost": 0.133161} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 3, "score": 0.583, "cost": 0.194482} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 3, "score": 0.917, "cost": 0.179029} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 3, "score": 0.667, "cost": 0.240832} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 4, "score": 0.5, "cost": 0.255856} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 4, "score": 0.917, "cost": 0.549721} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 4, "score": 0.667, "cost": 0.37438299999999997} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 5, "score": 0.583, "cost": 0.216631} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 5, "score": 0.917, "cost": 0.338349} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 5, "score": 0.667, "cost": 0.547085} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 6, "score": 0.583, "cost": 0.119545} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 6, "score": 0.917, "cost": 0.225548} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 6, "score": 0.917, "cost": 0.18648800000000001} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 7, "score": 0.0, "cost": 0.011179} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 7, "score": 0.917, "cost": 0.212241} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 7, "score": 0.917, "cost": 0.215362} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 8, "score": 0.583, "cost": 0.175848} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 8, "score": 0.917, "cost": 0.168271} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 8, "score": 0.5, "cost": 0.102829} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 9, "score": 0.583, "cost": 0.079859} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 9, "score": 0.5, "cost": 0.170653} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 9, "score": 0.667, "cost": 0.42117499999999997} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 10, "score": 0.0, "cost": 0.023852} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 10, "score": 0.917, "cost": 0.197244} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 10, "score": 0.417, "cost": 0.269322} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 1, "score": 1.0, "cost": 0.053168} +{"task": "ap_grade_school", "cond": "chained", "trial": 1, "score": 1.0, "cost": 0.089482} +{"task": "ap_grade_school", "cond": "segmented", "trial": 1, "score": 1.0, "cost": 0.100163} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 2, "score": 1.0, "cost": 0.061648} +{"task": "ap_grade_school", "cond": "chained", "trial": 2, "score": 1.0, "cost": 0.089717} +{"task": "ap_grade_school", "cond": "segmented", "trial": 2, "score": 1.0, "cost": 0.089521} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 3, "score": 1.0, "cost": 0.045618} +{"task": "ap_grade_school", "cond": "chained", "trial": 3, "score": 1.0, "cost": 0.066207} +{"task": "ap_grade_school", "cond": "segmented", "trial": 3, "score": 1.0, "cost": 0.10169900000000001} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 4, "score": 1.0, "cost": 0.042883} +{"task": "ap_grade_school", "cond": "chained", "trial": 4, "score": 1.0, "cost": 0.113449} +{"task": "ap_grade_school", "cond": "segmented", "trial": 4, "score": 1.0, "cost": 0.10062} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 5, "score": 1.0, "cost": 0.051494} +{"task": "ap_grade_school", "cond": "chained", "trial": 5, "score": 1.0, "cost": 0.074355} +{"task": "ap_grade_school", "cond": "segmented", "trial": 5, "score": 1.0, "cost": 0.086843} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 6, "score": 1.0, "cost": 0.051516} +{"task": "ap_grade_school", "cond": "chained", "trial": 6, "score": 1.0, "cost": 0.047979} +{"task": "ap_grade_school", "cond": "segmented", "trial": 6, "score": 1.0, "cost": 0.10292100000000001} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 7, "score": 1.0, "cost": 0.051104} +{"task": "ap_grade_school", "cond": "chained", "trial": 7, "score": 1.0, "cost": 0.096186} +{"task": "ap_grade_school", "cond": "segmented", "trial": 7, "score": 1.0, "cost": 0.099065} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 8, "score": 1.0, "cost": 0.042686} +{"task": "ap_grade_school", "cond": "chained", "trial": 8, "score": 1.0, "cost": 0.061836} +{"task": "ap_grade_school", "cond": "segmented", "trial": 8, "score": 1.0, "cost": 0.057517} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 9, "score": 1.0, "cost": 0.049021} +{"task": "ap_grade_school", "cond": "chained", "trial": 9, "score": 1.0, "cost": 0.077453} +{"task": "ap_grade_school", "cond": "segmented", "trial": 9, "score": 1.0, "cost": 0.075845} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 10, "score": 1.0, "cost": 0.074998} +{"task": "ap_grade_school", "cond": "chained", "trial": 10, "score": 1.0, "cost": 0.052932} +{"task": "ap_grade_school", "cond": "segmented", "trial": 10, "score": 1.0, "cost": 0.069306} +{"task": "ap_pov", "cond": "monolithic", "trial": 1, "score": 0.933, "cost": 0.185651} +{"task": "ap_pov", "cond": "chained", "trial": 1, "score": 0.533, "cost": 0.106338} +{"task": "ap_pov", "cond": "segmented", "trial": 1, "score": 1.0, "cost": 0.272141} +{"task": "ap_pov", "cond": "monolithic", "trial": 2, "score": 0.933, "cost": 0.180872} +{"task": "ap_pov", "cond": "chained", "trial": 2, "score": 1.0, "cost": 0.291128} +{"task": "ap_pov", "cond": "segmented", "trial": 2, "score": 1.0, "cost": 0.156616} +{"task": "ap_pov", "cond": "monolithic", "trial": 3, "score": 0.933, "cost": 0.113233} +{"task": "ap_pov", "cond": "chained", "trial": 3, "score": 1.0, "cost": 0.17191} +{"task": "ap_pov", "cond": "segmented", "trial": 3, "score": 1.0, "cost": 0.147053} +{"task": "ap_pov", "cond": "monolithic", "trial": 4, "score": 0.933, "cost": 0.136742} +{"task": "ap_pov", "cond": "chained", "trial": 4, "score": 1.0, "cost": 0.129588} +{"task": "ap_pov", "cond": "segmented", "trial": 4, "score": 1.0, "cost": 0.254353} +{"task": "ap_pov", "cond": "monolithic", "trial": 5, "score": 0.933, "cost": 0.110044} +{"task": "ap_pov", "cond": "chained", "trial": 5, "score": 1.0, "cost": 0.127067} +{"task": "ap_pov", "cond": "segmented", "trial": 5, "score": 1.0, "cost": 0.120614} +{"task": "ap_pov", "cond": "monolithic", "trial": 6, "score": 0.933, "cost": 0.159007} +{"task": "ap_pov", "cond": "chained", "trial": 6, "score": 1.0, "cost": 0.132126} +{"task": "ap_pov", "cond": "segmented", "trial": 6, "score": 1.0, "cost": 0.163681} +{"task": "ap_pov", "cond": "monolithic", "trial": 7, "score": 0.933, "cost": 0.315791} +{"task": "ap_pov", "cond": "chained", "trial": 7, "score": 1.0, "cost": 0.157985} +{"task": "ap_pov", "cond": "segmented", "trial": 7, "score": 1.0, "cost": 0.109949} +{"task": "ap_pov", "cond": "monolithic", "trial": 8, "score": 0.933, "cost": 0.155004} +{"task": "ap_pov", "cond": "chained", "trial": 8, "score": 1.0, "cost": 0.184958} +{"task": "ap_pov", "cond": "segmented", "trial": 8, "score": 1.0, "cost": 0.10896600000000001} +{"task": "ap_pov", "cond": "monolithic", "trial": 9, "score": 0.867, "cost": 0.109895} +{"task": "ap_pov", "cond": "chained", "trial": 9, "score": 1.0, "cost": 0.148656} +{"task": "ap_pov", "cond": "segmented", "trial": 9, "score": 1.0, "cost": 0.167844} +{"task": "ap_pov", "cond": "monolithic", "trial": 10, "score": 0.933, "cost": 0.092343} +{"task": "ap_pov", "cond": "chained", "trial": 10, "score": 1.0, "cost": 0.155014} +{"task": "ap_pov", "cond": "segmented", "trial": 10, "score": 1.0, "cost": 0.14584} diff --git a/scripts/eval/segmentation/results/h5-zai-coding-plan_glm-5.2-n10.jsonl b/scripts/eval/segmentation/results/h5-zai-coding-plan_glm-5.2-n10.jsonl new file mode 100644 index 0000000..4cbae40 --- /dev/null +++ b/scripts/eval/segmentation/results/h5-zai-coding-plan_glm-5.2-n10.jsonl @@ -0,0 +1,90 @@ +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 1, "score": 0.0, "cost": 0.112423} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 1, "score": 0.667, "cost": 0.058102} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 1, "score": 0.667, "cost": 0.109179} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 2, "score": 0.083, "cost": 0.172395} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 2, "score": 0.667, "cost": 0.154526} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 2, "score": 0.667, "cost": 0.119821} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 3, "score": 0.083, "cost": 0.224435} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 3, "score": 0.667, "cost": 0.116891} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 3, "score": 0.75, "cost": 0.138405} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 4, "score": 0.083, "cost": 0.076814} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 4, "score": 0.667, "cost": 0.153942} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 4, "score": 0.5, "cost": 0.089349} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 5, "score": 0.083, "cost": 0.053499} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 5, "score": 0.75, "cost": 0.100873} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 5, "score": 0.5, "cost": 0.025668} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 6, "score": 0.0, "cost": 0.017692} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 6, "score": 1.0, "cost": 0.151507} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 6, "score": 0.917, "cost": 0.212633} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 7, "score": 0.083, "cost": 0.067372} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 7, "score": 0.5, "cost": 0.111056} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 7, "score": 0.667, "cost": 0.151573} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 8, "score": 0.083, "cost": 0.049285} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 8, "score": 0.75, "cost": 0.060129} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 8, "score": 0.5, "cost": 0.131219} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 9, "score": 0.083, "cost": 0.111088} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 9, "score": 0.75, "cost": 0.128139} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 9, "score": 0.5, "cost": 0.1076} +{"task": "ap_dot_dsl", "cond": "monolithic", "trial": 10, "score": 0.083, "cost": 0.139247} +{"task": "ap_dot_dsl", "cond": "chained", "trial": 10, "score": 0.75, "cost": 0.148684} +{"task": "ap_dot_dsl", "cond": "segmented", "trial": 10, "score": 0.667, "cost": 0.10191499999999999} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 1, "score": 0.65, "cost": 0.027309} +{"task": "ap_grade_school", "cond": "chained", "trial": 1, "score": 1.0, "cost": 0.061694} +{"task": "ap_grade_school", "cond": "segmented", "trial": 1, "score": 1.0, "cost": 0.047206000000000005} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 2, "score": 1.0, "cost": 0.023457} +{"task": "ap_grade_school", "cond": "chained", "trial": 2, "score": 1.0, "cost": 0.051382} +{"task": "ap_grade_school", "cond": "segmented", "trial": 2, "score": 1.0, "cost": 0.056610999999999995} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 3, "score": 0.75, "cost": 0.025809} +{"task": "ap_grade_school", "cond": "chained", "trial": 3, "score": 1.0, "cost": 0.081511} +{"task": "ap_grade_school", "cond": "segmented", "trial": 3, "score": 1.0, "cost": 0.062561} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 4, "score": 0.75, "cost": 0.029446} +{"task": "ap_grade_school", "cond": "chained", "trial": 4, "score": 1.0, "cost": 0.075512} +{"task": "ap_grade_school", "cond": "segmented", "trial": 4, "score": 1.0, "cost": 0.063412} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 5, "score": 0.75, "cost": 0.026068} +{"task": "ap_grade_school", "cond": "chained", "trial": 5, "score": 1.0, "cost": 0.072312} +{"task": "ap_grade_school", "cond": "segmented", "trial": 5, "score": 1.0, "cost": 0.070007} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 6, "score": 0.75, "cost": 0.034361} +{"task": "ap_grade_school", "cond": "chained", "trial": 6, "score": 1.0, "cost": 0.062863} +{"task": "ap_grade_school", "cond": "segmented", "trial": 6, "score": 1.0, "cost": 0.06772} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 7, "score": 0.75, "cost": 0.037742} +{"task": "ap_grade_school", "cond": "chained", "trial": 7, "score": 1.0, "cost": 0.052679} +{"task": "ap_grade_school", "cond": "segmented", "trial": 7, "score": 1.0, "cost": 0.057592000000000004} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 8, "score": 0.75, "cost": 0.028229} +{"task": "ap_grade_school", "cond": "chained", "trial": 8, "score": 1.0, "cost": 0.083011} +{"task": "ap_grade_school", "cond": "segmented", "trial": 8, "score": 1.0, "cost": 0.10345399999999999} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 9, "score": 0.75, "cost": 0.028483} +{"task": "ap_grade_school", "cond": "chained", "trial": 9, "score": 1.0, "cost": 0.072627} +{"task": "ap_grade_school", "cond": "segmented", "trial": 9, "score": 1.0, "cost": 0.06532} +{"task": "ap_grade_school", "cond": "monolithic", "trial": 10, "score": 0.75, "cost": 0.033972} +{"task": "ap_grade_school", "cond": "chained", "trial": 10, "score": 1.0, "cost": 0.068537} +{"task": "ap_grade_school", "cond": "segmented", "trial": 10, "score": 1.0, "cost": 0.048803} +{"task": "ap_pov", "cond": "monolithic", "trial": 1, "score": 0.0, "cost": 0.024281} +{"task": "ap_pov", "cond": "chained", "trial": 1, "score": 1.0, "cost": 0.120206} +{"task": "ap_pov", "cond": "segmented", "trial": 1, "score": 1.0, "cost": 0.067585} +{"task": "ap_pov", "cond": "monolithic", "trial": 2, "score": 0.8, "cost": 0.039488} +{"task": "ap_pov", "cond": "chained", "trial": 2, "score": 0.533, "cost": 0.056386} +{"task": "ap_pov", "cond": "segmented", "trial": 2, "score": 1.0, "cost": 0.096887} +{"task": "ap_pov", "cond": "monolithic", "trial": 3, "score": 0.933, "cost": 0.068128} +{"task": "ap_pov", "cond": "chained", "trial": 3, "score": 0.0, "cost": 0.068052} +{"task": "ap_pov", "cond": "segmented", "trial": 3, "score": 1.0, "cost": 0.056244999999999996} +{"task": "ap_pov", "cond": "monolithic", "trial": 4, "score": 0.0, "cost": 0.012771} +{"task": "ap_pov", "cond": "chained", "trial": 4, "score": 1.0, "cost": 0.102978} +{"task": "ap_pov", "cond": "segmented", "trial": 4, "score": 1.0, "cost": 0.07569200000000001} +{"task": "ap_pov", "cond": "monolithic", "trial": 5, "score": 0.933, "cost": 0.054598} +{"task": "ap_pov", "cond": "chained", "trial": 5, "score": 1.0, "cost": 0.077884} +{"task": "ap_pov", "cond": "segmented", "trial": 5, "score": 1.0, "cost": 0.12985} +{"task": "ap_pov", "cond": "monolithic", "trial": 6, "score": 0.0, "cost": 0.012669} +{"task": "ap_pov", "cond": "chained", "trial": 6, "score": 1.0, "cost": 0.110582} +{"task": "ap_pov", "cond": "segmented", "trial": 6, "score": 1.0, "cost": 0.057485} +{"task": "ap_pov", "cond": "monolithic", "trial": 7, "score": 0.933, "cost": 0.055636} +{"task": "ap_pov", "cond": "chained", "trial": 7, "score": 1.0, "cost": 0.07025} +{"task": "ap_pov", "cond": "segmented", "trial": 7, "score": 1.0, "cost": 0.06523999999999999} +{"task": "ap_pov", "cond": "monolithic", "trial": 8, "score": 0.467, "cost": 0.037643} +{"task": "ap_pov", "cond": "chained", "trial": 8, "score": 1.0, "cost": 0.074312} +{"task": "ap_pov", "cond": "segmented", "trial": 8, "score": 1.0, "cost": 0.075598} +{"task": "ap_pov", "cond": "monolithic", "trial": 9, "score": 0.933, "cost": 0.070199} +{"task": "ap_pov", "cond": "chained", "trial": 9, "score": 1.0, "cost": 0.093794} +{"task": "ap_pov", "cond": "segmented", "trial": 9, "score": 1.0, "cost": 0.08077999999999999} +{"task": "ap_pov", "cond": "monolithic", "trial": 10, "score": 0.8, "cost": 0.107426} +{"task": "ap_pov", "cond": "chained", "trial": 10, "score": 1.0, "cost": 0.068282} +{"task": "ap_pov", "cond": "segmented", "trial": 10, "score": 1.0, "cost": 0.070141}