feat(tasks): add curated Campaign 20 kernel benchmarks

.gitignore

@@ -6,6 +6,7 @@ build/
 .idea
 .claude
 workspace_*
+/experiments/
 
 # Auto-cloned agent dependencies
 agents/geak_optimagentv2/GEAK-agent/

.pre-commit-config.yaml

@@ -6,4 +6,4 @@ repos:
         entry: make check-perf-helpers
         language: system
         pass_filenames: false
-        files: ^(src/tools/perf/|src/tools/sync_perf_helpers\.py|src/tools/materialize_perf_helpers\.py|src/perf_helper_materialization\.py|tasks/.*/rocmbench/.*/performance_utils_pytest\.py|tasks/triton2triton/vllm/.*/scripts/task_runner\.py)$
+        files: ^(src/tools/perf/|src/tools/sync_perf_helpers\.py|src/tools/materialize_perf_helpers\.py|src/perf_helper_materialization\.py|tasks/.*/rocmbench/.*/performance_utils_pytest\.py|tasks/.*/scripts/task_runner\.py)$

Makefile

@@ -39,7 +39,7 @@ help:
 	@echo "make check-perf-helpers  - Verify task perf-helper stubs and markers are valid"
 	@echo "make materialize-perf-workspace WORKSPACE=workspace_x - Inject canonical perf helpers into workspace(s)"
 	@echo "make materialize-perf-task TASK=tasks/... OUT=/tmp/aka-task - Copy task(s) and inject canonical perf helpers"
-	@echo "make cleanup-works       - Remove workspace_* directories and logs"
+	@echo "make cleanup-works       - Remove experiments/workspace_* directories and logs"
 	@echo "make install-cursor-agent - Install the Cursor Agent CLI on the host"
 
 DOCKER_RUNNER := src/scripts/docker_benchmark.sh
@@ -123,7 +123,7 @@ materialize-perf-task:
 
 cleanup-works:
 	@echo "Removing workspace directories and logs..."
-	@rm -rf workspace_*
+	@rm -rf experiments/workspace_*
 	@rm -rf logs
 	@echo "✓ Workspace directories and logs removed"
 

README.md

@@ -40,8 +40,8 @@ Compare the generated reports directly:
 
 ```bash
 python3 src/tools/compare_runs.py \
-  workspace_MI300_claude_code/run_<timestamp>_baseline \
-  workspace_MI300_claude_code/run_<timestamp>_treatment
+  experiments/workspace_MI300_claude_code/run_<timestamp>_baseline \
+  experiments/workspace_MI300_claude_code/run_<timestamp>_treatment
 ```
 
 For visual comparison, build the local dashboard as described in the
@@ -57,6 +57,7 @@ the observed deltas together with run-to-run variance.
 AgentKernelArena/
 ├── main.py                         # Run orchestration, resume, and parallel queue
 ├── example_configs/                # Quickstart and curated benchmark run configs
+├── experiments/                    # Ignored generated workspaces and run artifacts
 ├── src/
 │   ├── module_registration.py     # Agent registration and handler selection
 │   ├── preprocessing.py            # Workspace and repository setup

agents/geak_v3/README.md

@@ -95,10 +95,10 @@ with more GPUs via `GEAK_GPU_IDS` / `--num-parallel`.
 
 - **AKA run log:** `logs/*.log` (path from `log_directory` in `config.yaml`).
 - **Per-task result (authoritative):**
-  `workspace_<gpu>_geak_v3/run_<timestamp>/<task>_<timestamp>/task_result.yaml`
+  `experiments/workspace_<gpu>_geak_v3/run_<timestamp>/<task>_<timestamp>/task_result.yaml`
   (plus `baseline_perf.yaml`, `optimized_perf.yaml`, and
   `build/performance_report.json`).
 - **GEAK internals:**
-  `workspace_<gpu>_geak_v3/run_<timestamp>/<task>_<timestamp>_logs/`
+  `experiments/workspace_<gpu>_geak_v3/run_<timestamp>/<task>_<timestamp>_logs/`
   (`final_report.json`, `geak_agent.log`, the winning `.diff`).
-- **Aggregate:** `workspace_<gpu>_geak_v3/run_<timestamp>/reports/overall_summary.csv`.
+- **Aggregate:** `experiments/workspace_<gpu>_geak_v3/run_<timestamp>/reports/overall_summary.csv`.

agents/geak_v3_triton/README.md

@@ -118,7 +118,7 @@ Override per run via env: `GEAK_GPU_IDS`, `GEAK_NUM_PARALLEL`, `GEAK_MODEL`,
 docker exec geak-aka tail -f /workspace/logs/*.log
 
 # Per-kernel AKA results (authoritative)
-for f in workspace_*/run_*/*/task_result.yaml; do
+for f in experiments/workspace_*/run_*/*/task_result.yaml; do
   [ -f "$f" ] && echo "$(basename $(dirname "$f")): $(grep speedup_ratio "$f")"
 done
 ```

docs/examples/examples.md

@@ -34,7 +34,7 @@ Run the single-task Claude Code quickstart for the physical GPU.
 3. Inspect the result:
 
     ```text
-    workspace_MI300_claude_code/
+    experiments/workspace_MI300_claude_code/
     └── run_<timestamp>/
         └── hip2hip_gpumode_GELU_<timestamp>/
             └── task_result.yaml
@@ -89,7 +89,7 @@ make docker-run CONFIG=config_triton.yaml RUN_ARGS="--run-suffix baseline"
 make docker-run CONFIG=config_triton.yaml RUN_ARGS="--run-suffix with_capability"
 ```
 
-Both runs land in `workspace_MI300_<agent>/` with distinct run names. Build the
+Both runs land in `experiments/workspace_MI300_<agent>/` with distinct run names. Build the
 dashboard and compare them side-by-side:
 
 ```bash
@@ -153,7 +153,7 @@ make docker-parallel-run \
 The run directory contains normal per-task workspaces plus a scheduler queue:
 
 ```text
-workspace_MI355X_claude_code/
+experiments/workspace_MI355X_claude_code/
 └── run_<timestamp>_claude_parallel8/
     ├── .parallel/
     │   ├── pending/

docs/how-to/add-task.md

@@ -156,8 +156,9 @@ The shared performance timing helpers are generated from `src/tools/perf/` into
 run workspace. In committed task sources:
 
 - `tasks/*/rocmbench/**/performance_utils_pytest.py` is intentionally a stub.
-- The `AKA-GENERATED` region in `triton2triton/vllm/*/scripts/task_runner.py` is
-  intentionally a stub block.
+- The `AKA-GENERATED` region in any `tasks/**/scripts/task_runner.py` that opts
+  into shared timing is intentionally a stub block. The vLLM runners remain
+  mandatory users of this mechanism.
 
 Do not hand-edit those stubs. If a task needs shared timing behavior, add the
 stub/marker and run `make sync-perf-helpers`. If you need to change timing logic,

docs/how-to/agents.md

@@ -95,8 +95,8 @@ You can also generate a text comparison directly:
 
 ```bash
 python3 src/tools/compare_runs.py \
-  workspace_MI300_claude_code/run_<timestamp>_baseline \
-  workspace_MI300_claude_code/run_<timestamp>_with_capability
+  experiments/workspace_MI300_claude_code/run_<timestamp>_baseline \
+  experiments/workspace_MI300_claude_code/run_<timestamp>_with_capability
 ```
 
 The resulting `task_result.yaml` files expose compilation, correctness, timing,

docs/how-to/held-out-evaluation.md

@@ -57,7 +57,7 @@ Pass a completed run directory and the generated held-out configurations:
 
 ```bash
 python3 -m src.held_out.run_heldout_eval \
-  --run-dir workspace_MI300_claude_code/run_<timestamp> \
+  --run-dir experiments/workspace_MI300_claude_code/run_<timestamp> \
   --heldout-dir held_out_tests/ \
   --tasks-dir tasks/
 ```

docs/how-to/parallel-run.md

@@ -52,7 +52,7 @@ make docker-parallel-run \
 The runner creates a shared queue in the run directory:
 
 ```text
-workspace_<gpu>_<agent>/
+experiments/workspace_<gpu>_<agent>/
 └── run_<timestamp>[_suffix]/
     └── .parallel/
         ├── pending/

docs/how-to/run-evaluation.md

@@ -95,7 +95,7 @@ Add a suffix to label a run directory (useful for A/B testing):
 
 ```bash
 make docker-run CONFIG="$CONFIG_PATH" RUN_ARGS="--run-suffix cursor_with_mcp"
-# → workspace_MI300_cursor/run_20260617_101500_cursor_with_mcp
+# → experiments/workspace_MI300_cursor/run_20260617_101500_cursor_with_mcp
 ```
 
 For debugging, enter the same Docker runtime used by the experiment:
@@ -193,10 +193,10 @@ make docker-parallel-run CONFIG="$CONFIG_PATH" GPU_IDS=0,1,2,3 RUN_ARGS="--resum
 
 ## Read the results
 
-A run produces this layout under the workspace directory:
+A run produces this ignored layout under `experiments/`:
 
 ```text
-workspace_<gpu>_<agent>/
+experiments/workspace_<gpu>_<agent>/
 └── run_<timestamp>/
     ├── .parallel/              # present for docker-parallel-run
     │   ├── pending/

docs/how-to/task-validator.md

@@ -37,7 +37,7 @@ make docker-run CONFIG=config_validator.yaml
 
 Each task workspace receives a `validation_report.yaml` with per-check results,
 and a `validation_summary.yaml` with aggregated statistics is written to the
-workspace root. Tasks skipped by `platform_support.status: skip` or by a
+workspace root under `experiments/`. Tasks skipped by `platform_support.status: skip` or by a
 non-matching `platform_support.required_arch` are filtered before workspace
 creation, so they do not produce a validation report or appear in the summary
 counts.

docs/how-to/visualization.md

@@ -27,13 +27,13 @@ By default, it scans only visualization-specific report bundles:
 ```
 
 Workspace-run reports, which are usually located at
-`workspace_<gpu>_<agent>/run_<timestamp>/reports/`, can also be scanned, but this
+`experiments/workspace_<gpu>_<agent>/run_<timestamp>/reports/`, can also be scanned, but this
 is opt-in.
 
 ## Build the dashboard data and serve it
 
 After a normal AgentKernelArena run, reports land in
-`workspace_<gpu>_<agent>/run_<timestamp>/reports/`. Pass
+`experiments/workspace_<gpu>_<agent>/run_<timestamp>/reports/`. Pass
 `--include-workspace-runs` so the module picks them up. Run the following from
 the AgentKernelArena repository root:
 

docs/install/install.md

@@ -180,7 +180,7 @@ provider settings.
 
 The quickstart run above confirms that the framework, GPU, and agent CLI work
 together. It creates a timestamped workspace directory
-(`workspace_<gpu>_<agent>/run_<timestamp>/`), logs to `logs/`, and writes a
+(`experiments/workspace_<gpu>_<agent>/run_<timestamp>/`), logs to `logs/`, and writes a
 `task_result.yaml` for the task.
 
 To run across multiple GPUs, list host GPU IDs or omit `GPU_IDS` to discover

docs/reference/api-reference.md

@@ -21,7 +21,7 @@ A run configuration defines a single experiment. Start from a file under
 | `tasks` | list of strings | Task selectors relative to `tasks/`. Use `all` for every task, a category prefix for a group, or a full path for a single task. |
 | `target_gpu_model` | string | Target GPU model, for example `MI300` or `MI355X`. Used to select the Docker image architecture, set `PYTORCH_ROCM_ARCH`, and name the workspace. |
 | `log_directory` | string | Directory for run logs. |
-| `workspace_directory_prefix` | string | Prefix for the workspace directory. The full name is `<prefix>_<gpu>_<agent>`. |
+| `workspace_directory_prefix` | string | Prefix for the workspace directory. Runs are stored under the ignored path `experiments/<prefix>_<gpu>_<agent>`. |
 
 Specialized GEAK and mini-swe integrations also accept some optional top-level
 fields:

main.py

@@ -340,7 +340,9 @@ def _build_context(
     workspace_directory_name = (
         f"{config['workspace_directory_prefix']}_{config['target_gpu_model']}_{agent.value}"
     )
-    workspace_directory = (project_root / workspace_directory_name).resolve()
+    workspace_directory = (
+        project_root / "experiments" / workspace_directory_name
+    ).resolve()
     resolved_run = _resolve_run(args, workspace_directory)
     if resolved_run is None:
         return None

src/held_out/README.md

@@ -54,7 +54,7 @@ host-CLI integrations).
 
 ```bash
 python3 -m src.held_out.run_heldout_eval \
-    --run-dir workspace_MI300_cursor/run_20260417_142419 \
+    --run-dir experiments/workspace_MI300_cursor/run_20260417_142419 \
     --heldout-dir held_out_tests/ \
     --tasks-dir tasks/
 ```

src/held_out/run_heldout_eval.py

@@ -9,7 +9,7 @@
 
 Usage:
     python3 -m src.held_out.run_heldout_eval \
-        --run-dir workspace_MI300_cursor/run_20260417_142419 \
+        --run-dir experiments/workspace_MI300_cursor/run_20260417_142419 \
         --heldout-dir held_out_tests/ \
         --tasks-dir tasks/ \
         [--output-suffix _heldout]
@@ -507,7 +507,7 @@ def main():
     parser = argparse.ArgumentParser(description="Held-out evaluation for AgentKernelArena")
     parser.add_argument(
         "--run-dir", required=True,
-        help="Path to the completed run directory (e.g. workspace_MI300_cursor/run_20260417_142419)",
+        help="Path to the completed run directory (e.g. experiments/workspace_MI300_cursor/run_20260417_142419)",
     )
     parser.add_argument(
         "--heldout-dir", required=True,

src/perf_helper_materialization.py

@@ -92,7 +92,7 @@ def _marker_filtered_targets(root: Path, pattern: str) -> list[Path]:
     bespoke timer are left untouched.
     """
     targets = []
-    for p in sorted(glob.glob(str(root / pattern))):
+    for p in sorted(glob.glob(str(root / pattern), recursive=True)):
         text = Path(p).read_text()
         if any(marker in text for marker in MARK_STARTS) or MARK_END in text:
             targets.append(Path(p))
@@ -104,6 +104,17 @@ def image_kernel_targets(root: Path = ROOT) -> list[Path]:
     return _marker_filtered_targets(root, "tasks/image_kernel/*/scripts/task_runner.py")
 
 
+def marked_task_runner_targets(root: Path = ROOT) -> list[Path]:
+    """Return any task runner that opts into the shared inline timing helper.
+
+    vLLM runners remain mandatory targets through :func:`vllm_targets`. Other
+    task suites opt in by carrying the AKA-GENERATED marker block, which lets
+    new suites use the canonical helper without adding another path-specific
+    scanner here.
+    """
+    return _marker_filtered_targets(root, "tasks/**/scripts/task_runner.py")
+
+
 def canonical_rocmbench_helper(root: Path = ROOT) -> str:
     return (root / "src" / "tools" / "perf" / "performance_utils_pytest.py").read_text()
 

src/postprocessing.py

@@ -125,9 +125,9 @@ def _get_run_directory(workspace_paths: List[str]) -> Path:
     Extract run directory from workspace paths.
 
     Workspace paths are task directories like:
-    workspace_MI300_cursor/run_20250115_143022/task_hip2hip_silu_20250115_143022/
+    experiments/workspace_MI300_cursor/run_20250115_143022/task_hip2hip_silu_20250115_143022/
 
-    Returns the run directory: workspace_MI300_cursor/run_20250115_143022/
+    Returns the run directory: experiments/workspace_MI300_cursor/run_20250115_143022/
     """
     if not workspace_paths:
         raise ValueError("Cannot determine run directory: empty workspace_paths")
@@ -360,7 +360,7 @@ def _collect_all_tasks_from_run(run_directory: Path) -> List[str]:
     Collect all task directories from a run directory that have task_result.yaml.
 
     Args:
-        run_directory: Run-level directory (e.g., workspace_MI300_cursor/run_20250115_143022/)
+        run_directory: Run-level directory (e.g., experiments/workspace_MI300_cursor/run_20250115_143022/)
 
     Returns:
         List of task directory paths (as strings) that have task_result.yaml
@@ -665,5 +665,5 @@ def export_task_results_csv(
 if __name__ == "__main__":
 
     # manually generate report
-    workspace_path = "workspace_MI300_claude_code"
+    workspace_path = "experiments/workspace_MI300_claude_code"
     general_post_processing(workspace_path, logger = None)

src/preprocessing.py

@@ -448,7 +448,7 @@ def is_task_complete(
     Check if a task is already completed.
 
     Args:
-        run_directory: Run-level directory (e.g., workspace_MI300_cursor/run_20250115_143022/)
+        run_directory: Run-level directory (e.g., experiments/workspace_MI300_cursor/run_20250115_143022/)
         task_name: Full task name (e.g., "hip2hip/gpumode/SiLU")
         timestamp: Timestamp string used in task directory name
         agent_name: Agent name. task_validator uses validation_report.yaml;
@@ -478,7 +478,7 @@ def setup_workspace(task_config_dir: str, run_directory: Path, timestamp: str, l
 
     Args:
         task_config_dir: Path to task's config.yaml
-        run_directory: Run-level directory (e.g., workspace_MI300_cursor/run_20250115_143022/)
+        run_directory: Run-level directory (e.g., experiments/workspace_MI300_cursor/run_20250115_143022/)
         timestamp: Timestamp string for unique workspace naming
         logger: Logger instance
         task_name: Full task name (e.g., "hip2hip/gpumode/SiLU") for unique directory naming

src/scripts/docker_benchmark.sh

@@ -922,7 +922,7 @@ resolve_workspace_dir_for_config() {
     [[ -n "$prefix" ]] || die "workspace_directory_prefix not found in $config"
     [[ -n "$model" ]] || die "target_gpu_model not found in $config"
     [[ -n "$agent" ]] || die "agent.template not found in $config"
-    printf '%s/%s_%s_%s\n' "$HOST_ROOT" "$prefix" "$model" "$agent"
+    printf '%s/experiments/%s_%s_%s\n' "$HOST_ROOT" "$prefix" "$model" "$agent"
 }
 
 resolve_latest_run_name() {

src/tools/compare_runs.py

@@ -4,7 +4,7 @@
 
 Usage:
     python3 src/tools/compare_runs.py run1_path run2_path
-    python3 src/tools/compare_runs.py workspace_MI300_cursor/run_20260714_120000_baseline workspace_MI300_cursor/run_20260714_140000_treatment
+    python3 src/tools/compare_runs.py experiments/workspace_MI300_cursor/run_20260714_120000_baseline experiments/workspace_MI300_cursor/run_20260714_140000_treatment
 """
 
 import json
@@ -19,7 +19,7 @@ def load_run_data(run_path: Path) -> Dict[str, Any]:
     Load task_type_breakdown.json from a run directory.
 
     Args:
-        run_path: Path to run directory (e.g., workspace_MI300_cursor/run_20260714_120000_baseline)
+        run_path: Path to run directory (e.g., experiments/workspace_MI300_cursor/run_20260714_120000_baseline)
 
     Returns:
         Dictionary containing run data from JSON file
@@ -342,7 +342,7 @@ def main():
         epilog="""
 Examples:
   # Compare two runs
-  python3 src/tools/compare_runs.py workspace_MI300_cursor/run_20260714_120000_baseline workspace_MI300_cursor/run_20260714_140000_treatment
+  python3 src/tools/compare_runs.py experiments/workspace_MI300_cursor/run_20260714_120000_baseline experiments/workspace_MI300_cursor/run_20260714_140000_treatment
 
   # Compare and save to file
   python3 src/tools/compare_runs.py run1 run2 --output comparison_report.txt
@@ -352,13 +352,13 @@ def main():
     parser.add_argument(
         'run1',
         type=str,
-        help='Path to baseline/first run directory (e.g., workspace_MI300_cursor/run_20260714_120000_baseline)'
+        help='Path to baseline/first run directory (e.g., experiments/workspace_MI300_cursor/run_20260714_120000_baseline)'
     )
 
     parser.add_argument(
         'run2',
         type=str,
-        help='Path to treatment/second run directory (e.g., workspace_MI300_cursor/run_20260714_140000_treatment)'
+        help='Path to treatment/second run directory (e.g., experiments/workspace_MI300_cursor/run_20260714_140000_treatment)'
     )
 
     parser.add_argument(