From 7a50da5ea5e93c989a5f69dbb3f6df80456abaea Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Wed, 15 Jul 2026 17:19:33 -0400 Subject: [PATCH 1/6] feat: enrich SWE-bench deck with per-instance test_cmd and test_directives Derive the canonical gold-test command + directives for every deck instance straight from the installed swebench package, matching what swebench itself runs at eval time (swebench.harness.constants.MAP_REPO_VERSION_TO_SPECS + swebench.harness.test_spec.python.get_test_directives) rather than a driver-side per-repo heuristic. The upcoming measurement driver (Task 5a) reads test_cmd/test_directives straight off each deck instance. deckHash is unchanged (ff962cb83fe5c624): it hashes only instance_id+env_key, and neither changed. bake-list.json is untouched. Assisted-By: Claude (Anthropic AI) Signed-off-by: Paolo Dettori --- experiments/swebench/deck.json | 98 ++++++++++++++++++++++++++ experiments/test/swebench-deck.test.ts | 13 ++++ scripts/gen_swebench_deck.py | 24 +++++++ 3 files changed, 135 insertions(+) diff --git a/experiments/swebench/deck.json b/experiments/swebench/deck.json index 1a25892..23be35e 100644 --- a/experiments/swebench/deck.json +++ b/experiments/swebench/deck.json @@ -42,6 +42,11 @@ "test_stop_slicing (ordering.tests.OrderingTests)", "test_stop_start_slicing (ordering.tests.OrderingTests)" ], + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1", + "test_directives": [ + "ordering.models", + "ordering.tests" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -75,6 +80,10 @@ "test_timezone_unsupported (model_enums.tests.CustomChoicesTests)", "test_uuid_unsupported (model_enums.tests.CustomChoicesTests)" ], + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1", + "test_directives": [ + "model_enums.tests" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -91,6 +100,10 @@ "test_middleware_headers (project_template.test_settings.TestStartProjectSettings)" ], "pass_to_pass": [], + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1", + "test_directives": [ + "project_template.test_settings" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -149,6 +162,10 @@ "test_valid_image (files.tests.GetImageDimensionsTests)", "test_webp (files.tests.GetImageDimensionsTests)" ], + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1", + "test_directives": [ + "files.tests" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -196,6 +213,10 @@ "test_process_template_response_returns_none (middleware_exceptions.tests.AsyncMiddlewareTests)", "test_process_view_return_response (middleware_exceptions.tests.AsyncMiddlewareTests)" ], + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1", + "test_directives": [ + "middleware_exceptions.tests" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -495,6 +516,11 @@ "test_tickets_7204_7506 (queries.tests.Queries1Tests)", "test_tickets_7448_7707 (queries.tests.Queries1Tests)" ], + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1", + "test_directives": [ + "queries.models", + "queries.tests" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -673,6 +699,10 @@ "test_update_with_none (expressions.tests.BasicExpressionsTests)", "test_uuid_pk_subquery (expressions.tests.BasicExpressionsTests)" ], + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1", + "test_directives": [ + "expressions.tests" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -768,6 +798,10 @@ "Annotations can reference fields in a values clause,", "test_values_with_pk_annotation (annotations.tests.NonAggregateAnnotationTestCase)" ], + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1", + "test_directives": [ + "annotations.tests" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -821,6 +855,10 @@ "The autocomplete_fields, raw_id_fields, and radio_fields widgets may", "`obj` is passed from `InlineModelAdmin.get_fieldsets()` to" ], + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1", + "test_directives": [ + "modeladmin.tests" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -1606,6 +1644,10 @@ "lib/matplotlib/tests/test_axes.py::test_bar_leading_nan", "lib/matplotlib/tests/test_axes.py::test_bar_all_nan[png]" ], + "test_cmd": "pytest -rA", + "test_directives": [ + "lib/matplotlib/tests/test_axes.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -1987,6 +2029,10 @@ "xarray/tests/test_variable.py::TestBackendIndexing::test_MemoryCachedArray", "xarray/tests/test_variable.py::TestBackendIndexing::test_DaskIndexingAdapter" ], + "test_cmd": "pytest -rA", + "test_directives": [ + "xarray/tests/test_variable.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -2284,6 +2330,10 @@ "xarray/tests/test_computation.py::test_cross[a6-b6-ae6-be6-cartesian--1-False]", "xarray/tests/test_computation.py::test_cross[a6-b6-ae6-be6-cartesian--1-True]" ], + "test_cmd": "pytest -rA", + "test_directives": [ + "xarray/tests/test_computation.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -2423,6 +2473,10 @@ "tests/test_self.py::TestCallbackOptions::test_verbose", "tests/test_self.py::TestCallbackOptions::test_enable_all_extensions" ], + "test_cmd": "pytest -rA", + "test_directives": [ + "tests/test_self.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -2455,6 +2509,10 @@ "testing/logging/test_fixture.py::test_caplog_captures_despite_exception", "testing/logging/test_fixture.py::test_log_report_captures_according_to_config_option_upon_failure" ], + "test_cmd": "pytest -rA", + "test_directives": [ + "testing/logging/test_fixture.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -2477,6 +2535,10 @@ "sklearn/tree/tests/test_export.py::test_precision", "sklearn/tree/tests/test_export.py::test_export_text_errors" ], + "test_cmd": "pytest -rA", + "test_directives": [ + "sklearn/tree/tests/test_export.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -2682,6 +2744,10 @@ "sklearn/compose/tests/test_column_transformer.py::test_empty_selection_pandas_output[bool_int]", "sklearn/compose/tests/test_column_transformer.py::test_raise_error_if_index_not_aligned" ], + "test_cmd": "pytest -rA", + "test_directives": [ + "sklearn/compose/tests/test_column_transformer.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -2729,6 +2795,10 @@ "tests/test_ext_autodoc_configs.py::test_autodoc_default_options", "tests/test_ext_autodoc_configs.py::test_autodoc_default_options_with_values" ], + "test_cmd": "tox --current-env -epy39 -v --", + "test_directives": [ + "tests/test_ext_autodoc_configs.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -2778,6 +2848,10 @@ "tests/test_markup.py::test_default_role1", "tests/test_markup.py::test_default_role2" ], + "test_cmd": "tox --current-env -epy39 -v --", + "test_directives": [ + "tests/test_markup.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -2838,6 +2912,10 @@ "test_ITE", "test_Min_Max" ], + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose", + "test_directives": [ + "sympy/utilities/tests/test_lambdify.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -2864,6 +2942,10 @@ "test_copyin", "test_sparse_solve" ], + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose", + "test_directives": [ + "sympy/matrices/tests/test_sparse.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -2895,6 +2977,10 @@ "test_op_priority", "test_symbolic_indexing" ], + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose", + "test_directives": [ + "sympy/tensor/array/tests/test_immutable_ndim_array.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -3060,6 +3146,10 @@ "test_issue_17988", "test_issue_18205" ], + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose", + "test_directives": [ + "sympy/polys/tests/test_polytools.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -3231,6 +3321,10 @@ "test_issue_19360", "test_poly_copy_equals_original" ], + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose", + "test_directives": [ + "sympy/polys/tests/test_polytools.py" + ], "test_runtime_ms": null, "weight_bucket": null }, @@ -3309,6 +3403,10 @@ "test_beta_math", "test_lambdify_cse" ], + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose", + "test_directives": [ + "sympy/utilities/tests/test_lambdify.py" + ], "test_runtime_ms": null, "weight_bucket": null } diff --git a/experiments/test/swebench-deck.test.ts b/experiments/test/swebench-deck.test.ts index 8b36db0..29e32d2 100644 --- a/experiments/test/swebench-deck.test.ts +++ b/experiments/test/swebench-deck.test.ts @@ -57,6 +57,19 @@ describe("swebench deck", () => { } }); + it("each instance carries a canonical gold-test command and directives (Task 5-gen)", () => { + for (const inst of deck.instances) { + expect(typeof inst.test_cmd, `${inst.instance_id} test_cmd should be a string`).toBe("string"); + expect(inst.test_cmd.length, `${inst.instance_id} test_cmd should be non-empty`).toBeGreaterThan(0); + + expect(Array.isArray(inst.test_directives), `${inst.instance_id} test_directives should be an array`).toBe(true); + expect(inst.test_directives.length, `${inst.instance_id} test_directives should be non-empty`).toBeGreaterThan(0); + for (const d of inst.test_directives) { + expect(typeof d, `${inst.instance_id} test_directives entries should be strings`).toBe("string"); + } + } + }); + it("every deck env_key has a bake-list.envs entry whose representative_instance_id is a deck instance for that env_key", () => { expect(Array.isArray(bake.envs)).toBe(true); diff --git a/scripts/gen_swebench_deck.py b/scripts/gen_swebench_deck.py index a4f2f5d..43fa442 100644 --- a/scripts/gen_swebench_deck.py +++ b/scripts/gen_swebench_deck.py @@ -21,9 +21,30 @@ import random from pathlib import Path +from swebench.harness.constants import MAP_REPO_VERSION_TO_SPECS +from swebench.harness.test_spec.python import get_test_directives from swebench.harness.test_spec.test_spec import make_test_spec +def test_cmd_and_directives(row: dict) -> tuple[str, list[str]]: + """Derive the canonical gold-test command + directives for a full SWE-bench instance row. + + Mirrors, rather than re-derives, what swebench itself runs at eval time. In the installed + swebench (4.1.0), swebench.harness.test_spec.python.make_eval_script_list_py builds its test + invocation as ``" ".join([MAP_REPO_VERSION_TO_SPECS[repo][version]["test_cmd"], + *get_test_directives(instance)])`` -- MAP_REPO_VERSION_TO_SPECS lives in + swebench.harness.constants (same constant env_key derivation already leans on -- see the + Task 1 note above), and get_test_directives lives in swebench.harness.test_spec.python (NOT + swebench.harness.utils, despite that being the more discoverable-looking module name). + test_cmd is a str per repo/version in this installed version; defensively join if a future + swebench version ever returns a list. test_directives is always a list[str]. + """ + test_cmd = MAP_REPO_VERSION_TO_SPECS[row["repo"]][str(row["version"])]["test_cmd"] + if isinstance(test_cmd, list): + test_cmd = " ".join(test_cmd) + return test_cmd, list(get_test_directives(row)) + + def env_key_and_instance_key(row: dict, arch: str) -> tuple[str, str]: """Derive (env_image_key, instance_image_key) for a full SWE-bench instance row. @@ -91,6 +112,7 @@ def main() -> None: for row in rows: env_key, instance_image_key = env_key_and_instance_key(row, args.arch) instance_image_keys[row["instance_id"]] = instance_image_key + test_cmd, test_directives = test_cmd_and_directives(row) instances.append( { "instance_id": row["instance_id"], @@ -103,6 +125,8 @@ def main() -> None: "test_patch": row.get("test_patch", ""), "fail_to_pass": to_list(row.get("FAIL_TO_PASS")), "pass_to_pass": to_list(row.get("PASS_TO_PASS")), + "test_cmd": test_cmd, + "test_directives": test_directives, "test_runtime_ms": None, "weight_bucket": None, } From fcb4ac8c7df808915f3d381e045778419ba9163a Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Wed, 15 Jul 2026 17:37:58 -0400 Subject: [PATCH 2/6] feat: SWE-bench gold-test runtime measurement driver + weight buckets Adds deploy/knative/measure-swebench-runtimes.sh (Plan B Task 5a), a gated (MEASURE_LIVE=1) driver that measures each SWE-bench deck instance's gold-test wall-clock inside a live swebench sandbox pool pod and writes test_runtime_ms + weight_bucket (tercile: light/medium/heavy) back into experiments/swebench/deck.json. Per instance: git clone --local of the baked /repos/.git mirror + checkout base_commit (not a worktree -- worktree-from-baked-mirror fails as uid 65532), a per-instance venv --system-site-packages over the baked conda env, HOME=/workspace pip install -e with --no-build-isolation --no-cache-dir, then time under a 20-minute timeout with set -f + per-token single-quoting. Exit-code handling: at base_commit the FAIL_TO_PASS tests fail by definition, so a non-zero test exit is expected and the runtime is still recorded (status=ran). Only timeout (status=timeout, ms capped at the timeout) and setup failures (status=setupfail, ms=null, excluded from tercile) are special-cased. Terciles are computed deterministically (sort by ms ascending, split by rank into 3 as-equal-as- possible groups) in embedded python3 on the host, preserving the deck generator's JSON formatting. This is authoring-only: MEASURE_LIVE is unset by default (usage + exit 0), so this change does not run against a cluster and deck.json is unmodified. Validated via bash -n and shellcheck (clean); tercile/bucketing logic demonstrated offline on synthetic runtimes. Assisted-By: Claude (Anthropic AI) Signed-off-by: Paolo Dettori --- deploy/knative/measure-swebench-runtimes.sh | 294 ++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100755 deploy/knative/measure-swebench-runtimes.sh diff --git a/deploy/knative/measure-swebench-runtimes.sh b/deploy/knative/measure-swebench-runtimes.sh new file mode 100755 index 0000000..474159a --- /dev/null +++ b/deploy/knative/measure-swebench-runtimes.sh @@ -0,0 +1,294 @@ +#!/usr/bin/env bash +# deploy/knative/measure-swebench-runtimes.sh +# Plan B Task 5: measure each SWE-bench deck instance's gold-test wall-clock inside a live +# `swebench` sandbox pool pod, then write test_runtime_ms + weight_bucket (terciles) back into +# experiments/swebench/deck.json. OCP-only (the baked sandbox image is x86_64 and lives only in +# the OCP internal registry -- see deploy/knative/build-swebench-sandbox.sh). Gated by +# MEASURE_LIVE=1 (default: print usage and exit 0, like the sibling *-smoke.sh / e*-*.sh drivers). +# +# Forced per-instance mechanism (from the live spike -- do not deviate): +# 1. git clone --local (NOT worktree -- worktree-from-baked-mirror fails as uid 65532, /repos is +# read-only root-owned) the baked bare mirror at /repos/.git into a scratch checkout, +# then checkout base_commit. +# 2. A per-instance venv --system-site-packages over the baked conda env +# (/opt/miniconda3/envs/, env_dir = env_key with the trailing ":latest" tag +# stripped, dots kept). +# 3. HOME=/workspace pip install -e --no-build-isolation --no-cache-dir (the baked +# env has the repo's deps but not the repo package itself; the venv inherits deps via +# --system-site-packages; HOME=/workspace because / is unwritable as uid 65532). +# 4. Run + time ` ` under `timeout` (default 20 min), CWD = checkout +# root, PATH prefixed with the venv then the baked env. The test's own stdout/stderr goes to +# /dev/null on the pod -- we only want the wall-clock, not the output. +# +# CRITICAL exit-code semantics: at base_commit the FAIL_TO_PASS tests fail BY DEFINITION (they +# only pass once the real fix lands), so the gold-test command exits non-zero for essentially +# every instance. That non-zero exit is EXPECTED and is NOT a validity signal -- the measured +# wall-clock is still recorded. Only two outcomes are NOT a plain "ran": +# - timeout (`timeout` returns 124): hit the cap. Record the cap (MEASURE_TIMEOUT_SEC * 1000 ms) +# as a lower-bound estimate so the instance lands in the heavy bucket -- never null it. +# - setup failure (clone / checkout / venv / pip install returned non-zero, so the test itself +# never ran): record test_runtime_ms = null; excluded from the tercile computation. +# +# Usage: +# MEASURE_LIVE=1 [NS=default] \ +# [KAGENTI_SANDBOX_POOL_SELECTOR=sh.kagenti.io/sandbox-pool=swebench] [MEASURE_SAMPLES=1] \ +# [MEASURE_TIMEOUT_SEC=1200] [MEASURE_LIMIT=0] [DECK=experiments/swebench/deck.json] \ +# [LOG_DIR=/tmp/kagenti/planB] bash deploy/knative/measure-swebench-runtimes.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +# shellcheck disable=SC1091 # lib.sh is co-located; not resolvable at lint time in isolation +source "$SCRIPT_DIR/lib.sh" + +usage() { + cat <<'EOF' +measure-swebench-runtimes.sh -- SWE-bench gold-test runtime measurement driver (Plan B Task 5). + +Gated by MEASURE_LIVE=1 (this run is a no-op without it). Needs an OCP cluster with the swebench +sandbox pool deployed (deploy/knative/swebench-sandbox-pool.yaml) and reachable via kubectl. + + MEASURE_LIVE=1 required to actually run + NS namespace (default: default) + KAGENTI_SANDBOX_POOL_SELECTOR pool pod selector (default: sh.kagenti.io/sandbox-pool=swebench) + MEASURE_SAMPLES samples per instance, median-of-N (default: 1) + MEASURE_TIMEOUT_SEC per-run timeout in seconds (default: 1200 = 20 min) + MEASURE_LIMIT cap on number of instances measured, 0 = no limit (default: 0) + DECK path to deck.json (default: experiments/swebench/deck.json) + LOG_DIR per-instance pod transcripts (default: /tmp/kagenti/planB) + +Measures SEQUENTIALLY in one pool pod (isolated + quiet -> clean timing). Writes test_runtime_ms +and weight_bucket (tercile: light/medium/heavy) back into DECK. Re-runnable: overwrites both +fields for every instance it measures; leaves unmeasured instances (beyond MEASURE_LIMIT) as-is. +EOF +} + +[ "${MEASURE_LIVE:-0}" = "1" ] || { usage; exit 0; } + +NS="${NS:-default}" +KAGENTI_SANDBOX_POOL_SELECTOR="${KAGENTI_SANDBOX_POOL_SELECTOR:-sh.kagenti.io/sandbox-pool=swebench}" +MEASURE_SAMPLES="${MEASURE_SAMPLES:-1}" +MEASURE_TIMEOUT_SEC="${MEASURE_TIMEOUT_SEC:-1200}" +MEASURE_LIMIT="${MEASURE_LIMIT:-0}" +DECK="${DECK:-$REPO_ROOT/experiments/swebench/deck.json}" +LOG_DIR="${LOG_DIR:-/tmp/kagenti/planB}" +mkdir -p "$LOG_DIR" + +[ -f "$DECK" ] || { echo "FAIL: deck not found: $DECK" >&2; exit 1; } + +echo "--- measure-swebench-runtimes: pool=$KAGENTI_SANDBOX_POOL_SELECTOR ns=$NS samples=$MEASURE_SAMPLES timeout=${MEASURE_TIMEOUT_SEC}s deck=$DECK ---" + +POD="$(kubectl -n "$NS" get pod -l "$KAGENTI_SANDBOX_POOL_SELECTOR" \ + --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" +[ -n "$POD" ] || { echo "FAIL: no Running pod matching -l $KAGENTI_SANDBOX_POOL_SELECTOR in ns $NS" >&2; exit 1; } +echo "pod=$POD" + +# Single-quote a string for safe embedding as one shell word in the pod-side script. This alone +# defeats glob expansion of directive ids that contain "[param]" brackets even before the pod +# script's own `set -f` runs -- belt-and-braces, per the brief's quoting requirement. +qq() { + local s=$1 + printf "'%s'" "${s//\'/\'\\\'\'}" +} + +MANIFEST="$(mktemp "${LOG_DIR}/manifest.XXXXXX")" +RESULTS="$(mktemp "${LOG_DIR}/results.XXXXXX")" +trap 'rm -f "$MANIFEST" "$RESULTS"' EXIT + +# Manifest: one line per instance, tab-separated: instance_id, repo, base_commit, env_key, +# test_cmd, base64(json array of test_directives). Built host-side in python3 (proper JSON +# parsing -- bash has no JSON support and deck.json fields can contain arbitrary text). +python3 - "$DECK" "$MEASURE_LIMIT" >"$MANIFEST" <<'PY' +import base64 +import json +import sys + +deck_path, limit = sys.argv[1], int(sys.argv[2]) +with open(deck_path) as f: + deck = json.load(f) +instances = deck["instances"] +if limit > 0: + instances = instances[:limit] +for inst in instances: + directives_b64 = base64.b64encode(json.dumps(inst["test_directives"]).encode()).decode() + print( + "\t".join( + [ + inst["instance_id"], + inst["repo"], + inst["base_commit"], + inst["env_key"], + inst["test_cmd"], + directives_b64, + ] + ) + ) +PY + +TOTAL=$(wc -l <"$MANIFEST" | tr -d ' ') +echo "instances to measure: $TOTAL" + +N_RAN=0 +N_TIMEOUT=0 +N_SETUPFAIL=0 +IDX=0 + +while IFS=$'\t' read -r instance_id repo base_commit env_key test_cmd directives_b64; do + IDX=$((IDX + 1)) + # env_dir: env_key with the trailing ":latest" tag stripped (dots kept). + env_dir="${env_key%:latest}" + + mapfile -t directives < <(python3 -c ' +import base64, json, sys +for d in json.loads(base64.b64decode(sys.argv[1])): + print(d) +' "$directives_b64") + + # Build the single-quoted, space-joined test invocation: test_cmd tokens (word-split -- none of + # the known test_cmd values quote their own arguments) followed by each directive, each wrapped + # individually via qq() so brackets/dashes/dots/colons in either can never be re-interpreted by + # the pod shell as a glob or option boundary. + cmd_line="" + # shellcheck disable=SC2086 # intentional word-splitting of the test_cmd string into tokens + for tok in $test_cmd; do + cmd_line="$cmd_line $(qq "$tok")" + done + for d in "${directives[@]+"${directives[@]}"}"; do + cmd_line="$cmd_line $(qq "$d")" + done + + co="/workspace/co-${instance_id}" + venv="/workspace/venv-${instance_id}" + pod_log="${LOG_DIR}/${instance_id}.pod.log" + : >"$pod_log" + + sample_mss=() + sample_status="setupfail" + for _ in $(seq 1 "$MEASURE_SAMPLES"); do + # NOTE: inside this heredoc, ${...} tokens are host-expanded (repo, base_commit, env_dir, + # instance_id, cmd_line, MEASURE_TIMEOUT_SEC -- all known on the host); \$-escaped tokens + # (\$CO, \$VENV, \$ENV_PY, \$RC, \$PATH, \$( date ... ), \$((END - START))) are pod-side and + # must only be evaluated once this text is executed as a script inside the pod. A pod-side + # `trap ... EXIT` guarantees the checkout+venv are removed on every exit path (setup failure, + # timeout, or normal completion) without duplicating the rm -rf in each branch. + pod_script="$(cat </dev/null 2>&1 +RC=\$? +END=\$(date +%s%3N) +set +f +if [ "\$RC" -eq 124 ]; then + echo "RESULT ${instance_id} timeout $((MEASURE_TIMEOUT_SEC * 1000))" +else + echo "RESULT ${instance_id} ran \$((END - START))" +fi +PODEOF +)" + + out="$(kubectl -n "$NS" exec -i "$POD" -c sandbox -- bash -s <<<"$pod_script" 2>&1 || true)" + printf '%s\n' "$out" >>"$pod_log" + + line="$(printf '%s\n' "$out" | grep '^RESULT ' | tail -n1 || true)" + read -r _ _ status ms <<<"${line:-}" + if [ "${status:-}" = "ran" ] || [ "${status:-}" = "timeout" ]; then + sample_status="$status" + sample_mss+=("$ms") + fi + done + + if [ "${#sample_mss[@]}" -eq 0 ]; then + ms_final="NA" + N_SETUPFAIL=$((N_SETUPFAIL + 1)) + echo "[$IDX/$TOTAL] ${instance_id} ${repo} setupfail (see ${pod_log})" + else + ms_final="$(printf '%s\n' "${sample_mss[@]}" | median)" + if [ "$sample_status" = "timeout" ]; then + N_TIMEOUT=$((N_TIMEOUT + 1)) + else + N_RAN=$((N_RAN + 1)) + fi + echo "[$IDX/$TOTAL] ${instance_id} ${repo} ms=${ms_final} (${sample_status})" + fi + + printf '%s\t%s\n' "$instance_id" "$ms_final" >>"$RESULTS" +done <"$MANIFEST" + +echo "--- writing test_runtime_ms + weight_bucket into $DECK ---" +python3 - "$DECK" "$RESULTS" <<'PY' +import json +import sys + +deck_path, results_path = sys.argv[1], sys.argv[2] + +runtimes = {} +with open(results_path) as f: + for line in f: + line = line.rstrip("\n") + if not line: + continue + instance_id, ms = line.split("\t") + runtimes[instance_id] = None if ms == "NA" else int(ms) + +with open(deck_path) as f: + deck = json.load(f) + +for inst in deck["instances"]: + if inst["instance_id"] in runtimes: + inst["test_runtime_ms"] = runtimes[inst["instance_id"]] + +# Terciles over ALL instances in the deck with non-null test_runtime_ms (timeouts included, at +# their cap value; not just the ones measured this run -- a re-run with MEASURE_LIMIT set should +# not wipe buckets for instances measured in a prior run). Deterministic: sort ascending by ms, +# split by RANK into 3 as-equal-as-possible groups (rank * 3 // n -> 0/1/2 -> light/medium/heavy). +# Instances with a null runtime keep weight_bucket = null. +timed = sorted( + (inst for inst in deck["instances"] if inst["test_runtime_ms"] is not None), + key=lambda inst: inst["test_runtime_ms"], +) +n = len(timed) +labels = ["light", "medium", "heavy"] +for rank, inst in enumerate(timed): + inst["weight_bucket"] = labels[rank * 3 // n] +for inst in deck["instances"]: + if inst["test_runtime_ms"] is None: + inst["weight_bucket"] = None + +with open(deck_path, "w") as f: + f.write(json.dumps(deck, indent=2) + "\n") + +buckets = {"light": 0, "medium": 0, "heavy": 0} +nulls = 0 +for inst in deck["instances"]: + if inst["weight_bucket"] is None: + nulls += 1 + else: + buckets[inst["weight_bucket"]] += 1 +print(f"buckets: light={buckets['light']} medium={buckets['medium']} heavy={buckets['heavy']} null={nulls}") +PY + +echo "--- summary: ran=$N_RAN timeout=$N_TIMEOUT setupfail=$N_SETUPFAIL total=$TOTAL ---" From 4cbf69e8fe6c126cbb86ce6a7e32e8fa41563ce9 Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Wed, 15 Jul 2026 17:47:41 -0400 Subject: [PATCH 3/6] fix: clone repos with --no-hardlinks (cross-device /repos to /workspace) git clone --local hardlinks objects, but the baked mirror at /repos (image overlayfs) and the scratch checkout under /workspace (EBS PVC) are different filesystems, so the clone fails with 'Invalid cross-device link' and every instance lands in setupfail on the live OCP gate. Switch to --no-hardlinks, which keeps the local-path fast-path but copies objects instead of hardlinking them, so it works across devices. Assisted-By: Claude (Anthropic AI) Signed-off-by: Paolo Dettori --- deploy/knative/measure-swebench-runtimes.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/deploy/knative/measure-swebench-runtimes.sh b/deploy/knative/measure-swebench-runtimes.sh index 474159a..13bf3e3 100755 --- a/deploy/knative/measure-swebench-runtimes.sh +++ b/deploy/knative/measure-swebench-runtimes.sh @@ -7,9 +7,10 @@ # MEASURE_LIVE=1 (default: print usage and exit 0, like the sibling *-smoke.sh / e*-*.sh drivers). # # Forced per-instance mechanism (from the live spike -- do not deviate): -# 1. git clone --local (NOT worktree -- worktree-from-baked-mirror fails as uid 65532, /repos is -# read-only root-owned) the baked bare mirror at /repos/.git into a scratch checkout, -# then checkout base_commit. +# 1. git clone --no-hardlinks (NOT --local: /repos is a different filesystem from /workspace, +# so hardlinking objects fails cross-device; NOT worktree -- worktree-from-baked-mirror fails +# as uid 65532, /repos is read-only root-owned) the baked bare mirror at /repos/.git +# into a scratch checkout, then checkout base_commit. # 2. A per-instance venv --system-site-packages over the baked conda env # (/opt/miniconda3/envs/, env_dir = env_key with the trailing ":latest" tag # stripped, dots kept). @@ -178,7 +179,7 @@ VENV="${venv}" ENV_PY="/opt/miniconda3/envs/${env_dir}/bin/python" rm -rf "\$CO" "\$VENV" trap 'rm -rf "\$CO" "\$VENV"' EXIT -if ! git clone --local "/repos/${repo}.git" "\$CO"; then +if ! git clone --no-hardlinks "/repos/${repo}.git" "\$CO"; then echo "RESULT ${instance_id} setupfail NA" exit 0 fi From dc737df2fb19376b585f492edcaafa14e8c99fb1 Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Wed, 15 Jul 2026 18:36:32 -0400 Subject: [PATCH 4/6] fix: run gold test_cmd verbatim and fall back to build isolation Two fixes to measure-swebench-runtimes.sh proven live on the OCP sandbox pool: 1. Splice test_cmd VERBATIM into the pod-side shell instead of tokenizing it on whitespace and re-quoting each token. Tokenizing broke inline env-var-assignment prefixes such as sympy's PYTHONWARNINGS='...' bin/test -C --verbose: quoting the assignment as argv[0] made the pod shell try to exec a literal command named 'PYTHONWARNINGS=...', exiting 127 in a few ms and masquerading as a completed run. test_cmd is now passed unmodified into a nested `bash -c 'set -f; '` invocation under timeout, so shell prefixes and flags are parsed correctly; each directive is still individually single-quoted via the existing qq() helper so '[param]'-style brackets are never glob-expanded. 2. Two-attempt editable install for the per-instance venv: first try the existing --no-build-isolation --no-cache-dir (fast, already proven for 13 repos), and on failure retry the same install WITH build isolation (still --no-cache-dir). This fixes repos whose baked env has a setuptools/setuptools_scm skew (pytest, pylint) without touching the repos that already worked. Only a failure of BOTH attempts is treated as a setup failure. Exit-code semantics are unchanged: timeout (124) records the cap and buckets heavy; any other completion records the measured wall-clock; only a setup step failing (clone/checkout/venv/both pip attempts) records test_runtime_ms = null. Assisted-By: Claude (Anthropic AI) Signed-off-by: Paolo Dettori --- deploy/knative/measure-swebench-runtimes.sh | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/deploy/knative/measure-swebench-runtimes.sh b/deploy/knative/measure-swebench-runtimes.sh index 13bf3e3..9a18d7e 100755 --- a/deploy/knative/measure-swebench-runtimes.sh +++ b/deploy/knative/measure-swebench-runtimes.sh @@ -145,15 +145,13 @@ for d in json.loads(base64.b64decode(sys.argv[1])): print(d) ' "$directives_b64") - # Build the single-quoted, space-joined test invocation: test_cmd tokens (word-split -- none of - # the known test_cmd values quote their own arguments) followed by each directive, each wrapped - # individually via qq() so brackets/dashes/dots/colons in either can never be re-interpreted by - # the pod shell as a glob or option boundary. - cmd_line="" - # shellcheck disable=SC2086 # intentional word-splitting of the test_cmd string into tokens - for tok in $test_cmd; do - cmd_line="$cmd_line $(qq "$tok")" - done + # Build the test invocation: test_cmd is spliced VERBATIM (unquoted, un-tokenized) so that + # inline shell syntax such as sympy's `PYTHONWARNINGS='...' bin/test -C --verbose` env-var + # prefix is parsed by the pod shell as intended, rather than being word-split and re-quoted + # into a bogus argv[0]. Only each directive (a file/node path that may contain "[param]" + # globs) is individually single-quoted via qq() so it can never be re-interpreted as a glob + # or option boundary. + cmd_line="$test_cmd" for d in "${directives[@]+"${directives[@]}"}"; do cmd_line="$cmd_line $(qq "$d")" done @@ -192,14 +190,16 @@ if ! "\$ENV_PY" -m venv --system-site-packages "\$VENV"; then exit 0 fi if ! HOME=/workspace "\$VENV/bin/pip" install -e "\$CO" --no-build-isolation --no-cache-dir; then - echo "RESULT ${instance_id} setupfail NA" - exit 0 + if ! HOME=/workspace "\$VENV/bin/pip" install -e "\$CO" --no-cache-dir; then + echo "RESULT ${instance_id} setupfail NA" + exit 0 + fi fi export PATH="\$VENV/bin:/opt/miniconda3/envs/${env_dir}/bin:\$PATH" cd "\$CO" set -f START=\$(date +%s%3N) -timeout ${MEASURE_TIMEOUT_SEC} ${cmd_line} >/dev/null 2>&1 +timeout ${MEASURE_TIMEOUT_SEC} bash -c $(qq "set -f; ${cmd_line}") >/dev/null 2>&1 RC=\$? END=\$(date +%s%3N) set +f From 97883f5299bda061e00a8d67e6e875f95e7a4d4b Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Wed, 15 Jul 2026 19:09:38 -0400 Subject: [PATCH 5/6] feat: bake C/Cython build toolchain into swebench sandbox image Add build-essential, python3-dev, pkg-config, libfreetype6-dev, and libpng-dev to the base-tools apt-get install list so scikit-learn and matplotlib (and other C/Cython repos) can be editable-installed in the sandbox pool. Proven live on the OCP pod. Assisted-By: Claude (Anthropic AI) Signed-off-by: Paolo Dettori --- deploy/knative/build-swebench-sandbox.sh | 5 ++++- experiments/test/swebench-sandbox-build.test.ts | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/deploy/knative/build-swebench-sandbox.sh b/deploy/knative/build-swebench-sandbox.sh index 9e7ca66..636b73b 100755 --- a/deploy/knative/build-swebench-sandbox.sh +++ b/deploy/knative/build-swebench-sandbox.sh @@ -214,8 +214,11 @@ EOF # the '*' literal. if [ "$tools_enabled" = "1" ]; then cat <<'EOF' +# build-essential/python3-dev/pkg-config/libfreetype6-dev/libpng-dev: C/Cython +# build toolchain needed for editable installs of scikit-learn and matplotlib +# (proven live on the pool pod). RUN apt-get update \ - && apt-get install -y --no-install-recommends git ripgrep ca-certificates \ + && apt-get install -y --no-install-recommends git ripgrep ca-certificates build-essential python3-dev pkg-config libfreetype6-dev libpng-dev \ && rm -rf /var/lib/apt/lists/* # Repos are cloned as root below, but the pod runs as UID 65532; without this # git refuses to operate on them with 'detected dubious ownership'. diff --git a/experiments/test/swebench-sandbox-build.test.ts b/experiments/test/swebench-sandbox-build.test.ts index d22a155..b53e535 100644 --- a/experiments/test/swebench-sandbox-build.test.ts +++ b/experiments/test/swebench-sandbox-build.test.ts @@ -180,7 +180,10 @@ describe("swebench-sandbox emitter — iterative accumulation (--offset / --base it("batch-1 default base still emits apt + safe.directory (auto base-tools)", () => { const df = emit("--offset 0 --limit 5"); expect(df).toContain("FROM ubuntu:22.04 AS assembled"); - expect(df).toContain("apt-get install -y --no-install-recommends git ripgrep ca-certificates"); + expect(df).toContain( + "apt-get install -y --no-install-recommends git ripgrep ca-certificates " + + "build-essential python3-dev pkg-config libfreetype6-dev libpng-dev", + ); expect(df).toContain("git config --system --add safe.directory '*'"); }); From 64bfbcc6b2c98f026576c89a42e8e2d7f1438e1d Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Wed, 15 Jul 2026 20:56:48 -0400 Subject: [PATCH 6/6] feat: measure SWE-bench gold-test runtimes and assign weight buckets All 24 deck instances measured live on the OCP swebench pool (gcc-enabled image); test_runtime_ms populated and light/medium/heavy terciles assigned (8/8/8). Assisted-By: Claude (Anthropic AI) Signed-off-by: Paolo Dettori --- experiments/swebench/deck.json | 96 +++++++++++++++++----------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/experiments/swebench/deck.json b/experiments/swebench/deck.json index 23be35e..8853f1d 100644 --- a/experiments/swebench/deck.json +++ b/experiments/swebench/deck.json @@ -47,8 +47,8 @@ "ordering.models", "ordering.tests" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 945, + "weight_bucket": "medium" }, { "instance_id": "django__django-11964", @@ -84,8 +84,8 @@ "test_directives": [ "model_enums.tests" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 715, + "weight_bucket": "light" }, { "instance_id": "django__django-12419", @@ -104,8 +104,8 @@ "test_directives": [ "project_template.test_settings" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 905, + "weight_bucket": "medium" }, { "instance_id": "django__django-13410", @@ -166,8 +166,8 @@ "test_directives": [ "files.tests" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 685, + "weight_bucket": "light" }, { "instance_id": "django__django-13810", @@ -217,8 +217,8 @@ "test_directives": [ "middleware_exceptions.tests" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 736, + "weight_bucket": "light" }, { "instance_id": "django__django-15128", @@ -521,8 +521,8 @@ "queries.models", "queries.tests" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 1720, + "weight_bucket": "medium" }, { "instance_id": "django__django-15382", @@ -703,8 +703,8 @@ "test_directives": [ "expressions.tests" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 1061, + "weight_bucket": "medium" }, { "instance_id": "django__django-16032", @@ -802,8 +802,8 @@ "test_directives": [ "annotations.tests" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 977, + "weight_bucket": "medium" }, { "instance_id": "django__django-16661", @@ -859,8 +859,8 @@ "test_directives": [ "modeladmin.tests" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 868, + "weight_bucket": "light" }, { "instance_id": "matplotlib__matplotlib-24177", @@ -1648,8 +1648,8 @@ "test_directives": [ "lib/matplotlib/tests/test_axes.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 2277, + "weight_bucket": "heavy" }, { "instance_id": "pydata__xarray-2905", @@ -2033,8 +2033,8 @@ "test_directives": [ "xarray/tests/test_variable.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 26746, + "weight_bucket": "heavy" }, { "instance_id": "pydata__xarray-7229", @@ -2334,8 +2334,8 @@ "test_directives": [ "xarray/tests/test_computation.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 16822, + "weight_bucket": "heavy" }, { "instance_id": "pylint-dev__pylint-7277", @@ -2477,8 +2477,8 @@ "test_directives": [ "tests/test_self.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 431, + "weight_bucket": "light" }, { "instance_id": "pytest-dev__pytest-10051", @@ -2513,8 +2513,8 @@ "test_directives": [ "testing/logging/test_fixture.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 557, + "weight_bucket": "light" }, { "instance_id": "scikit-learn__scikit-learn-14053", @@ -2539,8 +2539,8 @@ "test_directives": [ "sklearn/tree/tests/test_export.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 1224, + "weight_bucket": "medium" }, { "instance_id": "scikit-learn__scikit-learn-26323", @@ -2748,8 +2748,8 @@ "test_directives": [ "sklearn/compose/tests/test_column_transformer.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 214, + "weight_bucket": "light" }, { "instance_id": "sphinx-doc__sphinx-10449", @@ -2799,8 +2799,8 @@ "test_directives": [ "tests/test_ext_autodoc_configs.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 3362, + "weight_bucket": "heavy" }, { "instance_id": "sphinx-doc__sphinx-8621", @@ -2852,8 +2852,8 @@ "test_directives": [ "tests/test_markup.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 770, + "weight_bucket": "light" }, { "instance_id": "sympy__sympy-12096", @@ -2916,8 +2916,8 @@ "test_directives": [ "sympy/utilities/tests/test_lambdify.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 2113, + "weight_bucket": "heavy" }, { "instance_id": "sympy__sympy-13031", @@ -2946,8 +2946,8 @@ "test_directives": [ "sympy/matrices/tests/test_sparse.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 1907, + "weight_bucket": "medium" }, { "instance_id": "sympy__sympy-15017", @@ -2981,8 +2981,8 @@ "test_directives": [ "sympy/tensor/array/tests/test_immutable_ndim_array.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 1447, + "weight_bucket": "medium" }, { "instance_id": "sympy__sympy-19040", @@ -3150,8 +3150,8 @@ "test_directives": [ "sympy/polys/tests/test_polytools.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 14786, + "weight_bucket": "heavy" }, { "instance_id": "sympy__sympy-20428", @@ -3325,8 +3325,8 @@ "test_directives": [ "sympy/polys/tests/test_polytools.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 14498, + "weight_bucket": "heavy" }, { "instance_id": "sympy__sympy-23262", @@ -3407,8 +3407,8 @@ "test_directives": [ "sympy/utilities/tests/test_lambdify.py" ], - "test_runtime_ms": null, - "weight_bucket": null + "test_runtime_ms": 4000, + "weight_bucket": "heavy" } ] }