feat: evidence-grade NIM discovery + all-modality cost-quality benchmark

.github/workflows/fuzz.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
   schedule:
     # Weekly deeper run (longer per-target budget via FUZZ_SECONDS).
     - cron: "41 4 * * 2"
@@ -26,6 +25,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Set up Python
@@ -50,6 +50,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Set up Python
@@ -83,6 +84,9 @@ jobs:
       - name: Fuzz orchestration engine
         run: python fuzz/fuzz_orchestration.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/orchestration
 
+      - name: Fuzz NIM model-catalog parser
+        run: python fuzz/fuzz_nim_catalog.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/nim_catalog
+
       - name: Upload crash artifacts
         if: failure()
         uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5

.github/workflows/nim-benchmark.yml

@@ -0,0 +1,149 @@
+name: NIM benchmark
+
+# Evidence-grade NVIDIA NIM model discovery and cost-quality benchmark.
+# Manual dispatch defaults to a deterministic dry run. Monthly scheduled runs
+# are live but use a conservative hard request cap. Dry execution receives no
+# provider credential; only the live step can read NVIDIA_NIM_API_KEY.
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Dry run without contacting NVIDIA"
+        type: boolean
+        default: true
+      max_total_requests:
+        description: "Hard cap on provider requests for this run"
+        type: number
+        default: 2000
+      pricing_scenario:
+        description: "Optional reviewed pricing-scenario JSON path"
+        type: string
+        default: ""
+  schedule:
+    - cron: "23 3 5 * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: nim-benchmark
+  cancel-in-progress: false
+
+jobs:
+  dry_run_benchmark:
+    name: Deterministic NIM benchmark dry run
+    if: github.event_name == 'workflow_dispatch' && inputs.dry_run == true
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install pinned runtime
+        run: |
+          python -m pip install --require-hashes -r requirements.lock
+          python -m pip install --no-deps -e .
+
+      - name: Run dry benchmark
+        env:
+          MAX_REQUESTS: ${{ inputs.max_total_requests }}
+          PRICING_SCENARIO: ${{ inputs.pricing_scenario }}
+          PROVENANCE_GIT_SHA: ${{ github.sha }}
+          PROVENANCE_RUN_ID: ${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          extra_args=()
+          if [ -n "$PRICING_SCENARIO" ]; then
+            extra_args+=(--pricing-scenario "$PRICING_SCENARIO")
+          fi
+          python -m contextual_orchestrator nim-benchmark \
+            --dry-run \
+            "${extra_args[@]}" \
+            --task-manifest examples/nim_task_manifest.json \
+            --output-dir benchmark_artifacts \
+            --max-total-requests "$MAX_REQUESTS" \
+            --git-sha "$PROVENANCE_GIT_SHA" \
+            --workflow-run-id "$PROVENANCE_RUN_ID"
+
+      - name: Upload dry-run artifacts
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5
+        with:
+          name: nim-benchmark-dry-${{ github.run_id }}
+          path: benchmark_artifacts/
+          retention-days: 30
+          if-no-files-found: error
+
+  live_benchmark:
+    name: Live NIM catalog benchmark
+    if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.dry_run != true)
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install pinned runtime
+        run: |
+          python -m pip install --require-hashes -r requirements.lock
+          python -m pip install --no-deps -e .
+
+      - name: Resolve live parameters
+        id: live_params
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_MAX_REQUESTS: ${{ inputs.max_total_requests }}
+          INPUT_PRICING: ${{ inputs.pricing_scenario }}
+        run: |
+          set -euo pipefail
+          if [ "$EVENT_NAME" = "schedule" ]; then
+            echo "max_requests=2000" >> "$GITHUB_OUTPUT"
+            echo "pricing_scenario=" >> "$GITHUB_OUTPUT"
+          else
+            echo "max_requests=${INPUT_MAX_REQUESTS}" >> "$GITHUB_OUTPUT"
+            echo "pricing_scenario=${INPUT_PRICING}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Run live benchmark
+        env:
+          NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}
+          MAX_REQUESTS: ${{ steps.live_params.outputs.max_requests }}
+          PRICING_SCENARIO: ${{ steps.live_params.outputs.pricing_scenario }}
+          PROVENANCE_GIT_SHA: ${{ github.sha }}
+          PROVENANCE_RUN_ID: ${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          extra_args=()
+          if [ -n "$PRICING_SCENARIO" ]; then
+            extra_args+=(--pricing-scenario "$PRICING_SCENARIO")
+          fi
+          python -m contextual_orchestrator nim-benchmark \
+            "${extra_args[@]}" \
+            --task-manifest examples/nim_task_manifest.json \
+            --output-dir benchmark_artifacts \
+            --max-total-requests "$MAX_REQUESTS" \
+            --git-sha "$PROVENANCE_GIT_SHA" \
+            --workflow-run-id "$PROVENANCE_RUN_ID"
+
+      - name: Upload live benchmark artifacts
+        if: always()
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5
+        with:
+          name: nim-benchmark-live-${{ github.run_id }}
+          path: benchmark_artifacts/
+          retention-days: 90
+          if-no-files-found: error

.github/workflows/security.yml

@@ -9,7 +9,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
   schedule:
     - cron: "17 3 * * 1"
   workflow_dispatch:
@@ -34,6 +33,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Initialize CodeQL
@@ -54,6 +54,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Set up Python

.github/workflows/tests.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
 
 permissions:
   contents: read
@@ -21,6 +20,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Set up Python
@@ -36,3 +36,63 @@ jobs:
 
       - name: Run full test suite
         run: python -m pytest -q
+
+  nim_benchmark_quality:
+    name: NIM benchmark coverage, docstrings, and package smoke
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install hash-locked quality tools
+        run: python -m pip install --require-hashes -r requirements-opencode-review-ci.txt
+
+      - name: Prove complete benchmark coverage and public docstrings
+        run: |
+          set -euo pipefail
+          python -m coverage erase
+          python -m coverage run --branch \
+            --source=contextual_orchestrator.nim_benchmark,contextual_orchestrator.nim_csv_evidence,contextual_orchestrator.nim_strict_scoring \
+            -m pytest \
+            tests/test_nim_benchmark.py \
+            tests/test_nim_benchmark_budget_view.py \
+            tests/test_nim_benchmark_release_acceptance.py \
+            tests/test_nim_benchmark_review_regressions.py \
+            tests/test_nim_benchmark_workflow_contract.py \
+            tests/test_nim_artifact_publication.py \
+            tests/test_nim_artifact_publication_edges.py \
+            tests/test_nim_assignment_step_ids.py \
+            tests/test_nim_csv_evidence.py \
+            tests/test_nim_csv_evidence_edges.py \
+            tests/test_nim_strict_scorer_validity.py \
+            tests/test_nim_strict_scoring_bounds.py \
+            tests/test_nim_strict_scoring_integration.py \
+            tests/test_nim_strict_scoring_leakage.py \
+            -q
+          python -m coverage report \
+            --include=contextual_orchestrator/nim_benchmark.py,contextual_orchestrator/nim_csv_evidence.py,contextual_orchestrator/nim_strict_scoring.py \
+            --show-missing \
+            --fail-under=100
+          python -m interrogate -f 100 contextual_orchestrator/nim_benchmark.py
+          python -m interrogate -f 100 contextual_orchestrator/nim_csv_evidence.py
+          python -m interrogate -f 100 contextual_orchestrator/nim_strict_scoring.py
+
+      - name: Build, install, and import the wheel
+        run: |
+          set -euo pipefail
+          rm -rf dist "$RUNNER_TEMP/nim-wheel-site"
+          python -m pip wheel --no-deps . --wheel-dir dist
+          python -m pip install --no-deps \
+            --target "$RUNNER_TEMP/nim-wheel-site" \
+            dist/contextual_orchestrator-*.whl
+          cd "$RUNNER_TEMP"
+          PYTHONPATH="$RUNNER_TEMP/nim-wheel-site" \
+            python -c "import contextual_orchestrator; import contextual_orchestrator.nim_benchmark; import contextual_orchestrator.nim_csv_evidence; import contextual_orchestrator.nim_strict_scoring"

.gitignore

@@ -12,3 +12,9 @@ tempcred.txt
 
 # hypothesis fuzzing DB
 .hypothesis/
+
+# coverage measurement data
+.coverage
+
+# local benchmark artifacts (uploaded by CI, not committed)
+benchmark_artifacts/

CHANGELOG.md

@@ -6,19 +6,31 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and
 
 ## [Unreleased]
 
+### Added
+
+- Add an optional provider-neutral NVIDIA NIM benchmark harness that dynamically discovers the live `/v1/models` catalog, probes every discovered model under bounded concurrency and a complete hard request plan, and compares every eligible direct model, route-once, bounded-conduct, and reviewed pricing-scenario policy under equal total token and call envelopes.
+- Add deterministic no-egress dry runs, valid media fixtures, secret-redacted transactional JSON/CSV/Markdown artifacts, complete role/agent/model assignment evidence, paired bootstrap uncertainty, evidence sufficiency, and quality-latency and reviewed-hypothetical-cost Pareto frontiers.
+- Add validation-time public-address-pinned provider transport, original-host authority/SNI/certificate verification, redirect and ambient-proxy rejection, credential-forwarding prevention, 8 MiB response bounds, and live-secret isolation to the NIM benchmark path.
+- Add versioned strict complete-answer scoring for locked tasks: finite full-response decimal comparison, NFC-normalized declared text alternatives with task-specific case semantics, explicit alias and prompt-disambiguation evidence, bounded value and prompt inputs, decimal-equivalent and declared-alias prompt-leakage rejection, complete token boundaries that avoid larger-word false positives, zero-score handling for oversized or unrepresentable model output, fail-before-egress manifest validation, derived-manifest provenance, and lazy optional-adapter activation.
+- Add permanent benchmark contracts for 100% production statement/branch coverage, 100% public docstrings, fuzzing, package build/install/import, exact contributor-head workflows, evidence-status semantics, and release acceptance without automatic routing authorization.
+
 ### Security
 
 - Restrict the private plain-HTTP provider seam to `localhost` or literal loopback IP addresses, reject URL userinfo before connection, dial directly without ambient proxy lookup, reject all redirect responses, and close failed resources deterministically.
 - Pin each HTTPS provider connection to the exact public addresses approved during validation, preserve the original hostname for TLS verification, bypass environment proxy resolution, and reject redirects to close DNS-rebinding and credential-forwarding SSRF paths.
+- Bound every provider response to 8 MiB of cumulative consumed bytes, including SSE iteration, reject oversized declared lengths before body consumption, fail closed on malformed or conflicting `Content-Length` and ambiguous `Content-Length` plus `Transfer-Encoding`, redact header-inspection failures, and never silently truncate an untrusted response.
 - Integrate DNS-pinned provider dispatch directly into `ModelClient` so package import performs no optional-adapter monkey-patching or order-dependent class mutation.
 - Reject provider hosts that resolve to any non-globally-routable address, including RFC 6598 shared address space, while retaining explicit multicast, private, loopback, link-local, and reserved-address protections.
 - Document narrowly scoped Semgrep suppressions for parameter-bound database queries, the explicit development-only TLS verification opt-out, and provider URLs that pass the egress guard.
 
 ### Changed
 
 - Pin Atheris by Python interpreter so the Python 3.11 fuzz job and the newer central coverage-evidence image both install a published, hash-locked wheel.
+- Run repository Tests, Fuzz, and Security workflows for stacked pull requests targeting any branch, bind every checkout to the literal contributor-head SHA, and keep checkout credentials non-persistent so local evidence cannot silently become absent or synthetic-merge-only evidence.
 
 ### Documentation
 
 - Add APA 7 doctoring for Python environment-marker semantics, Atheris artifact availability and hashes, and the supported-platform uncertainty boundary.
+- Add provider-response resource-bound doctoring covering the 8 MiB fail-closed limit, HTTP framing preflight, bounded SSE reads, batch-output partitioning, incident handling, and operational rollback.
+- Add pull-request exact-head workflow doctoring covering stacked-base support, contributor-head identity, untrusted-code execution, merge-tree separation, cancellation handling, and rollback.
 - Record the CI trust boundary between generic coverage and native fuzz execution, including the evidence-preserving retry rule for branch-referenced reusable workflows.

README.md

@@ -199,9 +199,36 @@ is read from a **KV config store**, never `os.getenv`.
   `pg_tiktoken` counting, and the production batch backend without adding a
   repository split here.
 
-Grounding papers (LLM cost, routing, load balancing) live in
+Grounding papers (LLM cost, routing, load balancing, evaluation) live in
 [docs/papers](docs/papers/README.md) with citations.
 
+### NIM cost-quality benchmark (optional harness)
+
+Evidence-grade benchmark of the routing policies against a **dynamically
+discovered** NVIDIA NIM catalog. It probes chat, completions, Responses,
+embeddings, image/video/audio understanding, transcription, and speech; compares
+direct, route-once, and bounded-conduct cells under one equal total-token and
+call budget; records paired uncertainty and Pareto frontiers; and keeps reviewed
+actual endpoint-access evidence separate from optional hypothetical paid rates.
+The bundled manifest contains thirty locked tasks. It may reach
+`evidence_review_required` when at least 90% of policy-task cells complete and
+the paired-task floor is met; otherwise it reports `insufficient_evidence`. No
+benchmark artifact automatically changes production routing.
+
+The adapter is lazy and optional: ordinary `import contextual_orchestrator` does
+not import or mutate it. Deterministic `--dry-run` receives no network access or
+NVIDIA secret. Live execution resolves `NVIDIA_NIM_API_KEY` from the credential
+registry, pins HTTPS connections to validation-time public addresses, rejects
+redirects and proxy routing, and fails closed on missing/expired evidence. See
+[docs/nim_benchmark.md](docs/nim_benchmark.md) and the
+[engineering decision record](docs/doctoring/nim-benchmark-evidence-grade.md).
+
+```bash
+python -m contextual_orchestrator nim-benchmark --dry-run \
+  --pricing-scenario examples/nim_pricing_scenario.json \
+  --output-dir benchmark_artifacts
+```
+
 ## Design Artifacts
 
 - [Library research](docs/library_research.md)
@@ -255,6 +282,7 @@ python tests/test_paper_contracts.py
 python tests/test_admin_contract.py
 python tests/test_conventions.py
 python tests/test_api_contract.py
+python tests/test_nim_benchmark.py
 python tests/test_security_hardening.py
 python tests/test_repository_security_metadata.py
 python tests/test_product_planning_contract.py

conductor/tracks.md

@@ -4,3 +4,4 @@
 |---|---|---|
 | 001-paper-grounded-orchestrator | active | Implement the source-backed orchestration contract with TDD, DDD, and CDD |
 | 002-enterprise-design-foundation | active | Add paper-grounded screen design, user stories, REST API, code/DB conventions, and i18n |
+| 003-nim-cost-quality-benchmark | active | Evidence-grade NIM catalog discovery, all-modality capability probes, and the route/conduct/single-worker cost-quality benchmark (docs/nim_benchmark.md) |