Vally harness and CI workflow updates

.github/aw/actions-lock.json

@@ -5,6 +5,11 @@
       "version": "v8",
       "sha": "ed597411d8f924073f98dfc5c65a23a2325f34cd"
     },
+    "github/gh-aw-actions/setup@v0.81.6": {
+      "repo": "github/gh-aw-actions/setup",
+      "version": "v0.81.6",
+      "sha": "ba6380cc6e5be5d21677bebe04d52fb48e3abec7"
+    },
     "githubnext/gh-aw/actions/setup@v0.36.0": {
       "repo": "githubnext/gh-aw/actions/setup",
       "version": "v0.36.0",

.github/skills/agent-framework-azure-ai-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/agent-framework-azure-ai-py

.github/skills/azure-ai-contentsafety-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-ai-contentsafety-py

.github/skills/azure-ai-contentunderstanding-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py

.github/skills/azure-ai-language-conversations-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-ai-language-conversations-py

.github/skills/azure-ai-ml-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-ai-ml-py

.github/skills/azure-ai-projects-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-ai-projects-py

.github/skills/azure-ai-textanalytics-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-ai-textanalytics-py

.github/skills/azure-ai-transcription-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-ai-transcription-py

.github/skills/azure-ai-translation-document-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-ai-translation-document-py

.github/skills/azure-ai-translation-text-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-ai-translation-text-py

.github/skills/azure-ai-vision-imageanalysis-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py

.github/skills/azure-ai-voicelive-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-ai-voicelive-py

.github/skills/azure-appconfiguration-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-appconfiguration-py

.github/skills/azure-containerregistry-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-containerregistry-py

.github/skills/azure-cosmos-db-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-cosmos-db-py

.github/skills/azure-cosmos-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-cosmos-py

.github/skills/azure-data-tables-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-data-tables-py

.github/skills/azure-eventgrid-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-eventgrid-py

.github/skills/azure-eventhub-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-eventhub-py

.github/skills/azure-identity-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-identity-py

.github/skills/azure-keyvault-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-keyvault-py

.github/skills/azure-messaging-webpubsubservice-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py

.github/skills/azure-mgmt-apicenter-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py

.github/skills/azure-mgmt-apimanagement-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py

.github/skills/azure-mgmt-botservice-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-mgmt-botservice-py

.github/skills/azure-mgmt-fabric-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-mgmt-fabric-py

.github/skills/azure-monitor-ingestion-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-monitor-ingestion-py

.github/skills/azure-monitor-opentelemetry-exporter-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py

.github/skills/azure-monitor-opentelemetry-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py

.github/skills/azure-monitor-query-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-monitor-query-py

.github/skills/azure-search-documents-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-search-documents-py

.github/skills/azure-servicebus-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-servicebus-py

.github/skills/azure-speech-to-text-rest-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py

.github/skills/azure-storage-blob-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-storage-blob-py

.github/skills/azure-storage-file-datalake-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-storage-file-datalake-py

.github/skills/azure-storage-file-share-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-storage-file-share-py

.github/skills/azure-storage-queue-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/azure-storage-queue-py

.github/skills/fastapi-router-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/fastapi-router-py

.github/skills/m365-agents-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/m365-agents-py

.github/skills/pydantic-models-py

@@ -0,0 +1 @@
+../plugins/azure-sdk-python/skills/pydantic-models-py

.github/workflows/vally-evaluation.yml → .github/workflows/run-vally-evaluations.yml

@@ -1,15 +1,15 @@
-name: Vally Evaluation
+name: Vally Workflow Evaluation
 
 on:
   pull_request:
     paths:
       - "tests/scenarios/**/vally/**"
-      - ".github/workflows/vally-evaluation.yml"
+      - ".github/workflows/run-vally-evaluations.yml"
   push:
     branches: [main]
     paths:
       - "tests/scenarios/**/vally/**"
-      - ".github/workflows/vally-evaluation.yml"
+      - ".github/workflows/run-vally-evaluations.yml"
   workflow_dispatch:
     inputs:
       eval_spec:
@@ -19,24 +19,38 @@ on:
 
 permissions:
   contents: read
+  # eval.yaml's `executor: copilot-sdk` makes vally spawn the real GitHub
+  # Copilot CLI (`@github/copilot`, pulled in transitively via the vally npm
+  # dependency tree -- no extra install step needed) to run each stimulus.
+  # That CLI authenticates via the GH_TOKEN/GITHUB_TOKEN env var, which
+  # requires the GITHUB_TOKEN to carry the "Copilot Requests" permission.
+  copilot-requests: write
 
 jobs:
   vally:
     runs-on: ubuntu-latest
     env:
-      GH_TOKEN: ${{ secrets.COPILOT_TOKEN || '' }}
-
+      # Authenticates Copilot CLI and Vally's Copilot SDK evaluator path.
+      GH_TOKEN: ${{ github.token }}
+      GITHUB_TOKEN: ${{ github.token }}
+      GITHUB_COPILOT_API_TOKEN: ${{ github.token }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@v7
         with:
-          node-version: "20"
+          node-version: "24"
 
       - name: Install Vally CLI
-        run: npm install -g @microsoft/vally-cli@0.6.0
+        run: npm install -g @microsoft/vally-cli@0.7.0
+
+      - name: Install Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
 
       - name: Verify Vally CLI
         run: vally --version
@@ -87,7 +101,7 @@ jobs:
               find tests/scenarios -type f \( -name 'eval.yaml' -o -name 'eval.yml' \) -path '*/vally/*' | sort -u > "$specs_file"
             fi
           elif [ "${{ github.event_name }}" = "pull_request" ]; then
-            collect_from_changed_dirs "${{ github.event.pull_request.base.sha }}" "${{ github.sha }}" | sort -u > "$specs_file"
+            collect_from_changed_dirs "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}" | sort -u > "$specs_file"
           else
             base_sha="${{ github.event.before }}"
             if [ -z "$base_sha" ] || [ "$base_sha" = '0000000000000000000000000000000000000000' ]; then
@@ -137,8 +151,18 @@ jobs:
               --strict
           done
 
+      - name: Vally PR scope notice (same-repo)
+        if: ${{ steps.specs.outputs.has_specs == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
+        run: |
+          echo "::notice::Running Vally evaluations for same-repo PR from ${{ github.event.pull_request.head.repo.full_name }}."
+
+      - name: Vally PR scope notice (cross-repo)
+        if: ${{ steps.specs.outputs.has_specs == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}
+        run: |
+          echo "::notice::Skipping Vally evaluations for cross-repo PR from ${{ github.event.pull_request.head.repo.full_name }} to ${{ github.repository }}. Evaluation runs only on same-repo PRs and non-PR events."
+
       - name: Run Vally evaluations
-        if: ${{ steps.specs.outputs.has_specs == 'true' && env.GH_TOKEN != '' }}
+        if: ${{ steps.specs.outputs.has_specs == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
         shell: bash
         run: |
           set -euo pipefail
@@ -156,11 +180,6 @@ jobs:
             --grader-plugin "$GITHUB_WORKSPACE/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure" \
             --junit --output-dir vally-results --workers 2
 
-      - name: Skip eval when token is unavailable
-        if: ${{ steps.specs.outputs.has_specs == 'true' && env.GH_TOKEN == '' }}
-        run: |
-          echo "::warning::COPILOT_TOKEN secret is not available. Skipping vally eval run and keeping lint-only validation."
-
       - name: Upload Vally artifacts
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/test-harness.yml

@@ -24,9 +24,10 @@ jobs:
           cache: "pnpm"
           cache-dependency-path: tests/pnpm-lock.yaml
 
-      - name: Verify Rust skill links are in sync
+      - name: Verify Skill links are in sync
         run: |
           python .github/scripts/sync_skill_links.py --plugin azure-sdk-rust --check
+          python .github/scripts/sync_skill_links.py --plugin azure-sdk-python --check
 
       - name: Install dependencies
         working-directory: tests

tests/harness/copilot-client.test.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  CopilotGenerationError,
+  SkillCopilotClient,
+  classifyCopilotError,
+} from "./copilot-client.js";
+
+describe("SkillCopilotClient.extractCode", () => {
+  it("keeps assignment line for multiline constructor call without fences", () => {
+    const client = new SkillCopilotClient(process.cwd(), true);
+
+    const response = [
+      "from azure.identity import DefaultAzureCredential",
+      "from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter",
+      "",
+      "exporter = AzureMonitorTraceExporter(",
+      "    credential=DefaultAzureCredential(),",
+      '    storage_directory="/path/to/storage",',
+      "    disable_offline_storage=False,",
+      ")",
+    ].join("\n");
+
+    const extracted = (
+      client as unknown as { extractCode: (r: string) => string }
+    ).extractCode(response);
+
+    expect(extracted).toContain("exporter = AzureMonitorTraceExporter(");
+    expect(extracted).toContain("disable_offline_storage=False,");
+    expect(extracted.trim().endsWith(")")).toBe(true);
+  });
+
+  it("prefers fenced code blocks when present", () => {
+    const client = new SkillCopilotClient(process.cwd(), true);
+
+    const response = [
+      "Here is the implementation:",
+      "```python",
+      "x = 1",
+      "print(x)",
+      "```",
+    ].join("\n");
+
+    const extracted = (
+      client as unknown as { extractCode: (r: string) => string }
+    ).extractCode(response);
+
+    expect(extracted).toBe("x = 1\nprint(x)");
+  });
+});
+
+describe("classifyCopilotError", () => {
+  it("classifies timeout and marks retryable", () => {
+    const classified = classifyCopilotError(
+      new Error("Timeout after 120000ms waiting for session.idle"),
+    );
+
+    expect(classified.kind).toBe("timeout");
+    expect(classified.retryable).toBe(true);
+  });
+
+  it("classifies auth and marks non-retryable", () => {
+    const classified = classifyCopilotError(
+      new Error(
+        "Authentication failed: Failed to fetch GitHub CLI user login (401): Bad credentials",
+      ),
+    );
+
+    expect(classified.kind).toBe("auth");
+    expect(classified.retryable).toBe(false);
+  });
+
+  it("returns existing classified errors unchanged", () => {
+    const original = new CopilotGenerationError("transient", "429", true);
+    const classified = classifyCopilotError(original);
+
+    expect(classified).toBe(original);
+  });
+});