diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json index 1f061b90..df05975d 100644 --- a/.github/aw/actions-lock.json +++ b/.github/aw/actions-lock.json @@ -5,6 +5,11 @@ "version": "v8", "sha": "ed597411d8f924073f98dfc5c65a23a2325f34cd" }, + "github/gh-aw-actions/setup@v0.81.6": { + "repo": "github/gh-aw-actions/setup", + "version": "v0.81.6", + "sha": "ba6380cc6e5be5d21677bebe04d52fb48e3abec7" + }, "githubnext/gh-aw/actions/setup@v0.36.0": { "repo": "githubnext/gh-aw/actions/setup", "version": "v0.36.0", diff --git a/.github/skills/agent-framework-azure-ai-py b/.github/skills/agent-framework-azure-ai-py new file mode 120000 index 00000000..0f574683 --- /dev/null +++ b/.github/skills/agent-framework-azure-ai-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/agent-framework-azure-ai-py \ No newline at end of file diff --git a/.github/skills/azure-ai-contentsafety-py b/.github/skills/azure-ai-contentsafety-py new file mode 120000 index 00000000..08ab0c46 --- /dev/null +++ b/.github/skills/azure-ai-contentsafety-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-contentsafety-py \ No newline at end of file diff --git a/.github/skills/azure-ai-contentunderstanding-py b/.github/skills/azure-ai-contentunderstanding-py new file mode 120000 index 00000000..8d4a2e04 --- /dev/null +++ b/.github/skills/azure-ai-contentunderstanding-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py \ No newline at end of file diff --git a/.github/skills/azure-ai-language-conversations-py b/.github/skills/azure-ai-language-conversations-py new file mode 120000 index 00000000..257a9470 --- /dev/null +++ b/.github/skills/azure-ai-language-conversations-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-language-conversations-py \ No newline at end of file diff --git a/.github/skills/azure-ai-ml-py b/.github/skills/azure-ai-ml-py new file mode 120000 index 00000000..eaac1e82 --- /dev/null +++ b/.github/skills/azure-ai-ml-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-ml-py \ No newline at end of file diff --git a/.github/skills/azure-ai-projects-py b/.github/skills/azure-ai-projects-py new file mode 120000 index 00000000..eaaf98bc --- /dev/null +++ b/.github/skills/azure-ai-projects-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-projects-py \ No newline at end of file diff --git a/.github/skills/azure-ai-textanalytics-py b/.github/skills/azure-ai-textanalytics-py new file mode 120000 index 00000000..650baacf --- /dev/null +++ b/.github/skills/azure-ai-textanalytics-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-textanalytics-py \ No newline at end of file diff --git a/.github/skills/azure-ai-transcription-py b/.github/skills/azure-ai-transcription-py new file mode 120000 index 00000000..f5b70e97 --- /dev/null +++ b/.github/skills/azure-ai-transcription-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-transcription-py \ No newline at end of file diff --git a/.github/skills/azure-ai-translation-document-py b/.github/skills/azure-ai-translation-document-py new file mode 120000 index 00000000..d81c331a --- /dev/null +++ b/.github/skills/azure-ai-translation-document-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-translation-document-py \ No newline at end of file diff --git a/.github/skills/azure-ai-translation-text-py b/.github/skills/azure-ai-translation-text-py new file mode 120000 index 00000000..a0d31c26 --- /dev/null +++ b/.github/skills/azure-ai-translation-text-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-translation-text-py \ No newline at end of file diff --git a/.github/skills/azure-ai-vision-imageanalysis-py b/.github/skills/azure-ai-vision-imageanalysis-py new file mode 120000 index 00000000..0cf02c59 --- /dev/null +++ b/.github/skills/azure-ai-vision-imageanalysis-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py \ No newline at end of file diff --git a/.github/skills/azure-ai-voicelive-py b/.github/skills/azure-ai-voicelive-py new file mode 120000 index 00000000..ee0d9929 --- /dev/null +++ b/.github/skills/azure-ai-voicelive-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-voicelive-py \ No newline at end of file diff --git a/.github/skills/azure-appconfiguration-py b/.github/skills/azure-appconfiguration-py new file mode 120000 index 00000000..0c8182d0 --- /dev/null +++ b/.github/skills/azure-appconfiguration-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-appconfiguration-py \ No newline at end of file diff --git a/.github/skills/azure-containerregistry-py b/.github/skills/azure-containerregistry-py new file mode 120000 index 00000000..ee9c2e53 --- /dev/null +++ b/.github/skills/azure-containerregistry-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-containerregistry-py \ No newline at end of file diff --git a/.github/skills/azure-cosmos-db-py b/.github/skills/azure-cosmos-db-py new file mode 120000 index 00000000..4b58584f --- /dev/null +++ b/.github/skills/azure-cosmos-db-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-cosmos-db-py \ No newline at end of file diff --git a/.github/skills/azure-cosmos-py b/.github/skills/azure-cosmos-py new file mode 120000 index 00000000..ab82f04f --- /dev/null +++ b/.github/skills/azure-cosmos-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-cosmos-py \ No newline at end of file diff --git a/.github/skills/azure-data-tables-py b/.github/skills/azure-data-tables-py new file mode 120000 index 00000000..a2dbba4d --- /dev/null +++ b/.github/skills/azure-data-tables-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-data-tables-py \ No newline at end of file diff --git a/.github/skills/azure-eventgrid-py b/.github/skills/azure-eventgrid-py new file mode 120000 index 00000000..d3caa828 --- /dev/null +++ b/.github/skills/azure-eventgrid-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-eventgrid-py \ No newline at end of file diff --git a/.github/skills/azure-eventhub-py b/.github/skills/azure-eventhub-py new file mode 120000 index 00000000..d2fc8221 --- /dev/null +++ b/.github/skills/azure-eventhub-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-eventhub-py \ No newline at end of file diff --git a/.github/skills/azure-identity-py b/.github/skills/azure-identity-py new file mode 120000 index 00000000..dfe20c2b --- /dev/null +++ b/.github/skills/azure-identity-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-identity-py \ No newline at end of file diff --git a/.github/skills/azure-keyvault-py b/.github/skills/azure-keyvault-py new file mode 120000 index 00000000..2335eac7 --- /dev/null +++ b/.github/skills/azure-keyvault-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-keyvault-py \ No newline at end of file diff --git a/.github/skills/azure-messaging-webpubsubservice-py b/.github/skills/azure-messaging-webpubsubservice-py new file mode 120000 index 00000000..41368c4a --- /dev/null +++ b/.github/skills/azure-messaging-webpubsubservice-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py \ No newline at end of file diff --git a/.github/skills/azure-mgmt-apicenter-py b/.github/skills/azure-mgmt-apicenter-py new file mode 120000 index 00000000..446781ba --- /dev/null +++ b/.github/skills/azure-mgmt-apicenter-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py \ No newline at end of file diff --git a/.github/skills/azure-mgmt-apimanagement-py b/.github/skills/azure-mgmt-apimanagement-py new file mode 120000 index 00000000..adf8958c --- /dev/null +++ b/.github/skills/azure-mgmt-apimanagement-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py \ No newline at end of file diff --git a/.github/skills/azure-mgmt-botservice-py b/.github/skills/azure-mgmt-botservice-py new file mode 120000 index 00000000..84a3ac19 --- /dev/null +++ b/.github/skills/azure-mgmt-botservice-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-mgmt-botservice-py \ No newline at end of file diff --git a/.github/skills/azure-mgmt-fabric-py b/.github/skills/azure-mgmt-fabric-py new file mode 120000 index 00000000..67651cbb --- /dev/null +++ b/.github/skills/azure-mgmt-fabric-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-mgmt-fabric-py \ No newline at end of file diff --git a/.github/skills/azure-monitor-ingestion-py b/.github/skills/azure-monitor-ingestion-py new file mode 120000 index 00000000..a31461c2 --- /dev/null +++ b/.github/skills/azure-monitor-ingestion-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-monitor-ingestion-py \ No newline at end of file diff --git a/.github/skills/azure-monitor-opentelemetry-exporter-py b/.github/skills/azure-monitor-opentelemetry-exporter-py new file mode 120000 index 00000000..23b34751 --- /dev/null +++ b/.github/skills/azure-monitor-opentelemetry-exporter-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py \ No newline at end of file diff --git a/.github/skills/azure-monitor-opentelemetry-py b/.github/skills/azure-monitor-opentelemetry-py new file mode 120000 index 00000000..bcee5c56 --- /dev/null +++ b/.github/skills/azure-monitor-opentelemetry-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py \ No newline at end of file diff --git a/.github/skills/azure-monitor-query-py b/.github/skills/azure-monitor-query-py new file mode 120000 index 00000000..0f8f3bd6 --- /dev/null +++ b/.github/skills/azure-monitor-query-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-monitor-query-py \ No newline at end of file diff --git a/.github/skills/azure-search-documents-py b/.github/skills/azure-search-documents-py new file mode 120000 index 00000000..13a4ec86 --- /dev/null +++ b/.github/skills/azure-search-documents-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-search-documents-py \ No newline at end of file diff --git a/.github/skills/azure-servicebus-py b/.github/skills/azure-servicebus-py new file mode 120000 index 00000000..5275c159 --- /dev/null +++ b/.github/skills/azure-servicebus-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-servicebus-py \ No newline at end of file diff --git a/.github/skills/azure-speech-to-text-rest-py b/.github/skills/azure-speech-to-text-rest-py new file mode 120000 index 00000000..c9bc2f35 --- /dev/null +++ b/.github/skills/azure-speech-to-text-rest-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py \ No newline at end of file diff --git a/.github/skills/azure-storage-blob-py b/.github/skills/azure-storage-blob-py new file mode 120000 index 00000000..6941148a --- /dev/null +++ b/.github/skills/azure-storage-blob-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-storage-blob-py \ No newline at end of file diff --git a/.github/skills/azure-storage-file-datalake-py b/.github/skills/azure-storage-file-datalake-py new file mode 120000 index 00000000..a4f13f7b --- /dev/null +++ b/.github/skills/azure-storage-file-datalake-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-storage-file-datalake-py \ No newline at end of file diff --git a/.github/skills/azure-storage-file-share-py b/.github/skills/azure-storage-file-share-py new file mode 120000 index 00000000..5e1e9b27 --- /dev/null +++ b/.github/skills/azure-storage-file-share-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-storage-file-share-py \ No newline at end of file diff --git a/.github/skills/azure-storage-queue-py b/.github/skills/azure-storage-queue-py new file mode 120000 index 00000000..3260a7ed --- /dev/null +++ b/.github/skills/azure-storage-queue-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-storage-queue-py \ No newline at end of file diff --git a/.github/skills/fastapi-router-py b/.github/skills/fastapi-router-py new file mode 120000 index 00000000..d190edc7 --- /dev/null +++ b/.github/skills/fastapi-router-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/fastapi-router-py \ No newline at end of file diff --git a/.github/skills/m365-agents-py b/.github/skills/m365-agents-py new file mode 120000 index 00000000..a4dddc0e --- /dev/null +++ b/.github/skills/m365-agents-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/m365-agents-py \ No newline at end of file diff --git a/.github/skills/pydantic-models-py b/.github/skills/pydantic-models-py new file mode 120000 index 00000000..5f5f6459 --- /dev/null +++ b/.github/skills/pydantic-models-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/pydantic-models-py \ No newline at end of file diff --git a/.github/workflows/vally-evaluation.yml b/.github/workflows/run-vally-evaluations.yml similarity index 71% rename from .github/workflows/vally-evaluation.yml rename to .github/workflows/run-vally-evaluations.yml index b5975531..a4359748 100644 --- a/.github/workflows/vally-evaluation.yml +++ b/.github/workflows/run-vally-evaluations.yml @@ -1,15 +1,15 @@ -name: Vally Evaluation +name: Vally Workflow Evaluation on: pull_request: paths: - "tests/scenarios/**/vally/**" - - ".github/workflows/vally-evaluation.yml" + - ".github/workflows/run-vally-evaluations.yml" push: branches: [main] paths: - "tests/scenarios/**/vally/**" - - ".github/workflows/vally-evaluation.yml" + - ".github/workflows/run-vally-evaluations.yml" workflow_dispatch: inputs: eval_spec: @@ -19,24 +19,38 @@ on: permissions: contents: read + # eval.yaml's `executor: copilot-sdk` makes vally spawn the real GitHub + # Copilot CLI (`@github/copilot`, pulled in transitively via the vally npm + # dependency tree -- no extra install step needed) to run each stimulus. + # That CLI authenticates via the GH_TOKEN/GITHUB_TOKEN env var, which + # requires the GITHUB_TOKEN to carry the "Copilot Requests" permission. + copilot-requests: write jobs: vally: runs-on: ubuntu-latest env: - GH_TOKEN: ${{ secrets.COPILOT_TOKEN || '' }} - + # Authenticates Copilot CLI and Vally's Copilot SDK evaluator path. + GH_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ github.token }} + GITHUB_COPILOT_API_TOKEN: ${{ github.token }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: - node-version: "20" + node-version: "24" - name: Install Vally CLI - run: npm install -g @microsoft/vally-cli@0.6.0 + run: npm install -g @microsoft/vally-cli@0.7.0 + + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true - name: Verify Vally CLI run: vally --version @@ -87,7 +101,7 @@ jobs: find tests/scenarios -type f \( -name 'eval.yaml' -o -name 'eval.yml' \) -path '*/vally/*' | sort -u > "$specs_file" fi elif [ "${{ github.event_name }}" = "pull_request" ]; then - collect_from_changed_dirs "${{ github.event.pull_request.base.sha }}" "${{ github.sha }}" | sort -u > "$specs_file" + collect_from_changed_dirs "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}" | sort -u > "$specs_file" else base_sha="${{ github.event.before }}" if [ -z "$base_sha" ] || [ "$base_sha" = '0000000000000000000000000000000000000000' ]; then @@ -137,8 +151,18 @@ jobs: --strict done + - name: Vally PR scope notice (same-repo) + if: ${{ steps.specs.outputs.has_specs == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + run: | + echo "::notice::Running Vally evaluations for same-repo PR from ${{ github.event.pull_request.head.repo.full_name }}." + + - name: Vally PR scope notice (cross-repo) + if: ${{ steps.specs.outputs.has_specs == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + run: | + echo "::notice::Skipping Vally evaluations for cross-repo PR from ${{ github.event.pull_request.head.repo.full_name }} to ${{ github.repository }}. Evaluation runs only on same-repo PRs and non-PR events." + - name: Run Vally evaluations - if: ${{ steps.specs.outputs.has_specs == 'true' && env.GH_TOKEN != '' }} + if: ${{ steps.specs.outputs.has_specs == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} shell: bash run: | set -euo pipefail @@ -156,11 +180,6 @@ jobs: --grader-plugin "$GITHUB_WORKSPACE/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure" \ --junit --output-dir vally-results --workers 2 - - name: Skip eval when token is unavailable - if: ${{ steps.specs.outputs.has_specs == 'true' && env.GH_TOKEN == '' }} - run: | - echo "::warning::COPILOT_TOKEN secret is not available. Skipping vally eval run and keeping lint-only validation." - - name: Upload Vally artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/test-harness.yml b/.github/workflows/test-harness.yml index 6dc78f91..1805c2da 100644 --- a/.github/workflows/test-harness.yml +++ b/.github/workflows/test-harness.yml @@ -24,9 +24,10 @@ jobs: cache: "pnpm" cache-dependency-path: tests/pnpm-lock.yaml - - name: Verify Rust skill links are in sync + - name: Verify Skill links are in sync run: | python .github/scripts/sync_skill_links.py --plugin azure-sdk-rust --check + python .github/scripts/sync_skill_links.py --plugin azure-sdk-python --check - name: Install dependencies working-directory: tests diff --git a/tests/harness/copilot-client.test.ts b/tests/harness/copilot-client.test.ts new file mode 100644 index 00000000..53100cdc --- /dev/null +++ b/tests/harness/copilot-client.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; + +import { + CopilotGenerationError, + SkillCopilotClient, + classifyCopilotError, +} from "./copilot-client.js"; + +describe("SkillCopilotClient.extractCode", () => { + it("keeps assignment line for multiline constructor call without fences", () => { + const client = new SkillCopilotClient(process.cwd(), true); + + const response = [ + "from azure.identity import DefaultAzureCredential", + "from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter", + "", + "exporter = AzureMonitorTraceExporter(", + " credential=DefaultAzureCredential(),", + ' storage_directory="/path/to/storage",', + " disable_offline_storage=False,", + ")", + ].join("\n"); + + const extracted = ( + client as unknown as { extractCode: (r: string) => string } + ).extractCode(response); + + expect(extracted).toContain("exporter = AzureMonitorTraceExporter("); + expect(extracted).toContain("disable_offline_storage=False,"); + expect(extracted.trim().endsWith(")")).toBe(true); + }); + + it("prefers fenced code blocks when present", () => { + const client = new SkillCopilotClient(process.cwd(), true); + + const response = [ + "Here is the implementation:", + "```python", + "x = 1", + "print(x)", + "```", + ].join("\n"); + + const extracted = ( + client as unknown as { extractCode: (r: string) => string } + ).extractCode(response); + + expect(extracted).toBe("x = 1\nprint(x)"); + }); +}); + +describe("classifyCopilotError", () => { + it("classifies timeout and marks retryable", () => { + const classified = classifyCopilotError( + new Error("Timeout after 120000ms waiting for session.idle"), + ); + + expect(classified.kind).toBe("timeout"); + expect(classified.retryable).toBe(true); + }); + + it("classifies auth and marks non-retryable", () => { + const classified = classifyCopilotError( + new Error( + "Authentication failed: Failed to fetch GitHub CLI user login (401): Bad credentials", + ), + ); + + expect(classified.kind).toBe("auth"); + expect(classified.retryable).toBe(false); + }); + + it("returns existing classified errors unchanged", () => { + const original = new CopilotGenerationError("transient", "429", true); + const classified = classifyCopilotError(original); + + expect(classified).toBe(original); + }); +}); diff --git a/tests/harness/copilot-client.ts b/tests/harness/copilot-client.ts index 1e05bb6a..9f9cdb50 100644 --- a/tests/harness/copilot-client.ts +++ b/tests/harness/copilot-client.ts @@ -16,6 +16,73 @@ import type { import { DEFAULT_GENERATION_CONFIG } from "./types.js"; import { CopilotClient as SDKCopilotClient } from "@github/copilot-sdk"; +export type CopilotGenerationErrorKind = + | "timeout" + | "auth" + | "transient" + | "fatal"; + +export class CopilotGenerationError extends Error { + readonly kind: CopilotGenerationErrorKind; + readonly retryable: boolean; + + constructor( + kind: CopilotGenerationErrorKind, + message: string, + retryable: boolean, + ) { + super(message); + this.name = "CopilotGenerationError"; + this.kind = kind; + this.retryable = retryable; + } +} + +export function classifyCopilotError(error: unknown): CopilotGenerationError { + if (error instanceof CopilotGenerationError) { + return error; + } + + const message = error instanceof Error ? error.message : String(error); + const normalized = message.toLowerCase(); + + if ( + normalized.includes("authentication failed") || + normalized.includes("bad credentials") || + normalized.includes("failed to fetch github cli user login") || + normalized.includes("401") + ) { + return new CopilotGenerationError("auth", message, false); + } + + if ( + normalized.includes("timeout") || + normalized.includes("session.idle") || + normalized.includes("timed out") + ) { + return new CopilotGenerationError("timeout", message, true); + } + + if ( + normalized.includes("429") || + normalized.includes("rate limit") || + normalized.includes("econnreset") || + normalized.includes("etimedout") || + normalized.includes("enotfound") || + normalized.includes("socket hang up") || + normalized.includes("temporarily unavailable") || + normalized.includes("service unavailable") + ) { + return new CopilotGenerationError("transient", message, true); + } + + return new CopilotGenerationError("fatal", message, false); +} + +function sleep(ms: number): Promise { + return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); +} + // ============================================================================= // Mock Client // ============================================================================= @@ -85,6 +152,8 @@ export class MockCopilotClient implements CopilotClient { export class SkillCopilotClient implements CopilotClient { private static readonly SKILLS_DIR = ".github/skills"; private static readonly PLUGINS_DIR = ".github/plugins"; + private static readonly DEFAULT_TIMEOUT_MS = 120000; + private static readonly DEFAULT_MAX_RETRIES = 2; private basePath: string; private skillsDir: string; @@ -227,23 +296,28 @@ Generate only code. Follow the patterns from the skill documentation exactly. ): Promise { const startTime = Date.now(); const fullPrompt = this.buildPrompt(prompt, skillContext); + const timeoutMs = this.getTimeoutMs(); + const maxRetries = this.getMaxRetries(); - const client = new SDKCopilotClient(); - - let rawResponse = ""; - - try { - const session = await client.createSession({ - model: config.model, - }); + for (let attempt = 0; attempt <= maxRetries; attempt++) { + const client = new SDKCopilotClient(); + let session: unknown; try { - const response = await session.sendAndWait( - { prompt: fullPrompt }, - 120000, - ); - - rawResponse = response?.data?.content ?? ""; + session = await client.createSession({ + model: config.model, + }); + + const response = await ( + session as { + sendAndWait: ( + request: { prompt: string }, + timeout: number, + ) => Promise<{ data?: { content?: string } }>; + } + ).sendAndWait({ prompt: fullPrompt }, timeoutMs); + + const rawResponse = response?.data?.content ?? ""; const code = this.extractCode(rawResponse); return { @@ -255,20 +329,51 @@ Generate only code. Follow the patterns from the skill documentation exactly. durationMs: Date.now() - startTime, rawResponse, }; + } catch (error) { + const classified = classifyCopilotError(error); + if (!classified.retryable || attempt === maxRetries) { + throw classified; + } + + // Exponential backoff with jitter for transient SDK/network failures. + const backoffMs = Math.min(1000 * 2 ** attempt, 8000); + const jitterMs = Math.floor(Math.random() * 300); + await sleep(backoffMs + jitterMs); } finally { - const s = session as unknown as { + const s = session as { disconnect?: () => Promise; destroy?: () => Promise; }; - if (typeof s.disconnect === "function") { + if (s && typeof s.disconnect === "function") { await s.disconnect(); - } else if (typeof s.destroy === "function") { + } else if (s && typeof s.destroy === "function") { await s.destroy(); } + await client.stop(); } - } finally { - await client.stop(); } + + throw new CopilotGenerationError( + "fatal", + "Code generation failed after all retry attempts.", + false, + ); + } + + private getTimeoutMs(): number { + const raw = process.env["HARNESS_COPILOT_TIMEOUT_MS"]; + const parsed = raw ? Number.parseInt(raw, 10) : NaN; + return Number.isFinite(parsed) && parsed > 0 + ? parsed + : SkillCopilotClient.DEFAULT_TIMEOUT_MS; + } + + private getMaxRetries(): number { + const raw = process.env["HARNESS_COPILOT_MAX_RETRIES"]; + const parsed = raw ? Number.parseInt(raw, 10) : NaN; + return Number.isFinite(parsed) && parsed >= 0 + ? parsed + : SkillCopilotClient.DEFAULT_MAX_RETRIES; } /** @@ -295,9 +400,12 @@ Generate only code. Follow the patterns from the skill documentation exactly. const codeLines: string[] = []; let inCode = false; + const assignmentRegex = /^[A-Za-z_][A-Za-z0-9_]*\s*=\s*.+$/; + const callStartRegex = /^[A-Za-z_][A-Za-z0-9_.]*\s*\(.+$/; + for (const line of lines) { - // Heuristic: lines starting with import, def, class, or indented - if ( + const trimmed = line.trim(); + const isCodeLikeLine = line.startsWith("import ") || line.startsWith("from ") || line.startsWith("def ") || @@ -307,8 +415,12 @@ Generate only code. Follow the patterns from the skill documentation exactly. line.startsWith("let ") || line.startsWith("using ") || line.startsWith(" ") || - line.startsWith("\t") - ) { + line.startsWith("\t") || + assignmentRegex.test(trimmed) || + callStartRegex.test(trimmed); + + // Heuristic: lines starting with import, def, class, or indented + if (isCodeLikeLine) { inCode = true; codeLines.push(line); } else if (inCode && line.trim() === "") { diff --git a/tests/harness/ralph-loop.test.ts b/tests/harness/ralph-loop.test.ts index c322be6e..019c1b42 100644 --- a/tests/harness/ralph-loop.test.ts +++ b/tests/harness/ralph-loop.test.ts @@ -55,10 +55,11 @@ class MockClient implements CopilotClient { _prompt: string, _skillName: string, _config?: GenerationConfig, - _scenarioName?: string + _scenarioName?: string, ): Promise { this.callCount++; - const code = this.responses.get(this.callCount) ?? "# Default mock response\npass"; + const code = + this.responses.get(this.callCount) ?? "# Default mock response\npass"; return { code, prompt: _prompt, @@ -111,13 +112,15 @@ class MockEvaluator { skillName: "test-skill", scenario, generatedCode: code, + rawResponse: code, findings, matchedCorrect: [], matchedIncorrect: score < 70 ? ["some-section"] : [], score, passed: score >= 50, errorCount: findings.filter((f) => f.severity === Severity.ERROR).length, - warningCount: findings.filter((f) => f.severity === Severity.WARNING).length, + warningCount: findings.filter((f) => f.severity === Severity.WARNING) + .length, }; } } @@ -161,7 +164,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80 } + { qualityThreshold: 80 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -184,7 +187,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, improvementThreshold: 10 } + { qualityThreshold: 80, improvementThreshold: 10 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -197,18 +200,14 @@ describe("RalphLoopController", () => { }); it("should stop at max iterations if threshold not met", async () => { - mockClient.setResponses([ - "# Attempt 1", - "# Attempt 2", - "# Attempt 3", - ]); + mockClient.setResponses(["# Attempt 1", "# Attempt 2", "# Attempt 3"]); mockEvaluator.setScores([30, 40, 50]); // Never reaches 80 const controller = new RalphLoopController( criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { maxIterations: 3, qualityThreshold: 80, improvementThreshold: 5 } + { maxIterations: 3, qualityThreshold: 80, improvementThreshold: 5 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -229,7 +228,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { earlyStopOnPerfect: true } + { earlyStopOnPerfect: true }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -247,7 +246,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, improvementThreshold: 5 } + { qualityThreshold: 80, improvementThreshold: 5 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -265,7 +264,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, improvementThreshold: 5 } + { qualityThreshold: 80, improvementThreshold: 5 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -285,7 +284,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { maxIterations: 3, qualityThreshold: 80, improvementThreshold: 0 } + { maxIterations: 3, qualityThreshold: 80, improvementThreshold: 0 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -305,7 +304,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, improvementThreshold: 10 } + { qualityThreshold: 80, improvementThreshold: 10 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -334,7 +333,11 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, includeFeedback: true, improvementThreshold: 10 } + { + qualityThreshold: 80, + includeFeedback: true, + improvementThreshold: 10, + }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -353,7 +356,7 @@ describe("RalphLoopController", () => { const controller = new RalphLoopController( criteria, mockEvaluator as unknown as CodeEvaluator, - mockClient + mockClient, ); const result = await controller.run("Generate code", "test-scenario"); diff --git a/tests/harness/runner.ts b/tests/harness/runner.ts index f106cea9..704ac972 100644 --- a/tests/harness/runner.ts +++ b/tests/harness/runner.ts @@ -20,8 +20,17 @@ import type { GenerationConfig, Finding, } from "./types.js"; -import { DEFAULT_GENERATION_CONFIG, Severity, createFinding } from "./types.js"; -import { SkillCopilotClient, checkCopilotAvailable } from "./copilot-client.js"; +import { + DEFAULT_GENERATION_CONFIG, + Severity, + createEvaluationResult, + createFinding, +} from "./types.js"; +import { + CopilotGenerationError, + SkillCopilotClient, + checkCopilotAvailable, +} from "./copilot-client.js"; import { AcceptanceCriteriaLoader } from "./criteria-loader.js"; import { CodeEvaluator } from "./evaluator.js"; import { @@ -282,19 +291,41 @@ export class SkillEvaluationRunner { ); } - // Generate code - const genResult = await this.copilotClient.generate( - scenario.prompt, - skillName, - suite.config, - scenario.name, - ); + let evalResult: EvaluationResult; - // Evaluate - const evalResult = evaluator.evaluate(genResult.code, scenario.name); + try { + // Generate code + const genResult = await this.copilotClient.generate( + scenario.prompt, + skillName, + suite.config, + scenario.name, + ); - // Add scenario-specific checks - this.checkScenarioPatterns(evalResult, scenario, genResult.code); + // Evaluate + evalResult = evaluator.evaluate(genResult.code, scenario.name); + evalResult.rawResponse = genResult.rawResponse; + + // Add scenario-specific checks + this.checkScenarioPatterns(evalResult, scenario, genResult.code); + } catch (error) { + const kind = + error instanceof CopilotGenerationError ? error.kind : "fatal"; + const message = error instanceof Error ? error.message : String(error); + + evalResult = createEvaluationResult(skillName, scenario.name, ""); + evalResult.passed = false; + evalResult.errorCount = 1; + evalResult.score = 0; + evalResult.findings.push( + createFinding({ + severity: Severity.ERROR, + rule: `runtime:${kind}`, + message: `Code generation failed (${kind}): ${message}`, + suggestion: this.getGenerationFailureSuggestion(kind), + }), + ); + } results.push(evalResult); @@ -394,6 +425,21 @@ export class SkillEvaluationRunner { } } + private getGenerationFailureSuggestion( + kind: "timeout" | "auth" | "transient" | "fatal", + ): string { + switch (kind) { + case "timeout": + return "Retry with HARNESS_COPILOT_TIMEOUT_MS set higher (for example 240000), or re-run this scenario."; + case "auth": + return "Refresh GitHub authentication (gh auth login) or set GH_TOKEN/GITHUB_TOKEN with valid Copilot access."; + case "transient": + return "This looks transient; retry the run. If it repeats, verify network stability and API availability."; + default: + return "Inspect diagnostics and rerun with --verbose for additional context."; + } + } + private printFinding(finding: Finding): void { const severityStyle = this.getSeverityStyle(finding.severity); const severityLabel = finding.severity.toUpperCase(); @@ -621,6 +667,7 @@ function summaryToDict(summary: EvaluationSummary): Record { skill_name: r.skillName, scenario: r.scenario, generated_code: r.generatedCode, + raw_response: r.rawResponse, passed: r.passed, score: r.score, error_count: r.errorCount, @@ -770,6 +817,7 @@ function convertRalphToSummary(ralph: RalphLoopSummary): EvaluationSummary { skillName: ralph.skillName, scenario: scenarioName, generatedCode: lastIteration.generatedCode, + rawResponse: "", findings: lastIteration.findings, matchedCorrect: [], matchedIncorrect: [], @@ -813,6 +861,8 @@ interface CLIOptions { threshold?: number; } +type CliErrorKind = "timeout" | "auth" | "transient" | "fatal"; + interface AllSkillsSummary { totalSkills: number; passedSkills: number; @@ -826,6 +876,52 @@ interface AllSkillsSummary { skills: EvaluationSummary[]; } +function classifyCliError(error: unknown): CliErrorKind { + if (error instanceof CopilotGenerationError) { + return error.kind; + } + + const message = error instanceof Error ? error.message.toLowerCase() : ""; + if ( + message.includes("authentication failed") || + message.includes("bad credentials") || + message.includes("401") + ) { + return "auth"; + } + if (message.includes("timeout") || message.includes("session.idle")) { + return "timeout"; + } + if (message.includes("rate limit") || message.includes("429")) { + return "transient"; + } + return "fatal"; +} + +function writeCliErrorOutput( + options: CLIOptions, + skillName: string | undefined, + error: unknown, +): void { + const message = error instanceof Error ? error.message : String(error); + const payload = { + status: "ERROR", + error_kind: classifyCliError(error), + error: message, + skill: skillName ?? null, + timestamp: new Date().toISOString(), + }; + const output = JSON.stringify(payload, null, 2); + + if (options.outputFile) { + writeFileSync(options.outputFile, output); + console.log(`Results written to: ${options.outputFile}`); + return; + } + + console.log(output); +} + async function main(): Promise { const program = new Command(); @@ -1087,6 +1183,10 @@ async function main(): Promise { } } catch (err) { const message = err instanceof Error ? err.message : String(err); + if (options.output === "json") { + writeCliErrorOutput(options, skillArg, err); + return 1; + } console.log(chalk.red(`Error: ${message}`)); return 1; } diff --git a/tests/harness/types.ts b/tests/harness/types.ts index 715fb494..37e5b525 100644 --- a/tests/harness/types.ts +++ b/tests/harness/types.ts @@ -83,6 +83,7 @@ export interface EvaluationResult { skillName: string; scenario: string; generatedCode: string; + rawResponse: string; findings: Finding[]; matchedCorrect: string[]; matchedIncorrect: string[]; @@ -110,7 +111,7 @@ export interface GenerationConfig { * Default generation configuration. */ export const DEFAULT_GENERATION_CONFIG: GenerationConfig = { - model: "gpt-4", + model: "gpt-5.5", maxTokens: 2000, temperature: 0.3, includeSkillContext: true, @@ -175,7 +176,7 @@ export interface CopilotClient { prompt: string, skillName: string, config?: GenerationConfig, - scenarioName?: string + scenarioName?: string, ): Promise; } @@ -223,12 +224,13 @@ export function detectLanguage(skillName: string): Language { export function createEvaluationResult( skillName: string, scenario: string, - generatedCode: string + generatedCode: string, ): EvaluationResult { return { skillName, scenario, generatedCode, + rawResponse: "", findings: [], matchedCorrect: [], matchedIncorrect: [], @@ -242,7 +244,9 @@ export function createEvaluationResult( /** * Create an empty code pattern. */ -export function createCodePattern(partial: Partial = {}): CodePattern { +export function createCodePattern( + partial: Partial = {}, +): CodePattern { return { code: "", language: "python", @@ -256,7 +260,9 @@ export function createCodePattern(partial: Partial = {}): CodePatte /** * Create an empty validation rule. */ -export function createValidationRule(partial: Partial = {}): ValidationRule { +export function createValidationRule( + partial: Partial = {}, +): ValidationRule { return { name: "", description: "", @@ -274,7 +280,7 @@ export function createValidationRule(partial: Partial = {}): Val * Create an empty acceptance criteria. */ export function createAcceptanceCriteria( - partial: Partial = {} + partial: Partial = {}, ): AcceptanceCriteria { return { skillName: "",