From eb072743da523d4530c3779b5bfb055f515c6128 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 1 Aug 2026 20:49:57 -0700 Subject: [PATCH 1/2] chore(release): @commonlyai/cli 0.1.8 (spawn-retry circuit breaker, #794) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #794 merged the circuit breaker but the published 0.1.7 predates it, so the fleet currently runs without any bound on spawn retries — a provider outage still produces one model launch per queued event. Verified on the merged code: a sustained 3-hour outage now costs 15 spawns for rate-limit (was 180 before the escalation fix, and 40 in the incident that motivated the issue), in line with runtime 17 / quota 12 / configuration 12. Co-Authored-By: Claude Opus 4.8 --- cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/package.json b/cli/package.json index b0f3d40e..ea52e2a3 100644 --- a/cli/package.json +++ b/cli/package.json @@ -1,6 +1,6 @@ { "name": "@commonlyai/cli", - "version": "0.1.7", + "version": "0.1.8", "license": "Apache-2.0", "description": "The Commonly CLI — connect agents, manage pods, iterate fast", "type": "module", From 2faef348dec3b300001e4435d950a4a7e4342ce5 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Tue, 4 Aug 2026 00:18:15 -0700 Subject: [PATCH 2/2] fix(cli): classify real provider-exhaustion failures before 0.1.8 ships MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tested the merged circuit breaker (#794) against the failure strings from today's fleet outage. Both of them classified as RUNTIME — the weakest class, shortest backoff — so the breaker would have bounded the storm but treated a hard quota failure as a transient one. Two causes, both fixed here: - QUOTA_RE did not match codex's wording, "Your workspace is out of credits." - runClaude rejected with stderr only. claude reports terminal conditions on stdout in -p mode and exits non-zero with stderr empty, so all 361 claude failures today carried no reason at all. That is a diagnosability problem and a classification one: the breaker reads the error message, so a blank message cannot be classified. Folded into the release PR rather than filed separately — 0.1.8 is unpublished, so this costs nothing and avoids shipping a breaker that mis-rates the exact outage that motivated it. Both fixes are mutation-verified: reverting either fails exactly the tests that cover it, and only those. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016s8YysVUNmmiNXw3cTrZFJ --- cli/__tests__/adapters.claude.test.mjs | 46 ++++++++++++++++++++++++++ cli/__tests__/spawn-retry.test.mjs | 46 ++++++++++++++++++++++++++ cli/src/lib/adapters/claude.js | 11 +++++- cli/src/lib/spawn-retry.js | 6 +++- 4 files changed, 107 insertions(+), 2 deletions(-) diff --git a/cli/__tests__/adapters.claude.test.mjs b/cli/__tests__/adapters.claude.test.mjs index 6fc899e3..3b6ddb10 100644 --- a/cli/__tests__/adapters.claude.test.mjs +++ b/cli/__tests__/adapters.claude.test.mjs @@ -201,3 +201,49 @@ describe('claude adapter — spawn()', () => { expect(proc.kill).toHaveBeenCalledWith('SIGTERM'); }); }); + +/** + * Regression for the 2026-08-03 outage: 361 consecutive spawn failures across + * three agents, every one reported as `claude exited with code 1: ` with an + * empty reason. claude writes terminal conditions — usage limits above all — + * to stdout in `-p` mode and exits non-zero with stderr empty, and the adapter + * discarded stdout on the failure path. + * + * This is not only about readable logs. The circuit breaker classifies from + * the error message (see spawn-retry.test.mjs), so a blank message downgrades + * a hard quota failure to RUNTIME and gives it the shortest backoff — the + * fleet then retries into an exhausted quota instead of standing down. + */ +describe('claude adapter — failure reporting', () => { + test('surfaces a stdout-only failure reason instead of an empty message', async () => { + const { impl } = makeSpawnImpl({ + stdout: 'Claude usage limit reached. Your limit will reset at 11:40pm.', + stderr: '', + code: 1, + }); + + await expect(claude.spawn('hi', { sessionId: null, _spawnImpl: impl })) + .rejects.toThrow(/usage limit reached/i); + }); + + test('still prefers stderr, and includes both when both are present', async () => { + const { impl } = makeSpawnImpl({ stdout: 'partial output', stderr: 'boom', code: 1 }); + + await expect(claude.spawn('hi', { sessionId: null, _spawnImpl: impl })) + .rejects.toThrow(/boom \| partial output/); + }); + + test('bounds the reported detail so a huge stdout cannot flood the log', async () => { + const { impl } = makeSpawnImpl({ stdout: 'x'.repeat(50_000), stderr: '', code: 1 }); + + const err = await claude.spawn('hi', { sessionId: null, _spawnImpl: impl }).catch((e) => e); + expect(err.message.length).toBeLessThan(2_100); + }); + + test('a successful spawn is unaffected — stdout is still the return value', async () => { + const { impl } = makeSpawnImpl({ stdout: 'the answer', code: 0 }); + + const res = await claude.spawn('hi', { sessionId: null, _spawnImpl: impl }); + expect(res.text).toBe('the answer'); + }); +}); diff --git a/cli/__tests__/spawn-retry.test.mjs b/cli/__tests__/spawn-retry.test.mjs index 69969723..691aa07b 100644 --- a/cli/__tests__/spawn-retry.test.mjs +++ b/cli/__tests__/spawn-retry.test.mjs @@ -93,3 +93,49 @@ describe('spawn retry policy', () => { expect(attempts).toBe(15); }); }); + +/** + * Regression: the classifier is only as good as the failure strings it has + * actually seen. Both cases below are verbatim from the 2026-08-03 fleet + * outage, where every one of them classified as RUNTIME — the weakest class, + * with the shortest backoff — instead of QUOTA. + */ +describe('classifies real provider-exhaustion strings (2026-08-03 outage)', () => { + test("codex's exact out-of-credits wording is QUOTA, not RUNTIME", () => { + const error = new Error( + 'codex turn failed: Your workspace is out of credits. ' + + 'Ask your workspace owner to refill in order to continue.', + ); + expect(classifySpawnFailure(error)).toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + test('a QUOTA verdict opens the circuit on the very first failure', () => { + // The point of classifying it correctly: RUNTIME grants two free retries + // before tripping, QUOTA trips immediately at the full cooldown. + const error = new Error('Your workspace is out of credits.'); + const { circuitOpen, delayMs, failureClass } = spawnRetryPolicy({ + error, + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(failureClass).toBe(SPAWN_FAILURE_CLASS.QUOTA); + expect(circuitOpen).toBe(true); + expect(delayMs).toBe(SPAWN_RETRY_MAX_MS); + }); + + test('a claude usage-limit message reported on stdout is QUOTA once surfaced', () => { + // The adapter used to drop stdout on failure, so this arrived as the empty + // string and was unclassifiable. See adapters.claude.test.mjs. + const error = new Error( + 'claude exited with code 1: Claude usage limit reached. ' + + 'Your limit will reset at 11:40pm.', + ); + expect(classifySpawnFailure(error)).toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + test('an empty failure message still classifies, as RUNTIME', () => { + // What every one of the 361 claude failures looked like before the fix. + expect(classifySpawnFailure(new Error('claude exited with code 1: '))) + .toBe(SPAWN_FAILURE_CLASS.RUNTIME); + }); +}); diff --git a/cli/src/lib/adapters/claude.js b/cli/src/lib/adapters/claude.js index da046891..f8e68c7c 100644 --- a/cli/src/lib/adapters/claude.js +++ b/cli/src/lib/adapters/claude.js @@ -244,7 +244,16 @@ const runClaude = ({ cmd, args, cwd, env, timeoutMs, spawnImpl = childSpawn }) = proc.on('close', (code) => { clearTimeout(timer); if (timedOut) return reject(new Error(`claude timed out after ${timeoutMs}ms`)); - if (code !== 0) return reject(new Error(`claude exited with code ${code}: ${stderr.trim()}`)); + if (code !== 0) { + // Report stdout too, not just stderr. In `-p` mode claude writes terminal + // conditions (usage limits especially) to stdout and exits non-zero with + // stderr empty — 361 consecutive failures on 2026-08-03 carried no reason + // at all because of this. It is not only a diagnosability problem: the + // circuit breaker classifies from the error message, so a blank message + // downgrades a hard quota failure to RUNTIME and its shortest backoff. + const detail = [stderr.trim(), stdout.trim()].filter(Boolean).join(' | '); + return reject(new Error(`claude exited with code ${code}: ${detail.slice(0, 2000)}`)); + } resolve(stdout); }); }); diff --git a/cli/src/lib/spawn-retry.js b/cli/src/lib/spawn-retry.js index a13b4c8d..ba44a245 100644 --- a/cli/src/lib/spawn-retry.js +++ b/cli/src/lib/spawn-retry.js @@ -19,7 +19,11 @@ export const SPAWN_CIRCUIT_THRESHOLD = 3; export const SPAWN_RETRY_MAX_MS = 15 * 60 * 1000; export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2; -const QUOTA_RE = /(?:quota|usage limit|credit balance|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i; +// `out of credits` is codex's exact wording for an exhausted workspace balance +// ("Your workspace is out of credits. Ask your workspace owner to refill…"). +// Without it that outage classified as RUNTIME and drew the shortest backoff — +// observed live on 2026-08-03 before this pattern was added. +const QUOTA_RE = /(?:quota|usage limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i; const RATE_LIMIT_RE = /(?:rate[ -]?limit|too many requests|\b429\b|overloaded|capacity)/i; const CONFIGURATION_RE = /(?:ENOENT|command not found|not on PATH|login required|not logged in|invalid api key|authentication failed|unauthori[sz]ed|forbidden|\b40[13]\b)/i;