From 2fde0c3e25134e0ffd70cb709901d3ca0c5fa686 Mon Sep 17 00:00:00 2001 From: luoye520ww <100058663+luoye520ww@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:47:14 +0800 Subject: [PATCH] test(kun): align LLM debug recorder trace attempts --- kun/tests/llm-debug-recorder.test.ts | 69 +++++++++++++++++++--------- kun/tests/runtime-factory.test.ts | 59 +++++++++++++++++++++++- 2 files changed, 105 insertions(+), 23 deletions(-) diff --git a/kun/tests/llm-debug-recorder.test.ts b/kun/tests/llm-debug-recorder.test.ts index 2baccfe24..cc3e90cc4 100644 --- a/kun/tests/llm-debug-recorder.test.ts +++ b/kun/tests/llm-debug-recorder.test.ts @@ -1,58 +1,83 @@ import { describe, expect, it } from 'vitest' import { LlmDebugRecorder } from '../src/services/llm-debug-recorder.js' -function record(recorder: LlmDebugRecorder, model: string): void { +function beginRequestAttempt( + recorder: LlmDebugRecorder, + round: ReturnType, + body: Record +): void { + recorder.beginHttpAttempt(round, { + endpointFormat: 'chat_completions', + attempt: 1, + reason: 'initial', + url: 'https://example.test/v1/chat/completions', + headers: {}, + bodyText: JSON.stringify(body) + }) +} + +async function record(recorder: LlmDebugRecorder, model: string): Promise { const round = recorder.start({ threadId: 't', turnId: 'u', provider: 'compat', model }) - recorder.captureRequest(round, { model }, 'https://example.test/v1/chat/completions') + beginRequestAttempt(recorder, round, { model }) recorder.captureChunk(round, { kind: 'assistant_text_delta', text: `out:${model}` }) - recorder.finish(round) + await recorder.finish(round) } describe('LlmDebugRecorder', () => { - it('keeps only the most recent 25 rounds', () => { + it('keeps only the most recent 25 rounds', async () => { const recorder = new LlmDebugRecorder() - for (let i = 1; i <= 30; i++) record(recorder, `m${i}`) + for (let i = 1; i <= 30; i++) await record(recorder, `m${i}`) const snapshot = recorder.snapshot() expect(snapshot).toHaveLength(25) // Oldest five (m1..m5) dropped; m6 is the oldest retained. expect(snapshot[snapshot.length - 1]?.model).toBe('m6') }) - it('returns the snapshot most-recent first', () => { + it('returns the snapshot most-recent first', async () => { const recorder = new LlmDebugRecorder() - record(recorder, 'a') - record(recorder, 'b') + await record(recorder, 'a') + await record(recorder, 'b') const snapshot = recorder.snapshot() expect(snapshot.map((r) => r.model)).toEqual(['b', 'a']) expect(snapshot[0]?.requestBody).toEqual({ model: 'b' }) + expect(snapshot[0]?.exchanges[0]).toMatchObject({ + endpointFormat: 'chat_completions', + attempt: 1, + attemptReason: 'initial', + request: { body: { text: JSON.stringify({ model: 'b' }) } } + }) expect(snapshot[0]?.output.text).toBe('out:b') }) - it('clear empties the buffer', () => { + it('clear empties the buffer', async () => { const recorder = new LlmDebugRecorder() - record(recorder, 'a') + await record(recorder, 'a') recorder.clear() expect(recorder.snapshot()).toHaveLength(0) }) - it('retains only a bounded prefix of oversized request bodies', () => { + it('retains only a bounded prefix of oversized request bodies', async () => { const recorder = new LlmDebugRecorder({ maxRequestBodyBytes: 96, maxRoundBytes: 1_024, maxTotalBytes: 4_096 }) const round = recorder.start({ threadId: 't', turnId: 'u', provider: 'compat', model: 'm' }) - recorder.captureRequest(round, { prompt: 'šŸ’”'.repeat(1_000) }, 'https://example.test/v1/chat/completions') - recorder.finish(round) + beginRequestAttempt(recorder, round, { prompt: 'šŸ’”'.repeat(1_000) }) + await recorder.finish(round) const captured = recorder.snapshot()[0] + const request = captured?.exchanges[0]?.request.body expect(captured?.requestBodyTruncated).toBe(true) expect(captured?.requestBodyOriginalBytes).toBeGreaterThan(96) expect(captured?.requestBody).toMatchObject({ __debugTruncated: true }) - expect(Buffer.byteLength(JSON.stringify(captured?.requestBody), 'utf8')).toBeLessThanOrEqual(96) + expect(request).toMatchObject({ truncated: true }) + expect(request?.originalBytes).toBeGreaterThan(96) + expect(request?.capturedBytes).toBeLessThanOrEqual(96) + expect(Buffer.byteLength(request?.text ?? '', 'utf8')).toBeLessThanOrEqual(96) }) - it('bounds streamed output bytes without repeatedly joining prior chunks', () => { + it('bounds streamed output bytes without repeatedly joining prior chunks', async () => { const recorder = new LlmDebugRecorder({ maxRequestBodyBytes: 64, maxRoundBytes: 128, @@ -60,11 +85,11 @@ describe('LlmDebugRecorder', () => { }) const round = recorder.start({ threadId: 't', turnId: 'u', provider: 'compat', model: 'm' }) expect(recorder.activeCaptureCount).toBe(1) - recorder.captureRequest(round, { model: 'm' }, 'https://example.test/v1/chat/completions') + beginRequestAttempt(recorder, round, { model: 'm' }) for (let index = 0; index < 100; index += 1) { recorder.captureChunk(round, { kind: 'assistant_text_delta', text: '"\\\nšŸ’”'.repeat(10) }) } - recorder.finish(round) + await recorder.finish(round) expect(recorder.activeCaptureCount).toBe(0) const captured = recorder.snapshot()[0] @@ -74,23 +99,23 @@ describe('LlmDebugRecorder', () => { expect(captured?.output.text).not.toContain('\ufffd') }) - it('evicts old rounds when the global byte budget is exhausted', () => { + it('evicts old rounds when the global byte budget is exhausted', async () => { const recorder = new LlmDebugRecorder({ capacity: 25, maxRequestBodyBytes: 64, maxRoundBytes: 512, - maxTotalBytes: 1_000 + maxTotalBytes: 2_000 }) for (const model of ['a', 'b', 'c']) { const round = recorder.start({ threadId: 't', turnId: model, provider: 'compat', model }) - recorder.captureRequest(round, { model }, 'https://example.test/v1/chat/completions') + beginRequestAttempt(recorder, round, { model }) recorder.captureChunk(round, { kind: 'assistant_text_delta', text: model.repeat(250) }) - recorder.finish(round) + await recorder.finish(round) } const snapshot = recorder.snapshot() expect(snapshot.length).toBeLessThan(3) expect(snapshot[0]?.model).toBe('c') - expect(snapshot.reduce((total, round) => total + (round.retainedBytes ?? 0), 0)).toBeLessThanOrEqual(1_000) + expect(snapshot.reduce((total, round) => total + (round.retainedBytes ?? 0), 0)).toBeLessThanOrEqual(2_000) }) }) diff --git a/kun/tests/runtime-factory.test.ts b/kun/tests/runtime-factory.test.ts index 1996ba5ae..810069090 100644 --- a/kun/tests/runtime-factory.test.ts +++ b/kun/tests/runtime-factory.test.ts @@ -126,7 +126,7 @@ describe('runtime factory usage carryover', () => { }) try { - expect(runtime.llmDebug).toBeUndefined() + expect(runtime.llmDebug).toBeDefined() expect(runtime.extensionPlatform).toBeDefined() expect(runtime.info().extensions).toMatchObject({ enabled: true, @@ -189,6 +189,63 @@ describe('runtime factory usage carryover', () => { } }) + it('keeps Agent Perspective capture available when runtime llmDebug is omitted or disabled', async () => { + for (const [name, runtimeOptions] of [ + ['omitted', undefined], + ['disabled', { llmDebug: { enabled: false } }] + ] as const) { + const dataDir = await mkdtemp(join(tmpdir(), `kun-runtime-llm-debug-${name}-`)) + tempDirs.push(dataDir) + const runtime = await createKunServeRuntime({ + host: '127.0.0.1', + port: 0, + dataDir, + runtimeToken: 'tok', + apiKey: 'sk-default', + baseUrl: 'https://api.example.test/v1', + model: 'model-before', + approvalPolicy: 'auto', + sandboxMode: 'danger-full-access', + tokenEconomyMode: false, + insecure: false, + storage: { backend: 'file' }, + ...(runtimeOptions ? { runtime: runtimeOptions } : {}), + capabilities: KunCapabilitiesConfig.parse({}) + }) + + try { + const recorder = runtime.llmDebug + expect(recorder).toBeDefined() + if (!recorder) throw new Error('expected Agent Perspective recorder') + const round = recorder.start({ + threadId: `thread-${name}`, + turnId: 'turn-1', + provider: 'compat', + model: 'model-before' + }) + recorder.beginHttpAttempt(round, { + endpointFormat: 'chat_completions', + attempt: 1, + reason: 'initial', + url: 'https://api.example.test/v1/chat/completions', + headers: {}, + bodyText: JSON.stringify({ model: 'model-before' }) + }) + await recorder.finish(round) + await expect(recorder.listThread(`thread-${name}`)).resolves.toMatchObject({ + records: [expect.objectContaining({ + provider: 'compat', + model: 'model-before', + attempt: 1, + endpointFormat: 'chat_completions' + })] + }) + } finally { + await runtime.shutdown?.() + } + } + }) + it('clears per-thread runtime memory when a thread is deleted', async () => { const dataDir = await mkdtemp(join(tmpdir(), 'kun-runtime-delete-')) tempDirs.push(dataDir)