feat(eve): send images to the model via toModelOutput content parts

.changeset/tool-model-output-content-parts.md

@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+`toModelOutput` can now return `{ type: "content", value }` with text and file parts, so a tool can send images (screenshots, rendered charts) to vision-capable models as actual pixels instead of descriptions. Build outputs with the new `toolOutput.text` / `toolOutput.json` / `toolOutput.content` helpers and parts with `toolOutputPart.text` / `toolOutputPart.file`, all from `eve/tools`; file payloads must be base64 strings.

docs/tools/overview.mdx

@@ -82,6 +82,33 @@ toModelOutput(output) {
 
 Tool outputs must be JSON-serializable. Return plain objects, arrays, strings, numbers, booleans, or `null`; convert values like `Date`, `Map`, `Set`, `NaN`, and cyclic objects before returning them from `execute` or from a `{ type: "json" }` `toModelOutput`.
 
+### Send images to the model with content parts
+
+A tool that produces an image — a screenshot, a rendered chart — can hand the pixels to a vision-capable model by returning a `content` output from `toModelOutput`. Build outputs with the `toolOutput` helpers and parts with the `toolOutputPart` helpers, both from `eve/tools`:
+
+```ts
+import { defineTool, toolOutput, toolOutputPart } from "eve/tools";
+
+export default defineTool({
+  description: "Capture a screenshot of the current page",
+  inputSchema: z.object({ url: z.string() }),
+  async execute(input) {
+    const png = await captureScreenshot(input.url);
+    return { path: png.path, screenshotBase64: png.base64 };
+  },
+  toModelOutput(output) {
+    return toolOutput.content([
+      toolOutputPart.text(`Screenshot of ${output.path}:`),
+      toolOutputPart.file(output.screenshotBase64, { mediaType: "image/png" }),
+    ]);
+  },
+});
+```
+
+The `toolOutput.text` and `toolOutput.json` builders construct the other two output shapes; hand-written literals remain valid everywhere.
+
+File payloads must be base64 strings — raw bytes (`Uint8Array`, `Buffer`) are rejected because they do not survive eve's durable JSON boundary. Keep payloads small: a content-part image is persisted in session history and re-sent on every subsequent model call, and eve warns above 3 MiB. Sending image parts to a model without vision support fails with that provider's error, the same as image parts in user messages.
+
 Do not return secrets, credentials, unnecessary personal data, or unbounded sensitive content from tools. Filter, minimize, and redact tool outputs before returning them.
 
 ## What to read next

e2e/fixtures/agent-tools/agent/tools/render-stripes.ts

@@ -0,0 +1,79 @@
+import { crc32, deflateSync } from "node:zlib";
+import { defineTool, toolOutput, toolOutputPart } from "eve/tools";
+import { z } from "zod";
+
+// Single-token names no model paraphrases (unlike cyan/teal or purple/violet),
+// so the eval can match the reply against the rendered sequence verbatim.
+const PALETTE = {
+  black: [0, 0, 0],
+  blue: [0, 0, 255],
+  green: [0, 160, 0],
+  orange: [255, 140, 0],
+  red: [255, 0, 0],
+  yellow: [255, 220, 0],
+} as const;
+
+type ColorName = keyof typeof PALETTE;
+
+const WIDTH = 240;
+const HEIGHT = 120;
+const STRIPE_COUNT = 3;
+
+function pngChunk(type: string, data: Buffer): Buffer {
+  const length = Buffer.alloc(4);
+  length.writeUInt32BE(data.length);
+  const typeBuffer = Buffer.from(type, "latin1");
+  const crc = Buffer.alloc(4);
+  crc.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data])) >>> 0);
+  return Buffer.concat([length, typeBuffer, data, crc]);
+}
+
+function renderStripesPng(stripes: readonly ColorName[]): Buffer {
+  const ihdr = Buffer.alloc(13);
+  ihdr.writeUInt32BE(WIDTH, 0);
+  ihdr.writeUInt32BE(HEIGHT, 4);
+  ihdr[8] = 8; // bit depth
+  ihdr[9] = 2; // truecolor
+
+  const stripeWidth = WIDTH / stripes.length;
+  const row = Buffer.alloc(1 + WIDTH * 3); // leading scanline filter byte 0
+  for (let x = 0; x < WIDTH; x += 1) {
+    const stripe = Math.min(Math.floor(x / stripeWidth), stripes.length - 1);
+    const color = PALETTE[stripes[stripe]!];
+    row.set(color, 1 + x * 3);
+  }
+  const idat = deflateSync(Buffer.concat(Array.from({ length: HEIGHT }, () => row)), { level: 9 });
+
+  return Buffer.concat([
+    Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
+    pngChunk("IHDR", ihdr),
+    pngChunk("IDAT", idat),
+    pngChunk("IEND", Buffer.alloc(0)),
+  ]);
+}
+
+export default defineTool({
+  description:
+    "Smoke-test fixture: renders an image of colored vertical stripes chosen at random. " +
+    "Only call when the user explicitly asks to use `render-stripes`. The colors appear " +
+    "ONLY in the returned image — inspect it visually; do not guess.",
+  inputSchema: z.object({}),
+  async execute() {
+    const names = Object.keys(PALETTE) as ColorName[];
+    const colors = [...names].sort(() => Math.random() - 0.5).slice(0, STRIPE_COUNT);
+    const png = renderStripesPng(colors);
+    // `colors` is the eval's answer key. It reaches action.result (and the
+    // eval's event stream) but never the model: the projection below sends
+    // only the pixels.
+    return { colors, imageBase64: png.toString("base64") };
+  },
+  toModelOutput(output) {
+    return toolOutput.content([
+      toolOutputPart.text("Rendered stripes:"),
+      toolOutputPart.file(output.imageBase64, {
+        filename: "stripes.png",
+        mediaType: "image/png",
+      }),
+    ]);
+  },
+});

e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts

@@ -0,0 +1,82 @@
+import type { HandleMessageStreamEvent } from "eve/client";
+import { defineEval } from "eve/evals";
+
+const TOOL_NAME = "render-stripes";
+
+// The stripe colors are randomized per run, so a blind model cannot pass by
+// guessing; the eval stays self-contained by validating the reply against
+// the answer key the tool records on action.result. The pixels reach the
+// model exclusively through `toModelOutput` content parts.
+export default defineEval({
+  description: "Static tools smoke: toModelOutput content parts deliver an image to the model.",
+  async test(t) {
+    await t.send(
+      `Call \`${TOOL_NAME}\` exactly once, look at the rendered image, and reply with only ` +
+        "the stripe colors left to right, comma-separated.",
+    );
+
+    t.succeeded();
+    t.noFailedActions();
+    t.calledTool(TOOL_NAME, { count: 1, output: isRenderStripesOutput });
+    t.eventsSatisfy("a reply names the rendered colors in order", (events) => {
+      const answer = assistantAnswers(events)[0];
+      return answer !== undefined && namesColorsInOrder(events, answer);
+    });
+
+    // The content part is baked into persisted history, so a follow-up turn
+    // must answer from replay without re-running the tool.
+    await t.send(
+      "Without calling any tool, repeat the stripe colors left to right, comma-separated.",
+    );
+
+    t.succeeded();
+    t.calledTool(TOOL_NAME, { count: 1 });
+    t.eventsSatisfy("the replayed image still answers the follow-up", (events) => {
+      const answer = assistantAnswers(events).at(-1);
+      return answer !== undefined && namesColorsInOrder(events, answer);
+    });
+  },
+});
+
+function isRenderStripesOutput(value: unknown): boolean {
+  if (typeof value !== "object" || value === null) return false;
+  const output = value as { readonly colors?: unknown; readonly imageBase64?: unknown };
+  return (
+    Array.isArray(output.colors) &&
+    output.colors.length > 0 &&
+    output.colors.every((color) => typeof color === "string") &&
+    typeof output.imageBase64 === "string" &&
+    output.imageBase64.startsWith("iVBOR") // PNG magic bytes, base64-encoded
+  );
+}
+
+function renderedColors(events: readonly HandleMessageStreamEvent[]): readonly string[] {
+  for (const event of events) {
+    if (event.type !== "action.result" || event.data.result.kind !== "tool-result") continue;
+    if (event.data.result.toolName !== TOOL_NAME) continue;
+    const output = event.data.result.output as { readonly colors?: unknown };
+    if (Array.isArray(output?.colors) && output.colors.every((c) => typeof c === "string")) {
+      return output.colors as string[];
+    }
+  }
+  return [];
+}
+
+/** Final (non-tool-call) assistant messages, in turn order. */
+function assistantAnswers(events: readonly HandleMessageStreamEvent[]): readonly string[] {
+  return events.flatMap((event) =>
+    event.type === "message.completed" &&
+    event.data.finishReason !== "tool-calls" &&
+    event.data.message !== null &&
+    event.data.message.trim().length > 0
+      ? [event.data.message]
+      : [],
+  );
+}
+
+function namesColorsInOrder(events: readonly HandleMessageStreamEvent[], answer: string): boolean {
+  const colors = renderedColors(events);
+  if (colors.length === 0) return false;
+  const pattern = new RegExp(colors.map((color) => `\\b${color}\\b`).join("[\\s\\S]*"), "iu");
+  return pattern.test(answer);
+}

packages/eve/extension-contracts/compatibility/dynamicTool/v3.ts

@@ -0,0 +1,26 @@
+import { defineDynamic, defineTool } from "#public/tools/index.js";
+
+export default defineDynamic({
+  events: {
+    "session.started": (_event, ctx) => ({
+      summarize_report: defineTool({
+        description: "Summarize a report for the model",
+        inputSchema: { type: "object", properties: {} },
+        async execute(_input, toolContext) {
+          return {
+            internal: "details",
+            resolverSessionId: ctx.session.id,
+            sessionId: toolContext.session.id,
+            summary: "Report generated",
+          };
+        },
+        toModelOutput(output) {
+          return {
+            type: "text",
+            value: (output as { summary: string }).summary,
+          };
+        },
+      }),
+    }),
+  },
+});

packages/eve/extension-contracts/compatibility/tool/v2.ts

@@ -0,0 +1,19 @@
+import { defineTool } from "#public/tools/index.js";
+
+export default defineTool({
+  description: "Summarize a report for the model",
+  inputSchema: { type: "object", properties: {} },
+  async execute(_input, ctx) {
+    return {
+      internal: "details",
+      sessionId: ctx.session.id,
+      summary: "Report generated",
+    };
+  },
+  toModelOutput(output) {
+    return {
+      type: "text",
+      value: (output as { summary: string }).summary,
+    };
+  },
+});

packages/eve/extension-contracts/entrypoints/tool.ts

@@ -9,5 +9,7 @@ export {
   experimental_workflow,
   isDisabledToolSentinel,
   isExperimentalWorkflowToolDefinition,
+  toolOutput,
+  toolOutputPart,
   toolResultFrom,
 } from "../../src/public/tools/index.ts";

packages/eve/extension-contracts/reports/dynamicTool/v4.json

@@ -0,0 +1,13 @@
+{
+  "kind": "eve-extension-capability-contract",
+  "capability": "dynamicTool",
+  "epoch": 4,
+  "sha256": "d5ed3d2576c48abe76044aaf8a9aa0e8ab734d3fc6d4d3bc7be91c3a6648a198",
+  "exports": [
+    "DynamicToolEntry",
+    "DynamicToolEvents",
+    "DynamicToolResult",
+    "DynamicToolSet",
+    "defineDynamic"
+  ]
+}

packages/eve/extension-contracts/reports/tool/v3.json

@@ -0,0 +1,21 @@
+{
+  "kind": "eve-extension-capability-contract",
+  "capability": "tool",
+  "epoch": 3,
+  "sha256": "64e6354ca546aaa2bbd07a05b5fb66596d34d14633222a41e5c647898211e14f",
+  "exports": [
+    "defineBashTool",
+    "defineGlobTool",
+    "defineGrepTool",
+    "defineReadFileTool",
+    "defineTool",
+    "defineWriteFileTool",
+    "disableTool",
+    "experimental_workflow",
+    "isDisabledToolSentinel",
+    "isExperimentalWorkflowToolDefinition",
+    "toolOutput",
+    "toolOutputPart",
+    "toolResultFrom"
+  ]
+}

packages/eve/src/compiler/extension-compatibility.ts

@@ -21,8 +21,8 @@ interface ExtensionCapabilityContract {
 
 const EXTENSION_CAPABILITY_CONTRACTS = {
   extension: { current: 1, supported: [1], dropped: {} },
-  tool: { current: 2, supported: [1, 2], dropped: {} },
-  dynamicTool: { current: 3, supported: [1, 2, 3], dropped: {} },
+  tool: { current: 3, supported: [1, 2, 3], dropped: {} },
+  dynamicTool: { current: 4, supported: [1, 2, 3, 4], dropped: {} },
   connection: { current: 2, supported: [1, 2], dropped: {} },
   hook: { current: 2, supported: [1, 2], dropped: {} },
   skill: { current: 1, supported: [1], dropped: {} },

packages/eve/src/discover/agent.integration.test.ts

@@ -3,6 +3,10 @@ import { resolve } from "node:path";
 import { describe, expect, it } from "vitest";
 
 import { buildMemoryAgentProject } from "#internal/testing/memory-agent-source.js";
+import {
+  EXTENSION_CAPABILITY_SUPPORT,
+  EXTENSION_CAPABILITY_VERSIONS,
+} from "#compiler/extension-compatibility.js";
 import { discoverAgent } from "#discover/discover-agent.js";
 import {
   DISCOVER_EXTENSION_CAPABILITY_INCOMPATIBLE,
@@ -983,6 +987,9 @@ describe("discoverAgent (memory)", () => {
   });
 
   it("rejects a mounted extension that requires an unsupported capability version", async () => {
+    // One past the current epoch is unsupported by construction, so the
+    // fixture keeps rejecting after future capability bumps.
+    const unsupportedToolVersion = EXTENSION_CAPABILITY_VERSIONS.tool + 1;
     const project = buildMemoryAgentProject({
       appFiles: {
         "node_modules/@acme/crm/package.json": JSON.stringify({
@@ -994,7 +1001,7 @@ describe("discoverAgent (memory)", () => {
           kind: "eve-extension",
           formatVersion: 1,
           builtWithEve: "9.0.0",
-          requires: { extension: 1, tool: 3 },
+          requires: { extension: 1, tool: unsupportedToolVersion },
         }),
         "node_modules/@acme/crm/extension/extension.ts": "export default {};\n",
         "node_modules/@acme/crm/extension/tools/search.ts": "export default {};\n",
@@ -1015,8 +1022,10 @@ describe("discoverAgent (memory)", () => {
       (diagnostic) => diagnostic.code === DISCOVER_EXTENSION_CAPABILITY_INCOMPATIBLE,
     );
     expect(incompatible).toBeDefined();
-    expect(incompatible?.message).toContain("tool contract v3");
-    expect(incompatible?.message).toContain("versions: v1, v2");
+    expect(incompatible?.message).toContain(`tool contract v${unsupportedToolVersion}`);
+    expect(incompatible?.message).toContain(
+      `versions: ${EXTENSION_CAPABILITY_SUPPORT.tool.map((version) => `v${version}`).join(", ")}`,
+    );
     expect(result.manifest.resolvedExtensions).toEqual([]);
   });