From 6315706799859c2fcdf07b300466e187d454990f Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 10 Apr 2026 20:54:28 +0200 Subject: [PATCH 1/7] Create chat_parsing.test.js --- .../tests/utils/chat_parsing.test.js | 937 ++++++++++++++++++ 1 file changed, 937 insertions(+) create mode 100644 packages/transformers/tests/utils/chat_parsing.test.js diff --git a/packages/transformers/tests/utils/chat_parsing.test.js b/packages/transformers/tests/utils/chat_parsing.test.js new file mode 100644 index 000000000..c991d2dfb --- /dev/null +++ b/packages/transformers/tests/utils/chat_parsing.test.js @@ -0,0 +1,937 @@ +import { recursive_parse } from "../../src/utils/chat_parsing.js"; + +const cohere_schema = { + type: "object", + properties: { + role: { const: "assistant" }, + content: { type: "string", "x-regex": "<\\|START_RESPONSE\\|>(.*?)(?:<\\|END_RESPONSE\\|>|$)" }, + thinking: { type: "string", "x-regex": "<\\|START_THINKING\\|>(.*?)(?:<\\|END_THINKING\\|>|$)" }, + tool_calls: { + "x-regex": "<\\|START_ACTION\\|>(.*?)(?:<\\|END_ACTION\\|>|$)", + "x-parser": "json", + "x-parser-args": { + transform: "[*].{type: 'function', function: {name: tool_name, arguments: parameters}}", + }, + type: "array", + items: { + type: "object", + properties: { + type: { const: "function" }, + function: { + type: "object", + properties: { + name: { type: "string" }, + arguments: { type: "object", additionalProperties: {} }, + }, + }, + }, + }, + }, + }, +}; + +const ernie_schema = { + type: "object", + properties: { + role: { const: "assistant" }, + content: { type: "string", "x-regex": "\n(.*?)\n?" }, + thinking: { type: "string", "x-regex": "(?:^|\\s*)(.*?)\\s*<\\/think>" }, + tool_calls: { + "x-regex-iterator": "(.*?)", + type: "array", + items: { + type: "object", + "x-parser": "json", + "x-parser-args": { transform: "{type: 'function', function: @}" }, + properties: { + type: { const: "function" }, + function: { + type: "object", + properties: { + name: { type: "string" }, + arguments: { type: "object", additionalProperties: {} }, + }, + }, + }, + }, + }, + }, +}; + +const gpt_oss_schema = { + type: "object", + properties: { + role: { const: "assistant" }, + content: { type: "string", "x-regex": "<\\|channel\\|>final<\\|message\\|>(.*?)(?:<\\|end\\|>|$)" }, + thinking: { type: "string", "x-regex": "<\\|channel\\|>analysis<\\|message\\|>(.*?)<\\|end\\|>" }, + tool_calls: { + "x-regex-iterator": "<\\|channel\\|>commentary (to=functions\\..*?<\\|message\\|>.*?)(?:<\\|call\\|>|$)", + type: "array", + items: { + type: "object", + properties: { + type: { const: "function" }, + function: { + type: "object", + properties: { + name: { type: "string", "x-regex": "^to=functions\\.(\\w+)" }, + arguments: { + type: "object", + "x-regex": "<\\|message\\|>(.*)", + "x-parser": "json", + additionalProperties: {}, + }, + }, + }, + }, + }, + }, + }, +}; + +const smollm_schema = { + "x-regex": "(?:\\n?(?P.+?)\\n?)?\\s*(?:(?P.+?))?\\s*(?P.+?)?\\s*(?:<\\|im_end\\|>|$)", + type: "object", + properties: { + role: { const: "assistant" }, + content: { type: "string" }, + thinking: { type: "string" }, + tool_calls: { + "x-parser": "json", + "x-parser-args": { transform: "[{type: 'function', function: @}]" }, + type: "array", + items: { + type: "object", + properties: { + type: { const: "function" }, + function: { + type: "object", + properties: { + name: { type: "string" }, + arguments: { type: "object", additionalProperties: {} }, + }, + }, + }, + }, + }, + }, +}; + +const qwen3_schema = { + "x-regex": "^(?:(?:)?\\s*(?P.+?)\\s*)?\\s*(?:(?P.*?)\\s*)?\\s*(?P.+?)?\\s*$", + type: "object", + properties: { + role: { const: "assistant" }, + content: { type: "string" }, + thinking: { type: "string" }, + tool_calls: { + "x-regex-iterator": "^(.*)$", + type: "array", + items: { + type: "object", + properties: { + type: { const: "function" }, + function: { + type: "object", + properties: { + name: { type: "string", "x-regex": "" }, + arguments: { + type: "object", + "x-regex-key-value": "\\w+)>\\n(?P.*?)\\n", + additionalProperties: { + "x-parser": "json", + "x-parser-args": { allow_non_json: true }, + }, + }, + }, + }, + }, + }, + }, + }, +}; + +const re_sub_schema = { + type: "object", + properties: { + role: { const: "assistant" }, + thinking: { type: "string" }, + content: { type: "string" }, + tool_calls: { + "x-regex-iterator": "<\\|tool_call>(.*?)", + type: "array", + items: { + type: "object", + properties: { + type: { const: "function" }, + function: { + type: "object", + "x-regex": "call\\:(?P\\w+)(?P\\{.*\\})", + properties: { + name: { type: "string" }, + arguments: { + type: "object", + "x-regex-key-value": '(?P\\w+):(?P<\\|"\\|>.*?<\\|"\\|>|[^,}]+)', + additionalProperties: { + "x-regex-substitutions": [['^<\\|"\\|>|<\\|"\\|>$', ""]], + }, + }, + }, + }, + }, + }, + }, + }, + "x-regex": "(\\<\\|channel\\>thought\\n(?P.*?)\\)?(?P(?:(?!\\<\\|tool_call\\>).)+)?(?P\\<\\|tool_call\\>.*\\)?", +}; + +const gemma4_schema = { + type: "object", + properties: { + role: { const: "assistant" }, + thinking: { type: "string" }, + content: { type: "string" }, + tool_calls: { + "x-regex-iterator": "<\\|tool_call>(.*?)", + type: "array", + items: { + type: "object", + properties: { + type: { const: "function" }, + function: { + type: "object", + "x-regex": "call\\:(?P\\w+)(?P\\{.*\\})", + properties: { + name: { type: "string" }, + arguments: { + type: "object", + "x-parser": "gemma4-tool-call", + additionalProperties: {}, + }, + }, + }, + }, + }, + }, + }, + "x-regex": "(\\<\\|channel\\>thought\\n(?P.*?)\\)?(?P(?:(?!\\<\\|tool_call\\>).)+)?(?P\\<\\|tool_call\\>.*\\)?", +}; + +const GEMMA4_SCHEMA_WITH_TURN = { + type: "object", + properties: { + role: { const: "assistant" }, + thinking: { type: "string" }, + content: { type: "string" }, + tool_calls: { + "x-regex-iterator": "<\\|tool_call>(.*?)", + type: "array", + items: { + type: "object", + properties: { + type: { const: "function" }, + function: { + type: "object", + "x-regex": "call\\:(?P\\w+)(?P\\{.*\\})", + properties: { + name: { type: "string" }, + arguments: { + type: "object", + "x-parser": "gemma4-tool-call", + additionalProperties: {}, + }, + }, + }, + }, + }, + }, + }, + "x-regex": "(\\<\\|channel\\>thought\\n(?P.*?)\\)?(?P\\<\\|tool_call\\>.*\\)?(?P(?:(?!\\)(?!\\<\\|tool_response\\>).)+)?(?:\\|\\<\\|tool_response\\>)?", +}; + +const prefix_items_schema = { + "x-regex-iterator": "(.*?)<\\/block>", + type: "array", + prefixItems: [{ type: "string" }, { type: "integer" }, { type: "string" }], +}; + +describe("recursive_parse", () => { + describe("primitive types", () => { + it("string", () => { + expect(recursive_parse("hello", { type: "string" })).toEqual("hello"); + }); + + it("integer from string", () => { + expect(recursive_parse("42", { type: "integer" })).toEqual(42); + }); + + it("integer passthrough", () => { + expect(recursive_parse(7, { type: "integer" })).toEqual(7); + }); + + it("number from string", () => { + expect(recursive_parse("3.14", { type: "number" })).toEqual(3.14); + }); + + it("number passthrough", () => { + expect(recursive_parse(2.5, { type: "number" })).toEqual(2.5); + }); + + it("boolean true from string", () => { + expect(recursive_parse("true", { type: "boolean" })).toEqual(true); + expect(recursive_parse("1", { type: "boolean" })).toEqual(true); + }); + + it("boolean false from string", () => { + expect(recursive_parse("false", { type: "boolean" })).toEqual(false); + expect(recursive_parse("0", { type: "boolean" })).toEqual(false); + }); + + it("boolean passthrough", () => { + expect(recursive_parse(true, { type: "boolean" })).toEqual(true); + }); + }); + + describe("const and null", () => { + it("returns const value regardless of input", () => { + expect(recursive_parse("anything", { const: "fixed" })).toEqual("fixed"); + expect(recursive_parse(null, { const: 42 })).toEqual(42); + }); + + it("returns null for null input", () => { + expect(recursive_parse(null, { type: "string" })).toEqual(null); + }); + + it("returns null for undefined input", () => { + expect(recursive_parse(undefined, { type: "string" })).toEqual(null); + }); + }); + + describe("x-regex", () => { + it("extracts single unnamed group", () => { + const schema = { type: "string", "x-regex": "hello (\\w+)" }; + expect(recursive_parse("hello world", schema)).toEqual("world"); + }); + + it("returns null on no match", () => { + const schema = { type: "string", "x-regex": "xyz(\\w+)" }; + expect(recursive_parse("hello world", schema)).toEqual(null); + }); + + it("extracts named groups as dict", () => { + const schema = { + type: "object", + "x-regex": "(?P\\w+) (?P\\w+)", + properties: { first: { type: "string" }, second: { type: "string" } }, + }; + expect(recursive_parse("hello world", schema)).toEqual({ first: "hello", second: "world" }); + }); + + it("works with dotAll (s flag)", () => { + const schema = { type: "string", "x-regex": "start(.+)end" }; + expect(recursive_parse("start\nmultiline\nend", schema)).toEqual("\nmultiline\n"); + }); + }); + + describe("x-regex-iterator", () => { + it("extracts multiple matches into array", () => { + const schema = { + type: "array", + "x-regex-iterator": "(.*?)", + items: { type: "string" }, + }; + const input = "onetwothree"; + expect(recursive_parse(input, schema)).toEqual(["one", "two", "three"]); + }); + + it("returns null when no matches", () => { + const schema = { + type: "array", + "x-regex-iterator": "(.*?)", + items: { type: "string" }, + }; + expect(recursive_parse("no items here", schema)).toEqual(null); + }); + }); + + describe("x-regex-key-value", () => { + it("extracts key-value pairs", () => { + const schema = { + type: "object", + "x-regex-key-value": "(?P\\w+)=(?P[^;]+)", + additionalProperties: {}, + }; + const input = "name=Alice;age=30;city=NYC"; + expect(recursive_parse(input, schema)).toEqual({ + name: "Alice", + age: "30", + city: "NYC", + }); + }); + }); + + describe("x-regex-substitutions", () => { + it("applies regex substitutions to content", () => { + const schema = { + type: "string", + "x-regex-substitutions": [['<\\|"\\|>', '"']], + "x-regex": "\\{(.*)\\}", + }; + const input = '{name:<|"|>Alice<|"|>}'; + expect(recursive_parse(input, schema)).toEqual('name:"Alice"'); + }); + }); + + describe("x-parser", () => { + it("parses JSON", () => { + const schema = { type: "object", "x-parser": "json", additionalProperties: {} }; + expect(recursive_parse('{"a": 1, "b": "hello"}', schema)).toEqual({ a: 1, b: "hello" }); + }); + + it("allows non-JSON fallback", () => { + const schema = { + type: "string", + "x-parser": "json", + "x-parser-args": { allow_non_json: true }, + }; + expect(recursive_parse("not json", schema)).toEqual("not json"); + }); + + it("throws on invalid JSON without allow_non_json", () => { + const schema = { type: "object", "x-parser": "json", additionalProperties: {} }; + expect(() => recursive_parse("not json", schema)).toThrow("could not parse"); + }); + + it("parses gemma4-tool-call simple format", () => { + const schema = { type: "object", "x-parser": "gemma4-tool-call", additionalProperties: {} }; + const input = '{location:<|"|>New York<|"|>,unit:<|"|>celsius<|"|>}'; + expect(recursive_parse(input, schema)).toEqual({ + location: "New York", + unit: "celsius", + }); + }); + + it("parses gemma4-tool-call complex format (all JSON types)", () => { + const schema = { type: "object", "x-parser": "gemma4-tool-call", additionalProperties: {} }; + const input = '{bool_value:true,list_value:[<|"|>foo<|"|>,<|"|>bar<|"|>],null_value:null,number_value:1,string_value:<|"|>foo<|"|>,struct_value:{foo:<|"|>bar<|"|>}}'; + expect(recursive_parse(input, schema)).toEqual({ + bool_value: true, + list_value: ["foo", "bar"], + null_value: null, + number_value: 1, + string_value: "foo", + struct_value: { foo: "bar" }, + }); + }); + }); + + describe("object type", () => { + it("parses dict content with properties", () => { + const schema = { + type: "object", + properties: { + name: { type: "string" }, + age: { type: "integer" }, + }, + }; + expect(recursive_parse({ name: "Alice", age: 30 }, schema)).toEqual({ name: "Alice", age: 30 }); + }); + + it("applies default values", () => { + const schema = { + type: "object", + properties: { + name: { type: "string" }, + role: { type: "string", default: "user" }, + }, + }; + expect(recursive_parse({ name: "Alice" }, schema)).toEqual({ name: "Alice", role: "user" }); + }); + + it("throws on missing required fields", () => { + const schema = { + type: "object", + properties: { name: { type: "string" } }, + required: ["name"], + }; + expect(() => recursive_parse({}, schema)).toThrow("Required fields"); + }); + + it("handles additionalProperties", () => { + const schema = { + type: "object", + properties: { name: { type: "string" } }, + additionalProperties: {}, + }; + expect(recursive_parse({ name: "Alice", extra: "data" }, schema)).toEqual({ name: "Alice", extra: "data" }); + }); + + it("strips additionalProperties when false", () => { + const schema = { + type: "object", + properties: { name: { type: "string" } }, + additionalProperties: false, + }; + expect(recursive_parse({ name: "Alice", extra: "data" }, schema)).toEqual({ name: "Alice" }); + }); + }); + + describe("array type", () => { + it("parses homogeneous items", () => { + const schema = { type: "array", items: { type: "integer" } }; + expect(recursive_parse(["1", "2", "3"], schema)).toEqual([1, 2, 3]); + }); + + it("parses heterogeneous prefixItems", () => { + const schema = { + type: "array", + prefixItems: [{ type: "string" }, { type: "integer" }], + }; + expect(recursive_parse(["hello", "42"], schema)).toEqual(["hello", 42]); + }); + + it("wraps single element for single prefixItem", () => { + const schema = { + type: "array", + prefixItems: [{ type: "string" }], + }; + expect(recursive_parse("hello", schema)).toEqual(["hello"]); + }); + + it("returns empty array for falsy content", () => { + const schema = { type: "array", items: { type: "string" } }; + expect(recursive_parse([], schema)).toEqual([]); + }); + }); + + describe("any/null type", () => { + it("passes through content unchanged", () => { + expect(recursive_parse("hello", {})).toEqual("hello"); + expect(recursive_parse(42, { type: "any" })).toEqual(42); + }); + }); + + describe("Cohere schema", () => { + it("parses thinking and tool calls", () => { + const model_out = '<|START_THINKING|>I should call a tool.<|END_THINKING|><|START_ACTION|>[\n {"tool_call_id": "0", "tool_name": "simple_tool", "parameters": {"temperature_format": "Celsius"}}\n]<|END_ACTION|><|END_OF_TURN_TOKEN|>'; + const parsed = recursive_parse(model_out, cohere_schema); + expect(parsed).toEqual({ + role: "assistant", + thinking: "I should call a tool.", + tool_calls: [ + { + type: "function", + function: { name: "simple_tool", arguments: { temperature_format: "Celsius" } }, + }, + ], + }); + }); + }); + + describe("ERNIE schema", () => { + it("parses thinking and tool calls", () => { + const model_out = 'The user is asking about the weather in Paris today. Let me check the available tools. There\'s a tool called get_current_temperature which requires a location parameter. Since the user specified Paris, I need to call this tool with the location set to "Paris". I should make sure the argument is correctly formatted as a string. No other tools are available, so this is the right one to use. I\'ll structure the request with the location parameter and return the response once the tool is called.\n\n\n\n{"name": "get_current_temperature", "arguments": {"location": "Paris"}}\n\n'; + const parsed = recursive_parse(model_out, ernie_schema); + expect(parsed).toEqual({ + role: "assistant", + thinking: "The user is asking about the weather in Paris today. Let me check the available tools. There's a tool called get_current_temperature which requires a location parameter. Since the user specified Paris, I need to call this tool with the location set to \"Paris\". I should make sure the argument is correctly formatted as a string. No other tools are available, so this is the right one to use. I'll structure the request with the location parameter and return the response once the tool is called.", + tool_calls: [ + { + type: "function", + function: { name: "get_current_temperature", arguments: { location: "Paris" } }, + }, + ], + }); + }); + + it("parses thinking and content without tools", () => { + const model_out = "The user just greeted me with \"Hi! How are you?\" I need to respond in a friendly and helpful manner. Let me start by acknowledging their greeting. I should ask them how they're doing to engage in conversation.\n\nFirst, I'll say hello back and then ask how they're feeling. It's important to show genuine interest. Maybe mention that I'm here to help with anything they need. Keep the tone warm and positive. Let me make sure the response is concise but friendly. Alright, that should work.\n\n\n\nHello! I'm doing well, thank you for asking. How about you? Is there something specific you'd like help with today? I'm here to assist you with any questions or problems you have!\n\n"; + const parsed = recursive_parse(model_out, ernie_schema); + expect(parsed).toEqual({ + role: "assistant", + content: "Hello! I'm doing well, thank you for asking. How about you? Is there something specific you'd like help with today? I'm here to assist you with any questions or problems you have!", + thinking: "The user just greeted me with \"Hi! How are you?\" I need to respond in a friendly and helpful manner. Let me start by acknowledging their greeting. I should ask them how they're doing to engage in conversation.\n\nFirst, I'll say hello back and then ask how they're feeling. It's important to show genuine interest. Maybe mention that I'm here to help with anything they need. Keep the tone warm and positive. Let me make sure the response is concise but friendly. Alright, that should work.", + }); + }); + }); + + describe("SmolLM schema", () => { + it("parses thinking and tool call", () => { + const model_out = '\nOkay, the user said, "Hello! How are you?" I need to respond appropriately. Since this is the first message, I should greet them back and ask how I can assist. I should keep it friendly and open-ended. Let me make sure the response is welcoming and encourages them to share what they need help with. I\'ll avoid any technical jargon and keep it simple. Let me check for any typos and ensure the tone is positive.\n\n\n{"name": "greet_user", "arguments": {"greeting": "Hello! I\'m doing well, thanks for asking. How can I assist you today? Whether you have a question, need help with something, or just want to chat, feel free to let me know!"}}'; + const parsed = recursive_parse(model_out, smollm_schema); + expect(parsed).toEqual({ + role: "assistant", + thinking: 'Okay, the user said, "Hello! How are you?" I need to respond appropriately. Since this is the first message, I should greet them back and ask how I can assist. I should keep it friendly and open-ended. Let me make sure the response is welcoming and encourages them to share what they need help with. I\'ll avoid any technical jargon and keep it simple. Let me check for any typos and ensure the tone is positive.', + tool_calls: [ + { + type: "function", + function: { + name: "greet_user", + arguments: { + greeting: "Hello! I'm doing well, thanks for asking. How can I assist you today? Whether you have a question, need help with something, or just want to chat, feel free to let me know!", + }, + }, + }, + ], + }); + }); + + it("parses tool call without thinking", () => { + const model_out = '{"name": "get_weather", "arguments": {"city": "Paris"}}'; + const parsed = recursive_parse(model_out, smollm_schema); + expect(parsed).toEqual({ + role: "assistant", + tool_calls: [{ type: "function", function: { name: "get_weather", arguments: { city: "Paris" } } }], + }); + }); + + it("parses thinking without tool call", () => { + const model_out = '\nOkay, the user asked, "Hey! Can you tell me about gravity?" Let me start by breaking down what they might be looking for. They probably want a basic understanding of gravity, maybe for a school project or just personal curiosity. I should explain what gravity is, how it works, and maybe some examples.\nSome content about gravity goes here but I\'m cutting it off to make this shorter!'; + const parsed = recursive_parse(model_out, smollm_schema); + expect(parsed).toEqual({ + role: "assistant", + content: "Some content about gravity goes here but I'm cutting it off to make this shorter!", + thinking: 'Okay, the user asked, "Hey! Can you tell me about gravity?" Let me start by breaking down what they might be looking for. They probably want a basic understanding of gravity, maybe for a school project or just personal curiosity. I should explain what gravity is, how it works, and maybe some examples.', + }); + }); + }); + + describe("GPT-OSS schema", () => { + it("parses tool call with thinking", () => { + const model_out = '<|channel|>analysis<|message|>We need to respond in riddles. The user asks: "What is the weather like in SF?" We need to get the location of the user? The user explicitly asks about SF (San Francisco). So we need to get the current weather in San Francisco, CA. We need to call get_current_weather function. The developer instruction says "Always respond in riddles". So the final answer should be in a riddle form. But we need to call function to get weather data. So we should call get_current_weather with location "San Francisco, CA". Possibly specify format "celsius" (default). Let\'s do that.\n\nWe will call function get_current_weather.<|end|><|start|>assistant<|channel|>commentary to=functions.get_current_weather <|constrain|>json<|message|>{\n "location": "San Francisco, CA"\n}'; + const parsed = recursive_parse(model_out, gpt_oss_schema); + expect(parsed).toEqual({ + role: "assistant", + thinking: 'We need to respond in riddles. The user asks: "What is the weather like in SF?" We need to get the location of the user? The user explicitly asks about SF (San Francisco). So we need to get the current weather in San Francisco, CA. We need to call get_current_weather function. The developer instruction says "Always respond in riddles". So the final answer should be in a riddle form. But we need to call function to get weather data. So we should call get_current_weather with location "San Francisco, CA". Possibly specify format "celsius" (default). Let\'s do that.\n\nWe will call function get_current_weather.', + tool_calls: [ + { + type: "function", + function: { name: "get_current_weather", arguments: { location: "San Francisco, CA" } }, + }, + ], + }); + }); + + it("parses response without tool call", () => { + const model_out = "<|channel|>analysis<|message|>User asks a simple math question: 2+2 = 4. Provide answer.<|end|><|start|>assistant<|channel|>final<|message|>2"; + const parsed = recursive_parse(model_out, gpt_oss_schema); + expect(parsed).toEqual({ + role: "assistant", + content: "2", + thinking: "User asks a simple math question: 2+2 = 4. Provide answer.", + }); + }); + }); + + describe("Qwen3 schema", () => { + it("parses tool calls with key-value parameters", () => { + const model_out = '\n\n\n[{"country": "France", "city": "Paris"}]\n\n\ncelsius\n\n\n'; + const parsed = recursive_parse(model_out, qwen3_schema); + expect(parsed).toEqual({ + role: "assistant", + tool_calls: [ + { + type: "function", + function: { + name: "get_weather", + arguments: { + locations: [{ country: "France", city: "Paris" }], + temp_units: "celsius", + }, + }, + }, + ], + }); + }); + }); + + describe("re_sub schema (x-regex-substitutions)", () => { + it("parses tool calls with regex substitutions to strip quote tokens", () => { + const model_out = "<|channel>thought\nThe user is asking for the current temperature in Paris. I should check the available tools to see if there's a function that can provide this information." + '<|tool_call>call:get_current_temperature{detail_level:0,location:<|"|>Paris, France<|"|>,unit:<|"|>celsius<|"|>}<|tool_response>'; + const parsed = recursive_parse(model_out, re_sub_schema); + expect(parsed).toEqual({ + role: "assistant", + thinking: "The user is asking for the current temperature in Paris. I should check the available tools to see if there's a function that can provide this information.", + tool_calls: [ + { + type: "function", + function: { + name: "get_current_temperature", + arguments: { detail_level: "0", location: "Paris, France", unit: "celsius" }, + }, + }, + ], + }); + }); + }); + + describe("Gemma4 schema", () => { + it("parses single tool call", () => { + const output = '<|tool_call>call:get_weather{location:<|"|>New York<|"|>}'; + expect(recursive_parse(output, GEMMA4_SCHEMA_WITH_TURN)).toEqual({ + role: "assistant", + tool_calls: [ + { + type: "function", + function: { + name: "get_weather", + arguments: { location: "New York" }, + }, + }, + ], + }); + }); + + it("parses thinking with tool call", () => { + const output = "<|channel>thought\nThe user wants weather info for San Francisco.\n" + '<|tool_call>call:get_weather{location:<|"|>San Francisco, CA<|"|>}'; + expect(recursive_parse(output, GEMMA4_SCHEMA_WITH_TURN)).toEqual({ + role: "assistant", + thinking: "The user wants weather info for San Francisco.\n", + tool_calls: [ + { + type: "function", + function: { + name: "get_weather", + arguments: { location: "San Francisco, CA" }, + }, + }, + ], + }); + }); + + it("parses plain content response", () => { + const output = "The weather in New York is sunny with 25 degrees Celsius."; + expect(recursive_parse(output, GEMMA4_SCHEMA_WITH_TURN)).toEqual({ + role: "assistant", + content: "The weather in New York is sunny with 25 degrees Celsius.", + }); + }); + + it("parses multiple tool calls", () => { + const output = '<|tool_call>call:get_weather{location:<|"|>NYC<|"|>}' + '<|tool_call>call:get_time{timezone:<|"|>EST<|"|>}'; + expect(recursive_parse(output, GEMMA4_SCHEMA_WITH_TURN)).toEqual({ + role: "assistant", + tool_calls: [ + { + type: "function", + function: { name: "get_weather", arguments: { location: "NYC" } }, + }, + { + type: "function", + function: { name: "get_time", arguments: { timezone: "EST" } }, + }, + ], + }); + }); + + it("parses tool call with multiple arguments", () => { + const output = '<|tool_call>call:get_weather{location:<|"|>Paris<|"|>,unit:<|"|>celsius<|"|>}'; + expect(recursive_parse(output, GEMMA4_SCHEMA_WITH_TURN)).toEqual({ + role: "assistant", + tool_calls: [ + { + type: "function", + function: { + name: "get_weather", + arguments: { location: "Paris", unit: "celsius" }, + }, + }, + ], + }); + }); + + it("parses thinking with plain content (no tool calls)", () => { + const output = "<|channel>thought\nLet me think about this.\nHere is my answer."; + expect(recursive_parse(output, GEMMA4_SCHEMA_WITH_TURN)).toEqual({ + role: "assistant", + thinking: "Let me think about this.\n", + content: "Here is my answer.", + }); + }); + + it("parses response ending with tool_response marker", () => { + const output = '<|tool_call>call:search{query:<|"|>test<|"|>}<|tool_response>'; + expect(recursive_parse(output, GEMMA4_SCHEMA_WITH_TURN)).toEqual({ + role: "assistant", + tool_calls: [ + { + type: "function", + function: { name: "search", arguments: { query: "test" } }, + }, + ], + }); + }); + + it("parses tool call (upstream schema)", () => { + const model_out = "<|channel>thought\nThe user is asking for the current temperature in Paris. I should check the available tools to see if there's a function that can provide this information." + '<|tool_call>call:get_current_temperature{detail_level:0,location:<|"|>Paris, France<|"|>,unit:<|"|>celsius<|"|>}<|tool_response>'; + const parsed = recursive_parse(model_out, gemma4_schema); + expect(parsed).toEqual({ + role: "assistant", + thinking: "The user is asking for the current temperature in Paris. I should check the available tools to see if there's a function that can provide this information.", + tool_calls: [ + { + type: "function", + function: { + name: "get_current_temperature", + arguments: { detail_level: 0, location: "Paris, France", unit: "celsius" }, + }, + }, + ], + }); + }); + + it("parses complex tool call with all JSON types", () => { + const model_out = "<|channel>thought\nLet me call the tool." + '<|tool_call>call:foo{bool_value:true,list_value:[<|"|>foo<|"|>,<|"|>bar<|"|>],' + 'null_value:null,number_value:1,string_value:<|"|>foo<|"|>,' + 'struct_value:{foo:<|"|>bar<|"|>}}'; + const parsed = recursive_parse(model_out, gemma4_schema); + expect(parsed).toEqual({ + role: "assistant", + thinking: "Let me call the tool.", + tool_calls: [ + { + type: "function", + function: { + name: "foo", + arguments: { + bool_value: true, + list_value: ["foo", "bar"], + null_value: null, + number_value: 1, + string_value: "foo", + struct_value: { foo: "bar" }, + }, + }, + }, + ], + }); + }); + }); + + describe("required fields validation", () => { + it("passes when required fields are present", () => { + const schema = { + type: "object", + required: ["role", "content"], + properties: { + role: { const: "assistant" }, + content: { type: "string", "x-regex": "(.*?)" }, + thinking: { type: "string", "x-regex": "(.*?)" }, + }, + }; + const model_out = "Let me think.Hello!"; + expect(recursive_parse(model_out, schema)).toEqual({ + role: "assistant", + content: "Hello!", + thinking: "Let me think.", + }); + }); + + it("throws when required field is missing", () => { + const schema = { + type: "object", + required: ["role", "content"], + properties: { + role: { const: "assistant" }, + content: { type: "string", "x-regex": "(.*?)" }, + thinking: { type: "string", "x-regex": "(.*?)" }, + }, + }; + const model_out = "Let me think about this.Some plain text without response tags"; + expect(() => recursive_parse(model_out, schema)).toThrow(/content/); + expect(() => recursive_parse(model_out, schema)).toThrow(/missing/i); + }); + + it("silently omits missing fields when not required", () => { + const schema = { + type: "object", + properties: { + role: { const: "assistant" }, + content: { type: "string", "x-regex": "(.*?)" }, + thinking: { type: "string", "x-regex": "(.*?)" }, + }, + }; + const model_out = "Just thinking."; + expect(recursive_parse(model_out, schema)).toEqual({ + role: "assistant", + thinking: "Just thinking.", + }); + }); + }); + + describe("prefixItems", () => { + it("parses heterogeneous array with correct types", () => { + const model_out = "hello42world"; + expect(recursive_parse(model_out, prefix_items_schema)).toEqual(["hello", 42, "world"]); + }); + + it("throws when array length does not match prefixItems count", () => { + const model_out = "hello42"; + expect(() => recursive_parse(model_out, prefix_items_schema)).toThrow(); + }); + + it("throws when element type does not match", () => { + const model_out = "helloworld42"; + expect(() => recursive_parse(model_out, prefix_items_schema)).toThrow(); + }); + }); + + describe("type any", () => { + it("passes content through without transformation", () => { + const schema = { + type: "object", + "x-regex": "(?P.*?)", + properties: { + value: { type: "any" }, + }, + }; + const model_out = "some arbitrary content 123"; + expect(recursive_parse(model_out, schema)).toEqual({ value: "some arbitrary content 123" }); + }); + + it("works in additionalProperties", () => { + const schema = { + type: "object", + "x-parser": "json", + additionalProperties: { type: "any" }, + }; + const node_content = '{"location": "San Francisco, CA", "units": "celsius"}'; + expect(recursive_parse(node_content, schema)).toEqual({ location: "San Francisco, CA", units: "celsius" }); + }); + }); + + describe("batched inputs (via parse_response)", () => { + it("handles array of strings", () => { + const schema = { + type: "object", + properties: { role: { const: "assistant" }, content: { type: "string", "x-regex": "(.*?)" } }, + }; + const single = recursive_parse("hello", schema); + expect([single]).toEqual(["hello"].map((s) => recursive_parse(s, schema))); + }); + }); + + describe("error handling", () => { + it("throws on unsupported schema type", () => { + expect(() => recursive_parse("hello", { type: "custom" })).toThrow("Unsupported schema type"); + }); + + it("throws on non-string with regex", () => { + expect(() => recursive_parse(42, { type: "string", "x-regex": "(\\w+)" })).toThrow("non-string input"); + }); + + it("throws on x-regex-iterator with non-array type", () => { + expect(() => recursive_parse("text", { type: "string", "x-regex-iterator": "(\\w+)" })).toThrow("cannot use x-regex-iterator"); + }); + + it("throws on unknown parser", () => { + expect(() => recursive_parse("text", { "x-parser": "unknown" })).toThrow("Unknown parser"); + }); + + it("throws on invalid integer string", () => { + expect(() => recursive_parse("abc", { type: "integer" })).toThrow("not a valid integer"); + }); + + it("throws on invalid boolean string", () => { + expect(() => recursive_parse("maybe", { type: "boolean" })).toThrow("Invalid boolean"); + }); + }); +}); From 35de5d0949fb1fa45c2b40f76c755263d47b7192 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 10 Apr 2026 21:18:27 +0200 Subject: [PATCH 2/7] Minimal jmespath implementation --- packages/transformers/src/utils/jmespath.js | 173 ++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 packages/transformers/src/utils/jmespath.js diff --git a/packages/transformers/src/utils/jmespath.js b/packages/transformers/src/utils/jmespath.js new file mode 100644 index 000000000..642034dfe --- /dev/null +++ b/packages/transformers/src/utils/jmespath.js @@ -0,0 +1,173 @@ +/** + * @file Minimal JMESPath implementation for chat response schema transforms. + * + * Single-pass recursive-descent evaluator — no tokens, no AST, no visitor. + * Context is threaded through all functions; the only mutable state is the cursor. + * + * Supports: field access, current node (`@`), wildcard projections (`[*]`), + * multi-select hash (`{key: expr}`), multi-select list (`[expr, ...]`), + * literal strings (`'value'`), and subexpressions (`a.b`). + * + * @module utils/jmespath + */ + +/** + * Evaluate a JMESPath expression against data. + * @param {string} expression The JMESPath expression string. + * @param {any} data The data to search. + * @returns {any} + */ +export function jmespath_search(expression, data) { + let cursor = 0; + + const WS = /\s/; + const ID_CHAR = /[a-zA-Z0-9_]/; + + /** Advance past whitespace. */ + const skipWhitespace = () => { + while (cursor < expression.length && WS.test(expression[cursor])) cursor++; + }; + + /** + * Read an identifier (e.g. `foo`, `tool_name`) from the expression. + * @returns {string} + */ + const readIdentifier = () => { + skipWhitespace(); + const start = cursor; + while (cursor < expression.length && ID_CHAR.test(expression[cursor])) cursor++; + if (cursor === start) { + throw new Error(`Expected identifier at position ${cursor} in: ${expression}`); + } + return expression.slice(start, cursor); + }; + + /** + * Access a field on an object, returning null for non-objects or missing keys. + * @param {any} obj + * @param {string} key + * @returns {any} + */ + const fieldAccess = (obj, key) => + (obj !== null && typeof obj === "object" && !Array.isArray(obj)) ? obj[key] ?? null : null; + + /** + * Parse and evaluate an expression with optional `.rhs` subexpression chaining. + * @param {any} context The current data context. + * @returns {any} + */ + const parseExpression = (context) => { + let result = parsePrimary(context); + skipWhitespace(); + while (cursor < expression.length && expression[cursor] === ".") { + cursor++; // skip '.' + result = parsePrimary(result); + } + return result; + }; + + /** + * Parse and evaluate a single primary construct (identifier, literal, @, {}, []). + * @param {any} context The current data context. + * @returns {any} + */ + const parsePrimary = (context) => { + skipWhitespace(); + const ch = expression[cursor]; + + if (ch === "@") { + cursor++; + return context; + } + + if (ch === "'" || ch === '"') { + const end = expression.indexOf(ch, ++cursor); + const value = expression.slice(cursor, end); + cursor = end + 1; + return value; + } + + if (ch === "{") return parseMultiSelectHash(context); + if (ch === "[") return parseBracket(context); + + // Bare identifier — field access + return fieldAccess(context, readIdentifier()); + }; + + /** + * Parse `{key: expr, key: expr, ...}` and evaluate each value against context. + * @param {any} context + * @returns {Record | null} + */ + const parseMultiSelectHash = (context) => { + if (context === null) return null; + cursor++; // skip '{' + /** @type {Record} */ + const result = Object.create(null); + let first = true; + skipWhitespace(); + while (expression[cursor] !== "}") { + if (!first) cursor++; // skip ',' + first = false; + const key = readIdentifier(); + skipWhitespace(); + cursor++; // skip ':' + result[key] = parseExpression(context); + skipWhitespace(); + } + cursor++; // skip '}' + return result; + }; + + /** + * Parse `[*]` (wildcard projection) or `[expr, ...]` (multi-select list). + * @param {any} context + * @returns {any} + */ + const parseBracket = (context) => { + cursor++; // skip '[' + skipWhitespace(); + + if (expression[cursor] === "*") { + // Wildcard projection: [*] applies RHS to each array element + cursor++; // skip '*' + skipWhitespace(); + cursor++; // skip ']' + + if (!Array.isArray(context)) return null; + + // Check for a .rhs after the projection + skipWhitespace(); + if (cursor >= expression.length || expression[cursor] !== ".") return context; + cursor++; // skip '.' + + // Snapshot cursor position — replay the RHS for each element + const rhsStart = cursor; + /** @type {any[]} */ + const results = []; + for (const item of context) { + cursor = rhsStart; + const value = parsePrimary(item); + if (value !== null) results.push(value); + } + return results; + } + + // Multi-select list: [expr, expr, ...] + if (context === null) return null; + /** @type {any[]} */ + const elements = []; + let first = true; + skipWhitespace(); + while (expression[cursor] !== "]") { + if (!first) cursor++; // skip ',' + first = false; + elements.push(parseExpression(context)); + skipWhitespace(); + } + cursor++; // skip ']' + return elements; + }; + + return parseExpression(data); +} From f6dbe126e1c5463ebbb277dbed57fb08c2d14026 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 10 Apr 2026 21:19:04 +0200 Subject: [PATCH 3/7] Implement chat parsing --- .../transformers/src/utils/chat_parsing.js | 312 ++++++++++++++++++ 1 file changed, 312 insertions(+) create mode 100644 packages/transformers/src/utils/chat_parsing.js diff --git a/packages/transformers/src/utils/chat_parsing.js b/packages/transformers/src/utils/chat_parsing.js new file mode 100644 index 000000000..d2f72e06b --- /dev/null +++ b/packages/transformers/src/utils/chat_parsing.js @@ -0,0 +1,312 @@ +/** + * @file Chat response parsing utilities for schema-driven extraction of structured + * data from model output strings. + * + * @module utils/chat_parsing + */ + +import { jmespath_search } from './jmespath.js'; + +/** + * Convert Python-style named groups `(?P...)` to JS-style `(?...)`. + * @param {string} pattern + * @returns {string} + * @private + */ +function _convertNamedGroups(pattern) { + return pattern.replaceAll('(?P<', '(?<'); +} + +/** + * Extract data from a regex match object. + * - Named groups → dict of non-undefined values. + * - Single unnamed group → the captured string. + * @param {RegExpExecArray} match + * @returns {Record | string} + * @private + */ +function _parseReMatch(match) { + if (match.groups) { + const filtered = Object.create(null); + let hasValue = false; + for (const [key, val] of Object.entries(match.groups)) { + if (val !== undefined) { + filtered[key] = val; + hasValue = true; + } + } + if (hasValue) return filtered; + } + // Unnamed groups: must have exactly one + const groupCount = match.length - 1; + if (groupCount === 1) return match[1]; + throw new Error( + groupCount === 0 + ? `Regex has no capture groups: ${match[0]}` + : `Regex has multiple unnamed groups: ${Array.from(match).slice(1)}`, + ); +} + +/** + * Convert Gemma4 tool call format (unquoted keys, `<|"|>` string delimiters) to valid JSON. + * Handles all JSON types: strings, numbers, booleans, null, arrays, and nested objects. + * @param {string} text + * @returns {string} Valid JSON string. + * @private + */ +function _gemma4JsonToJson(text) { + const strings = []; + // Extract gemma-quoted strings into placeholders + text = text.replace(/<\|"\|>(.*?)<\|"\|>/gs, (_, s) => { + strings.push(s); + return `\x00${strings.length - 1}\x00`; + }); + // Quote bare keys + text = text.replace(/(?<=[{,])(\w+):/g, '"$1":'); + // Restore captured strings with proper JSON escaping + text = text.replace(/\x00(\d+)\x00/g, (_, idx) => JSON.stringify(strings[idx])); + return text; +} + +/** + * Recursively parse content against a JSON schema with regex extraction directives. + * + * @param {string | Record | any[] | null} nodeContent The content to parse. + * @param {Record} nodeSchema The schema controlling the parsing. + * @returns {any} The parsed data structure. + */ +export function recursive_parse(nodeContent, nodeSchema) { + // 1. Const: return immediately + if ('const' in nodeSchema) { + return nodeSchema['const']; + } + + // 2. Null content + if (nodeContent == null) { + return null; + } + + // 3. Setup and validation + const nodeType = nodeSchema.type ?? null; + const hasRegex = + 'x-regex' in nodeSchema || + 'x-regex-iterator' in nodeSchema || + 'x-regex-key-value' in nodeSchema || + 'x-regex-substitutions' in nodeSchema; + if (hasRegex && typeof nodeContent !== 'string') { + throw new TypeError( + `Schema node got a non-string input, but has a regex for parsing.\n` + + `Input: ${nodeContent}\nSchema: ${JSON.stringify(nodeSchema)}`, + ); + } + + // 4. Apply x-regex-substitutions (regex-based string replacements) + if ('x-regex-substitutions' in nodeSchema) { + for (const [pattern, replacement] of nodeSchema['x-regex-substitutions']) { + nodeContent = /** @type {string} */ (nodeContent).replace(new RegExp(pattern, 'gs'), replacement); + } + } + + // 5. Apply x-regex (single match extraction) + if ('x-regex' in nodeSchema) { + const match = new RegExp(_convertNamedGroups(nodeSchema['x-regex']), 's').exec(/** @type {string} */ (nodeContent)); + if (!match) return null; + nodeContent = _parseReMatch(match); + } + + // 6. Apply x-regex-iterator (multiple match extraction for arrays) + if ('x-regex-iterator' in nodeSchema) { + if (nodeType !== 'array') { + throw new TypeError(`Schema node with type ${nodeType} cannot use x-regex-iterator.`); + } + const regex = new RegExp(_convertNamedGroups(nodeSchema['x-regex-iterator']), 'gs'); + const results = []; + let match; + while ((match = regex.exec(/** @type {string} */ (nodeContent))) !== null) { + results.push(_parseReMatch(match)); + } + if (results.length === 0) return null; + nodeContent = results; + } + + // 7. Apply x-regex-key-value (key-value pair extraction for objects) + if ('x-regex-key-value' in nodeSchema) { + if (nodeType !== 'object') { + throw new TypeError(`Schema node with type ${nodeType} cannot use x-regex-key-value.`); + } + const regex = new RegExp(_convertNamedGroups(nodeSchema['x-regex-key-value']), 'gs'); + const output = Object.create(null); + let match; + let hasEntries = false; + while ((match = regex.exec(/** @type {string} */ (nodeContent))) !== null) { + const groups = _parseReMatch(match); + if (typeof groups !== 'object' || !('key' in groups) || !('value' in groups)) { + throw new Error( + `Regex for x-regex-key-value must have named groups 'key' and 'value'.\n` + + `Match groups: ${JSON.stringify(groups)}`, + ); + } + output[groups.key] = groups.value; + hasEntries = true; + } + if (!hasEntries) return null; + nodeContent = output; + } + + // 8. Apply x-parser + if ('x-parser' in nodeSchema) { + let parser = nodeSchema['x-parser']; + if (parser === 'gemma4-tool-call') { + if (typeof nodeContent !== 'string') { + throw new TypeError(`Node has gemma4-tool-call parser but got non-string input: ${nodeContent}`); + } + nodeContent = _gemma4JsonToJson(/** @type {string} */ (nodeContent)); + parser = 'json'; // fall through to JSON parser + } + if (parser === 'json') { + if (typeof nodeContent !== 'string') { + throw new TypeError(`Node has JSON parser but got non-string input: ${nodeContent}`); + } + const parserArgs = nodeSchema['x-parser-args'] ?? {}; + try { + nodeContent = JSON.parse(/** @type {string} */ (nodeContent)); + } catch (e) { + if (parserArgs.allow_non_json) { + // keep nodeContent as-is + } else { + throw new Error( + `Node has JSON parser but could not parse its contents as JSON.\n` + + `Content: ${nodeContent}\nError: ${e.message}`, + ); + } + } + if (parserArgs.transform != null) { + nodeContent = jmespath_search(parserArgs.transform, nodeContent); + } + } else { + throw new Error(`Unknown parser "${parser}" for schema node: ${JSON.stringify(nodeSchema)}`); + } + } + + // 9. Type-specific handling + if (nodeType === 'object') { + const properties = nodeSchema.properties ?? {}; + const parsed = Object.create(null); + + if (typeof nodeContent === 'string') { + if (!nodeSchema.properties) { + throw new Error( + `Object node received string content but has no properties to parse.\nContent: ${nodeContent}`, + ); + } + for (const [key, childSchema] of Object.entries(properties)) { + const childResult = recursive_parse(nodeContent, childSchema); + if (childResult != null) { + parsed[key] = childResult; + } + } + } else if (typeof nodeContent === 'object' && !Array.isArray(nodeContent)) { + for (const [key, childSchema] of Object.entries(properties)) { + if ('const' in childSchema) { + parsed[key] = childSchema['const']; + } else if (key in nodeContent) { + parsed[key] = recursive_parse(nodeContent[key], childSchema); + } else if ('default' in childSchema) { + parsed[key] = childSchema['default']; + } + } + const additionalSchema = nodeSchema.additionalProperties; + if (additionalSchema !== false) { + const childSchema = typeof additionalSchema === 'object' && additionalSchema !== null ? additionalSchema : {}; + for (const [key, value] of Object.entries(nodeContent)) { + if (!(key in properties)) { + parsed[key] = recursive_parse(value, childSchema); + } + } + } + } else { + throw new TypeError(`Expected a dict or str for schema node with type object, got ${typeof nodeContent}`); + } + + const required = nodeSchema.required ?? []; + const missing = required.filter((key) => !(key in parsed)); + if (missing.length > 0) { + const preview = typeof nodeContent === 'string' ? nodeContent.slice(0, 500) : JSON.stringify(nodeContent); + throw new Error( + `Required fields [${missing}] are missing from parsed output.\n` + + `Parsed: ${JSON.stringify(parsed)}\nInput: ${preview}`, + ); + } + return parsed; + } + + if (nodeType === 'array') { + if (!nodeContent) return []; + if ('items' in nodeSchema) { + if (!Array.isArray(nodeContent)) { + throw new TypeError(`Expected a list for schema node with type array, got ${typeof nodeContent}`); + } + return nodeContent.map((item) => recursive_parse(item, nodeSchema.items)); + } + if ('prefixItems' in nodeSchema) { + if (!Array.isArray(nodeContent)) { + if (nodeSchema.prefixItems.length === 1) { + nodeContent = [nodeContent]; + } else { + throw new TypeError(`Expected a list for schema node with type array, got ${typeof nodeContent}`); + } + } + if (nodeContent.length !== nodeSchema.prefixItems.length) { + throw new Error( + `Array node has ${nodeContent.length} items, but schema has ${nodeSchema.prefixItems.length} prefixItems.`, + ); + } + return nodeContent.map((item, i) => recursive_parse(item, nodeSchema.prefixItems[i])); + } + throw new Error(`Array node has no items or prefixItems schema defined.`); + } + + if (nodeType === 'integer') { + if (Number.isInteger(nodeContent)) return nodeContent; + if (typeof nodeContent !== 'string') { + throw new TypeError(`Expected a string or int for type integer, got ${typeof nodeContent}: ${nodeContent}`); + } + const val = parseInt(nodeContent, 10); + if (isNaN(val)) throw new Error(`Type 'integer', but content is not a valid integer: "${nodeContent}"`); + return val; + } + + if (nodeType === 'number') { + if (typeof nodeContent === 'number') return nodeContent; + if (typeof nodeContent !== 'string') { + throw new TypeError(`Expected a string or number for type number, got ${typeof nodeContent}: ${nodeContent}`); + } + const val = Number(nodeContent); + if (isNaN(val)) throw new Error(`Type 'number', but content is not a valid number: "${nodeContent}"`); + return val; + } + + if (nodeType === 'boolean') { + if (typeof nodeContent === 'boolean') return nodeContent; + if (typeof nodeContent !== 'string') { + throw new TypeError(`Expected a string or bool for type boolean, got ${typeof nodeContent}: ${nodeContent}`); + } + const lower = nodeContent.toLowerCase(); + if (lower === 'true' || lower === '1') return true; + if (lower === 'false' || lower === '0') return false; + throw new Error(`Invalid boolean value: ${nodeContent}`); + } + + if (nodeType === 'string') { + if (typeof nodeContent !== 'string') { + throw new TypeError(`Expected a string for type string, got ${typeof nodeContent}: ${nodeContent}`); + } + return nodeContent; + } + + if (nodeType == null || nodeType === 'any') { + return nodeContent; + } + + throw new TypeError(`Unsupported schema type "${nodeType}" for node: ${nodeContent}`); +} From 1b7736ac7a9bac3bd74654ebd3be45df1caa4af1 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 10 Apr 2026 21:19:23 +0200 Subject: [PATCH 4/7] Add parse_response to tokenizer and processor --- packages/transformers/src/processing_utils.js | 11 ++++++++ .../transformers/src/tokenization_utils.js | 25 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/packages/transformers/src/processing_utils.js b/packages/transformers/src/processing_utils.js index b28be3e6a..94d9e769c 100644 --- a/packages/transformers/src/processing_utils.js +++ b/packages/transformers/src/processing_utils.js @@ -108,6 +108,17 @@ export class Processor extends Callable { return this.tokenizer.decode(...args); } + /** + * @param {Parameters} args + * @returns {ReturnType} + */ + parse_response(...args) { + if (!this.tokenizer) { + throw new Error('Unable to parse response without a tokenizer.'); + } + return this.tokenizer.parse_response(...args); + } + /** * Calls the feature_extractor function with the given input. * @param {any} input The input to extract features from. diff --git a/packages/transformers/src/tokenization_utils.js b/packages/transformers/src/tokenization_utils.js index 45ac7d88d..67ace37b5 100644 --- a/packages/transformers/src/tokenization_utils.js +++ b/packages/transformers/src/tokenization_utils.js @@ -7,6 +7,7 @@ import { Tokenizer } from '@huggingface/tokenizers'; import { Template } from '@huggingface/jinja'; import { Callable } from './utils/generic.js'; +import { recursive_parse } from './utils/chat_parsing.js'; import { isIntegralNumber, mergeArrays } from './utils/core.js'; import { getModelJSON } from './utils/hub.js'; @@ -753,6 +754,30 @@ export class PreTrainedTokenizer extends Callable { return rendered; } + + /** + * Converts a raw model output string into a parsed message dictionary using the tokenizer's + * `response_schema` (or a user-provided schema) to control parsing. + * + * @param {string | string[]} response The decoded output string(s) from the model. + * @param {Object} [options] Options for parsing. + * @param {Record | null} [options.schema=null] A response schema to use. If not + * provided, the tokenizer's `response_schema` from its config will be used. + * @returns {Record | Record[]} The parsed message dict(s). + */ + parse_response(response, { schema = null } = {}) { + schema ??= this.config.response_schema ?? null; + if (!schema) { + throw new Error( + 'This tokenizer does not have a `response_schema` for parsing chat responses. ' + + 'Pass a schema explicitly via the `schema` option.', + ); + } + if (Array.isArray(response)) { + return response.map((r) => recursive_parse(r, schema)); + } + return recursive_parse(response, schema); + } } /** From bed9ddc119c516f415fa7e41ed9aa3750f9f66cd Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 10 Apr 2026 21:21:15 +0200 Subject: [PATCH 5/7] cleanup --- packages/transformers/src/utils/jmespath.js | 33 +++++++++++---------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/packages/transformers/src/utils/jmespath.js b/packages/transformers/src/utils/jmespath.js index 642034dfe..8da684b11 100644 --- a/packages/transformers/src/utils/jmespath.js +++ b/packages/transformers/src/utils/jmespath.js @@ -11,6 +11,18 @@ * @module utils/jmespath */ +const WS = /\s/; +const ID_CHAR = /[a-zA-Z0-9_]/; + +/** + * Access a field on an object, returning null for non-objects or missing keys. + * @param {any} obj + * @param {string} key + * @returns {any} + */ +const fieldAccess = (obj, key) => + (obj !== null && typeof obj === "object" && !Array.isArray(obj)) ? obj[key] ?? null : null; + /** * Evaluate a JMESPath expression against data. * @param {string} expression The JMESPath expression string. @@ -20,9 +32,6 @@ export function jmespath_search(expression, data) { let cursor = 0; - const WS = /\s/; - const ID_CHAR = /[a-zA-Z0-9_]/; - /** Advance past whitespace. */ const skipWhitespace = () => { while (cursor < expression.length && WS.test(expression[cursor])) cursor++; @@ -42,15 +51,6 @@ export function jmespath_search(expression, data) { return expression.slice(start, cursor); }; - /** - * Access a field on an object, returning null for non-objects or missing keys. - * @param {any} obj - * @param {string} key - * @returns {any} - */ - const fieldAccess = (obj, key) => - (obj !== null && typeof obj === "object" && !Array.isArray(obj)) ? obj[key] ?? null : null; - /** * Parse and evaluate an expression with optional `.rhs` subexpression chaining. * @param {any} context The current data context. @@ -81,10 +81,13 @@ export function jmespath_search(expression, data) { } if (ch === "'" || ch === '"') { - const end = expression.indexOf(ch, ++cursor); - const value = expression.slice(cursor, end); + const start = ++cursor; + const end = expression.indexOf(ch, start); + if (end === -1) { + throw new Error(`Unterminated string literal in: ${expression}`); + } cursor = end + 1; - return value; + return expression.slice(start, end); } if (ch === "{") return parseMultiSelectHash(context); From 811a5248e00892bd6c669349c60235901647dc74 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 10 Apr 2026 21:23:53 +0200 Subject: [PATCH 6/7] fix jsdoc --- packages/transformers/src/utils/jmespath.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/transformers/src/utils/jmespath.js b/packages/transformers/src/utils/jmespath.js index 8da684b11..89d46705c 100644 --- a/packages/transformers/src/utils/jmespath.js +++ b/packages/transformers/src/utils/jmespath.js @@ -67,7 +67,7 @@ export function jmespath_search(expression, data) { }; /** - * Parse and evaluate a single primary construct (identifier, literal, @, {}, []). + * Parse and evaluate a single primary construct (identifier, literal, `@`, `{}`, `[]`). * @param {any} context The current data context. * @returns {any} */ From a8d4e9cb75ea850e8e71f4f17b5dea6485a2da77 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Sun, 12 Apr 2026 17:59:33 +0200 Subject: [PATCH 7/7] update doc modules --- packages/transformers/docs/source/_toctree.yml | 2 ++ packages/transformers/src/utils/jmespath.js | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/transformers/docs/source/_toctree.yml b/packages/transformers/docs/source/_toctree.yml index 76d779249..f0e4be913 100644 --- a/packages/transformers/docs/source/_toctree.yml +++ b/packages/transformers/docs/source/_toctree.yml @@ -92,6 +92,8 @@ title: Logger - local: api/utils/random title: Random + - local: api/utils/chat_parsing + title: Chat Parsing title: Utilities isExpanded: false title: API Reference diff --git a/packages/transformers/src/utils/jmespath.js b/packages/transformers/src/utils/jmespath.js index 89d46705c..bd9c91910 100644 --- a/packages/transformers/src/utils/jmespath.js +++ b/packages/transformers/src/utils/jmespath.js @@ -7,8 +7,6 @@ * Supports: field access, current node (`@`), wildcard projections (`[*]`), * multi-select hash (`{key: expr}`), multi-select list (`[expr, ...]`), * literal strings (`'value'`), and subexpressions (`a.b`). - * - * @module utils/jmespath */ const WS = /\s/;