From 00deb593c17307d3cabb2f231f29707796b49fef Mon Sep 17 00:00:00 2001 From: Nico Martin Date: Fri, 10 Jul 2026 11:27:55 +0200 Subject: [PATCH 1/5] first POC --- packages/transformers-llguidance/README.md | 28 ++ .../transformers-llguidance/jest.config.mjs | 11 + packages/transformers-llguidance/package.json | 68 +++++ .../transformers-llguidance/scripts/build.mjs | 26 ++ .../transformers-llguidance/scripts/dev.mjs | 34 +++ packages/transformers-llguidance/src/index.ts | 250 ++++++++++++++++++ .../tests/llguidance-constraint.test.js | 52 ++++ .../transformers-llguidance/tsconfig.json | 21 ++ pnpm-lock.yaml | 54 ++-- 9 files changed, 520 insertions(+), 24 deletions(-) create mode 100644 packages/transformers-llguidance/README.md create mode 100644 packages/transformers-llguidance/jest.config.mjs create mode 100644 packages/transformers-llguidance/package.json create mode 100644 packages/transformers-llguidance/scripts/build.mjs create mode 100644 packages/transformers-llguidance/scripts/dev.mjs create mode 100644 packages/transformers-llguidance/src/index.ts create mode 100644 packages/transformers-llguidance/tests/llguidance-constraint.test.js create mode 100644 packages/transformers-llguidance/tsconfig.json diff --git a/packages/transformers-llguidance/README.md b/packages/transformers-llguidance/README.md new file mode 100644 index 000000000..f1776465c --- /dev/null +++ b/packages/transformers-llguidance/README.md @@ -0,0 +1,28 @@ +# @huggingface/transformers-llguidance + +Experimental constrained-generation helpers for Transformers.js. + +This package exports `LlguidanceConstraint`, which turns an llguidance response format into the `logits_processor` and `stopping_criteria` objects accepted by Transformers.js generation. + +```js +import { LlguidanceConstraint } from "@huggingface/transformers-llguidance"; + +const { logits_processor, stopping_criteria } = + await LlguidanceConstraint.fromResponseFormat(tokenizer, { + type: "json_schema", + json_schema: { + type: "object", + properties: { + answer: { type: "string" }, + }, + required: ["answer"], + additionalProperties: false, + }, + }); + +await model.generate({ + ...inputs, + logits_processor, + stopping_criteria, +}); +``` diff --git a/packages/transformers-llguidance/jest.config.mjs b/packages/transformers-llguidance/jest.config.mjs new file mode 100644 index 000000000..b905ca752 --- /dev/null +++ b/packages/transformers-llguidance/jest.config.mjs @@ -0,0 +1,11 @@ +/** @type {import('jest').Config} */ +export default { + clearMocks: true, + collectCoverage: true, + coverageDirectory: "coverage", + coveragePathIgnorePatterns: ["node_modules", "tests"], + coverageProvider: "v8", + roots: ["./tests/"], + testTimeout: 32000, + transform: {}, +}; diff --git a/packages/transformers-llguidance/package.json b/packages/transformers-llguidance/package.json new file mode 100644 index 000000000..27fe44062 --- /dev/null +++ b/packages/transformers-llguidance/package.json @@ -0,0 +1,68 @@ +{ + "name": "@huggingface/transformers-llguidance", + "version": "0.0.0", + "description": "llguidance integration helpers for Transformers.js constrained generation", + "main": "./dist/index.cjs", + "types": "./types/index.d.ts", + "type": "module", + "exports": { + "import": { + "types": "./types/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./types/index.d.ts", + "default": "./dist/index.cjs" + } + }, + "scripts": { + "format": "prettier --write . --ignore-path ../../.prettierignore", + "format:check": "prettier --check . --ignore-path ../../.prettierignore", + "typegen": "tsc --build", + "dev": "node scripts/dev.mjs", + "build": "node scripts/build.mjs && pnpm typegen", + "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --verbose" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/huggingface/transformers.js.git" + }, + "keywords": [ + "transformers", + "transformers.js", + "huggingface", + "llguidance", + "constrained-generation" + ], + "author": "Hugging Face", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/huggingface/transformers.js/issues" + }, + "homepage": "https://github.com/huggingface/transformers.js#readme", + "peerDependencies": { + "@huggingface/transformers": "^4.2.0" + }, + "devDependencies": { + "@huggingface/transformers": "workspace:*", + "@types/jest": "^30.0.0", + "@types/node": "^24.1.0", + "esbuild": "^0.27.2", + "jest": "^30.2.0", + "typescript": "5.9.3" + }, + "files": [ + "src", + "dist", + "types", + "README.md", + "LICENSE", + "!**/*.tsbuildinfo" + ], + "publishConfig": { + "access": "public" + }, + "dependencies": { + "llguidance": "^0.1.7" + } +} diff --git a/packages/transformers-llguidance/scripts/build.mjs b/packages/transformers-llguidance/scripts/build.mjs new file mode 100644 index 000000000..9b5ca9f5a --- /dev/null +++ b/packages/transformers-llguidance/scripts/build.mjs @@ -0,0 +1,26 @@ +import { build } from "esbuild"; +import { rmSync } from "node:fs"; + +rmSync("dist", { recursive: true, force: true }); + +const common = { + entryPoints: ["src/index.ts"], + bundle: true, + platform: "neutral", + target: "es2022", + sourcemap: true, + external: ["@huggingface/transformers", "llguidance"], +}; + +await Promise.all([ + build({ + ...common, + format: "esm", + outfile: "dist/index.js", + }), + build({ + ...common, + format: "cjs", + outfile: "dist/index.cjs", + }), +]); diff --git a/packages/transformers-llguidance/scripts/dev.mjs b/packages/transformers-llguidance/scripts/dev.mjs new file mode 100644 index 000000000..4b75dfcad --- /dev/null +++ b/packages/transformers-llguidance/scripts/dev.mjs @@ -0,0 +1,34 @@ +import { context } from "esbuild"; +import { rmSync } from "node:fs"; + +rmSync("dist", { recursive: true, force: true }); + +const common = { + entryPoints: ["src/index.ts"], + bundle: true, + platform: "neutral", + target: "es2022", + sourcemap: true, + external: ["@huggingface/transformers", "llguidance"], +}; + +const contexts = await Promise.all([ + context({ + ...common, + format: "esm", + outfile: "dist/index.js", + }), + context({ + ...common, + format: "cjs", + outfile: "dist/index.cjs", + }), +]); + +await Promise.all(contexts.map((ctx) => ctx.watch())); +console.log("Watching @huggingface/transformers-llguidance..."); + +process.on("SIGINT", async () => { + await Promise.all(contexts.map((ctx) => ctx.dispose())); + process.exit(0); +}); diff --git a/packages/transformers-llguidance/src/index.ts b/packages/transformers-llguidance/src/index.ts new file mode 100644 index 000000000..a578cff5a --- /dev/null +++ b/packages/transformers-llguidance/src/index.ts @@ -0,0 +1,250 @@ +import { LogitsProcessor, LogitsProcessorList, StoppingCriteria, type Tensor } from '@huggingface/transformers'; +import { type LLGuidanceResponseFormat, loadBundledLLGuidance } from 'llguidance'; + +export type ResponseFormat = LLGuidanceResponseFormat; + +type GuidanceMask = Uint32Array | Uint8Array | boolean[] | number[]; + +type GuidanceMaskResult = + | { mask: GuidanceMask; vocabSize?: number; stop?: false } + | { stop: true } + | { backtrack?: number; ffTokens?: number[] }; + +type GuidanceCommitResult = { + stop?: boolean; + backtrack?: number; + ffTokens?: number[]; +}; + +type GuidanceInterpreter = { + computeMask(): GuidanceMaskResult; + commitToken(tokenId: number): GuidanceCommitResult | undefined; +}; + +type LlguidanceState = { + completed: boolean; + interpreter: GuidanceInterpreter; + step: number; +}; + +export class LlguidanceConstraint { + static async fromResponseFormat(tokenizer: unknown, response_format: ResponseFormat) { + console.log('[LlguidanceConstraint] loading llguidance', { + response_format, + }); + + const runtime = await loadBundledLLGuidance(); + const interpreter = runtime.createInterpreter({ + tokenizer, + response_format, + }) as GuidanceInterpreter; + + console.log('[LlguidanceConstraint] interpreter created'); + + const state: LlguidanceState = { + completed: false, + interpreter, + step: 0, + }; + + const logits_processor = new LogitsProcessorList(); + logits_processor.push(new LlguidanceLogitsProcessor(state)); + + return { + logits_processor, + stopping_criteria: new LlguidanceStoppingCriteria(state), + }; + } +} + +class LlguidanceLogitsProcessor extends LogitsProcessor { + private state: LlguidanceState; + + constructor(state: LlguidanceState) { + super(); + this.state = state; + } + + _call(_inputIds: bigint[][], logits: Tensor) { + if (this.state.completed) { + console.log('[LlguidanceLogitsProcessor] skip completed', { + step: this.state.step, + }); + return logits; + } + + this.state.step++; + const vocabSize = logits.dims.at(-1); + console.log('[LlguidanceLogitsProcessor] compute mask', { + step: this.state.step, + logitsDims: logits.dims, + vocabSize, + }); + + let result: GuidanceMaskResult; + try { + result = this.state.interpreter.computeMask(); + } catch (error) { + if (!String((error as Error).message).includes('compute_mask() called after stop')) { + throw error; + } + this.state.completed = true; + console.log('[LlguidanceLogitsProcessor] compute after stop', { + step: this.state.step, + }); + return logits; + } + + console.log('[LlguidanceLogitsProcessor] mask result', { + step: this.state.step, + result: summarizeMaskResult(result, vocabSize), + }); + + if ('stop' in result && result.stop) { + this.state.completed = true; + console.log('[LlguidanceLogitsProcessor] stopped by mask', { + step: this.state.step, + }); + return logits; + } + + if ('mask' in result) { + const applied = applyMask(logits, result.mask, result.vocabSize ?? vocabSize); + console.log('[LlguidanceLogitsProcessor] mask applied', { + step: this.state.step, + ...applied, + }); + } + + return logits; + } + + onTokenSampled(tokenId: number, batchIdx: number, inputIds: bigint[][]) { + console.log('[LlguidanceLogitsProcessor] token sampled', { + step: this.state.step, + tokenId, + batchIdx, + inputLength: inputIds[batchIdx]?.length, + completed: this.state.completed, + }); + + if (this.state.completed) return; + + const result = this.state.interpreter.commitToken(tokenId); + console.log('[LlguidanceLogitsProcessor] token committed', { + step: this.state.step, + tokenId, + result, + }); + + if (result?.stop) { + this.state.completed = true; + console.log('[LlguidanceLogitsProcessor] stopped by commit', { + step: this.state.step, + tokenId, + }); + } + } + + onTokensSampled(tokenIds: number[], inputIds: bigint[][]) { + console.log('[LlguidanceLogitsProcessor] tokens sampled', { + step: this.state.step, + tokenIds, + inputLengths: inputIds.map((ids) => ids.length), + completed: this.state.completed, + }); + + for (let batchIdx = 0; batchIdx < tokenIds.length; ++batchIdx) { + this.onTokenSampled(tokenIds[batchIdx], batchIdx, inputIds); + } + } +} + +class LlguidanceStoppingCriteria extends StoppingCriteria { + private state: LlguidanceState; + + constructor(state: LlguidanceState) { + super(); + this.state = state; + } + + _call(inputIds: ArrayLike[]) { + const result = new Array(inputIds.length).fill(this.state.completed); + console.log('[LlguidanceStoppingCriteria] call', { + step: this.state.step, + completed: this.state.completed, + result, + inputLengths: inputIds.map((ids) => ids.length), + }); + return result; + } +} + +function summarizeMaskResult(result: GuidanceMaskResult, vocabSize?: number) { + if ('stop' in result && result.stop) { + return { stop: true }; + } + + if (!('mask' in result)) { + return result; + } + + return { + maskLength: result.mask.length, + vocabSize: result.vocabSize ?? vocabSize, + allowed: countAllowed(result.mask, result.vocabSize ?? vocabSize), + sampleAllowedTokenIds: sampleAllowedTokenIds(result.mask, result.vocabSize ?? vocabSize), + }; +} + +function countAllowed(mask: GuidanceMask, vocabSize?: number) { + if (!vocabSize) return undefined; + + let allowed = 0; + for (let tokenId = 0; tokenId < vocabSize; ++tokenId) { + if (isAllowed(mask, tokenId, vocabSize)) allowed++; + } + return allowed; +} + +function sampleAllowedTokenIds(mask: GuidanceMask, vocabSize?: number) { + if (!vocabSize) return []; + + const tokenIds: number[] = []; + for (let tokenId = 0; tokenId < vocabSize && tokenIds.length < 25; ++tokenId) { + if (isAllowed(mask, tokenId, vocabSize)) tokenIds.push(tokenId); + } + return tokenIds; +} + +function isAllowed(mask: GuidanceMask, tokenId: number, vocabSize: number) { + if (mask.length >= vocabSize) { + return Boolean(mask[tokenId]); + } + return Boolean(Number(mask[tokenId >> 5]) & (1 << (tokenId & 31))); +} + +function applyMask(logits: Tensor, mask: GuidanceMask, vocabSize?: number) { + if (!vocabSize) { + return { vocabSize, batchSize: 0, masked: 0, allowed: undefined }; + } + + const data = logits.data as Float32Array | Float64Array | number[]; + const batchSize = Math.max(1, data.length / vocabSize); + let masked = 0; + let allowed = 0; + + for (let batch = 0; batch < batchSize; ++batch) { + const offset = batch * vocabSize; + for (let tokenId = 0; tokenId < vocabSize; ++tokenId) { + if (!isAllowed(mask, tokenId, vocabSize)) { + data[offset + tokenId] = -Infinity; + masked++; + } else if (batch === 0) { + allowed++; + } + } + } + + return { vocabSize, batchSize, masked, allowed }; +} diff --git a/packages/transformers-llguidance/tests/llguidance-constraint.test.js b/packages/transformers-llguidance/tests/llguidance-constraint.test.js new file mode 100644 index 000000000..53e5c462d --- /dev/null +++ b/packages/transformers-llguidance/tests/llguidance-constraint.test.js @@ -0,0 +1,52 @@ +import { jest } from "@jest/globals"; +import { Tensor } from "@huggingface/transformers"; + +const computeMask = jest.fn(); +const commitToken = jest.fn(); +const createInterpreter = jest.fn(() => ({ computeMask, commitToken })); +const loadBundledLLGuidance = jest.fn(async () => ({ createInterpreter })); + +jest.unstable_mockModule("llguidance", () => ({ + loadBundledLLGuidance, +})); + +const { LlguidanceConstraint } = await import("../dist/index.js"); + +describe("LlguidanceConstraint", () => { + beforeEach(() => { + computeMask.mockReset(); + commitToken.mockReset(); + createInterpreter.mockClear(); + loadBundledLLGuidance.mockClear(); + }); + + it("loads llguidance and applies masks", async () => { + computeMask.mockReturnValue({ mask: [true, false, true, false], vocabSize: 4 }); + + const tokenizer = { name: "tokenizer" }; + const response_format = { type: "json_schema", json_schema: { type: "object" } }; + const { logits_processor } = await LlguidanceConstraint.fromResponseFormat(tokenizer, response_format); + const logits = new Tensor("float32", new Float32Array([1, 2, 3, 4, 5, 6, 7, 8]), [2, 4]); + + logits_processor([[0n], [0n]], logits); + + expect(loadBundledLLGuidance).toHaveBeenCalledTimes(1); + expect(createInterpreter).toHaveBeenCalledWith({ tokenizer, response_format }); + expect(Array.from(logits.data)).toEqual([1, -Infinity, 3, -Infinity, 5, -Infinity, 7, -Infinity]); + }); + + it("commits sampled tokens and stops when llguidance stops", async () => { + computeMask.mockReturnValue({ mask: [true, true], vocabSize: 2 }); + commitToken.mockReturnValueOnce(undefined).mockReturnValueOnce({ stop: true }); + + const { logits_processor, stopping_criteria } = await LlguidanceConstraint.fromResponseFormat({}, { type: "json_object" }); + + expect(stopping_criteria([[0n]])).toEqual([false]); + + logits_processor.onTokensSampled([0, 1], [[0n, 0n], [0n, 1n]]); + + expect(commitToken).toHaveBeenCalledWith(0); + expect(commitToken).toHaveBeenCalledWith(1); + expect(stopping_criteria([[0n, 0n], [0n, 1n]])).toEqual([true, true]); + }); +}); diff --git a/packages/transformers-llguidance/tsconfig.json b/packages/transformers-llguidance/tsconfig.json new file mode 100644 index 000000000..0a88cabdd --- /dev/null +++ b/packages/transformers-llguidance/tsconfig.json @@ -0,0 +1,21 @@ +{ + "include": ["src/**/*"], + "compilerOptions": { + "target": "esnext", + "module": "esnext", + "moduleResolution": "bundler", + "outDir": "types", + "rootDir": "src", + "strict": true, + "skipLibCheck": true, + "declaration": true, + "declarationMap": true, + "noEmit": false, + "emitDeclarationOnly": true, + "esModuleInterop": true, + "composite": true + }, + "typeAcquisition": { + "include": ["jest"] + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3ed937a04..b0585df63 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -58,6 +58,31 @@ importers: specifier: 5.9.3 version: 5.9.3 + packages/transformers-llguidance: + dependencies: + llguidance: + specifier: ^0.1.7 + version: 0.1.7 + devDependencies: + '@huggingface/transformers': + specifier: workspace:* + version: link:../transformers + '@types/jest': + specifier: ^30.0.0 + version: 30.0.0 + '@types/node': + specifier: ^24.1.0 + version: 24.10.9 + esbuild: + specifier: ^0.27.2 + version: 0.27.2 + jest: + specifier: ^30.2.0 + version: 30.2.0(@types/node@24.10.9) + typescript: + specifier: 5.9.3 + version: 5.9.3 + packages: '@babel/code-frame@7.28.6': @@ -427,105 +452,89 @@ packages: resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-arm@1.2.4': resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-ppc64@1.2.4': resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} cpu: [ppc64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-riscv64@1.2.4': resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} cpu: [riscv64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-s390x@1.2.4': resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-x64@1.2.4': resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linuxmusl-arm64@1.2.4': resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-libvips-linuxmusl-x64@1.2.4': resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-linux-arm64@0.34.5': resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-linux-arm@0.34.5': resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-linux-ppc64@0.34.5': resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [ppc64] os: [linux] - libc: [glibc] '@img/sharp-linux-riscv64@0.34.5': resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [riscv64] os: [linux] - libc: [glibc] '@img/sharp-linux-s390x@0.34.5': resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-linux-x64@0.34.5': resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-linuxmusl-arm64@0.34.5': resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-linuxmusl-x64@0.34.5': resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-wasm32@0.34.5': resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} @@ -816,49 +825,41 @@ packages: resolution: {integrity: sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==} cpu: [arm64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-arm64-musl@1.11.1': resolution: {integrity: sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==} cpu: [arm64] os: [linux] - libc: [musl] '@unrs/resolver-binding-linux-ppc64-gnu@1.11.1': resolution: {integrity: sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==} cpu: [ppc64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-riscv64-gnu@1.11.1': resolution: {integrity: sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==} cpu: [riscv64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-riscv64-musl@1.11.1': resolution: {integrity: sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==} cpu: [riscv64] os: [linux] - libc: [musl] '@unrs/resolver-binding-linux-s390x-gnu@1.11.1': resolution: {integrity: sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==} cpu: [s390x] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-x64-gnu@1.11.1': resolution: {integrity: sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==} cpu: [x64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-x64-musl@1.11.1': resolution: {integrity: sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==} cpu: [x64] os: [linux] - libc: [musl] '@unrs/resolver-binding-wasm32-wasi@1.11.1': resolution: {integrity: sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==} @@ -1584,6 +1585,9 @@ packages: linkify-it@5.0.0: resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==} + llguidance@0.1.7: + resolution: {integrity: sha512-r55h3hDmkq313cFlqbKyx4IebqpreyrrnMJip7IR+ocwPGm7O6ZoDVCIAEg2wd6k7pQ5wWyuSzV9rSefqKVs6Q==} + locate-path@5.0.0: resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==} engines: {node: '>=8'} @@ -3746,6 +3750,8 @@ snapshots: dependencies: uc.micro: 2.1.0 + llguidance@0.1.7: {} + locate-path@5.0.0: dependencies: p-locate: 4.1.0 From 8b7dc30b1111b6f6dc639eeb03adac86fc4cb628 Mon Sep 17 00:00:00 2001 From: Nico Martin Date: Fri, 10 Jul 2026 12:03:01 +0200 Subject: [PATCH 2/5] added LlguidanceConstraint --- .../transformers-llguidance/scripts/dev.mjs | 31 +++++ packages/transformers-llguidance/src/index.ts | 126 +++++++++++++++--- .../tests/llguidance-constraint.test.js | 6 + packages/transformers/src/transformers.js | 2 + 4 files changed, 148 insertions(+), 17 deletions(-) diff --git a/packages/transformers-llguidance/scripts/dev.mjs b/packages/transformers-llguidance/scripts/dev.mjs index 4b75dfcad..4c5186e26 100644 --- a/packages/transformers-llguidance/scripts/dev.mjs +++ b/packages/transformers-llguidance/scripts/dev.mjs @@ -1,7 +1,30 @@ import { context } from "esbuild"; +import { spawn } from "node:child_process"; import { rmSync } from "node:fs"; rmSync("dist", { recursive: true, force: true }); +rmSync("types", { recursive: true, force: true }); + +const watchLogger = { + name: "watch-logger", + setup(build) { + let startTime = 0; + + build.onStart(() => { + startTime = performance.now(); + console.log(`[transformers-llguidance] rebuilding ${build.initialOptions.outfile}...`); + }); + + build.onEnd((result) => { + const duration = (performance.now() - startTime).toFixed(2); + if (result.errors.length > 0) { + console.log(`[transformers-llguidance] rebuild failed in ${duration}ms`); + } else { + console.log(`[transformers-llguidance] rebuilt ${build.initialOptions.outfile} in ${duration}ms`); + } + }); + }, +}; const common = { entryPoints: ["src/index.ts"], @@ -10,6 +33,7 @@ const common = { target: "es2022", sourcemap: true, external: ["@huggingface/transformers", "llguidance"], + plugins: [watchLogger], }; const contexts = await Promise.all([ @@ -26,9 +50,16 @@ const contexts = await Promise.all([ ]); await Promise.all(contexts.map((ctx) => ctx.watch())); + +const tscWatch = spawn("tsc", ["--build", "--watch", "--preserveWatchOutput"], { + stdio: "inherit", + shell: true, +}); + console.log("Watching @huggingface/transformers-llguidance..."); process.on("SIGINT", async () => { + tscWatch.kill(); await Promise.all(contexts.map((ctx) => ctx.dispose())); process.exit(0); }); diff --git a/packages/transformers-llguidance/src/index.ts b/packages/transformers-llguidance/src/index.ts index a578cff5a..7578d85dc 100644 --- a/packages/transformers-llguidance/src/index.ts +++ b/packages/transformers-llguidance/src/index.ts @@ -1,8 +1,27 @@ -import { LogitsProcessor, LogitsProcessorList, StoppingCriteria, type Tensor } from '@huggingface/transformers'; -import { type LLGuidanceResponseFormat, loadBundledLLGuidance } from 'llguidance'; +import { + LogitsProcessor, + LogitsProcessorList, + StoppingCriteria, + env, + loadWasmBinary, + loadWasmFactory, + logger, + type Tensor, +} from '@huggingface/transformers'; +import { type LLGuidanceResponseFormat, type LoadBundledLLGuidanceOptions, loadBundledLLGuidance } from 'llguidance'; export type ResponseFormat = LLGuidanceResponseFormat; +export type LlguidanceLoadOptions = LoadBundledLLGuidanceOptions & { + /** Whether to pre-load and cache llguidance WASM assets. Defaults to env.useWasmCache. */ + useWasmCache?: boolean; +}; + +const LLGUIDANCE_VERSION = '0.1.7'; +const LLGUIDANCE_WASM_BASE = `https://cdn.jsdelivr.net/npm/llguidance@${LLGUIDANCE_VERSION}/wasm/`; +const DEFAULT_LLGUIDANCE_WASM_URL = `${LLGUIDANCE_WASM_BASE}llguidance_wasm_bg.wasm`; +const DEFAULT_LLGUIDANCE_WASM_FACTORY_URL = `${LLGUIDANCE_WASM_BASE}llguidance_wasm.js`; + type GuidanceMask = Uint32Array | Uint8Array | boolean[] | number[]; type GuidanceMaskResult = @@ -28,18 +47,22 @@ type LlguidanceState = { }; export class LlguidanceConstraint { - static async fromResponseFormat(tokenizer: unknown, response_format: ResponseFormat) { - console.log('[LlguidanceConstraint] loading llguidance', { + static async fromResponseFormat( + tokenizer: unknown, + response_format: ResponseFormat, + loadOptions: LlguidanceLoadOptions = {}, + ) { + logger.debug('[LlguidanceConstraint] loading llguidance', { response_format, }); - const runtime = await loadBundledLLGuidance(); + const runtime = await loadLLGuidance(loadOptions); const interpreter = runtime.createInterpreter({ tokenizer, response_format, }) as GuidanceInterpreter; - console.log('[LlguidanceConstraint] interpreter created'); + logger.debug('[LlguidanceConstraint] interpreter created'); const state: LlguidanceState = { completed: false, @@ -57,6 +80,75 @@ export class LlguidanceConstraint { } } +async function loadLLGuidance(loadOptions: LlguidanceLoadOptions) { + const { useWasmCache = env.useWasmCache, ...options } = loadOptions; + if (!useWasmCache || isNodeLikeRuntime() || options.wasmFactory) { + return loadBundledLLGuidance(options); + } + + const wasmSource = options.wasm ?? options.wasmUrl ?? DEFAULT_LLGUIDANCE_WASM_URL; + const wasmFactorySource = options.wasmFactoryUrl ?? DEFAULT_LLGUIDANCE_WASM_FACTORY_URL; + const cachedOptions = { ...options }; + + const [wasm, wasmFactoryUrl] = await Promise.all([ + loadCacheableWasm(wasmSource), + loadCacheableWasmFactory(wasmFactorySource), + ]); + + if (wasm) { + cachedOptions.wasm = wasm; + delete cachedOptions.wasmUrl; + } + + if (wasm && wasmFactoryUrl) { + cachedOptions.wasmFactoryUrl = wasmFactoryUrl; + } + + return loadBundledLLGuidance(cachedOptions); +} + +async function loadCacheableWasm(source: LoadBundledLLGuidanceOptions['wasm']) { + const url = toCacheableURL(source); + if (!url) return null; + + try { + return await loadWasmBinary(url); + } catch (error) { + logger.warn('Failed to pre-load llguidance WASM binary:', error); + return null; + } +} + +async function loadCacheableWasmFactory(source: LoadBundledLLGuidanceOptions['wasmFactoryUrl']) { + const url = toCacheableURL(source); + if (!url) return null; + + try { + return await loadWasmFactory(url); + } catch (error) { + logger.warn('Failed to pre-load llguidance WASM factory:', error); + return null; + } +} + +function toCacheableURL(source: unknown) { + if (typeof source === 'string') { + return isBlobURL(source) ? null : new URL(source, globalThis.location?.href).href; + } + if (source instanceof URL) { + return isBlobURL(source.href) ? null : source.href; + } + return null; +} + +function isBlobURL(url: string) { + return url.startsWith('blob:'); +} + +function isNodeLikeRuntime() { + return typeof process !== 'undefined' && Boolean(process.versions?.node); +} + class LlguidanceLogitsProcessor extends LogitsProcessor { private state: LlguidanceState; @@ -67,7 +159,7 @@ class LlguidanceLogitsProcessor extends LogitsProcessor { _call(_inputIds: bigint[][], logits: Tensor) { if (this.state.completed) { - console.log('[LlguidanceLogitsProcessor] skip completed', { + logger.debug('[LlguidanceLogitsProcessor] skip completed', { step: this.state.step, }); return logits; @@ -75,7 +167,7 @@ class LlguidanceLogitsProcessor extends LogitsProcessor { this.state.step++; const vocabSize = logits.dims.at(-1); - console.log('[LlguidanceLogitsProcessor] compute mask', { + logger.debug('[LlguidanceLogitsProcessor] compute mask', { step: this.state.step, logitsDims: logits.dims, vocabSize, @@ -89,20 +181,20 @@ class LlguidanceLogitsProcessor extends LogitsProcessor { throw error; } this.state.completed = true; - console.log('[LlguidanceLogitsProcessor] compute after stop', { + logger.debug('[LlguidanceLogitsProcessor] compute after stop', { step: this.state.step, }); return logits; } - console.log('[LlguidanceLogitsProcessor] mask result', { + logger.debug('[LlguidanceLogitsProcessor] mask result', { step: this.state.step, result: summarizeMaskResult(result, vocabSize), }); if ('stop' in result && result.stop) { this.state.completed = true; - console.log('[LlguidanceLogitsProcessor] stopped by mask', { + logger.debug('[LlguidanceLogitsProcessor] stopped by mask', { step: this.state.step, }); return logits; @@ -110,7 +202,7 @@ class LlguidanceLogitsProcessor extends LogitsProcessor { if ('mask' in result) { const applied = applyMask(logits, result.mask, result.vocabSize ?? vocabSize); - console.log('[LlguidanceLogitsProcessor] mask applied', { + logger.debug('[LlguidanceLogitsProcessor] mask applied', { step: this.state.step, ...applied, }); @@ -120,7 +212,7 @@ class LlguidanceLogitsProcessor extends LogitsProcessor { } onTokenSampled(tokenId: number, batchIdx: number, inputIds: bigint[][]) { - console.log('[LlguidanceLogitsProcessor] token sampled', { + logger.debug('[LlguidanceLogitsProcessor] token sampled', { step: this.state.step, tokenId, batchIdx, @@ -131,7 +223,7 @@ class LlguidanceLogitsProcessor extends LogitsProcessor { if (this.state.completed) return; const result = this.state.interpreter.commitToken(tokenId); - console.log('[LlguidanceLogitsProcessor] token committed', { + logger.debug('[LlguidanceLogitsProcessor] token committed', { step: this.state.step, tokenId, result, @@ -139,7 +231,7 @@ class LlguidanceLogitsProcessor extends LogitsProcessor { if (result?.stop) { this.state.completed = true; - console.log('[LlguidanceLogitsProcessor] stopped by commit', { + logger.debug('[LlguidanceLogitsProcessor] stopped by commit', { step: this.state.step, tokenId, }); @@ -147,7 +239,7 @@ class LlguidanceLogitsProcessor extends LogitsProcessor { } onTokensSampled(tokenIds: number[], inputIds: bigint[][]) { - console.log('[LlguidanceLogitsProcessor] tokens sampled', { + logger.debug('[LlguidanceLogitsProcessor] tokens sampled', { step: this.state.step, tokenIds, inputLengths: inputIds.map((ids) => ids.length), @@ -170,7 +262,7 @@ class LlguidanceStoppingCriteria extends StoppingCriteria { _call(inputIds: ArrayLike[]) { const result = new Array(inputIds.length).fill(this.state.completed); - console.log('[LlguidanceStoppingCriteria] call', { + logger.debug('[LlguidanceStoppingCriteria] call', { step: this.state.step, completed: this.state.completed, result, diff --git a/packages/transformers-llguidance/tests/llguidance-constraint.test.js b/packages/transformers-llguidance/tests/llguidance-constraint.test.js index 53e5c462d..c9f32a42d 100644 --- a/packages/transformers-llguidance/tests/llguidance-constraint.test.js +++ b/packages/transformers-llguidance/tests/llguidance-constraint.test.js @@ -35,6 +35,12 @@ describe("LlguidanceConstraint", () => { expect(Array.from(logits.data)).toEqual([1, -Infinity, 3, -Infinity, 5, -Infinity, 7, -Infinity]); }); + it("passes explicit llguidance load options", async () => { + await LlguidanceConstraint.fromResponseFormat({}, { type: "json_object" }, { useWasmCache: false, wasmUrl: "custom.wasm" }); + + expect(loadBundledLLGuidance).toHaveBeenCalledWith({ wasmUrl: "custom.wasm" }); + }); + it("commits sampled tokens and stops when llguidance stops", async () => { computeMask.mockReturnValue({ mask: [true, true], vocabSize: 2 }); commitToken.mockReturnValueOnce(undefined).mockReturnValueOnce({ stop: true }); diff --git a/packages/transformers/src/transformers.js b/packages/transformers/src/transformers.js index ef01569ef..7d6168020 100644 --- a/packages/transformers/src/transformers.js +++ b/packages/transformers/src/transformers.js @@ -52,6 +52,8 @@ export { load_video, RawVideo, RawVideoFrame } from './utils/video.js'; export * from './utils/tensor.js'; export { softmax, log_softmax, dot, cos_sim } from './utils/maths.js'; export { random } from './utils/random.js'; +export { logger } from './utils/logger.js'; +export { loadWasmBinary, loadWasmFactory } from './backends/utils/cacheWasm.js'; export { DynamicCache } from './cache_utils.js'; From 097dd6555ce66a28ff3ea46e05e0d59c41c87522 Mon Sep 17 00:00:00 2001 From: Nico Martin Date: Fri, 10 Jul 2026 12:09:25 +0200 Subject: [PATCH 3/5] added regex example --- packages/transformers-llguidance/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/transformers-llguidance/README.md b/packages/transformers-llguidance/README.md index f1776465c..e167b245f 100644 --- a/packages/transformers-llguidance/README.md +++ b/packages/transformers-llguidance/README.md @@ -26,3 +26,23 @@ await model.generate({ stopping_criteria, }); ``` + +## Regex constraints + +Use `type: "regex"` to constrain generation to a regular expression. For example, this only allows ISO-like dates in `YYYY-MM-DD` format: + +```js +import { LlguidanceConstraint } from "@huggingface/transformers-llguidance"; + +const { logits_processor, stopping_criteria } = + await LlguidanceConstraint.fromResponseFormat(tokenizer, { + type: "regex", + regex: "\\d{4}-\\d{2}-\\d{2}", + }); + +const output = await model.generate({ + ...inputs, + logits_processor, + stopping_criteria, +}); +``` From e695be3707e12e12c730faf816f6547d1b8bdd51 Mon Sep 17 00:00:00 2001 From: Nico Martin Date: Sat, 11 Jul 2026 08:22:41 +0200 Subject: [PATCH 4/5] clean up and added more unit tests --- .../src/LlguidanceConstraint.ts | 166 +++++++++ .../transformers-llguidance/src/constants.ts | 4 + packages/transformers-llguidance/src/index.ts | 344 +----------------- .../transformers-llguidance/src/utils/mask.ts | 72 ++++ .../src/utils/runtime.ts | 3 + .../src/utils/types.ts | 23 ++ .../transformers-llguidance/src/utils/wasm.ts | 75 ++++ .../tests/llguidance-constraint.test.js | 73 +++- .../tests/wasm-loading.test.js | 123 +++++++ 9 files changed, 539 insertions(+), 344 deletions(-) create mode 100644 packages/transformers-llguidance/src/LlguidanceConstraint.ts create mode 100644 packages/transformers-llguidance/src/constants.ts create mode 100644 packages/transformers-llguidance/src/utils/mask.ts create mode 100644 packages/transformers-llguidance/src/utils/runtime.ts create mode 100644 packages/transformers-llguidance/src/utils/types.ts create mode 100644 packages/transformers-llguidance/src/utils/wasm.ts create mode 100644 packages/transformers-llguidance/tests/wasm-loading.test.js diff --git a/packages/transformers-llguidance/src/LlguidanceConstraint.ts b/packages/transformers-llguidance/src/LlguidanceConstraint.ts new file mode 100644 index 000000000..bee7b774e --- /dev/null +++ b/packages/transformers-llguidance/src/LlguidanceConstraint.ts @@ -0,0 +1,166 @@ +import { LogitsProcessor, LogitsProcessorList, StoppingCriteria, logger, type Tensor } from '@huggingface/transformers'; +import { type LLGuidanceResponseFormat } from 'llguidance'; + +import { applyMask, summarizeMaskResult } from './utils/mask'; +import { type GuidanceInterpreter, type GuidanceMaskResult, type LlguidanceState } from './utils/types'; +import { type LlguidanceLoadOptions, loadLLGuidance } from './utils/wasm'; + +export type ResponseFormat = LLGuidanceResponseFormat; +export type { LlguidanceLoadOptions }; + +export class LlguidanceConstraint { + static async fromResponseFormat( + tokenizer: unknown, + response_format: ResponseFormat, + loadOptions: LlguidanceLoadOptions = {}, + ) { + logger.debug('[LlguidanceConstraint] loading llguidance', { + response_format, + }); + + const runtime = await loadLLGuidance(loadOptions); + const interpreter = runtime.createInterpreter({ + tokenizer, + response_format, + }) as GuidanceInterpreter; + + logger.debug('[LlguidanceConstraint] interpreter created'); + + const state: LlguidanceState = { + completed: false, + interpreter, + step: 0, + }; + + const logits_processor = new LogitsProcessorList(); + logits_processor.push(new LlguidanceLogitsProcessor(state)); + + return { + logits_processor, + stopping_criteria: new LlguidanceStoppingCriteria(state), + }; + } +} + +class LlguidanceLogitsProcessor extends LogitsProcessor { + private state: LlguidanceState; + + constructor(state: LlguidanceState) { + super(); + this.state = state; + } + + _call(_inputIds: bigint[][], logits: Tensor) { + if (this.state.completed) { + logger.debug('[LlguidanceLogitsProcessor] skip completed', { + step: this.state.step, + }); + return logits; + } + + this.state.step++; + const vocabSize = logits.dims.at(-1); + logger.debug('[LlguidanceLogitsProcessor] compute mask', { + step: this.state.step, + logitsDims: logits.dims, + vocabSize, + }); + + let result: GuidanceMaskResult; + try { + result = this.state.interpreter.computeMask(); + } catch (error) { + if (!String((error as Error).message).includes('compute_mask() called after stop')) { + throw error; + } + this.state.completed = true; + logger.debug('[LlguidanceLogitsProcessor] compute after stop', { + step: this.state.step, + }); + return logits; + } + + logger.debug('[LlguidanceLogitsProcessor] mask result', { + step: this.state.step, + result: summarizeMaskResult(result, vocabSize), + }); + + if ('stop' in result && result.stop) { + this.state.completed = true; + logger.debug('[LlguidanceLogitsProcessor] stopped by mask', { + step: this.state.step, + }); + return logits; + } + + if ('mask' in result) { + const applied = applyMask(logits, result.mask, result.vocabSize ?? vocabSize); + logger.debug('[LlguidanceLogitsProcessor] mask applied', { + step: this.state.step, + ...applied, + }); + } + + return logits; + } + + onTokenSampled(tokenId: number, batchIdx: number, inputIds: bigint[][]) { + logger.debug('[LlguidanceLogitsProcessor] token sampled', { + step: this.state.step, + tokenId, + batchIdx, + inputLength: inputIds[batchIdx]?.length, + completed: this.state.completed, + }); + + if (this.state.completed) return; + + const result = this.state.interpreter.commitToken(tokenId); + logger.debug('[LlguidanceLogitsProcessor] token committed', { + step: this.state.step, + tokenId, + result, + }); + + if (result?.stop) { + this.state.completed = true; + logger.debug('[LlguidanceLogitsProcessor] stopped by commit', { + step: this.state.step, + tokenId, + }); + } + } + + onTokensSampled(tokenIds: number[], inputIds: bigint[][]) { + logger.debug('[LlguidanceLogitsProcessor] tokens sampled', { + step: this.state.step, + tokenIds, + inputLengths: inputIds.map((ids) => ids.length), + completed: this.state.completed, + }); + + for (let batchIdx = 0; batchIdx < tokenIds.length; ++batchIdx) { + this.onTokenSampled(tokenIds[batchIdx], batchIdx, inputIds); + } + } +} + +class LlguidanceStoppingCriteria extends StoppingCriteria { + private state: LlguidanceState; + + constructor(state: LlguidanceState) { + super(); + this.state = state; + } + + _call(inputIds: ArrayLike[]) { + const result = new Array(inputIds.length).fill(this.state.completed); + logger.debug('[LlguidanceStoppingCriteria] call', { + step: this.state.step, + completed: this.state.completed, + result, + inputLengths: inputIds.map((ids) => ids.length), + }); + return result; + } +} diff --git a/packages/transformers-llguidance/src/constants.ts b/packages/transformers-llguidance/src/constants.ts new file mode 100644 index 000000000..773108fac --- /dev/null +++ b/packages/transformers-llguidance/src/constants.ts @@ -0,0 +1,4 @@ +export const LLGUIDANCE_VERSION = '0.1.7'; +export const LLGUIDANCE_WASM_BASE = `https://cdn.jsdelivr.net/npm/llguidance@${LLGUIDANCE_VERSION}/wasm/`; +export const DEFAULT_LLGUIDANCE_WASM_URL = `${LLGUIDANCE_WASM_BASE}llguidance_wasm_bg.wasm`; +export const DEFAULT_LLGUIDANCE_WASM_FACTORY_URL = `${LLGUIDANCE_WASM_BASE}llguidance_wasm.js`; diff --git a/packages/transformers-llguidance/src/index.ts b/packages/transformers-llguidance/src/index.ts index 7578d85dc..a870a56f9 100644 --- a/packages/transformers-llguidance/src/index.ts +++ b/packages/transformers-llguidance/src/index.ts @@ -1,342 +1,2 @@ -import { - LogitsProcessor, - LogitsProcessorList, - StoppingCriteria, - env, - loadWasmBinary, - loadWasmFactory, - logger, - type Tensor, -} from '@huggingface/transformers'; -import { type LLGuidanceResponseFormat, type LoadBundledLLGuidanceOptions, loadBundledLLGuidance } from 'llguidance'; - -export type ResponseFormat = LLGuidanceResponseFormat; - -export type LlguidanceLoadOptions = LoadBundledLLGuidanceOptions & { - /** Whether to pre-load and cache llguidance WASM assets. Defaults to env.useWasmCache. */ - useWasmCache?: boolean; -}; - -const LLGUIDANCE_VERSION = '0.1.7'; -const LLGUIDANCE_WASM_BASE = `https://cdn.jsdelivr.net/npm/llguidance@${LLGUIDANCE_VERSION}/wasm/`; -const DEFAULT_LLGUIDANCE_WASM_URL = `${LLGUIDANCE_WASM_BASE}llguidance_wasm_bg.wasm`; -const DEFAULT_LLGUIDANCE_WASM_FACTORY_URL = `${LLGUIDANCE_WASM_BASE}llguidance_wasm.js`; - -type GuidanceMask = Uint32Array | Uint8Array | boolean[] | number[]; - -type GuidanceMaskResult = - | { mask: GuidanceMask; vocabSize?: number; stop?: false } - | { stop: true } - | { backtrack?: number; ffTokens?: number[] }; - -type GuidanceCommitResult = { - stop?: boolean; - backtrack?: number; - ffTokens?: number[]; -}; - -type GuidanceInterpreter = { - computeMask(): GuidanceMaskResult; - commitToken(tokenId: number): GuidanceCommitResult | undefined; -}; - -type LlguidanceState = { - completed: boolean; - interpreter: GuidanceInterpreter; - step: number; -}; - -export class LlguidanceConstraint { - static async fromResponseFormat( - tokenizer: unknown, - response_format: ResponseFormat, - loadOptions: LlguidanceLoadOptions = {}, - ) { - logger.debug('[LlguidanceConstraint] loading llguidance', { - response_format, - }); - - const runtime = await loadLLGuidance(loadOptions); - const interpreter = runtime.createInterpreter({ - tokenizer, - response_format, - }) as GuidanceInterpreter; - - logger.debug('[LlguidanceConstraint] interpreter created'); - - const state: LlguidanceState = { - completed: false, - interpreter, - step: 0, - }; - - const logits_processor = new LogitsProcessorList(); - logits_processor.push(new LlguidanceLogitsProcessor(state)); - - return { - logits_processor, - stopping_criteria: new LlguidanceStoppingCriteria(state), - }; - } -} - -async function loadLLGuidance(loadOptions: LlguidanceLoadOptions) { - const { useWasmCache = env.useWasmCache, ...options } = loadOptions; - if (!useWasmCache || isNodeLikeRuntime() || options.wasmFactory) { - return loadBundledLLGuidance(options); - } - - const wasmSource = options.wasm ?? options.wasmUrl ?? DEFAULT_LLGUIDANCE_WASM_URL; - const wasmFactorySource = options.wasmFactoryUrl ?? DEFAULT_LLGUIDANCE_WASM_FACTORY_URL; - const cachedOptions = { ...options }; - - const [wasm, wasmFactoryUrl] = await Promise.all([ - loadCacheableWasm(wasmSource), - loadCacheableWasmFactory(wasmFactorySource), - ]); - - if (wasm) { - cachedOptions.wasm = wasm; - delete cachedOptions.wasmUrl; - } - - if (wasm && wasmFactoryUrl) { - cachedOptions.wasmFactoryUrl = wasmFactoryUrl; - } - - return loadBundledLLGuidance(cachedOptions); -} - -async function loadCacheableWasm(source: LoadBundledLLGuidanceOptions['wasm']) { - const url = toCacheableURL(source); - if (!url) return null; - - try { - return await loadWasmBinary(url); - } catch (error) { - logger.warn('Failed to pre-load llguidance WASM binary:', error); - return null; - } -} - -async function loadCacheableWasmFactory(source: LoadBundledLLGuidanceOptions['wasmFactoryUrl']) { - const url = toCacheableURL(source); - if (!url) return null; - - try { - return await loadWasmFactory(url); - } catch (error) { - logger.warn('Failed to pre-load llguidance WASM factory:', error); - return null; - } -} - -function toCacheableURL(source: unknown) { - if (typeof source === 'string') { - return isBlobURL(source) ? null : new URL(source, globalThis.location?.href).href; - } - if (source instanceof URL) { - return isBlobURL(source.href) ? null : source.href; - } - return null; -} - -function isBlobURL(url: string) { - return url.startsWith('blob:'); -} - -function isNodeLikeRuntime() { - return typeof process !== 'undefined' && Boolean(process.versions?.node); -} - -class LlguidanceLogitsProcessor extends LogitsProcessor { - private state: LlguidanceState; - - constructor(state: LlguidanceState) { - super(); - this.state = state; - } - - _call(_inputIds: bigint[][], logits: Tensor) { - if (this.state.completed) { - logger.debug('[LlguidanceLogitsProcessor] skip completed', { - step: this.state.step, - }); - return logits; - } - - this.state.step++; - const vocabSize = logits.dims.at(-1); - logger.debug('[LlguidanceLogitsProcessor] compute mask', { - step: this.state.step, - logitsDims: logits.dims, - vocabSize, - }); - - let result: GuidanceMaskResult; - try { - result = this.state.interpreter.computeMask(); - } catch (error) { - if (!String((error as Error).message).includes('compute_mask() called after stop')) { - throw error; - } - this.state.completed = true; - logger.debug('[LlguidanceLogitsProcessor] compute after stop', { - step: this.state.step, - }); - return logits; - } - - logger.debug('[LlguidanceLogitsProcessor] mask result', { - step: this.state.step, - result: summarizeMaskResult(result, vocabSize), - }); - - if ('stop' in result && result.stop) { - this.state.completed = true; - logger.debug('[LlguidanceLogitsProcessor] stopped by mask', { - step: this.state.step, - }); - return logits; - } - - if ('mask' in result) { - const applied = applyMask(logits, result.mask, result.vocabSize ?? vocabSize); - logger.debug('[LlguidanceLogitsProcessor] mask applied', { - step: this.state.step, - ...applied, - }); - } - - return logits; - } - - onTokenSampled(tokenId: number, batchIdx: number, inputIds: bigint[][]) { - logger.debug('[LlguidanceLogitsProcessor] token sampled', { - step: this.state.step, - tokenId, - batchIdx, - inputLength: inputIds[batchIdx]?.length, - completed: this.state.completed, - }); - - if (this.state.completed) return; - - const result = this.state.interpreter.commitToken(tokenId); - logger.debug('[LlguidanceLogitsProcessor] token committed', { - step: this.state.step, - tokenId, - result, - }); - - if (result?.stop) { - this.state.completed = true; - logger.debug('[LlguidanceLogitsProcessor] stopped by commit', { - step: this.state.step, - tokenId, - }); - } - } - - onTokensSampled(tokenIds: number[], inputIds: bigint[][]) { - logger.debug('[LlguidanceLogitsProcessor] tokens sampled', { - step: this.state.step, - tokenIds, - inputLengths: inputIds.map((ids) => ids.length), - completed: this.state.completed, - }); - - for (let batchIdx = 0; batchIdx < tokenIds.length; ++batchIdx) { - this.onTokenSampled(tokenIds[batchIdx], batchIdx, inputIds); - } - } -} - -class LlguidanceStoppingCriteria extends StoppingCriteria { - private state: LlguidanceState; - - constructor(state: LlguidanceState) { - super(); - this.state = state; - } - - _call(inputIds: ArrayLike[]) { - const result = new Array(inputIds.length).fill(this.state.completed); - logger.debug('[LlguidanceStoppingCriteria] call', { - step: this.state.step, - completed: this.state.completed, - result, - inputLengths: inputIds.map((ids) => ids.length), - }); - return result; - } -} - -function summarizeMaskResult(result: GuidanceMaskResult, vocabSize?: number) { - if ('stop' in result && result.stop) { - return { stop: true }; - } - - if (!('mask' in result)) { - return result; - } - - return { - maskLength: result.mask.length, - vocabSize: result.vocabSize ?? vocabSize, - allowed: countAllowed(result.mask, result.vocabSize ?? vocabSize), - sampleAllowedTokenIds: sampleAllowedTokenIds(result.mask, result.vocabSize ?? vocabSize), - }; -} - -function countAllowed(mask: GuidanceMask, vocabSize?: number) { - if (!vocabSize) return undefined; - - let allowed = 0; - for (let tokenId = 0; tokenId < vocabSize; ++tokenId) { - if (isAllowed(mask, tokenId, vocabSize)) allowed++; - } - return allowed; -} - -function sampleAllowedTokenIds(mask: GuidanceMask, vocabSize?: number) { - if (!vocabSize) return []; - - const tokenIds: number[] = []; - for (let tokenId = 0; tokenId < vocabSize && tokenIds.length < 25; ++tokenId) { - if (isAllowed(mask, tokenId, vocabSize)) tokenIds.push(tokenId); - } - return tokenIds; -} - -function isAllowed(mask: GuidanceMask, tokenId: number, vocabSize: number) { - if (mask.length >= vocabSize) { - return Boolean(mask[tokenId]); - } - return Boolean(Number(mask[tokenId >> 5]) & (1 << (tokenId & 31))); -} - -function applyMask(logits: Tensor, mask: GuidanceMask, vocabSize?: number) { - if (!vocabSize) { - return { vocabSize, batchSize: 0, masked: 0, allowed: undefined }; - } - - const data = logits.data as Float32Array | Float64Array | number[]; - const batchSize = Math.max(1, data.length / vocabSize); - let masked = 0; - let allowed = 0; - - for (let batch = 0; batch < batchSize; ++batch) { - const offset = batch * vocabSize; - for (let tokenId = 0; tokenId < vocabSize; ++tokenId) { - if (!isAllowed(mask, tokenId, vocabSize)) { - data[offset + tokenId] = -Infinity; - masked++; - } else if (batch === 0) { - allowed++; - } - } - } - - return { vocabSize, batchSize, masked, allowed }; -} +export { LlguidanceConstraint } from './LlguidanceConstraint'; +export type { LlguidanceLoadOptions, ResponseFormat } from './LlguidanceConstraint'; diff --git a/packages/transformers-llguidance/src/utils/mask.ts b/packages/transformers-llguidance/src/utils/mask.ts new file mode 100644 index 000000000..f1ee29ba2 --- /dev/null +++ b/packages/transformers-llguidance/src/utils/mask.ts @@ -0,0 +1,72 @@ +import { type Tensor } from '@huggingface/transformers'; + +import { type GuidanceMask, type GuidanceMaskResult } from './types'; + +export function summarizeMaskResult(result: GuidanceMaskResult, vocabSize?: number) { + if ('stop' in result && result.stop) { + return { stop: true }; + } + + if (!('mask' in result)) { + return result; + } + + return { + maskLength: result.mask.length, + vocabSize: result.vocabSize ?? vocabSize, + allowed: countAllowed(result.mask, result.vocabSize ?? vocabSize), + sampleAllowedTokenIds: sampleAllowedTokenIds(result.mask, result.vocabSize ?? vocabSize), + }; +} + +export function applyMask(logits: Tensor, mask: GuidanceMask, vocabSize?: number) { + if (!vocabSize) { + return { vocabSize, batchSize: 0, masked: 0, allowed: undefined }; + } + + const data = logits.data as Float32Array | Float64Array | number[]; + const batchSize = Math.max(1, data.length / vocabSize); + let masked = 0; + let allowed = 0; + + for (let batch = 0; batch < batchSize; ++batch) { + const offset = batch * vocabSize; + for (let tokenId = 0; tokenId < vocabSize; ++tokenId) { + if (!isAllowed(mask, tokenId, vocabSize)) { + data[offset + tokenId] = -Infinity; + masked++; + } else if (batch === 0) { + allowed++; + } + } + } + + return { vocabSize, batchSize, masked, allowed }; +} + +function countAllowed(mask: GuidanceMask, vocabSize?: number) { + if (!vocabSize) return undefined; + + let allowed = 0; + for (let tokenId = 0; tokenId < vocabSize; ++tokenId) { + if (isAllowed(mask, tokenId, vocabSize)) allowed++; + } + return allowed; +} + +function sampleAllowedTokenIds(mask: GuidanceMask, vocabSize?: number) { + if (!vocabSize) return []; + + const tokenIds: number[] = []; + for (let tokenId = 0; tokenId < vocabSize && tokenIds.length < 25; ++tokenId) { + if (isAllowed(mask, tokenId, vocabSize)) tokenIds.push(tokenId); + } + return tokenIds; +} + +function isAllowed(mask: GuidanceMask, tokenId: number, vocabSize: number) { + if (mask.length >= vocabSize) { + return Boolean(mask[tokenId]); + } + return Boolean(Number(mask[tokenId >> 5]) & (1 << (tokenId & 31))); +} diff --git a/packages/transformers-llguidance/src/utils/runtime.ts b/packages/transformers-llguidance/src/utils/runtime.ts new file mode 100644 index 000000000..f2ba25566 --- /dev/null +++ b/packages/transformers-llguidance/src/utils/runtime.ts @@ -0,0 +1,3 @@ +export function isNodeLikeRuntime() { + return typeof process !== 'undefined' && Boolean(process.versions?.node); +} diff --git a/packages/transformers-llguidance/src/utils/types.ts b/packages/transformers-llguidance/src/utils/types.ts new file mode 100644 index 000000000..3024c3601 --- /dev/null +++ b/packages/transformers-llguidance/src/utils/types.ts @@ -0,0 +1,23 @@ +export type GuidanceMask = Uint32Array | Uint8Array | boolean[] | number[]; + +export type GuidanceMaskResult = + | { mask: GuidanceMask; vocabSize?: number; stop?: false } + | { stop: true } + | { backtrack?: number; ffTokens?: number[] }; + +export type GuidanceCommitResult = { + stop?: boolean; + backtrack?: number; + ffTokens?: number[]; +}; + +export type GuidanceInterpreter = { + computeMask(): GuidanceMaskResult; + commitToken(tokenId: number): GuidanceCommitResult | undefined; +}; + +export type LlguidanceState = { + completed: boolean; + interpreter: GuidanceInterpreter; + step: number; +}; diff --git a/packages/transformers-llguidance/src/utils/wasm.ts b/packages/transformers-llguidance/src/utils/wasm.ts new file mode 100644 index 000000000..1e0324104 --- /dev/null +++ b/packages/transformers-llguidance/src/utils/wasm.ts @@ -0,0 +1,75 @@ +import { env, loadWasmBinary, loadWasmFactory, logger } from '@huggingface/transformers'; +import { type LoadBundledLLGuidanceOptions, loadBundledLLGuidance } from 'llguidance'; + +import { DEFAULT_LLGUIDANCE_WASM_FACTORY_URL, DEFAULT_LLGUIDANCE_WASM_URL } from '../constants'; +import { isNodeLikeRuntime } from './runtime'; + +export type LlguidanceLoadOptions = LoadBundledLLGuidanceOptions & { + /** Whether to pre-load and cache llguidance WASM assets. Defaults to env.useWasmCache. */ + useWasmCache?: boolean; +}; + +export async function loadLLGuidance(loadOptions: LlguidanceLoadOptions) { + const { useWasmCache = env.useWasmCache, ...options } = loadOptions; + if (!useWasmCache || isNodeLikeRuntime() || options.wasmFactory) { + return loadBundledLLGuidance(options); + } + + const wasmSource = options.wasm ?? options.wasmUrl ?? DEFAULT_LLGUIDANCE_WASM_URL; + const wasmFactorySource = options.wasmFactoryUrl ?? DEFAULT_LLGUIDANCE_WASM_FACTORY_URL; + const cachedOptions = { ...options }; + + const [wasm, wasmFactoryUrl] = await Promise.all([ + loadCacheableWasm(wasmSource), + loadCacheableWasmFactory(wasmFactorySource), + ]); + + if (wasm) { + cachedOptions.wasm = wasm; + delete cachedOptions.wasmUrl; + } + + if (wasm && wasmFactoryUrl) { + cachedOptions.wasmFactoryUrl = wasmFactoryUrl; + } + + return loadBundledLLGuidance(cachedOptions); +} + +async function loadCacheableWasm(source: LoadBundledLLGuidanceOptions['wasm']) { + const url = toCacheableURL(source); + if (!url) return null; + + try { + return await loadWasmBinary(url); + } catch (error) { + logger.warn('Failed to pre-load llguidance WASM binary:', error); + return null; + } +} + +async function loadCacheableWasmFactory(source: LoadBundledLLGuidanceOptions['wasmFactoryUrl']) { + const url = toCacheableURL(source); + if (!url) return null; + + try { + return await loadWasmFactory(url); + } catch (error) { + logger.warn('Failed to pre-load llguidance WASM factory:', error); + return null; + } +} + +function toCacheableURL(source: unknown) { + if (typeof source === 'string') { + return isBlobURL(source) ? null : new URL(source, globalThis.location?.href).href; + } + if (source instanceof URL) { + return isBlobURL(source.href) ? null : source.href; + } + return null; +} + +function isBlobURL(url: string) { + return url.startsWith('blob:'); +} diff --git a/packages/transformers-llguidance/tests/llguidance-constraint.test.js b/packages/transformers-llguidance/tests/llguidance-constraint.test.js index c9f32a42d..b12f66259 100644 --- a/packages/transformers-llguidance/tests/llguidance-constraint.test.js +++ b/packages/transformers-llguidance/tests/llguidance-constraint.test.js @@ -20,6 +20,7 @@ describe("LlguidanceConstraint", () => { loadBundledLLGuidance.mockClear(); }); + // Verifies the core integration path: llguidance creates an interpreter and its mask is applied across batches. it("loads llguidance and applies masks", async () => { computeMask.mockReturnValue({ mask: [true, false, true, false], vocabSize: 4 }); @@ -35,12 +36,14 @@ describe("LlguidanceConstraint", () => { expect(Array.from(logits.data)).toEqual([1, -Infinity, 3, -Infinity, 5, -Infinity, 7, -Infinity]); }); + // Ensures caller-provided load options are forwarded without the cache-only control flag. it("passes explicit llguidance load options", async () => { await LlguidanceConstraint.fromResponseFormat({}, { type: "json_object" }, { useWasmCache: false, wasmUrl: "custom.wasm" }); expect(loadBundledLLGuidance).toHaveBeenCalledWith({ wasmUrl: "custom.wasm" }); }); + // Confirms sampled tokens are committed back to llguidance so stopping criteria can end generation. it("commits sampled tokens and stops when llguidance stops", async () => { computeMask.mockReturnValue({ mask: [true, true], vocabSize: 2 }); commitToken.mockReturnValueOnce(undefined).mockReturnValueOnce({ stop: true }); @@ -49,10 +52,76 @@ describe("LlguidanceConstraint", () => { expect(stopping_criteria([[0n]])).toEqual([false]); - logits_processor.onTokensSampled([0, 1], [[0n, 0n], [0n, 1n]]); + logits_processor.onTokensSampled( + [0, 1], + [ + [0n, 0n], + [0n, 1n], + ], + ); expect(commitToken).toHaveBeenCalledWith(0); expect(commitToken).toHaveBeenCalledWith(1); - expect(stopping_criteria([[0n, 0n], [0n, 1n]])).toEqual([true, true]); + expect( + stopping_criteria([ + [0n, 0n], + [0n, 1n], + ]), + ).toEqual([true, true]); + }); + + // Covers llguidance's compact bitset mask format, not just boolean arrays. + it("applies packed uint32 masks", async () => { + computeMask.mockReturnValue({ mask: new Uint32Array([0b0101]), vocabSize: 4 }); + + const { logits_processor } = await LlguidanceConstraint.fromResponseFormat({}, { type: "json_object" }); + const logits = new Tensor("float32", new Float32Array([1, 2, 3, 4]), [1, 4]); + + logits_processor([[0n]], logits); + + expect(Array.from(logits.data)).toEqual([1, -Infinity, 3, -Infinity]); + }); + + // Ensures a stop response from computeMask marks the shared state complete without committing more tokens. + it("stops generation when computeMask returns stop", async () => { + computeMask.mockReturnValue({ stop: true }); + + const { logits_processor, stopping_criteria } = await LlguidanceConstraint.fromResponseFormat({}, { type: "json_object" }); + const logits = new Tensor("float32", new Float32Array([1, 2]), [1, 2]); + + logits_processor([[0n]], logits); + logits_processor.onTokensSampled([0], [[0n]]); + + expect(Array.from(logits.data)).toEqual([1, 2]); + expect(commitToken).not.toHaveBeenCalled(); + expect(stopping_criteria([[0n]])).toEqual([true]); + }); + + // Handles llguidance's post-stop compute error as a normal completion signal. + it("treats compute_mask after stop as completed", async () => { + computeMask.mockImplementation(() => { + throw new Error("compute_mask() called after stop"); + }); + + const { logits_processor, stopping_criteria } = await LlguidanceConstraint.fromResponseFormat({}, { type: "json_object" }); + const logits = new Tensor("float32", new Float32Array([1, 2]), [1, 2]); + + logits_processor([[0n]], logits); + + expect(Array.from(logits.data)).toEqual([1, 2]); + expect(stopping_criteria([[0n]])).toEqual([true]); + }); + + // Keeps unexpected interpreter failures visible instead of swallowing real bugs. + it("rethrows unexpected computeMask errors", async () => { + const error = new Error("unexpected"); + computeMask.mockImplementation(() => { + throw error; + }); + + const { logits_processor } = await LlguidanceConstraint.fromResponseFormat({}, { type: "json_object" }); + const logits = new Tensor("float32", new Float32Array([1, 2]), [1, 2]); + + expect(() => logits_processor([[0n]], logits)).toThrow(error); }); }); diff --git a/packages/transformers-llguidance/tests/wasm-loading.test.js b/packages/transformers-llguidance/tests/wasm-loading.test.js new file mode 100644 index 000000000..853a75da9 --- /dev/null +++ b/packages/transformers-llguidance/tests/wasm-loading.test.js @@ -0,0 +1,123 @@ +import { jest } from "@jest/globals"; + +const createInterpreter = jest.fn(() => ({ + computeMask: jest.fn(() => ({ stop: true })), + commitToken: jest.fn(), +})); +const loadBundledLLGuidance = jest.fn(async () => ({ createInterpreter })); +const loadWasmBinary = jest.fn(); +const loadWasmFactory = jest.fn(); +const logger = { + debug: jest.fn(), + warn: jest.fn(), +}; + +class LogitsProcessor {} +class LogitsProcessorList extends Array {} +class StoppingCriteria {} + +jest.unstable_mockModule("@huggingface/transformers", () => ({ + LogitsProcessor, + LogitsProcessorList, + StoppingCriteria, + env: { useWasmCache: true }, + loadWasmBinary, + loadWasmFactory, + logger, +})); + +jest.unstable_mockModule("llguidance", () => ({ + loadBundledLLGuidance, +})); + +const originalProcess = globalThis.process; +const originalLocationDescriptor = Object.getOwnPropertyDescriptor(globalThis, "location"); +const { LlguidanceConstraint } = await import("../dist/index.js"); + +describe("llguidance WASM loading", () => { + beforeEach(() => { + delete globalThis.process; + Object.defineProperty(globalThis, "location", { + configurable: true, + value: { href: "https://example.test/app/" }, + }); + + createInterpreter.mockClear(); + loadBundledLLGuidance.mockClear(); + loadWasmBinary.mockReset(); + loadWasmFactory.mockReset(); + logger.debug.mockClear(); + logger.warn.mockClear(); + }); + + afterEach(() => { + globalThis.process = originalProcess; + if (originalLocationDescriptor) { + Object.defineProperty(globalThis, "location", originalLocationDescriptor); + } else { + delete globalThis.location; + } + }); + + // Exercises browser-like cache preloading with the package's default CDN asset URLs. + it("preloads default WASM assets when cache is enabled outside Node", async () => { + const wasm = new Uint8Array([1, 2, 3]); + loadWasmBinary.mockResolvedValue(wasm); + loadWasmFactory.mockResolvedValue("blob:factory-url"); + + await LlguidanceConstraint.fromResponseFormat({}, { type: "json_object" }); + + expect(loadWasmBinary).toHaveBeenCalledWith("https://cdn.jsdelivr.net/npm/llguidance@0.1.7/wasm/llguidance_wasm_bg.wasm"); + expect(loadWasmFactory).toHaveBeenCalledWith("https://cdn.jsdelivr.net/npm/llguidance@0.1.7/wasm/llguidance_wasm.js"); + expect(loadBundledLLGuidance).toHaveBeenCalledWith({ + wasm, + wasmFactoryUrl: "blob:factory-url", + }); + }); + + // Verifies custom relative and absolute URLs are normalized before using the transformers cache helpers. + it("resolves custom cacheable WASM URLs", async () => { + const wasm = new Uint8Array([4, 5, 6]); + loadWasmBinary.mockResolvedValue(wasm); + loadWasmFactory.mockResolvedValue("blob:custom-factory-url"); + + await LlguidanceConstraint.fromResponseFormat( + {}, + { type: "json_object" }, + { + wasmUrl: "assets/custom.wasm", + wasmFactoryUrl: new URL("https://cdn.test/custom-factory.js"), + }, + ); + + expect(loadWasmBinary).toHaveBeenCalledWith("https://example.test/app/assets/custom.wasm"); + expect(loadWasmFactory).toHaveBeenCalledWith("https://cdn.test/custom-factory.js"); + expect(loadBundledLLGuidance).toHaveBeenCalledWith({ + wasm, + wasmFactoryUrl: "blob:custom-factory-url", + }); + }); + + // Ensures a failed preload does not prevent llguidance from loading with the original options. + it("falls back to uncached options when WASM preload fails", async () => { + const error = new Error("network failure"); + loadWasmBinary.mockRejectedValue(error); + loadWasmFactory.mockResolvedValue("blob:factory-url"); + + await LlguidanceConstraint.fromResponseFormat({}, { type: "json_object" }, { wasmUrl: "custom.wasm" }); + + expect(logger.warn).toHaveBeenCalledWith("Failed to pre-load llguidance WASM binary:", error); + expect(loadBundledLLGuidance).toHaveBeenCalledWith({ wasmUrl: "custom.wasm" }); + }); + + // Avoids cache preloading when the caller already supplied an initialized factory. + it("skips preloading when a WASM factory is provided", async () => { + const wasmFactory = jest.fn(); + + await LlguidanceConstraint.fromResponseFormat({}, { type: "json_object" }, { wasmFactory }); + + expect(loadWasmBinary).not.toHaveBeenCalled(); + expect(loadWasmFactory).not.toHaveBeenCalled(); + expect(loadBundledLLGuidance).toHaveBeenCalledWith({ wasmFactory }); + }); +}); From a7ab7f3327bc038e2bf8fed40702636081f2f143 Mon Sep 17 00:00:00 2001 From: Nico Martin Date: Sat, 11 Jul 2026 08:41:38 +0200 Subject: [PATCH 5/5] clean up --- packages/transformers-llguidance/src/utils/mask.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/transformers-llguidance/src/utils/mask.ts b/packages/transformers-llguidance/src/utils/mask.ts index f1ee29ba2..d73df19e3 100644 --- a/packages/transformers-llguidance/src/utils/mask.ts +++ b/packages/transformers-llguidance/src/utils/mask.ts @@ -1,5 +1,4 @@ import { type Tensor } from '@huggingface/transformers'; - import { type GuidanceMask, type GuidanceMaskResult } from './types'; export function summarizeMaskResult(result: GuidanceMaskResult, vocabSize?: number) {