Introduce @huggingface/transformers-llguidance

packages/transformers-llguidance/README.md

@@ -0,0 +1,48 @@
+# @huggingface/transformers-llguidance
+
+Experimental constrained-generation helpers for Transformers.js.
+
+This package exports `LlguidanceConstraint`, which turns an llguidance response format into the `logits_processor` and `stopping_criteria` objects accepted by Transformers.js generation.
+
+```js
+import { LlguidanceConstraint } from "@huggingface/transformers-llguidance";
+
+const { logits_processor, stopping_criteria } =
+  await LlguidanceConstraint.fromResponseFormat(tokenizer, {
+    type: "json_schema",
+    json_schema: {
+      type: "object",
+      properties: {
+        answer: { type: "string" },
+      },
+      required: ["answer"],
+      additionalProperties: false,
+    },
+  });
+
+await model.generate({
+  ...inputs,
+  logits_processor,
+  stopping_criteria,
+});
+```
+
+## Regex constraints
+
+Use `type: "regex"` to constrain generation to a regular expression. For example, this only allows ISO-like dates in `YYYY-MM-DD` format:
+
+```js
+import { LlguidanceConstraint } from "@huggingface/transformers-llguidance";
+
+const { logits_processor, stopping_criteria } =
+  await LlguidanceConstraint.fromResponseFormat(tokenizer, {
+    type: "regex",
+    regex: "\\d{4}-\\d{2}-\\d{2}",
+  });
+
+const output = await model.generate({
+  ...inputs,
+  logits_processor,
+  stopping_criteria,
+});
+```

packages/transformers-llguidance/jest.config.mjs

@@ -0,0 +1,11 @@
+/** @type {import('jest').Config} */
+export default {
+  clearMocks: true,
+  collectCoverage: true,
+  coverageDirectory: "coverage",
+  coveragePathIgnorePatterns: ["node_modules", "tests"],
+  coverageProvider: "v8",
+  roots: ["./tests/"],
+  testTimeout: 32000,
+  transform: {},
+};

packages/transformers-llguidance/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "@huggingface/transformers-llguidance",
+  "version": "0.0.0",
+  "description": "llguidance integration helpers for Transformers.js constrained generation",
+  "main": "./dist/index.cjs",
+  "types": "./types/index.d.ts",
+  "type": "module",
+  "exports": {
+    "import": {
+      "types": "./types/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "require": {
+      "types": "./types/index.d.ts",
+      "default": "./dist/index.cjs"
+    }
+  },
+  "scripts": {
+    "format": "prettier --write . --ignore-path ../../.prettierignore",
+    "format:check": "prettier --check . --ignore-path ../../.prettierignore",
+    "typegen": "tsc --build",
+    "dev": "node scripts/dev.mjs",
+    "build": "node scripts/build.mjs && pnpm typegen",
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --verbose"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/huggingface/transformers.js.git"
+  },
+  "keywords": [
+    "transformers",
+    "transformers.js",
+    "huggingface",
+    "llguidance",
+    "constrained-generation"
+  ],
+  "author": "Hugging Face",
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://github.com/huggingface/transformers.js/issues"
+  },
+  "homepage": "https://github.com/huggingface/transformers.js#readme",
+  "peerDependencies": {
+    "@huggingface/transformers": "^4.2.0"
+  },
+  "devDependencies": {
+    "@huggingface/transformers": "workspace:*",
+    "@types/jest": "^30.0.0",
+    "@types/node": "^24.1.0",
+    "esbuild": "^0.27.2",
+    "jest": "^30.2.0",
+    "typescript": "5.9.3"
+  },
+  "files": [
+    "src",
+    "dist",
+    "types",
+    "README.md",
+    "LICENSE",
+    "!**/*.tsbuildinfo"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "llguidance": "^0.1.7"
+  }
+}

packages/transformers-llguidance/scripts/build.mjs

@@ -0,0 +1,26 @@
+import { build } from "esbuild";
+import { rmSync } from "node:fs";
+
+rmSync("dist", { recursive: true, force: true });
+
+const common = {
+  entryPoints: ["src/index.ts"],
+  bundle: true,
+  platform: "neutral",
+  target: "es2022",
+  sourcemap: true,
+  external: ["@huggingface/transformers", "llguidance"],
+};
+
+await Promise.all([
+  build({
+    ...common,
+    format: "esm",
+    outfile: "dist/index.js",
+  }),
+  build({
+    ...common,
+    format: "cjs",
+    outfile: "dist/index.cjs",
+  }),
+]);

packages/transformers-llguidance/scripts/dev.mjs

@@ -0,0 +1,65 @@
+import { context } from "esbuild";
+import { spawn } from "node:child_process";
+import { rmSync } from "node:fs";
+
+rmSync("dist", { recursive: true, force: true });
+rmSync("types", { recursive: true, force: true });
+
+const watchLogger = {
+  name: "watch-logger",
+  setup(build) {
+    let startTime = 0;
+
+    build.onStart(() => {
+      startTime = performance.now();
+      console.log(`[transformers-llguidance] rebuilding ${build.initialOptions.outfile}...`);
+    });
+
+    build.onEnd((result) => {
+      const duration = (performance.now() - startTime).toFixed(2);
+      if (result.errors.length > 0) {
+        console.log(`[transformers-llguidance] rebuild failed in ${duration}ms`);
+      } else {
+        console.log(`[transformers-llguidance] rebuilt ${build.initialOptions.outfile} in ${duration}ms`);
+      }
+    });
+  },
+};
+
+const common = {
+  entryPoints: ["src/index.ts"],
+  bundle: true,
+  platform: "neutral",
+  target: "es2022",
+  sourcemap: true,
+  external: ["@huggingface/transformers", "llguidance"],
+  plugins: [watchLogger],
+};
+
+const contexts = await Promise.all([
+  context({
+    ...common,
+    format: "esm",
+    outfile: "dist/index.js",
+  }),
+  context({
+    ...common,
+    format: "cjs",
+    outfile: "dist/index.cjs",
+  }),
+]);
+
+await Promise.all(contexts.map((ctx) => ctx.watch()));
+
+const tscWatch = spawn("tsc", ["--build", "--watch", "--preserveWatchOutput"], {
+  stdio: "inherit",
+  shell: true,
+});
+
+console.log("Watching @huggingface/transformers-llguidance...");
+
+process.on("SIGINT", async () => {
+  tscWatch.kill();
+  await Promise.all(contexts.map((ctx) => ctx.dispose()));
+  process.exit(0);
+});

packages/transformers-llguidance/src/LlguidanceConstraint.ts

@@ -0,0 +1,166 @@
+import { LogitsProcessor, LogitsProcessorList, StoppingCriteria, logger, type Tensor } from '@huggingface/transformers';
+import { type LLGuidanceResponseFormat } from 'llguidance';
+
+import { applyMask, summarizeMaskResult } from './utils/mask';
+import { type GuidanceInterpreter, type GuidanceMaskResult, type LlguidanceState } from './utils/types';
+import { type LlguidanceLoadOptions, loadLLGuidance } from './utils/wasm';
+
+export type ResponseFormat = LLGuidanceResponseFormat;
+export type { LlguidanceLoadOptions };
+
+export class LlguidanceConstraint {
+    static async fromResponseFormat(
+        tokenizer: unknown,
+        response_format: ResponseFormat,
+        loadOptions: LlguidanceLoadOptions = {},
+    ) {
+        logger.debug('[LlguidanceConstraint] loading llguidance', {
+            response_format,
+        });
+
+        const runtime = await loadLLGuidance(loadOptions);
+        const interpreter = runtime.createInterpreter({
+            tokenizer,
+            response_format,
+        }) as GuidanceInterpreter;
+
+        logger.debug('[LlguidanceConstraint] interpreter created');
+
+        const state: LlguidanceState = {
+            completed: false,
+            interpreter,
+            step: 0,
+        };
+
+        const logits_processor = new LogitsProcessorList();
+        logits_processor.push(new LlguidanceLogitsProcessor(state));
+
+        return {
+            logits_processor,
+            stopping_criteria: new LlguidanceStoppingCriteria(state),
+        };
+    }
+}
+
+class LlguidanceLogitsProcessor extends LogitsProcessor {
+    private state: LlguidanceState;
+
+    constructor(state: LlguidanceState) {
+        super();
+        this.state = state;
+    }
+
+    _call(_inputIds: bigint[][], logits: Tensor) {
+        if (this.state.completed) {
+            logger.debug('[LlguidanceLogitsProcessor] skip completed', {
+                step: this.state.step,
+            });
+            return logits;
+        }
+
+        this.state.step++;
+        const vocabSize = logits.dims.at(-1);
+        logger.debug('[LlguidanceLogitsProcessor] compute mask', {
+            step: this.state.step,
+            logitsDims: logits.dims,
+            vocabSize,
+        });
+
+        let result: GuidanceMaskResult;
+        try {
+            result = this.state.interpreter.computeMask();
+        } catch (error) {
+            if (!String((error as Error).message).includes('compute_mask() called after stop')) {
+                throw error;
+            }
+            this.state.completed = true;
+            logger.debug('[LlguidanceLogitsProcessor] compute after stop', {
+                step: this.state.step,
+            });
+            return logits;
+        }
+
+        logger.debug('[LlguidanceLogitsProcessor] mask result', {
+            step: this.state.step,
+            result: summarizeMaskResult(result, vocabSize),
+        });
+
+        if ('stop' in result && result.stop) {
+            this.state.completed = true;
+            logger.debug('[LlguidanceLogitsProcessor] stopped by mask', {
+                step: this.state.step,
+            });
+            return logits;
+        }
+
+        if ('mask' in result) {
+            const applied = applyMask(logits, result.mask, result.vocabSize ?? vocabSize);
+            logger.debug('[LlguidanceLogitsProcessor] mask applied', {
+                step: this.state.step,
+                ...applied,
+            });
+        }
+
+        return logits;
+    }
+
+    onTokenSampled(tokenId: number, batchIdx: number, inputIds: bigint[][]) {
+        logger.debug('[LlguidanceLogitsProcessor] token sampled', {
+            step: this.state.step,
+            tokenId,
+            batchIdx,
+            inputLength: inputIds[batchIdx]?.length,
+            completed: this.state.completed,
+        });
+
+        if (this.state.completed) return;
+
+        const result = this.state.interpreter.commitToken(tokenId);
+        logger.debug('[LlguidanceLogitsProcessor] token committed', {
+            step: this.state.step,
+            tokenId,
+            result,
+        });
+
+        if (result?.stop) {
+            this.state.completed = true;
+            logger.debug('[LlguidanceLogitsProcessor] stopped by commit', {
+                step: this.state.step,
+                tokenId,
+            });
+        }
+    }
+
+    onTokensSampled(tokenIds: number[], inputIds: bigint[][]) {
+        logger.debug('[LlguidanceLogitsProcessor] tokens sampled', {
+            step: this.state.step,
+            tokenIds,
+            inputLengths: inputIds.map((ids) => ids.length),
+            completed: this.state.completed,
+        });
+
+        for (let batchIdx = 0; batchIdx < tokenIds.length; ++batchIdx) {
+            this.onTokenSampled(tokenIds[batchIdx], batchIdx, inputIds);
+        }
+    }
+}
+
+class LlguidanceStoppingCriteria extends StoppingCriteria {
+    private state: LlguidanceState;
+
+    constructor(state: LlguidanceState) {
+        super();
+        this.state = state;
+    }
+
+    _call(inputIds: ArrayLike<unknown>[]) {
+        const result = new Array(inputIds.length).fill(this.state.completed);
+        logger.debug('[LlguidanceStoppingCriteria] call', {
+            step: this.state.step,
+            completed: this.state.completed,
+            result,
+            inputLengths: inputIds.map((ids) => ids.length),
+        });
+        return result;
+    }
+}

packages/transformers-llguidance/src/constants.ts

@@ -0,0 +1,4 @@
+export const LLGUIDANCE_VERSION = '0.1.7';
+export const LLGUIDANCE_WASM_BASE = `https://cdn.jsdelivr.net/npm/llguidance@${LLGUIDANCE_VERSION}/wasm/`;
+export const DEFAULT_LLGUIDANCE_WASM_URL = `${LLGUIDANCE_WASM_BASE}llguidance_wasm_bg.wasm`;
+export const DEFAULT_LLGUIDANCE_WASM_FACTORY_URL = `${LLGUIDANCE_WASM_BASE}llguidance_wasm.js`;

packages/transformers-llguidance/src/index.ts

@@ -0,0 +1,2 @@
+export { LlguidanceConstraint } from './LlguidanceConstraint';
+export type { LlguidanceLoadOptions, ResponseFormat } from './LlguidanceConstraint';