fix(kb): CJK-safe search + slugs, and provenance frontmatter (K-B)

src/kb/search.test.ts

@@ -0,0 +1,67 @@
+import { test, describe, after, before } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+// Point lisaHome() at a temp dir before importing (see store.test.ts).
+const TMP = mkdtempSync(path.join(os.tmpdir(), "lisa-kb-search-"));
+process.env.LISA_HOME = TMP;
+process.env.LISA_KB_NO_GIT = "1";
+
+const store = await import("./store.js");
+const { searchKb, clearKbIndexCache } = await import("./search.js");
+
+after(() => rmSync(TMP, { recursive: true, force: true }));
+
+before(async () => {
+  await store.addSource({
+    title: "知识库的设计",
+    body: "这篇公众号文章讲的是知识库的设计，重点在检索和索引。",
+    tags: ["知识库"],
+  });
+  await store.addSource({
+    title: "OAuth PKCE",
+    body: "PKCE adds a code_verifier to the authorization code flow.",
+    tags: ["oauth"],
+  });
+  await store.addSource({
+    title: "苹果手机评测",
+    body: "关于苹果手机的一些使用感受。",
+  });
+  clearKbIndexCache();
+});
+
+describe("kb search — CJK", () => {
+  test("a Chinese substring query finds the Chinese entry", async () => {
+    // Regression: with whitespace-only tokenization a Chinese clause became one
+    // token, so this query returned nothing at all.
+    const hits = await searchKb("知识库", 5);
+    assert.ok(hits.length > 0, "expected at least one hit for 知识库");
+    assert.equal(hits[0]!.title, "知识库的设计");
+  });
+
+  test("a mid-clause phrase that is not a title still matches", async () => {
+    const hits = await searchKb("检索", 5);
+    assert.ok(hits.some((h) => h.title === "知识库的设计"));
+  });
+
+  test("an unrelated Chinese query does not pull the wrong entry first", async () => {
+    const hits = await searchKb("苹果手机", 5);
+    assert.equal(hits[0]!.title, "苹果手机评测");
+  });
+
+  test("mixed latin/CJK queries work from either side", async () => {
+    assert.ok((await searchKb("pkce", 5)).some((h) => h.title === "OAuth PKCE"));
+    assert.ok((await searchKb("公众号", 5)).some((h) => h.title === "知识库的设计"));
+  });
+
+  test("english search is unchanged", async () => {
+    const hits = await searchKb("code_verifier authorization", 5);
+    assert.equal(hits[0]!.title, "OAuth PKCE");
+  });
+
+  test("a query with no overlap returns nothing", async () => {
+    assert.equal((await searchKb("quantum chromodynamics", 5)).length, 0);
+  });
+});

src/kb/search.ts

@@ -1,9 +1,11 @@
 /**
  * TF-IDF search over the knowledge base (sources + wiki), mirroring the memory
  * transcript index (memory/vector.ts) but over KB entries keyed by layer/slug.
- * Kept self-contained (own tokenizer) so the KB doesn't couple to memory's
- * session-specific index. The index is fingerprint-cached over the KB dirs, so
- * repeated searches in one conversation are free until the KB actually changes.
+ * Kept self-contained (own index, own cache) so the KB doesn't couple to
+ * memory's session-specific index; only the tokenizer is shared (../tokenize.ts
+ * — see there for why CJK needs bigrams). The index is fingerprint-cached over
+ * the KB dirs, so repeated searches in one conversation are free until the KB
+ * actually changes.
  *
  * (Semantic/embedding search is a deliberate future enhancement — it can reuse
  * memory/embedding.ts the same way memory_search does; TF-IDF is the robust,
@@ -12,6 +14,7 @@
 import fs from "node:fs/promises";
 import path from "node:path";
 import { pathExists } from "../fs-utils.js";
+import { tokenize } from "../tokenize.js";
 import { kbSourcesDir, kbWikiDir, layerDir, type KbLayer } from "./paths.js";
 import { readEntry } from "./store.js";
 
@@ -36,20 +39,6 @@ export interface KbHit {
   excerpt: string;
 }
 
-const STOPWORDS = new Set([
-  "the", "a", "an", "of", "to", "in", "and", "or", "for", "is", "it",
-  "this", "that", "be", "are", "was", "were", "with", "on", "as", "at",
-  "by", "from", "but", "not", "i", "you", "we", "they", "he", "she",
-]);
-
-function tokenize(text: string): string[] {
-  return text
-    .toLowerCase()
-    .replace(/[^a-z0-9一-鿿\s]/g, " ")
-    .split(/\s+/)
-    .filter((t) => t.length >= 2 && !STOPWORDS.has(t));
-}
-
 function excerptAround(text: string, qTokens: string[], width: number): string {
   const lower = text.toLowerCase();
   for (const t of qTokens) {

src/kb/slug.test.ts

@@ -0,0 +1,81 @@
+import { test, describe } from "node:test";
+import assert from "node:assert/strict";
+import { kbSlug, canonicalUrl, shortHash } from "./slug.js";
+
+describe("kbSlug", () => {
+  test("latin titles slug exactly as before", () => {
+    assert.equal(kbSlug({ title: "OAuth PKCE notes" }), "oauth-pkce-notes");
+  });
+
+  test("a Chinese title gets a stable date+hash slug, not entry-<Date.now()>", () => {
+    // The bug: normalizeSlug strips every non-[a-z0-9] char, so a CJK title
+    // returned "" and the caller fell back to a timestamp — unstable, opaque,
+    // and impossible to dedupe against.
+    const a = kbSlug({ title: "知识库的设计", url: "https://x.test/a", date: "2026-07-23" });
+    const b = kbSlug({ title: "知识库的设计", url: "https://x.test/a", date: "2026-07-23" });
+    assert.equal(a, b, "same input → same slug");
+    assert.match(a, /^2026-07-23-[0-9a-f]{8}$/);
+  });
+
+  test("different URLs on the same day get different slugs", () => {
+    const a = kbSlug({ title: "文章", url: "https://x.test/a", date: "2026-07-23" });
+    const b = kbSlug({ title: "文章", url: "https://x.test/b", date: "2026-07-23" });
+    assert.notEqual(a, b);
+  });
+
+  test("falls back to hashing the title when there is no URL", () => {
+    const s = kbSlug({ title: "关于检索的笔记", date: "2026-01-02T03:04:05.000Z" });
+    assert.match(s, /^2026-01-02-[0-9a-f]{8}$/);
+  });
+
+  test("a too-short latin title also takes the hashed form", () => {
+    // "AI" normalizes to 2 chars — too thin to be a useful, collision-safe id.
+    assert.match(kbSlug({ title: "AI", date: "2026-07-23" }), /^2026-07-23-[0-9a-f]{8}$/);
+  });
+
+  test("emoji-only and punctuation-only titles never produce an empty slug", () => {
+    for (const title of ["🎉🎉", "!!!", "  "]) {
+      assert.ok(kbSlug({ title, date: "2026-07-23" }).length > 10, title);
+    }
+  });
+
+  test("the slug stays ASCII (NFD/NFC filename hazard, URLs, git paths)", () => {
+    const s = kbSlug({ title: "日本語のタイトル", date: "2026-07-23" });
+    // eslint-disable-next-line no-control-regex
+    assert.match(s, /^[\x21-\x7e]+$/);
+  });
+});
+
+describe("canonicalUrl", () => {
+  test("drops utm_* and known tracking params", () => {
+    assert.equal(
+      canonicalUrl("https://Example.com/post?utm_source=x&utm_medium=y&id=7"),
+      "https://example.com/post?id=7",
+    );
+  });
+
+  test("drops the fragment and a trailing slash", () => {
+    assert.equal(canonicalUrl("https://example.com/post/#section"), "https://example.com/post");
+  });
+
+  test("keeps params that carry meaning (t, from, timestamp)", () => {
+    const u = canonicalUrl("https://youtu.be/abc?t=120&si=track");
+    assert.match(u, /t=120/);
+    assert.doesNotMatch(u, /si=/);
+  });
+
+  test("strips the bilibili/wechat app appendages that break dedupe", () => {
+    const bili = canonicalUrl("https://www.bilibili.com/video/BV1x?spm_id_from=333&vd_source=abc");
+    assert.equal(bili, "https://www.bilibili.com/video/BV1x");
+  });
+
+  test("two shares of the same article canonicalize identically", () => {
+    const a = canonicalUrl("https://mp.weixin.qq.com/s?__biz=A&sn=B&scene=21&srcid=0101");
+    const b = canonicalUrl("https://mp.weixin.qq.com/s?__biz=A&sn=B&isappinstalled=0");
+    assert.equal(shortHash(a), shortHash(b));
+  });
+
+  test("an unparseable value is returned trimmed rather than thrown", () => {
+    assert.equal(canonicalUrl("  not a url  "), "not a url");
+  });
+});

src/kb/slug.ts

@@ -0,0 +1,97 @@
+/**
+ * Slug minting for KB entries.
+ *
+ * The shared `normalizeSlug` (soul/slug.ts) does `replace(/[^a-z0-9]+/g, "-")`,
+ * which is right for soul entries (Lisa names those herself, in English) and
+ * wrong the moment a title is Chinese: the whole string is stripped, the slug
+ * comes back empty, and the caller falls back to `entry-<Date.now()>`. Every
+ * Chinese source in the KB therefore got an opaque, unstable, un-dedupable id.
+ *
+ * `kbSlug` keeps the latin path exactly as it was and gives non-latin titles a
+ * stable, meaningful id instead: `<date>-<8 hex of sha256(url ?? title)>`.
+ *
+ * Why the slug stays ASCII rather than just using the Chinese title:
+ *   - macOS stores filenames as NFD, Linux/git as NFC. A CJK filename written
+ *     on one and read on the other is a different byte string — the file "does
+ *     not exist" on the other machine, which is a miserable bug to chase in a
+ *     dir that syncs.
+ *   - slugs land in URLs (/api/kb/entry?slug=…) and git paths.
+ *   - readability is the `title:` frontmatter's job, and every surface (index,
+ *     search, kb_list, the web UI) shows the title, not the slug.
+ *
+ * Hashing the URL (when there is one) also makes the slug the natural dedupe
+ * key: re-ingesting the same link mints the same slug.
+ */
+import { createHash } from "node:crypto";
+import { normalizeSlug } from "../soul/slug.js";
+
+/** Shortest latin slug we'll accept before falling back to the hashed form. */
+const MIN_LATIN_LEN = 3;
+
+export interface KbSlugInput {
+  title: string;
+  /** Canonical URL, when the entry came from one — makes the slug dedupe-stable. */
+  url?: string;
+  /** ISO timestamp or YYYY-MM-DD; defaults to now. Only the date part is used. */
+  date?: string;
+}
+
+/**
+ * Mint a slug for a new KB entry. Latin titles slug as before; anything that
+ * normalizes to less than 3 usable characters (CJK, emoji, punctuation-only)
+ * gets `YYYY-MM-DD-<hash8>`.
+ */
+export function kbSlug(input: KbSlugInput): string {
+  const latin = normalizeSlug(input.title);
+  if (latin.length >= MIN_LATIN_LEN) return latin;
+  const date = (input.date ?? new Date().toISOString()).slice(0, 10);
+  return `${date}-${shortHash(input.url || input.title)}`;
+}
+
+/** First 8 hex chars of sha256 — the KB's short, stable content id. */
+export function shortHash(value: string): string {
+  return createHash("sha256").update(value).digest("hex").slice(0, 8);
+}
+
+/**
+ * Canonical form of a URL for dedupe: lowercase host, no fragment, no trailing
+ * slash on the path, and the usual tracking params dropped so the same article
+ * shared from two places hashes the same.
+ *
+ * Returns the input unchanged if it isn't parseable — the caller still gets a
+ * deterministic dedupe key, just a less forgiving one.
+ */
+export function canonicalUrl(raw: string): string {
+  let u: URL;
+  try {
+    u = new URL(raw);
+  } catch {
+    return raw.trim();
+  }
+  u.hash = "";
+  u.hostname = u.hostname.toLowerCase();
+  for (const p of [...u.searchParams.keys()]) {
+    if (TRACKING_PARAMS.has(p.toLowerCase()) || p.toLowerCase().startsWith("utm_")) {
+      u.searchParams.delete(p);
+    }
+  }
+  if (u.pathname.length > 1 && u.pathname.endsWith("/")) {
+    u.pathname = u.pathname.replace(/\/+$/, "");
+  }
+  return u.toString();
+}
+
+/**
+ * Only unambiguous tracking params. Deliberately NOT here: `from`, `t`,
+ * `timestamp` — those carry real meaning on some sites (a video seek position,
+ * a real query filter), and this canonical URL is also what we store as the
+ * entry's `url:`, so stripping a meaningful param would hand the user a broken
+ * link back.
+ */
+const TRACKING_PARAMS = new Set([
+  "fbclid", "gclid", "msclkid", "mc_cid", "mc_eid", "ref_src",
+  "spm", "spm_id_from", "vd_source", // bilibili
+  "share_source", "share_medium", "share_plat", "share_tag", "unique_k",
+  "srcid", "isappinstalled", "scene", "clicktime", // wechat app appendages
+  "si", "feature", "pp", // youtube share links
+]);

src/kb/store.test.ts

@@ -104,6 +104,85 @@ describe("kb store", () => {
     assert.equal(await store.removeEntry("sources", "dup-2"), false, "already gone");
   });
 
+  test("a Chinese title gets a readable date+hash slug, not entry-<timestamp>", async () => {
+    const e = await store.addSource({
+      title: "知识库的设计笔记",
+      body: "三层结构：来源、维基、schema。",
+      tags: ["知识库"],
+    });
+    assert.match(e.slug, /^\d{4}-\d{2}-\d{2}-[0-9a-f]{8}$/);
+    assert.doesNotMatch(e.slug, /^entry-/);
+    const back = await store.readEntry("sources", e.slug);
+    assert.equal(back!.title, "知识库的设计笔记", "the real title lives in frontmatter");
+  });
+
+  test("provenance frontmatter round-trips through `extra`", async () => {
+    const e = await store.addSource({
+      title: "Ingested article",
+      body: "body text",
+      origin: "web",
+      extra: {
+        url: "https://example.com/a?b=1",
+        site: "example.com",
+        author: "Someone",
+        published: "2026-07-20",
+        hash: "deadbeef",
+      },
+    });
+    const back = await store.readEntry("sources", e.slug);
+    assert.equal(back!.extra?.url, "https://example.com/a?b=1");
+    assert.equal(back!.extra?.site, "example.com");
+    assert.equal(back!.extra?.hash, "deadbeef");
+    assert.equal(back!.origin, "web");
+    // Known keys must not leak into extra (they'd be written twice).
+    assert.equal(back!.extra?.title, undefined);
+    assert.equal(back!.extra?.tags, undefined);
+    const raw = readFileSync(entryFile("sources", e.slug), "utf8");
+    assert.equal(raw.match(/^title:/gm)?.length, 1, "title written exactly once");
+  });
+
+  test("a newline in an extra value cannot forge extra frontmatter lines", async () => {
+    const e = await store.addSource({
+      title: "Injection attempt",
+      body: "b",
+      extra: { author: "x\norigin: spoofed\ntitle: spoofed" },
+    });
+    const back = await store.readEntry("sources", e.slug);
+    assert.equal(back!.title, "Injection attempt");
+    assert.notEqual(back!.origin, "spoofed");
+    assert.match(back!.extra!.author!, /^x origin: spoofed title: spoofed$/);
+  });
+
+  test("a newline in a title cannot forge frontmatter or truncate the block", async () => {
+    // Untrusted, remote-authored titles (ingested page <title>/og:title) reach this
+    // path; an embedded `\n---\n` must not close the frontmatter early and push
+    // url/hash/origin into the body, nor forge new metadata lines.
+    const e = await store.addSource({
+      title: "Real Title\n---\nInjected body\norigin: spoofed",
+      body: "b",
+    });
+    const back = await store.readEntry("sources", e.slug);
+    assert.equal(back!.title, "Real Title --- Injected body origin: spoofed");
+    assert.notEqual(back!.origin, "spoofed");
+    assert.equal(back!.body, "b");
+  });
+
+  test("an untrusted-shaped extra key cannot inject a frontmatter line", async () => {
+    const e = await store.addSource({
+      title: "K",
+      body: "b",
+      extra: { "bad\nsources": "x", ok: "y" },
+    });
+    const back = await store.readEntry("sources", e.slug);
+    assert.equal(back!.extra?.ok, "y");
+    assert.ok(!("bad" in (back!.extra ?? {})), "malformed key is dropped, not written");
+  });
+
+  test("listEntries surfaces extra so the index/UI can show provenance", async () => {
+    const sources = await store.listEntries("sources");
+    assert.ok(sources.some((s) => s.extra?.site === "example.com"));
+  });
+
   test("slug jail: a traversal slug throws rather than escaping the KB", async () => {
     await assert.rejects(() => store.readEntry("wiki", "../../etc/passwd"));
     await assert.rejects(() =>