feat(kb): ingest engine — url → provenance-stamped Layer-1 markdown (K-E)

src/kb/ingest/dedupe.ts

@@ -0,0 +1,69 @@
+/**
+ * Ingest dedupe ledger — kb/.ingested.json, a flat `urlHash → slug` map.
+ *
+ * Sources are immutable (decision D2), so "ingest the same URL twice" must not
+ * silently pile up near-identical captures: the default is to return the
+ * existing slug, and only `force:true` writes a fresh capture (which then
+ * records `supersedes:` pointing at the one it replaces).
+ *
+ * The ledger is a cache, not a source of truth: every ingested source carries
+ * its own `hash:` frontmatter, so a missing or corrupt ledger is rebuilt from
+ * the sources dir. Writes happen inside the store's KB write path, so the file
+ * lives next to the entries it indexes (and travels with KB backups).
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { atomicWrite, pathExists } from "../../fs-utils.js";
+import { kbDir } from "../paths.js";
+import { listFullEntries } from "../store.js";
+
+export function ingestLedgerFile(): string {
+  return path.join(kbDir(), ".ingested.json");
+}
+
+type Ledger = Record<string, string>;
+
+async function rebuildFromSources(): Promise<Ledger> {
+  const ledger: Ledger = {};
+  for (const e of await listFullEntries("sources")) {
+    const hash = e.extra?.hash;
+    // listFullEntries is newest-first; first-write-wins means the NEWEST
+    // capture of a URL owns its hash — matching the live ledger, which is
+    // repointed to the fresh slug after every forced re-ingest.
+    if (hash && !(hash in ledger)) ledger[hash] = e.slug;
+  }
+  return ledger;
+}
+
+export async function readLedger(): Promise<Ledger> {
+  const file = ingestLedgerFile();
+  if (await pathExists(file)) {
+    try {
+      const parsed = JSON.parse(await fs.readFile(file, "utf8")) as unknown;
+      if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+        const ledger: Ledger = {};
+        for (const [k, v] of Object.entries(parsed)) {
+          if (typeof v === "string") ledger[k] = v;
+        }
+        return ledger;
+      }
+    } catch {
+      // corrupt → fall through to rebuild
+    }
+  }
+  const rebuilt = await rebuildFromSources();
+  await atomicWrite(file, JSON.stringify(rebuilt, null, 2) + "\n");
+  return rebuilt;
+}
+
+/** The slug previously ingested for this url-hash, if any. */
+export async function lookupIngested(hash: string): Promise<string | null> {
+  return (await readLedger())[hash] ?? null;
+}
+
+/** Record (or repoint, after a forced re-ingest) a hash → slug mapping. */
+export async function recordIngested(hash: string, slug: string): Promise<void> {
+  const ledger = await readLedger();
+  ledger[hash] = slug;
+  await atomicWrite(ingestLedgerFile(), JSON.stringify(ledger, null, 2) + "\n");
+}

src/kb/ingest/html-to-md.test.ts

@@ -0,0 +1,97 @@
+import { test, describe } from "node:test";
+import assert from "node:assert/strict";
+import { htmlToMarkdown, decodeEntities } from "./html-to-md.js";
+
+describe("htmlToMarkdown", () => {
+  test("headings and paragraphs", () => {
+    assert.equal(
+      htmlToMarkdown("<h1>Title</h1><p>One.</p><h2>Sub</h2><p>Two.</p>"),
+      "# Title\n\nOne.\n\n## Sub\n\nTwo.",
+    );
+  });
+
+  test("nested and ordered lists", () => {
+    const md = htmlToMarkdown(
+      "<ul><li>a<ul><li>a1</li><li>a2</li></ul></li><li>b</li></ul><ol><li>x</li><li>y</li></ol>",
+    );
+    assert.equal(md, "- a\n  - a1\n  - a2\n- b\n\n1. x\n2. y");
+  });
+
+  test("unclosed <li> (the real-world common case)", () => {
+    assert.equal(htmlToMarkdown("<ul><li>one<li>two</ul>"), "- one\n- two");
+  });
+
+  test("fenced code with language, fence-proof body", () => {
+    const md = htmlToMarkdown(
+      `<pre><code class="language-ts">const a = 1;\nif (a &lt; 2) {}</code></pre>`,
+    );
+    assert.equal(md, "```ts\nconst a = 1;\nif (a < 2) {}\n```");
+    // A body containing ``` must get a longer fence.
+    const evil = htmlToMarkdown("<pre><code>```\ninjection\n```</code></pre>");
+    assert.match(evil, /^````\n/);
+  });
+
+  test("inline code keeps backticks safe", () => {
+    assert.equal(htmlToMarkdown("<p>run <code>a`b</code> now</p>"), "run ``a`b`` now");
+  });
+
+  test("blockquote wraps nested markdown", () => {
+    assert.equal(
+      htmlToMarkdown("<blockquote><p>quoted</p><p>lines</p></blockquote>"),
+      "> quoted\n>\n> lines",
+    );
+  });
+
+  test("tables become pipe tables", () => {
+    const md = htmlToMarkdown(
+      "<table><thead><tr><th>A</th><th>B</th></tr></thead><tbody><tr><td>1</td><td>2</td></tr></tbody></table>",
+    );
+    assert.equal(md, "| A | B |\n| --- | --- |\n| 1 | 2 |");
+  });
+
+  test("links, images (incl. data-src lazy-loading), emphasis, hr", () => {
+    const md = htmlToMarkdown(
+      `<p><a href="https://x.dev">site</a> <strong>bold</strong> <em>it</em></p><hr><p><img data-src="https://img/x.png" alt="pic"></p>`,
+    );
+    assert.equal(md, "[site](https://x.dev) **bold** *it*\n\n---\n\n![pic](https://img/x.png)");
+  });
+
+  test("markdown special characters in prose are escaped — no syntax forgery", () => {
+    const md = htmlToMarkdown("<p>weight *not bold* and [[oauth]] and a|b</p>");
+    assert.equal(md, "weight \\*not bold\\* and \\[\\[oauth\\]\\] and a\\|b");
+  });
+
+  test("javascript: links render as plain text", () => {
+    assert.equal(htmlToMarkdown(`<p><a href="javascript:alert(1)">hi</a></p>`), "hi");
+  });
+
+  test("script/style/iframe contents are dropped, even with tricky bodies", () => {
+    const md = htmlToMarkdown(
+      `<p>keep</p><script>if (a < b) document.write("<p>evil</p>")</script><style>p{color:red}</style><p>also</p>`,
+    );
+    assert.equal(md, "keep\n\nalso");
+  });
+
+  test("a self-closing drop tag does not swallow the rest of the doc", () => {
+    // Self-closing <svg/>/<iframe/> have no matching close; the parser used to
+    // jump to EOF and silently discard everything after them.
+    assert.equal(htmlToMarkdown(`<p>before</p><svg/><p>after</p>`), "before\n\nafter");
+    assert.equal(htmlToMarkdown(`<p>before</p><iframe src="x"/><p>after</p>`), "before\n\nafter");
+    // A genuinely unclosed <script> (malformed) still drops to EOF — safer than
+    // leaking raw script source — so the tail is gone but "x=1" never leaks.
+    const md = htmlToMarkdown(`<p>before</p><script>x=1<p>after</p>`);
+    assert.equal(md, "before");
+    assert.ok(!md.includes("x=1"), "raw script source must not leak into output");
+  });
+
+  test("serializes from <body> when a full document is given", () => {
+    const md = htmlToMarkdown(
+      "<html><head><title>t</title></head><body><p>content</p></body></html>",
+    );
+    assert.equal(md, "content");
+  });
+
+  test("entities: named, decimal, hex", () => {
+    assert.equal(decodeEntities("a &amp; b &#65; &#x4e2d; &nbsp;"), "a & b A 中  ");
+  });
+});