diff --git a/src/kb/ingest/dedupe.ts b/src/kb/ingest/dedupe.ts new file mode 100644 index 0000000..9a49027 --- /dev/null +++ b/src/kb/ingest/dedupe.ts @@ -0,0 +1,69 @@ +/** + * Ingest dedupe ledger — kb/.ingested.json, a flat `urlHash → slug` map. + * + * Sources are immutable (decision D2), so "ingest the same URL twice" must not + * silently pile up near-identical captures: the default is to return the + * existing slug, and only `force:true` writes a fresh capture (which then + * records `supersedes:` pointing at the one it replaces). + * + * The ledger is a cache, not a source of truth: every ingested source carries + * its own `hash:` frontmatter, so a missing or corrupt ledger is rebuilt from + * the sources dir. Writes happen inside the store's KB write path, so the file + * lives next to the entries it indexes (and travels with KB backups). + */ +import fs from "node:fs/promises"; +import path from "node:path"; +import { atomicWrite, pathExists } from "../../fs-utils.js"; +import { kbDir } from "../paths.js"; +import { listFullEntries } from "../store.js"; + +export function ingestLedgerFile(): string { + return path.join(kbDir(), ".ingested.json"); +} + +type Ledger = Record; + +async function rebuildFromSources(): Promise { + const ledger: Ledger = {}; + for (const e of await listFullEntries("sources")) { + const hash = e.extra?.hash; + // listFullEntries is newest-first; first-write-wins means the NEWEST + // capture of a URL owns its hash — matching the live ledger, which is + // repointed to the fresh slug after every forced re-ingest. + if (hash && !(hash in ledger)) ledger[hash] = e.slug; + } + return ledger; +} + +export async function readLedger(): Promise { + const file = ingestLedgerFile(); + if (await pathExists(file)) { + try { + const parsed = JSON.parse(await fs.readFile(file, "utf8")) as unknown; + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + const ledger: Ledger = {}; + for (const [k, v] of Object.entries(parsed)) { + if (typeof v === "string") ledger[k] = v; + } + return ledger; + } + } catch { + // corrupt → fall through to rebuild + } + } + const rebuilt = await rebuildFromSources(); + await atomicWrite(file, JSON.stringify(rebuilt, null, 2) + "\n"); + return rebuilt; +} + +/** The slug previously ingested for this url-hash, if any. */ +export async function lookupIngested(hash: string): Promise { + return (await readLedger())[hash] ?? null; +} + +/** Record (or repoint, after a forced re-ingest) a hash → slug mapping. */ +export async function recordIngested(hash: string, slug: string): Promise { + const ledger = await readLedger(); + ledger[hash] = slug; + await atomicWrite(ingestLedgerFile(), JSON.stringify(ledger, null, 2) + "\n"); +} diff --git a/src/kb/ingest/html-to-md.test.ts b/src/kb/ingest/html-to-md.test.ts new file mode 100644 index 0000000..28b96fc --- /dev/null +++ b/src/kb/ingest/html-to-md.test.ts @@ -0,0 +1,97 @@ +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; +import { htmlToMarkdown, decodeEntities } from "./html-to-md.js"; + +describe("htmlToMarkdown", () => { + test("headings and paragraphs", () => { + assert.equal( + htmlToMarkdown("

Title

One.

Sub

Two.

"), + "# Title\n\nOne.\n\n## Sub\n\nTwo.", + ); + }); + + test("nested and ordered lists", () => { + const md = htmlToMarkdown( + "
  • a
    • a1
    • a2
  • b
  1. x
  2. y
", + ); + assert.equal(md, "- a\n - a1\n - a2\n- b\n\n1. x\n2. y"); + }); + + test("unclosed
  • (the real-world common case)", () => { + assert.equal(htmlToMarkdown("
    • one
    • two
    "), "- one\n- two"); + }); + + test("fenced code with language, fence-proof body", () => { + const md = htmlToMarkdown( + `
    const a = 1;\nif (a < 2) {}
    `, + ); + assert.equal(md, "```ts\nconst a = 1;\nif (a < 2) {}\n```"); + // A body containing ``` must get a longer fence. + const evil = htmlToMarkdown("
    ```\ninjection\n```
    "); + assert.match(evil, /^````\n/); + }); + + test("inline code keeps backticks safe", () => { + assert.equal(htmlToMarkdown("

    run a`b now

    "), "run ``a`b`` now"); + }); + + test("blockquote wraps nested markdown", () => { + assert.equal( + htmlToMarkdown("

    quoted

    lines

    "), + "> quoted\n>\n> lines", + ); + }); + + test("tables become pipe tables", () => { + const md = htmlToMarkdown( + "
    AB
    12
    ", + ); + assert.equal(md, "| A | B |\n| --- | --- |\n| 1 | 2 |"); + }); + + test("links, images (incl. data-src lazy-loading), emphasis, hr", () => { + const md = htmlToMarkdown( + `

    site bold it


    pic

    `, + ); + assert.equal(md, "[site](https://x.dev) **bold** *it*\n\n---\n\n![pic](https://img/x.png)"); + }); + + test("markdown special characters in prose are escaped — no syntax forgery", () => { + const md = htmlToMarkdown("

    weight *not bold* and [[oauth]] and a|b

    "); + assert.equal(md, "weight \\*not bold\\* and \\[\\[oauth\\]\\] and a\\|b"); + }); + + test("javascript: links render as plain text", () => { + assert.equal(htmlToMarkdown(`

    hi

    `), "hi"); + }); + + test("script/style/iframe contents are dropped, even with tricky bodies", () => { + const md = htmlToMarkdown( + `

    keep

    also

    `, + ); + assert.equal(md, "keep\n\nalso"); + }); + + test("a self-closing drop tag does not swallow the rest of the doc", () => { + // Self-closing /