diff --git a/package-lock.json b/package-lock.json index cf6e191..e6d5dae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,18 +1,20 @@ { "name": "context-compression-engine", - "version": "1.3.0", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "context-compression-engine", - "version": "1.3.0", + "version": "1.4.0", "license": "AGPL-3.0-only", "devDependencies": { "@arethetypeswrong/cli": "^0.18.2", "@eslint/js": "^10.0.1", "@google/genai": "^1.46.0", + "@types/better-sqlite3": "^7.6.13", "@vitest/coverage-v8": "^4.0.18", + "better-sqlite3": "^12.8.0", "esbuild": "^0.27.3", "eslint": "^10.0.2", "openai": "^6.25.0", @@ -1291,6 +1293,16 @@ "tslib": "^2.4.0" } }, + "node_modules/@types/better-sqlite3": { + "version": "7.6.13", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz", + "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -1876,6 +1888,21 @@ ], "license": "MIT" }, + "node_modules/better-sqlite3": { + "version": "12.8.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.8.0.tgz", + "integrity": "sha512-RxD2Vd96sQDjQr20kdP+F+dK/1OUNiVOl200vKBZY8u0vTwysfolF6Hq+3ZK2+h8My9YvZhHsF+RSGZW2VYrPQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + }, + "engines": { + "node": "20.x || 22.x || 23.x || 24.x || 25.x" + } + }, "node_modules/bignumber.js": { "version": "9.3.1", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", @@ -1886,6 +1913,28 @@ "node": "*" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/brace-expansion": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.3.tgz", @@ -1899,6 +1948,31 @@ "node": "18 || 20 || >=22" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -1943,6 +2017,13 @@ "node": ">=10" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true, + "license": "ISC" + }, "node_modules/cjs-module-lexer": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", @@ -2080,6 +2161,32 @@ } } }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -2121,6 +2228,16 @@ "dev": true, "license": "MIT" }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/environment": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", @@ -2368,6 +2485,16 @@ "node": ">=0.10.0" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "dev": true, + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, "node_modules/expect-type": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", @@ -2468,6 +2595,13 @@ "node": ">=16.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true, + "license": "MIT" + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -2519,6 +2653,13 @@ "node": ">=12.20.0" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true, + "license": "MIT" + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -2574,6 +2715,13 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "dev": true, + "license": "MIT" + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -2656,6 +2804,27 @@ "node": ">= 14" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -2676,6 +2845,20 @@ "node": ">=0.8.19" } }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "license": "ISC" + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -3220,6 +3403,19 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimatch": { "version": "10.2.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", @@ -3236,6 +3432,23 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true, + "license": "MIT" + }, "node_modules/mri": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", @@ -3284,6 +3497,13 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "dev": true, + "license": "MIT" + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -3291,6 +3511,19 @@ "dev": true, "license": "MIT" }, + "node_modules/node-abi": { + "version": "3.89.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.89.0.tgz", + "integrity": "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -3368,6 +3601,16 @@ ], "license": "MIT" }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/openai": { "version": "6.32.0", "resolved": "https://registry.npmjs.org/openai/-/openai-6.32.0.tgz", @@ -3561,6 +3804,34 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -3634,6 +3905,17 @@ "url": "https://bjornlu.com/sponsor" } }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -3644,6 +3926,37 @@ "node": ">=6" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -3775,6 +4088,53 @@ "dev": true, "license": "ISC" }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/skin-tone": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/skin-tone/-/skin-tone-2.0.0.tgz", @@ -3812,6 +4172,16 @@ "dev": true, "license": "MIT" }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -3850,6 +4220,16 @@ "node": ">=8" } }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -3880,6 +4260,36 @@ "url": "https://github.com/chalk/supports-hyperlinks?sponsor=1" } }, + "node_modules/tar-fs": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/thenify": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", @@ -3968,6 +4378,19 @@ "license": "0BSD", "optional": true }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -4046,6 +4469,13 @@ "punycode": "^2.1.0" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT" + }, "node_modules/validate-npm-package-name": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/validate-npm-package-name/-/validate-npm-package-name-5.0.1.tgz", @@ -4287,6 +4717,13 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, "node_modules/ws": { "version": "8.19.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", diff --git a/package.json b/package.json index ffc6e02..48858db 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "context-compression-engine", - "version": "1.3.0", + "version": "1.4.0", "description": "Lossless context compression engine for LLMs", "type": "module", "engines": { @@ -67,11 +67,21 @@ "bugs": { "url": "https://github.com/SimplyLiz/ContextCompressionEngine/issues" }, + "peerDependencies": { + "better-sqlite3": ">=9.0.0" + }, + "peerDependenciesMeta": { + "better-sqlite3": { + "optional": true + } + }, "devDependencies": { "@arethetypeswrong/cli": "^0.18.2", "@eslint/js": "^10.0.1", "@google/genai": "^1.46.0", + "@types/better-sqlite3": "^7.6.13", "@vitest/coverage-v8": "^4.0.18", + "better-sqlite3": "^12.8.0", "esbuild": "^0.27.3", "eslint": "^10.0.2", "openai": "^6.25.0", diff --git a/src/index.ts b/src/index.ts index 1a75719..daf1a98 100644 --- a/src/index.ts +++ b/src/index.ts @@ -72,6 +72,10 @@ export { splitSentences, normalizeScores, combineScores } from './entropy.js'; export { analyzeContradictions } from './contradiction.js'; export type { ContradictionAnnotation } from './contradiction.js'; +// Persistence (optional peer dependency: better-sqlite3) +export { SqliteStore } from './sqlite-store.js'; +export type { SqliteStoreLoadResult } from './sqlite-store.js'; + // Types export type { Classifier, diff --git a/src/sqlite-store.ts b/src/sqlite-store.ts new file mode 100644 index 0000000..724e9e5 --- /dev/null +++ b/src/sqlite-store.ts @@ -0,0 +1,172 @@ +import type BetterSqlite3 from 'better-sqlite3'; +import type { CompressResult, Message, VerbatimMap } from './types.js'; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type DatabaseConstructor = new (path: string, options?: any) => BetterSqlite3.Database; + +async function loadBetterSqlite3(): Promise { + try { + const mod = await import('better-sqlite3'); + return (mod.default ?? mod) as unknown as DatabaseConstructor; + } catch { + throw new Error( + 'SqliteStore requires "better-sqlite3" as a peer dependency. ' + + 'Install it with: npm install better-sqlite3', + ); + } +} + +const SCHEMA = ` + CREATE TABLE IF NOT EXISTS conversations ( + conversation_id TEXT NOT NULL, + message_index INTEGER NOT NULL, + message_json TEXT NOT NULL, + PRIMARY KEY (conversation_id, message_index) + ); + + CREATE TABLE IF NOT EXISTS verbatim ( + conversation_id TEXT NOT NULL, + message_id TEXT NOT NULL, + message_json TEXT NOT NULL, + PRIMARY KEY (conversation_id, message_id) + ); +`; + +export interface SqliteStoreLoadResult { + messages: Message[]; + verbatim: VerbatimMap; +} + +/** + * Persistent verbatim store backed by SQLite via better-sqlite3. + * + * Provides atomic save/load of CompressResult data (messages + verbatim) + * and a StoreLookup function for use with uncompress(). + * + * Requires `better-sqlite3` as an optional peer dependency. + * + * ```ts + * const store = await SqliteStore.open(':memory:'); + * store.save('conv-1', compressResult); + * const data = store.load('conv-1'); + * const restored = uncompress(data.messages, store.lookup('conv-1')); + * store.close(); + * ``` + */ +export class SqliteStore { + private db: BetterSqlite3.Database; + + private constructor(db: BetterSqlite3.Database) { + this.db = db; + db.exec(SCHEMA); + } + + /** + * Open a SQLite store at the given path. Use `:memory:` for an in-memory database. + */ + static async open(path: string): Promise { + const Database = await loadBetterSqlite3(); + const db = new Database(path); + db.pragma('journal_mode = WAL'); + return new SqliteStore(db); + } + + /** + * Atomically save compressed messages and their verbatim originals. + * + * Messages are replaced entirely for the conversation. Verbatim entries + * are upserted — older entries from prior compression rounds are preserved + * so that recursive uncompress() works across multiple rounds. + */ + save(conversationId: string, result: CompressResult): void { + const insertMsg = this.db.prepare( + 'INSERT INTO conversations (conversation_id, message_index, message_json) VALUES (?, ?, ?)', + ); + const upsertVerbatim = this.db.prepare( + 'INSERT OR REPLACE INTO verbatim (conversation_id, message_id, message_json) VALUES (?, ?, ?)', + ); + const deleteMessages = this.db.prepare('DELETE FROM conversations WHERE conversation_id = ?'); + + this.db.transaction(() => { + deleteMessages.run(conversationId); + + for (let i = 0; i < result.messages.length; i++) { + insertMsg.run(conversationId, i, JSON.stringify(result.messages[i])); + } + + for (const [id, msg] of Object.entries(result.verbatim)) { + upsertVerbatim.run(conversationId, id, JSON.stringify(msg)); + } + })(); + } + + /** + * Load compressed messages and the full verbatim map for a conversation. + * Returns null if the conversation does not exist. + */ + load(conversationId: string): SqliteStoreLoadResult | null { + const rows = this.db + .prepare( + 'SELECT message_json FROM conversations WHERE conversation_id = ? ORDER BY message_index', + ) + .all(conversationId) as Array<{ message_json: string }>; + + if (rows.length === 0) return null; + + const messages: Message[] = rows.map((r) => JSON.parse(r.message_json) as Message); + + const verbatimRows = this.db + .prepare('SELECT message_id, message_json FROM verbatim WHERE conversation_id = ?') + .all(conversationId) as Array<{ message_id: string; message_json: string }>; + + const verbatim: VerbatimMap = {}; + for (const r of verbatimRows) { + verbatim[r.message_id] = JSON.parse(r.message_json) as Message; + } + + return { messages, verbatim }; + } + + /** + * Return a StoreLookup function for use with uncompress(). + * + * Each call executes a single-row SELECT — no need to load the + * entire verbatim map into memory. + */ + lookup(conversationId: string): (id: string) => Message | undefined { + const stmt = this.db.prepare( + 'SELECT message_json FROM verbatim WHERE conversation_id = ? AND message_id = ?', + ); + return (id: string): Message | undefined => { + const row = stmt.get(conversationId, id) as { message_json: string } | undefined; + return row ? (JSON.parse(row.message_json) as Message) : undefined; + }; + } + + /** + * Delete a conversation and all its verbatim entries. + */ + delete(conversationId: string): void { + this.db.transaction(() => { + this.db.prepare('DELETE FROM conversations WHERE conversation_id = ?').run(conversationId); + this.db.prepare('DELETE FROM verbatim WHERE conversation_id = ?').run(conversationId); + })(); + } + + /** + * List all stored conversation IDs. + */ + list(): string[] { + const rows = this.db + .prepare('SELECT DISTINCT conversation_id FROM conversations ORDER BY conversation_id') + .all() as Array<{ conversation_id: string }>; + return rows.map((r) => r.conversation_id); + } + + /** + * Close the database connection. + */ + close(): void { + this.db.close(); + } +} diff --git a/tests/sqlite-store.test.ts b/tests/sqlite-store.test.ts new file mode 100644 index 0000000..a305b83 --- /dev/null +++ b/tests/sqlite-store.test.ts @@ -0,0 +1,225 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { SqliteStore } from '../src/sqlite-store.js'; +import { compress } from '../src/compress.js'; +import { uncompress } from '../src/expand.js'; +import type { Message } from '../src/types.js'; + +function msg(id: string, index: number, role: string, content: string): Message { + return { id, index, role, content, metadata: {} }; +} + +function makeMessages(): Message[] { + return [ + msg('1', 0, 'system', 'You are a helpful assistant.'), + msg('2', 1, 'user', 'Explain how caching works in distributed systems.'), + msg( + '3', + 2, + 'assistant', + 'Caching in distributed systems involves storing frequently accessed data in a fast storage layer. ' + + 'The cache invalidation strategy determines when stale entries are removed. ' + + 'Common approaches include TTL-based expiration, write-through caching, and cache-aside patterns. ' + + 'Each approach has different trade-offs for consistency, latency, and complexity. '.repeat( + 3, + ), + ), + msg('4', 3, 'user', 'What about Redis specifically?'), + msg( + '5', + 4, + 'assistant', + 'Redis is an in-memory data structure store that supports multiple data types including strings, hashes, lists, and sorted sets. ' + + 'It provides built-in replication, Lua scripting, LRU eviction, and persistence options. '.repeat( + 3, + ), + ), + ]; +} + +describe('SqliteStore', () => { + let store: SqliteStore; + + beforeEach(async () => { + store = await SqliteStore.open(':memory:'); + }); + + afterEach(() => { + try { + store.close(); + } catch { + // already closed + } + }); + + it('opens an in-memory database', async () => { + const s = await SqliteStore.open(':memory:'); + expect(s).toBeInstanceOf(SqliteStore); + s.close(); + }); + + it('save and load roundtrip', () => { + const messages = makeMessages(); + const result = compress(messages, { recencyWindow: 0 }); + + store.save('conv-1', result); + const loaded = store.load('conv-1'); + + expect(loaded).not.toBeNull(); + expect(loaded!.messages).toEqual(result.messages); + expect(loaded!.verbatim).toEqual(result.verbatim); + }); + + it('load returns null for nonexistent conversation', () => { + expect(store.load('nonexistent')).toBeNull(); + }); + + it('save replaces messages but accumulates verbatim', () => { + const messages = makeMessages(); + const result1 = compress(messages, { recencyWindow: 0 }); + store.save('conv-1', result1); + + const verbatimCount1 = Object.keys(result1.verbatim).length; + + // Second compression with different messages + const messages2 = [ + msg( + '10', + 0, + 'user', + 'New conversation content about deployment pipelines and CI/CD. '.repeat(5), + ), + msg( + '11', + 1, + 'assistant', + 'Deployment pipelines automate the build, test, and release process. '.repeat(5), + ), + ]; + const result2 = compress(messages2, { recencyWindow: 0 }); + store.save('conv-1', result2); + + const loaded = store.load('conv-1'); + expect(loaded).not.toBeNull(); + + // Messages should be from the second save only + expect(loaded!.messages).toEqual(result2.messages); + + // Verbatim should contain entries from both rounds + const totalVerbatim = Object.keys(loaded!.verbatim).length; + const verbatimCount2 = Object.keys(result2.verbatim).length; + expect(totalVerbatim).toBe(verbatimCount1 + verbatimCount2); + }); + + it('lookup returns a working StoreLookup function for uncompress', () => { + const messages = makeMessages(); + const result = compress(messages, { recencyWindow: 0 }); + store.save('conv-1', result); + + const lookupFn = store.lookup('conv-1'); + const expanded = uncompress(result.messages, lookupFn); + + expect(expanded.missing_ids).toEqual([]); + expect(expanded.messages).toEqual(messages); + }); + + it('lookup returns undefined for missing IDs', () => { + const lookupFn = store.lookup('conv-1'); + expect(lookupFn('nonexistent-id')).toBeUndefined(); + }); + + it('delete removes conversation', () => { + const messages = makeMessages(); + const result = compress(messages, { recencyWindow: 0 }); + store.save('conv-1', result); + + store.delete('conv-1'); + expect(store.load('conv-1')).toBeNull(); + + // Verbatim should also be gone + const lookupFn = store.lookup('conv-1'); + for (const id of Object.keys(result.verbatim)) { + expect(lookupFn(id)).toBeUndefined(); + } + }); + + it('multiple conversations are isolated', () => { + const messages = makeMessages(); + const result = compress(messages, { recencyWindow: 0 }); + store.save('conv-a', result); + + const other = [msg('20', 0, 'user', 'Different conversation entirely.')]; + const result2 = compress(other, { recencyWindow: 0 }); + store.save('conv-b', result2); + + const loadedA = store.load('conv-a'); + const loadedB = store.load('conv-b'); + + expect(loadedA!.messages).toEqual(result.messages); + expect(loadedB!.messages).toEqual(result2.messages); + expect(Object.keys(loadedA!.verbatim).length).toBe(Object.keys(result.verbatim).length); + }); + + it('list returns stored conversation IDs', () => { + const messages = makeMessages(); + const result = compress(messages, { recencyWindow: 0 }); + store.save('conv-b', result); + store.save('conv-a', result); + + const ids = store.list(); + expect(ids).toEqual(['conv-a', 'conv-b']); + }); + + it('list returns empty array when no conversations stored', () => { + expect(store.list()).toEqual([]); + }); + + it('messages with extra fields survive serialization', () => { + const messages: Message[] = [ + { + id: '1', + index: 0, + role: 'assistant', + content: 'Test content that is long enough to trigger compression in the pipeline. '.repeat( + 5, + ), + metadata: { custom: 'value' }, + tool_calls: [{ id: 'tc1', function: { name: 'test', arguments: '{}' } }], + customField: 42, + }, + ]; + const result = compress(messages, { recencyWindow: 0 }); + store.save('conv-1', result); + + const loaded = store.load('conv-1'); + expect(loaded).not.toBeNull(); + + // The messages array from compress might have modified the content, + // but all fields present in the result should survive the roundtrip + expect(loaded!.messages).toEqual(result.messages); + }); + + it('recursive uncompress works across multiple compression rounds', () => { + const messages = makeMessages(); + + // Round 1: compress + const round1 = compress(messages, { recencyWindow: 0 }); + store.save('conv-1', round1); + + // Round 2: compress the already-compressed messages again + const round2 = compress(round1.messages, { recencyWindow: 0 }); + store.save('conv-1', round2); + + // Recursive uncompress should restore originals through the chain + const lookupFn = store.lookup('conv-1'); + const expanded = uncompress(round2.messages, lookupFn, { recursive: true }); + + // Should have no missing IDs — both rounds' verbatim entries are in the store + expect(expanded.missing_ids).toEqual([]); + expect(expanded.messages).toEqual(messages); + }); + + it('close prevents subsequent operations', () => { + store.close(); + expect(() => store.list()).toThrow(); + }); +});