diff --git a/JS/edgechains/arakoodev/package.json b/JS/edgechains/arakoodev/package.json index 0b0bd3784..38a35d128 100644 --- a/JS/edgechains/arakoodev/package.json +++ b/JS/edgechains/arakoodev/package.json @@ -33,6 +33,7 @@ "cheerio": "^1.0.0-rc.12", "cors": "^2.8.5", "document": "^0.4.7", + "dotenv": "^16.4.5", "dts-bundle-generator": "^9.3.1", "esbuild": "^0.20.2", "hono": "3.9", diff --git a/JS/edgechains/arakoodev/src/vector-db/src/index.ts b/JS/edgechains/arakoodev/src/vector-db/src/index.ts index 557104a14..8e6b58a61 100644 --- a/JS/edgechains/arakoodev/src/vector-db/src/index.ts +++ b/JS/edgechains/arakoodev/src/vector-db/src/index.ts @@ -1 +1,16 @@ export { Supabase } from "./lib/supabase/supabase.js"; +export { + Qdrant, + type CreateCollectionArgs, + type InsertVectorDataArgs, + type PointByIdArgs, + type QdrantClient, + type QdrantDistance, + type QdrantPayload, + type QdrantPoint, + type QdrantPointId, + type QdrantVector, + type ScrollArgs, + type SearchArgs, + type UpdateByIdArgs, +} from "./lib/qdrant/qdrant.js"; diff --git a/JS/edgechains/arakoodev/src/vector-db/src/lib/qdrant/qdrant.ts b/JS/edgechains/arakoodev/src/vector-db/src/lib/qdrant/qdrant.ts new file mode 100644 index 000000000..46e810a2e --- /dev/null +++ b/JS/edgechains/arakoodev/src/vector-db/src/lib/qdrant/qdrant.ts @@ -0,0 +1,354 @@ +import { randomUUID } from "crypto"; + +export type QdrantDistance = "Cosine" | "Euclid" | "Dot" | "Manhattan"; +export type QdrantPointId = string | number; +export type QdrantPayload = Record; +export type QdrantVector = number[]; + +export interface QdrantClient { + url: string; + apiKey?: string; + fetch: FetchLike; +} + +export interface QdrantPoint { + id: QdrantPointId; + vector: QdrantVector; + payload?: QdrantPayload; +} + +export interface CreateCollectionArgs { + client: QdrantClient; + collectionName?: string; + tableName?: string; + vectorSize: number; + distance?: QdrantDistance; +} + +export interface InsertVectorDataArgs { + client: QdrantClient; + tableName?: string; + collectionName?: string; + id?: QdrantPointId; + vector?: QdrantVector; + embedding?: QdrantVector; + payload?: QdrantPayload; + wait?: boolean; + [key: string]: unknown; +} + +export interface SearchArgs { + client: QdrantClient; + tableName?: string; + collectionName?: string; + queryVector?: QdrantVector; + query_embedding?: QdrantVector; + vector?: QdrantVector; + embedding?: QdrantVector; + limit?: number; + match_count?: number; + scoreThreshold?: number; + similarity_threshold?: number; + filter?: QdrantPayload; + withPayload?: boolean; + withVector?: boolean; +} + +export interface ScrollArgs { + client: QdrantClient; + tableName?: string; + collectionName?: string; + limit?: number; + offset?: QdrantPointId | Record; + filter?: QdrantPayload; + withPayload?: boolean; + withVector?: boolean; +} + +export interface PointByIdArgs { + client: QdrantClient; + tableName?: string; + collectionName?: string; + id: QdrantPointId; +} + +export interface UpdateByIdArgs extends PointByIdArgs { + updatedContent: QdrantPayload; + wait?: boolean; +} + +interface FetchLike { + ( + input: string, + init: { + method: string; + headers: Record; + body?: string; + } + ): Promise<{ + ok: boolean; + status: number; + statusText: string; + text(): Promise; + json(): Promise; + }>; +} + +export class Qdrant { + QDRANT_URL: string; + QDRANT_API_KEY?: string; + fetcher: FetchLike; + + constructor( + QDRANT_URL?: string, + QDRANT_API_KEY?: string, + fetcher?: FetchLike + ) { + this.QDRANT_URL = + QDRANT_URL || process.env.QDRANT_URL || "http://localhost:6333"; + this.QDRANT_API_KEY = QDRANT_API_KEY || process.env.QDRANT_API_KEY; + this.fetcher = fetcher || getGlobalFetch(); + } + + createClient(): QdrantClient { + return { + url: this.QDRANT_URL.replace(/\/$/, ""), + apiKey: this.QDRANT_API_KEY, + fetch: this.fetcher, + }; + } + + async createCollection({ + client, + collectionName, + tableName, + vectorSize, + distance = "Cosine", + }: CreateCollectionArgs): Promise { + return this.request(client, "PUT", `/collections/${collection(collectionName, tableName)}`, { + vectors: { + size: vectorSize, + distance, + }, + }); + } + + async insertVectorData({ + client, + tableName, + collectionName, + id, + vector, + embedding, + payload, + wait = true, + ...rest + }: InsertVectorDataArgs): Promise { + const pointVector = vector || embedding; + if (!pointVector) { + throw new Error("Qdrant insertVectorData requires vector or embedding."); + } + + const pointPayload = payload || restPayload(rest); + const point: QdrantPoint = { + id: id || randomUUID(), + vector: pointVector, + payload: pointPayload, + }; + + return this.request( + client, + "PUT", + `/collections/${collection(collectionName, tableName)}/points?wait=${wait}`, + { points: [point] } + ); + } + + async getDataFromQuery({ + client, + tableName, + collectionName, + queryVector, + query_embedding, + vector, + embedding, + limit, + match_count, + scoreThreshold, + similarity_threshold, + filter, + withPayload = true, + withVector = false, + }: SearchArgs): Promise { + const searchVector = queryVector || query_embedding || vector || embedding; + if (!searchVector) { + throw new Error( + "Qdrant getDataFromQuery requires queryVector, query_embedding, vector, or embedding." + ); + } + + return this.request( + client, + "POST", + `/collections/${collection(collectionName, tableName)}/points/search`, + { + vector: searchVector, + limit: limit || match_count || 10, + score_threshold: scoreThreshold ?? similarity_threshold, + filter, + with_payload: withPayload, + with_vector: withVector, + } + ); + } + + async getData({ + client, + tableName, + collectionName, + limit = 10, + offset, + filter, + withPayload = true, + withVector = false, + }: ScrollArgs): Promise { + return this.request( + client, + "POST", + `/collections/${collection(collectionName, tableName)}/points/scroll`, + { + limit, + offset, + filter, + with_payload: withPayload, + with_vector: withVector, + } + ); + } + + async getDataById({ + client, + tableName, + collectionName, + id, + }: PointByIdArgs): Promise { + const response = (await this.request( + client, + "POST", + `/collections/${collection(collectionName, tableName)}/points`, + { + ids: [id], + with_payload: true, + with_vector: true, + } + )) as { result?: unknown[] }; + + return response.result?.[0] || null; + } + + async updateById({ + client, + tableName, + collectionName, + id, + updatedContent, + wait = true, + }: UpdateByIdArgs): Promise { + return this.request( + client, + "POST", + `/collections/${collection(collectionName, tableName)}/points/payload?wait=${wait}`, + { + payload: updatedContent, + points: [id], + } + ); + } + + async deleteById({ + client, + tableName, + collectionName, + id, + }: PointByIdArgs): Promise { + return this.request( + client, + "POST", + `/collections/${collection(collectionName, tableName)}/points/delete`, + { + points: [id], + } + ); + } + + private async request( + client: QdrantClient, + method: string, + path: string, + body?: Record + ): Promise { + const response = await client.fetch(`${client.url}${path}`, { + method, + headers: { + "Content-Type": "application/json", + ...(client.apiKey ? { "api-key": client.apiKey } : {}), + }, + body: body ? JSON.stringify(stripUndefined(body)) : undefined, + }); + + if (!response.ok) { + throw new Error( + `Qdrant request failed with ${response.status} ${response.statusText}: ${await response.text()}` + ); + } + + return response.json(); + } +} + +function collection(collectionName?: string, tableName?: string): string { + const value = collectionName || tableName; + if (!value) { + throw new Error("Qdrant collectionName or tableName is required."); + } + + return encodeURIComponent(value); +} + +function restPayload(rest: Record): QdrantPayload { + const { + client, + tableName, + collectionName, + id, + vector, + embedding, + wait, + ...payload + } = rest; + return payload; +} + +function stripUndefined(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(stripUndefined); + } + + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value as Record) + .filter(([, entryValue]) => entryValue !== undefined) + .map(([entryKey, entryValue]) => [entryKey, stripUndefined(entryValue)]) + ); + } + + return value; +} + +function getGlobalFetch(): FetchLike { + if (!globalThis.fetch) { + throw new Error("A fetch implementation is required to call Qdrant."); + } + + return globalThis.fetch as unknown as FetchLike; +} diff --git a/JS/edgechains/arakoodev/src/vector-db/src/tests/qdrant/qdrant.test.ts b/JS/edgechains/arakoodev/src/vector-db/src/tests/qdrant/qdrant.test.ts new file mode 100644 index 000000000..77fb97f8d --- /dev/null +++ b/JS/edgechains/arakoodev/src/vector-db/src/tests/qdrant/qdrant.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it, vi } from "vitest"; +import { Qdrant } from "../../lib/qdrant/qdrant.js"; + +function createFetch() { + return vi.fn(async (_url: string, _init: { method: string; headers: Record; body?: string }) => ({ + ok: true, + status: 200, + statusText: "OK", + text: async () => "", + json: async () => ({ result: [{ id: "point-1", payload: { content: "doc" } }], status: "ok" }), + })); +} + +describe("Qdrant", () => { + it("creates a collection with vector size and distance", async () => { + const fetch = createFetch(); + const qdrant = new Qdrant("http://localhost:6333", "secret", fetch); + const client = qdrant.createClient(); + + await qdrant.createCollection({ + client, + collectionName: "documents", + vectorSize: 1536, + distance: "Cosine", + }); + + expect(fetch).toHaveBeenCalledWith("http://localhost:6333/collections/documents", { + method: "PUT", + headers: { + "Content-Type": "application/json", + "api-key": "secret", + }, + body: JSON.stringify({ vectors: { size: 1536, distance: "Cosine" } }), + }); + }); + + it("upserts vector data using the existing tableName style", async () => { + const fetch = createFetch(); + const qdrant = new Qdrant("http://localhost:6333", undefined, fetch); + const client = qdrant.createClient(); + + await qdrant.insertVectorData({ + client, + tableName: "documents", + id: "doc-1", + content: "hello", + embedding: [0.1, 0.2, 0.3], + }); + + expect(fetch).toHaveBeenCalledWith( + "http://localhost:6333/collections/documents/points?wait=true", + expect.objectContaining({ + method: "PUT", + body: JSON.stringify({ + points: [ + { + id: "doc-1", + vector: [0.1, 0.2, 0.3], + payload: { content: "hello" }, + }, + ], + }), + }) + ); + }); + + it("searches with Qdrant points/search from getDataFromQuery", async () => { + const fetch = createFetch(); + const qdrant = new Qdrant("http://localhost:6333", undefined, fetch); + const client = qdrant.createClient(); + + await qdrant.getDataFromQuery({ + client, + tableName: "documents", + query_embedding: [0.4, 0.5, 0.6], + similarity_threshold: 0.75, + match_count: 3, + }); + + expect(fetch).toHaveBeenCalledWith( + "http://localhost:6333/collections/documents/points/search", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ + vector: [0.4, 0.5, 0.6], + limit: 3, + score_threshold: 0.75, + with_payload: true, + with_vector: false, + }), + }) + ); + }); + + it("scrolls, retrieves, updates payload, and deletes by id", async () => { + const fetch = createFetch(); + const qdrant = new Qdrant("http://localhost:6333", undefined, fetch); + const client = qdrant.createClient(); + + await qdrant.getData({ client, tableName: "documents", limit: 5 }); + const point = await qdrant.getDataById({ client, tableName: "documents", id: "doc-1" }); + await qdrant.updateById({ + client, + tableName: "documents", + id: "doc-1", + updatedContent: { content: "updated" }, + }); + await qdrant.deleteById({ client, tableName: "documents", id: "doc-1" }); + + expect(point).toEqual({ id: "point-1", payload: { content: "doc" } }); + expect(fetch.mock.calls.map(([url]) => url)).toEqual([ + "http://localhost:6333/collections/documents/points/scroll", + "http://localhost:6333/collections/documents/points", + "http://localhost:6333/collections/documents/points/payload?wait=true", + "http://localhost:6333/collections/documents/points/delete", + ]); + }); + + it("throws useful errors for failed Qdrant responses", async () => { + const fetch = vi.fn(async () => ({ + ok: false, + status: 500, + statusText: "Server Error", + text: async () => "boom", + json: async () => ({}), + })); + const qdrant = new Qdrant("http://localhost:6333", undefined, fetch); + const client = qdrant.createClient(); + + await expect( + qdrant.createCollection({ client, collectionName: "documents", vectorSize: 3 }) + ).rejects.toThrow("Qdrant request failed with 500 Server Error: boom"); + }); +}); diff --git a/JS/edgechains/examples/qdrant-vector-db/package.json b/JS/edgechains/examples/qdrant-vector-db/package.json new file mode 100644 index 000000000..d3b0b4ef5 --- /dev/null +++ b/JS/edgechains/examples/qdrant-vector-db/package.json @@ -0,0 +1,14 @@ +{ + "name": "qdrant-vector-db", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "build": "tsc", + "dev": "npm run build && node dist/index.js" + }, + "dependencies": { + "@arakoodev/edgechains.js": "file:../../arakoodev", + "typescript": "^5.6.3" + } +} diff --git a/JS/edgechains/examples/qdrant-vector-db/readme.md b/JS/edgechains/examples/qdrant-vector-db/readme.md new file mode 100644 index 000000000..23779299b --- /dev/null +++ b/JS/edgechains/examples/qdrant-vector-db/readme.md @@ -0,0 +1,10 @@ +# Qdrant Vector DB + +This example demonstrates the dependency-free Qdrant REST client exported from `@arakoodev/edgechains.js/vector-db`. + +```bash +npm install +npm run dev +``` + +The demo injects a mock `fetch` implementation so it can show the Qdrant REST requests without requiring a local Qdrant server. diff --git a/JS/edgechains/examples/qdrant-vector-db/src/index.ts b/JS/edgechains/examples/qdrant-vector-db/src/index.ts new file mode 100644 index 000000000..64dede940 --- /dev/null +++ b/JS/edgechains/examples/qdrant-vector-db/src/index.ts @@ -0,0 +1,43 @@ +import { Qdrant } from "@arakoodev/edgechains.js/vector-db"; + +const calls: Array<{ url: string; init: unknown }> = []; +const fetch = async (url: string, init: { method: string; headers: Record; body?: string }) => { + calls.push({ url, init }); + return { + ok: true, + status: 200, + statusText: "OK", + text: async () => "", + json: async () => ({ result: [], status: "ok" }), + }; +}; + +async function run() { + const qdrant = new Qdrant("http://localhost:6333", "demo-key", fetch); + const client = qdrant.createClient(); + + await qdrant.createCollection({ + client, + collectionName: "documents", + vectorSize: 3, + }); + await qdrant.insertVectorData({ + client, + tableName: "documents", + id: "doc-1", + content: "hello qdrant", + embedding: [0.1, 0.2, 0.3], + }); + await qdrant.getDataFromQuery({ + client, + tableName: "documents", + query_embedding: [0.1, 0.2, 0.3], + match_count: 1, + }); + + console.log(JSON.stringify(calls, null, 2)); +} + +run().catch((error) => { + console.error(error); +}); diff --git a/JS/edgechains/examples/qdrant-vector-db/tsconfig.json b/JS/edgechains/examples/qdrant-vector-db/tsconfig.json new file mode 100644 index 000000000..e6ee5c4f1 --- /dev/null +++ b/JS/edgechains/examples/qdrant-vector-db/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "rootDir": "./src", + "outDir": "./dist" + } +}