diff --git a/ts/packages/agents/browser/src/agent/agentServiceHandlers.mts b/ts/packages/agents/browser/src/agent/agentServiceHandlers.mts index 6a9ca594f7..df7959b4d3 100644 --- a/ts/packages/agents/browser/src/agent/agentServiceHandlers.mts +++ b/ts/packages/agents/browser/src/agent/agentServiceHandlers.mts @@ -116,6 +116,8 @@ export function createAgentInvokeHandlers( websiteHandler("getWebsiteStats", params), // Macros + autoDiscoverActions: (params: any) => + discoveryHandler("autoDiscoverActions", params), detectPageActions: (params: any) => discoveryHandler("detectPageActions", params), registerPageDynamicAgent: (params: any) => diff --git a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts index 5c9f659db6..2fbd9f72ff 100644 --- a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts +++ b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts @@ -82,6 +82,110 @@ interface DiscoveryActionHandlerContext { sessionContext: SessionContext; } +// ── Discovery cache ───────────────────────────────────────────────────────── +// +// Shared by auto-discovery (on navigation) and manual discovery (context menu). +// Two-tier lookup with a TTL: +// +// 1. Exact URL match (O(1) map lookup) +// 2. Pattern match — the LLM can return a urlPattern regex indicating that +// pages with similar URLs share the same available actions. For example, +// all Amazon product pages (https://www.amazon.com/dp/...) have the same +// action set regardless of product ID. On a cache miss for the exact URL, +// we iterate pattern entries and test each regex against the current URL. +// +// When auto-discovery runs on navigation, it populates the cache. When the +// user triggers "Discover page macros", the cache is checked first to avoid +// a redundant LLM call. + +interface DiscoveryCacheEntry { + flows: WebFlowDefinition[]; + mode: "scope" | "content" | "scope-fallback"; + timestamp: number; + urlPattern?: string | undefined; // Regex matching URLs with the same available actions +} + +const DISCOVERY_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes +const discoveryCache = new Map(); + +// Separate index for pattern-based entries, keyed by the original URL that +// produced the pattern. Patterns are tested in insertion order. +const patternEntries: DiscoveryCacheEntry[] = []; + +function isExpired(entry: DiscoveryCacheEntry): boolean { + return Date.now() - entry.timestamp > DISCOVERY_CACHE_TTL_MS; +} + +function testUrlPattern(pattern: string, url: string): boolean { + try { + return new RegExp(pattern).test(url); + } catch { + debug(`Invalid urlPattern regex: ${pattern}`); + return false; + } +} + +function getCachedDiscovery(url: string): DiscoveryCacheEntry | undefined { + // 1. Exact URL match + const exact = discoveryCache.get(url); + if (exact) { + if (isExpired(exact)) { + discoveryCache.delete(url); + } else { + return exact; + } + } + + // 2. Pattern match — check entries that have a urlPattern regex + for (let i = patternEntries.length - 1; i >= 0; i--) { + const entry = patternEntries[i]; + if (isExpired(entry)) { + patternEntries.splice(i, 1); + continue; + } + if (entry.urlPattern && testUrlPattern(entry.urlPattern, url)) { + debug( + `Discovery cache pattern hit: ${entry.urlPattern} matched ${url}`, + ); + return entry; + } + } + + return undefined; +} + +function setCachedDiscovery( + url: string, + flows: WebFlowDefinition[], + mode: "scope" | "content" | "scope-fallback", + urlPattern?: string, +): void { + const entry: DiscoveryCacheEntry = { + flows, + mode, + timestamp: Date.now(), + urlPattern, + }; + + discoveryCache.set(url, entry); + + if (urlPattern) { + patternEntries.push(entry); + } + + // Evict expired entries periodically + if (discoveryCache.size > 100) { + const now = Date.now(); + for (const [key, e] of discoveryCache) { + if (now - e.timestamp > DISCOVERY_CACHE_TTL_MS) { + discoveryCache.delete(key); + } + } + } +} + +// ── Discovery handlers ────────────────────────────────────────────────────── + async function handleFindUserActions( action: any, ctx: DiscoveryActionHandlerContext, @@ -90,6 +194,21 @@ async function handleFindUserActions( ctx.sessionContext.agentContext, ).getPageUrl(); + // Check cache first — auto-discovery may have already computed this + if (url) { + const cached = getCachedDiscovery(url); + if (cached) { + debug( + `Discovery cache hit for ${url}: ${cached.flows.length} flows (mode: ${cached.mode})`, + ); + return { + displayText: `Found ${cached.flows.length} applicable actions (cached)`, + entities: ctx.entities.getEntities(), + data: { actions: cached.flows, cached: true }, + }; + } + } + // Get existing WebFlows for this domain const webFlowStore = ctx.sessionContext.agentContext.webFlowStore; let candidateFlows: WebFlowDefinition[] = []; @@ -145,8 +264,11 @@ async function handleFindUserActions( } // The LLM returns a CandidateActionList with the subset of flows - // that actually apply to this page. - const selected = response.data as { actions: { actionName: string }[] }; + // that actually apply to this page, plus an optional urlPattern. + const selected = response.data as { + actions: { actionName: string }[]; + urlPattern?: string; + }; const selectedNames = new Set(selected.actions.map((a) => a.actionName)); // Filter the full WebFlowDefinitions to only those the LLM selected @@ -154,6 +276,19 @@ async function handleFindUserActions( selectedNames.has(f.name), ); + // Cache the result for this URL (with optional pattern for similar URLs) + if (url) { + setCachedDiscovery( + url, + applicableFlows, + "content", + selected.urlPattern, + ); + if (selected.urlPattern) { + debug(`Discovery urlPattern: ${selected.urlPattern}`); + } + } + debug( `Discovery: ${candidateFlows.length} candidates → ${applicableFlows.length} applicable to page`, ); @@ -165,7 +300,127 @@ async function handleFindUserActions( }; } -function generateDiscoverySchema(flows: WebFlowDefinition[]): string { +async function handleAutoDiscoverActions( + action: any, + ctx: DiscoveryActionHandlerContext, +): Promise { + const url = action.parameters?.url; + const domain = action.parameters?.domain; + const mode = action.parameters?.mode ?? "content"; + + const webFlowStore = ctx.sessionContext.agentContext.webFlowStore; + if (!webFlowStore || !domain) { + return { + displayText: "No web flow store available", + entities: [], + data: { actions: [], flowCount: 0 }, + }; + } + + // Check cache first + if (url) { + const cached = getCachedDiscovery(url); + if (cached) { + debug( + `Auto-discovery cache hit for ${url}: ${cached.flows.length} flows`, + ); + return { + displayText: `Found ${cached.flows.length} actions for ${domain} (cached)`, + entities: [], + data: { + actions: cached.flows, + flowCount: cached.flows.length, + mode: cached.mode, + cached: true, + }, + }; + } + } + + // Layer 1: Scope-based discovery — fast domain matching + const scopedFlows = await webFlowStore.listForDomainWithDetails(domain); + + if (mode === "scope" || scopedFlows.length === 0) { + debug( + `Auto-discovery (scope): ${scopedFlows.length} flows for ${domain}`, + ); + if (url) { + setCachedDiscovery(url, scopedFlows, "scope"); + } + return { + displayText: `Found ${scopedFlows.length} actions for ${domain}`, + entities: [], + data: { + actions: scopedFlows, + flowCount: scopedFlows.length, + mode: "scope", + }, + }; + } + + // Layer 2: Content-based discovery — LLM filters to applicable flows + const discoverySchema = generateDiscoverySchema(scopedFlows); + const htmlFragments = await ctx.browser.getHtmlFragments(); + + const response = await ctx.agent.getCandidateUserActions( + discoverySchema, + htmlFragments, + ); + + if (!response.success) { + debug("Auto-discovery LLM call failed, falling back to scope results"); + if (url) { + setCachedDiscovery(url, scopedFlows, "scope-fallback"); + } + return { + displayText: `Found ${scopedFlows.length} actions for ${domain}`, + entities: [], + data: { + actions: scopedFlows, + flowCount: scopedFlows.length, + mode: "scope-fallback", + }, + }; + } + + const selected = response.data as { + actions: { actionName: string }[]; + urlPattern?: string; + }; + const selectedNames = new Set(selected.actions.map((a) => a.actionName)); + const applicableFlows = scopedFlows.filter((f) => + selectedNames.has(f.name), + ); + + // Cache the content-based result (with optional pattern for similar URLs) + if (url) { + setCachedDiscovery( + url, + applicableFlows, + "content", + selected.urlPattern, + ); + if (selected.urlPattern) { + debug(`Auto-discovery urlPattern: ${selected.urlPattern}`); + } + } + + debug( + `Auto-discovery (content): ${scopedFlows.length} scoped → ${applicableFlows.length} applicable for ${domain}`, + ); + + return { + displayText: `Found ${applicableFlows.length} applicable actions for ${domain}`, + entities: [], + data: { + actions: applicableFlows, + flowCount: applicableFlows.length, + mode: "content", + }, + }; +} + +export function generateDiscoverySchema(flows: WebFlowDefinition[]): string { const typeNames: string[] = []; const typeDefs: string[] = []; @@ -213,7 +468,15 @@ function generateDiscoverySchema(flows: WebFlowDefinition[]): string { "\n\n" + union + "\n\n" + - `export type CandidateActionList = {\n actions: CandidateActions[];\n};` + `export type CandidateActionList = {\n` + + ` actions: CandidateActions[];\n` + + ` // If other pages on this site with different URLs would have the same\n` + + ` // set of available actions (e.g. all product detail pages), provide a\n` + + ` // regex that matches those URLs. This avoids re-analyzing similar pages.\n` + + ` // Example: "https://www\\\\.amazon\\\\.com/dp/[A-Z0-9]+" for Amazon product pages.\n` + + ` // Omit if the actions are specific to this exact URL.\n` + + ` urlPattern?: string;\n` + + `};` ); } @@ -515,6 +778,19 @@ export async function handleSchemaDiscoveryAction( let result: DiscoveryActionResult; + // autoDiscoverActions is not part of SchemaDiscoveryActions — handle before switch + if ((action as any).actionName === "autoDiscoverActions") { + const adResult = await handleAutoDiscoverActions( + action, + discoveryContext, + ); + return { + displayText: adResult.displayText, + data: adResult.data, + entities: adResult.entities, + }; + } + switch (action.actionName) { case "detectPageActions": result = await handleFindUserActions(action, discoveryContext); diff --git a/ts/packages/agents/browser/src/agent/webFlows/samples/buyAllInList.json b/ts/packages/agents/browser/src/agent/webFlows/samples/buyAllInList.json new file mode 100644 index 0000000000..a9c14ef1dd --- /dev/null +++ b/ts/packages/agents/browser/src/agent/webFlows/samples/buyAllInList.json @@ -0,0 +1,28 @@ +{ + "name": "buyAllInList", + "description": "Navigate to a shopping list and add all available items to the cart", + "version": 1, + "parameters": { + "listName": { + "type": "string", + "required": true, + "description": "Name of the shopping list to buy from" + }, + "storeName": { + "type": "string", + "required": false, + "description": "Store to shop from (uses current store if not specified)" + } + }, + "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise {\n await browser.navigateTo('https://www.instacart.com/');\n await browser.awaitPageLoad();\n\n if (params.storeName) {\n const store = await browser.extractComponent<{ name: string; linkSelector?: string }>({\n typeName: 'StoreInfo',\n schema: '{ name: string; subtitle?: string; linkSelector?: string; }'\n }, params.storeName);\n if (store && store.linkSelector) {\n await browser.clickAndWait(store.linkSelector);\n }\n }\n\n const listsNav = await browser.extractComponent<{ linkSelector: string }>({\n typeName: 'ListsNavigationLink',\n schema: '{ linkSelector: string; }'\n });\n if (!listsNav || !listsNav.linkSelector) {\n throw new Error('Could not find lists navigation link');\n }\n await browser.clickAndWait(listsNav.linkSelector);\n\n const list = await browser.extractComponent<{ name: string; detailsLinkSelector: string }>({\n typeName: 'ListInfo',\n schema: '{ name: string; detailsLinkSelector: string; }'\n }, params.listName);\n if (!list || !list.detailsLinkSelector) {\n throw new Error('Could not find list: ' + params.listName);\n }\n await browser.clickAndWait(list.detailsLinkSelector);\n\n const details = await browser.extractComponent<{ name: string; products?: { name: string; price: string; availability?: string; addToCartButtonSelector?: string }[] }>({\n typeName: 'ListDetailsInfo',\n schema: '{ name: string; storeName?: string; products?: { name: string; price: string; availability?: string; addToCartButtonSelector?: string; }[]; }'\n });\n if (!details || !details.products || details.products.length === 0) {\n return { success: false, error: 'No products found in list ' + params.listName };\n }\n\n let added = 0;\n const unavailable: string[] = [];\n for (const product of details.products) {\n if (product.availability === 'Out of stock') {\n unavailable.push(product.name);\n } else if (product.addToCartButtonSelector) {\n await browser.click(product.addToCartButtonSelector);\n await browser.awaitPageInteraction();\n added++;\n }\n }\n\n let message = 'Added ' + added + ' items from \"' + params.listName + '\" to cart';\n if (unavailable.length > 0) {\n message += '. Unavailable: ' + unavailable.join(', ');\n }\n return { success: true, message };\n}", + "grammarPatterns": [ + "buy (all)? (items)? (in | from) (my)? $(listName:wildcard) list", + "add (all)? (items)? from $(listName:wildcard) list to cart", + "shop (my)? $(listName:wildcard) list" + ], + "scope": { + "type": "site", + "domains": ["instacart.com"] + }, + "source": { "type": "manual", "timestamp": "2026-04-03T00:00:00.000Z" } +} diff --git a/ts/packages/agents/browser/src/agent/webFlows/samples/buyAllInRecipe.json b/ts/packages/agents/browser/src/agent/webFlows/samples/buyAllInRecipe.json new file mode 100644 index 0000000000..b6ff26302a --- /dev/null +++ b/ts/packages/agents/browser/src/agent/webFlows/samples/buyAllInRecipe.json @@ -0,0 +1,23 @@ +{ + "name": "buyAllInRecipe", + "description": "Find a recipe on Instacart and add all its ingredients to the cart", + "version": 1, + "parameters": { + "recipeName": { + "type": "string", + "required": true, + "description": "Name of the recipe to buy ingredients for" + } + }, + "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise {\n await browser.navigateTo('https://www.instacart.com/');\n await browser.awaitPageLoad();\n\n const searchInput = await browser.extractComponent<{ cssSelector: string; submitButtonCssSelector: string }>({\n typeName: 'SearchInput',\n schema: '{ cssSelector: string; submitButtonCssSelector: string; }'\n });\n if (!searchInput || !searchInput.cssSelector) {\n throw new Error('Could not find search input');\n }\n await browser.clearAndType(searchInput.cssSelector, 'recipes: ' + params.recipeName);\n if (searchInput.submitButtonCssSelector) {\n await browser.clickAndWait(searchInput.submitButtonCssSelector);\n } else {\n await browser.pressKey('Enter');\n await browser.awaitPageLoad();\n }\n\n const recipe = await browser.extractComponent<{ name: string; detailsLinkSelector: string }>({\n typeName: 'RecipeInfo',\n schema: '{ name: string; subtitle: string; detailsLinkSelector: string; }'\n }, params.recipeName);\n if (!recipe || !recipe.detailsLinkSelector) {\n throw new Error('Could not find recipe: ' + params.recipeName);\n }\n await browser.clickAndWait(recipe.detailsLinkSelector);\n\n const hero = await browser.extractComponent<{ recipeName: string; addAllIngredientsSelector: string; ingredients: { name: string; price: string; addToCartButtonSelector?: string }[] }>({\n typeName: 'RecipeHeroSection',\n schema: '{ recipeName: string; summary: string; addAllIngredientsSelector: string; saveButtonSelector: string; ingredients: { name: string; price: string; addToCartButtonSelector?: string; }[]; }'\n });\n if (!hero || !hero.addAllIngredientsSelector) {\n throw new Error('Could not find recipe details for: ' + params.recipeName);\n }\n await browser.clickAndWait(hero.addAllIngredientsSelector);\n const count = hero.ingredients ? hero.ingredients.length : 0;\n return { success: true, message: 'Added ' + count + ' ingredients from \"' + hero.recipeName + '\" to cart' };\n}", + "grammarPatterns": [ + "buy (all)? ingredients for $(recipeName:wildcard) recipe", + "add $(recipeName:wildcard) recipe ingredients to cart", + "shop for $(recipeName:wildcard) recipe" + ], + "scope": { + "type": "site", + "domains": ["instacart.com"] + }, + "source": { "type": "manual", "timestamp": "2026-04-03T00:00:00.000Z" } +} diff --git a/ts/packages/agents/browser/src/agent/webFlows/samples/buyItAgain.json b/ts/packages/agents/browser/src/agent/webFlows/samples/buyItAgain.json new file mode 100644 index 0000000000..59a8e7ea29 --- /dev/null +++ b/ts/packages/agents/browser/src/agent/webFlows/samples/buyItAgain.json @@ -0,0 +1,29 @@ +{ + "name": "buyItAgain", + "description": "Reorder items from your past purchases on Instacart", + "version": 1, + "parameters": { + "storeName": { + "type": "string", + "required": true, + "description": "Store to reorder from" + }, + "productName": { + "type": "string", + "required": false, + "description": "Specific product to reorder. If not specified, adds all past items." + } + }, + "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise {\n await browser.navigateTo('https://www.instacart.com/');\n await browser.awaitPageLoad();\n\n const store = await browser.extractComponent<{ name: string; linkSelector?: string }>({\n typeName: 'StoreInfo',\n schema: '{ name: string; subtitle?: string; linkSelector?: string; }'\n }, params.storeName);\n if (store && store.linkSelector) {\n await browser.clickAndWait(store.linkSelector);\n }\n\n const buyAgainNav = await browser.extractComponent<{ linkSelector: string }>({\n typeName: 'BuyItAgainNavigationLink',\n schema: '{ linkSelector: string; }'\n });\n if (!buyAgainNav || !buyAgainNav.linkSelector) {\n throw new Error('Could not find Buy It Again section');\n }\n await browser.clickAndWait(buyAgainNav.linkSelector);\n\n const section = await browser.extractComponent<{ products?: { name: string; price: string; availability?: string; addToCartButtonSelector?: string }[] }>({\n typeName: 'BuyItAgainHeaderSection',\n schema: '{ allItemsSelector: string; pastOrdersSelector: string; products?: { name: string; price: string; availability?: string; addToCartButtonSelector?: string; }[]; }'\n });\n if (!section || !section.products || section.products.length === 0) {\n return { success: false, error: 'No past purchase items found' };\n }\n\n if (params.productName) {\n const match = section.products.find(\n p => p.name.toLowerCase().includes(params.productName.toLowerCase())\n );\n if (!match || !match.addToCartButtonSelector) {\n return { success: false, error: 'Could not find ' + params.productName + ' in past purchases' };\n }\n await browser.click(match.addToCartButtonSelector);\n await browser.awaitPageInteraction();\n return { success: true, message: 'Reordered ' + match.name + ' from Buy It Again' };\n }\n\n let added = 0;\n for (const product of section.products) {\n if (product.availability !== 'Out of stock' && product.addToCartButtonSelector) {\n await browser.click(product.addToCartButtonSelector);\n await browser.awaitPageInteraction();\n added++;\n }\n }\n return { success: true, message: 'Added ' + added + ' items from Buy It Again to cart' };\n}", + "grammarPatterns": [ + "buy it again from $(storeName:wildcard)", + "reorder (from)? $(storeName:wildcard)", + "buy $(productName:wildcard) again from $(storeName:wildcard)", + "reorder (all)? (past)? items from $(storeName:wildcard)" + ], + "scope": { + "type": "site", + "domains": ["instacart.com"] + }, + "source": { "type": "manual", "timestamp": "2026-04-03T00:00:00.000Z" } +} diff --git a/ts/packages/agents/browser/src/agent/webFlows/samples/buyProduct.json b/ts/packages/agents/browser/src/agent/webFlows/samples/buyProduct.json new file mode 100644 index 0000000000..4d848e643b --- /dev/null +++ b/ts/packages/agents/browser/src/agent/webFlows/samples/buyProduct.json @@ -0,0 +1,36 @@ +{ + "name": "buyProduct", + "description": "Search for a product and add it to the shopping cart. If already on the product page, adds directly without searching.", + "version": 1, + "parameters": { + "productName": { + "type": "string", + "required": true, + "description": "The product to find and add to cart" + } + }, + "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise {\n const pageState = await browser.checkPageState(\n `Product detail page for \"${params.productName}\" with an add-to-cart button visible`\n );\n\n if (!pageState.matched) {\n const searchInput = await browser.extractComponent<{ cssSelector: string; submitButtonCssSelector: string }>({\n typeName: 'SearchInput',\n schema: '{ cssSelector: string; submitButtonCssSelector: string; }'\n }, params.productName);\n if (!searchInput || !searchInput.cssSelector) {\n throw new Error('Could not find search input on this page');\n }\n await browser.clearAndType(searchInput.cssSelector, params.productName);\n if (searchInput.submitButtonCssSelector) {\n await browser.clickAndWait(searchInput.submitButtonCssSelector);\n } else {\n await browser.pressKey('Enter');\n await browser.awaitPageLoad();\n }\n\n const product = await browser.extractComponent<{ name: string; detailsLinkSelector: string }>({\n typeName: 'ProductTile',\n schema: '{ name: string; price: string; detailsLinkSelector: string; }'\n }, params.productName);\n if (!product || !product.detailsLinkSelector) {\n throw new Error('Could not find product: ' + params.productName);\n }\n await browser.clickAndWait(product.detailsLinkSelector);\n }\n\n const hero = await browser.extractComponent<{ name: string; addToCartButtonSelector?: string }>({\n typeName: 'ProductDetailsHero',\n schema: '{ name: string; price: string; addToCartButtonSelector?: string; }'\n }, params.productName);\n if (!hero || !hero.addToCartButtonSelector) {\n throw new Error('Could not find add-to-cart button on this page');\n }\n await browser.click(hero.addToCartButtonSelector);\n await browser.awaitPageInteraction();\n return { success: true, message: 'Added ' + (hero.name || params.productName) + ' to cart' };\n}", + "grammarPatterns": [ + "buy $(productName:wildcard)", + "purchase $(productName:wildcard)", + "order $(productName:wildcard)", + "buy (me)? (a)? $(productName:wildcard)" + ], + "scope": { + "type": "site", + "domains": [ + "amazon.com", + "amazon.co.uk", + "amazon.de", + "amazon.fr", + "target.com", + "walmart.com", + "bestbuy.com", + "homedepot.com", + "lowes.com", + "costco.com", + "ebay.com" + ] + }, + "source": { "type": "manual", "timestamp": "2026-04-03T00:00:00.000Z" } +} diff --git a/ts/packages/agents/browser/src/agent/webFlows/samples/saveRecipe.json b/ts/packages/agents/browser/src/agent/webFlows/samples/saveRecipe.json new file mode 100644 index 0000000000..8e9c3bd626 --- /dev/null +++ b/ts/packages/agents/browser/src/agent/webFlows/samples/saveRecipe.json @@ -0,0 +1,23 @@ +{ + "name": "saveRecipe", + "description": "Save the currently viewed recipe to your Instacart collection", + "version": 1, + "parameters": { + "recipeName": { + "type": "string", + "required": true, + "description": "Name of the recipe to save" + } + }, + "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise {\n const hero = await browser.extractComponent<{ recipeName: string; saveButtonSelector: string }>({\n typeName: 'RecipeHeroSection',\n schema: '{ recipeName: string; summary: string; addAllIngredientsSelector: string; saveButtonSelector: string; ingredients: { name: string; price: string; }[]; }'\n }, params.recipeName);\n if (!hero || !hero.saveButtonSelector) {\n throw new Error('Could not find recipe save button for: ' + params.recipeName);\n }\n await browser.click(hero.saveButtonSelector);\n await browser.awaitPageInteraction();\n return { success: true, message: 'Saved recipe: ' + hero.recipeName };\n}", + "grammarPatterns": [ + "save (this)? $(recipeName:wildcard) recipe", + "bookmark (the)? $(recipeName:wildcard) recipe", + "save recipe $(recipeName:wildcard)" + ], + "scope": { + "type": "site", + "domains": ["instacart.com"] + }, + "source": { "type": "manual", "timestamp": "2026-04-03T00:00:00.000Z" } +} diff --git a/ts/packages/agents/browser/src/agent/webFlows/samples/searchForRecipe.json b/ts/packages/agents/browser/src/agent/webFlows/samples/searchForRecipe.json new file mode 100644 index 0000000000..d608a27cc6 --- /dev/null +++ b/ts/packages/agents/browser/src/agent/webFlows/samples/searchForRecipe.json @@ -0,0 +1,23 @@ +{ + "name": "searchForRecipe", + "description": "Search for a recipe on Instacart", + "version": 1, + "parameters": { + "keyword": { + "type": "string", + "required": true, + "description": "Recipe keyword to search for (e.g. 'chicken parmesan')" + } + }, + "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise {\n await browser.navigateTo('https://www.instacart.com/');\n await browser.awaitPageLoad();\n\n const searchInput = await browser.extractComponent<{ cssSelector: string; submitButtonCssSelector: string }>({\n typeName: 'SearchInput',\n schema: '{ cssSelector: string; submitButtonCssSelector: string; }'\n });\n if (!searchInput || !searchInput.cssSelector) {\n throw new Error('Could not find search input');\n }\n await browser.clearAndType(searchInput.cssSelector, 'recipes: ' + params.keyword);\n if (searchInput.submitButtonCssSelector) {\n await browser.clickAndWait(searchInput.submitButtonCssSelector);\n } else {\n await browser.pressKey('Enter');\n await browser.awaitPageLoad();\n }\n\n const recipe = await browser.extractComponent<{ name: string; subtitle: string; detailsLinkSelector: string }>({\n typeName: 'RecipeInfo',\n schema: '{ name: string; subtitle: string; detailsLinkSelector: string; }'\n }, params.keyword);\n if (!recipe || !recipe.detailsLinkSelector) {\n return { success: false, error: 'No recipes found for: ' + params.keyword };\n }\n await browser.clickAndWait(recipe.detailsLinkSelector);\n return { success: true, message: 'Found recipe: ' + recipe.name, data: { name: recipe.name, subtitle: recipe.subtitle } };\n}", + "grammarPatterns": [ + "search for (a)? $(keyword:wildcard) recipe", + "find (a)? recipe for $(keyword:wildcard)", + "look up $(keyword:wildcard) recipe (on instacart)?" + ], + "scope": { + "type": "site", + "domains": ["instacart.com"] + }, + "source": { "type": "manual", "timestamp": "2026-04-03T00:00:00.000Z" } +} diff --git a/ts/packages/agents/browser/src/agent/webFlows/samples/searchForReservation.json b/ts/packages/agents/browser/src/agent/webFlows/samples/searchForReservation.json new file mode 100644 index 0000000000..fb1961baee --- /dev/null +++ b/ts/packages/agents/browser/src/agent/webFlows/samples/searchForReservation.json @@ -0,0 +1,30 @@ +{ + "name": "searchForReservation", + "description": "Search for a restaurant and show available reservation time slots", + "version": 1, + "parameters": { + "restaurantName": { + "type": "string", + "required": true, + "description": "Name of the restaurant to find" + }, + "numberOfPeople": { + "type": "number", + "required": false, + "default": 2, + "description": "Party size for the reservation" + } + }, + "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise {\n const searchInput = await browser.extractComponent<{ cssSelector: string; submitButtonCssSelector: string }>({\n typeName: 'SearchInput',\n schema: '{ cssSelector: string; submitButtonCssSelector: string; }'\n }, params.restaurantName);\n if (!searchInput || !searchInput.cssSelector) {\n throw new Error('Could not find search input on this page');\n }\n await browser.clearAndType(searchInput.cssSelector, params.restaurantName);\n if (searchInput.submitButtonCssSelector) {\n await browser.clickAndWait(searchInput.submitButtonCssSelector);\n } else {\n await browser.pressKey('Enter');\n await browser.awaitPageLoad();\n }\n\n const restaurant = await browser.extractComponent<{ restaurantName: string; detailsLinkSelector: string }>({\n typeName: 'RestaurantResult',\n schema: '{ restaurantName: string; rating: string; detailsLinkSelector: string; }'\n }, params.restaurantName);\n if (!restaurant || !restaurant.detailsLinkSelector) {\n throw new Error('Could not find restaurant: ' + params.restaurantName);\n }\n await browser.clickAndWait(restaurant.detailsLinkSelector);\n\n const reservations = await browser.extractComponent<{ date: string; availableTimeSlots?: { time?: string; cssSelector: string }[] }>({\n typeName: 'BookReservationsModule',\n schema: '{ date: string; targetTime: string; availableTimeSlots?: { time?: string; cssSelector: string; }[]; numberOfPeople?: number; }'\n });\n if (!reservations || !reservations.availableTimeSlots || reservations.availableTimeSlots.length === 0) {\n return { success: false, error: 'No available time slots found for ' + params.restaurantName };\n }\n const times = reservations.availableTimeSlots.map(s => s.time).filter(Boolean).join(', ');\n return { success: true, message: 'Available times at ' + params.restaurantName + ': ' + times, data: reservations };\n}", + "grammarPatterns": [ + "find (a)? reservation at $(restaurantName:wildcard)", + "book $(restaurantName:wildcard) for $(numberOfPeople:number) (people)?", + "search for reservation at $(restaurantName:wildcard)", + "check availability at $(restaurantName:wildcard)" + ], + "scope": { + "type": "site", + "domains": ["opentable.com", "resy.com"] + }, + "source": { "type": "manual", "timestamp": "2026-04-03T00:00:00.000Z" } +} diff --git a/ts/packages/agents/browser/src/agent/webFlows/samples/selectReservation.json b/ts/packages/agents/browser/src/agent/webFlows/samples/selectReservation.json new file mode 100644 index 0000000000..4ac4f0281e --- /dev/null +++ b/ts/packages/agents/browser/src/agent/webFlows/samples/selectReservation.json @@ -0,0 +1,23 @@ +{ + "name": "selectReservation", + "description": "Select a specific reservation time slot from the available options on a restaurant page", + "version": 1, + "parameters": { + "time": { + "type": "string", + "required": true, + "description": "The reservation time to select (e.g. '7:00 PM')" + } + }, + "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise {\n const reservations = await browser.extractComponent<{ date: string; availableTimeSlots?: { time?: string; cssSelector: string }[] }>({\n typeName: 'BookReservationsModule',\n schema: '{ date: string; targetTime: string; availableTimeSlots?: { time?: string; cssSelector: string; }[]; numberOfPeople?: number; }'\n });\n if (!reservations || !reservations.availableTimeSlots || reservations.availableTimeSlots.length === 0) {\n return { success: false, error: 'No available time slots found on this page' };\n }\n const targetSlot = reservations.availableTimeSlots.find(\n s => s.time && s.time.toLowerCase().includes(params.time.toLowerCase())\n );\n if (!targetSlot) {\n const available = reservations.availableTimeSlots.map(s => s.time).filter(Boolean).join(', ');\n return { success: false, error: 'Time ' + params.time + ' not available. Available: ' + available };\n }\n await browser.clickAndWait(targetSlot.cssSelector);\n return { success: true, message: 'Selected reservation at ' + targetSlot.time };\n}", + "grammarPatterns": [ + "select (the)? $(time:wildcard) reservation", + "book (the)? $(time:wildcard) (time)? slot", + "reserve (at)? $(time:wildcard)" + ], + "scope": { + "type": "site", + "domains": ["opentable.com", "resy.com"] + }, + "source": { "type": "manual", "timestamp": "2026-04-03T00:00:00.000Z" } +} diff --git a/ts/packages/agents/browser/src/agent/webFlows/samples/setPreferredStore.json b/ts/packages/agents/browser/src/agent/webFlows/samples/setPreferredStore.json new file mode 100644 index 0000000000..42e881893e --- /dev/null +++ b/ts/packages/agents/browser/src/agent/webFlows/samples/setPreferredStore.json @@ -0,0 +1,23 @@ +{ + "name": "setPreferredStore", + "description": "Select a store as your preferred shopping location on Instacart", + "version": 1, + "parameters": { + "storeName": { + "type": "string", + "required": true, + "description": "Name of the store to set as preferred (e.g. 'Costco', 'Safeway')" + } + }, + "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise {\n await browser.navigateTo('https://www.instacart.com/');\n await browser.awaitPageLoad();\n\n const store = await browser.extractComponent<{ name: string; linkSelector?: string }>({\n typeName: 'StoreInfo',\n schema: '{ name: string; subtitle?: string; linkSelector?: string; }'\n }, params.storeName);\n if (!store || !store.linkSelector) {\n throw new Error('Could not find store: ' + params.storeName);\n }\n await browser.clickAndWait(store.linkSelector);\n return { success: true, message: 'Selected ' + store.name + ' as your store' };\n}", + "grammarPatterns": [ + "set (my)? (preferred)? store to $(storeName:wildcard)", + "switch (to)? $(storeName:wildcard) (store)?", + "shop at $(storeName:wildcard)" + ], + "scope": { + "type": "site", + "domains": ["instacart.com"] + }, + "source": { "type": "manual", "timestamp": "2026-04-03T00:00:00.000Z" } +} diff --git a/ts/packages/agents/browser/src/common/serviceTypes.mts b/ts/packages/agents/browser/src/common/serviceTypes.mts index 836827fe37..73e75b3cb0 100644 --- a/ts/packages/agents/browser/src/common/serviceTypes.mts +++ b/ts/packages/agents/browser/src/common/serviceTypes.mts @@ -194,6 +194,11 @@ export type BrowserAgentInvokeFunctions = { getLibraryStats(params: any): Promise; // Macros + autoDiscoverActions(params: { + url: string; + domain: string; + mode: "scope" | "content"; + }): Promise; detectPageActions(params: { registerAgent?: boolean }): Promise; registerPageDynamicAgent(params: { agentName: string }): Promise; @@ -446,6 +451,7 @@ export type AllServiceWorkerInvokeFunctions = ExtensionLocalInvokeFunctions & analyzeKnowledgeGaps(params: any): Promise; indexPageContentDirect(params: any): Promise; autoIndexPage(params: any): Promise; + autoDiscoverActions(params: any): Promise; getPageIndexStatus(params: any): Promise; getPageIndexedKnowledge(params: any): Promise; indexExtractedKnowledge(params: any): Promise; diff --git a/ts/packages/agents/browser/src/extension/contentScript/autoDiscovery.ts b/ts/packages/agents/browser/src/extension/contentScript/autoDiscovery.ts new file mode 100644 index 0000000000..78c67a1330 --- /dev/null +++ b/ts/packages/agents/browser/src/extension/contentScript/autoDiscovery.ts @@ -0,0 +1,264 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +interface AutoDiscoverySettings { + autoDiscovery: boolean; + autoDiscoveryMode: "scope" | "content"; + excludeSensitiveSites: boolean; +} + +class AutoDiscoveryManager { + private discoveryTimeout: number | null = null; + private lastUrl: string = ""; + private isDiscovering: boolean = false; + private settings: AutoDiscoverySettings = { + autoDiscovery: true, + autoDiscoveryMode: "content", + excludeSensitiveSites: true, + }; + + async initialize() { + console.log("Initializing AutoDiscoveryManager"); + + await this.loadSettings(); + + chrome.storage.onChanged.addListener((changes) => { + if (this.settingsChanged(changes)) { + this.loadSettings(); + } + }); + + this.setupNavigationListeners(); + + await this.checkForAutoDiscovery(); + } + + private settingsChanged(changes: { + [key: string]: chrome.storage.StorageChange; + }): boolean { + const relevantKeys = [ + "autoDiscovery", + "autoDiscoveryMode", + "excludeSensitiveSites", + ]; + return relevantKeys.some((key) => changes[key]); + } + + private async loadSettings() { + try { + const result = await chrome.storage.sync.get([ + "autoDiscovery", + "autoDiscoveryMode", + "excludeSensitiveSites", + ]); + + this.settings = { + autoDiscovery: result.autoDiscovery !== false, // default true + autoDiscoveryMode: result.autoDiscoveryMode || "content", + excludeSensitiveSites: result.excludeSensitiveSites !== false, // default true + }; + + console.log("Auto-discovery settings loaded:", this.settings); + } catch (error) { + console.error("Error loading auto-discovery settings:", error); + } + } + + private setupNavigationListeners() { + let currentUrl = window.location.href; + + const observer = new MutationObserver(() => { + if (window.location.href !== currentUrl) { + currentUrl = window.location.href; + this.onNavigationChange(); + } + }); + + observer.observe(document.body, { + childList: true, + subtree: true, + }); + + window.addEventListener("popstate", () => this.onNavigationChange()); + + const originalPushState = history.pushState; + const originalReplaceState = history.replaceState; + + history.pushState = function (...args) { + originalPushState.apply(history, args); + window.dispatchEvent(new Event("pushstate")); + }; + + history.replaceState = function (...args) { + originalReplaceState.apply(history, args); + window.dispatchEvent(new Event("replacestate")); + }; + + window.addEventListener("pushstate", () => this.onNavigationChange()); + window.addEventListener("replacestate", () => + this.onNavigationChange(), + ); + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", () => + this.onPageLoad(), + ); + } else { + this.onPageLoad(); + } + } + + private async onNavigationChange() { + if (this.discoveryTimeout) { + clearTimeout(this.discoveryTimeout); + this.discoveryTimeout = null; + } + + await this.checkForAutoDiscovery(); + } + + private async onPageLoad() { + await this.checkForAutoDiscovery(); + } + + private async checkForAutoDiscovery() { + if (!this.settings.autoDiscovery) { + return; + } + + const currentUrl = window.location.href; + if (currentUrl === this.lastUrl) { + return; + } + + this.lastUrl = currentUrl; + + if ( + this.settings.excludeSensitiveSites && + this.isSensitiveSite(currentUrl) + ) { + console.log( + "Skipping auto-discovery for sensitive site:", + currentUrl, + ); + return; + } + + if (!this.shouldDiscoverUrl(currentUrl)) { + return; + } + + const delay = 300; + + this.discoveryTimeout = window.setTimeout(async () => { + if (!this.isDiscovering && this.isPageReady()) { + await this.performDiscovery(); + } + }, delay); + } + + private isSensitiveSite(url: string): boolean { + const sensitivePatterns = [ + /banking/i, + /bank\./i, + /credit.*union/i, + /paypal/i, + /payment/i, + /checkout/i, + /billing/i, + /financial/i, + /login/i, + /signin/i, + /auth/i, + /password/i, + /healthcare/i, + /medical/i, + /patient/i, + /pharmacy/i, + ]; + + return sensitivePatterns.some((pattern) => pattern.test(url)); + } + + private shouldDiscoverUrl(url: string): boolean { + if (!url.startsWith("http://") && !url.startsWith("https://")) { + return false; + } + + const mediaExtensions = + /\.(pdf|doc|docx|xls|xlsx|ppt|pptx|zip|rar|exe|dmg|pkg|mp4|mp3|avi|mov|jpg|jpeg|png|gif|svg)$/i; + if (mediaExtensions.test(url)) { + return false; + } + + return true; + } + + private isPageReady(): boolean { + return ( + document.readyState === "complete" && + document.body && + document.body.children.length > 0 + ); + } + + private async performDiscovery() { + if (this.isDiscovering) return; + + this.isDiscovering = true; + + try { + const url = window.location.href; + const domain = new URL(url).hostname; + + console.log("Auto-discovering actions for:", domain); + + const response = await chrome.runtime.sendMessage({ + type: "autoDiscoverActions", + url, + domain, + mode: this.settings.autoDiscoveryMode, + }); + + if (response?.success) { + console.log( + `Auto-discovery found ${response.flowCount ?? 0} actions for:`, + domain, + ); + } + } catch (error) { + console.error("Auto-discovery error:", error); + } finally { + this.isDiscovering = false; + } + } + + isAutoDiscoveryEnabled(): boolean { + return this.settings.autoDiscovery; + } +} + +let autoDiscoveryManager: AutoDiscoveryManager | null = null; + +if (window.location.href.startsWith("http")) { + autoDiscoveryManager = new AutoDiscoveryManager(); + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", () => { + autoDiscoveryManager?.initialize(); + }); + } else { + autoDiscoveryManager.initialize(); + } +} + +chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { + if (message.type === "getAutoDiscoveryStatus") { + sendResponse({ + enabled: autoDiscoveryManager?.isAutoDiscoveryEnabled() || false, + discovering: autoDiscoveryManager?.["isDiscovering"] || false, + }); + } +}); + +export { AutoDiscoveryManager }; diff --git a/ts/packages/agents/browser/src/extension/contentScript/index.ts b/ts/packages/agents/browser/src/extension/contentScript/index.ts index b109b807db..3dd260a7c4 100644 --- a/ts/packages/agents/browser/src/extension/contentScript/index.ts +++ b/ts/packages/agents/browser/src/extension/contentScript/index.ts @@ -7,6 +7,7 @@ import { interceptHistory } from "./spaNavigation"; import { PDFInterceptor } from "./pdfInterceptor"; import { initializeContinuationHandler } from "./continuationHandler"; import "./autoIndexing"; // Initialize auto-indexing +import "./autoDiscovery"; // Initialize auto-discovery of webflow actions // Imports to help with bundling import "./domUtils"; diff --git a/ts/packages/agents/browser/src/extension/manifest.json b/ts/packages/agents/browser/src/extension/manifest.json index faeb6c35ef..aac24bc6fe 100644 --- a/ts/packages/agents/browser/src/extension/manifest.json +++ b/ts/packages/agents/browser/src/extension/manifest.json @@ -57,40 +57,6 @@ "run_at": "document_start", "world": "MAIN" }, - { - "matches": [ - "https://*.amazon.com/*", - "https://*.amazon.co.uk/*", - "https://*.amazon.de/*", - "https://*.amazon.fr/*", - "https://*.amazon.es/*", - "https://*.amazon.it/*", - "https://*.amazon.nl/*", - "https://*.amazon.in/*", - "https://*.amazon.ca/*", - "https://*.amazon.com.mx/*", - "https://*.amazon.com.br/*", - "https://*.amazon.com.au/*", - "https://*.target.com/*", - "https://*.walmart.com/*", - "https://*.bestbuy.com/*", - "https://*.homedepot.com/*", - "https://*.lowes.com/*", - "https://*.costco.com/*", - "https://*.ebay.com/*", - "https://*.opentable.com/*", - "https://*.resy.com/*" - ], - "js": ["sites/commerce.js"], - "run_at": "document_start", - "world": "MAIN" - }, - { - "matches": ["https://*.instacart.com/*"], - "js": ["sites/instacart.js"], - "run_at": "document_start", - "world": "MAIN" - }, { "matches": ["https://paleobiodb.org/*"], "js": ["sites/paleobiodb.js"], diff --git a/ts/packages/agents/browser/src/extension/serviceWorker/serviceWorkerRpcHandlers.ts b/ts/packages/agents/browser/src/extension/serviceWorker/serviceWorkerRpcHandlers.ts index 228dc434f5..b2cd71f629 100644 --- a/ts/packages/agents/browser/src/extension/serviceWorker/serviceWorkerRpcHandlers.ts +++ b/ts/packages/agents/browser/src/extension/serviceWorker/serviceWorkerRpcHandlers.ts @@ -440,6 +440,26 @@ export function createAllHandlers(): AllServiceWorkerInvokeFunctions { }; }, + async autoDiscoverActions(params: any) { + try { + const result = await forward("autoDiscoverActions", { + url: params.url, + domain: params.domain, + mode: params.mode || "scope", + }); + return { + success: true, + flowCount: result?.flowCount ?? 0, + }; + } catch (error) { + return { + success: false, + error: + error instanceof Error ? error.message : String(error), + }; + } + }, + async indexExtractedKnowledge(params: any) { try { const result = await forward("indexWebPageContent", { diff --git a/ts/packages/agents/browser/src/extension/views/options.html b/ts/packages/agents/browser/src/extension/views/options.html index deb21c2a9f..890d556f04 100644 --- a/ts/packages/agents/browser/src/extension/views/options.html +++ b/ts/packages/agents/browser/src/extension/views/options.html @@ -209,6 +209,45 @@

TypeAgent Knowledge Settings

+ +
+
Action Auto-Discovery
+
+ + +
+
+ + +
+
+ + +
+
+
+ + Lists + Cart + + +
+

Choose your store

+
+ +

Costco

+

Delivery by 3:00 PM

+ + +

Safeway

+

Delivery by 2:00 PM

+ + +

Sprouts

+

Delivery by 4:00 PM

+ +
+
+

Buy It Again

+ View all past items +
+
+ + diff --git a/ts/packages/agents/browser/test/fixtures/discovery-pages/instacart-recipe.html b/ts/packages/agents/browser/test/fixtures/discovery-pages/instacart-recipe.html new file mode 100644 index 0000000000..124e0dbb76 --- /dev/null +++ b/ts/packages/agents/browser/test/fixtures/discovery-pages/instacart-recipe.html @@ -0,0 +1,49 @@ + + + + + + Chicken Parmesan Recipe - Instacart + + +
+ +
+
+
+

Classic Chicken Parmesan

+

+ A family favorite with crispy breaded chicken and melted mozzarella. +

+ + +
+
+

Ingredients

+
    +
  • + Chicken Breast (2 lbs) + $8.99 + +
    • +
  • + Marinara Sauce (24 oz) + $3.49 + +
    • +
  • + Mozzarella Cheese (8 oz) + $4.29 + +
    • +
+
+
+ + diff --git a/ts/packages/agents/browser/test/fixtures/discovery-pages/known-failures.json b/ts/packages/agents/browser/test/fixtures/discovery-pages/known-failures.json new file mode 100644 index 0000000000..9d58d82df9 --- /dev/null +++ b/ts/packages/agents/browser/test/fixtures/discovery-pages/known-failures.json @@ -0,0 +1,17 @@ +{ + "description": "Known content-based discovery failures to track and improve over time. Listed failures are reported in the benchmark but do not block the exit code.", + "failures": [ + { + "page": "product-detail.html", + "missingFlow": "getLocationInStore", + "reason": "After HTML reduction, the 'Aisle 7, Shelf B' store location text loses context. The LLM doesn't associate it with an in-store location action.", + "filed": "2026-04-03" + }, + { + "page": "instacart-home.html", + "missingFlow": "searchForProduct", + "reason": "After HTML reduction, the search input loses identifying attributes. The LLM doesn't recognize it as a product search capability.", + "filed": "2026-04-03" + } + ] +} diff --git a/ts/packages/agents/browser/test/fixtures/discovery-pages/non-commerce.html b/ts/packages/agents/browser/test/fixtures/discovery-pages/non-commerce.html new file mode 100644 index 0000000000..02c6979d62 --- /dev/null +++ b/ts/packages/agents/browser/test/fixtures/discovery-pages/non-commerce.html @@ -0,0 +1,38 @@ + + + + + + TypeScript Documentation + + +
+ +
+
+

TypeScript Handbook

+
+

Getting Started

+

+ TypeScript is a typed superset of JavaScript that compiles to plain + JavaScript. +

+

Basic Types

+

+ TypeScript supports the same basic types as JavaScript with some + additions. +

+ +
+
+ + diff --git a/ts/packages/agents/browser/test/fixtures/discovery-pages/product-detail.html b/ts/packages/agents/browser/test/fixtures/discovery-pages/product-detail.html new file mode 100644 index 0000000000..cf214b7217 --- /dev/null +++ b/ts/packages/agents/browser/test/fixtures/discovery-pages/product-detail.html @@ -0,0 +1,41 @@ + + + + + + Widget Pro 3000 - Example Store + + +
+ +
+
+
+

Widget Pro 3000

+ $49.99 + 4.5 stars (128 reviews) +

The ultimate widget for professionals.

+
+ In Stock + Aisle 7, Shelf B +
+ + +
+
+ + diff --git a/ts/packages/agents/browser/test/fixtures/discovery-pages/reservation-page.html b/ts/packages/agents/browser/test/fixtures/discovery-pages/reservation-page.html new file mode 100644 index 0000000000..9a2f1199a7 --- /dev/null +++ b/ts/packages/agents/browser/test/fixtures/discovery-pages/reservation-page.html @@ -0,0 +1,47 @@ + + + + + + Chez Marie - Make a Reservation - OpenTable + + +
+ +
+
+
+

Chez Marie

+ 4.8 (342 reviews) + French Fine Dining +
+
+

Make a Reservation

+
+ +
+
+ +
+
+

Available Times

+ + + + + +
+
+
+ + diff --git a/ts/packages/agents/browser/test/fixtures/discovery-pages/restaurant-listing.html b/ts/packages/agents/browser/test/fixtures/discovery-pages/restaurant-listing.html new file mode 100644 index 0000000000..43c689cbe2 --- /dev/null +++ b/ts/packages/agents/browser/test/fixtures/discovery-pages/restaurant-listing.html @@ -0,0 +1,45 @@ + + + + + + Restaurants near you - OpenTable + + +
+ +
+
+

Restaurants near you

+
+
+ +

Chez Marie

+ 4.8 + French + $$$$ + +
+
+ +

Sushi Zen

+ 4.5 + Japanese + $$$ + +
+
+
+ + diff --git a/ts/packages/agents/browser/test/fixtures/discovery-pages/search-results.html b/ts/packages/agents/browser/test/fixtures/discovery-pages/search-results.html new file mode 100644 index 0000000000..81192f1863 --- /dev/null +++ b/ts/packages/agents/browser/test/fixtures/discovery-pages/search-results.html @@ -0,0 +1,66 @@ + + + + + + Search results for "headphones" - Example Store + + +
+ +
+
+

Search results for "headphones"

+
+ + +
+
+
+ +

Sony WH-1000XM5

+ $348.00 + 4.7 stars + + +
+
+ +

Apple AirPods Pro

+ $249.00 + 4.5 stars + + +
+
+ +

Bose QC45

+ $279.00 + 4.3 stars + + +
+
+ +
+ + diff --git a/ts/packages/agents/browser/test/fixtures/discovery-pages/shopping-cart.html b/ts/packages/agents/browser/test/fixtures/discovery-pages/shopping-cart.html new file mode 100644 index 0000000000..86ccb600de --- /dev/null +++ b/ts/packages/agents/browser/test/fixtures/discovery-pages/shopping-cart.html @@ -0,0 +1,39 @@ + + + + + + Shopping Cart - Example Store + + +
+ +
+
+

Shopping Cart

+
+
+ Widget Pro 3000 + $49.99 + Qty: 1 + +
+
+ Gadget Z100 + $29.99 + Qty: 2 + +
+
+
+ Subtotal: $109.97 + Total: $119.47 + +
+
+ + diff --git a/ts/packages/agents/browser/test/run-content-discovery.mts b/ts/packages/agents/browser/test/run-content-discovery.mts new file mode 100644 index 0000000000..c4769b8ee2 --- /dev/null +++ b/ts/packages/agents/browser/test/run-content-discovery.mts @@ -0,0 +1,488 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Standalone content-based discovery benchmark. + * Uses the production CrossContextHtmlReducer pipeline for accurate latency. + * + * Usage: + * npx tsx test/run-content-discovery.mts [--verbose] + * + * Requires .env with Azure OpenAI API keys in the ts/ root. + */ + +import fs from "fs"; +import path from "path"; +import { fileURLToPath } from "url"; +import dotenv from "dotenv"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +// Load .env +const envPath = path.resolve(__dirname, "..", "..", "..", "..", ".env"); +if (fs.existsSync(envPath)) { + dotenv.config({ path: envPath }); +} + +import { openai as ai } from "aiclient"; +import { createJsonTranslator } from "typechat"; +import { createTypeScriptJsonValidator } from "typechat/ts"; +import { createNodeHtmlReducer } from "../src/common/crossContextHtmlReducer.js"; + +const verbose = process.argv.includes("--verbose"); + +// ── Types ─────────────────────────────────────────────────────────────────── + +interface WebFlowDef { + name: string; + description: string; + parameters: Record< + string, + { type: string; required?: boolean; description?: string } + >; + scope: { type: string; domains?: string[] }; +} + +interface HtmlFragment { + frameId: string; + content: string; +} + +interface ExpectedPage { + simulatedDomain: string; + contentDiscovery?: { + shouldInclude: string[]; + shouldExclude: string[]; + }; +} + +interface KnownFailure { + page: string; + missingFlow: string; + reason: string; + filed: string; +} + +interface PageResult { + page: string; + domain: string; + rawHtmlSize: number; + reducedHtmlSize: number; + htmlReduceMs: number; + schemaGenMs: number; + llmMs: number; + totalMs: number; + candidateCount: number; + selectedCount: number; + selectedFlows: string[]; + success: boolean; + error?: string; + includePass: boolean; + excludePass: boolean; + knownFailureCount: number; + unexpectedFailureCount: number; +} + +// ── Paths ─────────────────────────────────────────────────────────────────── + +const samplesDir = path.resolve( + __dirname, + "..", + "src", + "agent", + "webFlows", + "samples", +); +const fixturesDir = path.resolve(__dirname, "fixtures", "discovery-pages"); + +// ── Helpers ───────────────────────────────────────────────────────────────── + +function loadAllFlows(): WebFlowDef[] { + return fs + .readdirSync(samplesDir) + .filter((f) => f.endsWith(".json")) + .map((f) => + JSON.parse(fs.readFileSync(path.join(samplesDir, f), "utf8")), + ); +} + +function discoverByScope(domain: string, flows: WebFlowDef[]): WebFlowDef[] { + return flows.filter((flow) => { + if (flow.scope.type === "global") return true; + if (flow.scope.type === "site" && flow.scope.domains?.length) { + return flow.scope.domains.some((d) => domain.endsWith(d)); + } + return false; + }); +} + +function generateDiscoverySchema(flows: WebFlowDef[]): string { + const typeNames: string[] = []; + const typeDefs: string[] = []; + + for (const flow of flows) { + const typeName = flow.name.charAt(0).toUpperCase() + flow.name.slice(1); + typeNames.push(typeName); + + const paramFields: string[] = []; + for (const [name, param] of Object.entries(flow.parameters)) { + const tsType = + param.type === "number" + ? "number" + : param.type === "boolean" + ? "boolean" + : "string"; + const optional = param.required ? "" : "?"; + const comment = param.description ? ` // ${param.description}` : ""; + paramFields.push( + ` ${name}${optional}: ${tsType};${comment}`, + ); + } + + const description = flow.description ? `// ${flow.description}\n` : ""; + const paramsBlock = + paramFields.length > 0 + ? ` parameters: {\n${paramFields.join("\n")}\n };` + : ""; + + typeDefs.push( + `${description}export type ${typeName} = {\n` + + ` actionName: "${flow.name}";\n` + + (paramsBlock ? `${paramsBlock}\n` : "") + + `};`, + ); + } + + const unionMembers = typeNames.join("\n | "); + const union = + typeNames.length > 0 + ? `export type CandidateActions = \n | ${unionMembers};` + : `export type CandidateActions = never;`; + + return ( + typeDefs.join("\n\n") + + "\n\n" + + union + + "\n\n" + + `export type CandidateActionList = {\n actions: CandidateActions[];\n};` + ); +} + +// ── Main ──────────────────────────────────────────────────────────────────── + +async function main() { + console.log("Content-Based Discovery Benchmark"); + console.log("==================================\n"); + + // Initialize HTML reducer (production pipeline) + console.log( + "Initializing HTML reducer (production CrossContextHtmlReducer)...", + ); + const reducer = await createNodeHtmlReducer(); + reducer.removeDivs = false; + + // Initialize LLM + console.log("Initializing LLM (GPT_4_O_MINI)..."); + const apiSettings = ai.azureApiSettingsFromEnv( + ai.ModelType.Chat, + undefined, + "GPT_4_O_MINI", + ); + const model = ai.createChatModel(apiSettings); + + // Load flows and expectations + const allFlows = loadAllFlows(); + const expectedAll: Record = JSON.parse( + fs.readFileSync(path.join(fixturesDir, "expected-flows.json"), "utf8"), + ).pages; + + const knownFailures: KnownFailure[] = JSON.parse( + fs.readFileSync(path.join(fixturesDir, "known-failures.json"), "utf8"), + ).failures; + + const knownFailureSet = new Set( + knownFailures.map((kf) => `${kf.page}:${kf.missingFlow}`), + ); + + console.log(`Loaded ${allFlows.length} sample flows`); + console.log(`Loaded ${Object.keys(expectedAll).length} test pages`); + console.log(`Loaded ${knownFailures.length} known failures\n`); + + const testPages = [ + "product-detail.html", + "search-results.html", + "shopping-cart.html", + "instacart-home.html", + "instacart-recipe.html", + "restaurant-listing.html", + "reservation-page.html", + "non-commerce.html", + ]; + + const results: PageResult[] = []; + + for (const page of testPages) { + const pageExpected = expectedAll[page]; + const domain = pageExpected.simulatedDomain; + const htmlPath = path.join(fixturesDir, page); + const rawHtml = fs.readFileSync(htmlPath, "utf8"); + + const totalStart = Date.now(); + + // Step 1: HTML reduction (production pipeline) + const reduceStart = Date.now(); + const reducedHtml = reducer.reduce(rawHtml); + const htmlReduceMs = Date.now() - reduceStart; + + const fragments: HtmlFragment[] = [ + { frameId: "main", content: reducedHtml }, + ]; + + // Step 2: Scope-based filtering + const scopedFlows = discoverByScope(domain, allFlows); + + // Step 3: Schema generation + const schemaStart = Date.now(); + const schema = generateDiscoverySchema(scopedFlows); + const schemaGenMs = Date.now() - schemaStart; + + // Step 4: LLM content analysis + const validator = createTypeScriptJsonValidator( + schema, + "CandidateActionList", + ); + const translator = createJsonTranslator(model, validator); + + const htmlText = fragments.map((f) => f.content).join("\n"); + const prompt = `You are given a list of known user actions. Examine the page layout and content, then determine which of these actions can actually be performed on THIS page. Only include actions that the page supports. If none of the known actions apply, return an empty actions array.\n\nPage HTML:\n${htmlText.substring(0, 30000)}\n\nReturn a SINGLE "CandidateActionList" response using the typescript schema:\n\`\`\`\n${schema}\n\`\`\``; + + const llmStart = Date.now(); + const response = await translator.translate(prompt); + const llmMs = Date.now() - llmStart; + const totalMs = Date.now() - totalStart; + + const result: PageResult = { + page, + domain, + rawHtmlSize: rawHtml.length, + reducedHtmlSize: reducedHtml.length, + htmlReduceMs, + schemaGenMs, + llmMs, + totalMs, + candidateCount: scopedFlows.length, + selectedCount: 0, + selectedFlows: [], + success: response.success, + includePass: true, + excludePass: true, + knownFailureCount: 0, + unexpectedFailureCount: 0, + }; + + if (response.success) { + const selected = response.data as { + actions: { actionName: string }[]; + }; + result.selectedFlows = [ + ...new Set(selected.actions.map((a) => a.actionName)), + ]; + result.selectedCount = result.selectedFlows.length; + + // Validate against expectations + if (pageExpected.contentDiscovery) { + for (const flowName of pageExpected.contentDiscovery + .shouldInclude) { + if (!result.selectedFlows.includes(flowName)) { + const isKnown = knownFailureSet.has( + `${page}:${flowName}`, + ); + if (isKnown) { + result.knownFailureCount++; + if (verbose) { + console.log( + ` KNOWN: ${page} missing expected flow: ${flowName}`, + ); + } + } else { + result.includePass = false; + result.unexpectedFailureCount++; + if (verbose) { + console.log( + ` FAIL: ${page} missing expected flow: ${flowName}`, + ); + } + } + } + } + for (const flowName of pageExpected.contentDiscovery + .shouldExclude) { + if (result.selectedFlows.includes(flowName)) { + result.excludePass = false; + result.unexpectedFailureCount++; + if (verbose) { + console.log( + ` FAIL: ${page} has unexpected flow: ${flowName}`, + ); + } + } + } + } + } else { + result.error = (response as any).message; + } + + const hasOnlyKnown = + result.success && + result.includePass && + result.excludePass && + result.knownFailureCount > 0; + const passIcon = + result.success && result.includePass && result.excludePass + ? hasOnlyKnown + ? "KNOWN" + : "PASS" + : "FAIL"; + console.log( + ` [${passIcon}] ${page} (${domain}): ${result.candidateCount} scoped -> ${result.selectedCount} selected, reduce: ${htmlReduceMs}ms (${rawHtml.length} -> ${reducedHtml.length} chars), LLM: ${llmMs}ms, total: ${totalMs}ms`, + ); + + results.push(result); + } + + // ── Report ────────────────────────────────────────────────────────────── + + console.log("\n=== Content-Based Discovery Latency Report ===\n"); + console.log( + "Page".padEnd(28) + + "Domain".padEnd(24) + + "Raw".padEnd(7) + + "Reduced".padEnd(9) + + "Scoped".padEnd(8) + + "Selected".padEnd(10) + + "Reduce".padEnd(10) + + "LLM".padEnd(10) + + "Total".padEnd(10) + + "Status".padEnd(8) + + "Selected Flows", + ); + console.log("-".repeat(160)); + + for (const r of results) { + const status = + r.success && r.includePass && r.excludePass ? "PASS" : "FAIL"; + console.log( + r.page.padEnd(28) + + r.domain.padEnd(24) + + String(r.rawHtmlSize).padEnd(7) + + String(r.reducedHtmlSize).padEnd(9) + + String(r.candidateCount).padEnd(8) + + String(r.selectedCount).padEnd(10) + + `${r.htmlReduceMs}ms`.padEnd(10) + + `${r.llmMs}ms`.padEnd(10) + + `${r.totalMs}ms`.padEnd(10) + + status.padEnd(8) + + r.selectedFlows.join(", "), + ); + } + + console.log("-".repeat(160)); + + const successful = results.filter((r) => r.success); + const passed = results.filter( + (r) => r.success && r.includePass && r.excludePass, + ); + const knownOnly = results.filter( + (r) => + r.success && + r.includePass && + r.excludePass && + r.knownFailureCount > 0, + ); + const unexpected = results.filter( + (r) => r.success && r.unexpectedFailureCount > 0, + ); + + if (successful.length > 0) { + const avg = (arr: number[]) => + Math.round(arr.reduce((a, b) => a + b, 0) / arr.length); + const min = (arr: number[]) => Math.min(...arr); + const max = (arr: number[]) => Math.max(...arr); + + const reduceTimes = successful.map((r) => r.htmlReduceMs); + const llmTimes = successful.map((r) => r.llmMs); + const totalTimes = successful.map((r) => r.totalMs); + const reductions = successful.map((r) => + Math.round((1 - r.reducedHtmlSize / r.rawHtmlSize) * 100), + ); + + console.log(`\nSummary (${successful.length} pages):`); + console.log( + ` HTML reduce: avg ${avg(reduceTimes)}ms, min ${min(reduceTimes)}ms, max ${max(reduceTimes)}ms`, + ); + console.log(` HTML reduction: avg ${avg(reductions)}% size reduction`); + console.log( + ` LLM latency: avg ${avg(llmTimes)}ms, min ${min(llmTimes)}ms, max ${max(llmTimes)}ms`, + ); + console.log(` Total latency: avg ${avg(totalTimes)}ms`); + console.log( + ` Pass rate: ${passed.length}/${results.length} (${Math.round((passed.length / results.length) * 100)}%)`, + ); + if (knownOnly.length > 0) { + console.log( + ` Known failures: ${knownOnly.length} pages (${results.reduce((s, r) => s + r.knownFailureCount, 0)} flows tracked)`, + ); + } + if (unexpected.length > 0) { + console.log( + ` NEW failures: ${unexpected.length} pages (${results.reduce((s, r) => s + r.unexpectedFailureCount, 0)} flows) ← investigate these`, + ); + } + } + + const failed = results.filter((r) => !r.success); + if (failed.length > 0) { + console.log(`\nLLM failures:`); + for (const r of failed) { + console.log(` ${r.page}: ${r.error}`); + } + } + + const expectationFailures = results.filter( + (r) => r.success && (!r.includePass || !r.excludePass), + ); + if (expectationFailures.length > 0) { + console.log(`\nExpectation failures:`); + for (const r of expectationFailures) { + if (!r.includePass) { + const expected = + expectedAll[r.page]?.contentDiscovery?.shouldInclude ?? []; + const missing = expected.filter( + (f) => !r.selectedFlows.includes(f), + ); + console.log( + ` ${r.page}: missing expected flows: ${missing.join(", ")}`, + ); + } + if (!r.excludePass) { + const excluded = + expectedAll[r.page]?.contentDiscovery?.shouldExclude ?? []; + const unexpected = excluded.filter((f) => + r.selectedFlows.includes(f), + ); + console.log( + ` ${r.page}: unexpected flows present: ${unexpected.join(", ")}`, + ); + } + } + } + + // Exit with failure only for unexpected failures (known failures don't block) + const exitCode = unexpected.length === 0 && failed.length === 0 ? 0 : 1; + process.exit(exitCode); +} + +main().catch((err) => { + console.error("Benchmark failed:", err); + process.exit(2); +}); diff --git a/ts/packages/agents/browser/test/scopeDiscovery.test.ts b/ts/packages/agents/browser/test/scopeDiscovery.test.ts new file mode 100644 index 0000000000..4103716cb1 --- /dev/null +++ b/ts/packages/agents/browser/test/scopeDiscovery.test.ts @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import * as fs from "fs"; +import * as path from "path"; + +interface FlowScope { + type: "site" | "global"; + domains?: string[]; + urlPatterns?: string[]; +} + +interface SampleFlow { + name: string; + scope: FlowScope; +} + +interface ExpectedPage { + simulatedDomain: string; + scopeDiscovery: { + shouldInclude: string[]; + shouldExclude: string[]; + }; +} + +interface ExpectedFlows { + pages: Record; +} + +function loadSampleFlows(): SampleFlow[] { + const samplesDir = path.resolve( + __dirname, + "..", + "src", + "agent", + "webFlows", + "samples", + ); + const files = fs.readdirSync(samplesDir).filter((f) => f.endsWith(".json")); + return files.map((f) => { + const content = JSON.parse( + fs.readFileSync(path.join(samplesDir, f), "utf8"), + ); + return { name: content.name, scope: content.scope }; + }); +} + +function discoverByScope(domain: string, flows: SampleFlow[]): string[] { + return flows + .filter((flow) => { + if (flow.scope.type === "global") return true; + if (flow.scope.type === "site" && flow.scope.domains?.length) { + return flow.scope.domains.some((d) => domain.endsWith(d)); + } + return false; + }) + .map((f) => f.name); +} + +describe("scope-based discovery", () => { + let flows: SampleFlow[]; + let expected: ExpectedFlows; + + beforeAll(() => { + flows = loadSampleFlows(); + const expectedPath = path.resolve( + __dirname, + "fixtures", + "discovery-pages", + "expected-flows.json", + ); + expected = JSON.parse(fs.readFileSync(expectedPath, "utf8")); + }); + + it("loads all sample flows", () => { + expect(flows.length).toBeGreaterThanOrEqual(21); + }); + + for (const [pageName, pageExpected] of Object.entries( + JSON.parse( + fs.readFileSync( + path.resolve( + __dirname, + "fixtures", + "discovery-pages", + "expected-flows.json", + ), + "utf8", + ), + ).pages as Record, + )) { + describe(`${pageName} (${pageExpected.simulatedDomain})`, () => { + it("includes expected flows", () => { + const samplesDir = path.resolve( + __dirname, + "..", + "src", + "agent", + "webFlows", + "samples", + ); + const files = fs + .readdirSync(samplesDir) + .filter((f) => f.endsWith(".json")); + const allFlows: SampleFlow[] = files.map((f) => { + const content = JSON.parse( + fs.readFileSync(path.join(samplesDir, f), "utf8"), + ); + return { name: content.name, scope: content.scope }; + }); + + const discovered = discoverByScope( + pageExpected.simulatedDomain, + allFlows, + ); + + for (const flowName of pageExpected.scopeDiscovery + .shouldInclude) { + expect(discovered).toContain(flowName); + } + }); + + it("excludes expected flows", () => { + const samplesDir = path.resolve( + __dirname, + "..", + "src", + "agent", + "webFlows", + "samples", + ); + const files = fs + .readdirSync(samplesDir) + .filter((f) => f.endsWith(".json")); + const allFlows: SampleFlow[] = files.map((f) => { + const content = JSON.parse( + fs.readFileSync(path.join(samplesDir, f), "utf8"), + ); + return { name: content.name, scope: content.scope }; + }); + + const discovered = discoverByScope( + pageExpected.simulatedDomain, + allFlows, + ); + + for (const flowName of pageExpected.scopeDiscovery + .shouldExclude) { + expect(discovered).not.toContain(flowName); + } + }); + }); + } +});