Enable action discovery when user natigates to a page

ts/packages/agents/browser/src/agent/agentServiceHandlers.mts

@@ -116,6 +116,8 @@ export function createAgentInvokeHandlers(
             websiteHandler("getWebsiteStats", params),
 
         // Macros
+        autoDiscoverActions: (params: any) =>
+            discoveryHandler("autoDiscoverActions", params),
         detectPageActions: (params: any) =>
             discoveryHandler("detectPageActions", params),
         registerPageDynamicAgent: (params: any) =>

ts/packages/agents/browser/src/agent/discovery/actionHandler.mts

@@ -82,6 +82,110 @@ interface DiscoveryActionHandlerContext {
     sessionContext: SessionContext<BrowserActionContext>;
 }
 
+// ── Discovery cache ─────────────────────────────────────────────────────────
+//
+// Shared by auto-discovery (on navigation) and manual discovery (context menu).
+// Two-tier lookup with a TTL:
+//
+//   1. Exact URL match (O(1) map lookup)
+//   2. Pattern match — the LLM can return a urlPattern regex indicating that
+//      pages with similar URLs share the same available actions. For example,
+//      all Amazon product pages (https://www.amazon.com/dp/...) have the same
+//      action set regardless of product ID. On a cache miss for the exact URL,
+//      we iterate pattern entries and test each regex against the current URL.
+//
+// When auto-discovery runs on navigation, it populates the cache. When the
+// user triggers "Discover page macros", the cache is checked first to avoid
+// a redundant LLM call.
+
+interface DiscoveryCacheEntry {
+    flows: WebFlowDefinition[];
+    mode: "scope" | "content" | "scope-fallback";
+    timestamp: number;
+    urlPattern?: string | undefined; // Regex matching URLs with the same available actions
+}
+
+const DISCOVERY_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
+const discoveryCache = new Map<string, DiscoveryCacheEntry>();
+
+// Separate index for pattern-based entries, keyed by the original URL that
+// produced the pattern. Patterns are tested in insertion order.
+const patternEntries: DiscoveryCacheEntry[] = [];
+
+function isExpired(entry: DiscoveryCacheEntry): boolean {
+    return Date.now() - entry.timestamp > DISCOVERY_CACHE_TTL_MS;
+}
+
+function testUrlPattern(pattern: string, url: string): boolean {
+    try {
+        return new RegExp(pattern).test(url);
+    } catch {
+        debug(`Invalid urlPattern regex: ${pattern}`);
+        return false;
+    }
+}
+
+function getCachedDiscovery(url: string): DiscoveryCacheEntry | undefined {
+    // 1. Exact URL match
+    const exact = discoveryCache.get(url);
+    if (exact) {
+        if (isExpired(exact)) {
+            discoveryCache.delete(url);
+        } else {
+            return exact;
+        }
+    }
+
+    // 2. Pattern match — check entries that have a urlPattern regex
+    for (let i = patternEntries.length - 1; i >= 0; i--) {
+        const entry = patternEntries[i];
+        if (isExpired(entry)) {
+            patternEntries.splice(i, 1);
+            continue;
+        }
+        if (entry.urlPattern && testUrlPattern(entry.urlPattern, url)) {
+            debug(
+                `Discovery cache pattern hit: ${entry.urlPattern} matched ${url}`,
+            );
+            return entry;
+        }
+    }
+
+    return undefined;
+}
+
+function setCachedDiscovery(
+    url: string,
+    flows: WebFlowDefinition[],
+    mode: "scope" | "content" | "scope-fallback",
+    urlPattern?: string,
+): void {
+    const entry: DiscoveryCacheEntry = {
+        flows,
+        mode,
+        timestamp: Date.now(),
+        urlPattern,
+    };
+
+    discoveryCache.set(url, entry);
+
+    if (urlPattern) {
+        patternEntries.push(entry);
+    }
+
+    // Evict expired entries periodically
+    if (discoveryCache.size > 100) {
+        const now = Date.now();
+        for (const [key, e] of discoveryCache) {
+            if (now - e.timestamp > DISCOVERY_CACHE_TTL_MS) {
+                discoveryCache.delete(key);
+            }
+        }
+    }
+}
+
+// ── Discovery handlers ──────────────────────────────────────────────────────
+
 async function handleFindUserActions(
     action: any,
     ctx: DiscoveryActionHandlerContext,
@@ -90,6 +194,21 @@ async function handleFindUserActions(
         ctx.sessionContext.agentContext,
     ).getPageUrl();
 
+    // Check cache first — auto-discovery may have already computed this
+    if (url) {
+        const cached = getCachedDiscovery(url);
+        if (cached) {
+            debug(
+                `Discovery cache hit for ${url}: ${cached.flows.length} flows (mode: ${cached.mode})`,
+            );
+            return {
+                displayText: `Found ${cached.flows.length} applicable actions (cached)`,
+                entities: ctx.entities.getEntities(),
+                data: { actions: cached.flows, cached: true },
+            };
+        }
+    }
+
     // Get existing WebFlows for this domain
     const webFlowStore = ctx.sessionContext.agentContext.webFlowStore;
     let candidateFlows: WebFlowDefinition[] = [];
@@ -145,15 +264,31 @@ async function handleFindUserActions(
     }
 
     // The LLM returns a CandidateActionList with the subset of flows
-    // that actually apply to this page.
-    const selected = response.data as { actions: { actionName: string }[] };
+    // that actually apply to this page, plus an optional urlPattern.
+    const selected = response.data as {
+        actions: { actionName: string }[];
+        urlPattern?: string;
+    };
     const selectedNames = new Set(selected.actions.map((a) => a.actionName));
 
     // Filter the full WebFlowDefinitions to only those the LLM selected
     const applicableFlows = candidateFlows.filter((f) =>
         selectedNames.has(f.name),
     );
 
+    // Cache the result for this URL (with optional pattern for similar URLs)
+    if (url) {
+        setCachedDiscovery(
+            url,
+            applicableFlows,
+            "content",
+            selected.urlPattern,
+        );
+        if (selected.urlPattern) {
+            debug(`Discovery urlPattern: ${selected.urlPattern}`);
+        }
+    }
+
     debug(
         `Discovery: ${candidateFlows.length} candidates → ${applicableFlows.length} applicable to page`,
     );
@@ -165,7 +300,127 @@ async function handleFindUserActions(
     };
 }
 
-function generateDiscoverySchema(flows: WebFlowDefinition[]): string {
+async function handleAutoDiscoverActions(
+    action: any,
+    ctx: DiscoveryActionHandlerContext,
+): Promise<DiscoveryActionResult> {
+    const url = action.parameters?.url;
+    const domain = action.parameters?.domain;
+    const mode = action.parameters?.mode ?? "content";
+
+    const webFlowStore = ctx.sessionContext.agentContext.webFlowStore;
+    if (!webFlowStore || !domain) {
+        return {
+            displayText: "No web flow store available",
+            entities: [],
+            data: { actions: [], flowCount: 0 },
+        };
+    }
+
+    // Check cache first
+    if (url) {
+        const cached = getCachedDiscovery(url);
+        if (cached) {
+            debug(
+                `Auto-discovery cache hit for ${url}: ${cached.flows.length} flows`,
+            );
+            return {
+                displayText: `Found ${cached.flows.length} actions for ${domain} (cached)`,
+                entities: [],
+                data: {
+                    actions: cached.flows,
+                    flowCount: cached.flows.length,
+                    mode: cached.mode,
+                    cached: true,
+                },
+            };
+        }
+    }
+
+    // Layer 1: Scope-based discovery — fast domain matching
+    const scopedFlows = await webFlowStore.listForDomainWithDetails(domain);
+
+    if (mode === "scope" || scopedFlows.length === 0) {
+        debug(
+            `Auto-discovery (scope): ${scopedFlows.length} flows for ${domain}`,
+        );
+        if (url) {
+            setCachedDiscovery(url, scopedFlows, "scope");
+        }
+        return {
+            displayText: `Found ${scopedFlows.length} actions for ${domain}`,
+            entities: [],
+            data: {
+                actions: scopedFlows,
+                flowCount: scopedFlows.length,
+                mode: "scope",
+            },
+        };
+    }
+
+    // Layer 2: Content-based discovery — LLM filters to applicable flows
+    const discoverySchema = generateDiscoverySchema(scopedFlows);
+    const htmlFragments = await ctx.browser.getHtmlFragments();
+
+    const response = await ctx.agent.getCandidateUserActions(
+        discoverySchema,
+        htmlFragments,
+    );
+
+    if (!response.success) {
+        debug("Auto-discovery LLM call failed, falling back to scope results");
+        if (url) {
+            setCachedDiscovery(url, scopedFlows, "scope-fallback");
+        }
+        return {
+            displayText: `Found ${scopedFlows.length} actions for ${domain}`,
+            entities: [],
+            data: {
+                actions: scopedFlows,
+                flowCount: scopedFlows.length,
+                mode: "scope-fallback",
+            },
+        };
+    }
+
+    const selected = response.data as {
+        actions: { actionName: string }[];
+        urlPattern?: string;
+    };
+    const selectedNames = new Set(selected.actions.map((a) => a.actionName));
+    const applicableFlows = scopedFlows.filter((f) =>
+        selectedNames.has(f.name),
+    );
+
+    // Cache the content-based result (with optional pattern for similar URLs)
+    if (url) {
+        setCachedDiscovery(
+            url,
+            applicableFlows,
+            "content",
+            selected.urlPattern,
+        );
+        if (selected.urlPattern) {
+            debug(`Auto-discovery urlPattern: ${selected.urlPattern}`);
+        }
+    }
+
+    debug(
+        `Auto-discovery (content): ${scopedFlows.length} scoped → ${applicableFlows.length} applicable for ${domain}`,
+    );
+
+    return {
+        displayText: `Found ${applicableFlows.length} applicable actions for ${domain}`,
+        entities: [],
+        data: {
+            actions: applicableFlows,
+            flowCount: applicableFlows.length,
+            mode: "content",
+        },
+    };
+}
+
+export function generateDiscoverySchema(flows: WebFlowDefinition[]): string {
     const typeNames: string[] = [];
     const typeDefs: string[] = [];
 
@@ -213,7 +468,15 @@ function generateDiscoverySchema(flows: WebFlowDefinition[]): string {
         "\n\n" +
         union +
         "\n\n" +
-        `export type CandidateActionList = {\n    actions: CandidateActions[];\n};`
+        `export type CandidateActionList = {\n` +
+        `    actions: CandidateActions[];\n` +
+        `    // If other pages on this site with different URLs would have the same\n` +
+        `    // set of available actions (e.g. all product detail pages), provide a\n` +
+        `    // regex that matches those URLs. This avoids re-analyzing similar pages.\n` +
+        `    // Example: "https://www\\\\.amazon\\\\.com/dp/[A-Z0-9]+" for Amazon product pages.\n` +
+        `    // Omit if the actions are specific to this exact URL.\n` +
+        `    urlPattern?: string;\n` +
+        `};`
     );
 }
 
@@ -515,6 +778,19 @@ export async function handleSchemaDiscoveryAction(
 
     let result: DiscoveryActionResult;
 
+    // autoDiscoverActions is not part of SchemaDiscoveryActions — handle before switch
+    if ((action as any).actionName === "autoDiscoverActions") {
+        const adResult = await handleAutoDiscoverActions(
+            action,
+            discoveryContext,
+        );
+        return {
+            displayText: adResult.displayText,
+            data: adResult.data,
+            entities: adResult.entities,
+        };
+    }
+
     switch (action.actionName) {
         case "detectPageActions":
             result = await handleFindUserActions(action, discoveryContext);

ts/packages/agents/browser/src/agent/webFlows/samples/buyAllInList.json

@@ -0,0 +1,28 @@
+{
+  "name": "buyAllInList",
+  "description": "Navigate to a shopping list and add all available items to the cart",
+  "version": 1,
+  "parameters": {
+    "listName": {
+      "type": "string",
+      "required": true,
+      "description": "Name of the shopping list to buy from"
+    },
+    "storeName": {
+      "type": "string",
+      "required": false,
+      "description": "Store to shop from (uses current store if not specified)"
+    }
+  },
+  "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise<WebFlowResult> {\n    await browser.navigateTo('https://www.instacart.com/');\n    await browser.awaitPageLoad();\n\n    if (params.storeName) {\n        const store = await browser.extractComponent<{ name: string; linkSelector?: string }>({\n            typeName: 'StoreInfo',\n            schema: '{ name: string; subtitle?: string; linkSelector?: string; }'\n        }, params.storeName);\n        if (store && store.linkSelector) {\n            await browser.clickAndWait(store.linkSelector);\n        }\n    }\n\n    const listsNav = await browser.extractComponent<{ linkSelector: string }>({\n        typeName: 'ListsNavigationLink',\n        schema: '{ linkSelector: string; }'\n    });\n    if (!listsNav || !listsNav.linkSelector) {\n        throw new Error('Could not find lists navigation link');\n    }\n    await browser.clickAndWait(listsNav.linkSelector);\n\n    const list = await browser.extractComponent<{ name: string; detailsLinkSelector: string }>({\n        typeName: 'ListInfo',\n        schema: '{ name: string; detailsLinkSelector: string; }'\n    }, params.listName);\n    if (!list || !list.detailsLinkSelector) {\n        throw new Error('Could not find list: ' + params.listName);\n    }\n    await browser.clickAndWait(list.detailsLinkSelector);\n\n    const details = await browser.extractComponent<{ name: string; products?: { name: string; price: string; availability?: string; addToCartButtonSelector?: string }[] }>({\n        typeName: 'ListDetailsInfo',\n        schema: '{ name: string; storeName?: string; products?: { name: string; price: string; availability?: string; addToCartButtonSelector?: string; }[]; }'\n    });\n    if (!details || !details.products || details.products.length === 0) {\n        return { success: false, error: 'No products found in list ' + params.listName };\n    }\n\n    let added = 0;\n    const unavailable: string[] = [];\n    for (const product of details.products) {\n        if (product.availability === 'Out of stock') {\n            unavailable.push(product.name);\n        } else if (product.addToCartButtonSelector) {\n            await browser.click(product.addToCartButtonSelector);\n            await browser.awaitPageInteraction();\n            added++;\n        }\n    }\n\n    let message = 'Added ' + added + ' items from \"' + params.listName + '\" to cart';\n    if (unavailable.length > 0) {\n        message += '. Unavailable: ' + unavailable.join(', ');\n    }\n    return { success: true, message };\n}",
+  "grammarPatterns": [
+    "buy (all)? (items)? (in | from) (my)? $(listName:wildcard) list",
+    "add (all)? (items)? from $(listName:wildcard) list to cart",
+    "shop (my)? $(listName:wildcard) list"
+  ],
+  "scope": {
+    "type": "site",
+    "domains": ["instacart.com"]
+  },
+  "source": { "type": "manual", "timestamp": "2026-04-03T00:00:00.000Z" }
+}

ts/packages/agents/browser/src/agent/webFlows/samples/buyAllInRecipe.json

@@ -0,0 +1,23 @@
+{
+  "name": "buyAllInRecipe",
+  "description": "Find a recipe on Instacart and add all its ingredients to the cart",
+  "version": 1,
+  "parameters": {
+    "recipeName": {
+      "type": "string",
+      "required": true,
+      "description": "Name of the recipe to buy ingredients for"
+    }
+  },
+  "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise<WebFlowResult> {\n    await browser.navigateTo('https://www.instacart.com/');\n    await browser.awaitPageLoad();\n\n    const searchInput = await browser.extractComponent<{ cssSelector: string; submitButtonCssSelector: string }>({\n        typeName: 'SearchInput',\n        schema: '{ cssSelector: string; submitButtonCssSelector: string; }'\n    });\n    if (!searchInput || !searchInput.cssSelector) {\n        throw new Error('Could not find search input');\n    }\n    await browser.clearAndType(searchInput.cssSelector, 'recipes: ' + params.recipeName);\n    if (searchInput.submitButtonCssSelector) {\n        await browser.clickAndWait(searchInput.submitButtonCssSelector);\n    } else {\n        await browser.pressKey('Enter');\n        await browser.awaitPageLoad();\n    }\n\n    const recipe = await browser.extractComponent<{ name: string; detailsLinkSelector: string }>({\n        typeName: 'RecipeInfo',\n        schema: '{ name: string; subtitle: string; detailsLinkSelector: string; }'\n    }, params.recipeName);\n    if (!recipe || !recipe.detailsLinkSelector) {\n        throw new Error('Could not find recipe: ' + params.recipeName);\n    }\n    await browser.clickAndWait(recipe.detailsLinkSelector);\n\n    const hero = await browser.extractComponent<{ recipeName: string; addAllIngredientsSelector: string; ingredients: { name: string; price: string; addToCartButtonSelector?: string }[] }>({\n        typeName: 'RecipeHeroSection',\n        schema: '{ recipeName: string; summary: string; addAllIngredientsSelector: string; saveButtonSelector: string; ingredients: { name: string; price: string; addToCartButtonSelector?: string; }[]; }'\n    });\n    if (!hero || !hero.addAllIngredientsSelector) {\n        throw new Error('Could not find recipe details for: ' + params.recipeName);\n    }\n    await browser.clickAndWait(hero.addAllIngredientsSelector);\n    const count = hero.ingredients ? hero.ingredients.length : 0;\n    return { success: true, message: 'Added ' + count + ' ingredients from \"' + hero.recipeName + '\" to cart' };\n}",
+  "grammarPatterns": [
+    "buy (all)? ingredients for $(recipeName:wildcard) recipe",
+    "add $(recipeName:wildcard) recipe ingredients to cart",
+    "shop for $(recipeName:wildcard) recipe"
+  ],
+  "scope": {
+    "type": "site",
+    "domains": ["instacart.com"]
+  },
+  "source": { "type": "manual", "timestamp": "2026-04-03T00:00:00.000Z" }
+}