Skip to content

Commit

Permalink
Enable recipe and list actions (#522)
Browse files Browse the repository at this point in the history 
Co-authored-by: robgruen <[email protected]>
  • Loading branch information
@hillary-mutisya @robgruen
hillary-mutisya and robgruen authored Jan 7, 2025
1 parent 99cc0c3 commit 62a28c8
Show file tree
Hide file tree
Showing 10 changed files with 301 additions and 46 deletions.
12 changes: 11 additions & 1 deletion ts/packages/agents/browser/src/agent/browserConnector.mts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,15 @@ export class BrowserConnector {
});
}

async getHtmlFragments() {
async getHtmlFragments(useTimestampIds?: boolean) {
const timeoutPromise = new Promise((f) => setTimeout(f, 120000));
const htmlAction = {
actionName: "getHTML",
parameters: {
fullHTML: false,
downloadAsFile: false,
extractText: false,
useTimestampIds: useTimestampIds,
},
};

Expand Down Expand Up @@ -224,4 +225,13 @@ export class BrowserConnector {
return actionPromise;
}
}

async awaitPageInteraction(timeout?: number) {
if (!timeout) {
timeout = 400;
}

const timeoutPromise = new Promise((f) => setTimeout(f, timeout));
return timeoutPromise;
}
}
45 changes: 28 additions & 17 deletions ts/packages/agents/browser/src/agent/commerce/translator.mts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ export async function createCommercePageTranslator(
model: "GPT_35_TURBO" | "GPT_4" | "GPT_v" | "GPT_4_O" | "GPT_4_O_MINI",
) {
const packageRoot = path.join("..", "..", "..");
const pageSchema = await fs.promises.readFile(
const actionSchema = await fs.promises.readFile(
fileURLToPath(
new URL(
path.join(packageRoot, "./src/agent/commerce/schema/userActions.mts"),
Expand All @@ -108,8 +108,22 @@ export async function createCommercePageTranslator(
"utf8",
);

const pageSchema = await fs.promises.readFile(
fileURLToPath(
new URL(
path.join(
packageRoot,
"./src/agent/commerce/schema/pageComponents.mts",
),
import.meta.url,
),
),
"utf8",
);

const agent = new ECommerceSiteAgent<ShoppingActions>(
pageSchema,
actionSchema,
"ShoppingActions",
model,
);
Expand All @@ -118,12 +132,19 @@ export async function createCommercePageTranslator(

export class ECommerceSiteAgent<T extends object> {
schema: string;
pageComponentsSchema: string;

model: TypeChatLanguageModel;
translator: TypeChatJsonTranslator<T>;

constructor(schema: string, schemaName: string, fastModelName: string) {
this.schema = schema;
constructor(
pageComponentsSchema: string,
actionSchema: string,
schemaName: string,
fastModelName: string,
) {
this.pageComponentsSchema = pageComponentsSchema;
this.schema = actionSchema;

const apiSettings = ai.azureApiSettingsFromEnv(
ai.ModelType.Chat,
Expand Down Expand Up @@ -167,6 +188,7 @@ export class ECommerceSiteAgent<T extends object> {
type: "text",
text: `
Use the layout information provided and the user request below to generate a SINGLE "${translator.validator.getTypeName()}" response using the typescript schema below.
For schemas that include CSS selectors, construct the selector based on the element's Id attribute if the id is present.
You should stop searching and return current result as soon as you find a result that matches the user's criteria:
'''
Expand All @@ -185,16 +207,8 @@ export class ECommerceSiteAgent<T extends object> {
return promptSections;
}

private getBootstrapTranslator(fileName: string, targetType: string) {
const packageRoot = path.join("..", "..", "..");
const schemaPath = fileURLToPath(
new URL(
path.join(packageRoot, `./src/agent/commerce/schema/${fileName}`),
import.meta.url,
),
);

const pageSchema = fs.readFileSync(schemaPath, "utf8");
private getBootstrapTranslator(targetType: string) {
const pageSchema = this.pageComponentsSchema;

const validator = createTypeScriptJsonValidator(pageSchema, targetType);
const bootstrapTranslator = createJsonTranslator(this.model, validator);
Expand All @@ -213,10 +227,7 @@ export class ECommerceSiteAgent<T extends object> {
fragments?: HtmlFragments[],
screenshot?: string,
) {
const bootstrapTranslator = this.getBootstrapTranslator(
"pageComponents.mts",
componentTypeName,
);
const bootstrapTranslator = this.getBootstrapTranslator(componentTypeName);

const promptSections = this.getCssSelectorForElementPrompt(
bootstrapTranslator,
Expand Down
160 changes: 150 additions & 10 deletions ts/packages/agents/browser/src/agent/instacart/actionHandler.mts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,16 @@ import { BrowserConnector } from "../browserConnector.mjs";
import { createInstacartPageTranslator } from "./translator.mjs";
import {
AllListsInfo,
AllRecipeSearchResults,
BuyItAgainHeaderSection,
BuyItAgainNavigationLink,
HomeLink,
ListDetailsInfo,
ListInfo,
ListsNavigationLink,
ProductDetailsHeroTile,
ProductTile,
RecipeHeroSection,
SearchInput,
StoreInfo,
} from "./schema/pageComponents.mjs";
Expand Down Expand Up @@ -42,16 +49,28 @@ export async function handleInstacartAction(
case "findNearbyStoreAction":
await handleFindStores(action);
break;
case "searchForRecipeAction":
await handleFindRecipe(action);
break;
case "buyAllInRecipeAction":
await handleBuyRecipeIngredients(action);
break;
case "buyAllInListAction":
await handleBuyListContents(action);
break;
case "setPreferredStoreAction":
await handleSetPreferredStore(action);
break;
case "buyItAgainAction":
await handleBuyItAgain(action);
break;
}

async function getComponentFromPage(
componentType: string,
selectionCondition?: string,
) {
const htmlFragments = await browser.getHtmlFragments();
const htmlFragments = await browser.getHtmlFragments(true);
const timerName = `getting ${componentType} section`;

console.time(timerName);
Expand Down Expand Up @@ -79,7 +98,7 @@ export async function handleInstacartAction(
await browser.clickOn(searchSelector);
await browser.enterTextIn(productName, searchSelector);
await browser.clickOn(selector.submitButtonCssSelector);
await new Promise((r) => setTimeout(r, 400));
await browser.awaitPageInteraction();
await browser.awaitPageLoad();
}

Expand All @@ -91,7 +110,7 @@ export async function handleInstacartAction(
)) as ProductTile;

await browser.clickOn(targetProduct.detailsLinkSelector);
await new Promise((r) => setTimeout(r, 200));
await browser.awaitPageInteraction();
await browser.awaitPageLoad();
}

Expand Down Expand Up @@ -130,43 +149,44 @@ export async function handleInstacartAction(

async function goToHomepage() {
const link = (await getComponentFromPage("HomeLink")) as HomeLink;
console.log(link);

if (link.linkCssSelector) {
await browser.clickOn(link.linkCssSelector);
await browser.awaitPageInteraction();
await browser.awaitPageLoad(5000);
}
}

async function handleFindStores(action: any) {
await goToHomepage();

const stores = (await getComponentFromPage("StoreInfo")) as StoreInfo[];

console.log(stores);

return stores;
}

async function searchForStore(storeName: string) {
await goToHomepage();
const selector = (await getComponentFromPage("SearchInput")) as SearchInput;
const searchSelector = selector.cssSelector;

await browser.clickOn(searchSelector);
await browser.enterTextIn("store: " + storeName, searchSelector);
await browser.clickOn(selector.submitButtonCssSelector);
await new Promise((r) => setTimeout(r, 400));
await browser.awaitPageInteraction();
await browser.awaitPageLoad();
}

async function selectStoreSearchResult(storeName: string) {
const request = `Search result: ${storeName}`;
const request = `${storeName}`;
const targetStore = (await getComponentFromPage(
"StoreInfo",
request,
)) as StoreInfo;

await browser.clickOn(targetStore.storeLinkCssSelector);
await new Promise((r) => setTimeout(r, 200));
await browser.awaitPageLoad();
await browser.awaitPageInteraction();
await browser.awaitPageLoad(5000);
}

async function handleSetPreferredStore(action: any) {
Expand All @@ -176,5 +196,125 @@ export async function handleInstacartAction(
// TODO: persist preferrences
}

async function searchForRecipe(recipeKeywords: string) {
// await goToHomepage();
const selector = (await getComponentFromPage("SearchInput")) as SearchInput;
const searchSelector = selector.cssSelector;

await browser.clickOn(searchSelector);
await browser.enterTextIn("recipe: " + recipeKeywords, searchSelector);
await browser.clickOn(selector.submitButtonCssSelector);
await browser.awaitPageInteraction();
await browser.awaitPageLoad(5000);
}

async function selectRecipeSearchResult(recipeKeywords: string) {
const request = `${recipeKeywords}`;
const allRecipes = (await getComponentFromPage(
"AllRecipeSearchResults",
request,
)) as AllRecipeSearchResults;

console.log(allRecipes);
if (allRecipes.recipes.length > 0) {
await browser.clickOn(allRecipes.recipes[0].recipeLinkCssSelector);
await browser.awaitPageInteraction();
console.log(
"Clicked on search result: " +
allRecipes.recipes[0].recipeLinkCssSelector,
);
await browser.awaitPageLoad();
}
}

async function handleFindRecipe(action: any) {
await searchForRecipe(action.parameters.keyword);
await selectRecipeSearchResult(action.parameters.keyword);
}

async function handleBuyRecipeIngredients(action: any) {
await searchForRecipe(action.parameters.keywords);
await selectRecipeSearchResult(action.parameters.keywords);

const targetRecipe = (await getComponentFromPage(
"RecipeHeroSection",
)) as RecipeHeroSection;

if (targetRecipe && targetRecipe.addAllIngridientsCssSelector) {
await browser.clickOn(targetRecipe.addAllIngridientsCssSelector);
}
}

async function handleBuyListContents(action: any) {
const navigationLink = (await getComponentFromPage(
"ListsNavigationLink",
)) as ListsNavigationLink;

if (navigationLink) {
await browser.clickOn(navigationLink.linkCssSelector);
await browser.awaitPageLoad();

const request = `List name: ${action.listName}`;
const targetList = (await getComponentFromPage(
"ListInfo",
request,
)) as ListInfo;

if (targetList && targetList.detailsLinkCssSelector) {
await browser.clickOn(targetList.detailsLinkCssSelector);
await browser.awaitPageLoad();

const listDetails = (await getComponentFromPage(
"ListDetailsInfo",
)) as ListDetailsInfo;

if (listDetails && listDetails.products) {
for (let product of listDetails.products) {
if (product.addToCartButton) {
await browser.clickOn(product.addToCartButton.cssSelector);
}
}
}
}
}
}

async function handleBuyItAgain(action: any) {
await searchForStore(action.parameters.storeName);
await selectStoreSearchResult(action.parameters.storeName);

const navigationLink = (await getComponentFromPage(
"BuyItAgainNavigationLink",
)) as BuyItAgainNavigationLink;

if (navigationLink) {
await browser.clickOn(navigationLink.linkCssSelector);
await browser.awaitPageLoad();

const headerSection = (await getComponentFromPage(
"BuyItAgainHeaderSection",
)) as BuyItAgainHeaderSection;

if (headerSection && headerSection.products) {
if (action.parameters.allItems) {
for (let product of headerSection.products) {
if (product.addToCartButton) {
await browser.clickOn(product.addToCartButton.cssSelector);
}
}
} else {
const request = `Product: ${action.productName}`;
const targetProduct = (await getComponentFromPage(
"ProductTile",
request,
)) as ProductTile;
if (targetProduct && targetProduct.addToCartButton) {
await browser.clickOn(targetProduct.addToCartButton.cssSelector);
}
}
}
}
}

return message;
}
Loading

0 comments on commit 62a28c8

Please sign in to comment.