Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions infra/containers/sandbox/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,11 @@ The driver returns hyphenated page refs while retaining their XPath map server-s
one bounded method/value against a single-use ref from the latest active-page state. Successful acts
atomically replace the consumed ref map with the fresh post-action tree and its server-held map, so
multi-step verification chains without another observation inference.
The serialized driver derives its exact-URL and allowed-origin guards from that same retained
observation, so no separate state request can race a browser restart or let a Worker-provided URL
weaken the binding. If a deterministic action fails while the origin interceptor and active-page
integrity checks remain healthy, the driver preserves the page and requires a fresh observation;
only a failed security or browser-integrity check discards the browser runtime.
Stagehand observation inference, natural-language actions, and self-healing are disabled; browser
behavior is independent of the selected model's structured-output quirks. Navigation clears the
observation, and origin interception remains active for deterministic execution.
Expand Down
143 changes: 95 additions & 48 deletions infra/containers/sandbox/browser-driver/server.js
Original file line number Diff line number Diff line change
Expand Up @@ -448,48 +448,86 @@ async function runAction(runtime, action) {

async function runGuardedAct(runtime, page, action) {
const { stagehand } = runtime;
assertExpectedBrowserTarget(page.url(), action.expectedUrl, action.allowedOrigin);
const observedAction = requireBoundAction(action.action, page.url());
const binding = requireBoundAction(action.action, page.url());
latestObservation = undefined;
let failure;
let originInterceptor;
let response;
const interceptor = await requireOriginInterceptor(runtime, binding.allowedOrigin);
const actionResult = await runBoundAction(stagehand, page, action, binding, interceptor);
await closeOriginInterceptor(runtime, interceptor);
if (actionResult.integrityError) {
await discardBrowserRuntime(runtime);
throw actionResult.integrityError;
}
const activePage = await requireAllowedActivePage(runtime, binding.allowedOrigin);
if (actionResult.actionError) {
throw new RequestError(
409,
"Browser action did not complete; observe the current page before retrying",
);
}

const observation = await capturePageObservation(activePage);
latestObservation = observation.boundary;
return {
result: {
action: { method: action.action.method, ref: action.action.ref },
state: observation.state,
},
type: action.type,
url: activePage.url(),
};
}

async function requireOriginInterceptor(runtime, allowedOrigin) {
try {
return await installOriginInterceptor(runtime.stagehand, allowedOrigin);
} catch (error) {
await discardBrowserRuntime(runtime);
throw error;
}
}

async function runBoundAction(stagehand, page, action, binding, interceptor) {
try {
await interceptor.assertHealthy();
assertExpectedBrowserTarget(page.url(), binding.expectedUrl, binding.allowedOrigin);
} catch (integrityError) {
return { integrityError };
}
let actionError;
try {
originInterceptor = await installOriginInterceptor(stagehand, action.allowedOrigin);
await originInterceptor.assertHealthy();
assertExpectedBrowserTarget(page.url(), action.expectedUrl, action.allowedOrigin);
await stagehand.act(observedAction, {
await stagehand.act(binding.action, {
page,
timeout: action.timeoutMs || 10000,
});
await originInterceptor.assertHealthy();
const activePage = await stagehand.context.awaitActivePage();
assertAllowedBrowserOrigin(activePage.url(), action.allowedOrigin);
const observation = await capturePageObservation(activePage);
latestObservation = observation.boundary;
response = {
result: {
action: { method: action.action.method, ref: action.action.ref },
state: observation.state,
},
type: action.type,
url: activePage.url(),
};
} catch (error) {
failure = error;
actionError = error;
}
if (originInterceptor) {
try {
await originInterceptor.close();
} catch (error) {
failure ??= error;
}
try {
await interceptor.assertHealthy();
} catch (integrityError) {
return { actionError, integrityError };
}
return { actionError };
}

async function closeOriginInterceptor(runtime, interceptor) {
try {
await interceptor.close();
} catch (error) {
await discardBrowserRuntime(runtime);
throw error;
}
if (failure) {
}

async function requireAllowedActivePage(runtime, allowedOrigin) {
try {
const activePage = await runtime.stagehand.context.awaitActivePage();
assertAllowedBrowserOrigin(activePage.url(), allowedOrigin);
return activePage;
} catch (error) {
await discardBrowserRuntime(runtime);
throw failure;
throw error;
}
return response;
}

async function capturePageObservation(page) {
Expand All @@ -516,13 +554,34 @@ function requireBoundAction(action, pageUrl) {
const selector = observedSelector(observation.refs, normalized.ref);
const args = browserActionArguments(observation.refs, normalized);
return {
arguments: args,
description: `${normalized.method} observed element [${normalized.ref}]`,
method: normalized.method,
selector,
action: {
arguments: args,
description: `${normalized.method} observed element [${normalized.ref}]`,
method: normalized.method,
selector,
},
allowedOrigin: observedHttpOrigin(observation.url),
expectedUrl: observation.url,
};
}

function observedHttpOrigin(value) {
let url;
try {
url = new URL(value);
} catch {
throw new RequestError(409, "Browser action requires an observed HTTP page");
}
if (
(url.protocol !== "http:" && url.protocol !== "https:") ||
url.username ||
url.password
) {
throw new RequestError(409, "Browser action requires an observed HTTP page");
}
return url.origin;
}

function observedSelector(refs, ref) {
const xpath = refs.get(ref);
if (!xpath) {
Expand Down Expand Up @@ -676,11 +735,6 @@ function validateAction(action) {
}
if (action.type === "act") {
action.action = validateBoundAction(action.action);
const expectedUrl = assertHttpUrl(action.expectedUrl);
const allowedOrigin = assertHttpUrl(action.allowedOrigin);
if (allowedOrigin.href !== `${allowedOrigin.origin}/` || expectedUrl.origin !== allowedOrigin.origin) {
throw new RequestError(400, "Browser action origin guard is invalid");
}
if (
action.timeoutMs !== undefined &&
(!Number.isInteger(action.timeoutMs) || action.timeoutMs < 1 || action.timeoutMs > 120_000)
Expand Down Expand Up @@ -779,13 +833,6 @@ const server = createServer(async (request, response) => {
return;
}

if (request.method === "GET" && request.url === "/state") {
const { stagehand } = await browserRuntime();
const page = await stagehand.context.awaitActivePage();
jsonResponse(response, 200, { ok: true, url: page.url() });
return;
}

if (request.method === "POST" && request.url === "/actions") {
const rawBody = await readBody(request);
const actions = parseActionsInput(rawBody);
Expand Down
7 changes: 5 additions & 2 deletions packages/agent-core/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,11 @@ ports and Git destinations stable between resolution and execution. The managed
the same boundary: observation reads Stagehand's native accessibility snapshot without model
inference and returns page-bound element refs. Execution accepts only a single-use ref from the
latest state tree plus a bounded method/value, resolves its server-held XPath, and atomically binds
the returned post-action tree as the next actionable state. A click or fill therefore cannot
invoke a hidden model decision, expose a selector, reuse a stale ref, or cross the active origin.
the returned post-action tree as the next actionable state. The sandbox driver derives the exact
URL and allowed origin from that same server-held observation inside the serialized action request;
the Worker cannot supply or inspect that security binding in a separate request. A click or fill
therefore cannot invoke a hidden model decision, expose a selector, reuse a stale ref, or cross the
active origin.
Every first-party browser tool advertises its strict JSON schema to providers that support strict
tool calling, while the same Zod contract remains the provider-independent runtime boundary.
The model-facing action schema is the same method-specific union used by the driver: every action
Expand Down
16 changes: 2 additions & 14 deletions packages/agent-core/src/mastra/tool-defs/browser-tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import {
executeBrowserObserve,
executeBrowserOpen,
executeBrowserScreenshot,
inspectBrowserPage,
} from "../../tools/browser";
import { browserRuntimeFromContext } from "./tool-runtime-context";

Expand Down Expand Up @@ -48,19 +47,8 @@ export const mastraBrowserAct = createTool({
],
outputSchema: BrowserActionsOutputSchema,
strict: true,
execute: async (input, context) => {
const parsedInput = BrowserActInputSchema.parse(input);
const runtimeContext = await browserRuntimeFromContext(context);
const page = await inspectBrowserPage(runtimeContext);
const expectedUrl = new URL(page.url);
if (expectedUrl.username || expectedUrl.password) {
throw new Error("Browser action URL must not contain embedded credentials.");
}
return executeBrowserAct(parsedInput, runtimeContext, {
allowedOrigin: expectedUrl.origin,
expectedUrl: page.url,
});
},
execute: async (input, context) =>
executeBrowserAct(input, await browserRuntimeFromContext(context)),
});

export const mastraBrowserObserve = createTool({
Expand Down
36 changes: 0 additions & 36 deletions packages/agent-core/src/tools/browser/actions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,6 @@ export const BrowserOpenInputSchema = z.strictObject({
waitUntil: WaitUntilSchema.default("domcontentloaded").describe("Navigation wait strategy."),
});

const BrowserActGuardSchema = z.strictObject({
allowedOrigin: BrowserUrlSchema,
expectedUrl: BrowserUrlSchema,
});

export const BrowserObserveInputSchema = z.strictObject({});

export const BrowserExtractInputSchema = z.strictObject({
Expand Down Expand Up @@ -91,8 +86,6 @@ const BrowserActionSchema = z.discriminatedUnion("type", [
}),
z.strictObject({
action: BrowserBoundActionSchema,
allowedOrigin: BrowserUrlSchema,
expectedUrl: BrowserUrlSchema,
type: z.literal("act"),
timeoutMs: z.number().int().positive().max(120_000).default(10_000),
}),
Expand Down Expand Up @@ -165,7 +158,6 @@ const BrowserDriverErrorSchema = z

type BrowserOpenInput = z.input<typeof BrowserOpenInputSchema>;
type BrowserActInput = z.input<typeof BrowserActInputSchema>;
type BrowserActGuard = z.infer<typeof BrowserActGuardSchema>;
type BrowserObserveInput = z.input<typeof BrowserObserveInputSchema>;
type BrowserExtractInput = z.input<typeof BrowserExtractInputSchema>;
type BrowserScreenshotInput = z.input<typeof BrowserScreenshotInputSchema>;
Expand All @@ -189,17 +181,13 @@ export async function executeBrowserOpen(
export async function executeBrowserAct(
input: BrowserActInput,
runtimeContext: BrowserRuntimeContext,
guard: BrowserActGuard,
): Promise<BrowserActionsOutput> {
const parsedInput = BrowserActInputSchema.parse(input);
const parsedGuard = BrowserActGuardSchema.parse(guard);
return executeBrowserActions(
{
actions: [
{
action: browserBoundActionFromInput(parsedInput),
allowedOrigin: parsedGuard.allowedOrigin,
expectedUrl: parsedGuard.expectedUrl,
type: "act",
timeoutMs: BROWSER_ACT_TIMEOUT_MS,
},
Expand All @@ -209,25 +197,6 @@ export async function executeBrowserAct(
);
}

/** Reads the exact active-page URL used to classify and bind a browser action. */
export async function inspectBrowserPage(
runtimeContext: BrowserRuntimeContext,
): Promise<{ url: string }> {
const ready = await readyBrowserDriver(runtimeContext);
const result = await requestBrowserDriver(
null,
"/state",
ready.runtimeContext,
ready.connection,
30_000,
);
if (!result.success) {
throw browserDriverRequestError(result);
}
const state = BrowserDriverStateSchema.parse(result.body);
return { url: state.url };
}

export async function executeBrowserObserve(
input: BrowserObserveInput,
runtimeContext: BrowserRuntimeContext,
Expand Down Expand Up @@ -654,11 +623,6 @@ const BrowserDriverHealthSchema = z.strictObject({
runId: z.string().min(1),
});

const BrowserDriverStateSchema = z.strictObject({
ok: z.literal(true),
url: BrowserUrlSchema,
});

function stagehandModel(credential: BrowserRuntimeContext["credential"]): string {
return `${credential.provider}/${credential.modelId}`;
}
Expand Down
1 change: 0 additions & 1 deletion packages/agent-core/src/tools/browser/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,5 @@ export {
executeBrowserObserve,
executeBrowserOpen,
executeBrowserScreenshot,
inspectBrowserPage,
} from "./actions";
export type { BrowserProvider, BrowserRuntimeContext } from "./runtime";