diff --git a/apps/web/src/components/chat/message-activity-model.ts b/apps/web/src/components/chat/message-activity-model.ts index 845283b1..046f6f92 100644 --- a/apps/web/src/components/chat/message-activity-model.ts +++ b/apps/web/src/components/chat/message-activity-model.ts @@ -64,6 +64,8 @@ const TOOL_VERBS: Record = { docs_generate_pdf: { verb: "Generated a PDF" }, docs_generate_slides: { verb: "Generated slides" }, docs_generate_xlsx: { verb: "Generated a spreadsheet" }, + generate_or_edit_image: { verb: "Generated or edited an image" }, + generate_or_extend_video: { verb: "Generated or extended a video" }, search_scrape: { verb: "Scraped", argKeys: ["url"] }, search_web_content: { verb: "Searched the web", argKeys: ["query", "q"] }, search_extract: { verb: "Extracted", argKeys: ["url"] }, diff --git a/packages/agent-core/README.md b/packages/agent-core/README.md index d43d5ae8..c735eb3f 100644 --- a/packages/agent-core/README.md +++ b/packages/agent-core/README.md @@ -27,8 +27,11 @@ Sandbox and artifact capabilities cross tool-domain boundaries only through Single-consumer data, document, and media implementations live under `src/tools/`. Data tools profile and normalize bounded tabular inputs and render deterministic SVG/Recharts output. Document tools generate sandbox-side PPTX, -DOCX, XLSX, and PDF source against `/opt/cheatcode-doc-runtime`. Media and Google-backed -browser tools resolve the user's Google AI BYOK key lazily when invoked. All three receive +DOCX, XLSX, and PDF source against `/opt/cheatcode-doc-runtime`; routine XLSX generation +supports bounded scalar and workbook-local formula cells with cached results and standard number +formats, while complex models remain on the custom Office path. Image and video generation use +separate strict contracts so one mode cannot acquire the other mode's arguments. Media and +Google-backed browser tools resolve the user's Google AI BYOK key lazily when invoked. All three receive sandbox and R2 artifact capabilities through request-scoped contracts; they do not read environment variables, persist credentials, or log keys. Published media is rendered automatically in the chat artifact card and is available in Files; diff --git a/packages/agent-core/src/mastra/agent-tool-policy.ts b/packages/agent-core/src/mastra/agent-tool-policy.ts index 4758b9f3..dbfc0518 100644 --- a/packages/agent-core/src/mastra/agent-tool-policy.ts +++ b/packages/agent-core/src/mastra/agent-tool-policy.ts @@ -54,7 +54,10 @@ const DOCUMENT_TOOLS = [ "docs_generate_xlsx", ] as const satisfies readonly ToolCapabilityName[]; -const MEDIA_TOOLS = ["generate_or_edit_media"] as const satisfies readonly ToolCapabilityName[]; +const MEDIA_TOOLS = [ + "generate_or_edit_image", + "generate_or_extend_video", +] as const satisfies readonly ToolCapabilityName[]; const RESEARCH_TOOLS = [ "search_extract", diff --git a/packages/agent-core/src/mastra/system-prompt.ts b/packages/agent-core/src/mastra/system-prompt.ts index 89c288e9..8a05e699 100644 --- a/packages/agent-core/src/mastra/system-prompt.ts +++ b/packages/agent-core/src/mastra/system-prompt.ts @@ -322,7 +322,7 @@ This project is scaffolded at the workspace root and its dev server + live previ const DOCS_MODULE = `## Building documents & slides -Build decks and docs from scratch with the preinstalled libraries — pptxgenjs for .pptx, docx, @react-pdf/renderer, exceljs — when the request needs custom layout control, or use docs_generate_slides / docs_generate_docx / docs_generate_pdf / docs_generate_xlsx for a fast structured deliverable. Preserve exact requested counts and scope. The structured generators validate their bounded layouts, publish the file, return its canonical read-only project path, and report the authoritative slide, section, or sheet count. For routine structured output, confirm that returned count and finish; do not search for the file, convert it, start a local server, or take screenshots. Use the custom path and inspect real renders when the user explicitly requests visual QA, the layout is custom or template-driven, or the document is high-stakes. In that path, check for faint text, overflow, placeholder text, and unfilled sections; re-render only affected pages after a fix. The file lands in the Deliverables automatically — refer to it naturally ("your deck is ready below"), don't paste a download link.`; +Build decks and docs from scratch with the preinstalled libraries — pptxgenjs for .pptx, docx, @react-pdf/renderer, exceljs — when the request needs custom layout control, or use docs_generate_slides / docs_generate_docx / docs_generate_pdf / docs_generate_xlsx for a fast structured deliverable. Preserve exact requested counts and scope. The XLSX generator accepts scalar cells plus typed formula cells with cached results and number formats. The structured generators validate their bounded layouts, publish the file, return its canonical read-only project path, and report the authoritative slide, section, or sheet count. For routine structured output, confirm that returned count and finish; do not search for the file, convert it, start a local server, or take screenshots. Use the custom path and inspect real renders when the user explicitly requests visual QA, the layout is custom or template-driven, or the document is high-stakes. In that path, check for faint text, overflow, placeholder text, and unfilled sections; re-render only affected pages after a fix. The file lands in the Deliverables automatically — refer to it naturally ("your deck is ready below"), don't paste a download link.`; const DATA_MODULE = `## Data & analysis @@ -330,7 +330,7 @@ For a quick question, compute the answer and just tell the user — a small code const MEDIA_MODULE = `## Images & video -Load the generate-media skill before creating or editing an image or generating or extending a video. Use generate_or_edit_media for the final asset, choose the requested aspect ratio intentionally, preserve stated invariants when editing, and refer to the resulting Deliverable naturally without pasting its download URL. A successful result is rendered automatically in the chat and can be opened in Files; do not open its sandbox path in the browser or claim that the user cannot preview it.`; +Load the generate-media skill before creating or editing an image or generating or extending a video. Use generate_or_edit_image for an image and generate_or_extend_video for a video. Call the selected tool once, choose the requested aspect ratio intentionally, preserve stated invariants when editing, and refer to the resulting Deliverable naturally without pasting its download URL. A successful result is rendered automatically in the chat and can be opened in Files; do not open its sandbox path in the browser or claim that the user cannot preview it. If the provider rejects or fails the request, report that failure instead of retrying the same request or substituting a hand-authored asset.`; const RESEARCH_MODULE = `## Research @@ -343,7 +343,7 @@ Pick the path that fits and load the matching skill (skill_invoke) for its full - Web or mobile app → build it in the sandbox (React / Next.js, or Expo for mobile), start the dev server, and verify it in the browser; the running app shows in the Computer panel. - Slides or documents → use the fast structured generator when it fits, or the document libraries for custom control; preserve exact counts and verify proportionately before finishing. The file lands in the Deliverables. - Data → profile it (data_analyze_csv, or pandas / Node) and chart it (data_chart) when it adds insight; verify the numbers. -- Image or video → load generate-media, then use generate_or_edit_media; the asset lands in the project and Deliverables and renders automatically in chat and Files. +- Image or video → load generate-media, then use generate_or_edit_image for an image or generate_or_extend_video for a video; the asset lands in the project and Deliverables and renders automatically in chat and Files. - Research → gather and cross-check real sources (search_web / firecrawl_* / research_deep); cite everything, and use the deep-research workflow when the user asks for a report so its PDF is delivered automatically. - Acting in the user's connected apps → composio_list_tools then composio_execute, only when they ask.`; diff --git a/packages/agent-core/src/mastra/tool-defs/document-tools.ts b/packages/agent-core/src/mastra/tool-defs/document-tools.ts index cda015c8..c814fad3 100644 --- a/packages/agent-core/src/mastra/tool-defs/document-tools.ts +++ b/packages/agent-core/src/mastra/tool-defs/document-tools.ts @@ -45,7 +45,7 @@ export const mastraDocsGenerateDocx = createTool({ export const mastraDocsGenerateXlsx = createTool({ id: "docs_generate_xlsx", description: - "Generate an XLSX workbook from sheets, columns, and rows. Publishes it as a Deliverable and returns its exact project filePath.", + "Generate an XLSX workbook from sheets, columns, scalar cells, and typed formula cells with cached results and number formats. Publishes it as a Deliverable and returns its exact project filePath; routine workbooks need no custom script or recalculation pass.", inputSchema: GenerateSpreadsheetInputSchema, outputSchema: GenerateXlsxOutputSchema, execute: async (input, context) => diff --git a/packages/agent-core/src/mastra/tool-defs/media-tools.ts b/packages/agent-core/src/mastra/tool-defs/media-tools.ts index c45fefff..e0ade625 100644 --- a/packages/agent-core/src/mastra/tool-defs/media-tools.ts +++ b/packages/agent-core/src/mastra/tool-defs/media-tools.ts @@ -1,24 +1,47 @@ import { createTool } from "@mastra/core/tools"; -import { executeGenerateOrEditMedia } from "../../tools/media/execute"; import { - GenerateOrEditMediaInputSchema, - GenerateOrEditMediaOutputSchema, + executeGenerateOrEditImage, + executeGenerateOrExtendVideo, +} from "../../tools/media/execute"; +import { + GenerateImageOutputSchema, + GenerateOrEditImageInputSchema, + GenerateOrExtendVideoInputSchema, + GenerateVideoOutputSchema, } from "../../tools/media/schemas"; import { resolveGoogleToolApiKey } from "./request-context"; import { requestContextFromToolContext, workspaceRuntimeFromContext } from "./tool-runtime-context"; -/** Generates or edits media after lazily resolving the user's Google AI tool key. */ -export const mastraGenerateOrEditMedia = createTool({ - id: "generate_or_edit_media", +/** Generates or edits one image through a contract that cannot accept video-only fields. */ +export const mastraGenerateOrEditImage = createTool({ + id: "generate_or_edit_image", + description: + "Generate a new image or edit/reference provided images with Google AI. Publishes the result to the project and Deliverables in one bounded call.", + inputSchema: GenerateOrEditImageInputSchema, + outputSchema: GenerateImageOutputSchema, + execute: async (input, context) => { + const requestContext = requestContextFromToolContext(context); + const googleApiKey = await resolveGoogleToolApiKey(requestContext); + return executeGenerateOrEditImage( + GenerateOrEditImageInputSchema.parse(input), + await workspaceRuntimeFromContext(context), + googleApiKey ?? "", + ); + }, +}); + +/** Generates or extends one video through a contract that cannot accept image-edit fields. */ +export const mastraGenerateOrExtendVideo = createTool({ + id: "generate_or_extend_video", description: - "Generate or edit an image, or generate/extend a video, using Google media models. Stores the result in the project and Deliverables.", - inputSchema: GenerateOrEditMediaInputSchema, - outputSchema: GenerateOrEditMediaOutputSchema, + "Generate a new video or extend a provided video with Google AI. Publishes the result to the project and Deliverables in one bounded call.", + inputSchema: GenerateOrExtendVideoInputSchema, + outputSchema: GenerateVideoOutputSchema, execute: async (input, context) => { const requestContext = requestContextFromToolContext(context); const googleApiKey = await resolveGoogleToolApiKey(requestContext); - return executeGenerateOrEditMedia( - GenerateOrEditMediaInputSchema.parse(input), + return executeGenerateOrExtendVideo( + GenerateOrExtendVideoInputSchema.parse(input), await workspaceRuntimeFromContext(context), googleApiKey ?? "", ); diff --git a/packages/agent-core/src/mastra/tool-defs/registry.ts b/packages/agent-core/src/mastra/tool-defs/registry.ts index a9b29cf2..f7412081 100644 --- a/packages/agent-core/src/mastra/tool-defs/registry.ts +++ b/packages/agent-core/src/mastra/tool-defs/registry.ts @@ -32,7 +32,7 @@ export { mastraDocsGenerateSlides, mastraDocsGenerateXlsx, } from "./document-tools"; -export { mastraGenerateOrEditMedia } from "./media-tools"; +export { mastraGenerateOrEditImage, mastraGenerateOrExtendVideo } from "./media-tools"; export { mastraDeepResearch, mastraFirecrawlExtract, diff --git a/packages/agent-core/src/mastra/tool-defs/tool-set.ts b/packages/agent-core/src/mastra/tool-defs/tool-set.ts index 187c157c..750cb517 100644 --- a/packages/agent-core/src/mastra/tool-defs/tool-set.ts +++ b/packages/agent-core/src/mastra/tool-defs/tool-set.ts @@ -24,7 +24,8 @@ import { mastraFsRead, mastraFsSearch, mastraFsWrite, - mastraGenerateOrEditMedia, + mastraGenerateOrEditImage, + mastraGenerateOrExtendVideo, mastraGitClone, mastraGitCommit, mastraGitPush, @@ -74,7 +75,8 @@ export const cheatcodeTools = { fs_read: mastraFsRead, fs_search: mastraFsSearch, fs_write: mastraFsWrite, - generate_or_edit_media: mastraGenerateOrEditMedia, + generate_or_edit_image: mastraGenerateOrEditImage, + generate_or_extend_video: mastraGenerateOrExtendVideo, git_clone: mastraGitClone, git_commit: mastraGitCommit, git_push: mastraGitPush, diff --git a/packages/agent-core/src/tools/docs/schemas.ts b/packages/agent-core/src/tools/docs/schemas.ts index c896d745..d53b520b 100644 --- a/packages/agent-core/src/tools/docs/schemas.ts +++ b/packages/agent-core/src/tools/docs/schemas.ts @@ -53,7 +53,26 @@ export const GenerateMarkdownPdfInputSchema = z.strictObject({ title: TextValueSchema.optional(), }); -const SpreadsheetCellSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]); +const SpreadsheetScalarSchema = z.union([ + z.string().max(10_000), + z.number(), + z.boolean(), + z.null(), +]); + +const SpreadsheetFormulaSchema = z.strictObject({ + formula: z + .string() + .trim() + .min(1) + .max(1_000) + .refine(isSafeSpreadsheetFormula, "Formula must use workbook-local values and functions."), + numberFormat: z.enum(["general", "integer", "decimal", "percent", "currency"]).optional(), + result: z.union([z.string().max(10_000), z.number(), z.boolean()]), + type: z.literal("formula"), +}); + +const SpreadsheetCellSchema = z.union([SpreadsheetScalarSchema, SpreadsheetFormulaSchema]); const SpreadsheetRowSchema = z.record(z.string().min(1).max(80), SpreadsheetCellSchema); const SpreadsheetSheetSchema = z.strictObject({ @@ -107,3 +126,9 @@ export type GenerateSlidesInput = z.input; export type GenerateSlidesOutput = z.output; export type GenerateSpreadsheetInput = z.input; export type GenerateXlsxOutput = z.output; + +function isSafeSpreadsheetFormula(value: string): boolean { + const formula = value.startsWith("=") ? value.slice(1) : value; + const blockedFunction = /\b(?:CALL|DDE|EXEC|FILTERXML|HYPERLINK|RTD|WEBSERVICE)\s*\(/i; + return formula.length > 0 && !/[[\]"'|@]/.test(formula) && !blockedFunction.test(formula); +} diff --git a/packages/agent-core/src/tools/docs/scripts.ts b/packages/agent-core/src/tools/docs/scripts.ts index da8edb71..94aa06d6 100644 --- a/packages/agent-core/src/tools/docs/scripts.ts +++ b/packages/agent-core/src/tools/docs/scripts.ts @@ -107,6 +107,17 @@ export function buildXlsxScript(inputPath: string, outputPath: string, filename: 'workbook.creator = "Cheatcode";', "workbook.created = new Date();", "workbook.title = input.title;", + "const numberFormats = {", + ' general: "General",', + ' integer: "#,##0",', + ' decimal: "#,##0.00",', + ' percent: "0.0%",', + ' currency: "$#,##0.00;($#,##0.00);-",', + "};", + "function cellValue(value) {", + ' if (!value || typeof value !== "object" || value.type !== "formula") return value;', + ' return { formula: value.formula.replace(/^=/, ""), result: value.result };', + "}", "for (const sheet of input.sheets) {", " const worksheet = workbook.addWorksheet(sheet.name);", " worksheet.columns = sheet.columns.map((column) => ({", @@ -114,7 +125,16 @@ export function buildXlsxScript(inputPath: string, outputPath: string, filename: " key: column,", " width: Math.max(12, Math.min(36, column.length + 8)),", " }));", - " worksheet.addRows(sheet.rows);", + " for (const inputRow of sheet.rows) {", + " const values = Object.fromEntries(sheet.columns.map((column) => [column, cellValue(inputRow[column])]));", + " const row = worksheet.addRow(values);", + " sheet.columns.forEach((column, columnIndex) => {", + " const value = inputRow[column];", + ' if (value && typeof value === "object" && value.type === "formula" && value.numberFormat) {', + " row.getCell(columnIndex + 1).numFmt = numberFormats[value.numberFormat];", + " }", + " });", + " }", " worksheet.getRow(1).font = { bold: true };", " worksheet.views = [{ state: 'frozen', ySplit: 1 }];", "}", diff --git a/packages/agent-core/src/tools/media/execute.ts b/packages/agent-core/src/tools/media/execute.ts index 694587f6..a365b6e9 100644 --- a/packages/agent-core/src/tools/media/execute.ts +++ b/packages/agent-core/src/tools/media/execute.ts @@ -3,10 +3,14 @@ import type { ArtifactUploadResult, CodeRuntimeContext } from "@cheatcode/sandbo import { GoogleGenAI, type Image, type Video, VideoGenerationReferenceType } from "@google/genai"; import { z } from "zod"; import { - type GenerateOrEditMediaInput, - GenerateOrEditMediaInputSchema, - type GenerateOrEditMediaOutput, - GenerateOrEditMediaOutputSchema, + type GenerateImageOutput, + GenerateImageOutputSchema, + type GenerateOrEditImageInput, + GenerateOrEditImageInputSchema, + type GenerateOrExtendVideoInput, + GenerateOrExtendVideoInputSchema, + type GenerateVideoOutput, + GenerateVideoOutputSchema, } from "./schemas"; const IMAGE_MODEL = "gemini-3.1-flash-image"; @@ -43,37 +47,53 @@ const InteractionStepsSchema = z.array( }), ); -export async function executeGenerateOrEditMedia( - input: GenerateOrEditMediaInput, +export async function executeGenerateOrEditImage( + input: GenerateOrEditImageInput, runtimeContext: CodeRuntimeContext, googleApiKey: string, -): Promise { - const parsed = GenerateOrEditMediaInputSchema.parse(input); +): Promise { + const parsed = GenerateOrEditImageInputSchema.parse(input); const apiKey = requiredApiKey(googleApiKey); const client = new GoogleGenAI({ apiKey }); - const media = - parsed.type === "image" - ? await generateImage(client, parsed, runtimeContext) - : await generateVideo(client, apiKey, parsed, runtimeContext); - const filename = generatedFilename(parsed.prompt, media.extension); - const sandboxPath = await writeMediaToWorkspace( - runtimeContext, - parsed.type, - filename, - media.bytes, - ); - const artifact = await storeArtifact(runtimeContext, parsed.type, filename, media); - return GenerateOrEditMediaOutputSchema.parse({ - artifact, - model: media.model, - sandboxPath, - type: parsed.type, + const media = await generateImage(client, parsed, runtimeContext); + const result = await persistGeneratedMedia(runtimeContext, "image", parsed.prompt, media); + return GenerateImageOutputSchema.parse({ + ...result, + type: "image", }); } +export async function executeGenerateOrExtendVideo( + input: GenerateOrExtendVideoInput, + runtimeContext: CodeRuntimeContext, + googleApiKey: string, +): Promise { + const parsed = GenerateOrExtendVideoInputSchema.parse(input); + const apiKey = requiredApiKey(googleApiKey); + const client = new GoogleGenAI({ apiKey }); + const media = await generateVideo(client, apiKey, parsed, runtimeContext); + const result = await persistGeneratedMedia(runtimeContext, "video", parsed.prompt, media); + return GenerateVideoOutputSchema.parse({ + ...result, + type: "video", + }); +} + +async function persistGeneratedMedia( + runtime: CodeRuntimeContext, + type: "image" | "video", + prompt: string, + media: GeneratedMedia, +) { + const filename = generatedFilename(prompt, media.extension); + const sandboxPath = await writeMediaToWorkspace(runtime, type, filename, media.bytes); + const artifact = await storeArtifact(runtime, type, filename, media); + return { artifact, model: media.model, sandboxPath }; +} + async function generateImage( client: GoogleGenAI, - input: GenerateOrEditMediaInput, + input: GenerateOrEditImageInput, runtime: CodeRuntimeContext, ): Promise { const references = await loadReferences(input.reference_images ?? [], runtime); @@ -110,7 +130,7 @@ function imageInteractionInput(prompt: string, references: MediaReference[]) { ]; } -function imagePrompt(input: GenerateOrEditMediaInput): string { +function imagePrompt(input: GenerateOrEditImageInput): string { if (!input.reference_images?.length) { return input.prompt; } @@ -155,7 +175,7 @@ function extractInteractionImage(steps: unknown): MediaReference | null { async function generateVideo( client: GoogleGenAI, apiKey: string, - input: GenerateOrEditMediaInput, + input: GenerateOrExtendVideoInput, runtime: CodeRuntimeContext, ): Promise { const references = await loadReferences(input.reference_images ?? [], runtime); @@ -186,7 +206,7 @@ async function generateVideo( return generatedMedia(bytesToBase64(bytes), generated.mimeType ?? "video/mp4", VIDEO_MODEL); } -function videoConfig(input: GenerateOrEditMediaInput, references: MediaReference[]) { +function videoConfig(input: GenerateOrExtendVideoInput, references: MediaReference[]) { return { aspectRatio: input.aspect_ratio ?? "16:9", durationSeconds: input.duration ?? 8, diff --git a/packages/agent-core/src/tools/media/schemas.ts b/packages/agent-core/src/tools/media/schemas.ts index f9de2b76..10dd2111 100644 --- a/packages/agent-core/src/tools/media/schemas.ts +++ b/packages/agent-core/src/tools/media/schemas.ts @@ -7,19 +7,25 @@ const MediaReferenceSchema = z .max(2_048) .describe("A project-relative or absolute sandbox path, or a public HTTPS URL."); -export const GenerateOrEditMediaInputSchema = z +const MediaPromptSchema = z.string().trim().min(3).max(20_000); + +export const GenerateOrEditImageInputSchema = z.strictObject({ + aspect_ratio: z + .enum(["1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9"]) + .optional(), + image_reference_mode: z.enum(["reference_generate", "edit"]).optional(), + prompt: MediaPromptSchema, + reference_images: z.array(MediaReferenceSchema).max(8).optional(), +}); + +export const GenerateOrExtendVideoInputSchema = z .strictObject({ - aspect_ratio: z - .enum(["1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9"]) - .optional(), + aspect_ratio: z.enum(["16:9", "9:16"]).optional(), duration: z.union([z.literal(4), z.literal(6), z.literal(8)]).optional(), - image_reference_mode: z.enum(["reference_generate", "edit"]).optional(), - prompt: z.string().trim().min(3).max(20_000), - reference_images: z.array(MediaReferenceSchema).max(8).optional(), + prompt: MediaPromptSchema, + reference_images: z.array(MediaReferenceSchema).max(3).optional(), reference_video: MediaReferenceSchema.optional(), - type: z.enum(["image", "video"]), }) - .superRefine((input, context) => { if (input.reference_images?.length && input.reference_video) { context.addIssue({ @@ -28,47 +34,30 @@ export const GenerateOrEditMediaInputSchema = z path: ["reference_video"], }); } - if (input.type === "image" && (input.duration || input.reference_video)) { - context.addIssue({ - code: "custom", - message: "duration and reference_video are video-only parameters.", - path: ["type"], - }); - } - if (input.type === "video" && (input.reference_images?.length ?? 0) > 3) { - context.addIssue({ - code: "custom", - message: "Video generation supports at most three reference images.", - path: ["reference_images"], - }); - } - if ( - input.type === "video" && - input.aspect_ratio && - !["16:9", "9:16"].includes(input.aspect_ratio) - ) { - context.addIssue({ - code: "custom", - message: "Video generation supports 16:9 or 9:16.", - path: ["aspect_ratio"], - }); - } }); -const MediaArtifactSchema = z.strictObject({ +const MediaArtifactShape = { filename: z.string().min(1), - kind: z.enum(["image", "video"]), mimeType: z.string().min(1), outputId: z.string().min(1), sizeBytes: z.number().int().nonnegative(), +} as const; + +export const GenerateImageOutputSchema = z.strictObject({ + artifact: z.strictObject({ ...MediaArtifactShape, kind: z.literal("image") }), + model: z.string().min(1), + sandboxPath: z.string().min(1), + type: z.literal("image"), }); -export const GenerateOrEditMediaOutputSchema = z.strictObject({ - artifact: MediaArtifactSchema, +export const GenerateVideoOutputSchema = z.strictObject({ + artifact: z.strictObject({ ...MediaArtifactShape, kind: z.literal("video") }), model: z.string().min(1), sandboxPath: z.string().min(1), - type: z.enum(["image", "video"]), + type: z.literal("video"), }); -export type GenerateOrEditMediaInput = z.input; -export type GenerateOrEditMediaOutput = z.output; +export type GenerateOrEditImageInput = z.input; +export type GenerateOrExtendVideoInput = z.input; +export type GenerateImageOutput = z.output; +export type GenerateVideoOutput = z.output; diff --git a/packages/skills/src/generated-manifest.ts b/packages/skills/src/generated-manifest.ts index b31fce70..7d7e630b 100644 --- a/packages/skills/src/generated-manifest.ts +++ b/packages/skills/src/generated-manifest.ts @@ -91,7 +91,7 @@ export const SKILL_MANIFEST: SkillManifestEntry[] = [ }, { "name": "generate-media", - "description": "Use this skill whenever you need to generate or edit images, or generate or extend videos using the generate_or_edit_media tool. Covers prompt engineering, parameter selection, reference image usage, and video generation/extension. Trigger when the user asks to create, generate, edit, modify, or extend any image or video asset.", + "description": "Use this skill whenever you need to generate or edit an image, or generate or extend a video. Route images to generate_or_edit_image and videos to generate_or_extend_video, with one bounded generation call and no cross-mode parameters. Trigger when the user asks to create, generate, edit, modify, or extend any image or video asset.", "category": "Data & Media", "tags": [ "image", diff --git a/packages/types/src/capabilities.ts b/packages/types/src/capabilities.ts index 4c60e576..c4562890 100644 --- a/packages/types/src/capabilities.ts +++ b/packages/types/src/capabilities.ts @@ -54,10 +54,11 @@ export const TOOL_CAPABILITIES = [ tool("docs", "docs_generate_pdf", "Generate a signed PDF artifact.", DELIVERABLE_TOOL), tool("docs", "docs_generate_slides", "Generate a signed PPTX artifact.", DELIVERABLE_TOOL), tool("docs", "docs_generate_xlsx", "Generate a signed XLSX artifact.", DELIVERABLE_TOOL), + tool("data", "generate_or_edit_image", "Generate or edit an image artifact.", DELIVERABLE_TOOL), tool( "data", - "generate_or_edit_media", - "Generate or edit an image or video artifact.", + "generate_or_extend_video", + "Generate or extend a video artifact.", DELIVERABLE_TOOL, ), tool( diff --git a/skills/generate-media/SKILL.md b/skills/generate-media/SKILL.md index 04cb9707..61765eea 100644 --- a/skills/generate-media/SKILL.md +++ b/skills/generate-media/SKILL.md @@ -1,230 +1,95 @@ --- name: generate-media -description: "Use this skill whenever you need to generate or edit images, or generate or extend videos using the generate_or_edit_media tool. Covers prompt engineering, parameter selection, reference image usage, and video generation/extension. Trigger when the user asks to create, generate, edit, modify, or extend any image or video asset." +description: "Use this skill whenever you need to generate or edit an image, or generate or extend a video. Route images to generate_or_edit_image and videos to generate_or_extend_video, with one bounded generation call and no cross-mode parameters. Trigger when the user asks to create, generate, edit, modify, or extend any image or video asset." category: Data & Media tags: image, video, generation, editing, gemini, veo -compatibility: Requires the generate_or_edit_media tool and a Google BYOK key. +compatibility: Requires generate_or_edit_image or generate_or_extend_video and a Google AI BYOK key. --- -# generate_or_edit_media Tool Guide +# Media generation -## Tool Parameters +Choose exactly one tool from the requested output type: -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `type` | `"image"` or `"video"` | Yes | Whether to generate an image or video | -| `prompt` | string (min 3 chars) | Yes | The generation/edit prompt | -| `reference_images` | string[] (max 8) | No | Paths or URLs to reference images. Mutually exclusive with `reference_video` | -| `image_reference_mode` | `"reference_generate"` or `"edit"` | No | Image-only. `reference_generate` (default) creates a new image inspired by references. `edit` directly transforms a referenced image | -| `reference_video` | string | No | Video extension only. Use the **real** `sandboxPath` from a prior generate_or_edit_media video result (e.g. `.cheatcode/assets/videos/...mp4`), or an HTTPS URL to that file. Mutually exclusive with `reference_images` | -| `aspect_ratio` | string | No | Image: `1:1`, `2:3`, `3:2`, `3:4`, `4:3`, `4:5`, `5:4`, `9:16`, `16:9`, `21:9`. Video: `16:9` (default) or `9:16` | -| `duration` | `4`, `6`, or `8` | No | Video only. Length in seconds | +- Static image generation, image references, or direct image editing: `generate_or_edit_image` +- Video generation from text or reference images, or video extension: `generate_or_extend_video` -## Underlying Models +The contracts are intentionally separate. Never add video fields to an image call, image-edit fields to a video call, or a synthetic `type` discriminator to either call. -- **Images**: Gemini 3.1 Flash Image (`gemini-3.1-flash-image`) -- **Videos**: Veo 3.1 (`veo-3.1-generate-preview`) — generates at 1080p with native audio +## Image tool -## Decision Tree +`generate_or_edit_image` accepts: -### 1. Image or video? +| Parameter | Required | Description | +|---|---|---| +| `prompt` | Yes | Scene, subject, details, style, and constraints | +| `aspect_ratio` | No | `1:1`, `2:3`, `3:2`, `3:4`, `4:3`, `4:5`, `5:4`, `9:16`, `16:9`, or `21:9` | +| `reference_images` | No | Up to 8 real project paths or public HTTPS URLs | +| `image_reference_mode` | No | `reference_generate` for inspiration/composition, or `edit` to transform the supplied image directly | -- Static visual asset → `type: "image"` -- Motion, animation, cinematic clip, anything with audio → `type: "video"` +With no references, provide only the prompt and intentional aspect ratio. For an edit, state what changes and what must remain invariant. -### 2. Generate or edit? - -- **No reference images** → Pure generation (just `prompt`) -- **Reference images for style/mood/composition guidance** → Set `reference_images`, leave `image_reference_mode` unset (defaults to `reference_generate`) -- **Directly transforming an existing image** (inpainting, object removal, style transfer on the image itself) → Set `reference_images` + `image_reference_mode: "edit"` -- **Extending a prior generated video** → Set `reference_video` to the prior result's `sandboxPath` (under `.cheatcode/assets/videos/`). Do NOT combine with `reference_images` - -### 3. Aspect ratio - -Pick the ratio that fits the intended use: - -| Use case | Recommended ratio | -|----------|-------------------| -| Square social post, avatar, icon | `1:1` | -| Instagram story, mobile portrait video | `9:16` | -| YouTube thumbnail, desktop hero, landscape video | `16:9` | -| Product photo (portrait) | `4:5` or `3:4` | -| Ultra-wide banner | `21:9` | -| Standard photo landscape | `3:2` | - -## Writing Effective Prompts - -### Prompt Structure - -Order your prompt: **scene/backdrop → subject → key details → style → constraints**. - -### For Photorealistic Images - -Use photography language: camera angles, lens types, lighting, and fine details. - -``` -A photorealistic close-up portrait of an elderly Japanese ceramicist in a -dimly lit workshop, hands shaping wet clay on a pottery wheel. Warm side -lighting from a single window, shallow depth of field with a 85mm f/1.4 -lens. Weathered hands with clay residue, focused expression. 3:2 landscape. -``` - -### For Stylized Illustrations and Stickers - -Be explicit about the art style and background. - -``` -A kawaii-style sticker of a happy red panda holding a bubble tea, pastel -color palette with soft pink and mint green, clean bold outlines, flat -shading, white background. -``` - -### For Text in Images - -Gemini excels at rendering text. Be explicit about the text content, style, and placement. For tricky words, spell them letter-by-letter. - -``` -Create a modern minimalist logo for a coffee shop with the text "The Daily -Grind" in a clean sans-serif font. Earthy brown and cream color scheme, -centered composition on white background. -``` - -### For Product Mockups - -Describe the product, materials, lighting setup, and camera angle. - -``` -A high-resolution studio-lit product photograph of a minimalist ceramic -coffee mug on a light marble surface. Three-point softbox lighting for even -illumination, slight three-quarter angle, ultra-realistic with sharp focus -on the matte texture. No logos, no text, no watermark. -``` - -### For Video Prompts - -Include subject, action, style, camera motion, and audio cues. - -``` -A cinematic tracking shot of a woman walking through a rain-soaked Tokyo -street at night, neon reflections on wet pavement. She opens an umbrella as -thunder rumbles in the distance. Slow dolly forward, shallow depth of field, -moody blue-orange color grading. -``` - -### Prompting for Video Audio - -Veo 3.1 natively generates audio. Provide cues for dialogue, sound effects, and ambiance: - -- **Dialogue**: Use quotes. `"This must be the key," he murmured.` -- **Sound effects**: Describe explicitly. `Tires screeching loudly, engine roaring.` -- **Ambient noise**: Describe the soundscape. `A faint hum resonates in the background, birds chirping.` - -## Editing Images - -### Adding/Removing Elements - -Provide the image via `reference_images` and set `image_reference_mode: "edit"`. - -``` -prompt: "Using the provided image, add a small knitted wizard hat on the cat's head. Match the original lighting and style." -reference_images: ["/path/to/cat-photo.png"] -image_reference_mode: "edit" -``` - -### Inpainting (Semantic Masking) - -Conversationally define a mask — no explicit mask file needed. - -``` -prompt: "Change only the blue sofa to a vintage brown leather chesterfield sofa. Keep everything else exactly the same." -reference_images: ["/path/to/living-room.png"] -image_reference_mode: "edit" -``` - -### Style Transfer - -Provide the style reference and describe the transformation. - -``` -prompt: "Transform this photograph into a watercolor painting style. Preserve the original composition but render with visible brushstrokes and soft color bleeding." -reference_images: ["/path/to/photo.jpg"] +```yaml +prompt: "Change only the blue sofa to vintage brown leather. Preserve the room, lighting, perspective, and every other object." +aspect_ratio: "4:3" +reference_images: + - "/workspace/project/living-room.png" image_reference_mode: "edit" ``` -### Compositing Multiple Images - -Provide multiple references and describe how they combine. - -``` -prompt: "Create a professional e-commerce fashion photo. Place the dress from the first image on the model from the second image. Match lighting and perspective." -reference_images: ["/path/to/dress.png", "/path/to/model.png"] -image_reference_mode: "reference_generate" -``` +## Video tool -## Video-Specific Features +`generate_or_extend_video` accepts: -### Image-to-Video +| Parameter | Required | Description | +|---|---|---| +| `prompt` | Yes | Subject, action, visual style, camera motion, and audio cues | +| `aspect_ratio` | No | `16:9` or `9:16` | +| `duration` | No | `4`, `6`, or `8` seconds | +| `reference_images` | No | Up to 3 real project paths or public HTTPS URLs | +| `reference_video` | No | The exact `sandboxPath` from an earlier video result, or a public HTTPS URL | -Generate an image first, then use it as the starting frame. +`reference_images` and `reference_video` are mutually exclusive. Never invent a reference path, operation ID, or URL. -``` -type: "video" -prompt: "Panning wide shot of this calico kitten waking up and stretching in warm sunshine" -reference_images: ["/path/to/kitten-image.png"] +```yaml +prompt: "A cinematic tracking shot of a cyclist crossing a rain-soaked city street at night, neon reflections, slow dolly forward, distant traffic and rainfall." +aspect_ratio: "16:9" +duration: 8 ``` -### Video Extension +For extension, copy the exact `sandboxPath` returned by the earlier `generate_or_extend_video` result: -Extend a video you already generated with this tool. Copy `sandboxPath` from that run's JSON output — do **not** invent or guess a URI. - -``` -type: "video" -prompt: "The butterfly lands gently on an orange flower as a puppy runs up" -reference_video: ".cheatcode/assets/videos/a-fluffy-orange-tabby-cat-sitting-on-a-sunny-win-50ad8b35.mp4" +```yaml +prompt: "The butterfly lands on an orange flower as the camera slowly pushes in. Preserve the established visual style and ambience." +reference_video: ".cheatcode/assets/videos/butterfly-50ad8b35.mp4" duration: 8 ``` -Where to get `reference_video`: -- **Always prefer** the `sandboxPath` field from the previous generate_or_edit_media result (paths look like `.cheatcode/assets/videos/-.mp4`). -- A public **HTTPS URL** to the same file also works when you have one. -- **Do not** use placeholder examples, Vertex operation IDs, or `projects/.../operations/...` strings unless that exact value was returned by a prior tool call on your stack. +## Prompt construction -Constraints for video extension: -- Source must be a video this tool (or a compatible prior generation) already produced — not an arbitrary upload unless readable at the path/URL you provide -- Maximum input length: 141 seconds -- Cannot combine `reference_video` with `reference_images` +Order prompts as: **scene and backdrop → subject → action or key details → style and camera → constraints**. -### Reference Images for Video +- For photographs, name lighting, framing, lens feel, material detail, and depth of field. +- For illustrations, name the art style, palette, line quality, shading, and background. +- For product visuals, name the material, camera angle, lighting setup, and whether text, logos, or watermarks are allowed. +- For rendered text, provide the exact copy, typography direction, and placement. +- For video, include action, camera movement, pacing, dialogue in quotes, sound effects, and ambience. +- Prefer concrete positive descriptions. State edit invariants explicitly. -Veo supports up to 3 reference images to guide video content (character appearance, product identity). +## Execution contract -``` -type: "video" -prompt: "A woman in a flamingo feather dress walks through shallow turquoise lagoon water. Cinematic, dreamlike atmosphere." -reference_images: ["/path/to/dress.png", "/path/to/woman.png", "/path/to/sunglasses.png"] -``` +1. Call the selected generation tool once with only its supported fields. +2. If the tool reports a field-level validation issue, correct that specific field once. +3. If the provider rejects, times out, or fails the request, explain the provider failure. Do not resubmit the same request repeatedly. +4. Do not substitute a hand-authored SVG, canvas drawing, placeholder, or unrelated asset unless the user explicitly requests that output. +5. A successful result is already published, shown in chat, and available in Files. Refer to the finished asset naturally; do not open its sandbox path in the browser or publish it again. -**Important**: Video reference images are capped at 3 (unlike images which allow up to 8). - -## Best Practices - -1. **Be specific over generic**: Instead of "fantasy armor," write "ornate elven plate armor, etched with silver leaf patterns, with a high collar and pauldrons shaped like falcon wings." -2. **State intent**: "Create a logo for a high-end minimalist skincare brand" beats "Create a logo." -3. **Iterate with single changes**: Don't rewrite the whole prompt. Follow up with "Keep everything the same, but make the lighting warmer." -4. **Use step-by-step for complex scenes**: "First, a misty forest at dawn. Then, a moss-covered stone altar in the foreground. Finally, a glowing sword on the altar." -5. **Positive framing over negation**: Instead of "no cars," write "an empty, deserted street with no signs of traffic." -6. **Camera language for composition**: Use terms like `wide-angle shot`, `macro shot`, `low-angle perspective`, `dolly shot`, `aerial view`. -7. **For edits, state invariants explicitly**: "Change only the background; keep the product, edges, and shadows unchanged." -8. **Re-state invariants on each iteration**: Gemini can drift on follow-ups; repeat what must stay the same. -9. **Finish from the published asset**: A successful result is already shown to the user in the chat and can be opened in Files. Refer to that preview naturally; do not try to open the sandbox path in the browser and do not say the asset cannot be previewed. - -## Common Mistakes - -- Setting `image_reference_mode: "edit"` when you just want style inspiration (use default `reference_generate` instead) -- Providing `reference_images` alongside `reference_video` (mutually exclusive) -- Using placeholder or made-up `reference_video` values (e.g. `projects/xxx/.../operations/xxx`) instead of the actual `sandboxPath` from the last video result -- Pointing `reference_video` at a path that does not exist in the sandbox (read the path from the tool output; do not fabricate it) -- Providing too many references for video (max 3 for video, max 8 for images) -- Using `duration` for image generation (video-only parameter) -- Forgetting to specify `type` — always required -- Trying to preview the returned sandbox path in the browser. Local project paths are not browser URLs, and the product already renders the published asset in chat and Files. +## Aspect-ratio guide + +| Use case | Ratio | +|---|---| +| Square post, avatar, icon | `1:1` | +| Story or mobile portrait | `9:16` | +| Desktop hero or landscape video | `16:9` | +| Portrait product photo | `4:5` or `3:4` | +| Ultra-wide banner | `21:9` | +| Standard landscape photo | `3:2` | diff --git a/skills/xlsx/SKILL.md b/skills/xlsx/SKILL.md index 546a324c..53eabc28 100644 --- a/skills/xlsx/SKILL.md +++ b/skills/xlsx/SKILL.md @@ -4,12 +4,31 @@ description: "Use this skill any time a spreadsheet file is the primary input or category: Data & Media tags: xlsx, excel, spreadsheet, formulas license: Proprietary. LICENSE.txt has complete terms -compatibility: Requires the snapshot-bundled Python Office scripts, OOXML schemas, openpyxl, and LibreOffice Calc. +compatibility: Routine workbooks use docs_generate_xlsx. Complex or existing-workbook operations use the snapshot-bundled Python Office scripts, OOXML schemas, openpyxl, and LibreOffice Calc. --- # Requirements for Outputs -Before using this skill, always install the required dependencies first or verify they are already installed. +The sandbox snapshot already contains the supported spreadsheet dependencies. Do not install packages or run dependency probes. + +## Choose the production path + +Use `docs_generate_xlsx` exactly once for a new routine workbook made from bounded tables, simple workbook-local formulas, and standard number formats. The tool creates and publishes the finished XLSX, so do not write a script, recalculate it, search for it, publish it again, or open a browser. + +Use the custom Python/LibreOffice path only when the user needs an existing workbook edited, a template preserved, charts or macros, non-standard formatting, complex financial modeling, or high-stakes formula verification. + +Typed formula cells for `docs_generate_xlsx` have this shape: + +```json +{ + "type": "formula", + "formula": "=SUM(B2:B5)", + "result": 700, + "numberFormat": "integer" +} +``` + +`result` is the expected cached display value; the workbook still contains the editable Excel formula. Supported number formats are `general`, `integer`, `decimal`, `percent`, and `currency`. ## All Excel files @@ -76,7 +95,7 @@ A user may ask you to create, edit, or analyze the contents of an .xlsx file. Yo ## Important Requirements -**LibreOffice Required for Formula Recalculation**: You can assume LibreOffice is installed for recalculating formula values using the `scripts/recalc.py` script. The script automatically configures LibreOffice on first run, including in sandboxed environments where Unix sockets are restricted (handled by `scripts/office/soffice.py`) +**LibreOffice is required only for custom formula workbooks**: It is installed for recalculating formula values using `scripts/recalc.py`. The script configures LibreOffice on first run, including in sandboxed environments where Unix sockets are restricted. ## Reading and analyzing data @@ -134,14 +153,14 @@ sheet['D20'] = '=AVERAGE(D2:D19)' This applies to ALL calculations - totals, percentages, ratios, differences, etc. The spreadsheet should be able to recalculate when source data changes. -## Common Workflow +## Custom Workflow 1. **Choose tool**: pandas for data, openpyxl for formulas/formatting -2. **Create/Load**: Create new workbook or load existing file +2. **Create/Load**: Create a custom workbook or load an existing file 3. **Modify**: Add/edit data, formulas, and formatting -4. **Save**: Write to file -5. **Recalculate formulas (MANDATORY IF USING FORMULAS)**: Use the scripts/recalc.py script +4. **Save safely**: Generate or recalculate a ZIP-based workbook under `/tmp`, then copy the completed file into the project. Do not finalize XLSX bytes directly on the project mount. +5. **Recalculate custom formulas**: Use `scripts/recalc.py` ```bash - python scripts/recalc.py output.xlsx + python3 scripts/recalc.py /tmp/output.xlsx ``` 6. **Verify and fix any errors**: - The script returns JSON with error details @@ -179,7 +198,7 @@ sheet['A1'].alignment = Alignment(horizontal='center') # Column width sheet.column_dimensions['A'].width = 20 -wb.save('output.xlsx') +wb.save('/tmp/output.xlsx') ``` ### Editing existing Excel files @@ -206,7 +225,7 @@ sheet.delete_cols(3) # Delete column 3 new_sheet = wb.create_sheet('NewSheet') new_sheet['A1'] = 'Data' -wb.save('modified.xlsx') +wb.save('/tmp/modified.xlsx') ``` ## Recalculating formulas @@ -214,12 +233,12 @@ wb.save('modified.xlsx') Excel files created or modified by openpyxl contain formulas as strings but not calculated values. Use the provided `scripts/recalc.py` script to recalculate formulas: ```bash -python scripts/recalc.py [timeout_seconds] +python3 scripts/recalc.py [timeout_seconds] ``` Example: ```bash -python scripts/recalc.py output.xlsx 30 +python3 scripts/recalc.py /tmp/output.xlsx 30 ``` The script: