diff --git a/apps/web/src/components/chat/message-deliverables.tsx b/apps/web/src/components/chat/message-deliverables.tsx index 8a721e63..8f1eb0e0 100644 --- a/apps/web/src/components/chat/message-deliverables.tsx +++ b/apps/web/src/components/chat/message-deliverables.tsx @@ -11,6 +11,8 @@ import { type OutputImagePreviewState, useLazyOutputImagePreview, } from "@/components/chat/output-image-preview"; +import type { OutputPreviewState } from "@/components/chat/output-preview-support"; +import { useLazyOutputVideoPreview } from "@/components/chat/output-video-preview"; import { Code, Download, @@ -35,7 +37,7 @@ interface DeliverablesBlockProps { } export function DeliverablesBlock({ items, threadId }: DeliverablesBlockProps) { - useAutoOpenLatestImage(items, threadId); + useAutoOpenLatestMedia(items, threadId); return (
@@ -65,9 +67,99 @@ function DeliverableCard({ data, threadId }: { data: ArtifactData; threadId: str /> ); } + if (isVideoArtifact(data)) { + return ( + + ); + } return ; } +function VideoDeliverableCard({ + data, + download, + getToken, + threadId, +}: { + data: ArtifactData; + download: DeliverableDownload; + getToken: () => Promise; + threadId: string; +}) { + const preview = useLazyOutputVideoPreview(data, getToken); + const openInFiles = useOpenMediaInFiles(data, threadId, "video"); + return ( +
+ +
+
+
+ ); +} + +function VideoPreview({ data, preview }: { data: ArtifactData; preview: OutputPreviewState }) { + return ( +
+ {preview.url ? ( + // Generated assets do not include a timed-text sidecar to attach here. + // biome-ignore lint/a11y/useMediaCaption: No caption track exists for arbitrary provider output. +
+ ); +} + +function VideoPreviewPlaceholder({ + message, + status, +}: Pick) { + const isLoading = status === "idle" || status === "loading"; + return ( +
+ {isLoading ? ( +
+ ); +} + function FileDeliverableCard({ data, download, @@ -98,7 +190,7 @@ function ImageDeliverableCard({ }) { const [isViewerOpen, setViewerOpen] = useState(false); const preview = useLazyOutputImagePreview(data, getToken); - const openInFiles = useOpenImageInFiles(data, threadId, () => setViewerOpen(false)); + const openInFiles = useOpenMediaInFiles(data, threadId, "image", () => setViewerOpen(false)); return (
setViewerOpen(true)} preview={preview} /> @@ -349,14 +441,19 @@ function useDeliverableDownload( return { isPreparing, start }; } -function useOpenImageInFiles(data: ArtifactData, threadId: string, afterOpen: () => void) { +function useOpenMediaInFiles( + data: ArtifactData, + threadId: string, + type: "image" | "video", + afterOpen: () => void = () => undefined, +) { const requestFileOpen = useAppStore((state) => state.requestFileOpen); const setActiveComputerTab = useAppStore((state) => state.setActiveComputerTab); const setPreviewPanelOpen = useAppStore((state) => state.setPreviewPanelOpen); return () => { - const path = imageWorkspacePath(data); + const path = mediaWorkspacePath(data, type); if (!path) { - toast.error("This image does not have a safe workspace path."); + toast.error(`This ${type} does not have a safe workspace path.`); return; } requestFileOpen(threadId, path); @@ -366,38 +463,49 @@ function useOpenImageInFiles(data: ArtifactData, threadId: string, afterOpen: () }; } -function useAutoOpenLatestImage(items: readonly ArtifactData[], threadId: string): void { +function useAutoOpenLatestMedia(items: readonly ArtifactData[], threadId: string): void { const activeComputerTab = useAppStore((state) => state.activeComputerTab); const previewPanelOpen = useAppStore((state) => state.previewPanelOpen); const requestFileOpen = useAppStore((state) => state.requestFileOpen); - const latestImage = items.findLast(isImageArtifact); - const latestImagePath = latestImage ? imageWorkspacePath(latestImage) : null; + const latestMedia = items.findLast((item) => isImageArtifact(item) || isVideoArtifact(item)); + const latestMediaPath = latestMedia + ? mediaWorkspacePath(latestMedia, isVideoArtifact(latestMedia) ? "video" : "image") + : null; useEffect(() => { - if (!previewPanelOpen || activeComputerTab !== "files" || !latestImagePath) return; - requestFileOpen(threadId, latestImagePath); - }, [activeComputerTab, latestImagePath, previewPanelOpen, requestFileOpen, threadId]); + if (!previewPanelOpen || activeComputerTab !== "files" || !latestMediaPath) return; + requestFileOpen(threadId, latestMediaPath); + }, [activeComputerTab, latestMediaPath, previewPanelOpen, requestFileOpen, threadId]); } -function imageWorkspacePath(data: ArtifactData): string | null { - if (!isImageArtifact(data) || /[/\\]/u.test(data.filename) || data.filename.includes("..")) { +function mediaWorkspacePath(data: ArtifactData, type: "image" | "video"): string | null { + const matchesType = type === "image" ? isImageArtifact(data) : isVideoArtifact(data); + if (!matchesType || /[/\\]/u.test(data.filename) || data.filename.includes("..")) { return null; } - return `.cheatcode/assets/images/${data.filename}`; + return `.cheatcode/assets/${type === "image" ? "images" : "videos"}/${data.filename}`; } function imageAlt(filename: string): string { - const readable = filename + const readable = readableFilename(filename); + return readable ? `Generated image: ${readable}` : "Generated image"; +} + +function readableFilename(filename: string): string { + return filename .replace(/\.[^.]+$/u, "") .replace(/[-_]+/gu, " ") .replace(/\s+/gu, " ") .trim(); - return readable ? `Generated image: ${readable}` : "Generated image"; } function isImageArtifact(data: ArtifactData): boolean { return data.kind === "image" || data.mimeType.startsWith("image/"); } +function isVideoArtifact(data: ArtifactData): boolean { + return data.kind === "video" || data.mimeType.startsWith("video/"); +} + function deliverableIcon(kind: ArtifactData["kind"], mimeType: string) { if (kind === "slide") return Presentation; if (kind === "xlsx") return FileSpreadsheet; diff --git a/apps/web/src/components/chat/output-image-preview.ts b/apps/web/src/components/chat/output-image-preview.ts index 59a1dd37..6f32207b 100644 --- a/apps/web/src/components/chat/output-image-preview.ts +++ b/apps/web/src/components/chat/output-image-preview.ts @@ -1,22 +1,17 @@ "use client"; -import { type RefObject, useEffect, useRef, useState } from "react"; +import { useEffect, useRef, useState } from "react"; +import { + type OutputPreviewIdentity, + type OutputPreviewState, + useNearViewport, +} from "@/components/chat/output-preview-support"; import { loadOutputImagePreview } from "@/lib/api/outputs"; -export interface OutputImagePreviewState { - hostRef: RefObject; - message: string | null; - status: "error" | "idle" | "loading" | "ready"; - url: string | null; -} - -interface OutputImageIdentity { - outputId: string; - sizeBytes: number; -} +export type OutputImagePreviewState = OutputPreviewState; export function useLazyOutputImagePreview( - data: OutputImageIdentity, + data: OutputPreviewIdentity, getToken: () => Promise, ): OutputImagePreviewState { const hostRef = useRef(null); @@ -63,25 +58,3 @@ export function useLazyOutputImagePreview( return { ...preview, hostRef }; } - -function useNearViewport(ref: RefObject): boolean { - const [isNear, setNear] = useState(false); - useEffect(() => { - const element = ref.current; - if (!element || typeof IntersectionObserver === "undefined") { - setNear(true); - return; - } - const observer = new IntersectionObserver( - ([entry]) => { - if (!entry?.isIntersecting) return; - setNear(true); - observer.disconnect(); - }, - { rootMargin: "320px" }, - ); - observer.observe(element); - return () => observer.disconnect(); - }, [ref]); - return isNear; -} diff --git a/apps/web/src/components/chat/output-preview-support.ts b/apps/web/src/components/chat/output-preview-support.ts new file mode 100644 index 00000000..03c6f2de --- /dev/null +++ b/apps/web/src/components/chat/output-preview-support.ts @@ -0,0 +1,37 @@ +"use client"; + +import { type RefObject, useEffect, useState } from "react"; + +export interface OutputPreviewIdentity { + outputId: string; + sizeBytes: number; +} + +export interface OutputPreviewState { + hostRef: RefObject; + message: string | null; + status: "error" | "idle" | "loading" | "ready"; + url: string | null; +} + +export function useNearViewport(ref: RefObject): boolean { + const [isNear, setNear] = useState(false); + useEffect(() => { + const element = ref.current; + if (!element || typeof IntersectionObserver === "undefined") { + setNear(true); + return; + } + const observer = new IntersectionObserver( + ([entry]) => { + if (!entry?.isIntersecting) return; + setNear(true); + observer.disconnect(); + }, + { rootMargin: "320px" }, + ); + observer.observe(element); + return () => observer.disconnect(); + }, [ref]); + return isNear; +} diff --git a/apps/web/src/components/chat/output-video-preview.ts b/apps/web/src/components/chat/output-video-preview.ts new file mode 100644 index 00000000..c6f5f3a2 --- /dev/null +++ b/apps/web/src/components/chat/output-video-preview.ts @@ -0,0 +1,48 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import { + type OutputPreviewIdentity, + type OutputPreviewState, + useNearViewport, +} from "@/components/chat/output-preview-support"; +import { createOutputDownloadUrl } from "@/lib/api/outputs"; + +export function useLazyOutputVideoPreview( + data: OutputPreviewIdentity, + getToken: () => Promise, +): OutputPreviewState { + const hostRef = useRef(null); + const getTokenRef = useRef(getToken); + const isNearViewport = useNearViewport(hostRef); + const [preview, setPreview] = useState>({ + message: null, + status: "idle", + url: null, + }); + + useEffect(() => { + getTokenRef.current = getToken; + }, [getToken]); + + useEffect(() => { + if (!isNearViewport) return; + const controller = new AbortController(); + setPreview({ message: null, status: "loading", url: null }); + void createOutputDownloadUrl(() => getTokenRef.current(), data.outputId, controller.signal) + .then((capability) => { + setPreview({ message: null, status: "ready", url: capability.downloadUrl }); + }) + .catch((error: unknown) => { + if (controller.signal.aborted) return; + setPreview({ + message: error instanceof Error ? error.message : "Preview unavailable", + status: "error", + url: null, + }); + }); + return () => controller.abort(); + }, [data.outputId, isNearViewport]); + + return { ...preview, hostRef }; +} diff --git a/packages/agent-core/src/tools/media/execute.ts b/packages/agent-core/src/tools/media/execute.ts index a365b6e9..f6d9ce15 100644 --- a/packages/agent-core/src/tools/media/execute.ts +++ b/packages/agent-core/src/tools/media/execute.ts @@ -207,16 +207,24 @@ async function generateVideo( } function videoConfig(input: GenerateOrExtendVideoInput, references: MediaReference[]) { + if (input.reference_video) { + return { + numberOfVideos: 1, + resolution: "720p", + }; + } return { aspectRatio: input.aspect_ratio ?? "16:9", durationSeconds: input.duration ?? 8, - generateAudio: true, numberOfVideos: 1, - referenceImages: references.map((reference) => ({ - image: referenceImage(reference), - referenceType: VideoGenerationReferenceType.ASSET, - })), - resolution: "1080p", + ...(references.length > 0 + ? { + referenceImages: references.map((reference) => ({ + image: referenceImage(reference), + referenceType: VideoGenerationReferenceType.ASSET, + })), + } + : {}), }; } diff --git a/packages/agent-core/src/tools/media/schemas.ts b/packages/agent-core/src/tools/media/schemas.ts index 10dd2111..97ca11e1 100644 --- a/packages/agent-core/src/tools/media/schemas.ts +++ b/packages/agent-core/src/tools/media/schemas.ts @@ -34,6 +34,17 @@ export const GenerateOrExtendVideoInputSchema = z path: ["reference_video"], }); } + if ( + (input.reference_images?.length || input.reference_video) && + input.duration !== undefined && + input.duration !== 8 + ) { + context.addIssue({ + code: "custom", + message: "Reference-based video generation and extension require an 8-second duration.", + path: ["duration"], + }); + } }); const MediaArtifactShape = { diff --git a/skills/generate-media/SKILL.md b/skills/generate-media/SKILL.md index 61765eea..6035c13d 100644 --- a/skills/generate-media/SKILL.md +++ b/skills/generate-media/SKILL.md @@ -44,7 +44,7 @@ image_reference_mode: "edit" |---|---|---| | `prompt` | Yes | Subject, action, visual style, camera motion, and audio cues | | `aspect_ratio` | No | `16:9` or `9:16` | -| `duration` | No | `4`, `6`, or `8` seconds | +| `duration` | No | `4`, `6`, or `8` seconds; reference-image generation and extension require `8` | | `reference_images` | No | Up to 3 real project paths or public HTTPS URLs | | `reference_video` | No | The exact `sandboxPath` from an earlier video result, or a public HTTPS URL | @@ -73,6 +73,7 @@ Order prompts as: **scene and backdrop → subject → action or key details → - For product visuals, name the material, camera angle, lighting setup, and whether text, logos, or watermarks are allowed. - For rendered text, provide the exact copy, typography direction, and placement. - For video, include action, camera movement, pacing, dialogue in quotes, sound effects, and ambience. +- Veo generates synchronized audio natively from those prompt cues; do not add an audio configuration field. - Prefer concrete positive descriptions. State edit invariants explicitly. ## Execution contract