diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index ef93c84..cc10da9 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -2,13 +2,14 @@ import { type Message, convertToCoreMessages, createDataStreamResponse, + experimental_generateImage, streamObject, streamText, } from 'ai'; import { z } from 'zod'; import { auth } from '@/app/(auth)/auth'; -import { customModel } from '@/lib/ai'; +import { customModel, imageGenerationModel } from '@/lib/ai'; import { models } from '@/lib/ai/models'; import { codePrompt, @@ -124,10 +125,10 @@ export async function POST(request: Request) { }, createDocument: { description: - 'Create a document for a writing activity. This tool will call other functions that will generate the contents of the document based on the title and kind.', + 'Create a document for a writing or content creation activities like image generation. This tool will call other functions that will generate the contents of the document based on the title and kind.', parameters: z.object({ title: z.string(), - kind: z.enum(['text', 'code']), + kind: z.enum(['text', 'code', 'image']), }), execute: async ({ title, kind }) => { const id = generateUUID(); @@ -204,6 +205,21 @@ export async function POST(request: Request) { } } + dataStream.writeData({ type: 'finish', content: '' }); + } else if (kind === 'image') { + const { image } = await experimental_generateImage({ + model: imageGenerationModel, + prompt: title, + n: 1, + }); + + draftText = image.base64; + + dataStream.writeData({ + type: 'image-delta', + content: image.base64, + }); + dataStream.writeData({ type: 'finish', content: '' }); } @@ -309,6 +325,21 @@ export async function POST(request: Request) { } } + dataStream.writeData({ type: 'finish', content: '' }); + } else if (document.kind === 'image') { + const { image } = await experimental_generateImage({ + model: imageGenerationModel, + prompt: description, + n: 1, + }); + + draftText = image.base64; + + dataStream.writeData({ + type: 'image-delta', + content: image.base64, + }); + dataStream.writeData({ type: 'finish', content: '' }); } diff --git a/components/block-actions.tsx b/components/block-actions.tsx index de24d81..4cbb8d1 100644 --- a/components/block-actions.tsx +++ b/components/block-actions.tsx @@ -2,11 +2,11 @@ import { cn } from '@/lib/utils'; import { ClockRewind, CopyIcon, RedoIcon, UndoIcon } from './icons'; import { Button } from './ui/button'; import { Tooltip, TooltipContent, TooltipTrigger } from './ui/tooltip'; -import { useCopyToClipboard } from 'usehooks-ts'; import { toast } from 'sonner'; import { ConsoleOutput, UIBlock } from './block'; import { Dispatch, memo, SetStateAction } from 'react'; import { RunCodeButton } from './run-code-button'; +import { useMultimodalCopyToClipboard } from '@/hooks/use-multimodal-copy-to-clipboard'; interface BlockActionsProps { block: UIBlock; @@ -25,7 +25,8 @@ function PureBlockActions({ mode, setConsoleOutputs, }: BlockActionsProps) { - const [_, copyToClipboard] = useCopyToClipboard(); + const { copyTextToClipboard, copyImageToClipboard } = + useMultimodalCopyToClipboard(); return (
@@ -96,7 +97,12 @@ function PureBlockActions({ variant="outline" className="p-2 h-fit dark:hover:bg-zinc-700" onClick={() => { - copyToClipboard(block.content); + if (block.kind === 'image') { + copyImageToClipboard(block.content); + } else { + copyTextToClipboard(block.content); + } + toast.success('Copied to clipboard!'); }} disabled={block.status === 'streaming'} diff --git a/components/block.tsx b/components/block.tsx index 7dc5b53..dd08654 100644 --- a/components/block.tsx +++ b/components/block.tsx @@ -34,8 +34,9 @@ import { Console } from './console'; import { useSidebar } from './ui/sidebar'; import { useBlock } from '@/hooks/use-block'; import equal from 'fast-deep-equal'; +import { ImageEditor } from './image-editor'; -export type BlockKind = 'text' | 'code'; +export type BlockKind = 'text' | 'code' | 'image'; export interface UIBlock { title: string; @@ -476,7 +477,7 @@ function PureBlock({ })} > {isDocumentsFetching && !block.content ? ( - + ) : block.kind === 'code' ? ( ) + ) : block.kind === 'image' ? ( + ) : null} - {suggestions ? ( + {suggestions && suggestions.length > 0 ? (
) : null} diff --git a/components/data-stream-handler.tsx b/components/data-stream-handler.tsx index 6c50b77..120416e 100644 --- a/components/data-stream-handler.tsx +++ b/components/data-stream-handler.tsx @@ -6,13 +6,13 @@ import { BlockKind } from './block'; import { Suggestion } from '@/lib/db/schema'; import { initialBlockData, useBlock } from '@/hooks/use-block'; import { useUserMessageId } from '@/hooks/use-user-message-id'; -import { cx } from 'class-variance-authority'; import { useSWRConfig } from 'swr'; type DataStreamDelta = { type: | 'text-delta' | 'code-delta' + | 'image-delta' | 'title' | 'id' | 'suggestion' @@ -107,6 +107,14 @@ export function DataStreamHandler({ id }: { id: string }) { status: 'streaming', }; + case 'image-delta': + return { + ...draftBlock, + content: delta.content as string, + isVisible: true, + status: 'streaming', + }; + case 'suggestion': setTimeout(() => { setOptimisticSuggestions((currentSuggestions) => [ diff --git a/components/document-preview.tsx b/components/document-preview.tsx index e0fe907..247ed08 100644 --- a/components/document-preview.tsx +++ b/components/document-preview.tsx @@ -8,8 +8,8 @@ import { useMemo, useRef, } from 'react'; -import { UIBlock } from './block'; -import { FileIcon, FullscreenIcon, LoaderIcon } from './icons'; +import { BlockKind, UIBlock } from './block'; +import { FileIcon, FullscreenIcon, ImageIcon, LoaderIcon } from './icons'; import { cn, fetcher } from '@/lib/utils'; import { Document } from '@/lib/db/schema'; import { InlineDocumentSkeleton } from './document-skeleton'; @@ -19,6 +19,7 @@ import { DocumentToolCall, DocumentToolResult } from './document'; import { CodeEditor } from './code-editor'; import { useBlock } from '@/hooks/use-block'; import equal from 'fast-deep-equal'; +import { ImageEditor } from './image-editor'; interface DocumentPreviewProps { isReadonly: boolean; @@ -78,7 +79,7 @@ export function DocumentPreview({ } if (isDocumentsFetching) { - return ; + return ; } const document: Document | null = previewDocument @@ -94,13 +95,14 @@ export function DocumentPreview({ } : null; - if (!document) return ; + if (!document) return ; return (
@@ -108,7 +110,7 @@ export function DocumentPreview({ ); } -const LoadingSkeleton = () => ( +const LoadingSkeleton = ({ blockKind }: { blockKind: BlockKind }) => (
@@ -121,9 +123,15 @@ const LoadingSkeleton = () => (
-
- -
+ {blockKind === 'image' ? ( +
+
+
+ ) : ( +
+ +
+ )}
); @@ -185,9 +193,11 @@ const HitboxLayer = memo(PureHitboxLayer, (prevProps, nextProps) => { const PureDocumentHeader = ({ title, + kind, isStreaming, }: { title: string; + kind: BlockKind; isStreaming: boolean; }) => (
@@ -197,6 +207,8 @@ const PureDocumentHeader = ({
+ ) : kind === 'image' ? ( + ) : ( )} @@ -244,6 +256,15 @@ const DocumentContent = ({ document }: { document: Document }) => {
+ ) : document.kind === 'image' ? ( + ) : null}
); diff --git a/components/document-skeleton.tsx b/components/document-skeleton.tsx index 7a45794..ae8ace9 100644 --- a/components/document-skeleton.tsx +++ b/components/document-skeleton.tsx @@ -1,7 +1,13 @@ 'use client'; -export const DocumentSkeleton = () => { - return ( +import { BlockKind } from './block'; + +export const DocumentSkeleton = ({ blockKind }: { blockKind: BlockKind }) => { + return blockKind === 'image' ? ( +
+
+
+ ) : (
diff --git a/components/image-editor.tsx b/components/image-editor.tsx new file mode 100644 index 0000000..7376fef --- /dev/null +++ b/components/image-editor.tsx @@ -0,0 +1,50 @@ +import { LoaderIcon } from './icons'; +import cn from 'classnames'; + +interface ImageEditorProps { + title: string; + content: string; + isCurrentVersion: boolean; + currentVersionIndex: number; + status: string; + isInline: boolean; +} + +export function ImageEditor({ + title, + content, + isCurrentVersion, + currentVersionIndex, + status, + isInline, +}: ImageEditorProps) { + return ( +
+ {status === 'streaming' ? ( +
+ {!isInline && ( +
+ +
+ )} +
Generating Image...
+
+ ) : ( + + {title} + + )} +
+ ); +} diff --git a/components/toolbar.tsx b/components/toolbar.tsx index 8cb9aee..64b5441 100644 --- a/components/toolbar.tsx +++ b/components/toolbar.tsx @@ -29,13 +29,12 @@ import { sanitizeUIMessages } from '@/lib/utils'; import { ArrowUpIcon, CodeIcon, - FileIcon, LogsIcon, MessageIcon, PenIcon, + SparklesIcon, StopIcon, SummarizeIcon, - TerminalIcon, } from './icons'; import { BlockKind } from './block'; @@ -327,6 +326,7 @@ const toolsByBlockKind: Record< icon: , }, ], + image: [], }; export const Tools = ({ @@ -347,7 +347,7 @@ export const Tools = ({ ) => Promise; isAnimating: boolean; setIsToolbarVisible: Dispatch>; - blockKind: 'text' | 'code'; + blockKind: BlockKind; }) => { const [primaryTool, ...secondaryTools] = toolsByBlockKind[blockKind]; @@ -407,7 +407,7 @@ const PureToolbar = ({ ) => Promise; stop: () => void; setMessages: Dispatch>; - blockKind: 'text' | 'code'; + blockKind: BlockKind; }) => { const toolbarRef = useRef(null); const timeoutRef = useRef>(); @@ -451,6 +451,10 @@ const PureToolbar = ({ } }, [isLoading, setIsToolbarVisible]); + if (toolsByBlockKind[blockKind].length === 0) { + return null; + } + return ( res.blob(), + ); + + const item = new ClipboardItem({ + 'image/png': blob, + }); + + await navigator.clipboard.write([item]); + } catch (error) { + console.error('Failed to copy image to clipboard:', error); + } +} + +export function useMultimodalCopyToClipboard() { + const [_, copyTextToClipboard] = useCopyToClipboard(); + return { copyTextToClipboard, copyImageToClipboard }; +} diff --git a/lib/ai/index.ts b/lib/ai/index.ts index e1bf2dd..851564a 100644 --- a/lib/ai/index.ts +++ b/lib/ai/index.ts @@ -9,3 +9,5 @@ export const customModel = (apiIdentifier: string) => { middleware: customMiddleware, }); }; + +export const imageGenerationModel = openai.image('dall-e-3'); diff --git a/lib/db/schema.ts b/lib/db/schema.ts index 712cc36..be5e80f 100644 --- a/lib/db/schema.ts +++ b/lib/db/schema.ts @@ -72,7 +72,7 @@ export const document = pgTable( createdAt: timestamp('createdAt').notNull(), title: text('title').notNull(), content: text('content'), - kind: varchar('text', { enum: ['text', 'code'] }) + kind: varchar('text', { enum: ['text', 'code', 'image'] }) .notNull() .default('text'), userId: uuid('userId') diff --git a/package.json b/package.json index 5f8f745..ce6d63d 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "db:up": "drizzle-kit up" }, "dependencies": { - "@ai-sdk/openai": "1.0.6", + "@ai-sdk/openai": "1.0.19", "@codemirror/lang-javascript": "^6.2.2", "@codemirror/lang-python": "^6.1.6", "@codemirror/state": "^6.5.0", @@ -37,7 +37,7 @@ "@vercel/analytics": "^1.3.1", "@vercel/blob": "^0.24.1", "@vercel/postgres": "^0.10.0", - "ai": "4.0.20", + "ai": "4.0.36", "bcrypt-ts": "^5.0.2", "class-variance-authority": "^0.7.0", "classnames": "^2.5.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f8a40e9..2718890 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@ai-sdk/openai': - specifier: 1.0.6 - version: 1.0.6(zod@3.23.8) + specifier: 1.0.19 + version: 1.0.19(zod@3.23.8) '@codemirror/lang-javascript': specifier: ^6.2.2 version: 6.2.2 @@ -66,8 +66,8 @@ importers: specifier: ^0.10.0 version: 0.10.0 ai: - specifier: 4.0.20 - version: 4.0.20(react@19.0.0-rc-45804af1-20241021)(zod@3.23.8) + specifier: 4.0.36 + version: 4.0.36(react@19.0.0-rc-45804af1-20241021)(zod@3.23.8) bcrypt-ts: specifier: ^5.0.2 version: 5.0.2 @@ -240,14 +240,14 @@ importers: packages: - '@ai-sdk/openai@1.0.6': - resolution: {integrity: sha512-AhNILXn/hVD91mrokg9Wph78BiWNP/N0tJiua+UCS5TYXCbSeeqKqiodRiNxw6tDA5sGnrC6yCv8TjgY1CwzWg==} + '@ai-sdk/openai@1.0.19': + resolution: {integrity: sha512-7qmLgppWpGUhSgrH0a6CtgD9hZeRh2hARppl1B7fNhVbekYftSMucsdCiVlKbQzSKPxox0vkNMmwjKa/7xf8bQ==} engines: {node: '>=18'} peerDependencies: zod: ^3.0.0 - '@ai-sdk/provider-utils@2.0.2': - resolution: {integrity: sha512-IAvhKhdlXqiSmvx/D4uNlFYCl8dWT+M9K+IuEcSgnE2Aj27GWu8sDIpAf4r4Voc+wOUkOECVKQhFo8g9pozdjA==} + '@ai-sdk/provider-utils@2.0.7': + resolution: {integrity: sha512-4sfPlKEALHPXLmMFcPlYksst3sWBJXmCDZpIBJisRrmwGG6Nn3mq0N1Zu/nZaGcrWZoOY+HT2Wbxla1oTElYHQ==} engines: {node: '>=18'} peerDependencies: zod: ^3.0.0 @@ -255,25 +255,12 @@ packages: zod: optional: true - '@ai-sdk/provider-utils@2.0.4': - resolution: {integrity: sha512-GMhcQCZbwM6RoZCri0MWeEWXRt/T+uCxsmHEsTwNvEH3GDjNzchfX25C8ftry2MeEOOn6KfqCLSKomcgK6RoOg==} - engines: {node: '>=18'} - peerDependencies: - zod: ^3.0.0 - peerDependenciesMeta: - zod: - optional: true - - '@ai-sdk/provider@1.0.1': - resolution: {integrity: sha512-mV+3iNDkzUsZ0pR2jG0sVzU6xtQY5DtSCBy3JFycLp6PwjyLw/iodfL3MwdmMCRJWgs3dadcHejRnMvF9nGTBg==} + '@ai-sdk/provider@1.0.4': + resolution: {integrity: sha512-lJi5zwDosvvZER3e/pB8lj1MN3o3S7zJliQq56BRr4e9V3fcRyFtwP0JRxaRS5vHYX3OJ154VezVoQNrk0eaKw==} engines: {node: '>=18'} - '@ai-sdk/provider@1.0.2': - resolution: {integrity: sha512-YYtP6xWQyaAf5LiWLJ+ycGTOeBLWrED7LUrvc+SQIWhGaneylqbaGsyQL7VouQUeQ4JZ1qKYZuhmi3W56HADPA==} - engines: {node: '>=18'} - - '@ai-sdk/react@1.0.6': - resolution: {integrity: sha512-8Hkserq0Ge6AEi7N4hlv2FkfglAGbkoAXEZ8YSp255c3PbnZz6+/5fppw+aROmZMOfNwallSRuy1i/iPa2rBpQ==} + '@ai-sdk/react@1.0.11': + resolution: {integrity: sha512-ndBPA7dx2DqUr7s4zO1cRAPkFGS+wWvSri6OWfCuhfyTAADQ4vdd56vFP9zdTZl4cyL27Vh0hKLfFJMGx83MUQ==} engines: {node: '>=18'} peerDependencies: react: ^18 || ^19 || ^19.0.0-rc @@ -284,8 +271,8 @@ packages: zod: optional: true - '@ai-sdk/ui-utils@1.0.5': - resolution: {integrity: sha512-DGJSbDf+vJyWmFNexSPUsS1AAy7gtsmFmoSyNbNbJjwl9hRIf2dknfA1V0ahx6pg3NNklNYFm53L8Nphjovfvg==} + '@ai-sdk/ui-utils@1.0.10': + resolution: {integrity: sha512-wZfZNH2IloTx5b1O8CU7/R/icm8EsmURElPckYwNYj2YZrKk9X5XeYSDBF/1/J83obzsn0i7VKkIf40qhRzVVA==} engines: {node: '>=18'} peerDependencies: zod: ^3.0.0 @@ -1626,8 +1613,8 @@ packages: engines: {node: '>=0.4.0'} hasBin: true - ai@4.0.20: - resolution: {integrity: sha512-dYevYKtREcjSVopBDFWVNca7WJEI1p9Vr9eo7V7fZHzi2vXGDyEa2WYatjFbpR6z6gpVAxKHsof8EoN+B1IAsA==} + ai@4.0.36: + resolution: {integrity: sha512-KeXQe+eKQK57WYzN+K6VUBZqj/n70V+cCwrXAhXa7enbZCAsErvQKqrnxkF1qm9UfPuNKNEFV5BRoMKJ5gRfAw==} engines: {node: '>=18'} peerDependencies: react: ^18 || ^19 || ^19.0.0-rc @@ -3738,10 +3725,10 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - zod-to-json-schema@3.23.5: - resolution: {integrity: sha512-5wlSS0bXfF/BrL4jPAbz9da5hDlDptdEppYfe+x4eIJ7jioqKG9uUxOwPzqof09u/XeVdrgFu29lZi+8XNDJtA==} + zod-to-json-schema@3.24.1: + resolution: {integrity: sha512-3h08nf3Vw3Wl3PK+q3ow/lIil81IT2Oa7YpQyUUDsEWbXveMesdfK1xBd2RhCkynwZndAxixji/7SYJJowr62w==} peerDependencies: - zod: ^3.23.3 + zod: ^3.24.1 zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} @@ -3751,53 +3738,40 @@ packages: snapshots: - '@ai-sdk/openai@1.0.6(zod@3.23.8)': + '@ai-sdk/openai@1.0.19(zod@3.23.8)': dependencies: - '@ai-sdk/provider': 1.0.1 - '@ai-sdk/provider-utils': 2.0.2(zod@3.23.8) + '@ai-sdk/provider': 1.0.4 + '@ai-sdk/provider-utils': 2.0.7(zod@3.23.8) zod: 3.23.8 - '@ai-sdk/provider-utils@2.0.2(zod@3.23.8)': + '@ai-sdk/provider-utils@2.0.7(zod@3.23.8)': dependencies: - '@ai-sdk/provider': 1.0.1 - eventsource-parser: 3.0.0 - nanoid: 3.3.7 - secure-json-parse: 2.7.0 - optionalDependencies: - zod: 3.23.8 - - '@ai-sdk/provider-utils@2.0.4(zod@3.23.8)': - dependencies: - '@ai-sdk/provider': 1.0.2 + '@ai-sdk/provider': 1.0.4 eventsource-parser: 3.0.0 nanoid: 3.3.8 secure-json-parse: 2.7.0 optionalDependencies: zod: 3.23.8 - '@ai-sdk/provider@1.0.1': + '@ai-sdk/provider@1.0.4': dependencies: json-schema: 0.4.0 - '@ai-sdk/provider@1.0.2': + '@ai-sdk/react@1.0.11(react@19.0.0-rc-45804af1-20241021)(zod@3.23.8)': dependencies: - json-schema: 0.4.0 - - '@ai-sdk/react@1.0.6(react@19.0.0-rc-45804af1-20241021)(zod@3.23.8)': - dependencies: - '@ai-sdk/provider-utils': 2.0.4(zod@3.23.8) - '@ai-sdk/ui-utils': 1.0.5(zod@3.23.8) + '@ai-sdk/provider-utils': 2.0.7(zod@3.23.8) + '@ai-sdk/ui-utils': 1.0.10(zod@3.23.8) swr: 2.2.5(react@19.0.0-rc-45804af1-20241021) throttleit: 2.1.0 optionalDependencies: react: 19.0.0-rc-45804af1-20241021 zod: 3.23.8 - '@ai-sdk/ui-utils@1.0.5(zod@3.23.8)': + '@ai-sdk/ui-utils@1.0.10(zod@3.23.8)': dependencies: - '@ai-sdk/provider': 1.0.2 - '@ai-sdk/provider-utils': 2.0.4(zod@3.23.8) - zod-to-json-schema: 3.23.5(zod@3.23.8) + '@ai-sdk/provider': 1.0.4 + '@ai-sdk/provider-utils': 2.0.7(zod@3.23.8) + zod-to-json-schema: 3.24.1(zod@3.23.8) optionalDependencies: zod: 3.23.8 @@ -4896,15 +4870,14 @@ snapshots: acorn@8.14.0: {} - ai@4.0.20(react@19.0.0-rc-45804af1-20241021)(zod@3.23.8): + ai@4.0.36(react@19.0.0-rc-45804af1-20241021)(zod@3.23.8): dependencies: - '@ai-sdk/provider': 1.0.2 - '@ai-sdk/provider-utils': 2.0.4(zod@3.23.8) - '@ai-sdk/react': 1.0.6(react@19.0.0-rc-45804af1-20241021)(zod@3.23.8) - '@ai-sdk/ui-utils': 1.0.5(zod@3.23.8) + '@ai-sdk/provider': 1.0.4 + '@ai-sdk/provider-utils': 2.0.7(zod@3.23.8) + '@ai-sdk/react': 1.0.11(react@19.0.0-rc-45804af1-20241021)(zod@3.23.8) + '@ai-sdk/ui-utils': 1.0.10(zod@3.23.8) '@opentelemetry/api': 1.9.0 jsondiffpatch: 0.6.0 - zod-to-json-schema: 3.23.5(zod@3.23.8) optionalDependencies: react: 19.0.0-rc-45804af1-20241021 zod: 3.23.8 @@ -7499,7 +7472,7 @@ snapshots: yocto-queue@0.1.0: {} - zod-to-json-schema@3.23.5(zod@3.23.8): + zod-to-json-schema@3.24.1(zod@3.23.8): dependencies: zod: 3.23.8