From 9dd9a9898c0995bd14a6a154f0e54a33f48b1f69 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 4 Mar 2025 17:25:46 -0800 Subject: [PATCH] feat: add tests (#843) --- .github/workflows/playwright.yml | 72 +++++++ .gitignore | 7 + app/(auth)/login/page.tsx | 12 +- app/(auth)/register/page.tsx | 14 +- app/(chat)/actions.ts | 2 +- app/(chat)/api/chat/route.ts | 185 ++++++++-------- artifacts/code/server.ts | 2 +- artifacts/image/server.ts | 2 +- artifacts/sheet/server.ts | 2 +- artifacts/text/server.ts | 2 +- components/artifact-messages.tsx | 1 + components/console.tsx | 2 +- components/icons.tsx | 19 ++ components/message-editor.tsx | 2 + components/message-reasoning.tsx | 5 +- components/message.tsx | 20 +- components/messages.tsx | 1 + components/multimodal-input.tsx | 9 +- components/preview-attachment.tsx | 7 +- components/suggested-actions.tsx | 5 +- components/toast.tsx | 44 ++++ docs/03-artifacts.md | 2 +- lib/ai/models.test.ts | 226 ++++++++++++++++++++ lib/ai/models.ts | 25 --- lib/ai/providers.ts | 41 ++++ lib/ai/tools/request-suggestions.ts | 2 +- lib/constants.ts | 7 + package.json | 8 +- playwright.config.ts | 105 +++++++++ pnpm-lock.yaml | 125 +++++++---- public/images/mouth of the seine, monet.jpg | Bin 0 -> 33497 bytes tailwind.config.ts | 3 + tests/auth.test.ts | 77 +++++++ tests/chat.test.ts | 194 +++++++++++++++++ tests/global.setup.ts | 24 +++ 35 files changed, 1063 insertions(+), 191 deletions(-) create mode 100644 .github/workflows/playwright.yml create mode 100644 components/toast.tsx create mode 100644 lib/ai/models.test.ts create mode 100644 lib/ai/providers.ts create mode 100644 lib/constants.ts create mode 100644 playwright.config.ts create mode 100644 public/images/mouth of the seine, monet.jpg create mode 100644 tests/auth.test.ts create mode 100644 tests/chat.test.ts create mode 100644 tests/global.setup.ts diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 0000000..9bbae17 --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,72 @@ +name: Playwright Tests +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +jobs: + test: + timeout-minutes: 30 + runs-on: ubuntu-latest + env: + AUTH_SECRET: ${{ secrets.AUTH_SECRET }} + POSTGRES_URL: ${{ secrets.POSTGRES_URL }} + BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - uses: actions/setup-node@v4 + with: + node-version: lts/* + + - name: Install pnpm + uses: pnpm/action-setup@v2 + with: + version: latest + run_install: false + + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + + - uses: actions/cache@v3 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - uses: actions/setup-node@v4 + with: + node-version: lts/* + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Cache Playwright browsers + uses: actions/cache@v3 + id: playwright-cache + with: + path: ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ hashFiles('**/pnpm-lock.yaml') }} + + - name: Install Playwright Browsers + if: steps.playwright-cache.outputs.cache-hit != 'true' + run: pnpm exec playwright install --with-deps chromium + + - name: Run Playwright tests + run: pnpm test + + - uses: actions/upload-artifact@v4 + if: always() && !cancelled() + with: + name: playwright-report + path: playwright-report/ + retention-days: 7 diff --git a/.gitignore b/.gitignore index dd019e4..7ece070 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,10 @@ yarn-error.log* .vercel .vscode .env*.local + +# Playwright +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ +/playwright/.auth/ diff --git a/app/(auth)/login/page.tsx b/app/(auth)/login/page.tsx index 7ff4c73..4b44103 100644 --- a/app/(auth)/login/page.tsx +++ b/app/(auth)/login/page.tsx @@ -3,7 +3,7 @@ import Link from 'next/link'; import { useRouter } from 'next/navigation'; import { useActionState, useEffect, useState } from 'react'; -import { toast } from 'sonner'; +import { toast } from '@/components/toast'; import { AuthForm } from '@/components/auth-form'; import { SubmitButton } from '@/components/submit-button'; @@ -25,9 +25,15 @@ export default function Page() { useEffect(() => { if (state.status === 'failed') { - toast.error('Invalid credentials!'); + toast({ + type: 'error', + description: 'Invalid credentials!', + }); } else if (state.status === 'invalid_data') { - toast.error('Failed validating your submission!'); + toast({ + type: 'error', + description: 'Failed validating your submission!', + }); } else if (state.status === 'success') { setIsSuccessful(true); router.refresh(); diff --git a/app/(auth)/register/page.tsx b/app/(auth)/register/page.tsx index 69e079b..224f0c0 100644 --- a/app/(auth)/register/page.tsx +++ b/app/(auth)/register/page.tsx @@ -3,12 +3,12 @@ import Link from 'next/link'; import { useRouter } from 'next/navigation'; import { useActionState, useEffect, useState } from 'react'; -import { toast } from 'sonner'; import { AuthForm } from '@/components/auth-form'; import { SubmitButton } from '@/components/submit-button'; import { register, type RegisterActionState } from '../actions'; +import { toast } from '@/components/toast'; export default function Page() { const router = useRouter(); @@ -25,13 +25,17 @@ export default function Page() { useEffect(() => { if (state.status === 'user_exists') { - toast.error('Account already exists'); + toast({ type: 'error', description: 'Account already exists!' }); } else if (state.status === 'failed') { - toast.error('Failed to create account'); + toast({ type: 'error', description: 'Failed to create account!' }); } else if (state.status === 'invalid_data') { - toast.error('Failed validating your submission!'); + toast({ + type: 'error', + description: 'Failed validating your submission!', + }); } else if (state.status === 'success') { - toast.success('Account created successfully'); + toast({ type: 'success', description: 'Account created successfully!' }); + setIsSuccessful(true); router.refresh(); } diff --git a/app/(chat)/actions.ts b/app/(chat)/actions.ts index c559934..8e5bd02 100644 --- a/app/(chat)/actions.ts +++ b/app/(chat)/actions.ts @@ -9,7 +9,7 @@ import { updateChatVisiblityById, } from '@/lib/db/queries'; import { VisibilityType } from '@/components/visibility-selector'; -import { myProvider } from '@/lib/ai/models'; +import { myProvider } from '@/lib/ai/providers'; export async function saveChatModelAsCookie(model: string) { const cookieStore = await cookies(); diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index e075d23..4e98455 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -4,9 +4,7 @@ import { smoothStream, streamText, } from 'ai'; - import { auth } from '@/app/(auth)/auth'; -import { myProvider } from '@/lib/ai/models'; import { systemPrompt } from '@/lib/ai/prompts'; import { deleteChatById, @@ -19,113 +17,124 @@ import { getMostRecentUserMessage, sanitizeResponseMessages, } from '@/lib/utils'; - import { generateTitleFromUserMessage } from '../../actions'; import { createDocument } from '@/lib/ai/tools/create-document'; import { updateDocument } from '@/lib/ai/tools/update-document'; import { requestSuggestions } from '@/lib/ai/tools/request-suggestions'; import { getWeather } from '@/lib/ai/tools/get-weather'; +import { isProductionEnvironment } from '@/lib/constants'; +import { NextResponse } from 'next/server'; +import { myProvider } from '@/lib/ai/providers'; export const maxDuration = 60; export async function POST(request: Request) { - const { - id, - messages, - selectedChatModel, - }: { id: string; messages: Array; selectedChatModel: string } = - await request.json(); + try { + const { + id, + messages, + selectedChatModel, + }: { + id: string; + messages: Array; + selectedChatModel: string; + } = await request.json(); - const session = await auth(); + const session = await auth(); - if (!session || !session.user || !session.user.id) { - return new Response('Unauthorized', { status: 401 }); - } + if (!session || !session.user || !session.user.id) { + return new Response('Unauthorized', { status: 401 }); + } - const userMessage = getMostRecentUserMessage(messages); + const userMessage = getMostRecentUserMessage(messages); - if (!userMessage) { - return new Response('No user message found', { status: 400 }); - } + if (!userMessage) { + return new Response('No user message found', { status: 400 }); + } - const chat = await getChatById({ id }); + const chat = await getChatById({ id }); - if (!chat) { - const title = await generateTitleFromUserMessage({ message: userMessage }); - await saveChat({ id, userId: session.user.id, title }); - } + if (!chat) { + const title = await generateTitleFromUserMessage({ + message: userMessage, + }); + await saveChat({ id, userId: session.user.id, title }); + } - await saveMessages({ - messages: [{ ...userMessage, createdAt: new Date(), chatId: id }], - }); + await saveMessages({ + messages: [{ ...userMessage, createdAt: new Date(), chatId: id }], + }); - return createDataStreamResponse({ - execute: (dataStream) => { - const result = streamText({ - model: myProvider.languageModel(selectedChatModel), - system: systemPrompt({ selectedChatModel }), - messages, - maxSteps: 5, - experimental_activeTools: - selectedChatModel === 'chat-model-reasoning' - ? [] - : [ - 'getWeather', - 'createDocument', - 'updateDocument', - 'requestSuggestions', - ], - experimental_transform: smoothStream({ chunking: 'word' }), - experimental_generateMessageId: generateUUID, - tools: { - getWeather, - createDocument: createDocument({ session, dataStream }), - updateDocument: updateDocument({ session, dataStream }), - requestSuggestions: requestSuggestions({ - session, - dataStream, - }), - }, - onFinish: async ({ response, reasoning }) => { - if (session.user?.id) { - try { - const sanitizedResponseMessages = sanitizeResponseMessages({ - messages: response.messages, - reasoning, - }); + return createDataStreamResponse({ + execute: (dataStream) => { + const result = streamText({ + model: myProvider.languageModel(selectedChatModel), + system: systemPrompt({ selectedChatModel }), + messages, + maxSteps: 5, + experimental_activeTools: + selectedChatModel === 'chat-model-reasoning' + ? [] + : [ + 'getWeather', + 'createDocument', + 'updateDocument', + 'requestSuggestions', + ], + experimental_transform: smoothStream({ chunking: 'word' }), + experimental_generateMessageId: generateUUID, + tools: { + getWeather, + createDocument: createDocument({ session, dataStream }), + updateDocument: updateDocument({ session, dataStream }), + requestSuggestions: requestSuggestions({ + session, + dataStream, + }), + }, + onFinish: async ({ response, reasoning }) => { + if (session.user?.id) { + try { + const sanitizedResponseMessages = sanitizeResponseMessages({ + messages: response.messages, + reasoning, + }); - await saveMessages({ - messages: sanitizedResponseMessages.map((message) => { - return { - id: message.id, - chatId: id, - role: message.role, - content: message.content, - createdAt: new Date(), - }; - }), - }); - } catch (error) { - console.error('Failed to save chat'); + await saveMessages({ + messages: sanitizedResponseMessages.map((message) => { + return { + id: message.id, + chatId: id, + role: message.role, + content: message.content, + createdAt: new Date(), + }; + }), + }); + } catch (error) { + console.error('Failed to save chat'); + } } - } - }, - experimental_telemetry: { - isEnabled: true, - functionId: 'stream-text', - }, - }); + }, + experimental_telemetry: { + isEnabled: isProductionEnvironment, + functionId: 'stream-text', + }, + }); - result.consumeStream(); + result.consumeStream(); - result.mergeIntoDataStream(dataStream, { - sendReasoning: true, - }); - }, - onError: () => { - return 'Oops, an error occured!'; - }, - }); + result.mergeIntoDataStream(dataStream, { + sendReasoning: true, + }); + }, + onError: () => { + return 'Oops, an error occured!'; + }, + }); + } catch (error) { + return NextResponse.json({ error }, { status: 400 }); + } } export async function DELETE(request: Request) { diff --git a/artifacts/code/server.ts b/artifacts/code/server.ts index 3ecdb1e..0b74019 100644 --- a/artifacts/code/server.ts +++ b/artifacts/code/server.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; import { streamObject } from 'ai'; -import { myProvider } from '@/lib/ai/models'; +import { myProvider } from '@/lib/ai/providers'; import { codePrompt, updateDocumentPrompt } from '@/lib/ai/prompts'; import { createDocumentHandler } from '@/lib/artifacts/server'; diff --git a/artifacts/image/server.ts b/artifacts/image/server.ts index 31e00ee..96c2bc5 100644 --- a/artifacts/image/server.ts +++ b/artifacts/image/server.ts @@ -1,4 +1,4 @@ -import { myProvider } from '@/lib/ai/models'; +import { myProvider } from '@/lib/ai/providers'; import { createDocumentHandler } from '@/lib/artifacts/server'; import { experimental_generateImage } from 'ai'; diff --git a/artifacts/sheet/server.ts b/artifacts/sheet/server.ts index f343e21..7a557bd 100644 --- a/artifacts/sheet/server.ts +++ b/artifacts/sheet/server.ts @@ -1,4 +1,4 @@ -import { myProvider } from '@/lib/ai/models'; +import { myProvider } from '@/lib/ai/providers'; import { sheetPrompt, updateDocumentPrompt } from '@/lib/ai/prompts'; import { createDocumentHandler } from '@/lib/artifacts/server'; import { streamObject } from 'ai'; diff --git a/artifacts/text/server.ts b/artifacts/text/server.ts index b35cb60..58578bf 100644 --- a/artifacts/text/server.ts +++ b/artifacts/text/server.ts @@ -1,5 +1,5 @@ import { smoothStream, streamText } from 'ai'; -import { myProvider } from '@/lib/ai/models'; +import { myProvider } from '@/lib/ai/providers'; import { createDocumentHandler } from '@/lib/artifacts/server'; import { updateDocumentPrompt } from '@/lib/ai/prompts'; diff --git a/components/artifact-messages.tsx b/components/artifact-messages.tsx index de170f3..b7cddaa 100644 --- a/components/artifact-messages.tsx +++ b/components/artifact-messages.tsx @@ -44,6 +44,7 @@ function PureArtifactMessages({ key={message.id} message={message} isLoading={isLoading && index === messages.length - 1} + index={index} vote={ votes ? votes.find((vote) => vote.messageId === message.id) diff --git a/components/console.tsx b/components/console.tsx index f4a06e9..aba7bff 100644 --- a/components/console.tsx +++ b/components/console.tsx @@ -155,7 +155,7 @@ export function Console({ consoleOutputs, setConsoleOutputs }: ConsoleProps) { output ) : ( diff --git a/components/icons.tsx b/components/icons.tsx index 17aac08..e7bd244 100644 --- a/components/icons.tsx +++ b/components/icons.tsx @@ -1153,3 +1153,22 @@ export const LineChartIcon = ({ size = 16 }: { size?: number }) => ( > ); + +export const WarningIcon = ({ size = 16 }: { size?: number }) => { + return ( + + + + ); +}; diff --git a/components/message-editor.tsx b/components/message-editor.tsx index 9e845d6..1b5a8a9 100644 --- a/components/message-editor.tsx +++ b/components/message-editor.tsx @@ -50,6 +50,7 @@ export function MessageEditor({ return (