chore: organize tests and tweak stream delays (#942)

tests/e2e/artifacts.test.ts

@@ -0,0 +1,74 @@
+import { expect, test } from '@playwright/test';
+import { ChatPage } from '../pages/chat';
+import { ArtifactPage } from '../pages/artifact';
+
+test.describe('artifacts activity', () => {
+  let chatPage: ChatPage;
+  let artifactPage: ArtifactPage;
+
+  test.beforeEach(async ({ page }) => {
+    chatPage = new ChatPage(page);
+    artifactPage = new ArtifactPage(page);
+
+    await chatPage.createNewChat();
+  });
+
+  test('create a text artifact', async () => {
+    await chatPage.createNewChat();
+
+    await chatPage.sendUserMessage(
+      'Help me write an essay about Silicon Valley',
+    );
+    await artifactPage.isGenerationComplete();
+
+    expect(artifactPage.artifact).toBeVisible();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+    expect(assistantMessage.content).toBe(
+      'A document was created and is now visible to the user.',
+    );
+
+    await chatPage.hasChatIdInUrl();
+  });
+
+  test('toggle artifact visibility', async () => {
+    await chatPage.createNewChat();
+
+    await chatPage.sendUserMessage(
+      'Help me write an essay about Silicon Valley',
+    );
+    await artifactPage.isGenerationComplete();
+
+    expect(artifactPage.artifact).toBeVisible();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+    expect(assistantMessage.content).toBe(
+      'A document was created and is now visible to the user.',
+    );
+
+    await artifactPage.closeArtifact();
+    await chatPage.isElementNotVisible('artifact');
+  });
+
+  test('send follow up message after generation', async () => {
+    await chatPage.createNewChat();
+
+    await chatPage.sendUserMessage(
+      'Help me write an essay about Silicon Valley',
+    );
+    await artifactPage.isGenerationComplete();
+
+    expect(artifactPage.artifact).toBeVisible();
+
+    const assistantMessage = await artifactPage.getRecentAssistantMessage();
+    expect(assistantMessage.content).toBe(
+      'A document was created and is now visible to the user.',
+    );
+
+    await artifactPage.sendUserMessage('Thanks!');
+    await artifactPage.isGenerationComplete();
+
+    const secondAssistantMessage = await chatPage.getRecentAssistantMessage();
+    expect(secondAssistantMessage.content).toBe("You're welcome!");
+  });
+});

tests/e2e/auth.setup.ts

@@ -0,0 +1,24 @@
+import path from 'node:path';
+import { generateId } from 'ai';
+import { getUnixTime } from 'date-fns';
+import { expect, test as setup } from '@playwright/test';
+
+const authFile = path.join(__dirname, '../../playwright/.auth/session.json');
+
+setup('authenticate', async ({ page }) => {
+  const testEmail = `test-${getUnixTime(new Date())}@playwright.com`;
+  const testPassword = generateId(16);
+
+  await page.goto('http://localhost:3000/register');
+  await page.getByPlaceholder('user@acme.com').click();
+  await page.getByPlaceholder('user@acme.com').fill(testEmail);
+  await page.getByLabel('Password').click();
+  await page.getByLabel('Password').fill(testPassword);
+  await page.getByRole('button', { name: 'Sign Up' }).click();
+
+  await expect(page.getByTestId('toast')).toContainText(
+    'Account created successfully!',
+  );
+
+  await page.context().storageState({ path: authFile });
+});

tests/e2e/auth.test.ts

@@ -0,0 +1,77 @@
+import { generateId } from 'ai';
+import { getUnixTime } from 'date-fns';
+import { test, expect, type Page } from '@playwright/test';
+
+test.use({ storageState: { cookies: [], origins: [] } });
+
+const testEmail = `test-${getUnixTime(new Date())}@playwright.com`;
+const testPassword = generateId(16);
+
+class AuthPage {
+  constructor(private page: Page) {}
+
+  async gotoLogin() {
+    await this.page.goto('/login');
+    await expect(this.page.getByRole('heading')).toContainText('Sign In');
+  }
+
+  async gotoRegister() {
+    await this.page.goto('/register');
+    await expect(this.page.getByRole('heading')).toContainText('Sign Up');
+  }
+
+  async register(email: string, password: string) {
+    await this.gotoRegister();
+    await this.page.getByPlaceholder('user@acme.com').click();
+    await this.page.getByPlaceholder('user@acme.com').fill(email);
+    await this.page.getByLabel('Password').click();
+    await this.page.getByLabel('Password').fill(password);
+    await this.page.getByRole('button', { name: 'Sign Up' }).click();
+  }
+
+  async login(email: string, password: string) {
+    await this.gotoLogin();
+    await this.page.getByPlaceholder('user@acme.com').click();
+    await this.page.getByPlaceholder('user@acme.com').fill(email);
+    await this.page.getByLabel('Password').click();
+    await this.page.getByLabel('Password').fill(password);
+    await this.page.getByRole('button', { name: 'Sign In' }).click();
+  }
+
+  async expectToastToContain(text: string) {
+    await expect(this.page.getByTestId('toast')).toContainText(text);
+  }
+}
+
+test.describe
+  .serial('authentication', () => {
+    let authPage: AuthPage;
+
+    test.beforeEach(async ({ page }) => {
+      authPage = new AuthPage(page);
+    });
+
+    test('redirect to login page when unauthenticated', async ({ page }) => {
+      await page.goto('/');
+      await expect(page.getByRole('heading')).toContainText('Sign In');
+    });
+
+    test('register a test account', async ({ page }) => {
+      await authPage.register(testEmail, testPassword);
+      await expect(page).toHaveURL('/');
+      await authPage.expectToastToContain('Account created successfully!');
+    });
+
+    test('register test account with existing email', async () => {
+      await authPage.register(testEmail, testPassword);
+      await authPage.expectToastToContain('Account already exists!');
+    });
+
+    test('log into account', async ({ page }) => {
+      await authPage.login(testEmail, testPassword);
+
+      await page.waitForURL('/');
+      await expect(page).toHaveURL('/');
+      await expect(page.getByPlaceholder('Send a message...')).toBeVisible();
+    });
+  });

tests/e2e/chat.test.ts

@@ -0,0 +1,141 @@
+import { ChatPage } from '../pages/chat';
+import { test, expect } from '@playwright/test';
+
+test.describe('chat activity', () => {
+  let chatPage: ChatPage;
+
+  test.beforeEach(async ({ page }) => {
+    chatPage = new ChatPage(page);
+    await chatPage.createNewChat();
+  });
+
+  test('send a user message and receive response', async () => {
+    await chatPage.sendUserMessage('Why is grass green?');
+    await chatPage.isGenerationComplete();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+    expect(assistantMessage.content).toContain("It's just green duh!");
+  });
+
+  test('redirect to /chat/:id after submitting message', async () => {
+    await chatPage.sendUserMessage('Why is grass green?');
+    await chatPage.isGenerationComplete();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+    expect(assistantMessage.content).toContain("It's just green duh!");
+    await chatPage.hasChatIdInUrl();
+  });
+
+  test('send a user message from suggestion', async () => {
+    await chatPage.sendUserMessageFromSuggestion();
+    await chatPage.isGenerationComplete();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+    expect(assistantMessage.content).toContain(
+      'With Next.js, you can ship fast!',
+    );
+  });
+
+  test('toggle between send/stop button based on activity', async () => {
+    await expect(chatPage.sendButton).toBeVisible();
+    await expect(chatPage.sendButton).toBeDisabled();
+
+    await chatPage.sendUserMessage('Why is grass green?');
+
+    await expect(chatPage.sendButton).not.toBeVisible();
+    await expect(chatPage.stopButton).toBeVisible();
+
+    await chatPage.isGenerationComplete();
+
+    await expect(chatPage.stopButton).not.toBeVisible();
+    await expect(chatPage.sendButton).toBeVisible();
+  });
+
+  test('stop generation during submission', async () => {
+    await chatPage.sendUserMessage('Why is grass green?');
+    await expect(chatPage.stopButton).toBeVisible();
+    await chatPage.stopButton.click();
+    await expect(chatPage.sendButton).toBeVisible();
+  });
+
+  test('edit user message and resubmit', async () => {
+    await chatPage.sendUserMessage('Why is grass green?');
+    await chatPage.isGenerationComplete();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+    expect(assistantMessage.content).toContain("It's just green duh!");
+
+    const userMessage = await chatPage.getRecentUserMessage();
+    await userMessage.edit('Why is the sky blue?');
+
+    await chatPage.isGenerationComplete();
+
+    const updatedAssistantMessage = await chatPage.getRecentAssistantMessage();
+    expect(updatedAssistantMessage.content).toContain("It's just blue duh!");
+  });
+
+  test('hide suggested actions after sending message', async () => {
+    await chatPage.isElementVisible('suggested-actions');
+    await chatPage.sendUserMessageFromSuggestion();
+    await chatPage.isElementNotVisible('suggested-actions');
+  });
+
+  test('upload file and send image attachment with message', async () => {
+    await chatPage.addImageAttachment();
+
+    await chatPage.isElementVisible('attachments-preview');
+    await chatPage.isElementVisible('input-attachment-loader');
+    await chatPage.isElementNotVisible('input-attachment-loader');
+
+    await chatPage.sendUserMessage('Who painted this?');
+
+    const userMessage = await chatPage.getRecentUserMessage();
+    expect(userMessage.attachments).toHaveLength(1);
+
+    await chatPage.isGenerationComplete();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+    expect(assistantMessage.content).toBe('This painting is by Monet!');
+  });
+
+  test('call weather tool', async () => {
+    await chatPage.sendUserMessage("What's the weather in sf?");
+    await chatPage.isGenerationComplete();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+
+    expect(assistantMessage.content).toBe(
+      'The current temperature in San Francisco is 17°C.',
+    );
+  });
+
+  test('upvote message', async () => {
+    await chatPage.sendUserMessage('Why is the sky blue?');
+    await chatPage.isGenerationComplete();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+    await assistantMessage.upvote();
+    await chatPage.isVoteComplete();
+  });
+
+  test('downvote message', async () => {
+    await chatPage.sendUserMessage('Why is the sky blue?');
+    await chatPage.isGenerationComplete();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+    await assistantMessage.downvote();
+    await chatPage.isVoteComplete();
+  });
+
+  test('update vote', async () => {
+    await chatPage.sendUserMessage('Why is the sky blue?');
+    await chatPage.isGenerationComplete();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+    await assistantMessage.upvote();
+    await chatPage.isVoteComplete();
+
+    await assistantMessage.downvote();
+    await chatPage.isVoteComplete();
+  });
+});

tests/e2e/reasoning.setup.ts

@@ -0,0 +1,20 @@
+import path from 'node:path';
+import { expect, test as setup } from '@playwright/test';
+import { ChatPage } from '../pages/chat';
+
+const reasoningFile = path.join(
+  __dirname,
+  '../../playwright/.reasoning/session.json',
+);
+
+setup('switch to reasoning model', async ({ page }) => {
+  const chatPage = new ChatPage(page);
+  await chatPage.createNewChat();
+
+  await chatPage.chooseModelFromSelector('chat-model-reasoning');
+
+  await expect(chatPage.getSelectedModel()).resolves.toEqual('Reasoning model');
+
+  await page.waitForTimeout(1000);
+  await page.context().storageState({ path: reasoningFile });
+});

tests/e2e/reasoning.test.ts

@@ -0,0 +1,62 @@
+import { ChatPage } from '../pages/chat';
+import { test, expect } from '@playwright/test';
+
+test.describe('chat activity with reasoning', () => {
+  let chatPage: ChatPage;
+
+  test.beforeEach(async ({ page }) => {
+    chatPage = new ChatPage(page);
+    await chatPage.createNewChat();
+  });
+
+  test('send user message and generate response with reasoning', async () => {
+    await chatPage.sendUserMessage('Why is the sky blue?');
+    await chatPage.isGenerationComplete();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+    expect(assistantMessage.content).toBe("It's just blue duh!");
+
+    expect(assistantMessage.reasoning).toBe(
+      'The sky is blue because of rayleigh scattering!',
+    );
+  });
+
+  test('toggle reasoning visibility', async () => {
+    await chatPage.sendUserMessage('Why is the sky blue?');
+    await chatPage.isGenerationComplete();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+    const reasoningElement =
+      assistantMessage.element.getByTestId('message-reasoning');
+    expect(reasoningElement).toBeVisible();
+
+    await assistantMessage.toggleReasoningVisibility();
+    await expect(reasoningElement).not.toBeVisible();
+
+    await assistantMessage.toggleReasoningVisibility();
+    await expect(reasoningElement).toBeVisible();
+  });
+
+  test('edit message and resubmit', async () => {
+    await chatPage.sendUserMessage('Why is the sky blue?');
+    await chatPage.isGenerationComplete();
+
+    const assistantMessage = await chatPage.getRecentAssistantMessage();
+    const reasoningElement =
+      assistantMessage.element.getByTestId('message-reasoning');
+    expect(reasoningElement).toBeVisible();
+
+    const userMessage = await chatPage.getRecentUserMessage();
+
+    await userMessage.edit('Why is grass green?');
+    await chatPage.isGenerationComplete();
+
+    const updatedAssistantMessage = await chatPage.getRecentAssistantMessage();
+
+    expect(updatedAssistantMessage.content).toBe("It's just green duh!");
+
+    expect(updatedAssistantMessage.reasoning).toBe(
+      'Grass is green because of chlorophyll absorption!',
+    );
+  });
+});