dmitry.galkin/chatbot-template
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add reasoning tests (#856)

Browse source
This commit is contained in:
Jeremy 2025-03-09 21:02:19 -07:00 • committed by GitHub
parent 39729644df
commit 9628c54755
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 420 additions and 155 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/global.setup.ts → tests/auth.setup.ts
View file

@ -3,7 +3,7 @@ import { generateId } from 'ai';
import { getUnixTime } from 'date-fns';
import { expect, test as setup } from '@playwright/test';
const authFile = path.join(__dirname, '../playwright/.auth/user.json');
const authFile = path.join(__dirname, '../playwright/.auth/session.json');
setup('authenticate', async ({ page }) => {
const testEmail = `test-${getUnixTime(new Date())}@playwright.com`;

146
tests/chat.test.ts
View file

@ -1,132 +1,37 @@
import fs from 'fs';
import path from 'path';
import { test, expect, Page } from '@playwright/test';
class ChatPage {
constructor(private page: Page) {}
async goto() {
await this.page.goto('/');
}
public getCurrentURL(): string {
return this.page.url();
}
async sendUserMessage(message: string) {
await this.page.getByTestId('multimodal-input').click();
await this.page.getByTestId('multimodal-input').fill(message);
await this.page.getByTestId('send-button').click();
await this.page.getByTestId('message-user-0').isVisible();
expect(await this.page.getByTestId('message-user-0').innerText()).toContain(
message,
);
}
async getAssistantResponse() {
return this.page.getByTestId('message-assistant-1').innerText();
}
async isGenerationComplete() {
await expect(this.page.getByTestId('send-button')).toBeVisible();
await this.page.waitForTimeout(2000);
}
async hasChatIdInUrl() {
await expect(this.page).toHaveURL(
/^http:\/\/localhost:3000\/chat\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/,
);
}
async sendUserMessageFromSuggestion() {
await this.page
.getByRole('button', { name: 'What are the advantages of' })
.click();
}
async editAndResendUserMessage(message: string) {
await this.page.getByTestId('edit-user-0').click();
await this.page.getByTestId('message-editor').fill(message);
await this.page.getByTestId('message-editor-send-button').click();
await this.page.getByTestId('message-user-0').isVisible();
await expect(
this.page.getByTestId('message-editor-send-button'),
).not.toBeVisible();
expect(await this.page.getByTestId('message-user-0').innerText()).toContain(
message,
);
}
async isElementVisible(elementId: string) {
await expect(this.page.getByTestId(elementId)).toBeVisible();
}
async isElementNotVisible(elementId: string) {
await expect(this.page.getByTestId(elementId)).not.toBeVisible();
}
async addImageAttachment() {
this.page.on('filechooser', async (fileChooser) => {
const filePath = path.join(
process.cwd(),
'public',
'images',
'mouth of the seine, monet.jpg',
);
const imageBuffer = fs.readFileSync(filePath);
await fileChooser.setFiles({
name: 'mouth of the seine, monet.jpg',
mimeType: 'image/jpeg',
buffer: imageBuffer,
});
});
await this.page.getByTestId('attachments-button').click();
}
public get sendButton() {
return this.page.getByTestId('send-button');
}
public get stopButton() {
return this.page.getByTestId('stop-button');
}
public get multimodalInput() {
return this.page.getByTestId('multimodal-input');
}
}
import { test, expect } from '@playwright/test';
import { ChatPage } from './pages/chat';
test.describe('chat activity', () => {
let chatPage: ChatPage;
test.beforeEach(async ({ page }) => {
chatPage = new ChatPage(page);
await chatPage.goto();
await chatPage.createNewChat();
});
test('send a user message and receive response', async () => {
await chatPage.sendUserMessage('why is grass green?');
await chatPage.isGenerationComplete();
expect(await chatPage.getAssistantResponse()).toContain(
"it's just green duh!",
);
const assistantMessage = await chatPage.getRecentAssistantMessage();
expect(assistantMessage.content).toContain("it's just green duh!");
});
test('redirect to /chat/:id after submitting message', async () => {
await chatPage.sendUserMessage('why is grass green?');
await chatPage.isGenerationComplete();
expect(await chatPage.getAssistantResponse()).toContain(
"it's just green duh!",
);
const assistantMessage = await chatPage.getRecentAssistantMessage();
expect(assistantMessage.content).toContain("it's just green duh!");
await chatPage.hasChatIdInUrl();
});
test('send a user message from suggestion', async () => {
await chatPage.sendUserMessageFromSuggestion();
await chatPage.isGenerationComplete();
expect(await chatPage.getAssistantResponse()).toContain(
const assistantMessage = await chatPage.getRecentAssistantMessage();
expect(assistantMessage.content).toContain(
'with next.js you can ship fast!',
);
});
@ -148,6 +53,7 @@ test.describe('chat activity', () => {
test('stop generation during submission', async () => {
await chatPage.sendUserMessage('why is grass green?');
await expect(chatPage.stopButton).toBeVisible();
await chatPage.stopButton.click();
await expect(chatPage.sendButton).toBeVisible();
});
@ -156,16 +62,16 @@ test.describe('chat activity', () => {
await chatPage.sendUserMessage('why is grass green?');
await chatPage.isGenerationComplete();
expect(await chatPage.getAssistantResponse()).toContain(
"it's just green duh!",
);
const assistantMessage = await chatPage.getRecentAssistantMessage();
expect(assistantMessage.content).toContain("it's just green duh!");
const userMessage = await chatPage.getRecentUserMessage();
await userMessage.edit('why is the sky blue?');
await chatPage.editAndResendUserMessage('why is the sky blue?');
await chatPage.isGenerationComplete();
expect(await chatPage.getAssistantResponse()).toContain(
"it's just blue duh!",
);
const updatedAssistantMessage = await chatPage.getRecentAssistantMessage();
expect(updatedAssistantMessage.content).toContain("it's just blue duh!");
});
test('hide suggested actions after sending message', async () => {
@ -174,7 +80,7 @@ test.describe('chat activity', () => {
await chatPage.isElementNotVisible('suggested-actions');
});
test('handle file upload and send image attachment with message', async () => {
test('upload file and send image attachment with message', async () => {
await chatPage.addImageAttachment();
await chatPage.isElementVisible('attachments-preview');
@ -182,13 +88,13 @@ test.describe('chat activity', () => {
await chatPage.isElementNotVisible('input-attachment-loader');
await chatPage.sendUserMessage('who painted this?');
await chatPage.isElementVisible('message-attachments-0');
const userMessage = await chatPage.getRecentUserMessage();
expect(userMessage.attachments).toHaveLength(1);
await chatPage.isGenerationComplete();
await chatPage.isElementVisible('message-assistant-1');
expect(await chatPage.getAssistantResponse()).toContain(
'this painting is by monet!',
);
const assistantMessage = await chatPage.getRecentAssistantMessage();
expect(assistantMessage.content).toBe('this painting is by monet!');
});
});

186
tests/pages/chat.ts Normal file
View file

@ -0,0 +1,186 @@
import fs from 'fs';
import path from 'path';
import { chatModels } from '@/lib/ai/models';
import { expect, Page } from '@playwright/test';
export class ChatPage {
constructor(private page: Page) {}
public get sendButton() {
return this.page.getByTestId('send-button');
}
public get stopButton() {
return this.page.getByTestId('stop-button');
}
public get multimodalInput() {
return this.page.getByTestId('multimodal-input');
}
async createNewChat() {
await this.page.goto('/');
}
public getCurrentURL(): string {
return this.page.url();
}
async sendUserMessage(message: string) {
await this.page.getByTestId('multimodal-input').click();
await this.page.getByTestId('multimodal-input').fill(message);
await this.page.getByTestId('send-button').click();
await this.page.getByTestId('message-user').isVisible();
expect(await this.page.getByTestId('message-user').innerText()).toContain(
message,
);
}
async isGenerationComplete() {
await expect(this.page.getByTestId('send-button')).toBeVisible();
await this.page.waitForTimeout(2000);
}
async hasChatIdInUrl() {
await expect(this.page).toHaveURL(
/^http:\/\/localhost:3000\/chat\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/,
);
}
async sendUserMessageFromSuggestion() {
await this.page
.getByRole('button', { name: 'What are the advantages of' })
.click();
}
async isElementVisible(elementId: string) {
await expect(this.page.getByTestId(elementId)).toBeVisible();
}
async isElementNotVisible(elementId: string) {
await expect(this.page.getByTestId(elementId)).not.toBeVisible();
}
async addImageAttachment() {
this.page.on('filechooser', async (fileChooser) => {
const filePath = path.join(
process.cwd(),
'public',
'images',
'mouth of the seine, monet.jpg',
);
const imageBuffer = fs.readFileSync(filePath);
await fileChooser.setFiles({
name: 'mouth of the seine, monet.jpg',
mimeType: 'image/jpeg',
buffer: imageBuffer,
});
});
await this.page.getByTestId('attachments-button').click();
}
public async getSelectedModel() {
const modelId = await this.page.getByTestId('model-selector').innerText();
return modelId;
}
public async chooseModelFromSelector(chatModelId: string) {
const chatModel = chatModels.find(
(chatModel) => chatModel.id === chatModelId,
);
if (!chatModel) {
throw new Error(`Model with id ${chatModelId} not found`);
}
await this.page.getByTestId('model-selector').click();
await this.page.getByTestId(`model-selector-item-${chatModelId}`).click();
expect(await this.getSelectedModel()).toBe(chatModel.name);
}
async getRecentAssistantMessage() {
const messageElements = await this.page
.getByTestId('message-assistant')
.all();
const lastMessageElement = messageElements[messageElements.length - 1];
const content = await lastMessageElement
.getByTestId('message-content')
.innerText()
.catch(() => null);
const reasoningElement = await lastMessageElement
.getByTestId('message-reasoning')
.isVisible()
.then(async (visible) =>
visible
? await lastMessageElement
.getByTestId('message-reasoning')
.innerText()
: null,
)
.catch(() => null);
const page = this.page;
return {
element: lastMessageElement,
content,
reasoning: reasoningElement,
async waitForGenerationComplete() {
await expect(page.getByTestId('send-button')).toBeVisible();
await page.waitForTimeout(2000);
},
async toggleReasoningVisibility() {
await lastMessageElement
.getByTestId('message-reasoning-toggle')
.click();
},
};
}
async getRecentUserMessage() {
const messageElements = await this.page.getByTestId('message-user').all();
const lastMessageElement = messageElements[messageElements.length - 1];
const content = await lastMessageElement.innerText();
const hasAttachments = await lastMessageElement
.getByTestId('message-attachments')
.isVisible()
.catch(() => false);
const attachments = hasAttachments
? await lastMessageElement.getByTestId('message-attachments').all()
: [];
const page = this.page;
return {
element: lastMessageElement,
content,
attachments,
async edit(newMessage: string) {
await page.getByTestId('message-edit').click();
await page.getByTestId('message-editor').fill(newMessage);
await page.getByTestId('message-editor-send-button').click();
await expect(
page.getByTestId('message-editor-send-button'),
).not.toBeVisible();
},
};
}
async waitForMessageGeneration(timeout = 10000) {
await this.page.waitForFunction(
() => {
return document.querySelector('[data-testid="send-button"]') !== null;
},
{ timeout },
);
await this.page.waitForTimeout(500);
}
}

20
tests/reasoning.setup.ts Normal file
View file

@ -0,0 +1,20 @@
import path from 'path';
import { expect, test as setup } from '@playwright/test';
import { ChatPage } from './pages/chat';
const reasoningFile = path.join(
__dirname,
'../playwright/.reasoning/session.json',
);
setup('switch to reasoning model', async ({ page }) => {
const chatPage = new ChatPage(page);
await chatPage.createNewChat();
await chatPage.chooseModelFromSelector('chat-model-reasoning');
await expect(chatPage.getSelectedModel()).resolves.toEqual('Reasoning model');
await page.waitForTimeout(1000);
await page.context().storageState({ path: reasoningFile });
});

64
tests/reasoning.test.ts Normal file
View file

@ -0,0 +1,64 @@
import { useAssistant } from '@ai-sdk/react';
import { ChatPage } from './pages/chat';
import { test, expect } from '@playwright/test';
test.describe('chat activity with reasoning', () => {
let chatPage: ChatPage;
test.beforeEach(async ({ page }) => {
chatPage = new ChatPage(page);
await chatPage.createNewChat();
});
test('send user message and generate response with reasoning', async () => {
await chatPage.sendUserMessage('why is the sky blue?');
await chatPage.waitForMessageGeneration();
const assistantMessage = await chatPage.getRecentAssistantMessage();
expect(assistantMessage.content).toBe("it's just blue duh!");
expect(assistantMessage.reasoning).toBe(
'the sky is blue because of rayleigh scattering!',
);
});
test('toggle reasoning visibility', async () => {
await chatPage.sendUserMessage('why is the sky blue?');
await chatPage.waitForMessageGeneration();
const assistantMessage = await chatPage.getRecentAssistantMessage();
const reasoningElement =
assistantMessage.element.getByTestId('message-reasoning');
expect(reasoningElement).toBeVisible();
await assistantMessage.toggleReasoningVisibility();
await expect(reasoningElement).not.toBeVisible();
await assistantMessage.toggleReasoningVisibility();
await expect(reasoningElement).toBeVisible();
});
test('edit message and resubmit', async () => {
await chatPage.sendUserMessage('why is the sky blue?');
await chatPage.waitForMessageGeneration();
const assistantMessage = await chatPage.getRecentAssistantMessage();
const reasoningElement =
assistantMessage.element.getByTestId('message-reasoning');
expect(reasoningElement).toBeVisible();
const userMessage = await chatPage.getRecentUserMessage();
await userMessage.edit('why is grass green?');
await chatPage.waitForMessageGeneration();
const updatedAssistantMessage = await chatPage.getRecentAssistantMessage();
expect(updatedAssistantMessage.content).toBe("it's just green duh!");
expect(updatedAssistantMessage.reasoning).toBe(
'grass is green because of chlorophyll absorption!',
);
});
});