feat: add reasoning tests (#856)

This commit is contained in:
Jeremy 2025-03-09 21:02:19 -07:00 committed by GitHub
parent 39729644df
commit 9628c54755
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 420 additions and 155 deletions

64
tests/reasoning.test.ts Normal file
View file

@ -0,0 +1,64 @@
import { useAssistant } from '@ai-sdk/react';
import { ChatPage } from './pages/chat';
import { test, expect } from '@playwright/test';
test.describe('chat activity with reasoning', () => {
let chatPage: ChatPage;
test.beforeEach(async ({ page }) => {
chatPage = new ChatPage(page);
await chatPage.createNewChat();
});
test('send user message and generate response with reasoning', async () => {
await chatPage.sendUserMessage('why is the sky blue?');
await chatPage.waitForMessageGeneration();
const assistantMessage = await chatPage.getRecentAssistantMessage();
expect(assistantMessage.content).toBe("it's just blue duh!");
expect(assistantMessage.reasoning).toBe(
'the sky is blue because of rayleigh scattering!',
);
});
test('toggle reasoning visibility', async () => {
await chatPage.sendUserMessage('why is the sky blue?');
await chatPage.waitForMessageGeneration();
const assistantMessage = await chatPage.getRecentAssistantMessage();
const reasoningElement =
assistantMessage.element.getByTestId('message-reasoning');
expect(reasoningElement).toBeVisible();
await assistantMessage.toggleReasoningVisibility();
await expect(reasoningElement).not.toBeVisible();
await assistantMessage.toggleReasoningVisibility();
await expect(reasoningElement).toBeVisible();
});
test('edit message and resubmit', async () => {
await chatPage.sendUserMessage('why is the sky blue?');
await chatPage.waitForMessageGeneration();
const assistantMessage = await chatPage.getRecentAssistantMessage();
const reasoningElement =
assistantMessage.element.getByTestId('message-reasoning');
expect(reasoningElement).toBeVisible();
const userMessage = await chatPage.getRecentUserMessage();
await userMessage.edit('why is grass green?');
await chatPage.waitForMessageGeneration();
const updatedAssistantMessage = await chatPage.getRecentAssistantMessage();
expect(updatedAssistantMessage.content).toBe("it's just green duh!");
expect(updatedAssistantMessage.reasoning).toBe(
'grass is green because of chlorophyll absorption!',
);
});
});