feat: dynamic model discovery from vercel ai gateway (#1353)
This commit is contained in:
josh
GitHub
parent
2b0b42d144
commit
b1da86062e
74 changed files with 7426 additions and 2277 deletions
|
|
@ -1,77 +0,0 @@
|
|||
import { expect, test } from "../fixtures";
|
||||
import { ArtifactPage } from "../pages/artifact";
|
||||
import { ChatPage } from "../pages/chat";
|
||||
|
||||
test.describe("Artifacts activity", () => {
|
||||
let chatPage: ChatPage;
|
||||
let artifactPage: ArtifactPage;
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
chatPage = new ChatPage(page);
|
||||
artifactPage = new ArtifactPage(page);
|
||||
|
||||
await chatPage.createNewChat();
|
||||
});
|
||||
|
||||
test("Create a text artifact", async () => {
|
||||
test.fixme();
|
||||
await chatPage.createNewChat();
|
||||
|
||||
await chatPage.sendUserMessage(
|
||||
"Help me write an essay about Silicon Valley"
|
||||
);
|
||||
await artifactPage.isGenerationComplete();
|
||||
|
||||
expect(artifactPage.artifact).toBeVisible();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
expect(assistantMessage.content).toBe(
|
||||
"A document was created and is now visible to the user."
|
||||
);
|
||||
|
||||
await chatPage.hasChatIdInUrl();
|
||||
});
|
||||
|
||||
test("Toggle artifact visibility", async () => {
|
||||
test.fixme();
|
||||
await chatPage.createNewChat();
|
||||
|
||||
await chatPage.sendUserMessage(
|
||||
"Help me write an essay about Silicon Valley"
|
||||
);
|
||||
await artifactPage.isGenerationComplete();
|
||||
|
||||
expect(artifactPage.artifact).toBeVisible();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
expect(assistantMessage.content).toBe(
|
||||
"A document was created and is now visible to the user."
|
||||
);
|
||||
|
||||
await artifactPage.closeArtifact();
|
||||
await chatPage.isElementNotVisible("artifact");
|
||||
});
|
||||
|
||||
test("Send follow up message after generation", async () => {
|
||||
test.fixme();
|
||||
await chatPage.createNewChat();
|
||||
|
||||
await chatPage.sendUserMessage(
|
||||
"Help me write an essay about Silicon Valley"
|
||||
);
|
||||
await artifactPage.isGenerationComplete();
|
||||
|
||||
expect(artifactPage.artifact).toBeVisible();
|
||||
|
||||
const assistantMessage = await artifactPage.getRecentAssistantMessage();
|
||||
expect(assistantMessage.content).toBe(
|
||||
"A document was created and is now visible to the user."
|
||||
);
|
||||
|
||||
await artifactPage.sendUserMessage("Thanks!");
|
||||
await artifactPage.isGenerationComplete();
|
||||
|
||||
const secondAssistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
expect(secondAssistantMessage.content).toBe("You're welcome!");
|
||||
});
|
||||
});
|
||||
53
tests/e2e/auth.test.ts
Normal file
53
tests/e2e/auth.test.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import { expect, test } from "@playwright/test";
|
||||
|
||||
test.describe("Authentication", () => {
|
||||
test("can register a new account", async ({ page }) => {
|
||||
const timestamp = Date.now();
|
||||
const email = `test-${timestamp}@example.com`;
|
||||
const password = "testpassword123";
|
||||
|
||||
await page.goto("/register");
|
||||
await page.getByPlaceholder("user@acme.com").fill(email);
|
||||
await page.getByLabel("Password").fill(password);
|
||||
await page.getByRole("button", { name: "Sign Up" }).click();
|
||||
|
||||
await expect(page.getByTestId("toast")).toContainText(
|
||||
"Account created successfully"
|
||||
);
|
||||
await expect(page).toHaveURL("/");
|
||||
});
|
||||
|
||||
test("can login with credentials", async ({ page }) => {
|
||||
const timestamp = Date.now();
|
||||
const email = `test-${timestamp}@example.com`;
|
||||
const password = "testpassword123";
|
||||
|
||||
await page.goto("/register");
|
||||
await page.getByPlaceholder("user@acme.com").fill(email);
|
||||
await page.getByLabel("Password").fill(password);
|
||||
await page.getByRole("button", { name: "Sign Up" }).click();
|
||||
await expect(page).toHaveURL("/");
|
||||
|
||||
await page.goto("/login");
|
||||
await page.getByPlaceholder("user@acme.com").fill(email);
|
||||
await page.getByLabel("Password").fill(password);
|
||||
await page.getByRole("button", { name: "Sign In" }).click();
|
||||
|
||||
await expect(page).toHaveURL("/");
|
||||
});
|
||||
|
||||
test("shows error for invalid credentials", async ({ page }) => {
|
||||
await page.goto("/login");
|
||||
await page.getByPlaceholder("user@acme.com").fill("invalid@example.com");
|
||||
await page.getByLabel("Password").fill("wrongpassword");
|
||||
await page.getByRole("button", { name: "Sign In" }).click();
|
||||
|
||||
await expect(page.getByTestId("toast")).toBeVisible();
|
||||
});
|
||||
|
||||
test("can continue as guest", async ({ page }) => {
|
||||
await page.goto("/");
|
||||
await expect(page.getByTestId("multimodal-input")).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import { expect, test } from "../fixtures";
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { ChatPage } from "../pages/chat";
|
||||
|
||||
test.describe("Chat activity", () => {
|
||||
test.describe("Chat", () => {
|
||||
let chatPage: ChatPage;
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
|
|
@ -9,164 +9,60 @@ test.describe("Chat activity", () => {
|
|||
await chatPage.createNewChat();
|
||||
});
|
||||
|
||||
test("Send a user message and receive response", async () => {
|
||||
await chatPage.sendUserMessage("Why is grass green?");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
expect(assistantMessage.content).toContain("It's just green duh!");
|
||||
test("page loads with input ready", async () => {
|
||||
await chatPage.waitForInputToBeReady();
|
||||
});
|
||||
|
||||
test("Redirect to /chat/:id after submitting message", async () => {
|
||||
await chatPage.sendUserMessage("Why is grass green?");
|
||||
test("send button is disabled when input is empty", async () => {
|
||||
await expect(chatPage.sendButton).toBeDisabled();
|
||||
});
|
||||
|
||||
test("send button is enabled when input has text", async () => {
|
||||
await chatPage.multimodalInput.fill("Hello");
|
||||
await expect(chatPage.sendButton).toBeEnabled();
|
||||
});
|
||||
|
||||
test("can send a message and receive a response", async () => {
|
||||
await chatPage.sendUserMessage("Hello");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
expect(assistantMessage.content).toContain("It's just green duh!");
|
||||
const userContent = await chatPage.getLastUserMessageContent();
|
||||
expect(userContent).toBe("Hello");
|
||||
|
||||
const assistantContent = await chatPage.getLastAssistantMessageContent();
|
||||
expect(assistantContent).toBeTruthy();
|
||||
});
|
||||
|
||||
test("redirects to /chat/:id after sending message", async () => {
|
||||
await chatPage.sendUserMessage("Hello");
|
||||
await chatPage.isGenerationComplete();
|
||||
await chatPage.hasChatIdInUrl();
|
||||
});
|
||||
|
||||
test("Send a user message from suggestion", async () => {
|
||||
await chatPage.sendUserMessageFromSuggestion();
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
expect(assistantMessage.content).toContain(
|
||||
"With Next.js, you can ship fast!"
|
||||
);
|
||||
test("shows stop button during generation", async () => {
|
||||
await chatPage.multimodalInput.fill("Hello");
|
||||
await chatPage.sendButton.click();
|
||||
await expect(chatPage.stopButton).toBeVisible({ timeout: 5000 });
|
||||
});
|
||||
|
||||
test("Toggle between send/stop button based on activity", async () => {
|
||||
await expect(chatPage.sendButton).toBeVisible();
|
||||
await expect(chatPage.sendButton).toBeDisabled();
|
||||
|
||||
await chatPage.sendUserMessage("Why is grass green?");
|
||||
|
||||
await expect(chatPage.sendButton).not.toBeVisible();
|
||||
await expect(chatPage.stopButton).toBeVisible();
|
||||
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
await expect(chatPage.stopButton).not.toBeVisible();
|
||||
await expect(chatPage.sendButton).toBeVisible();
|
||||
});
|
||||
|
||||
test("Stop generation during submission", async () => {
|
||||
await chatPage.sendUserMessage("Why is grass green?");
|
||||
await expect(chatPage.stopButton).toBeVisible();
|
||||
test("can stop generation", async () => {
|
||||
await chatPage.multimodalInput.fill("Hello");
|
||||
await chatPage.sendButton.click();
|
||||
await expect(chatPage.stopButton).toBeVisible({ timeout: 5000 });
|
||||
await chatPage.stopButton.click();
|
||||
await expect(chatPage.sendButton).toBeVisible();
|
||||
});
|
||||
|
||||
test("Edit user message and resubmit", async () => {
|
||||
await chatPage.sendUserMessage("Why is grass green?");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
expect(assistantMessage.content).toContain("It's just green duh!");
|
||||
|
||||
const userMessage = await chatPage.getRecentUserMessage();
|
||||
await userMessage.edit("Why is the sky blue?");
|
||||
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const updatedAssistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
expect(updatedAssistantMessage.content).toContain("It's just blue duh!");
|
||||
});
|
||||
|
||||
test("Hide suggested actions after sending message", async () => {
|
||||
await chatPage.isElementVisible("suggested-actions");
|
||||
await chatPage.sendUserMessageFromSuggestion();
|
||||
await chatPage.isElementNotVisible("suggested-actions");
|
||||
});
|
||||
|
||||
test("Upload file and send image attachment with message", async () => {
|
||||
await chatPage.addImageAttachment();
|
||||
|
||||
await chatPage.isElementVisible("attachments-preview");
|
||||
await chatPage.isElementVisible("input-attachment-loader");
|
||||
await chatPage.isElementNotVisible("input-attachment-loader");
|
||||
|
||||
await chatPage.sendUserMessage("Who painted this?");
|
||||
|
||||
const userMessage = await chatPage.getRecentUserMessage();
|
||||
expect(userMessage.attachments).toHaveLength(1);
|
||||
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
expect(assistantMessage.content).toBe("This painting is by Monet!");
|
||||
});
|
||||
|
||||
test("Call weather tool", async () => {
|
||||
await chatPage.sendUserMessage("What's the weather in sf?");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
|
||||
expect(assistantMessage.content).toBe(
|
||||
"The current temperature in San Francisco is 17°C."
|
||||
);
|
||||
});
|
||||
|
||||
test("Upvote message", async () => {
|
||||
await chatPage.sendUserMessage("Why is the sky blue?");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
await assistantMessage.upvote();
|
||||
await chatPage.isVoteComplete();
|
||||
});
|
||||
|
||||
test("Downvote message", async () => {
|
||||
await chatPage.sendUserMessage("Why is the sky blue?");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
await assistantMessage.downvote();
|
||||
await chatPage.isVoteComplete();
|
||||
});
|
||||
|
||||
test("Update vote", async () => {
|
||||
await chatPage.sendUserMessage("Why is the sky blue?");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
await assistantMessage.upvote();
|
||||
await chatPage.isVoteComplete();
|
||||
|
||||
await assistantMessage.downvote();
|
||||
await chatPage.isVoteComplete();
|
||||
});
|
||||
|
||||
test("Create message from url query", async ({ page }) => {
|
||||
await page.goto("/?query=Why is the sky blue?");
|
||||
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const userMessage = await chatPage.getRecentUserMessage();
|
||||
expect(userMessage.content).toBe("Why is the sky blue?");
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
expect(assistantMessage.content).toContain("It's just blue duh!");
|
||||
});
|
||||
|
||||
test("auto-scrolls to bottom after submitting new messages", async () => {
|
||||
test.fixme();
|
||||
await chatPage.sendMultipleMessages(5, (i) => `filling message #${i}`);
|
||||
await chatPage.waitForScrollToBottom();
|
||||
});
|
||||
|
||||
test("scroll button appears when user scrolls up, hides on click", async () => {
|
||||
test.fixme();
|
||||
await chatPage.sendMultipleMessages(5, (i) => `filling message #${i}`);
|
||||
await expect(chatPage.scrollToBottomButton).not.toBeVisible();
|
||||
|
||||
await chatPage.scrollToTop();
|
||||
await expect(chatPage.scrollToBottomButton).toBeVisible();
|
||||
|
||||
await chatPage.scrollToBottomButton.click();
|
||||
await chatPage.waitForScrollToBottom();
|
||||
await expect(chatPage.scrollToBottomButton).not.toBeVisible();
|
||||
await expect(chatPage.sendButton).toBeVisible({ timeout: 5000 });
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("Chat - Guest User", () => {
|
||||
test("can use chat as guest", async ({ page }) => {
|
||||
const chatPage = new ChatPage(page);
|
||||
await chatPage.createNewChat();
|
||||
await chatPage.waitForInputToBeReady();
|
||||
await chatPage.sendUserMessage("Hello");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const content = await chatPage.getLastAssistantMessageContent();
|
||||
expect(content).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
54
tests/e2e/message-actions.test.ts
Normal file
54
tests/e2e/message-actions.test.ts
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
import { expect, test } from "@playwright/test";
|
||||
import { ChatPage } from "../pages/chat";
|
||||
|
||||
test.describe("Message Actions", () => {
|
||||
let chatPage: ChatPage;
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
chatPage = new ChatPage(page);
|
||||
await chatPage.createNewChat();
|
||||
await chatPage.sendUserMessage("Hello");
|
||||
await chatPage.isGenerationComplete();
|
||||
});
|
||||
|
||||
test("can upvote a message", async ({ page }) => {
|
||||
const upvoteButton = page.getByTestId("message-upvote").first();
|
||||
await upvoteButton.click();
|
||||
await expect(upvoteButton).toHaveAttribute("data-state", "active");
|
||||
});
|
||||
|
||||
test("can downvote a message", async ({ page }) => {
|
||||
const downvoteButton = page.getByTestId("message-downvote").first();
|
||||
await downvoteButton.click();
|
||||
await expect(downvoteButton).toHaveAttribute("data-state", "active");
|
||||
});
|
||||
|
||||
test("can copy message content", async ({ page, context }) => {
|
||||
await context.grantPermissions(["clipboard-read", "clipboard-write"]);
|
||||
|
||||
const copyButton = page.getByTestId("message-copy").first();
|
||||
await copyButton.click();
|
||||
|
||||
const clipboardContent = await page.evaluate(() =>
|
||||
navigator.clipboard.readText()
|
||||
);
|
||||
expect(clipboardContent.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test("can edit user message", async ({ page }) => {
|
||||
const editButton = page.getByTestId("message-edit-button").first();
|
||||
await editButton.click();
|
||||
|
||||
const editor = page.getByTestId("message-editor");
|
||||
await expect(editor).toBeVisible();
|
||||
await editor.fill("Updated message");
|
||||
|
||||
const sendButton = page.getByTestId("message-editor-send-button");
|
||||
await sendButton.click();
|
||||
|
||||
await chatPage.isGenerationComplete();
|
||||
const userContent = await chatPage.getLastUserMessageContent();
|
||||
expect(userContent).toBe("Updated message");
|
||||
});
|
||||
});
|
||||
|
||||
41
tests/e2e/model-selector.test.ts
Normal file
41
tests/e2e/model-selector.test.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
import { expect, test } from "@playwright/test";
|
||||
import { ChatPage } from "../pages/chat";
|
||||
|
||||
test.describe("Model Selector", () => {
|
||||
let chatPage: ChatPage;
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
chatPage = new ChatPage(page);
|
||||
await chatPage.createNewChat();
|
||||
});
|
||||
|
||||
test("displays default model on load", async ({ page }) => {
|
||||
const modelButton = page.locator("button").filter({ hasText: /Gemini|Claude|GPT/i }).first();
|
||||
await expect(modelButton).toBeVisible();
|
||||
});
|
||||
|
||||
test("opens model selector on click", async ({ page }) => {
|
||||
const modelButton = page.locator("button").filter({ hasText: /Gemini|Claude|GPT/i }).first();
|
||||
await modelButton.click();
|
||||
await expect(page.getByPlaceholder("Search models...")).toBeVisible();
|
||||
});
|
||||
|
||||
test("can search for models", async ({ page }) => {
|
||||
const modelButton = page.locator("button").filter({ hasText: /Gemini|Claude|GPT/i }).first();
|
||||
await modelButton.click();
|
||||
await page.getByPlaceholder("Search models...").fill("Claude");
|
||||
await expect(page.getByText("Claude", { exact: false })).toBeVisible();
|
||||
});
|
||||
|
||||
test("can select a different model", async ({ page }) => {
|
||||
const modelButton = page.locator("button").filter({ hasText: /Gemini|Claude|GPT/i }).first();
|
||||
await modelButton.click();
|
||||
|
||||
const modelOption = page.getByRole("option").first();
|
||||
const modelName = await modelOption.innerText();
|
||||
await modelOption.click();
|
||||
|
||||
await expect(page.locator("button").filter({ hasText: modelName.split("\n")[0] })).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
import { expect, test } from "../fixtures";
|
||||
import { ChatPage } from "../pages/chat";
|
||||
|
||||
test.describe("chat activity with reasoning", () => {
|
||||
let chatPage: ChatPage;
|
||||
|
||||
test.beforeEach(async ({ curieContext }) => {
|
||||
chatPage = new ChatPage(curieContext.page);
|
||||
await chatPage.createNewChat();
|
||||
});
|
||||
|
||||
test("Curie can send message and generate response with reasoning", async () => {
|
||||
await chatPage.sendUserMessage("Why is the sky blue?");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
expect(assistantMessage.content).toBe("It's just blue duh!");
|
||||
|
||||
expect(assistantMessage.reasoning).toBe(
|
||||
"The sky is blue because of rayleigh scattering!"
|
||||
);
|
||||
});
|
||||
|
||||
test("Curie can toggle reasoning visibility", async () => {
|
||||
await chatPage.sendUserMessage("Why is the sky blue?");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
const reasoningElement =
|
||||
assistantMessage.element.getByTestId("message-reasoning");
|
||||
expect(reasoningElement).toBeVisible();
|
||||
|
||||
await assistantMessage.toggleReasoningVisibility();
|
||||
await expect(reasoningElement).not.toBeVisible();
|
||||
|
||||
await assistantMessage.toggleReasoningVisibility();
|
||||
await expect(reasoningElement).toBeVisible();
|
||||
});
|
||||
|
||||
test("Curie can edit message and resubmit", async () => {
|
||||
await chatPage.sendUserMessage("Why is the sky blue?");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const assistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
const reasoningElement =
|
||||
assistantMessage.element.getByTestId("message-reasoning");
|
||||
expect(reasoningElement).toBeVisible();
|
||||
|
||||
const userMessage = await chatPage.getRecentUserMessage();
|
||||
|
||||
const generationCompletePromise = chatPage.isGenerationComplete();
|
||||
await userMessage.edit("Why is grass green?");
|
||||
await generationCompletePromise;
|
||||
|
||||
const updatedAssistantMessage = await chatPage.getRecentAssistantMessage();
|
||||
|
||||
expect(updatedAssistantMessage.content).toBe("It's just green duh!");
|
||||
|
||||
expect(updatedAssistantMessage.reasoning).toBe(
|
||||
"Grass is green because of chlorophyll absorption!"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,209 +0,0 @@
|
|||
import { getMessageByErrorCode } from "@/lib/errors";
|
||||
import { expect, test } from "../fixtures";
|
||||
import { generateRandomTestUser } from "../helpers";
|
||||
import { AuthPage } from "../pages/auth";
|
||||
import { ChatPage } from "../pages/chat";
|
||||
|
||||
test.describe
|
||||
.serial("Guest Session", () => {
|
||||
test("Authenticate as guest user when a new session is loaded", async ({
|
||||
page,
|
||||
}) => {
|
||||
const response = await page.goto("/");
|
||||
|
||||
if (!response) {
|
||||
throw new Error("Failed to load page");
|
||||
}
|
||||
|
||||
let request = response.request();
|
||||
|
||||
const chain: string[] = [];
|
||||
|
||||
while (request) {
|
||||
chain.unshift(request.url());
|
||||
request = request.redirectedFrom();
|
||||
}
|
||||
|
||||
expect(chain).toEqual([
|
||||
"http://localhost:3000/",
|
||||
"http://localhost:3000/api/auth/guest?redirectUrl=http%3A%2F%2Flocalhost%3A3000%2F",
|
||||
"http://localhost:3000/",
|
||||
]);
|
||||
});
|
||||
|
||||
test("Log out is not available for guest users", async ({ page }) => {
|
||||
await page.goto("/");
|
||||
|
||||
const sidebarToggleButton = page.getByTestId("sidebar-toggle-button");
|
||||
await sidebarToggleButton.click();
|
||||
|
||||
const userNavButton = page.getByTestId("user-nav-button");
|
||||
await expect(userNavButton).toBeVisible();
|
||||
|
||||
await userNavButton.click();
|
||||
const userNavMenu = page.getByTestId("user-nav-menu");
|
||||
await expect(userNavMenu).toBeVisible();
|
||||
|
||||
const authMenuItem = page.getByTestId("user-nav-item-auth");
|
||||
await expect(authMenuItem).toContainText("Login to your account");
|
||||
});
|
||||
|
||||
test("Do not authenticate as guest user when an existing non-guest session is active", async ({
|
||||
adaContext,
|
||||
}) => {
|
||||
const response = await adaContext.page.goto("/");
|
||||
|
||||
if (!response) {
|
||||
throw new Error("Failed to load page");
|
||||
}
|
||||
|
||||
let request = response.request();
|
||||
|
||||
const chain: string[] = [];
|
||||
|
||||
while (request) {
|
||||
chain.unshift(request.url());
|
||||
request = request.redirectedFrom();
|
||||
}
|
||||
|
||||
expect(chain).toEqual(["http://localhost:3000/"]);
|
||||
});
|
||||
|
||||
test("Allow navigating to /login as guest user", async ({ page }) => {
|
||||
await page.goto("/login");
|
||||
await page.waitForURL("/login");
|
||||
await expect(page).toHaveURL("/login");
|
||||
});
|
||||
|
||||
test("Allow navigating to /register as guest user", async ({ page }) => {
|
||||
await page.goto("/register");
|
||||
await page.waitForURL("/register");
|
||||
await expect(page).toHaveURL("/register");
|
||||
});
|
||||
|
||||
test("Do not show email in user menu for guest user", async ({ page }) => {
|
||||
await page.goto("/");
|
||||
|
||||
const sidebarToggleButton = page.getByTestId("sidebar-toggle-button");
|
||||
await sidebarToggleButton.click();
|
||||
|
||||
const userEmail = page.getByTestId("user-email");
|
||||
await expect(userEmail).toContainText("Guest");
|
||||
});
|
||||
});
|
||||
|
||||
test.describe
|
||||
.serial("Login and Registration", () => {
|
||||
let authPage: AuthPage;
|
||||
|
||||
const testUser = generateRandomTestUser();
|
||||
|
||||
test.beforeEach(({ page }) => {
|
||||
authPage = new AuthPage(page);
|
||||
});
|
||||
|
||||
test("Register new account", async () => {
|
||||
await authPage.register(testUser.email, testUser.password);
|
||||
await authPage.expectToastToContain("Account created successfully!");
|
||||
});
|
||||
|
||||
test("Register new account with existing email", async () => {
|
||||
await authPage.register(testUser.email, testUser.password);
|
||||
await authPage.expectToastToContain("Account already exists!");
|
||||
});
|
||||
|
||||
test("Log into account that exists", async ({ page }) => {
|
||||
await authPage.login(testUser.email, testUser.password);
|
||||
|
||||
await page.waitForURL("/");
|
||||
await expect(page.getByPlaceholder("Send a message...")).toBeVisible();
|
||||
});
|
||||
|
||||
test("Display user email in user menu", async ({ page }) => {
|
||||
await authPage.login(testUser.email, testUser.password);
|
||||
|
||||
await page.waitForURL("/");
|
||||
await expect(page.getByPlaceholder("Send a message...")).toBeVisible();
|
||||
|
||||
const userEmail = await page.getByTestId("user-email");
|
||||
await expect(userEmail).toHaveText(testUser.email);
|
||||
});
|
||||
|
||||
test("Log out as non-guest user", async () => {
|
||||
await authPage.logout(testUser.email, testUser.password);
|
||||
});
|
||||
|
||||
test("Do not force create a guest session if non-guest session already exists", async ({
|
||||
page,
|
||||
}) => {
|
||||
await authPage.login(testUser.email, testUser.password);
|
||||
await page.waitForURL("/");
|
||||
|
||||
const userEmail = await page.getByTestId("user-email");
|
||||
await expect(userEmail).toHaveText(testUser.email);
|
||||
|
||||
await page.goto("/api/auth/guest");
|
||||
await page.waitForURL("/");
|
||||
|
||||
const updatedUserEmail = await page.getByTestId("user-email");
|
||||
await expect(updatedUserEmail).toHaveText(testUser.email);
|
||||
});
|
||||
|
||||
test("Log out is available for non-guest users", async ({ page }) => {
|
||||
await authPage.login(testUser.email, testUser.password);
|
||||
await page.waitForURL("/");
|
||||
|
||||
authPage.openSidebar();
|
||||
|
||||
const userNavButton = page.getByTestId("user-nav-button");
|
||||
await expect(userNavButton).toBeVisible();
|
||||
|
||||
await userNavButton.click();
|
||||
const userNavMenu = page.getByTestId("user-nav-menu");
|
||||
await expect(userNavMenu).toBeVisible();
|
||||
|
||||
const authMenuItem = page.getByTestId("user-nav-item-auth");
|
||||
await expect(authMenuItem).toContainText("Sign out");
|
||||
});
|
||||
|
||||
test("Do not navigate to /register for non-guest users", async ({
|
||||
page,
|
||||
}) => {
|
||||
await authPage.login(testUser.email, testUser.password);
|
||||
await page.waitForURL("/");
|
||||
|
||||
await page.goto("/register");
|
||||
await expect(page).toHaveURL("/");
|
||||
});
|
||||
|
||||
test("Do not navigate to /login for non-guest users", async ({ page }) => {
|
||||
await authPage.login(testUser.email, testUser.password);
|
||||
await page.waitForURL("/");
|
||||
|
||||
await page.goto("/login");
|
||||
await expect(page).toHaveURL("/");
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("Entitlements", () => {
|
||||
let chatPage: ChatPage;
|
||||
|
||||
test.beforeEach(({ page }) => {
|
||||
chatPage = new ChatPage(page);
|
||||
});
|
||||
|
||||
test("Guest user cannot send more than 20 messages/day", async () => {
|
||||
test.fixme();
|
||||
await chatPage.createNewChat();
|
||||
|
||||
for (let i = 0; i <= 20; i++) {
|
||||
await chatPage.sendUserMessage("Why is the sky blue?");
|
||||
await chatPage.isGenerationComplete();
|
||||
}
|
||||
|
||||
await chatPage.sendUserMessage("Why is the sky blue?");
|
||||
await chatPage.expectToastToContain(
|
||||
getMessageByErrorCode("rate_limit:chat")
|
||||
);
|
||||
});
|
||||
});
|
||||
85
tests/e2e/sidebar.test.ts
Normal file
85
tests/e2e/sidebar.test.ts
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
import { expect, test } from "@playwright/test";
|
||||
import { ChatPage } from "../pages/chat";
|
||||
|
||||
test.describe("Sidebar", () => {
|
||||
test("can toggle sidebar open and closed", async ({ page }) => {
|
||||
const chatPage = new ChatPage(page);
|
||||
await chatPage.createNewChat();
|
||||
|
||||
const toggleButton = page.getByTestId("sidebar-toggle-button");
|
||||
const sidebar = page.getByTestId("sidebar");
|
||||
|
||||
await toggleButton.click();
|
||||
await expect(sidebar).toBeVisible();
|
||||
|
||||
await toggleButton.click();
|
||||
await expect(sidebar).not.toBeVisible();
|
||||
});
|
||||
|
||||
test("shows chat in history after sending message", async ({ page }) => {
|
||||
const chatPage = new ChatPage(page);
|
||||
await chatPage.createNewChat();
|
||||
await chatPage.sendUserMessage("Test message for history");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const toggleButton = page.getByTestId("sidebar-toggle-button");
|
||||
await toggleButton.click();
|
||||
|
||||
const historyItem = page.getByTestId("sidebar-chat-item").first();
|
||||
await expect(historyItem).toBeVisible();
|
||||
});
|
||||
|
||||
test("can navigate to chat from history", async ({ page }) => {
|
||||
const chatPage = new ChatPage(page);
|
||||
await chatPage.createNewChat();
|
||||
await chatPage.sendUserMessage("First chat message");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const firstChatUrl = page.url();
|
||||
|
||||
await page.goto("/");
|
||||
|
||||
const toggleButton = page.getByTestId("sidebar-toggle-button");
|
||||
await toggleButton.click();
|
||||
|
||||
const historyItem = page.getByTestId("sidebar-chat-item").first();
|
||||
await historyItem.click();
|
||||
|
||||
await expect(page).toHaveURL(firstChatUrl);
|
||||
});
|
||||
|
||||
test("can delete chat from history", async ({ page }) => {
|
||||
const chatPage = new ChatPage(page);
|
||||
await chatPage.createNewChat();
|
||||
await chatPage.sendUserMessage("Chat to delete");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const toggleButton = page.getByTestId("sidebar-toggle-button");
|
||||
await toggleButton.click();
|
||||
|
||||
const historyItem = page.getByTestId("sidebar-chat-item").first();
|
||||
await historyItem.hover();
|
||||
|
||||
const deleteButton = page.getByTestId("sidebar-chat-delete").first();
|
||||
await deleteButton.click();
|
||||
|
||||
const confirmButton = page.getByRole("button", { name: "Delete" });
|
||||
await confirmButton.click();
|
||||
|
||||
await expect(page.getByTestId("toast")).toContainText("deleted");
|
||||
});
|
||||
|
||||
test("can create new chat from sidebar", async ({ page }) => {
|
||||
const chatPage = new ChatPage(page);
|
||||
await chatPage.createNewChat();
|
||||
await chatPage.sendUserMessage("Initial message");
|
||||
await chatPage.isGenerationComplete();
|
||||
|
||||
const newChatButton = page.getByTestId("sidebar-new-chat");
|
||||
await newChatButton.click();
|
||||
|
||||
await expect(page).toHaveURL("/");
|
||||
await expect(page.getByTestId("multimodal-input")).toBeEmpty();
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -3,45 +3,19 @@ import { getUnixTime } from "date-fns";
|
|||
import { createAuthenticatedContext, type UserContext } from "./helpers";
|
||||
|
||||
type Fixtures = {
|
||||
adaContext: UserContext;
|
||||
babbageContext: UserContext;
|
||||
curieContext: UserContext;
|
||||
authenticatedContext: UserContext;
|
||||
};
|
||||
|
||||
export const test = baseTest.extend<object, Fixtures>({
|
||||
adaContext: [
|
||||
authenticatedContext: [
|
||||
async ({ browser }, use, workerInfo) => {
|
||||
const ada = await createAuthenticatedContext({
|
||||
const userContext = await createAuthenticatedContext({
|
||||
browser,
|
||||
name: `ada-${workerInfo.workerIndex}-${getUnixTime(new Date())}`,
|
||||
name: `user-${workerInfo.workerIndex}-${getUnixTime(new Date())}`,
|
||||
});
|
||||
|
||||
await use(ada);
|
||||
await ada.context.close();
|
||||
},
|
||||
{ scope: "worker" },
|
||||
],
|
||||
babbageContext: [
|
||||
async ({ browser }, use, workerInfo) => {
|
||||
const babbage = await createAuthenticatedContext({
|
||||
browser,
|
||||
name: `babbage-${workerInfo.workerIndex}-${getUnixTime(new Date())}`,
|
||||
});
|
||||
|
||||
await use(babbage);
|
||||
await babbage.context.close();
|
||||
},
|
||||
{ scope: "worker" },
|
||||
],
|
||||
curieContext: [
|
||||
async ({ browser }, use, workerInfo) => {
|
||||
const curie = await createAuthenticatedContext({
|
||||
browser,
|
||||
name: `curie-${workerInfo.workerIndex}-${getUnixTime(new Date())}`,
|
||||
});
|
||||
|
||||
await use(curie);
|
||||
await curie.context.close();
|
||||
await use(userContext);
|
||||
await userContext.context.close();
|
||||
},
|
||||
{ scope: "worker" },
|
||||
],
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import {
|
|||
} from "@playwright/test";
|
||||
import { generateId } from "ai";
|
||||
import { getUnixTime } from "date-fns";
|
||||
import { ChatPage } from "./pages/chat";
|
||||
|
||||
export type UserContext = {
|
||||
context: BrowserContext;
|
||||
|
|
@ -49,10 +48,8 @@ export async function createAuthenticatedContext({
|
|||
"Account created successfully!"
|
||||
);
|
||||
|
||||
const chatPage = new ChatPage(page);
|
||||
await chatPage.createNewChat();
|
||||
await chatPage.chooseModelFromSelector("chat-model-reasoning");
|
||||
await expect(chatPage.getSelectedModel()).resolves.toEqual("Reasoning model");
|
||||
// Wait for redirect to home page
|
||||
await page.waitForURL("/");
|
||||
|
||||
await page.waitForTimeout(1000);
|
||||
await context.storageState({ path: storageFile });
|
||||
|
|
|
|||
|
|
@ -1,120 +0,0 @@
|
|||
import { expect, type Page } from "@playwright/test";
|
||||
|
||||
export class ArtifactPage {
|
||||
private readonly page: Page;
|
||||
|
||||
constructor(page: Page) {
|
||||
this.page = page;
|
||||
}
|
||||
|
||||
get artifact() {
|
||||
return this.page.getByTestId("artifact");
|
||||
}
|
||||
|
||||
get sendButton() {
|
||||
return this.artifact.getByTestId("send-button");
|
||||
}
|
||||
|
||||
get stopButton() {
|
||||
return this.page.getByTestId("stop-button");
|
||||
}
|
||||
|
||||
get multimodalInput() {
|
||||
return this.page.getByTestId("multimodal-input");
|
||||
}
|
||||
|
||||
async isGenerationComplete() {
|
||||
const response = await this.page.waitForResponse((currentResponse) =>
|
||||
currentResponse.url().includes("/api/chat")
|
||||
);
|
||||
|
||||
await response.finished();
|
||||
}
|
||||
|
||||
async sendUserMessage(message: string) {
|
||||
await this.artifact.getByTestId("multimodal-input").click();
|
||||
await this.artifact.getByTestId("multimodal-input").fill(message);
|
||||
await this.artifact.getByTestId("send-button").click();
|
||||
}
|
||||
|
||||
async getRecentAssistantMessage() {
|
||||
const messageElements = await this.artifact
|
||||
.getByTestId("message-assistant")
|
||||
.all();
|
||||
const lastMessageElement = messageElements.at(-1);
|
||||
|
||||
if (!lastMessageElement) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const content = await lastMessageElement
|
||||
.getByTestId("message-content")
|
||||
.innerText()
|
||||
.catch(() => null);
|
||||
|
||||
const reasoningElement = await lastMessageElement
|
||||
.getByTestId("message-reasoning")
|
||||
.isVisible()
|
||||
.then(async (visible) =>
|
||||
visible
|
||||
? await lastMessageElement
|
||||
.getByTestId("message-reasoning")
|
||||
.innerText()
|
||||
: null
|
||||
)
|
||||
.catch(() => null);
|
||||
|
||||
return {
|
||||
element: lastMessageElement,
|
||||
content,
|
||||
reasoning: reasoningElement,
|
||||
async toggleReasoningVisibility() {
|
||||
await lastMessageElement
|
||||
.getByTestId("message-reasoning-toggle")
|
||||
.click();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async getRecentUserMessage() {
|
||||
const messageElements = await this.artifact
|
||||
.getByTestId("message-user")
|
||||
.all();
|
||||
const lastMessageElement = messageElements.at(-1);
|
||||
|
||||
if (!lastMessageElement) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const content = await lastMessageElement.innerText();
|
||||
|
||||
const hasAttachments = await lastMessageElement
|
||||
.getByTestId("message-attachments")
|
||||
.isVisible()
|
||||
.catch(() => false);
|
||||
|
||||
const attachments = hasAttachments
|
||||
? await lastMessageElement.getByTestId("message-attachments").all()
|
||||
: [];
|
||||
|
||||
const page = this.artifact;
|
||||
|
||||
return {
|
||||
element: lastMessageElement,
|
||||
content,
|
||||
attachments,
|
||||
async edit(newMessage: string) {
|
||||
await page.getByTestId("message-edit-button").click();
|
||||
await page.getByTestId("message-editor").fill(newMessage);
|
||||
await page.getByTestId("message-editor-send-button").click();
|
||||
await expect(
|
||||
page.getByTestId("message-editor-send-button")
|
||||
).not.toBeVisible();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
closeArtifact() {
|
||||
return this.page.getByTestId("artifact-close-button").click();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
import type { Page } from "@playwright/test";
|
||||
import { expect } from "../fixtures";
|
||||
|
||||
export class AuthPage {
|
||||
private readonly page: Page;
|
||||
|
||||
constructor(page: Page) {
|
||||
this.page = page;
|
||||
}
|
||||
|
||||
async gotoLogin() {
|
||||
await this.page.goto("/login");
|
||||
await expect(this.page.getByRole("heading")).toContainText("Sign In");
|
||||
}
|
||||
|
||||
async gotoRegister() {
|
||||
await this.page.goto("/register");
|
||||
await expect(this.page.getByRole("heading")).toContainText("Sign Up");
|
||||
}
|
||||
|
||||
async register(email: string, password: string) {
|
||||
await this.gotoRegister();
|
||||
await this.page.getByPlaceholder("user@acme.com").click();
|
||||
await this.page.getByPlaceholder("user@acme.com").fill(email);
|
||||
await this.page.getByLabel("Password").click();
|
||||
await this.page.getByLabel("Password").fill(password);
|
||||
await this.page.getByRole("button", { name: "Sign Up" }).click();
|
||||
}
|
||||
|
||||
async login(email: string, password: string) {
|
||||
await this.gotoLogin();
|
||||
await this.page.getByPlaceholder("user@acme.com").click();
|
||||
await this.page.getByPlaceholder("user@acme.com").fill(email);
|
||||
await this.page.getByLabel("Password").click();
|
||||
await this.page.getByLabel("Password").fill(password);
|
||||
await this.page.getByRole("button", { name: "Sign In" }).click();
|
||||
}
|
||||
|
||||
async logout(email: string, password: string) {
|
||||
await this.login(email, password);
|
||||
await this.page.waitForURL("/");
|
||||
|
||||
await this.openSidebar();
|
||||
|
||||
const userNavButton = this.page.getByTestId("user-nav-button");
|
||||
await expect(userNavButton).toBeVisible();
|
||||
|
||||
await userNavButton.click();
|
||||
const userNavMenu = this.page.getByTestId("user-nav-menu");
|
||||
await expect(userNavMenu).toBeVisible();
|
||||
|
||||
const authMenuItem = this.page.getByTestId("user-nav-item-auth");
|
||||
await expect(authMenuItem).toContainText("Sign out");
|
||||
|
||||
await authMenuItem.click();
|
||||
|
||||
const userEmail = this.page.getByTestId("user-email");
|
||||
await expect(userEmail).toContainText("Guest");
|
||||
}
|
||||
|
||||
async expectToastToContain(text: string) {
|
||||
await expect(this.page.getByTestId("toast")).toContainText(text);
|
||||
}
|
||||
|
||||
async openSidebar() {
|
||||
const sidebarToggleButton = this.page.getByTestId("sidebar-toggle-button");
|
||||
await sidebarToggleButton.click();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,4 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { expect, type Page } from "@playwright/test";
|
||||
import { chatModels } from "@/lib/ai/models";
|
||||
|
||||
const CHAT_ID_REGEX =
|
||||
/^http:\/\/localhost:3000\/chat\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
|
|
@ -25,16 +22,17 @@ export class ChatPage {
|
|||
return this.page.getByTestId("multimodal-input");
|
||||
}
|
||||
|
||||
get scrollContainer() {
|
||||
return this.page.locator(".overflow-y-scroll");
|
||||
get messagesContainer() {
|
||||
return this.page.locator("[data-testid='messages-container']");
|
||||
}
|
||||
|
||||
get scrollToBottomButton() {
|
||||
return this.page.getByTestId("scroll-to-bottom-button");
|
||||
async goto() {
|
||||
await this.page.goto("/");
|
||||
await this.page.waitForLoadState("networkidle");
|
||||
}
|
||||
|
||||
async createNewChat() {
|
||||
await this.page.goto("/");
|
||||
await this.goto();
|
||||
}
|
||||
|
||||
getCurrentURL(): string {
|
||||
|
|
@ -47,219 +45,69 @@ export class ChatPage {
|
|||
await this.sendButton.click();
|
||||
}
|
||||
|
||||
async isGenerationComplete() {
|
||||
const response = await this.page.waitForResponse((currentResponse) =>
|
||||
currentResponse.url().includes("/api/chat")
|
||||
async waitForResponse(timeout = 30_000) {
|
||||
const response = await this.page.waitForResponse(
|
||||
(res) => res.url().includes("/api/chat") && res.status() === 200,
|
||||
{ timeout }
|
||||
);
|
||||
|
||||
await response.finished();
|
||||
}
|
||||
|
||||
async isVoteComplete() {
|
||||
const response = await this.page.waitForResponse((currentResponse) =>
|
||||
currentResponse.url().includes("/api/vote")
|
||||
);
|
||||
|
||||
await response.finished();
|
||||
async isGenerationComplete(timeout = 30_000) {
|
||||
await this.waitForResponse(timeout);
|
||||
await this.page.waitForTimeout(500);
|
||||
}
|
||||
|
||||
async hasChatIdInUrl() {
|
||||
await expect(this.page).toHaveURL(CHAT_ID_REGEX);
|
||||
}
|
||||
|
||||
async sendUserMessageFromSuggestion() {
|
||||
await this.page
|
||||
.getByRole("button", { name: "What are the advantages of" })
|
||||
.click();
|
||||
async getAssistantMessages() {
|
||||
return await this.page.getByTestId("message-assistant").all();
|
||||
}
|
||||
|
||||
async isElementVisible(elementId: string) {
|
||||
await expect(this.page.getByTestId(elementId)).toBeVisible();
|
||||
async getUserMessages() {
|
||||
return await this.page.getByTestId("message-user").all();
|
||||
}
|
||||
|
||||
async isElementNotVisible(elementId: string) {
|
||||
await expect(this.page.getByTestId(elementId)).not.toBeVisible();
|
||||
}
|
||||
|
||||
async addImageAttachment() {
|
||||
this.page.on("filechooser", async (fileChooser) => {
|
||||
const filePath = path.join(
|
||||
process.cwd(),
|
||||
"public",
|
||||
"images",
|
||||
"mouth of the seine, monet.jpg"
|
||||
);
|
||||
const imageBuffer = fs.readFileSync(filePath);
|
||||
|
||||
await fileChooser.setFiles({
|
||||
name: "mouth of the seine, monet.jpg",
|
||||
mimeType: "image/jpeg",
|
||||
buffer: imageBuffer,
|
||||
});
|
||||
});
|
||||
|
||||
await this.page.getByTestId("attachments-button").click();
|
||||
}
|
||||
|
||||
async getSelectedModel() {
|
||||
const modelId = await this.page.getByTestId("model-selector").innerText();
|
||||
return modelId;
|
||||
}
|
||||
|
||||
async chooseModelFromSelector(chatModelId: string) {
|
||||
const chatModel = chatModels.find(
|
||||
(currentChatModel) => currentChatModel.id === chatModelId
|
||||
);
|
||||
|
||||
if (!chatModel) {
|
||||
throw new Error(`Model with id ${chatModelId} not found`);
|
||||
}
|
||||
|
||||
await this.page.getByTestId("model-selector").click();
|
||||
await this.page.getByTestId(`model-selector-item-${chatModelId}`).click();
|
||||
expect(await this.getSelectedModel()).toBe(chatModel.name);
|
||||
}
|
||||
|
||||
async getSelectedVisibility() {
|
||||
const visibilityId = await this.page
|
||||
.getByTestId("visibility-selector")
|
||||
.innerText();
|
||||
return visibilityId;
|
||||
}
|
||||
|
||||
async chooseVisibilityFromSelector(chatVisibility: "public" | "private") {
|
||||
await this.page.getByTestId("visibility-selector").click();
|
||||
await this.page
|
||||
.getByTestId(`visibility-selector-item-${chatVisibility}`)
|
||||
.click();
|
||||
expect(await this.getSelectedVisibility()).toBe(chatVisibility);
|
||||
}
|
||||
|
||||
async getRecentAssistantMessage() {
|
||||
const messageElements = await this.page
|
||||
.getByTestId("message-assistant")
|
||||
.all();
|
||||
const lastMessageElement = messageElements.at(-1);
|
||||
|
||||
if (!lastMessageElement) {
|
||||
async getLastAssistantMessageContent(): Promise<string | null> {
|
||||
const messages = await this.getAssistantMessages();
|
||||
const lastMessage = messages.at(-1);
|
||||
if (!lastMessage) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const content = await lastMessageElement
|
||||
const content = await lastMessage
|
||||
.getByTestId("message-content")
|
||||
.innerText()
|
||||
.catch(() => null);
|
||||
|
||||
const reasoningElement = await lastMessageElement
|
||||
.getByTestId("message-reasoning")
|
||||
.isVisible()
|
||||
.then(async (visible) =>
|
||||
visible
|
||||
? await lastMessageElement
|
||||
.getByTestId("message-reasoning")
|
||||
.innerText()
|
||||
: null
|
||||
)
|
||||
.catch(() => null);
|
||||
|
||||
return {
|
||||
element: lastMessageElement,
|
||||
content,
|
||||
reasoning: reasoningElement,
|
||||
async toggleReasoningVisibility() {
|
||||
await lastMessageElement
|
||||
.getByTestId("message-reasoning-toggle")
|
||||
.click();
|
||||
},
|
||||
async upvote() {
|
||||
await lastMessageElement.getByTestId("message-upvote").click();
|
||||
},
|
||||
async downvote() {
|
||||
await lastMessageElement.getByTestId("message-downvote").click();
|
||||
},
|
||||
};
|
||||
.innerText();
|
||||
return content;
|
||||
}
|
||||
|
||||
async getRecentUserMessage() {
|
||||
const messageElements = await this.page.getByTestId("message-user").all();
|
||||
const lastMessageElement = messageElements.at(-1);
|
||||
|
||||
if (!lastMessageElement) {
|
||||
throw new Error("No user message found");
|
||||
async getLastUserMessageContent(): Promise<string | null> {
|
||||
const messages = await this.getUserMessages();
|
||||
const lastMessage = messages.at(-1);
|
||||
if (!lastMessage) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const content = await lastMessageElement
|
||||
const content = await lastMessage
|
||||
.getByTestId("message-content")
|
||||
.innerText()
|
||||
.catch(() => null);
|
||||
.innerText();
|
||||
return content;
|
||||
}
|
||||
|
||||
const hasAttachments = await lastMessageElement
|
||||
.getByTestId("message-attachments")
|
||||
.isVisible()
|
||||
.catch(() => false);
|
||||
async isElementVisible(testId: string) {
|
||||
await expect(this.page.getByTestId(testId)).toBeVisible();
|
||||
}
|
||||
|
||||
const attachments = hasAttachments
|
||||
? await lastMessageElement.getByTestId("message-attachments").all()
|
||||
: [];
|
||||
|
||||
const page = this.page;
|
||||
|
||||
return {
|
||||
element: lastMessageElement,
|
||||
content,
|
||||
attachments,
|
||||
async edit(newMessage: string) {
|
||||
await page.getByTestId("message-edit-button").click();
|
||||
await page.getByTestId("message-editor").fill(newMessage);
|
||||
await page.getByTestId("message-editor-send-button").click();
|
||||
await expect(
|
||||
page.getByTestId("message-editor-send-button")
|
||||
).not.toBeVisible();
|
||||
},
|
||||
};
|
||||
async isElementNotVisible(testId: string) {
|
||||
await expect(this.page.getByTestId(testId)).not.toBeVisible();
|
||||
}
|
||||
|
||||
async expectToastToContain(text: string) {
|
||||
await expect(this.page.getByTestId("toast")).toContainText(text);
|
||||
}
|
||||
|
||||
async openSideBar() {
|
||||
const sidebarToggleButton = this.page.getByTestId("sidebar-toggle-button");
|
||||
await sidebarToggleButton.click();
|
||||
}
|
||||
|
||||
isScrolledToBottom(): Promise<boolean> {
|
||||
return this.scrollContainer.evaluate(
|
||||
(el) => Math.abs(el.scrollHeight - el.scrollTop - el.clientHeight) < 1
|
||||
);
|
||||
}
|
||||
|
||||
async waitForScrollToBottom(timeout = 5000): Promise<void> {
|
||||
const start = Date.now();
|
||||
|
||||
while (Date.now() - start < timeout) {
|
||||
if (await this.isScrolledToBottom()) {
|
||||
return;
|
||||
}
|
||||
await this.page.waitForTimeout(100);
|
||||
}
|
||||
|
||||
throw new Error(`Timed out waiting for scroll bottom after ${timeout}ms`);
|
||||
}
|
||||
|
||||
async sendMultipleMessages(
|
||||
count: number,
|
||||
makeMessage: (i: number) => string
|
||||
) {
|
||||
for (let i = 0; i < count; i++) {
|
||||
await this.sendUserMessage(makeMessage(i));
|
||||
await this.isGenerationComplete();
|
||||
}
|
||||
}
|
||||
|
||||
async scrollToTop(): Promise<void> {
|
||||
await this.scrollContainer.evaluate((element) => {
|
||||
element.scrollTop = 0;
|
||||
});
|
||||
async waitForInputToBeReady() {
|
||||
await expect(this.multimodalInput).toBeVisible();
|
||||
await expect(this.sendButton).toBeVisible();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,148 +0,0 @@
|
|||
import type { ModelMessage } from "ai";
|
||||
|
||||
export const TEST_PROMPTS: Record<string, ModelMessage> = {
|
||||
USER_SKY: {
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Why is the sky blue?" }],
|
||||
},
|
||||
USER_GRASS: {
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Why is grass green?" }],
|
||||
},
|
||||
USER_THANKS: {
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Thanks!" }],
|
||||
},
|
||||
USER_NEXTJS: {
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "What are the advantages of using Next.js?" },
|
||||
],
|
||||
},
|
||||
USER_IMAGE_ATTACHMENT: {
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "file",
|
||||
mediaType: "...",
|
||||
data: "...",
|
||||
},
|
||||
{
|
||||
type: "text",
|
||||
text: "Who painted this?",
|
||||
},
|
||||
],
|
||||
},
|
||||
USER_TEXT_ARTIFACT: {
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: "Help me write an essay about Silicon Valley",
|
||||
},
|
||||
],
|
||||
},
|
||||
CREATE_DOCUMENT_TEXT_CALL: {
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: "Essay about Silicon Valley",
|
||||
},
|
||||
],
|
||||
},
|
||||
CREATE_DOCUMENT_TEXT_RESULT: {
|
||||
role: "tool",
|
||||
content: [
|
||||
{
|
||||
type: "tool-result",
|
||||
toolCallId: "call_123",
|
||||
toolName: "createDocument",
|
||||
output: {
|
||||
type: "json",
|
||||
value: {
|
||||
id: "3ca386a4-40c6-4630-8ed1-84cbd46cc7eb",
|
||||
title: "Essay about Silicon Valley",
|
||||
kind: "text",
|
||||
content: "A document was created and is now visible to the user.",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
GET_WEATHER_CALL: {
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: "What's the weather in sf?",
|
||||
},
|
||||
],
|
||||
},
|
||||
GET_WEATHER_RESULT: {
|
||||
role: "tool",
|
||||
content: [
|
||||
{
|
||||
type: "tool-result",
|
||||
toolCallId: "call_456",
|
||||
toolName: "getWeather",
|
||||
output: {
|
||||
type: "json",
|
||||
value: {
|
||||
latitude: 37.763_283,
|
||||
longitude: -122.412_86,
|
||||
generationtime_ms: 0.064_492_225_646_972_66,
|
||||
utc_offset_seconds: -25_200,
|
||||
timezone: "America/Los_Angeles",
|
||||
timezone_abbreviation: "GMT-7",
|
||||
elevation: 18,
|
||||
current_units: {
|
||||
time: "iso8601",
|
||||
interval: "seconds",
|
||||
temperature_2m: "°C",
|
||||
},
|
||||
current: {
|
||||
time: "2025-03-10T14:00",
|
||||
interval: 900,
|
||||
temperature_2m: 17,
|
||||
},
|
||||
daily_units: {
|
||||
time: "iso8601",
|
||||
sunrise: "iso8601",
|
||||
sunset: "iso8601",
|
||||
},
|
||||
daily: {
|
||||
time: [
|
||||
"2025-03-10",
|
||||
"2025-03-11",
|
||||
"2025-03-12",
|
||||
"2025-03-13",
|
||||
"2025-03-14",
|
||||
"2025-03-15",
|
||||
"2025-03-16",
|
||||
],
|
||||
sunrise: [
|
||||
"2025-03-10T07:27",
|
||||
"2025-03-11T07:25",
|
||||
"2025-03-12T07:24",
|
||||
"2025-03-13T07:22",
|
||||
"2025-03-14T07:21",
|
||||
"2025-03-15T07:19",
|
||||
"2025-03-16T07:18",
|
||||
],
|
||||
sunset: [
|
||||
"2025-03-10T19:12",
|
||||
"2025-03-11T19:13",
|
||||
"2025-03-12T19:14",
|
||||
"2025-03-13T19:15",
|
||||
"2025-03-14T19:16",
|
||||
"2025-03-15T19:17",
|
||||
"2025-03-16T19:17",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
import { generateUUID } from "@/lib/utils";
|
||||
|
||||
export const TEST_PROMPTS = {
|
||||
SKY: {
|
||||
MESSAGE: {
|
||||
id: generateUUID(),
|
||||
createdAt: new Date().toISOString(),
|
||||
role: "user",
|
||||
content: "Why is the sky blue?",
|
||||
parts: [{ type: "text", text: "Why is the sky blue?" }],
|
||||
},
|
||||
OUTPUT_STREAM: [
|
||||
'data: {"type":"start-step"}',
|
||||
'data: {"type":"text-start","id":"STATIC_ID"}',
|
||||
'data: {"type":"text-delta","id":"STATIC_ID","delta":"It\'s "}',
|
||||
'data: {"type":"text-delta","id":"STATIC_ID","delta":"just "}',
|
||||
'data: {"type":"text-delta","id":"STATIC_ID","delta":"blue "}',
|
||||
'data: {"type":"text-delta","id":"STATIC_ID","delta":"duh! "}',
|
||||
'data: {"type":"text-end","id":"STATIC_ID"}',
|
||||
'data: {"type":"finish-step"}',
|
||||
'data: {"type":"finish"}',
|
||||
"data: [DONE]",
|
||||
],
|
||||
},
|
||||
GRASS: {
|
||||
MESSAGE: {
|
||||
id: generateUUID(),
|
||||
createdAt: new Date().toISOString(),
|
||||
role: "user",
|
||||
content: "Why is grass green?",
|
||||
parts: [{ type: "text", text: "Why is grass green?" }],
|
||||
},
|
||||
OUTPUT_STREAM: [
|
||||
'data: {"type":"start-step"}',
|
||||
'data: {"type":"text-start","id":"STATIC_ID"}',
|
||||
'data: {"type":"text-delta","id":"STATIC_ID","delta":"It\'s "}',
|
||||
'data: {"type":"text-delta","id":"STATIC_ID","delta":"just "}',
|
||||
'data: {"type":"text-delta","id":"STATIC_ID","delta":"green "}',
|
||||
'data: {"type":"text-delta","id":"STATIC_ID","delta":"duh! "}',
|
||||
'data: {"type":"text-end","id":"STATIC_ID"}',
|
||||
'data: {"type":"finish-step"}',
|
||||
'data: {"type":"finish"}',
|
||||
"data: [DONE]",
|
||||
],
|
||||
},
|
||||
};
|
||||
|
|
@ -1,282 +0,0 @@
|
|||
import type { LanguageModelV2StreamPart } from "@ai-sdk/provider";
|
||||
import { generateId, type ModelMessage } from "ai";
|
||||
import { TEST_PROMPTS } from "./basic";
|
||||
|
||||
export function compareMessages(
|
||||
firstMessage: ModelMessage,
|
||||
secondMessage: ModelMessage
|
||||
): boolean {
|
||||
if (firstMessage.role !== secondMessage.role) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (
|
||||
!Array.isArray(firstMessage.content) ||
|
||||
!Array.isArray(secondMessage.content)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (firstMessage.content.length !== secondMessage.content.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (let i = 0; i < firstMessage.content.length; i++) {
|
||||
const item1 = firstMessage.content[i];
|
||||
const item2 = secondMessage.content[i];
|
||||
|
||||
if (item1.type !== item2.type) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (item1.type === "file" && item2.type === "file") {
|
||||
// if (item1.image.toString() !== item2.image.toString()) return false;
|
||||
// if (item1.mimeType !== item2.mimeType) return false;
|
||||
} else if (item1.type === "text" && item2.type === "text") {
|
||||
if (item1.text !== item2.text) {
|
||||
return false;
|
||||
}
|
||||
} else if (item1.type === "tool-result" && item2.type === "tool-result") {
|
||||
if (item1.toolCallId !== item2.toolCallId) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
const textToDeltas = (text: string): LanguageModelV2StreamPart[] => {
|
||||
const id = generateId();
|
||||
|
||||
const deltas = text.split(" ").map((char) => ({
|
||||
id,
|
||||
type: "text-delta" as const,
|
||||
delta: `${char} `,
|
||||
}));
|
||||
|
||||
return [{ id, type: "text-start" }, ...deltas, { id, type: "text-end" }];
|
||||
};
|
||||
|
||||
const reasoningToDeltas = (text: string): LanguageModelV2StreamPart[] => {
|
||||
const id = generateId();
|
||||
|
||||
const deltas = text.split(" ").map((char) => ({
|
||||
id,
|
||||
type: "reasoning-delta" as const,
|
||||
delta: `${char} `,
|
||||
}));
|
||||
|
||||
return [
|
||||
{ id, type: "reasoning-start" },
|
||||
...deltas,
|
||||
{ id, type: "reasoning-end" },
|
||||
];
|
||||
};
|
||||
|
||||
export const getResponseChunksByPrompt = (
|
||||
prompt: ModelMessage[],
|
||||
isReasoningEnabled = false
|
||||
): LanguageModelV2StreamPart[] => {
|
||||
const recentMessage = prompt.at(-1);
|
||||
|
||||
if (!recentMessage) {
|
||||
throw new Error("No recent message found!");
|
||||
}
|
||||
|
||||
if (isReasoningEnabled) {
|
||||
if (compareMessages(recentMessage, TEST_PROMPTS.USER_SKY)) {
|
||||
return [
|
||||
...reasoningToDeltas("The sky is blue because of rayleigh scattering!"),
|
||||
...textToDeltas("It's just blue duh!"),
|
||||
{
|
||||
type: "finish",
|
||||
finishReason: "stop",
|
||||
usage: { inputTokens: 3, outputTokens: 10, totalTokens: 13 },
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (compareMessages(recentMessage, TEST_PROMPTS.USER_GRASS)) {
|
||||
return [
|
||||
...reasoningToDeltas(
|
||||
"Grass is green because of chlorophyll absorption!"
|
||||
),
|
||||
...textToDeltas("It's just green duh!"),
|
||||
{
|
||||
type: "finish",
|
||||
finishReason: "stop",
|
||||
usage: { inputTokens: 3, outputTokens: 10, totalTokens: 13 },
|
||||
},
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
if (compareMessages(recentMessage, TEST_PROMPTS.USER_THANKS)) {
|
||||
return [
|
||||
...textToDeltas("You're welcome!"),
|
||||
{
|
||||
type: "finish",
|
||||
finishReason: "stop",
|
||||
usage: { inputTokens: 3, outputTokens: 10, totalTokens: 13 },
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (compareMessages(recentMessage, TEST_PROMPTS.USER_GRASS)) {
|
||||
return [
|
||||
...textToDeltas("It's just green duh!"),
|
||||
{
|
||||
type: "finish",
|
||||
finishReason: "stop",
|
||||
usage: { inputTokens: 3, outputTokens: 10, totalTokens: 13 },
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (compareMessages(recentMessage, TEST_PROMPTS.USER_SKY)) {
|
||||
return [
|
||||
...textToDeltas("It's just blue duh!"),
|
||||
{
|
||||
type: "finish",
|
||||
finishReason: "stop",
|
||||
usage: { inputTokens: 3, outputTokens: 10, totalTokens: 13 },
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (compareMessages(recentMessage, TEST_PROMPTS.USER_NEXTJS)) {
|
||||
return [
|
||||
...textToDeltas("With Next.js, you can ship fast!"),
|
||||
|
||||
{
|
||||
type: "finish",
|
||||
finishReason: "stop",
|
||||
usage: { inputTokens: 3, outputTokens: 10, totalTokens: 13 },
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (compareMessages(recentMessage, TEST_PROMPTS.USER_IMAGE_ATTACHMENT)) {
|
||||
return [
|
||||
...textToDeltas("This painting is by Monet!"),
|
||||
{
|
||||
type: "finish",
|
||||
finishReason: "stop",
|
||||
usage: { inputTokens: 3, outputTokens: 10, totalTokens: 13 },
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (compareMessages(recentMessage, TEST_PROMPTS.USER_TEXT_ARTIFACT)) {
|
||||
const toolCallId = generateId();
|
||||
|
||||
return [
|
||||
{
|
||||
id: toolCallId,
|
||||
type: "tool-input-start",
|
||||
toolName: "createDocument",
|
||||
},
|
||||
{
|
||||
id: toolCallId,
|
||||
type: "tool-input-delta",
|
||||
delta: JSON.stringify({
|
||||
title: "Essay about Silicon Valley",
|
||||
kind: "text",
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: toolCallId,
|
||||
type: "tool-input-end",
|
||||
},
|
||||
{
|
||||
toolCallId,
|
||||
type: "tool-result",
|
||||
toolName: "createDocument",
|
||||
result: {
|
||||
id: "doc_123",
|
||||
title: "Essay about Silicon Valley",
|
||||
kind: "text",
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "finish",
|
||||
finishReason: "stop",
|
||||
usage: { inputTokens: 3, outputTokens: 10, totalTokens: 13 },
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (compareMessages(recentMessage, TEST_PROMPTS.CREATE_DOCUMENT_TEXT_CALL)) {
|
||||
return [
|
||||
...textToDeltas(`\n
|
||||
# Silicon Valley: The Epicenter of Innovation
|
||||
|
||||
## Origins and Evolution
|
||||
|
||||
Silicon Valley, nestled in the southern part of the San Francisco Bay Area, emerged as a global technology hub in the late 20th century. Its transformation began in the 1950s when Stanford University encouraged its graduates to start their own companies nearby, leading to the formation of pioneering semiconductor firms that gave the region its name.
|
||||
|
||||
## The Innovation Ecosystem
|
||||
|
||||
What makes Silicon Valley unique is its perfect storm of critical elements: prestigious universities like Stanford and Berkeley, abundant venture capital, a culture that celebrates risk-taking, and a dense network of talented individuals. This ecosystem has consistently nurtured groundbreaking technologies from personal computers to social media platforms to artificial intelligence.
|
||||
|
||||
## Challenges and Criticisms
|
||||
|
||||
Despite its remarkable success, Silicon Valley faces significant challenges including extreme income inequality, housing affordability crises, and questions about technology's impact on society. Critics argue the region has developed a monoculture that sometimes struggles with diversity and inclusion.
|
||||
|
||||
## Future Prospects
|
||||
|
||||
As we move forward, Silicon Valley continues to reinvent itself. While some predict its decline due to remote work trends and competition from other tech hubs, the region's adaptability and innovative spirit suggest it will remain influential in shaping our technological future for decades to come.
|
||||
`),
|
||||
{
|
||||
type: "finish",
|
||||
finishReason: "stop",
|
||||
usage: { inputTokens: 3, outputTokens: 10, totalTokens: 13 },
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (
|
||||
compareMessages(recentMessage, TEST_PROMPTS.CREATE_DOCUMENT_TEXT_RESULT)
|
||||
) {
|
||||
return [
|
||||
...textToDeltas("A document was created and is now visible to the user."),
|
||||
{
|
||||
type: "finish",
|
||||
finishReason: "stop",
|
||||
usage: { inputTokens: 3, outputTokens: 10, totalTokens: 13 },
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (compareMessages(recentMessage, TEST_PROMPTS.GET_WEATHER_CALL)) {
|
||||
return [
|
||||
{
|
||||
type: "tool-call",
|
||||
toolCallId: "call_456",
|
||||
toolName: "getWeather",
|
||||
input: JSON.stringify({ latitude: 37.7749, longitude: -122.4194 }),
|
||||
},
|
||||
{
|
||||
type: "finish",
|
||||
finishReason: "stop",
|
||||
usage: { inputTokens: 3, outputTokens: 10, totalTokens: 13 },
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (compareMessages(recentMessage, TEST_PROMPTS.GET_WEATHER_RESULT)) {
|
||||
return [
|
||||
...textToDeltas("The current temperature in San Francisco is 17°C."),
|
||||
{
|
||||
type: "finish",
|
||||
finishReason: "stop",
|
||||
usage: { inputTokens: 3, outputTokens: 10, totalTokens: 13 },
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
return [{ id: "6", type: "text-delta", delta: "Unknown test prompt!" }];
|
||||
};
|
||||
|
|
@ -1,366 +0,0 @@
|
|||
import { getMessageByErrorCode } from "@/lib/errors";
|
||||
import { generateUUID } from "@/lib/utils";
|
||||
import { expect, test } from "../fixtures";
|
||||
import { TEST_PROMPTS } from "../prompts/routes";
|
||||
|
||||
const chatIdsCreatedByAda: string[] = [];
|
||||
|
||||
// Helper function to normalize stream data for comparison
|
||||
function normalizeStreamData(lines: string[]): string[] {
|
||||
return lines.map((line) => {
|
||||
if (line.startsWith("data: ")) {
|
||||
try {
|
||||
const data = JSON.parse(line.slice(6)); // Remove 'data: ' prefix
|
||||
if (data.id) {
|
||||
// Replace dynamic id with a static one for comparison
|
||||
return `data: ${JSON.stringify({ ...data, id: "STATIC_ID" })}`;
|
||||
}
|
||||
return line;
|
||||
} catch {
|
||||
return line; // Return as-is if it's not valid JSON
|
||||
}
|
||||
}
|
||||
return line;
|
||||
});
|
||||
}
|
||||
|
||||
test.describe
|
||||
.serial("/api/chat", () => {
|
||||
test("Ada cannot invoke a chat generation with empty request body", async ({
|
||||
adaContext,
|
||||
}) => {
|
||||
const response = await adaContext.request.post("/api/chat", {
|
||||
data: JSON.stringify({}),
|
||||
});
|
||||
expect(response.status()).toBe(400);
|
||||
|
||||
const { code, message } = await response.json();
|
||||
expect(code).toEqual("bad_request:api");
|
||||
expect(message).toEqual(getMessageByErrorCode("bad_request:api"));
|
||||
});
|
||||
|
||||
test("Ada can invoke chat generation", async ({ adaContext }) => {
|
||||
const chatId = generateUUID();
|
||||
|
||||
const response = await adaContext.request.post("/api/chat", {
|
||||
data: {
|
||||
id: chatId,
|
||||
message: TEST_PROMPTS.SKY.MESSAGE,
|
||||
selectedChatModel: "chat-model",
|
||||
selectedVisibilityType: "private",
|
||||
},
|
||||
});
|
||||
expect(response.status()).toBe(200);
|
||||
|
||||
const text = await response.text();
|
||||
const lines = text.split("\n");
|
||||
|
||||
const [_, ...rest] = lines;
|
||||
const actualNormalized = normalizeStreamData(rest.filter(Boolean));
|
||||
const expectedNormalized = normalizeStreamData(
|
||||
TEST_PROMPTS.SKY.OUTPUT_STREAM
|
||||
);
|
||||
|
||||
expect(actualNormalized).toEqual(expectedNormalized);
|
||||
|
||||
chatIdsCreatedByAda.push(chatId);
|
||||
});
|
||||
|
||||
test("Babbage cannot append message to Ada's chat", async ({
|
||||
babbageContext,
|
||||
}) => {
|
||||
const [chatId] = chatIdsCreatedByAda;
|
||||
|
||||
const response = await babbageContext.request.post("/api/chat", {
|
||||
data: {
|
||||
id: chatId,
|
||||
message: TEST_PROMPTS.GRASS.MESSAGE,
|
||||
selectedChatModel: "chat-model",
|
||||
selectedVisibilityType: "private",
|
||||
},
|
||||
});
|
||||
expect(response.status()).toBe(403);
|
||||
|
||||
const { code, message } = await response.json();
|
||||
expect(code).toEqual("forbidden:chat");
|
||||
expect(message).toEqual(getMessageByErrorCode("forbidden:chat"));
|
||||
});
|
||||
|
||||
test("Babbage cannot delete Ada's chat", async ({ babbageContext }) => {
|
||||
const [chatId] = chatIdsCreatedByAda;
|
||||
|
||||
const response = await babbageContext.request.delete(
|
||||
`/api/chat?id=${chatId}`
|
||||
);
|
||||
expect(response.status()).toBe(403);
|
||||
|
||||
const { code, message } = await response.json();
|
||||
expect(code).toEqual("forbidden:chat");
|
||||
expect(message).toEqual(getMessageByErrorCode("forbidden:chat"));
|
||||
});
|
||||
|
||||
test("Ada can delete her own chat", async ({ adaContext }) => {
|
||||
const [chatId] = chatIdsCreatedByAda;
|
||||
|
||||
const response = await adaContext.request.delete(
|
||||
`/api/chat?id=${chatId}`
|
||||
);
|
||||
expect(response.status()).toBe(200);
|
||||
|
||||
const deletedChat = await response.json();
|
||||
expect(deletedChat).toMatchObject({ id: chatId });
|
||||
});
|
||||
|
||||
test("Ada cannot resume stream of chat that does not exist", async ({
|
||||
adaContext,
|
||||
}) => {
|
||||
const response = await adaContext.request.get(
|
||||
`/api/chat/${generateUUID()}/stream`
|
||||
);
|
||||
expect(response.status()).toBe(404);
|
||||
});
|
||||
|
||||
test("Ada can resume chat generation", async ({ adaContext }) => {
|
||||
const chatId = generateUUID();
|
||||
|
||||
const firstRequest = adaContext.request.post("/api/chat", {
|
||||
data: {
|
||||
id: chatId,
|
||||
message: {
|
||||
id: generateUUID(),
|
||||
role: "user",
|
||||
content: "Help me write an essay about Silcon Valley",
|
||||
parts: [
|
||||
{
|
||||
type: "text",
|
||||
text: "Help me write an essay about Silicon Valley",
|
||||
},
|
||||
],
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
selectedChatModel: "chat-model",
|
||||
selectedVisibilityType: "private",
|
||||
},
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
|
||||
const secondRequest = adaContext.request.get(
|
||||
`/api/chat/${chatId}/stream`
|
||||
);
|
||||
|
||||
const [firstResponse, secondResponse] = await Promise.all([
|
||||
firstRequest,
|
||||
secondRequest,
|
||||
]);
|
||||
|
||||
const [firstStatusCode, secondStatusCode] = await Promise.all([
|
||||
firstResponse.status(),
|
||||
secondResponse.status(),
|
||||
]);
|
||||
|
||||
expect(firstStatusCode).toBe(200);
|
||||
expect(secondStatusCode).toBe(200);
|
||||
|
||||
const [firstResponseBody, secondResponseBody] = await Promise.all([
|
||||
await firstResponse.body(),
|
||||
await secondResponse.body(),
|
||||
]);
|
||||
|
||||
expect(firstResponseBody.toString()).toEqual(
|
||||
secondResponseBody.toString()
|
||||
);
|
||||
});
|
||||
|
||||
test("Ada can resume chat generation that has ended during request", async ({
|
||||
adaContext,
|
||||
}) => {
|
||||
const chatId = generateUUID();
|
||||
|
||||
const firstRequest = await adaContext.request.post("/api/chat", {
|
||||
data: {
|
||||
id: chatId,
|
||||
message: {
|
||||
id: generateUUID(),
|
||||
role: "user",
|
||||
content: "Help me write an essay about Silcon Valley",
|
||||
parts: [
|
||||
{
|
||||
type: "text",
|
||||
text: "Help me write an essay about Silicon Valley",
|
||||
},
|
||||
],
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
selectedChatModel: "chat-model",
|
||||
selectedVisibilityType: "private",
|
||||
},
|
||||
});
|
||||
|
||||
const secondRequest = adaContext.request.get(
|
||||
`/api/chat/${chatId}/stream`
|
||||
);
|
||||
|
||||
const [firstResponse, secondResponse] = await Promise.all([
|
||||
firstRequest,
|
||||
secondRequest,
|
||||
]);
|
||||
|
||||
const [firstStatusCode, secondStatusCode] = await Promise.all([
|
||||
firstResponse.status(),
|
||||
secondResponse.status(),
|
||||
]);
|
||||
|
||||
expect(firstStatusCode).toBe(200);
|
||||
expect(secondStatusCode).toBe(200);
|
||||
|
||||
const [, secondResponseContent] = await Promise.all([
|
||||
firstResponse.text(),
|
||||
secondResponse.text(),
|
||||
]);
|
||||
|
||||
expect(secondResponseContent).toContain("appendMessage");
|
||||
});
|
||||
|
||||
test("Ada cannot resume chat generation that has ended", async ({
|
||||
adaContext,
|
||||
}) => {
|
||||
const chatId = generateUUID();
|
||||
|
||||
const firstResponse = await adaContext.request.post("/api/chat", {
|
||||
data: {
|
||||
id: chatId,
|
||||
message: {
|
||||
id: generateUUID(),
|
||||
role: "user",
|
||||
content: "Help me write an essay about Silcon Valley",
|
||||
parts: [
|
||||
{
|
||||
type: "text",
|
||||
text: "Help me write an essay about Silicon Valley",
|
||||
},
|
||||
],
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
selectedChatModel: "chat-model",
|
||||
selectedVisibilityType: "private",
|
||||
},
|
||||
});
|
||||
|
||||
const firstStatusCode = firstResponse.status();
|
||||
expect(firstStatusCode).toBe(200);
|
||||
|
||||
await firstResponse.text();
|
||||
await new Promise((resolve) => setTimeout(resolve, 15 * 1000));
|
||||
await new Promise((resolve) => setTimeout(resolve, 15_000));
|
||||
const secondResponse = await adaContext.request.get(
|
||||
`/api/chat/${chatId}/stream`
|
||||
);
|
||||
|
||||
const secondStatusCode = secondResponse.status();
|
||||
expect(secondStatusCode).toBe(200);
|
||||
|
||||
const secondResponseContent = await secondResponse.text();
|
||||
expect(secondResponseContent).toEqual("");
|
||||
});
|
||||
|
||||
test("Babbage cannot resume a private chat generation that belongs to Ada", async ({
|
||||
adaContext,
|
||||
babbageContext,
|
||||
}) => {
|
||||
const chatId = generateUUID();
|
||||
|
||||
const firstRequest = adaContext.request.post("/api/chat", {
|
||||
data: {
|
||||
id: chatId,
|
||||
message: {
|
||||
id: generateUUID(),
|
||||
role: "user",
|
||||
content: "Help me write an essay about Silcon Valley",
|
||||
parts: [
|
||||
{
|
||||
type: "text",
|
||||
text: "Help me write an essay about Silicon Valley",
|
||||
},
|
||||
],
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
selectedChatModel: "chat-model",
|
||||
selectedVisibilityType: "private",
|
||||
},
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
|
||||
const secondRequest = babbageContext.request.get(
|
||||
`/api/chat/${chatId}/stream`
|
||||
);
|
||||
|
||||
const [firstResponse, secondResponse] = await Promise.all([
|
||||
firstRequest,
|
||||
secondRequest,
|
||||
]);
|
||||
|
||||
const [firstStatusCode, secondStatusCode] = await Promise.all([
|
||||
firstResponse.status(),
|
||||
secondResponse.status(),
|
||||
]);
|
||||
|
||||
expect(firstStatusCode).toBe(200);
|
||||
expect(secondStatusCode).toBe(403);
|
||||
});
|
||||
|
||||
test("Babbage can resume a public chat generation that belongs to Ada", async ({
|
||||
adaContext,
|
||||
babbageContext,
|
||||
}) => {
|
||||
test.fixme();
|
||||
const chatId = generateUUID();
|
||||
|
||||
const firstRequest = adaContext.request.post("/api/chat", {
|
||||
data: {
|
||||
id: chatId,
|
||||
message: {
|
||||
id: generateUUID(),
|
||||
role: "user",
|
||||
content: "Help me write an essay about Silicon Valley",
|
||||
parts: [
|
||||
{
|
||||
type: "text",
|
||||
text: "Help me write an essay about Silicon Valley",
|
||||
},
|
||||
],
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
selectedChatModel: "chat-model",
|
||||
selectedVisibilityType: "public",
|
||||
},
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 10 * 1000));
|
||||
|
||||
const secondRequest = babbageContext.request.get(
|
||||
`/api/chat/${chatId}/stream`
|
||||
);
|
||||
|
||||
const [firstResponse, secondResponse] = await Promise.all([
|
||||
firstRequest,
|
||||
secondRequest,
|
||||
]);
|
||||
|
||||
const [firstStatusCode, secondStatusCode] = await Promise.all([
|
||||
firstResponse.status(),
|
||||
secondResponse.status(),
|
||||
]);
|
||||
|
||||
expect(firstStatusCode).toBe(200);
|
||||
expect(secondStatusCode).toBe(200);
|
||||
|
||||
const [firstResponseContent, secondResponseContent] = await Promise.all([
|
||||
firstResponse.text(),
|
||||
secondResponse.text(),
|
||||
]);
|
||||
|
||||
expect(firstResponseContent).toEqual(secondResponseContent);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,211 +0,0 @@
|
|||
import type { Document } from "@/lib/db/schema";
|
||||
import { getMessageByErrorCode } from "@/lib/errors";
|
||||
import { generateUUID } from "@/lib/utils";
|
||||
import { expect, test } from "../fixtures";
|
||||
|
||||
const documentsCreatedByAda: Document[] = [];
|
||||
|
||||
test.describe
|
||||
.serial("/api/document", () => {
|
||||
test("Ada cannot retrieve a document without specifying an id", async ({
|
||||
adaContext,
|
||||
}) => {
|
||||
const response = await adaContext.request.get("/api/document");
|
||||
expect(response.status()).toBe(400);
|
||||
|
||||
const { code, message } = await response.json();
|
||||
expect(code).toEqual("bad_request:api");
|
||||
expect(message).toEqual(getMessageByErrorCode(code));
|
||||
});
|
||||
|
||||
test("Ada cannot retrieve a document that does not exist", async ({
|
||||
adaContext,
|
||||
}) => {
|
||||
const documentId = generateUUID();
|
||||
|
||||
const response = await adaContext.request.get(
|
||||
`/api/document?id=${documentId}`
|
||||
);
|
||||
expect(response.status()).toBe(404);
|
||||
|
||||
const { code, message } = await response.json();
|
||||
expect(code).toEqual("not_found:document");
|
||||
expect(message).toEqual(getMessageByErrorCode(code));
|
||||
});
|
||||
|
||||
test("Ada can create a document", async ({ adaContext }) => {
|
||||
const documentId = generateUUID();
|
||||
|
||||
const draftDocument = {
|
||||
title: "Ada's Document",
|
||||
kind: "text",
|
||||
content: "Created by Ada",
|
||||
};
|
||||
|
||||
const response = await adaContext.request.post(
|
||||
`/api/document?id=${documentId}`,
|
||||
{
|
||||
data: draftDocument,
|
||||
}
|
||||
);
|
||||
expect(response.status()).toBe(200);
|
||||
|
||||
const [createdDocument] = await response.json();
|
||||
expect(createdDocument).toMatchObject(draftDocument);
|
||||
|
||||
documentsCreatedByAda.push(createdDocument);
|
||||
});
|
||||
|
||||
test("Ada can retrieve a created document", async ({ adaContext }) => {
|
||||
const [document] = documentsCreatedByAda;
|
||||
|
||||
const response = await adaContext.request.get(
|
||||
`/api/document?id=${document.id}`
|
||||
);
|
||||
expect(response.status()).toBe(200);
|
||||
|
||||
const retrievedDocuments = await response.json();
|
||||
expect(retrievedDocuments).toHaveLength(1);
|
||||
|
||||
const [retrievedDocument] = retrievedDocuments;
|
||||
expect(retrievedDocument).toMatchObject(document);
|
||||
});
|
||||
|
||||
test("Ada can save a new version of the document", async ({
|
||||
adaContext,
|
||||
}) => {
|
||||
const [firstDocument] = documentsCreatedByAda;
|
||||
|
||||
const draftDocument = {
|
||||
title: "Ada's Document",
|
||||
kind: "text",
|
||||
content: "Updated by Ada",
|
||||
};
|
||||
|
||||
const response = await adaContext.request.post(
|
||||
`/api/document?id=${firstDocument.id}`,
|
||||
{
|
||||
data: draftDocument,
|
||||
}
|
||||
);
|
||||
expect(response.status()).toBe(200);
|
||||
|
||||
const [createdDocument] = await response.json();
|
||||
expect(createdDocument).toMatchObject(draftDocument);
|
||||
|
||||
documentsCreatedByAda.push(createdDocument);
|
||||
});
|
||||
|
||||
test("Ada can retrieve all versions of her documents", async ({
|
||||
adaContext,
|
||||
}) => {
|
||||
const [firstDocument, secondDocument] = documentsCreatedByAda;
|
||||
|
||||
const response = await adaContext.request.get(
|
||||
`/api/document?id=${firstDocument.id}`
|
||||
);
|
||||
expect(response.status()).toBe(200);
|
||||
|
||||
const retrievedDocuments = await response.json();
|
||||
expect(retrievedDocuments).toHaveLength(2);
|
||||
|
||||
const [firstRetrievedDocument, secondRetrievedDocument] =
|
||||
retrievedDocuments;
|
||||
expect(firstRetrievedDocument).toMatchObject(firstDocument);
|
||||
expect(secondRetrievedDocument).toMatchObject(secondDocument);
|
||||
});
|
||||
|
||||
test("Ada cannot delete a document without specifying an id", async ({
|
||||
adaContext,
|
||||
}) => {
|
||||
const response = await adaContext.request.delete("/api/document");
|
||||
expect(response.status()).toBe(400);
|
||||
|
||||
const { code, message } = await response.json();
|
||||
expect(code).toEqual("bad_request:api");
|
||||
expect(message).toEqual(getMessageByErrorCode(code));
|
||||
});
|
||||
|
||||
test("Ada cannot delete a document without specifying a timestamp", async ({
|
||||
adaContext,
|
||||
}) => {
|
||||
const [firstDocument] = documentsCreatedByAda;
|
||||
|
||||
const response = await adaContext.request.delete(
|
||||
`/api/document?id=${firstDocument.id}`
|
||||
);
|
||||
expect(response.status()).toBe(400);
|
||||
|
||||
const { code, message } = await response.json();
|
||||
expect(code).toEqual("bad_request:api");
|
||||
expect(message).toEqual(getMessageByErrorCode(code));
|
||||
});
|
||||
|
||||
test("Ada can delete a document by specifying id and timestamp", async ({
|
||||
adaContext,
|
||||
}) => {
|
||||
const [firstDocument, secondDocument] = documentsCreatedByAda;
|
||||
|
||||
const response = await adaContext.request.delete(
|
||||
`/api/document?id=${firstDocument.id}×tamp=${firstDocument.createdAt}`
|
||||
);
|
||||
expect(response.status()).toBe(200);
|
||||
|
||||
const deletedDocuments = await response.json();
|
||||
expect(deletedDocuments).toHaveLength(1);
|
||||
|
||||
const [deletedDocument] = deletedDocuments;
|
||||
expect(deletedDocument).toMatchObject(secondDocument);
|
||||
});
|
||||
|
||||
test("Ada can retrieve documents without deleted versions", async ({
|
||||
adaContext,
|
||||
}) => {
|
||||
const [firstDocument] = documentsCreatedByAda;
|
||||
|
||||
const response = await adaContext.request.get(
|
||||
`/api/document?id=${firstDocument.id}`
|
||||
);
|
||||
expect(response.status()).toBe(200);
|
||||
|
||||
const retrievedDocuments = await response.json();
|
||||
expect(retrievedDocuments).toHaveLength(1);
|
||||
|
||||
const [firstRetrievedDocument] = retrievedDocuments;
|
||||
expect(firstRetrievedDocument).toMatchObject(firstDocument);
|
||||
});
|
||||
|
||||
test("Babbage cannot update Ada's document", async ({ babbageContext }) => {
|
||||
const [firstDocument] = documentsCreatedByAda;
|
||||
|
||||
const draftDocument = {
|
||||
title: "Babbage's Document",
|
||||
kind: "text",
|
||||
content: "Created by Babbage",
|
||||
};
|
||||
|
||||
const response = await babbageContext.request.post(
|
||||
`/api/document?id=${firstDocument.id}`,
|
||||
{
|
||||
data: draftDocument,
|
||||
}
|
||||
);
|
||||
expect(response.status()).toBe(403);
|
||||
|
||||
const { code, message } = await response.json();
|
||||
expect(code).toEqual("forbidden:document");
|
||||
expect(message).toEqual(getMessageByErrorCode(code));
|
||||
});
|
||||
|
||||
test("Ada's documents did not get updated", async ({ adaContext }) => {
|
||||
const [firstDocument] = documentsCreatedByAda;
|
||||
|
||||
const response = await adaContext.request.get(
|
||||
`/api/document?id=${firstDocument.id}`
|
||||
);
|
||||
expect(response.status()).toBe(200);
|
||||
|
||||
const documentsRetrieved = await response.json();
|
||||
expect(documentsRetrieved).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue