refactor: replace message.content with message.parts (#868)

This commit is contained in:
Jeremy 2025-03-16 18:42:29 -07:00 committed by GitHub
parent 553a3d825a
commit 47a630fd53
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 1311 additions and 311 deletions

View file

@ -0,0 +1,196 @@
import { config } from 'dotenv';
import postgres from 'postgres';
import {
chat,
message,
messageDeprecated,
vote,
voteDeprecated,
} from '../schema';
import { drizzle } from 'drizzle-orm/postgres-js';
import { inArray } from 'drizzle-orm';
import { appendResponseMessages, UIMessage } from 'ai';
config({
path: '.env.local',
});
if (!process.env.POSTGRES_URL) {
throw new Error('POSTGRES_URL environment variable is not set');
}
const client = postgres(process.env.POSTGRES_URL);
const db = drizzle(client);
const BATCH_SIZE = 50; // Process 10 chats at a time
const INSERT_BATCH_SIZE = 100; // Insert 100 messages at a time
type NewMessageInsert = {
id: string;
chatId: string;
parts: any[];
role: string;
attachments: any[];
createdAt: Date;
};
type NewVoteInsert = {
messageId: string;
chatId: string;
isUpvoted: boolean;
};
async function createNewTable() {
const chats = await db.select().from(chat);
let processedCount = 0;
// Process chats in batches
for (let i = 0; i < chats.length; i += BATCH_SIZE) {
const chatBatch = chats.slice(i, i + BATCH_SIZE);
const chatIds = chatBatch.map((chat) => chat.id);
// Fetch all messages and votes for the current batch of chats in bulk
const allMessages = await db
.select()
.from(messageDeprecated)
.where(inArray(messageDeprecated.chatId, chatIds));
const allVotes = await db
.select()
.from(voteDeprecated)
.where(inArray(voteDeprecated.chatId, chatIds));
// Prepare batches for insertion
const newMessagesToInsert: NewMessageInsert[] = [];
const newVotesToInsert: NewVoteInsert[] = [];
// Process each chat in the batch
for (const chat of chatBatch) {
processedCount++;
console.info(`Processed ${processedCount}/${chats.length} chats`);
// Filter messages and votes for this specific chat
const messages = allMessages.filter((msg) => msg.chatId === chat.id);
const votes = allVotes.filter((v) => v.chatId === chat.id);
// Group messages into sections
const messageSection: Array<UIMessage> = [];
const messageSections: Array<Array<UIMessage>> = [];
for (const message of messages) {
const { role } = message;
if (role === 'user' && messageSection.length > 0) {
messageSections.push([...messageSection]);
messageSection.length = 0;
}
// @ts-expect-error message.content has different type
messageSection.push(message);
}
if (messageSection.length > 0) {
messageSections.push([...messageSection]);
}
// Process each message section
for (const section of messageSections) {
const [userMessage, ...assistantMessages] = section;
const [firstAssistantMessage] = assistantMessages;
try {
const uiSection = appendResponseMessages({
messages: [userMessage],
// @ts-expect-error: message.content has different type
responseMessages: assistantMessages,
_internal: {
currentDate: () => firstAssistantMessage.createdAt ?? new Date(),
},
});
const projectedUISection = uiSection
.map((message) => {
if (message.role === 'user') {
return {
id: message.id,
chatId: chat.id,
parts: [{ type: 'text', text: message.content }],
role: message.role,
createdAt: message.createdAt,
attachments: [],
} as NewMessageInsert;
} else if (message.role === 'assistant') {
return {
id: message.id,
chatId: chat.id,
parts: message.parts || [],
role: message.role,
createdAt: message.createdAt,
attachments: [],
} as NewMessageInsert;
}
return null;
})
.filter((msg): msg is NewMessageInsert => msg !== null);
// Add messages to batch
for (const msg of projectedUISection) {
newMessagesToInsert.push(msg);
if (msg.role === 'assistant') {
const voteByMessage = votes.find((v) => v.messageId === msg.id);
if (voteByMessage) {
newVotesToInsert.push({
messageId: msg.id,
chatId: msg.chatId,
isUpvoted: voteByMessage.isUpvoted,
});
}
}
}
} catch (error) {
console.error(`Error processing chat ${chat.id}: ${error}`);
}
}
}
// Batch insert messages
for (let j = 0; j < newMessagesToInsert.length; j += INSERT_BATCH_SIZE) {
const messageBatch = newMessagesToInsert.slice(j, j + INSERT_BATCH_SIZE);
if (messageBatch.length > 0) {
// Ensure all required fields are present
const validMessageBatch = messageBatch.map((msg) => ({
id: msg.id,
chatId: msg.chatId,
parts: msg.parts,
role: msg.role,
attachments: msg.attachments,
createdAt: msg.createdAt,
}));
await db.insert(message).values(validMessageBatch);
}
}
// Batch insert votes
for (let j = 0; j < newVotesToInsert.length; j += INSERT_BATCH_SIZE) {
const voteBatch = newVotesToInsert.slice(j, j + INSERT_BATCH_SIZE);
if (voteBatch.length > 0) {
await db.insert(vote).values(voteBatch);
}
}
}
console.info(`Migration completed: ${processedCount} chats processed`);
}
createNewTable()
.then(() => {
console.info('Script completed successfully');
process.exit(0);
})
.catch((error) => {
console.error('Script failed:', error);
process.exit(1);
});

View file

@ -0,0 +1,33 @@
CREATE TABLE IF NOT EXISTS "Message_v2" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"chatId" uuid NOT NULL,
"role" varchar NOT NULL,
"parts" json NOT NULL,
"attachments" json NOT NULL,
"createdAt" timestamp NOT NULL
);
--> statement-breakpoint
CREATE TABLE IF NOT EXISTS "Vote_v2" (
"chatId" uuid NOT NULL,
"messageId" uuid NOT NULL,
"isUpvoted" boolean NOT NULL,
CONSTRAINT "Vote_v2_chatId_messageId_pk" PRIMARY KEY("chatId","messageId")
);
--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "Message_v2" ADD CONSTRAINT "Message_v2_chatId_Chat_id_fk" FOREIGN KEY ("chatId") REFERENCES "public"."Chat"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "Vote_v2" ADD CONSTRAINT "Vote_v2_chatId_Chat_id_fk" FOREIGN KEY ("chatId") REFERENCES "public"."Chat"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "Vote_v2" ADD CONSTRAINT "Vote_v2_messageId_Message_v2_id_fk" FOREIGN KEY ("messageId") REFERENCES "public"."Message_v2"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN null;
END $$;

View file

@ -0,0 +1,515 @@
{
"id": "c6c102e6-b64e-4f0c-a7a6-32df758de437",
"prevId": "30ad8233-1432-428b-93fc-2bb1ba867ff1",
"version": "7",
"dialect": "postgresql",
"tables": {
"public.Chat": {
"name": "Chat",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"createdAt": {
"name": "createdAt",
"type": "timestamp",
"primaryKey": false,
"notNull": true
},
"title": {
"name": "title",
"type": "text",
"primaryKey": false,
"notNull": true
},
"userId": {
"name": "userId",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"visibility": {
"name": "visibility",
"type": "varchar",
"primaryKey": false,
"notNull": true,
"default": "'private'"
}
},
"indexes": {},
"foreignKeys": {
"Chat_userId_User_id_fk": {
"name": "Chat_userId_User_id_fk",
"tableFrom": "Chat",
"tableTo": "User",
"columnsFrom": [
"userId"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"public.Document": {
"name": "Document",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": false,
"notNull": true,
"default": "gen_random_uuid()"
},
"createdAt": {
"name": "createdAt",
"type": "timestamp",
"primaryKey": false,
"notNull": true
},
"title": {
"name": "title",
"type": "text",
"primaryKey": false,
"notNull": true
},
"content": {
"name": "content",
"type": "text",
"primaryKey": false,
"notNull": false
},
"text": {
"name": "text",
"type": "varchar",
"primaryKey": false,
"notNull": true,
"default": "'text'"
},
"userId": {
"name": "userId",
"type": "uuid",
"primaryKey": false,
"notNull": true
}
},
"indexes": {},
"foreignKeys": {
"Document_userId_User_id_fk": {
"name": "Document_userId_User_id_fk",
"tableFrom": "Document",
"tableTo": "User",
"columnsFrom": [
"userId"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {
"Document_id_createdAt_pk": {
"name": "Document_id_createdAt_pk",
"columns": [
"id",
"createdAt"
]
}
},
"uniqueConstraints": {}
},
"public.Message_v2": {
"name": "Message_v2",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"chatId": {
"name": "chatId",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"role": {
"name": "role",
"type": "varchar",
"primaryKey": false,
"notNull": true
},
"parts": {
"name": "parts",
"type": "json",
"primaryKey": false,
"notNull": true
},
"attachments": {
"name": "attachments",
"type": "json",
"primaryKey": false,
"notNull": true
},
"createdAt": {
"name": "createdAt",
"type": "timestamp",
"primaryKey": false,
"notNull": true
}
},
"indexes": {},
"foreignKeys": {
"Message_v2_chatId_Chat_id_fk": {
"name": "Message_v2_chatId_Chat_id_fk",
"tableFrom": "Message_v2",
"tableTo": "Chat",
"columnsFrom": [
"chatId"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"public.Message": {
"name": "Message",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"chatId": {
"name": "chatId",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"role": {
"name": "role",
"type": "varchar",
"primaryKey": false,
"notNull": true
},
"content": {
"name": "content",
"type": "json",
"primaryKey": false,
"notNull": true
},
"createdAt": {
"name": "createdAt",
"type": "timestamp",
"primaryKey": false,
"notNull": true
}
},
"indexes": {},
"foreignKeys": {
"Message_chatId_Chat_id_fk": {
"name": "Message_chatId_Chat_id_fk",
"tableFrom": "Message",
"tableTo": "Chat",
"columnsFrom": [
"chatId"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"public.Suggestion": {
"name": "Suggestion",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": false,
"notNull": true,
"default": "gen_random_uuid()"
},
"documentId": {
"name": "documentId",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"documentCreatedAt": {
"name": "documentCreatedAt",
"type": "timestamp",
"primaryKey": false,
"notNull": true
},
"originalText": {
"name": "originalText",
"type": "text",
"primaryKey": false,
"notNull": true
},
"suggestedText": {
"name": "suggestedText",
"type": "text",
"primaryKey": false,
"notNull": true
},
"description": {
"name": "description",
"type": "text",
"primaryKey": false,
"notNull": false
},
"isResolved": {
"name": "isResolved",
"type": "boolean",
"primaryKey": false,
"notNull": true,
"default": false
},
"userId": {
"name": "userId",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"createdAt": {
"name": "createdAt",
"type": "timestamp",
"primaryKey": false,
"notNull": true
}
},
"indexes": {},
"foreignKeys": {
"Suggestion_userId_User_id_fk": {
"name": "Suggestion_userId_User_id_fk",
"tableFrom": "Suggestion",
"tableTo": "User",
"columnsFrom": [
"userId"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
},
"Suggestion_documentId_documentCreatedAt_Document_id_createdAt_fk": {
"name": "Suggestion_documentId_documentCreatedAt_Document_id_createdAt_fk",
"tableFrom": "Suggestion",
"tableTo": "Document",
"columnsFrom": [
"documentId",
"documentCreatedAt"
],
"columnsTo": [
"id",
"createdAt"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {
"Suggestion_id_pk": {
"name": "Suggestion_id_pk",
"columns": [
"id"
]
}
},
"uniqueConstraints": {}
},
"public.User": {
"name": "User",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"email": {
"name": "email",
"type": "varchar(64)",
"primaryKey": false,
"notNull": true
},
"password": {
"name": "password",
"type": "varchar(64)",
"primaryKey": false,
"notNull": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"public.Vote_v2": {
"name": "Vote_v2",
"schema": "",
"columns": {
"chatId": {
"name": "chatId",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"messageId": {
"name": "messageId",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"isUpvoted": {
"name": "isUpvoted",
"type": "boolean",
"primaryKey": false,
"notNull": true
}
},
"indexes": {},
"foreignKeys": {
"Vote_v2_chatId_Chat_id_fk": {
"name": "Vote_v2_chatId_Chat_id_fk",
"tableFrom": "Vote_v2",
"tableTo": "Chat",
"columnsFrom": [
"chatId"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
},
"Vote_v2_messageId_Message_v2_id_fk": {
"name": "Vote_v2_messageId_Message_v2_id_fk",
"tableFrom": "Vote_v2",
"tableTo": "Message_v2",
"columnsFrom": [
"messageId"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {
"Vote_v2_chatId_messageId_pk": {
"name": "Vote_v2_chatId_messageId_pk",
"columns": [
"chatId",
"messageId"
]
}
},
"uniqueConstraints": {}
},
"public.Vote": {
"name": "Vote",
"schema": "",
"columns": {
"chatId": {
"name": "chatId",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"messageId": {
"name": "messageId",
"type": "uuid",
"primaryKey": false,
"notNull": true
},
"isUpvoted": {
"name": "isUpvoted",
"type": "boolean",
"primaryKey": false,
"notNull": true
}
},
"indexes": {},
"foreignKeys": {
"Vote_chatId_Chat_id_fk": {
"name": "Vote_chatId_Chat_id_fk",
"tableFrom": "Vote",
"tableTo": "Chat",
"columnsFrom": [
"chatId"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
},
"Vote_messageId_Message_id_fk": {
"name": "Vote_messageId_Message_id_fk",
"tableFrom": "Vote",
"tableTo": "Message",
"columnsFrom": [
"messageId"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {
"Vote_chatId_messageId_pk": {
"name": "Vote_chatId_messageId_pk",
"columns": [
"chatId",
"messageId"
]
}
},
"uniqueConstraints": {}
}
},
"enums": {},
"schemas": {},
"sequences": {},
"_meta": {
"columns": {},
"schemas": {},
"tables": {}
}
}

View file

@ -36,6 +36,13 @@
"when": 1733945232355,
"tag": "0004_odd_slayback",
"breakpoints": true
},
{
"idx": 5,
"version": "7",
"when": 1741934630596,
"tag": "0005_wooden_whistler",
"breakpoints": true
}
]
}

View file

@ -12,9 +12,9 @@ import {
document,
type Suggestion,
suggestion,
type Message,
message,
vote,
type DBMessage,
} from './schema';
import { ArtifactKind } from '@/components/artifact';
@ -104,7 +104,11 @@ export async function getChatById({ id }: { id: string }) {
}
}
export async function saveMessages({ messages }: { messages: Array<Message> }) {
export async function saveMessages({
messages,
}: {
messages: Array<DBMessage>;
}) {
try {
return await db.insert(message).values(messages);
} catch (error) {

View file

@ -33,7 +33,9 @@ export const chat = pgTable('Chat', {
export type Chat = InferSelectModel<typeof chat>;
export const message = pgTable('Message', {
// DEPRECATED: The following schema is deprecated and will be removed in the future.
// Read the migration guide at https://github.com/vercel/ai-chatbot/blob/main/docs/04-migrate-to-parts.md
export const messageDeprecated = pgTable('Message', {
id: uuid('id').primaryKey().notNull().defaultRandom(),
chatId: uuid('chatId')
.notNull()
@ -43,10 +45,45 @@ export const message = pgTable('Message', {
createdAt: timestamp('createdAt').notNull(),
});
export type Message = InferSelectModel<typeof message>;
export type MessageDeprecated = InferSelectModel<typeof messageDeprecated>;
export const message = pgTable('Message_v2', {
id: uuid('id').primaryKey().notNull().defaultRandom(),
chatId: uuid('chatId')
.notNull()
.references(() => chat.id),
role: varchar('role').notNull(),
parts: json('parts').notNull(),
attachments: json('attachments').notNull(),
createdAt: timestamp('createdAt').notNull(),
});
export type DBMessage = InferSelectModel<typeof message>;
// DEPRECATED: The following schema is deprecated and will be removed in the future.
// Read the migration guide at https://github.com/vercel/ai-chatbot/blob/main/docs/04-migrate-to-parts.md
export const voteDeprecated = pgTable(
'Vote',
{
chatId: uuid('chatId')
.notNull()
.references(() => chat.id),
messageId: uuid('messageId')
.notNull()
.references(() => messageDeprecated.id),
isUpvoted: boolean('isUpvoted').notNull(),
},
(table) => {
return {
pk: primaryKey({ columns: [table.chatId, table.messageId] }),
};
},
);
export type VoteDeprecated = InferSelectModel<typeof voteDeprecated>;
export const vote = pgTable(
'Vote',
'Vote_v2',
{
chatId: uuid('chatId')
.notNull()