faster!
This commit is contained in:
Thomas G. Lopes
parent
7c216dc18b
commit
d8d0d0c9b1
2 changed files with 209 additions and 117 deletions
|
|
@ -45,6 +45,7 @@ IDK, calm down
|
|||
- [ ] Syntax highlighting with Shiki/markdown renderer
|
||||
- [ ] Eliminate FOUC
|
||||
- [ ] Cascade deletes and shit in Convex
|
||||
- [ ] Error notification central, specially for BYOK models like o3
|
||||
|
||||
### Extra
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { PUBLIC_CONVEX_URL } from '$env/static/public';
|
||||
import { api } from '$lib/backend/convex/_generated/api';
|
||||
import type { Id } from '$lib/backend/convex/_generated/dataModel';
|
||||
import type { SessionObj } from '$lib/backend/convex/betterAuth';
|
||||
import { Provider } from '$lib/types';
|
||||
import { error, json, type RequestHandler } from '@sveltejs/kit';
|
||||
import { ConvexHttpClient } from 'convex/browser';
|
||||
|
|
@ -9,6 +10,9 @@ import OpenAI from 'openai';
|
|||
|
||||
import { z } from 'zod/v4';
|
||||
|
||||
// Set to true to enable debug logging
|
||||
const ENABLE_LOGGING = true;
|
||||
|
||||
const reqBodySchema = z.object({
|
||||
message: z.string(),
|
||||
model_id: z.string(),
|
||||
|
|
@ -28,162 +32,249 @@ function response(res: GenerateMessageResponse) {
|
|||
return json(res);
|
||||
}
|
||||
|
||||
function log(message: string, startTime: number): void {
|
||||
if (!ENABLE_LOGGING) return;
|
||||
const elapsed = Date.now() - startTime;
|
||||
console.log(`[GenerateMessage] ${message} (${elapsed}ms)`);
|
||||
}
|
||||
|
||||
const client = new ConvexHttpClient(PUBLIC_CONVEX_URL);
|
||||
|
||||
export const POST: RequestHandler = async ({ request }) => {
|
||||
const bodyResult = await ResultAsync.fromPromise(
|
||||
request.json(),
|
||||
() => 'Failed to parse request body'
|
||||
async function generateAIResponse(
|
||||
conversationId: string,
|
||||
session: SessionObj,
|
||||
modelId: string,
|
||||
startTime: number
|
||||
) {
|
||||
log('Starting AI response generation in background', startTime);
|
||||
|
||||
const modelResult = await ResultAsync.fromPromise(
|
||||
client.query(api.user_enabled_models.get, {
|
||||
provider: Provider.OpenRouter,
|
||||
model_id: modelId,
|
||||
user_id: session.userId,
|
||||
}),
|
||||
(e) => `Failed to get model: ${e}`
|
||||
);
|
||||
|
||||
if (bodyResult.isErr()) {
|
||||
return error(400, 'Failed to parse request body');
|
||||
if (modelResult.isErr()) {
|
||||
log(`Background model query failed: ${modelResult.error}`, startTime);
|
||||
return;
|
||||
}
|
||||
|
||||
const parsed = reqBodySchema.safeParse(bodyResult.value);
|
||||
if (!parsed.success) {
|
||||
return error(400, parsed.error);
|
||||
}
|
||||
const args = parsed.data;
|
||||
|
||||
const session = await client.query(api.betterAuth.publicGetSession, {
|
||||
session_token: args.session_token,
|
||||
});
|
||||
|
||||
if (!session) {
|
||||
throw new Error('Unauthorized');
|
||||
}
|
||||
|
||||
const model = await client.query(api.user_enabled_models.get, {
|
||||
provider: Provider.OpenRouter,
|
||||
model_id: args.model_id,
|
||||
user_id: session.userId,
|
||||
});
|
||||
|
||||
const model = modelResult.value;
|
||||
if (!model) {
|
||||
throw new Error('Model not found or not enabled');
|
||||
log('Background: Model not found or not enabled', startTime);
|
||||
return;
|
||||
}
|
||||
|
||||
let conversationId = args.conversation_id;
|
||||
if (!conversationId) {
|
||||
conversationId = await client.mutation(api.conversations.create, {
|
||||
session_token: args.session_token,
|
||||
});
|
||||
}
|
||||
|
||||
if (args.message) {
|
||||
await client.mutation(api.messages.create, {
|
||||
conversation_id: conversationId as Id<'conversations'>,
|
||||
content: args.message,
|
||||
session_token: args.session_token,
|
||||
model_id: args.model_id,
|
||||
role: 'user',
|
||||
});
|
||||
}
|
||||
log('Background: Model found and enabled', startTime);
|
||||
|
||||
const messagesQuery = await ResultAsync.fromPromise(
|
||||
client.query(api.messages.getAllFromConversation, {
|
||||
conversation_id: conversationId as Id<'conversations'>,
|
||||
session_token: args.session_token,
|
||||
session_token: session.token,
|
||||
}),
|
||||
(e) => e
|
||||
(e) => `Failed to get messages: ${e}`
|
||||
);
|
||||
|
||||
if (messagesQuery.isErr()) {
|
||||
throw new Error('Failed to get messages');
|
||||
log(`Background messages query failed: ${messagesQuery.error}`, startTime);
|
||||
return;
|
||||
}
|
||||
|
||||
const messages = messagesQuery.value;
|
||||
log(`Background: Retrieved ${messages.length} messages from conversation`, startTime);
|
||||
|
||||
console.log(messages);
|
||||
const keyResult = await ResultAsync.fromPromise(
|
||||
client.query(api.user_keys.get, {
|
||||
provider: Provider.OpenRouter,
|
||||
session_token: session.token,
|
||||
}),
|
||||
(e) => `Failed to get API key: ${e}`
|
||||
);
|
||||
|
||||
const key = await client.query(api.user_keys.get, {
|
||||
provider: Provider.OpenRouter,
|
||||
session_token: session.token,
|
||||
});
|
||||
|
||||
if (!key) {
|
||||
throw new Error('No key found');
|
||||
if (keyResult.isErr()) {
|
||||
log(`Background API key query failed: ${keyResult.error}`, startTime);
|
||||
return;
|
||||
}
|
||||
|
||||
const key = keyResult.value;
|
||||
if (!key) {
|
||||
log('Background: No API key found', startTime);
|
||||
return;
|
||||
}
|
||||
|
||||
log('Background: API key retrieved successfully', startTime);
|
||||
|
||||
const openai = new OpenAI({
|
||||
baseURL: 'https://openrouter.ai/api/v1',
|
||||
apiKey: key,
|
||||
});
|
||||
|
||||
const stream = await openai.chat.completions.create({
|
||||
model: model.model_id,
|
||||
messages: messages.map((m) => ({ role: m.role, content: m.content })),
|
||||
max_tokens: 1000,
|
||||
temperature: 0.7,
|
||||
stream: true,
|
||||
});
|
||||
const streamResult = await ResultAsync.fromPromise(
|
||||
openai.chat.completions.create({
|
||||
model: model.model_id,
|
||||
messages: messages.map((m) => ({ role: m.role, content: m.content })),
|
||||
max_tokens: 1000,
|
||||
temperature: 0.7,
|
||||
stream: true,
|
||||
}),
|
||||
(e) => `OpenAI API call failed: ${e}`
|
||||
);
|
||||
|
||||
// Create first message
|
||||
const mid = await client.mutation(api.messages.create, {
|
||||
conversation_id: conversationId,
|
||||
content: '',
|
||||
role: 'assistant',
|
||||
session_token: session.token,
|
||||
});
|
||||
if (streamResult.isErr()) {
|
||||
log(`Background OpenAI stream creation failed: ${streamResult.error}`, startTime);
|
||||
return;
|
||||
}
|
||||
|
||||
async function handleStream() {
|
||||
if (!session) return;
|
||||
const stream = streamResult.value;
|
||||
log('Background: OpenAI stream created successfully', startTime);
|
||||
|
||||
let content = '';
|
||||
// Create assistant message
|
||||
const messageCreationResult = await ResultAsync.fromPromise(
|
||||
client.mutation(api.messages.create, {
|
||||
conversation_id: conversationId,
|
||||
content: '',
|
||||
role: 'assistant',
|
||||
session_token: session.token,
|
||||
}),
|
||||
(e) => `Failed to create assistant message: ${e}`
|
||||
);
|
||||
|
||||
if (messageCreationResult.isErr()) {
|
||||
log(`Background assistant message creation failed: ${messageCreationResult.error}`, startTime);
|
||||
return;
|
||||
}
|
||||
|
||||
const mid = messageCreationResult.value;
|
||||
log('Background: Assistant message created', startTime);
|
||||
|
||||
let content = '';
|
||||
let chunkCount = 0;
|
||||
|
||||
try {
|
||||
for await (const chunk of stream) {
|
||||
chunkCount++;
|
||||
content += chunk.choices[0]?.delta?.content || '';
|
||||
if (!content) continue;
|
||||
|
||||
await client.mutation(api.messages.updateContent, {
|
||||
message_id: mid,
|
||||
content,
|
||||
session_token: session.token,
|
||||
});
|
||||
const updateResult = await ResultAsync.fromPromise(
|
||||
client.mutation(api.messages.updateContent, {
|
||||
message_id: mid,
|
||||
content,
|
||||
session_token: session.token,
|
||||
}),
|
||||
(e) => `Failed to update message content: ${e}`
|
||||
);
|
||||
|
||||
if (updateResult.isErr()) {
|
||||
log(
|
||||
`Background message update failed on chunk ${chunkCount}: ${updateResult.error}`,
|
||||
startTime
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
log(
|
||||
`Background stream processing completed. Processed ${chunkCount} chunks, final content length: ${content.length}`,
|
||||
startTime
|
||||
);
|
||||
} catch (error) {
|
||||
log(`Background stream processing error: ${error}`, startTime);
|
||||
}
|
||||
}
|
||||
|
||||
export const POST: RequestHandler = async ({ request }) => {
|
||||
const startTime = Date.now();
|
||||
log('Starting message generation request', startTime);
|
||||
|
||||
const bodyResult = await ResultAsync.fromPromise(
|
||||
request.json(),
|
||||
() => 'Failed to parse request body'
|
||||
);
|
||||
|
||||
if (bodyResult.isErr()) {
|
||||
log(`Request body parsing failed: ${bodyResult.error}`, startTime);
|
||||
return error(400, 'Failed to parse request body');
|
||||
}
|
||||
|
||||
handleStream();
|
||||
log('Request body parsed successfully', startTime);
|
||||
|
||||
const parsed = reqBodySchema.safeParse(bodyResult.value);
|
||||
if (!parsed.success) {
|
||||
log(`Schema validation failed: ${parsed.error}`, startTime);
|
||||
return error(400, parsed.error);
|
||||
}
|
||||
const args = parsed.data;
|
||||
|
||||
log('Schema validation passed', startTime);
|
||||
|
||||
const sessionResult = await ResultAsync.fromPromise(
|
||||
client.query(api.betterAuth.publicGetSession, {
|
||||
session_token: args.session_token,
|
||||
}),
|
||||
(e) => `Failed to get session: ${e}`
|
||||
);
|
||||
|
||||
if (sessionResult.isErr()) {
|
||||
log(`Session query failed: ${sessionResult.error}`, startTime);
|
||||
return error(401, 'Failed to authenticate');
|
||||
}
|
||||
|
||||
const session = sessionResult.value;
|
||||
if (!session) {
|
||||
log('No session found - unauthorized', startTime);
|
||||
return error(401, 'Unauthorized');
|
||||
}
|
||||
|
||||
log('Session authenticated successfully', startTime);
|
||||
|
||||
let conversationId = args.conversation_id;
|
||||
if (!conversationId) {
|
||||
const conversationResult = await ResultAsync.fromPromise(
|
||||
client.mutation(api.conversations.create, {
|
||||
session_token: args.session_token,
|
||||
}),
|
||||
(e) => `Failed to create conversation: ${e}`
|
||||
);
|
||||
|
||||
if (conversationResult.isErr()) {
|
||||
log(`Conversation creation failed: ${conversationResult.error}`, startTime);
|
||||
return error(500, 'Failed to create conversation');
|
||||
}
|
||||
|
||||
conversationId = conversationResult.value;
|
||||
log('New conversation created', startTime);
|
||||
} else {
|
||||
log('Using existing conversation', startTime);
|
||||
}
|
||||
|
||||
if (args.message) {
|
||||
const userMessageResult = await ResultAsync.fromPromise(
|
||||
client.mutation(api.messages.create, {
|
||||
conversation_id: conversationId as Id<'conversations'>,
|
||||
content: args.message,
|
||||
session_token: args.session_token,
|
||||
model_id: args.model_id,
|
||||
role: 'user',
|
||||
}),
|
||||
(e) => `Failed to create user message: ${e}`
|
||||
);
|
||||
|
||||
if (userMessageResult.isErr()) {
|
||||
log(`User message creation failed: ${userMessageResult.error}`, startTime);
|
||||
return error(500, 'Failed to create user message');
|
||||
}
|
||||
|
||||
log('User message created', startTime);
|
||||
}
|
||||
|
||||
// Start AI response generation in background - don't await
|
||||
generateAIResponse(conversationId, session, args.model_id, startTime).catch((error) => {
|
||||
log(`Background AI response generation error: ${error}`, startTime);
|
||||
});
|
||||
|
||||
log('Response sent, AI generation started in background', startTime);
|
||||
return response({ ok: true, conversation_id: conversationId });
|
||||
|
||||
// const completionResult = await ResultAsync.fromPromise(
|
||||
// openai.chat.completions.create({
|
||||
// model,
|
||||
// messages: [{ role: 'user', content: message }],
|
||||
// max_tokens: 1000,
|
||||
// temperature: 0.7,
|
||||
// stream: true,
|
||||
// }),
|
||||
// () => 'OpenRouter API failed'
|
||||
// );
|
||||
//
|
||||
// if (completionResult.isErr()) {
|
||||
// return new Response(JSON.stringify({ error: completionResult.error }), {
|
||||
// status: 500,
|
||||
// headers: { 'Content-Type': 'application/json' },
|
||||
// });
|
||||
// }
|
||||
//
|
||||
// const stream = completionResult.value;
|
||||
//
|
||||
//
|
||||
// const readable = new ReadableStream({
|
||||
// async start(controller) {
|
||||
// for await (const chunk of stream) {
|
||||
// const content = chunk.choices[0]?.delta?.content || '';
|
||||
// if (content) {
|
||||
// controller.enqueue(new TextEncoder().encode(content));
|
||||
// }
|
||||
// }
|
||||
// controller.close();
|
||||
// },
|
||||
// });
|
||||
//
|
||||
// return new Response(readable, {
|
||||
// headers: {
|
||||
// 'Content-Type': 'text/plain',
|
||||
// 'Cache-Control': 'no-cache',
|
||||
// },
|
||||
// });
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue