diff --git a/README.md b/README.md index 10dbe84..70bde96 100644 --- a/README.md +++ b/README.md @@ -12,20 +12,18 @@ Clone of [T3 Chat](https://t3.chat/) - **Framework**: SvelteKit - **Language**: TypeScript - **Styling**: Tailwind +- **Backend**: Convex +- **Auth**: BetterAuth + Convex - **Components**: Melt UI (next-gen) - **Testing**: Humans - **Package Manager**: pnpm - **Linting**: ESLint - **Formatting**: Prettier -### Discussion - -- Vercel SDK? - - Nah, too limited - ## 📦 Self-hosting -IDK, calm down +TODO: test self-hosting, including Convex self-hosting perhaps +TODO: add instructions ## TODO @@ -39,11 +37,10 @@ IDK, calm down - ~[ ] OpenAI~ - [ ] File upload - [x] Ensure responsiveness -- [ ] File support -- [x] Streams on the server +- [x] Streams on the server (Resumable streams) - [x] Syntax highlighting with Shiki/markdown renderer - [ ] Eliminate FOUC -- [ ] Cascade deletes and shit in Convex +- [x] Cascade deletes - [ ] Error notification central, specially for BYOK models like o3 - [ ] Google Auth - [ ] Fix light mode (urgh) @@ -67,3 +64,4 @@ IDK, calm down - [ ] Chat sharing - [ ] 404 page/redirect - [ ] Test link with free credits +- [x] Cursor-like Rules (@ieedan's idea!) diff --git a/src/lib/auth.ts b/src/lib/auth.ts index 581cbc0..68eae36 100644 --- a/src/lib/auth.ts +++ b/src/lib/auth.ts @@ -22,4 +22,10 @@ export const auth = betterAuth({ }, }, plugins: [], + session: { + cookieCache: { + enabled: true, + maxAge: 5 * 60, // Cache duration in seconds + }, + }, }); diff --git a/src/lib/backend/convex/conversations.ts b/src/lib/backend/convex/conversations.ts index 2cc6913..aaf2b78 100644 --- a/src/lib/backend/convex/conversations.ts +++ b/src/lib/backend/convex/conversations.ts @@ -89,6 +89,11 @@ export const createAndAddMessage = mutation({ content: v.string(), role: messageRoleValidator, session_token: v.string(), + images: v.optional(v.array(v.object({ + url: v.string(), + storage_id: v.string(), + fileName: v.optional(v.string()), + }))), }, handler: async ( ctx, @@ -118,6 +123,7 @@ export const createAndAddMessage = mutation({ role: args.role, conversation_id: conversationId, session_token: args.session_token, + images: args.images, }); return { diff --git a/src/lib/backend/convex/messages.ts b/src/lib/backend/convex/messages.ts index da6708e..3df1f1f 100644 --- a/src/lib/backend/convex/messages.ts +++ b/src/lib/backend/convex/messages.ts @@ -40,6 +40,12 @@ export const create = mutation({ model_id: v.optional(v.string()), provider: v.optional(providerValidator), token_count: v.optional(v.number()), + // Optional image attachments + images: v.optional(v.array(v.object({ + url: v.string(), + storage_id: v.string(), + fileName: v.optional(v.string()), + }))), }, handler: async (ctx, args): Promise> => { const session = await ctx.runQuery(api.betterAuth.publicGetSession, { @@ -72,6 +78,8 @@ export const create = mutation({ model_id: args.model_id, provider: args.provider, token_count: args.token_count, + // Optional image attachments + images: args.images, }), ctx.db.patch(args.conversation_id as Id<'conversations'>, { generating: true, diff --git a/src/lib/backend/convex/schema.ts b/src/lib/backend/convex/schema.ts index 64fc342..3304de7 100644 --- a/src/lib/backend/convex/schema.ts +++ b/src/lib/backend/convex/schema.ts @@ -46,7 +46,7 @@ export default defineSchema({ title: v.string(), updated_at: v.optional(v.number()), pinned: v.optional(v.boolean()), - generating: v.boolean(), + generating: v.optional(v.boolean()), cost_usd: v.optional(v.number()), }).index('by_user', ['user_id']), messages: defineTable({ @@ -57,6 +57,16 @@ export default defineSchema({ model_id: v.optional(v.string()), provider: v.optional(providerValidator), token_count: v.optional(v.number()), + // Optional image attachments + images: v.optional( + v.array( + v.object({ + url: v.string(), + storage_id: v.string(), + fileName: v.optional(v.string()), + }) + ) + ), cost_usd: v.optional(v.number()), generation_id: v.optional(v.string()), }).index('by_conversation', ['conversation_id']), diff --git a/src/lib/backend/convex/storage.ts b/src/lib/backend/convex/storage.ts new file mode 100644 index 0000000..6b14d40 --- /dev/null +++ b/src/lib/backend/convex/storage.ts @@ -0,0 +1,58 @@ +import { v } from 'convex/values'; +import { api } from './_generated/api'; +import { query } from './_generated/server'; +import { mutation } from './functions'; + +export const generateUploadUrl = mutation({ + args: { + session_token: v.string(), + }, + handler: async (ctx, args) => { + const session = await ctx.runQuery(api.betterAuth.publicGetSession, { + session_token: args.session_token, + }); + + if (!session) { + throw new Error('Unauthorized'); + } + + return await ctx.storage.generateUploadUrl(); + }, +}); + +export const getUrl = query({ + args: { + storage_id: v.string(), + session_token: v.string(), + }, + handler: async (ctx, args) => { + const session = await ctx.runQuery(api.betterAuth.publicGetSession, { + session_token: args.session_token, + }); + + if (!session) { + throw new Error('Unauthorized'); + } + + return await ctx.storage.getUrl(args.storage_id); + }, +}); + +export const deleteFile = mutation({ + args: { + storage_id: v.string(), + session_token: v.string(), + }, + handler: async (ctx, args) => { + const session = await ctx.runQuery(api.betterAuth.publicGetSession, { + session_token: args.session_token, + }); + + if (!session) { + throw new Error('Unauthorized'); + } + + await ctx.storage.delete(args.storage_id); + }, +}); + diff --git a/src/lib/components/ui/button/button.svelte b/src/lib/components/ui/button/button.svelte index a9b71fe..37c5887 100644 --- a/src/lib/components/ui/button/button.svelte +++ b/src/lib/components/ui/button/button.svelte @@ -24,6 +24,7 @@ sm: 'h-8 gap-1.5 rounded-md px-3 has-[>svg]:px-2.5', lg: 'h-10 rounded-md px-6 has-[>svg]:px-4', icon: 'size-9', + iconSm: 'size-7', }, }, defaultVariants: { diff --git a/src/lib/components/ui/image-modal/image-modal.svelte b/src/lib/components/ui/image-modal/image-modal.svelte new file mode 100644 index 0000000..d7552cf --- /dev/null +++ b/src/lib/components/ui/image-modal/image-modal.svelte @@ -0,0 +1,67 @@ + + + +
+

{fileName}

+
+ + {#snippet trigger(tooltip)} + + {/snippet} + Download image + + + {#snippet trigger(tooltip)} + + {/snippet} + Open in new tab + + + {#snippet trigger(tooltip)} + + {/snippet} + Close + +
+
+ +
+ {fileName} +
+ diff --git a/src/lib/components/ui/image-modal/index.ts b/src/lib/components/ui/image-modal/index.ts new file mode 100644 index 0000000..0a335d0 --- /dev/null +++ b/src/lib/components/ui/image-modal/index.ts @@ -0,0 +1 @@ +export { default as ImageModal } from './image-modal.svelte'; \ No newline at end of file diff --git a/src/lib/utils/image-compression.ts b/src/lib/utils/image-compression.ts new file mode 100644 index 0000000..d48e223 --- /dev/null +++ b/src/lib/utils/image-compression.ts @@ -0,0 +1,73 @@ +export function compressImage(file: File, maxSizeBytes: number = 1024 * 1024): Promise { + return new Promise((resolve, reject) => { + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + const img = new Image(); + + img.onload = () => { + // Calculate new dimensions to maintain aspect ratio + let { width, height } = img; + const maxDimension = 1920; // Max width or height + + if (width > maxDimension || height > maxDimension) { + if (width > height) { + height = (height * maxDimension) / width; + width = maxDimension; + } else { + width = (width * maxDimension) / height; + height = maxDimension; + } + } + + canvas.width = width; + canvas.height = height; + + if (!ctx) { + reject(new Error('Could not get canvas context')); + return; + } + + // Draw and compress + ctx.drawImage(img, 0, 0, width, height); + + // Start with high quality and reduce until under size limit + let quality = 0.9; + let compressed: File | null = null; + + const tryCompress = () => { + canvas.toBlob( + (blob) => { + if (!blob) { + reject(new Error('Failed to compress image')); + return; + } + + compressed = new File([blob], file.name, { + type: 'image/jpeg', + lastModified: Date.now(), + }); + + // If under size limit or quality is too low, return result + if (compressed.size <= maxSizeBytes || quality <= 0.1) { + resolve(compressed); + } else { + // Reduce quality and try again + quality -= 0.1; + tryCompress(); + } + }, + 'image/jpeg', + quality + ); + }; + + tryCompress(); + }; + + img.onerror = () => { + reject(new Error('Failed to load image')); + }; + + img.src = URL.createObjectURL(file); + }); +} \ No newline at end of file diff --git a/src/lib/utils/model-capabilities.ts b/src/lib/utils/model-capabilities.ts new file mode 100644 index 0000000..94d2e3e --- /dev/null +++ b/src/lib/utils/model-capabilities.ts @@ -0,0 +1,9 @@ +import type { OpenRouterModel } from '$lib/backend/models/open-router'; + +export function supportsImages(model: OpenRouterModel): boolean { + return model.architecture.input_modalities.includes('image'); +} + +export function getImageSupportedModels(models: OpenRouterModel[]): OpenRouterModel[] { + return models.filter(supportsImages); +} \ No newline at end of file diff --git a/src/routes/api/generate-message/+server.ts b/src/routes/api/generate-message/+server.ts index 4d5b8c3..3b0dbdc 100644 --- a/src/routes/api/generate-message/+server.ts +++ b/src/routes/api/generate-message/+server.ts @@ -20,6 +20,15 @@ const reqBodySchema = z.object({ session_token: z.string(), conversation_id: z.string().optional(), + images: z + .array( + z.object({ + url: z.string(), + storage_id: z.string(), + fileName: z.string().optional(), + }) + ) + .optional(), }); export type GenerateMessageRequestBody = z.infer; @@ -189,14 +198,12 @@ async function generateAIResponse({ log('Background: Model found and enabled', startTime); - const messagesQuery = await messagesQueryResult; - - if (messagesQuery.isErr()) { - log(`Background messages query failed: ${messagesQuery.error}`, startTime); + if (messagesQueryResult.isErr()) { + log(`Background messages query failed: ${messagesQueryResult.error}`, startTime); return; } - const messages = messagesQuery.value; + const messages = messagesQueryResult.value; log(`Background: Retrieved ${messages.length} messages from conversation`, startTime); if (keyResult.isErr()) { @@ -246,10 +253,29 @@ ${attachedRules.map((r) => `- ${r.name}: ${r.rule}`).join('\n')}`, apiKey: key, }); + const formattedMessages = messages.map((m) => { + if (m.images && m.images.length > 0 && m.role === 'user') { + return { + role: 'user' as const, + content: [ + { type: 'text' as const, text: m.content }, + ...m.images.map((img) => ({ + type: 'image_url' as const, + image_url: { url: img.url }, + })), + ], + }; + } + return { + role: m.role as 'user' | 'assistant' | 'system', + content: m.content, + }; + }); + const streamResult = await ResultAsync.fromPromise( openai.chat.completions.create({ model: model.model_id, - messages: [...messages.map((m) => ({ role: m.role, content: m.content })), systemMessage], + messages: [...formattedMessages, systemMessage], temperature: 0.7, stream: true, }), @@ -463,6 +489,7 @@ export const POST: RequestHandler = async ({ request }) => { client.mutation(api.conversations.createAndAddMessage, { content: args.message, role: 'user', + images: args.images, session_token: sessionToken, }), (e) => `Failed to create conversation: ${e}` @@ -497,6 +524,7 @@ export const POST: RequestHandler = async ({ request }) => { session_token: args.session_token, model_id: args.model_id, role: 'user', + images: args.images, }), (e) => `Failed to create user message: ${e}` ); diff --git a/src/routes/chat/+layout.svelte b/src/routes/chat/+layout.svelte index e1e0576..5a9e9c6 100644 --- a/src/routes/chat/+layout.svelte +++ b/src/routes/chat/+layout.svelte @@ -6,33 +6,39 @@ import { useCachedQuery } from '$lib/cache/cached-query.svelte.js'; import * as Icons from '$lib/components/icons'; import { Button } from '$lib/components/ui/button'; + import { ImageModal } from '$lib/components/ui/image-modal'; import { LightSwitch } from '$lib/components/ui/light-switch/index.js'; import { callModal } from '$lib/components/ui/modal/global-modal.svelte'; import * as Sidebar from '$lib/components/ui/sidebar'; import Tooltip from '$lib/components/ui/tooltip.svelte'; + import { cmdOrCtrl } from '$lib/hooks/is-mac.svelte.js'; import { TextareaAutosize } from '$lib/spells/textarea-autosize.svelte.js'; + import { models } from '$lib/state/models.svelte'; import { usePrompt } from '$lib/state/prompt.svelte.js'; import { session } from '$lib/state/session.svelte.js'; import { settings } from '$lib/state/settings.svelte.js'; + import { Provider } from '$lib/types'; + import { compressImage } from '$lib/utils/image-compression'; import { isString } from '$lib/utils/is.js'; - import { pick } from '$lib/utils/object.js'; + import { supportsImages } from '$lib/utils/model-capabilities'; + import { omit, pick } from '$lib/utils/object.js'; import { cn } from '$lib/utils/utils.js'; import { useConvexClient } from 'convex-svelte'; - import { Popover } from 'melt/builders'; + import { FileUpload, Popover } from 'melt/builders'; import { Avatar } from 'melt/components'; import { Debounced, ElementSize, IsMounted, ScrollState } from 'runed'; import SendIcon from '~icons/lucide/arrow-up'; import ChevronDownIcon from '~icons/lucide/chevron-down'; + import ImageIcon from '~icons/lucide/image'; import LoaderCircleIcon from '~icons/lucide/loader-circle'; import PanelLeftIcon from '~icons/lucide/panel-left'; import PinIcon from '~icons/lucide/pin'; import PinOffIcon from '~icons/lucide/pin-off'; import Settings2Icon from '~icons/lucide/settings-2'; + import UploadIcon from '~icons/lucide/upload'; import XIcon from '~icons/lucide/x'; import { callGenerateMessage } from '../api/generate-message/call.js'; import ModelPicker from './model-picker.svelte'; - import { cmdOrCtrl } from '$lib/hooks/is-mac.svelte.js'; - import { Provider } from '$lib/types.js'; const client = useConvexClient(); @@ -48,11 +54,16 @@ if (!isString(message) || !session.current?.user.id || !settings.modelId) return; if (textarea) textarea.value = ''; + const messageCopy = message; + const imagesCopy = [...selectedImages]; + selectedImages = []; + const res = await callGenerateMessage({ - message, + message: messageCopy, session_token: session.current?.session.token, conversation_id: page.params.id ?? undefined, model_id: settings.modelId, + images: imagesCopy.length > 0 ? imagesCopy : undefined, }); if (res.isErr()) return; // TODO: Handle error @@ -165,12 +176,107 @@ ]); let message = $state(''); + let selectedImages = $state<{ url: string; storage_id: string; fileName?: string }[]>([]); + let isUploading = $state(false); + let fileInput = $state(); + let imageModal = $state<{ open: boolean; imageUrl: string; fileName: string }>({ + open: false, + imageUrl: '', + fileName: '', + }); usePrompt( () => message, (v) => (message = v) ); + models.init(); + + const currentModelSupportsImages = $derived.by(() => { + if (!settings.modelId) return false; + const openRouterModels = models.from(Provider.OpenRouter); + const currentModel = openRouterModels.find((m) => m.id === settings.modelId); + return currentModel ? supportsImages(currentModel) : false; + }); + + const fileUpload = new FileUpload({ + multiple: true, + accept: 'image/*', + maxSize: 10 * 1024 * 1024, // 10MB + }); + + async function handleFileChange(files: File[]) { + if (!files.length || !session.current?.session.token) return; + + isUploading = true; + const uploadedFiles: { url: string; storage_id: string; fileName?: string }[] = []; + + try { + for (const file of files) { + // Skip non-image files + if (!file.type.startsWith('image/')) { + console.warn('Skipping non-image file:', file.name); + continue; + } + + // Compress image to max 1MB + const compressedFile = await compressImage(file, 1024 * 1024); + + // Generate upload URL + const uploadUrl = await client.mutation(api.storage.generateUploadUrl, { + session_token: session.current.session.token, + }); + + // Upload compressed file + const result = await fetch(uploadUrl, { + method: 'POST', + body: compressedFile, + }); + + if (!result.ok) { + throw new Error(`Upload failed: ${result.statusText}`); + } + + const { storageId } = await result.json(); + + // Get the URL for the uploaded file + const url = await client.query(api.storage.getUrl, { + storage_id: storageId, + session_token: session.current.session.token, + }); + + if (url) { + uploadedFiles.push({ url, storage_id: storageId, fileName: file.name }); + } + } + + selectedImages = [...selectedImages, ...uploadedFiles]; + } catch (error) { + console.error('Upload failed:', error); + } finally { + isUploading = false; + } + } + + function removeImage(index: number) { + selectedImages = selectedImages.filter((_, i) => i !== index); + } + + function openImageModal(imageUrl: string, fileName: string) { + imageModal = { + open: true, + imageUrl, + fileName, + }; + } + + $effect(() => { + if (fileUpload.selected.size > 0) { + handleFileChange(Array.from(fileUpload.selected)); + fileUpload.clear(); + } + }); + const suggestedRules = $derived.by(() => { if (!rulesQuery.data || rulesQuery.data.length === 0) return; if (!textarea) return; @@ -293,7 +399,10 @@ Chat | Thom.chat - +
Thom.chat @@ -526,7 +635,36 @@
{/if}
-
+ {#if selectedImages.length > 0} +
+ {#each selectedImages as image, index (image.storage_id)} +
+ + +
+ {/each} +
+ {/if} +
+