wip by claude

src/lib/backend/convex/conversations.ts

@@ -89,6 +89,10 @@ export const createAndAddMessage = mutation({
 		content: v.string(),
 		role: messageRoleValidator,
 		session_token: v.string(),
+		images: v.optional(v.array(v.object({
+			url: v.string(),
+			storage_id: v.string(),
+		}))),
 	},
 	handler: async (
 		ctx,
@@ -118,6 +122,7 @@ export const createAndAddMessage = mutation({
 			role: args.role,
 			conversation_id: conversationId,
 			session_token: args.session_token,
+			images: args.images,
 		});
 
 		return {

src/lib/backend/convex/messages.ts

@@ -40,6 +40,11 @@ export const create = mutation({
 		model_id: v.optional(v.string()),
 		provider: v.optional(providerValidator),
 		token_count: v.optional(v.number()),
+		// Optional image attachments
+		images: v.optional(v.array(v.object({
+			url: v.string(),
+			storage_id: v.string(),
+		}))),
 	},
 	handler: async (ctx, args): Promise<Id<'messages'>> => {
 		const session = await ctx.runQuery(api.betterAuth.publicGetSession, {
@@ -72,6 +77,8 @@ export const create = mutation({
 				model_id: args.model_id,
 				provider: args.provider,
 				token_count: args.token_count,
+				// Optional image attachments
+				images: args.images,
 			}),
 			ctx.db.patch(args.conversation_id as Id<'conversations'>, {
 				generating: true,

src/lib/backend/convex/schema.ts

@@ -46,7 +46,7 @@ export default defineSchema({
 		title: v.string(),
 		updated_at: v.optional(v.number()),
 		pinned: v.optional(v.boolean()),
-		generating: v.boolean(),
+		generating: v.optional(v.boolean()),
 	}).index('by_user', ['user_id']),
 	messages: defineTable({
 		conversation_id: v.string(),
@@ -56,5 +56,14 @@ export default defineSchema({
 		model_id: v.optional(v.string()),
 		provider: v.optional(providerValidator),
 		token_count: v.optional(v.number()),
+		// Optional image attachments
+		images: v.optional(
+			v.array(
+				v.object({
+					url: v.string(),
+					storage_id: v.string(),
+				})
+			)
+		),
 	}).index('by_conversation', ['conversation_id']),
 });

src/lib/backend/convex/storage.ts

@@ -0,0 +1,56 @@
+import { v } from 'convex/values';
+import { api } from './_generated/api';
+import { mutation, query } from './_generated/server';
+
+export const generateUploadUrl = mutation({
+	args: {
+		session_token: v.string(),
+	},
+	handler: async (ctx, args) => {
+		const session = await ctx.runQuery(api.betterAuth.publicGetSession, {
+			session_token: args.session_token,
+		});
+
+		if (!session) {
+			throw new Error('Unauthorized');
+		}
+
+		return await ctx.storage.generateUploadUrl();
+	},
+});
+
+export const getUrl = query({
+	args: {
+		storage_id: v.string(),
+		session_token: v.string(),
+	},
+	handler: async (ctx, args) => {
+		const session = await ctx.runQuery(api.betterAuth.publicGetSession, {
+			session_token: args.session_token,
+		});
+
+		if (!session) {
+			throw new Error('Unauthorized');
+		}
+
+		return await ctx.storage.getUrl(args.storage_id);
+	},
+});
+
+export const deleteFile = mutation({
+	args: {
+		storage_id: v.string(),
+		session_token: v.string(),
+	},
+	handler: async (ctx, args) => {
+		const session = await ctx.runQuery(api.betterAuth.publicGetSession, {
+			session_token: args.session_token,
+		});
+
+		if (!session) {
+			throw new Error('Unauthorized');
+		}
+
+		await ctx.storage.delete(args.storage_id);
+	},
+});

src/lib/components/ui/file-upload/file-upload.svelte

@@ -0,0 +1,131 @@
+<script lang="ts">
+	import { api } from '$lib/backend/convex/_generated/api';
+	import { session } from '$lib/state/session.svelte';
+	import { cn } from '$lib/utils/utils';
+	import { useConvexClient } from 'convex-svelte';
+	import { FileUpload } from 'melt/builders';
+	import ImageIcon from '~icons/lucide/image';
+	import XIcon from '~icons/lucide/x';
+
+	type Props = {
+		onFilesSelected?: (files: { url: string; storage_id: string }[]) => void;
+		selectedFiles?: { url: string; storage_id: string }[];
+		class?: string;
+		disabled?: boolean;
+	};
+
+	let { onFilesSelected, selectedFiles = $bindable([]), class: className, disabled = false }: Props = $props();
+
+	const client = useConvexClient();
+	const fileUpload = new FileUpload({
+		multiple: true,
+		accept: 'image/*',
+		maxSize: 10 * 1024 * 1024, // 10MB
+	});
+
+	let isUploading = $state(false);
+
+	async function handleFileChange(files: File[]) {
+		if (!files.length || !session.current?.session.token) return;
+
+		isUploading = true;
+		const uploadedFiles: { url: string; storage_id: string }[] = [];
+
+		try {
+			for (const file of files) {
+				// Generate upload URL
+				const uploadUrl = await client.mutation(api.storage.generateUploadUrl, {
+					session_token: session.current.session.token,
+				});
+
+				// Upload file
+				const result = await fetch(uploadUrl, {
+					method: 'POST',
+					body: file,
+				});
+
+				if (!result.ok) {
+					throw new Error(`Upload failed: ${result.statusText}`);
+				}
+
+				const { storageId } = await result.json();
+
+				// Get the URL for the uploaded file
+				const url = await client.query(api.storage.getUrl, {
+					storage_id: storageId,
+					session_token: session.current.session.token,
+				});
+
+				if (url) {
+					uploadedFiles.push({ url, storage_id: storageId });
+				}
+			}
+
+			const newFiles = [...selectedFiles, ...uploadedFiles];
+			onFilesSelected?.(newFiles);
+		} catch (error) {
+			console.error('Upload failed:', error);
+		} finally {
+			isUploading = false;
+		}
+	}
+
+	function removeFile(index: number) {
+		const newFiles = selectedFiles.filter((_, i) => i !== index);
+		onFilesSelected?.(newFiles);
+	}
+
+	$effect(() => {
+		if (fileUpload.selected.size > 0) {
+			handleFileChange(Array.from(fileUpload.selected));
+			fileUpload.clear();
+		}
+	});
+</script>
+
+<div class={cn('flex flex-col gap-2', className)}>
+	{#if selectedFiles.length > 0}
+		<div class="flex flex-wrap gap-2">
+			{#each selectedFiles as file, index}
+				<div class="relative">
+					<img src={file.url} alt="Uploaded" class="h-16 w-16 rounded-lg object-cover" />
+					<button
+						type="button"
+						onclick={() => removeFile(index)}
+						class="absolute -top-2 -right-2 flex h-5 w-5 items-center justify-center rounded-full bg-red-500 text-white hover:bg-red-600"
+					>
+						<XIcon class="h-3 w-3" />
+					</button>
+				</div>
+			{/each}
+		</div>
+	{/if}
+
+	<div>
+		<input {...fileUpload.input} />
+		<div
+			{...fileUpload.dropzone}
+			class={cn(
+				'flex flex-col items-center justify-center rounded-lg border-2 border-dashed border-gray-300 p-4 text-center transition-colors hover:border-gray-400',
+				{
+					'border-blue-400 bg-blue-50': fileUpload.isDragging,
+					'opacity-50 cursor-not-allowed': disabled || isUploading,
+				}
+			)}
+		>
+			{#if isUploading}
+				<div class="animate-spin rounded-full h-6 w-6 border-b-2 border-gray-900"></div>
+				<p class="mt-2 text-sm text-gray-600">Uploading...</p>
+			{:else if fileUpload.isDragging}
+				<ImageIcon class="h-8 w-8 text-blue-500" />
+				<p class="mt-2 text-sm text-blue-600">Drop images here</p>
+			{:else}
+				<ImageIcon class="h-8 w-8 text-gray-400" />
+				<p class="mt-2 text-sm text-gray-600">
+					Click to upload or drag and drop images
+				</p>
+				<p class="text-xs text-gray-500">PNG, JPG, GIF up to 10MB</p>
+			{/if}
+		</div>
+	</div>
+</div>

src/lib/components/ui/file-upload/index.ts

@@ -0,0 +1 @@
+export { default as FileUpload } from './file-upload.svelte';

src/lib/utils/model-capabilities.ts

@@ -0,0 +1,9 @@
+import type { OpenRouterModel } from '$lib/backend/models/open-router';
+
+export function supportsImages(model: OpenRouterModel): boolean {
+	return model.architecture.input_modalities.includes('image');
+}
+
+export function getImageSupportedModels(models: OpenRouterModel[]): OpenRouterModel[] {
+	return models.filter(supportsImages);
+}

src/routes/api/generate-message/+server.ts

@@ -21,6 +21,10 @@ const reqBodySchema = z.object({
 
 	session_token: z.string(),
 	conversation_id: z.string().optional(),
+	images: z.array(z.object({
+		url: z.string(),
+		storage_id: z.string(),
+	})).optional(),
 });
 
 export type GenerateMessageRequestBody = z.infer<typeof reqBodySchema>;
@@ -247,10 +251,29 @@ ${attachedRules.map((r) => `- ${r.name}: ${r.rule}`).join('\n')}`,
 		apiKey: key,
 	});
 
+	const formattedMessages = messages.map((m) => {
+		if (m.images && m.images.length > 0 && m.role === 'user') {
+			return {
+				role: 'user' as const,
+				content: [
+					{ type: 'text' as const, text: m.content },
+					...m.images.map(img => ({
+						type: 'image_url' as const,
+						image_url: { url: img.url }
+					}))
+				]
+			};
+		}
+		return { 
+			role: m.role as 'user' | 'assistant' | 'system', 
+			content: m.content 
+		};
+	});
+
 	const streamResult = await ResultAsync.fromPromise(
 		openai.chat.completions.create({
 			model: model.model_id,
-			messages: [...messages.map((m) => ({ role: m.role, content: m.content })), systemMessage],
+			messages: [...formattedMessages, systemMessage],
 			temperature: 0.7,
 			stream: true,
 		}),
@@ -411,6 +434,7 @@ export const POST: RequestHandler = async ({ request }) => {
 				content: args.message,
 				role: 'user',
 				session_token: session.token,
+				images: args.images,
 			}),
 			(e) => `Failed to create conversation: ${e}`
 		);
@@ -444,6 +468,7 @@ export const POST: RequestHandler = async ({ request }) => {
 				session_token: args.session_token,
 				model_id: args.model_id,
 				role: 'user',
+				images: args.images,
 			}),
 			(e) => `Failed to create user message: ${e}`
 		);

src/routes/chat/+layout.svelte

@@ -31,6 +31,11 @@
 	import XIcon from '~icons/lucide/x';
 	import { callGenerateMessage } from '../api/generate-message/call.js';
 	import ModelPicker from './model-picker.svelte';
+	import { models } from '$lib/state/models.svelte';
+	import { supportsImages } from '$lib/utils/model-capabilities';
+	import { Provider } from '$lib/types';
+	import { FileUpload } from 'melt/builders';
+	import ImageIcon from '~icons/lucide/image';
 
 	const client = useConvexClient();
 
@@ -46,11 +51,16 @@
 		if (!isString(message) || !session.current?.user.id || !settings.modelId) return;
 
 		if (textarea) textarea.value = '';
+		const messageCopy = message;
+		const imagesCopy = [...selectedImages];
+		selectedImages = [];
+		
 		const res = await callGenerateMessage({
-			message,
+			message: messageCopy,
 			session_token: session.current?.session.token,
 			conversation_id: page.params.id ?? undefined,
 			model_id: settings.modelId,
+			images: imagesCopy.length > 0 ? imagesCopy : undefined,
 		});
 		if (res.isErr()) return; // TODO: Handle error
 
@@ -158,12 +168,85 @@
 	]);
 
 	let message = $state('');
+	let selectedImages = $state<{ url: string; storage_id: string }[]>([]);
+	let isUploading = $state(false);
+	let fileInput = $state<HTMLInputElement>();
 
 	usePrompt(
 		() => message,
 		(v) => (message = v)
 	);
 
+	models.init();
+
+	const currentModelSupportsImages = $derived.by(() => {
+		if (!settings.modelId) return false;
+		const openRouterModels = models.from(Provider.OpenRouter);
+		const currentModel = openRouterModels.find(m => m.id === settings.modelId);
+		return currentModel ? supportsImages(currentModel) : false;
+	});
+
+	const fileUpload = new FileUpload({
+		multiple: true,
+		accept: 'image/*',
+		maxSize: 10 * 1024 * 1024, // 10MB
+	});
+
+	async function handleFileChange(files: File[]) {
+		if (!files.length || !session.current?.session.token) return;
+
+		isUploading = true;
+		const uploadedFiles: { url: string; storage_id: string }[] = [];
+
+		try {
+			for (const file of files) {
+				// Generate upload URL
+				const uploadUrl = await client.mutation(api.storage.generateUploadUrl, {
+					session_token: session.current.session.token,
+				});
+
+				// Upload file
+				const result = await fetch(uploadUrl, {
+					method: 'POST',
+					body: file,
+				});
+
+				if (!result.ok) {
+					throw new Error(`Upload failed: ${result.statusText}`);
+				}
+
+				const { storageId } = await result.json();
+
+				// Get the URL for the uploaded file
+				const url = await client.query(api.storage.getUrl, {
+					storage_id: storageId,
+					session_token: session.current.session.token,
+				});
+
+				if (url) {
+					uploadedFiles.push({ url, storage_id: storageId });
+				}
+			}
+
+			selectedImages = [...selectedImages, ...uploadedFiles];
+		} catch (error) {
+			console.error('Upload failed:', error);
+		} finally {
+			isUploading = false;
+		}
+	}
+
+	function removeImage(index: number) {
+		selectedImages = selectedImages.filter((_, i) => i !== index);
+	}
+
+	$effect(() => {
+		if (fileUpload.selected.size > 0) {
+			handleFileChange(Array.from(fileUpload.selected));
+			fileUpload.clear();
+		}
+	});
+
 	const suggestedRules = $derived.by(() => {
 		if (!rulesQuery.data || rulesQuery.data.length === 0) return;
 		if (!textarea) return;
@@ -508,7 +591,32 @@
 							</div>
 						{/if}
 						<div class="flex flex-grow flex-col">
-							<div class="flex flex-grow flex-row items-start">
+							{#if selectedImages.length > 0}
+								<div class="flex flex-wrap gap-2 mb-2">
+									{#each selectedImages as image, index}
+										<div class="relative">
+											<img src={image.url} alt="Uploaded" class="h-16 w-16 rounded-lg object-cover" />
+											<button
+												type="button"
+												onclick={() => removeImage(index)}
+												class="absolute -top-2 -right-2 flex h-5 w-5 items-center justify-center rounded-full bg-red-500 text-white hover:bg-red-600"
+											>
+												<XIcon class="h-3 w-3" />
+											</button>
+										</div>
+									{/each}
+								</div>
+							{/if}
+							<div 
+								{...fileUpload.dropzone}
+								class={cn(
+									"flex flex-grow flex-row items-start relative transition-colors",
+									{
+										'bg-blue-50 border-2 border-dashed border-blue-400 rounded-lg': fileUpload.isDragging && currentModelSupportsImages,
+									}
+								)}
+							>
+								<input {...fileUpload.input} bind:this={fileInput} />
 								<!-- TODO: Figure out better autofocus solution -->
 								<!-- svelte-ignore a11y_autofocus -->
 								<textarea
@@ -552,9 +660,37 @@
 									autocomplete="off"
 									{@attach autosize.attachment}
 								></textarea>
+								{#if fileUpload.isDragging && currentModelSupportsImages}
+									<div class="absolute inset-0 flex items-center justify-center bg-blue-50/90 rounded-lg">
+										<div class="text-blue-600 text-center">
+											<ImageIcon class="h-8 w-8 mx-auto mb-2" />
+											<p class="text-sm font-medium">Drop images here</p>
+										</div>
+									</div>
+								{/if}
 							</div>
 							<div class="mt-2 -mb-px flex w-full flex-row-reverse justify-between">
 								<div class="-mt-0.5 -mr-0.5 flex items-center justify-center gap-2">
+									{#if currentModelSupportsImages}
+										<Tooltip placement="top">
+											{#snippet trigger(tooltip)}
+												<button
+													type="button"
+													onclick={() => fileInput?.click()}
+													disabled={isUploading}
+													class="border-reflect button-reflect hover:bg-secondary/90 active:bg-secondary text-secondary-foreground relative h-9 w-9 rounded-lg p-2 font-medium shadow transition disabled:opacity-50"
+													{...tooltip.trigger}
+												>
+													{#if isUploading}
+														<div class="animate-spin rounded-full h-4 w-4 border-2 border-current border-t-transparent"></div>
+													{:else}
+														<ImageIcon class="!size-4" />
+													{/if}
+												</button>
+											{/snippet}
+											{isUploading ? 'Uploading...' : 'Add images'}
+										</Tooltip>
+									{/if}
 									<Tooltip placement="top">
 										{#snippet trigger(tooltip)}
 											<button

src/routes/chat/[id]/message.svelte

@@ -25,6 +25,13 @@
 
 {#if message.role !== 'system' && !(message.role === 'assistant' && message.content.length === 0)}
 	<div class={cn('group flex max-w-[80%] flex-col gap-1', { 'self-end': message.role === 'user' })}>
+		{#if message.images && message.images.length > 0}
+			<div class="flex flex-wrap gap-2 mb-2">
+				{#each message.images as image}
+					<img src={image.url} alt="Uploaded" class="max-w-xs rounded-lg" />
+				{/each}
+			</div>
+		{/if}
 		<div class={style({ role: message.role })}>
 			<svelte:boundary>
 				<MarkdownRenderer content={message.content} />
@@ -32,7 +39,7 @@
 				{#snippet failed(error)}
 					<div class="text-destructive">
 						<span>Error rendering markdown:</span>
-						<pre class="!bg-sidebar"><code>{error.message}</code></pre>
+						<pre class="!bg-sidebar"><code>{error instanceof Error ? error.message : String(error)}</code></pre>
 					</div>
 				{/snippet}
 			</svelte:boundary>