Merge pull request #15 from TGlide/file-upload

wip by claude
This commit is contained in:
Thomas G. Lopes 2025-06-17 23:29:32 +01:00 committed by GitHub
commit 614410b076
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 500 additions and 24 deletions

View file

@ -12,20 +12,18 @@ Clone of [T3 Chat](https://t3.chat/)
- **Framework**: SvelteKit
- **Language**: TypeScript
- **Styling**: Tailwind
- **Backend**: Convex
- **Auth**: BetterAuth + Convex
- **Components**: Melt UI (next-gen)
- **Testing**: Humans
- **Package Manager**: pnpm
- **Linting**: ESLint
- **Formatting**: Prettier
### Discussion
- Vercel SDK?
- Nah, too limited
## 📦 Self-hosting
IDK, calm down
TODO: test self-hosting, including Convex self-hosting perhaps
TODO: add instructions
## TODO
@ -39,11 +37,10 @@ IDK, calm down
- ~[ ] OpenAI~
- [ ] File upload
- [x] Ensure responsiveness
- [ ] File support
- [x] Streams on the server
- [x] Streams on the server (Resumable streams)
- [x] Syntax highlighting with Shiki/markdown renderer
- [ ] Eliminate FOUC
- [ ] Cascade deletes and shit in Convex
- [x] Cascade deletes
- [ ] Error notification central, specially for BYOK models like o3
- [ ] Google Auth
- [ ] Fix light mode (urgh)
@ -67,3 +64,4 @@ IDK, calm down
- [ ] Chat sharing
- [ ] 404 page/redirect
- [ ] Test link with free credits
- [x] Cursor-like Rules (@ieedan's idea!)

View file

@ -22,4 +22,10 @@ export const auth = betterAuth({
},
},
plugins: [],
session: {
cookieCache: {
enabled: true,
maxAge: 5 * 60, // Cache duration in seconds
},
},
});

View file

@ -89,6 +89,11 @@ export const createAndAddMessage = mutation({
content: v.string(),
role: messageRoleValidator,
session_token: v.string(),
images: v.optional(v.array(v.object({
url: v.string(),
storage_id: v.string(),
fileName: v.optional(v.string()),
}))),
},
handler: async (
ctx,
@ -118,6 +123,7 @@ export const createAndAddMessage = mutation({
role: args.role,
conversation_id: conversationId,
session_token: args.session_token,
images: args.images,
});
return {

View file

@ -40,6 +40,12 @@ export const create = mutation({
model_id: v.optional(v.string()),
provider: v.optional(providerValidator),
token_count: v.optional(v.number()),
// Optional image attachments
images: v.optional(v.array(v.object({
url: v.string(),
storage_id: v.string(),
fileName: v.optional(v.string()),
}))),
},
handler: async (ctx, args): Promise<Id<'messages'>> => {
const session = await ctx.runQuery(api.betterAuth.publicGetSession, {
@ -72,6 +78,8 @@ export const create = mutation({
model_id: args.model_id,
provider: args.provider,
token_count: args.token_count,
// Optional image attachments
images: args.images,
}),
ctx.db.patch(args.conversation_id as Id<'conversations'>, {
generating: true,

View file

@ -46,7 +46,7 @@ export default defineSchema({
title: v.string(),
updated_at: v.optional(v.number()),
pinned: v.optional(v.boolean()),
generating: v.boolean(),
generating: v.optional(v.boolean()),
cost_usd: v.optional(v.number()),
}).index('by_user', ['user_id']),
messages: defineTable({
@ -57,6 +57,16 @@ export default defineSchema({
model_id: v.optional(v.string()),
provider: v.optional(providerValidator),
token_count: v.optional(v.number()),
// Optional image attachments
images: v.optional(
v.array(
v.object({
url: v.string(),
storage_id: v.string(),
fileName: v.optional(v.string()),
})
)
),
cost_usd: v.optional(v.number()),
generation_id: v.optional(v.string()),
}).index('by_conversation', ['conversation_id']),

View file

@ -0,0 +1,58 @@
import { v } from 'convex/values';
import { api } from './_generated/api';
import { query } from './_generated/server';
import { mutation } from './functions';
export const generateUploadUrl = mutation({
args: {
session_token: v.string(),
},
handler: async (ctx, args) => {
const session = await ctx.runQuery(api.betterAuth.publicGetSession, {
session_token: args.session_token,
});
if (!session) {
throw new Error('Unauthorized');
}
return await ctx.storage.generateUploadUrl();
},
});
export const getUrl = query({
args: {
storage_id: v.string(),
session_token: v.string(),
},
handler: async (ctx, args) => {
const session = await ctx.runQuery(api.betterAuth.publicGetSession, {
session_token: args.session_token,
});
if (!session) {
throw new Error('Unauthorized');
}
return await ctx.storage.getUrl(args.storage_id);
},
});
export const deleteFile = mutation({
args: {
storage_id: v.string(),
session_token: v.string(),
},
handler: async (ctx, args) => {
const session = await ctx.runQuery(api.betterAuth.publicGetSession, {
session_token: args.session_token,
});
if (!session) {
throw new Error('Unauthorized');
}
await ctx.storage.delete(args.storage_id);
},
});

View file

@ -24,6 +24,7 @@
sm: 'h-8 gap-1.5 rounded-md px-3 has-[>svg]:px-2.5',
lg: 'h-10 rounded-md px-6 has-[>svg]:px-4',
icon: 'size-9',
iconSm: 'size-7',
},
},
defaultVariants: {

View file

@ -0,0 +1,67 @@
<script lang="ts">
import { Modal } from '$lib/components/ui/modal';
import { Button } from '$lib/components/ui/button';
import DownloadIcon from '~icons/lucide/download';
import ExternalLinkIcon from '~icons/lucide/external-link';
import XIcon from '~icons/lucide/x';
import Tooltip from '../tooltip.svelte';
type Props = {
open?: boolean;
imageUrl: string;
fileName?: string;
};
let { open = $bindable(false), imageUrl, fileName = 'image' }: Props = $props();
function openInNewTab() {
window.open(imageUrl, '_blank');
}
</script>
<Modal bind:open>
<div class="flex items-center justify-between">
<h2 class="text-lg font-semibold">{fileName}</h2>
<div class="flex items-center gap-2">
<Tooltip>
{#snippet trigger(tooltip)}
<Button
size="iconSm"
variant="outline"
download={fileName}
href={imageUrl}
{...tooltip.trigger}
>
<DownloadIcon class="size-4" />
</Button>
{/snippet}
Download image
</Tooltip>
<Tooltip>
{#snippet trigger(tooltip)}
<Button size="iconSm" variant="outline" onclick={openInNewTab} {...tooltip.trigger}>
<ExternalLinkIcon class="size-4" />
</Button>
{/snippet}
Open in new tab
</Tooltip>
<Tooltip>
{#snippet trigger(tooltip)}
<Button
size="iconSm"
variant="outline"
onclick={() => (open = false)}
{...tooltip.trigger}
>
<XIcon class="size-4" />
</Button>
{/snippet}
Close
</Tooltip>
</div>
</div>
<div class="mt-2 flex items-center justify-center">
<img src={imageUrl} alt={fileName} class="max-h-[60vh] max-w-full rounded-lg object-contain" />
</div>
</Modal>

View file

@ -0,0 +1 @@
export { default as ImageModal } from './image-modal.svelte';

View file

@ -0,0 +1,73 @@
export function compressImage(file: File, maxSizeBytes: number = 1024 * 1024): Promise<File> {
return new Promise((resolve, reject) => {
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d');
const img = new Image();
img.onload = () => {
// Calculate new dimensions to maintain aspect ratio
let { width, height } = img;
const maxDimension = 1920; // Max width or height
if (width > maxDimension || height > maxDimension) {
if (width > height) {
height = (height * maxDimension) / width;
width = maxDimension;
} else {
width = (width * maxDimension) / height;
height = maxDimension;
}
}
canvas.width = width;
canvas.height = height;
if (!ctx) {
reject(new Error('Could not get canvas context'));
return;
}
// Draw and compress
ctx.drawImage(img, 0, 0, width, height);
// Start with high quality and reduce until under size limit
let quality = 0.9;
let compressed: File | null = null;
const tryCompress = () => {
canvas.toBlob(
(blob) => {
if (!blob) {
reject(new Error('Failed to compress image'));
return;
}
compressed = new File([blob], file.name, {
type: 'image/jpeg',
lastModified: Date.now(),
});
// If under size limit or quality is too low, return result
if (compressed.size <= maxSizeBytes || quality <= 0.1) {
resolve(compressed);
} else {
// Reduce quality and try again
quality -= 0.1;
tryCompress();
}
},
'image/jpeg',
quality
);
};
tryCompress();
};
img.onerror = () => {
reject(new Error('Failed to load image'));
};
img.src = URL.createObjectURL(file);
});
}

View file

@ -0,0 +1,9 @@
import type { OpenRouterModel } from '$lib/backend/models/open-router';
export function supportsImages(model: OpenRouterModel): boolean {
return model.architecture.input_modalities.includes('image');
}
export function getImageSupportedModels(models: OpenRouterModel[]): OpenRouterModel[] {
return models.filter(supportsImages);
}

View file

@ -20,6 +20,15 @@ const reqBodySchema = z.object({
session_token: z.string(),
conversation_id: z.string().optional(),
images: z
.array(
z.object({
url: z.string(),
storage_id: z.string(),
fileName: z.string().optional(),
})
)
.optional(),
});
export type GenerateMessageRequestBody = z.infer<typeof reqBodySchema>;
@ -189,14 +198,12 @@ async function generateAIResponse({
log('Background: Model found and enabled', startTime);
const messagesQuery = await messagesQueryResult;
if (messagesQuery.isErr()) {
log(`Background messages query failed: ${messagesQuery.error}`, startTime);
if (messagesQueryResult.isErr()) {
log(`Background messages query failed: ${messagesQueryResult.error}`, startTime);
return;
}
const messages = messagesQuery.value;
const messages = messagesQueryResult.value;
log(`Background: Retrieved ${messages.length} messages from conversation`, startTime);
if (keyResult.isErr()) {
@ -246,10 +253,29 @@ ${attachedRules.map((r) => `- ${r.name}: ${r.rule}`).join('\n')}`,
apiKey: key,
});
const formattedMessages = messages.map((m) => {
if (m.images && m.images.length > 0 && m.role === 'user') {
return {
role: 'user' as const,
content: [
{ type: 'text' as const, text: m.content },
...m.images.map((img) => ({
type: 'image_url' as const,
image_url: { url: img.url },
})),
],
};
}
return {
role: m.role as 'user' | 'assistant' | 'system',
content: m.content,
};
});
const streamResult = await ResultAsync.fromPromise(
openai.chat.completions.create({
model: model.model_id,
messages: [...messages.map((m) => ({ role: m.role, content: m.content })), systemMessage],
messages: [...formattedMessages, systemMessage],
temperature: 0.7,
stream: true,
}),
@ -463,6 +489,7 @@ export const POST: RequestHandler = async ({ request }) => {
client.mutation(api.conversations.createAndAddMessage, {
content: args.message,
role: 'user',
images: args.images,
session_token: sessionToken,
}),
(e) => `Failed to create conversation: ${e}`
@ -497,6 +524,7 @@ export const POST: RequestHandler = async ({ request }) => {
session_token: args.session_token,
model_id: args.model_id,
role: 'user',
images: args.images,
}),
(e) => `Failed to create user message: ${e}`
);

View file

@ -6,33 +6,39 @@
import { useCachedQuery } from '$lib/cache/cached-query.svelte.js';
import * as Icons from '$lib/components/icons';
import { Button } from '$lib/components/ui/button';
import { ImageModal } from '$lib/components/ui/image-modal';
import { LightSwitch } from '$lib/components/ui/light-switch/index.js';
import { callModal } from '$lib/components/ui/modal/global-modal.svelte';
import * as Sidebar from '$lib/components/ui/sidebar';
import Tooltip from '$lib/components/ui/tooltip.svelte';
import { cmdOrCtrl } from '$lib/hooks/is-mac.svelte.js';
import { TextareaAutosize } from '$lib/spells/textarea-autosize.svelte.js';
import { models } from '$lib/state/models.svelte';
import { usePrompt } from '$lib/state/prompt.svelte.js';
import { session } from '$lib/state/session.svelte.js';
import { settings } from '$lib/state/settings.svelte.js';
import { Provider } from '$lib/types';
import { compressImage } from '$lib/utils/image-compression';
import { isString } from '$lib/utils/is.js';
import { pick } from '$lib/utils/object.js';
import { supportsImages } from '$lib/utils/model-capabilities';
import { omit, pick } from '$lib/utils/object.js';
import { cn } from '$lib/utils/utils.js';
import { useConvexClient } from 'convex-svelte';
import { Popover } from 'melt/builders';
import { FileUpload, Popover } from 'melt/builders';
import { Avatar } from 'melt/components';
import { Debounced, ElementSize, IsMounted, ScrollState } from 'runed';
import SendIcon from '~icons/lucide/arrow-up';
import ChevronDownIcon from '~icons/lucide/chevron-down';
import ImageIcon from '~icons/lucide/image';
import LoaderCircleIcon from '~icons/lucide/loader-circle';
import PanelLeftIcon from '~icons/lucide/panel-left';
import PinIcon from '~icons/lucide/pin';
import PinOffIcon from '~icons/lucide/pin-off';
import Settings2Icon from '~icons/lucide/settings-2';
import UploadIcon from '~icons/lucide/upload';
import XIcon from '~icons/lucide/x';
import { callGenerateMessage } from '../api/generate-message/call.js';
import ModelPicker from './model-picker.svelte';
import { cmdOrCtrl } from '$lib/hooks/is-mac.svelte.js';
import { Provider } from '$lib/types.js';
const client = useConvexClient();
@ -48,11 +54,16 @@
if (!isString(message) || !session.current?.user.id || !settings.modelId) return;
if (textarea) textarea.value = '';
const messageCopy = message;
const imagesCopy = [...selectedImages];
selectedImages = [];
const res = await callGenerateMessage({
message,
message: messageCopy,
session_token: session.current?.session.token,
conversation_id: page.params.id ?? undefined,
model_id: settings.modelId,
images: imagesCopy.length > 0 ? imagesCopy : undefined,
});
if (res.isErr()) return; // TODO: Handle error
@ -165,12 +176,107 @@
]);
let message = $state('');
let selectedImages = $state<{ url: string; storage_id: string; fileName?: string }[]>([]);
let isUploading = $state(false);
let fileInput = $state<HTMLInputElement>();
let imageModal = $state<{ open: boolean; imageUrl: string; fileName: string }>({
open: false,
imageUrl: '',
fileName: '',
});
usePrompt(
() => message,
(v) => (message = v)
);
models.init();
const currentModelSupportsImages = $derived.by(() => {
if (!settings.modelId) return false;
const openRouterModels = models.from(Provider.OpenRouter);
const currentModel = openRouterModels.find((m) => m.id === settings.modelId);
return currentModel ? supportsImages(currentModel) : false;
});
const fileUpload = new FileUpload({
multiple: true,
accept: 'image/*',
maxSize: 10 * 1024 * 1024, // 10MB
});
async function handleFileChange(files: File[]) {
if (!files.length || !session.current?.session.token) return;
isUploading = true;
const uploadedFiles: { url: string; storage_id: string; fileName?: string }[] = [];
try {
for (const file of files) {
// Skip non-image files
if (!file.type.startsWith('image/')) {
console.warn('Skipping non-image file:', file.name);
continue;
}
// Compress image to max 1MB
const compressedFile = await compressImage(file, 1024 * 1024);
// Generate upload URL
const uploadUrl = await client.mutation(api.storage.generateUploadUrl, {
session_token: session.current.session.token,
});
// Upload compressed file
const result = await fetch(uploadUrl, {
method: 'POST',
body: compressedFile,
});
if (!result.ok) {
throw new Error(`Upload failed: ${result.statusText}`);
}
const { storageId } = await result.json();
// Get the URL for the uploaded file
const url = await client.query(api.storage.getUrl, {
storage_id: storageId,
session_token: session.current.session.token,
});
if (url) {
uploadedFiles.push({ url, storage_id: storageId, fileName: file.name });
}
}
selectedImages = [...selectedImages, ...uploadedFiles];
} catch (error) {
console.error('Upload failed:', error);
} finally {
isUploading = false;
}
}
function removeImage(index: number) {
selectedImages = selectedImages.filter((_, i) => i !== index);
}
function openImageModal(imageUrl: string, fileName: string) {
imageModal = {
open: true,
imageUrl,
fileName,
};
}
$effect(() => {
if (fileUpload.selected.size > 0) {
handleFileChange(Array.from(fileUpload.selected));
fileUpload.clear();
}
});
const suggestedRules = $derived.by(() => {
if (!rulesQuery.data || rulesQuery.data.length === 0) return;
if (!textarea) return;
@ -293,7 +399,10 @@
<title>Chat | Thom.chat</title>
</svelte:head>
<Sidebar.Root class="h-screen overflow-clip">
<Sidebar.Root
class="h-screen overflow-clip"
{...currentModelSupportsImages ? omit(fileUpload.dropzone, ['onclick']) : {}}
>
<Sidebar.Sidebar class="flex flex-col overflow-clip p-2">
<div class="flex place-items-center justify-center py-2">
<span class="text-center font-serif text-lg">Thom.chat</span>
@ -526,7 +635,36 @@
</div>
{/if}
<div class="flex flex-grow flex-col">
<div class="flex flex-grow flex-row items-start">
{#if selectedImages.length > 0}
<div class="mb-2 flex flex-wrap gap-2">
{#each selectedImages as image, index (image.storage_id)}
<div
class="group border-secondary-foreground/[0.08] bg-secondary-foreground/[0.02] hover:bg-secondary-foreground/10 relative flex h-12 w-12 max-w-full shrink-0 items-center justify-center gap-2 rounded-xl border border-solid p-0 transition-[width,height] duration-500"
>
<button
type="button"
onclick={() => openImageModal(image.url, image.fileName || 'image')}
class="rounded-lg"
>
<img
src={image.url}
alt="Uploaded"
class="size-10 rounded-lg object-cover opacity-100 transition-opacity"
/>
</button>
<button
type="button"
onclick={() => removeImage(index)}
class="bg-secondary hover:bg-muted absolute -top-1 -right-1 cursor-pointer rounded-full p-1 opacity-0 transition group-hover:opacity-100"
>
<XIcon class="h-3 w-3" />
</button>
</div>
{/each}
</div>
{/if}
<div class="relative flex flex-grow flex-row items-start">
<input {...fileUpload.input} bind:this={fileInput} />
<!-- TODO: Figure out better autofocus solution -->
<!-- svelte-ignore a11y_autofocus -->
<textarea
@ -589,6 +727,23 @@
</div>
<div class="flex flex-col gap-2 pr-2 sm:flex-row sm:items-center">
<ModelPicker />
{#if currentModelSupportsImages}
<button
type="button"
class="border-border hover:bg-muted flex items-center gap-1 rounded-full border px-2 py-1 text-xs transition-colors disabled:opacity-50"
onclick={() => fileInput?.click()}
disabled={isUploading}
>
{#if isUploading}
<div
class="size-3 animate-spin rounded-full border-2 border-current border-t-transparent"
></div>
{:else}
<ImageIcon class="!size-3" />
{/if}
<span>Attach image</span>
</button>
{/if}
</div>
</div>
</div>
@ -610,4 +765,20 @@
</div>
</div>
</Sidebar.Inset>
{#if fileUpload.isDragging && currentModelSupportsImages}
<div class="fixed inset-0 z-50 flex items-center justify-center bg-black/10 backdrop-blur-sm">
<div class="text-center">
<UploadIcon class="text-primary mx-auto mb-4 h-16 w-16" />
<p class="text-xl font-semibold">Add image</p>
<p class="mt-2 text-sm opacity-75">Drop an image here to attach it to your message.</p>
</div>
</div>
{/if}
<ImageModal
bind:open={imageModal.open}
imageUrl={imageModal.imageUrl}
fileName={imageModal.fileName}
/>
</Sidebar.Root>

View file

@ -5,6 +5,7 @@
import { CopyButton } from '$lib/components/ui/copy-button';
import '../../../markdown.css';
import MarkdownRenderer from './markdown-renderer.svelte';
import { ImageModal } from '$lib/components/ui/image-modal';
const style = tv({
base: 'prose rounded-lg p-2',
@ -21,10 +22,41 @@
};
let { message }: Props = $props();
let imageModal = $state<{ open: boolean; imageUrl: string; fileName: string }>({
open: false,
imageUrl: '',
fileName: '',
});
function openImageModal(imageUrl: string, fileName: string) {
imageModal = {
open: true,
imageUrl,
fileName,
};
}
</script>
{#if message.role !== 'system' && !(message.role === 'assistant' && message.content.length === 0)}
<div class={cn('group flex max-w-[80%] flex-col gap-1', { 'self-end': message.role === 'user' })}>
{#if message.images && message.images.length > 0}
<div class="mb-2 flex flex-wrap gap-2">
{#each message.images as image (image.storage_id)}
<button
type="button"
onclick={() => openImageModal(image.url, image.fileName || 'image')}
class="rounded-lg"
>
<img
src={image.url}
alt={image.fileName || 'Uploaded'}
class="max-w-xs rounded-lg transition-opacity hover:opacity-80"
/>
</button>
{/each}
</div>
{/if}
<div class={style({ role: message.role })}>
<svelte:boundary>
<MarkdownRenderer content={message.content} />
@ -32,7 +64,9 @@
{#snippet failed(error)}
<div class="text-destructive">
<span>Error rendering markdown:</span>
<pre class="!bg-sidebar"><code>{error.message}</code></pre>
<pre class="!bg-sidebar"><code
>{error instanceof Error ? error.message : String(error)}</code
></pre>
</div>
{/snippet}
</svelte:boundary>
@ -56,4 +90,10 @@
{/if}
</div>
</div>
<ImageModal
bind:open={imageModal.open}
imageUrl={imageModal.imageUrl}
fileName={imageModal.fileName}
/>
{/if}