diff --git a/src/lib/backend/convex/conversations.ts b/src/lib/backend/convex/conversations.ts index ec158d7..bf6c741 100644 --- a/src/lib/backend/convex/conversations.ts +++ b/src/lib/backend/convex/conversations.ts @@ -96,12 +96,15 @@ export const createAndAddMessage = mutation({ role: messageRoleValidator, session_token: v.string(), web_search_enabled: v.optional(v.boolean()), - images: v.optional( + attachments: v.optional( v.array( v.object({ + type: v.union(v.literal('image'), v.literal('video'), v.literal('audio'), v.literal('document')), url: v.string(), storage_id: v.string(), - fileName: v.optional(v.string()), + fileName: v.string(), + mimeType: v.string(), + size: v.number(), }) ) ), @@ -137,7 +140,7 @@ export const createAndAddMessage = mutation({ conversation_id: conversationId, session_token: args.session_token, web_search_enabled: args.web_search_enabled, - images: args.images, + attachments: args.attachments, }); return { diff --git a/src/lib/backend/convex/messages.ts b/src/lib/backend/convex/messages.ts index adcfc3c..b718ff1 100644 --- a/src/lib/backend/convex/messages.ts +++ b/src/lib/backend/convex/messages.ts @@ -48,13 +48,16 @@ export const create = mutation({ token_count: v.optional(v.number()), web_search_enabled: v.optional(v.boolean()), reasoning_effort: v.optional(reasoningEffortValidator), - // Optional image attachments - images: v.optional( + // Optional attachments + attachments: v.optional( v.array( v.object({ + type: v.union(v.literal('image'), v.literal('video'), v.literal('audio'), v.literal('document')), url: v.string(), storage_id: v.string(), - fileName: v.optional(v.string()), + fileName: v.string(), + mimeType: v.string(), + size: v.number(), }) ) ), @@ -96,8 +99,8 @@ export const create = mutation({ token_count: args.token_count, web_search_enabled: args.web_search_enabled, reasoning_effort: args.reasoning_effort, - // Optional image attachments - images: args.images, + // Optional attachments + attachments: args.attachments, }), ctx.db.patch(args.conversation_id as Id<'conversations'>, { generating: true, diff --git a/src/lib/backend/convex/schema.ts b/src/lib/backend/convex/schema.ts index 9add93b..e08211d 100644 --- a/src/lib/backend/convex/schema.ts +++ b/src/lib/backend/convex/schema.ts @@ -72,13 +72,16 @@ export default defineSchema({ model_id: v.optional(v.string()), provider: v.optional(providerValidator), token_count: v.optional(v.number()), - // Optional image attachments - images: v.optional( + // Optional attachments + attachments: v.optional( v.array( v.object({ + type: v.union(v.literal('image'), v.literal('video'), v.literal('audio'), v.literal('document')), url: v.string(), storage_id: v.string(), - fileName: v.optional(v.string()), + fileName: v.string(), + mimeType: v.string(), + size: v.number(), }) ) ), diff --git a/src/lib/components/model-picker/model-picker.svelte b/src/lib/components/model-picker/model-picker.svelte index 405a97e..8d0cf27 100644 --- a/src/lib/components/model-picker/model-picker.svelte +++ b/src/lib/components/model-picker/model-picker.svelte @@ -10,7 +10,15 @@ import { settings } from '$lib/state/settings.svelte'; import { Provider, PROVIDER_META } from '$lib/types'; import { fuzzysearch } from '$lib/utils/fuzzy-search'; - import { supportsImages, supportsReasoning, supportsStreaming, supportsToolCalls } from '$lib/utils/model-capabilities'; + import { + supportsImages, + supportsReasoning, + supportsStreaming, + supportsToolCalls, + supportsVideo, + supportsAudio, + supportsDocuments, + } from '$lib/utils/model-capabilities'; import { capitalize } from '$lib/utils/strings'; import { cn } from '$lib/utils/utils'; import { type Component } from 'svelte'; @@ -45,14 +53,33 @@ type Props = { class?: string; - /* When images are attached, we should not select models that don't support images */ - onlyImageModels?: boolean; + /* When attachments are present, restrict to models that support all attachment types */ + requiredCapabilities?: Array<'vision' | 'audio' | 'video' | 'documents'>; }; - let { class: className, onlyImageModels }: Props = $props(); + let { class: className, requiredCapabilities = [] }: Props = $props(); const client = useConvexClient(); + function meetsRequiredCapabilities(model: any): boolean { + if (requiredCapabilities.length === 0) return true; + + return requiredCapabilities.every((capability) => { + switch (capability) { + case 'vision': + return supportsImages(model); + case 'video': + return supportsVideo(model); + case 'audio': + return supportsAudio(model); + case 'documents': + return supportsDocuments(model); + default: + return true; + } + }); + } + const enabledModelsQuery = useCachedQuery(api.user_enabled_models.get_enabled, { session_token: session.current?.session.token ?? '', }); @@ -60,9 +87,9 @@ // Get enabled models from our models state with ModelInfo data const enabledArr = $derived.by(() => { const enabledModelIds = Object.keys(enabledModelsQuery.data ?? {}); - const enabledModels = modelsState.all().filter(model => - enabledModelIds.some(id => id.includes(model.id)) - ); + const enabledModels = modelsState + .all() + .filter((model) => enabledModelIds.some((id) => id.includes(model.id))); return enabledModels; }); @@ -146,7 +173,10 @@ // Group models by provider const groupedModels = $derived.by(() => { - const groups: Record = {} as Record; + const groups: Record = {} as Record< + Provider, + typeof filteredModels + >; filteredModels.forEach((model) => { const provider = model.provider as Provider; @@ -159,10 +189,13 @@ // Sort by provider order and name const result = Object.entries(groups) .sort(([a], [b]) => a.localeCompare(b)) - .map(([provider, models]) => [ - provider, - models.sort((a, b) => a.name.localeCompare(b.name)) - ] as [Provider, typeof models]); + .map( + ([provider, models]) => + [provider, models.sort((a, b) => a.name.localeCompare(b.name))] as [ + Provider, + typeof models, + ] + ); return result; }); @@ -338,8 +371,10 @@ {#if view === 'favorites' && pinnedModels.length > 0} {#each pinnedModels as model (model._id)} {@const modelInfo = enabledArr.find((m) => m.id === model.model_id)} - {@const formatted = modelInfo ? formatModelName(modelInfo) : { full: model.model_id, primary: model.model_id, secondary: '' }} - {@const disabled = onlyImageModels && modelInfo && !supportsImages(modelInfo)} + {@const formatted = modelInfo + ? formatModelName(modelInfo) + : { full: model.model_id, primary: model.model_id, secondary: '' }} + {@const disabled = modelInfo && !meetsRequiredCapabilities(modelInfo)} {/if} - + {#if modelInfo && supportsStreaming(modelInfo)} {#snippet trigger(tooltip)} @@ -499,8 +534,8 @@ {#snippet modelCard(model: (typeof enabledArr)[number])} {@const formatted = formatModelName(model)} - {@const disabled = onlyImageModels && !supportsImages(model)} - {@const enabledModelData = enabledModelsData.find(m => m.model_id === model.id)} + {@const disabled = !meetsRequiredCapabilities(model)} + {@const enabledModelData = enabledModelsData.find((m) => m.model_id === model.id)} = { supportsStreaming: true, supportsTools: true, supportsVision: true, + supportsVideo: false, + supportsAudio: true, + supportsDocuments: false, supportsEmbeddings: true, }, [Provider.Anthropic]: { @@ -65,6 +71,9 @@ export const PROVIDER_META: Record = { supportsStreaming: true, supportsTools: true, supportsVision: true, + supportsVideo: false, + supportsAudio: false, + supportsDocuments: true, supportsEmbeddings: false, }, [Provider.Gemini]: { @@ -77,6 +86,9 @@ export const PROVIDER_META: Record = { supportsStreaming: true, supportsTools: true, supportsVision: true, + supportsVideo: true, + supportsAudio: true, + supportsDocuments: true, supportsEmbeddings: true, }, [Provider.Mistral]: { @@ -89,6 +101,9 @@ export const PROVIDER_META: Record = { supportsStreaming: true, supportsTools: true, supportsVision: false, + supportsVideo: false, + supportsAudio: false, + supportsDocuments: false, supportsEmbeddings: true, }, [Provider.Cohere]: { @@ -101,6 +116,9 @@ export const PROVIDER_META: Record = { supportsStreaming: true, supportsTools: true, supportsVision: false, + supportsVideo: false, + supportsAudio: false, + supportsDocuments: false, supportsEmbeddings: true, }, [Provider.OpenRouter]: { @@ -113,6 +131,9 @@ export const PROVIDER_META: Record = { supportsStreaming: true, supportsTools: true, supportsVision: true, + supportsVideo: false, + supportsAudio: false, + supportsDocuments: false, supportsEmbeddings: false, }, }; diff --git a/src/lib/utils/attachment-manager.ts b/src/lib/utils/attachment-manager.ts new file mode 100644 index 0000000..cb3a4ad --- /dev/null +++ b/src/lib/utils/attachment-manager.ts @@ -0,0 +1,223 @@ +import type { ModelInfo } from '@keplersystems/kepler-ai-sdk'; +import { supportsImages, supportsVideo, supportsAudio, supportsDocuments } from './model-capabilities'; + +export type AttachmentType = 'image' | 'video' | 'audio' | 'document'; + +export interface AttachmentInfo { + type: AttachmentType; + mimeType: string; + maxSize: number; // in bytes + extensions: string[]; +} + +export interface ProcessedAttachment { + type: AttachmentType; + url: string; + storage_id: string; + fileName: string; + mimeType: string; + size: number; +} + +export class AttachmentManager { + private static readonly SUPPORTED_ATTACHMENTS: Record = { + image: { + type: 'image', + mimeType: 'image/*', + maxSize: 10 * 1024 * 1024, // 10MB + extensions: ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.svg'] + }, + video: { + type: 'video', + mimeType: 'video/*', + maxSize: 100 * 1024 * 1024, // 100MB + extensions: ['.mp4', '.webm', '.ogg', '.avi', '.mov', '.wmv', '.flv', '.mkv'] + }, + audio: { + type: 'audio', + mimeType: 'audio/*', + maxSize: 50 * 1024 * 1024, // 50MB + extensions: ['.mp3', '.wav', '.ogg', '.m4a', '.aac', '.flac', '.wma'] + }, + document: { + type: 'document', + mimeType: 'application/pdf,text/*', + maxSize: 20 * 1024 * 1024, // 20MB + extensions: ['.pdf', '.txt', '.md', '.doc', '.docx', '.rtf', '.csv', '.json', '.xml', '.html'] + } + }; + + private static readonly MIME_TYPE_MAPPING: Record = { + // Images + 'image/jpeg': 'image', + 'image/jpg': 'image', + 'image/png': 'image', + 'image/gif': 'image', + 'image/webp': 'image', + 'image/bmp': 'image', + 'image/svg+xml': 'image', + + // Videos + 'video/mp4': 'video', + 'video/webm': 'video', + 'video/ogg': 'video', + 'video/avi': 'video', + 'video/quicktime': 'video', + 'video/x-msvideo': 'video', + 'video/x-flv': 'video', + 'video/x-matroska': 'video', + + // Audio + 'audio/mpeg': 'audio', + 'audio/mp3': 'audio', + 'audio/wav': 'audio', + 'audio/ogg': 'audio', + 'audio/mp4': 'audio', + 'audio/aac': 'audio', + 'audio/flac': 'audio', + 'audio/x-ms-wma': 'audio', + + // Documents + 'application/pdf': 'document', + 'text/plain': 'document', + 'text/markdown': 'document', + 'text/csv': 'document', + 'text/html': 'document', + 'text/xml': 'document', + 'application/json': 'document', + 'application/msword': 'document', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'document', + 'application/rtf': 'document' + }; + + /** + * Get supported attachment types for a given model + */ + static getSupportedAttachmentTypes(model: ModelInfo): AttachmentType[] { + const types: AttachmentType[] = []; + + if (supportsImages(model)) types.push('image'); + if (supportsVideo(model)) types.push('video'); + if (supportsAudio(model)) types.push('audio'); + if (supportsDocuments(model)) types.push('document'); + + return types; + } + + /** + * Get attachment info for supported types + */ + static getAttachmentInfo(types: AttachmentType[]): AttachmentInfo[] { + return types.map(type => this.SUPPORTED_ATTACHMENTS[type]); + } + + /** + * Validate if a file is supported by the given model + */ + static validateFile(file: File, model: ModelInfo): { valid: boolean; error?: string; type?: AttachmentType } { + const supportedTypes = this.getSupportedAttachmentTypes(model); + const fileType = this.getFileType(file); + + if (!fileType) { + return { valid: false, error: `Unsupported file type: ${file.type}` }; + } + + if (!supportedTypes.includes(fileType)) { + return { valid: false, error: `${fileType} files are not supported by this model` }; + } + + const attachmentInfo = this.SUPPORTED_ATTACHMENTS[fileType]; + if (file.size > attachmentInfo.maxSize) { + return { + valid: false, + error: `File size (${this.formatFileSize(file.size)}) exceeds maximum allowed size (${this.formatFileSize(attachmentInfo.maxSize)}) for ${fileType} files` + }; + } + + return { valid: true, type: fileType }; + } + + /** + * Get file type from File object + */ + static getFileType(file: File): AttachmentType | null { + if (this.MIME_TYPE_MAPPING[file.type]) { + return this.MIME_TYPE_MAPPING[file.type]; + } + + // Fallback to extension-based detection + const extension = this.getFileExtension(file.name); + for (const [type, info] of Object.entries(this.SUPPORTED_ATTACHMENTS)) { + if (info.extensions.includes(extension)) { + return type as AttachmentType; + } + } + + return null; + } + + /** + * Get file extension from filename + */ + private static getFileExtension(filename: string): string { + return '.' + filename.split('.').pop()?.toLowerCase() || ''; + } + + /** + * Format file size for human reading + */ + static formatFileSize(bytes: number): string { + const sizes = ['Bytes', 'KB', 'MB', 'GB']; + if (bytes === 0) return '0 Bytes'; + const i = Math.floor(Math.log(bytes) / Math.log(1024)); + return Math.round((bytes / Math.pow(1024, i)) * 100) / 100 + ' ' + sizes[i]; + } + + /** + * Get accept string for HTML input element + */ + static getAcceptString(types: AttachmentType[]): string { + const mimeTypes = types.map(type => this.SUPPORTED_ATTACHMENTS[type].mimeType); + return mimeTypes.join(','); + } + + /** + * Get icon name for attachment type + */ + static getTypeIcon(type: AttachmentType): string { + const icons = { + image: 'image', + video: 'video', + audio: 'music', + document: 'file-text' + }; + return icons[type]; + } + + /** + * Get display name for attachment type + */ + static getTypeDisplayName(type: AttachmentType): string { + const names = { + image: 'Image', + video: 'Video', + audio: 'Audio', + document: 'Document' + }; + return names[type]; + } + + /** + * Create accept string for multiple types + */ + static createFileInputAccept(supportedTypes: AttachmentType[]): string { + if (supportedTypes.length === 0) return ''; + + const extensions: string[] = []; + supportedTypes.forEach(type => { + extensions.push(...this.SUPPORTED_ATTACHMENTS[type].extensions); + }); + + return extensions.join(','); + } +} \ No newline at end of file diff --git a/src/lib/utils/model-capabilities.ts b/src/lib/utils/model-capabilities.ts index 55280f5..c7a2f2a 100644 --- a/src/lib/utils/model-capabilities.ts +++ b/src/lib/utils/model-capabilities.ts @@ -13,13 +13,37 @@ export function supportsStreaming(model: ModelInfo): boolean { } export function supportsToolCalls(model: ModelInfo): boolean { - return model.capabilities.toolCalls; + return model.capabilities.functionCalling; +} + +export function supportsVideo(model: ModelInfo): boolean { + return model.capabilities.video ?? false; +} + +export function supportsAudio(model: ModelInfo): boolean { + return model.capabilities.audio; +} + +export function supportsDocuments(model: ModelInfo): boolean { + return model.capabilities.documents ?? false; } export function getImageSupportedModels(models: ModelInfo[]): ModelInfo[] { return models.filter(supportsImages); } +export function getVideoSupportedModels(models: ModelInfo[]): ModelInfo[] { + return models.filter(supportsVideo); +} + +export function getAudioSupportedModels(models: ModelInfo[]): ModelInfo[] { + return models.filter(supportsAudio); +} + +export function getDocumentSupportedModels(models: ModelInfo[]): ModelInfo[] { + return models.filter(supportsDocuments); +} + export function getReasoningModels(models: ModelInfo[]): ModelInfo[] { return models.filter(supportsReasoning); } diff --git a/src/routes/api/generate-message/+server.ts b/src/routes/api/generate-message/+server.ts index 90cb4ff..731363b 100644 --- a/src/routes/api/generate-message/+server.ts +++ b/src/routes/api/generate-message/+server.ts @@ -25,12 +25,15 @@ const reqBodySchema = z session_token: z.string(), conversation_id: z.string().optional(), web_search_enabled: z.boolean().optional(), - images: z + attachments: z .array( z.object({ + type: z.enum(['image', 'video', 'audio', 'document']), url: z.string(), storage_id: z.string(), - fileName: z.string().optional(), + fileName: z.string(), + mimeType: z.string(), + size: z.number(), }) ) .optional(), @@ -353,18 +356,57 @@ async function generateAIResponse({ log(`Background: ${attachedRules.length} rules attached`, startTime); const formattedMessages = messages.map((m) => { - if (m.images && m.images.length > 0 && m.role === 'user') { + // Handle attachments format + if (m.attachments && m.attachments.length > 0 && m.role === 'user') { + const contentParts: Array<{ + type: string; + text?: string; + imageUrl?: string; + videoUrl?: string; + audioUrl?: string; + documentUrl?: string; + mimeType?: string; + }> = [{ type: 'text', text: m.content }]; + + for (const attachment of m.attachments) { + switch (attachment.type) { + case 'image': + contentParts.push({ + type: 'image', + imageUrl: attachment.url, + mimeType: attachment.mimeType, + }); + break; + case 'video': + contentParts.push({ + type: 'video', + videoUrl: attachment.url, + mimeType: attachment.mimeType, + }); + break; + case 'audio': + contentParts.push({ + type: 'audio', + audioUrl: attachment.url, + mimeType: attachment.mimeType, + }); + break; + case 'document': + contentParts.push({ + type: 'document', + documentUrl: attachment.url, + mimeType: attachment.mimeType, + }); + break; + } + } + return { role: 'user' as const, - content: [ - { type: 'text', text: m.content }, - ...m.images.map((img) => ({ - type: 'image_url', - image_url: { url: img.url }, - })), - ], + content: contentParts, }; } + return { role: m.role as 'user' | 'assistant' | 'system', content: m.content, @@ -618,7 +660,7 @@ export const POST: RequestHandler = async ({ request }) => { content: args.message, content_html: '', role: 'user', - images: args.images, + attachments: args.attachments, web_search_enabled: args.web_search_enabled, session_token: sessionToken, }), @@ -657,7 +699,7 @@ export const POST: RequestHandler = async ({ request }) => { model_id: args.model_id, reasoning_effort: args.reasoning_effort, role: 'user', - images: args.images, + attachments: args.attachments, web_search_enabled: args.web_search_enabled, }), (e) => `Failed to create user message: ${e}` diff --git a/src/routes/chat/+layout.svelte b/src/routes/chat/+layout.svelte index be00c11..e2de67d 100644 --- a/src/routes/chat/+layout.svelte +++ b/src/routes/chat/+layout.svelte @@ -20,7 +20,8 @@ import { settings } from '$lib/state/settings.svelte.js'; import { Provider } from '$lib/types'; import { compressImage } from '$lib/utils/image-compression'; - import { supportsImages, supportsReasoning } from '$lib/utils/model-capabilities'; + import { supportsImages, supportsReasoning, supportsVideo, supportsAudio, supportsDocuments } from '$lib/utils/model-capabilities'; + import { AttachmentManager, type AttachmentType, type ProcessedAttachment } from '$lib/utils/attachment-manager'; import { omit, pick } from '$lib/utils/object.js'; import { cn } from '$lib/utils/utils.js'; import { useConvexClient } from 'convex-svelte'; @@ -30,6 +31,10 @@ import SendIcon from '~icons/lucide/arrow-up'; import ChevronDownIcon from '~icons/lucide/chevron-down'; import ImageIcon from '~icons/lucide/image'; + import VideoIcon from '~icons/lucide/video'; + import AudioIcon from '~icons/lucide/music'; + import FileTextIcon from '~icons/lucide/file-text'; + import PaperclipIcon from '~icons/lucide/paperclip'; import PanelLeftIcon from '~icons/lucide/panel-left'; import SearchIcon from '~icons/lucide/search'; import Settings2Icon from '~icons/lucide/settings-2'; @@ -119,8 +124,8 @@ loading = true; - const imagesCopy = [...selectedImages]; - selectedImages = []; + const attachmentsCopy = [...selectedAttachments]; + selectedAttachments = []; try { const res = await callGenerateMessage({ @@ -128,7 +133,7 @@ session_token: session.current?.session.token, conversation_id: page.params.id ?? undefined, model_id: settings.modelId, - images: imagesCopy.length > 0 ? imagesCopy : undefined, + attachments: attachmentsCopy.length > 0 ? attachmentsCopy : undefined, web_search_enabled: settings.webSearchEnabled, reasoning_effort: currentModelSupportsReasoning ? settings.reasoningEffort : undefined, }); @@ -195,7 +200,7 @@ const autosize = new TextareaAutosize(); const message = new PersistedState('prompt', ''); - let selectedImages = $state<{ url: string; storage_id: string; fileName?: string }[]>([]); + let selectedAttachments = $state([]); let isUploading = $state(false); let fileInput = $state(); let imageModal = $state<{ open: boolean; imageUrl: string; fileName: string }>({ @@ -211,11 +216,20 @@ models.init(); - const currentModelSupportsImages = $derived.by(() => { - if (!settings.modelId) return false; + const currentModel = $derived.by(() => { + if (!settings.modelId) return null; const allModels = models.all(); - const currentModel = allModels.find((m) => m.id === settings.modelId); - return currentModel ? supportsImages(currentModel) : false; + return allModels.find((m) => m.id === settings.modelId) || null; + }); + + const currentModelSupportsImages = $derived(currentModel ? supportsImages(currentModel) : false); + const currentModelSupportsVideo = $derived(currentModel ? supportsVideo(currentModel) : false); + const currentModelSupportsAudio = $derived(currentModel ? supportsAudio(currentModel) : false); + const currentModelSupportsDocuments = $derived(currentModel ? supportsDocuments(currentModel) : false); + + const supportedAttachmentTypes = $derived.by(() => { + if (!currentModel) return []; + return AttachmentManager.getSupportedAttachmentTypes(currentModel); }); const currentModelSupportsReasoning = $derived.by(() => { @@ -228,36 +242,49 @@ const fileUpload = new FileUpload({ multiple: true, - accept: 'image/*', - maxSize: 10 * 1024 * 1024, // 10MB + maxSize: 100 * 1024 * 1024, // 100MB max for any file type + }); + + // Update the file input accept attribute reactively + $effect(() => { + if (fileInput) { + const acceptString = AttachmentManager.getAcceptString(supportedAttachmentTypes); + fileInput.accept = acceptString; + } }); async function handleFileChange(files: File[]) { - if (!files.length || !session.current?.session.token) return; + if (!files.length || !session.current?.session.token || !currentModel) return; isUploading = true; - const uploadedFiles: { url: string; storage_id: string; fileName?: string }[] = []; + const uploadedFiles: ProcessedAttachment[] = []; try { for (const file of files) { - // Skip non-image files - if (!file.type.startsWith('image/')) { - console.warn('Skipping non-image file:', file.name); + // Validate file against current model capabilities + const validation = AttachmentManager.validateFile(file, currentModel); + if (!validation.valid) { + console.warn(`Skipping invalid file ${file.name}: ${validation.error}`); + // TODO: Show error toast to user continue; } - // Compress image to max 1MB - const compressedFile = await compressImage(file, 1024 * 1024); + let fileToUpload = file; + + // Compress images to max 1MB for better performance + if (validation.type === 'image') { + fileToUpload = await compressImage(file, 1024 * 1024); + } // Generate upload URL const uploadUrl = await client.mutation(api.storage.generateUploadUrl, { session_token: session.current.session.token, }); - // Upload compressed file + // Upload file const result = await fetch(uploadUrl, { method: 'POST', - body: compressedFile, + body: fileToUpload, }); if (!result.ok) { @@ -273,11 +300,18 @@ }); if (url) { - uploadedFiles.push({ url, storage_id: storageId, fileName: file.name }); + uploadedFiles.push({ + type: validation.type!, + url, + storage_id: storageId, + fileName: file.name, + mimeType: file.type, + size: file.size + }); } } - selectedImages = [...selectedImages, ...uploadedFiles]; + selectedAttachments = [...selectedAttachments, ...uploadedFiles]; } catch (error) { console.error('Upload failed:', error); } finally { @@ -285,8 +319,31 @@ } } - function removeImage(index: number) { - selectedImages = selectedImages.filter((_, i) => i !== index); + function removeAttachment(index: number) { + selectedAttachments = selectedAttachments.filter((_, i) => i !== index); + } + + function getRequiredCapabilities(attachments: ProcessedAttachment[]): Array<'vision' | 'audio' | 'video' | 'documents'> { + const capabilities: Array<'vision' | 'audio' | 'video' | 'documents'> = []; + + attachments.forEach(attachment => { + switch (attachment.type) { + case 'image': + if (!capabilities.includes('vision')) capabilities.push('vision'); + break; + case 'video': + if (!capabilities.includes('video')) capabilities.push('video'); + break; + case 'audio': + if (!capabilities.includes('audio')) capabilities.push('audio'); + break; + case 'document': + if (!capabilities.includes('documents')) capabilities.push('documents'); + break; + } + }); + + return capabilities; } function openImageModal(imageUrl: string, fileName: string) { @@ -446,7 +503,7 @@ 0 ? omit(fileUpload.dropzone, ['onclick']) : {}} > @@ -608,26 +665,45 @@ {/if}
- {#if selectedImages.length > 0} + {#if selectedAttachments.length > 0}
- {#each selectedImages as image, index (image.storage_id)} + {#each selectedAttachments as attachment, index (attachment.storage_id)}
+ {#if attachment.type === 'image'} + + {:else if attachment.type === 'video'} +
+ + {attachment.fileName} +
+ {:else if attachment.type === 'audio'} +
+ + {attachment.fileName} +
+ {:else if attachment.type === 'document'} +
+ + {attachment.fileName} +
+ {/if} -
- 0} /> + 0 ? getRequiredCapabilities(selectedAttachments) : []} />
{#if currentModelSupportsReasoning} @@ -746,7 +822,7 @@ - {#if currentModelSupportsImages} + {#if supportedAttachmentTypes.length > 0}
{:else} - + {/if} - + {/if} {#if session.current !== null && message.current.trim() !== ''} @@ -811,12 +893,20 @@
- {#if fileUpload.isDragging && currentModelSupportsImages} + {#if fileUpload.isDragging && supportedAttachmentTypes.length > 0}
-

Add image

-

Drop an image here to attach it to your message.

+

+ {#if supportedAttachmentTypes.length === 1 && supportedAttachmentTypes[0]} + Add {AttachmentManager.getTypeDisplayName(supportedAttachmentTypes[0]).toLowerCase()} + {:else} + Add files + {/if} +

+

+ Drop {supportedAttachmentTypes.length === 1 ? `${supportedAttachmentTypes[0]}s` : 'supported files'} here to attach to your message. +

{/if}