diff --git a/src/lib/components/model-picker/model-picker.svelte b/src/lib/components/model-picker/model-picker.svelte index 8d0cf27..eaa30c1 100644 --- a/src/lib/components/model-picker/model-picker.svelte +++ b/src/lib/components/model-picker/model-picker.svelte @@ -15,9 +15,6 @@ supportsReasoning, supportsStreaming, supportsToolCalls, - supportsVideo, - supportsAudio, - supportsDocuments, } from '$lib/utils/model-capabilities'; import { capitalize } from '$lib/utils/strings'; import { cn } from '$lib/utils/utils'; @@ -53,7 +50,7 @@ type Props = { class?: string; - /* When attachments are present, restrict to models that support all attachment types */ + /* Required capabilities that the selected model must support */ requiredCapabilities?: Array<'vision' | 'audio' | 'video' | 'documents'>; }; @@ -61,25 +58,6 @@ const client = useConvexClient(); - function meetsRequiredCapabilities(model: any): boolean { - if (requiredCapabilities.length === 0) return true; - - return requiredCapabilities.every((capability) => { - switch (capability) { - case 'vision': - return supportsImages(model); - case 'video': - return supportsVideo(model); - case 'audio': - return supportsAudio(model); - case 'documents': - return supportsDocuments(model); - default: - return true; - } - }); - } - const enabledModelsQuery = useCachedQuery(api.user_enabled_models.get_enabled, { session_token: session.current?.session.token ?? '', }); @@ -285,6 +263,18 @@ // until we migrate the pinning system to work with the new ModelInfo structure const enabledModelsData = $derived(Object.values(enabledModelsQuery.data ?? {})); const pinnedModels = $derived(enabledModelsData.filter((m) => isPinned(m))); + + function modelSupportsRequiredCapabilities(model: typeof enabledArr[number], required: Array<'vision' | 'audio' | 'video' | 'documents'>): boolean { + return required.every(capability => { + switch (capability) { + case 'vision': return model.capabilities.vision; + case 'audio': return model.capabilities.audio; + case 'video': return model.capabilities.video; + case 'documents': return model.capabilities.documents; + default: return false; + } + }); + } 0 && modelInfo && !modelSupportsRequiredCapabilities(modelInfo, requiredCapabilities)} 0 && !modelSupportsRequiredCapabilities(model, requiredCapabilities)} {@const enabledModelData = enabledModelsData.find((m) => m.model_id === model.id)} = { apiKeyName: 'OpenAI API Key', placeholder: 'sk-...', docsLink: 'https://platform.openai.com/docs', - supportsStreaming: true, - supportsTools: true, - supportsVision: true, - supportsVideo: false, - supportsAudio: true, - supportsDocuments: false, - supportsEmbeddings: true, }, [Provider.Anthropic]: { title: 'Anthropic', @@ -68,13 +54,6 @@ export const PROVIDER_META: Record = { apiKeyName: 'Anthropic API Key', placeholder: 'sk-ant-...', docsLink: 'https://docs.anthropic.com', - supportsStreaming: true, - supportsTools: true, - supportsVision: true, - supportsVideo: false, - supportsAudio: false, - supportsDocuments: true, - supportsEmbeddings: false, }, [Provider.Gemini]: { title: 'Google Gemini', @@ -83,13 +62,6 @@ export const PROVIDER_META: Record = { apiKeyName: 'Google AI API Key', placeholder: 'AIza...', docsLink: 'https://ai.google.dev/docs', - supportsStreaming: true, - supportsTools: true, - supportsVision: true, - supportsVideo: true, - supportsAudio: true, - supportsDocuments: true, - supportsEmbeddings: true, }, [Provider.Mistral]: { title: 'Mistral', @@ -98,13 +70,6 @@ export const PROVIDER_META: Record = { apiKeyName: 'Mistral API Key', placeholder: 'mistral-...', docsLink: 'https://docs.mistral.ai', - supportsStreaming: true, - supportsTools: true, - supportsVision: false, - supportsVideo: false, - supportsAudio: false, - supportsDocuments: false, - supportsEmbeddings: true, }, [Provider.Cohere]: { title: 'Cohere', @@ -113,13 +78,6 @@ export const PROVIDER_META: Record = { apiKeyName: 'Cohere API Key', placeholder: 'co_...', docsLink: 'https://docs.cohere.com', - supportsStreaming: true, - supportsTools: true, - supportsVision: false, - supportsVideo: false, - supportsAudio: false, - supportsDocuments: false, - supportsEmbeddings: true, }, [Provider.OpenRouter]: { title: 'OpenRouter', @@ -128,12 +86,5 @@ export const PROVIDER_META: Record = { apiKeyName: 'OpenRouter API Key', placeholder: 'sk-or-...', docsLink: 'https://openrouter.ai/docs', - supportsStreaming: true, - supportsTools: true, - supportsVision: true, - supportsVideo: false, - supportsAudio: false, - supportsDocuments: false, - supportsEmbeddings: false, }, }; diff --git a/src/lib/utils/attachment-manager.ts b/src/lib/utils/attachment-manager.ts index cb3a4ad..1830887 100644 --- a/src/lib/utils/attachment-manager.ts +++ b/src/lib/utils/attachment-manager.ts @@ -1,15 +1,7 @@ import type { ModelInfo } from '@keplersystems/kepler-ai-sdk'; -import { supportsImages, supportsVideo, supportsAudio, supportsDocuments } from './model-capabilities'; export type AttachmentType = 'image' | 'video' | 'audio' | 'document'; -export interface AttachmentInfo { - type: AttachmentType; - mimeType: string; - maxSize: number; // in bytes - extensions: string[]; -} - export interface ProcessedAttachment { type: AttachmentType; url: string; @@ -19,205 +11,30 @@ export interface ProcessedAttachment { size: number; } -export class AttachmentManager { - private static readonly SUPPORTED_ATTACHMENTS: Record = { - image: { - type: 'image', - mimeType: 'image/*', - maxSize: 10 * 1024 * 1024, // 10MB - extensions: ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.svg'] - }, - video: { - type: 'video', - mimeType: 'video/*', - maxSize: 100 * 1024 * 1024, // 100MB - extensions: ['.mp4', '.webm', '.ogg', '.avi', '.mov', '.wmv', '.flv', '.mkv'] - }, - audio: { - type: 'audio', - mimeType: 'audio/*', - maxSize: 50 * 1024 * 1024, // 50MB - extensions: ['.mp3', '.wav', '.ogg', '.m4a', '.aac', '.flac', '.wma'] - }, - document: { - type: 'document', - mimeType: 'application/pdf,text/*', - maxSize: 20 * 1024 * 1024, // 20MB - extensions: ['.pdf', '.txt', '.md', '.doc', '.docx', '.rtf', '.csv', '.json', '.xml', '.html'] - } - }; +const MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB - private static readonly MIME_TYPE_MAPPING: Record = { - // Images - 'image/jpeg': 'image', - 'image/jpg': 'image', - 'image/png': 'image', - 'image/gif': 'image', - 'image/webp': 'image', - 'image/bmp': 'image', - 'image/svg+xml': 'image', - - // Videos - 'video/mp4': 'video', - 'video/webm': 'video', - 'video/ogg': 'video', - 'video/avi': 'video', - 'video/quicktime': 'video', - 'video/x-msvideo': 'video', - 'video/x-flv': 'video', - 'video/x-matroska': 'video', - - // Audio - 'audio/mpeg': 'audio', - 'audio/mp3': 'audio', - 'audio/wav': 'audio', - 'audio/ogg': 'audio', - 'audio/mp4': 'audio', - 'audio/aac': 'audio', - 'audio/flac': 'audio', - 'audio/x-ms-wma': 'audio', - - // Documents - 'application/pdf': 'document', - 'text/plain': 'document', - 'text/markdown': 'document', - 'text/csv': 'document', - 'text/html': 'document', - 'text/xml': 'document', - 'application/json': 'document', - 'application/msword': 'document', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'document', - 'application/rtf': 'document' - }; +export function getSupportedAttachmentTypes(model: ModelInfo): AttachmentType[] { + const types: AttachmentType[] = []; + if (model.capabilities.vision) types.push('image'); + if (model.capabilities.video) types.push('video'); + if (model.capabilities.audio) types.push('audio'); + if (model.capabilities.documents) types.push('document'); + return types; +} - /** - * Get supported attachment types for a given model - */ - static getSupportedAttachmentTypes(model: ModelInfo): AttachmentType[] { - const types: AttachmentType[] = []; - - if (supportsImages(model)) types.push('image'); - if (supportsVideo(model)) types.push('video'); - if (supportsAudio(model)) types.push('audio'); - if (supportsDocuments(model)) types.push('document'); - - return types; - } - - /** - * Get attachment info for supported types - */ - static getAttachmentInfo(types: AttachmentType[]): AttachmentInfo[] { - return types.map(type => this.SUPPORTED_ATTACHMENTS[type]); - } - - /** - * Validate if a file is supported by the given model - */ - static validateFile(file: File, model: ModelInfo): { valid: boolean; error?: string; type?: AttachmentType } { - const supportedTypes = this.getSupportedAttachmentTypes(model); - const fileType = this.getFileType(file); - - if (!fileType) { - return { valid: false, error: `Unsupported file type: ${file.type}` }; - } - - if (!supportedTypes.includes(fileType)) { - return { valid: false, error: `${fileType} files are not supported by this model` }; - } - - const attachmentInfo = this.SUPPORTED_ATTACHMENTS[fileType]; - if (file.size > attachmentInfo.maxSize) { - return { - valid: false, - error: `File size (${this.formatFileSize(file.size)}) exceeds maximum allowed size (${this.formatFileSize(attachmentInfo.maxSize)}) for ${fileType} files` - }; - } - - return { valid: true, type: fileType }; - } - - /** - * Get file type from File object - */ - static getFileType(file: File): AttachmentType | null { - if (this.MIME_TYPE_MAPPING[file.type]) { - return this.MIME_TYPE_MAPPING[file.type]; - } - - // Fallback to extension-based detection - const extension = this.getFileExtension(file.name); - for (const [type, info] of Object.entries(this.SUPPORTED_ATTACHMENTS)) { - if (info.extensions.includes(extension)) { - return type as AttachmentType; - } - } - - return null; - } - - /** - * Get file extension from filename - */ - private static getFileExtension(filename: string): string { - return '.' + filename.split('.').pop()?.toLowerCase() || ''; - } - - /** - * Format file size for human reading - */ - static formatFileSize(bytes: number): string { - const sizes = ['Bytes', 'KB', 'MB', 'GB']; - if (bytes === 0) return '0 Bytes'; - const i = Math.floor(Math.log(bytes) / Math.log(1024)); - return Math.round((bytes / Math.pow(1024, i)) * 100) / 100 + ' ' + sizes[i]; - } - - /** - * Get accept string for HTML input element - */ - static getAcceptString(types: AttachmentType[]): string { - const mimeTypes = types.map(type => this.SUPPORTED_ATTACHMENTS[type].mimeType); - return mimeTypes.join(','); - } - - /** - * Get icon name for attachment type - */ - static getTypeIcon(type: AttachmentType): string { - const icons = { - image: 'image', - video: 'video', - audio: 'music', - document: 'file-text' - }; - return icons[type]; - } - - /** - * Get display name for attachment type - */ - static getTypeDisplayName(type: AttachmentType): string { - const names = { - image: 'Image', - video: 'Video', - audio: 'Audio', - document: 'Document' - }; - return names[type]; - } - - /** - * Create accept string for multiple types - */ - static createFileInputAccept(supportedTypes: AttachmentType[]): string { - if (supportedTypes.length === 0) return ''; - - const extensions: string[] = []; - supportedTypes.forEach(type => { - extensions.push(...this.SUPPORTED_ATTACHMENTS[type].extensions); - }); - - return extensions.join(','); +export function getFileType(file: File): AttachmentType | null { + const { type } = file; + if (type.startsWith('image/')) return 'image'; + if (type.startsWith('video/')) return 'video'; + if (type.startsWith('audio/')) return 'audio'; + if (type === 'application/pdf' || type.startsWith('text/')) return 'document'; + return null; +} + +export function getAcceptString(types: AttachmentType[]): string { + const mimeTypes = types.map(type => `${type}/*`); + if (types.includes('document')) { + mimeTypes.push('application/pdf', 'text/*'); } + return mimeTypes.join(','); } \ No newline at end of file diff --git a/src/lib/utils/model-capabilities.ts b/src/lib/utils/model-capabilities.ts index c7a2f2a..97b0675 100644 --- a/src/lib/utils/model-capabilities.ts +++ b/src/lib/utils/model-capabilities.ts @@ -16,34 +16,10 @@ export function supportsToolCalls(model: ModelInfo): boolean { return model.capabilities.functionCalling; } -export function supportsVideo(model: ModelInfo): boolean { - return model.capabilities.video ?? false; -} - -export function supportsAudio(model: ModelInfo): boolean { - return model.capabilities.audio; -} - -export function supportsDocuments(model: ModelInfo): boolean { - return model.capabilities.documents ?? false; -} - export function getImageSupportedModels(models: ModelInfo[]): ModelInfo[] { return models.filter(supportsImages); } -export function getVideoSupportedModels(models: ModelInfo[]): ModelInfo[] { - return models.filter(supportsVideo); -} - -export function getAudioSupportedModels(models: ModelInfo[]): ModelInfo[] { - return models.filter(supportsAudio); -} - -export function getDocumentSupportedModels(models: ModelInfo[]): ModelInfo[] { - return models.filter(supportsDocuments); -} - export function getReasoningModels(models: ModelInfo[]): ModelInfo[] { return models.filter(supportsReasoning); } diff --git a/src/routes/api/generate-message/+server.ts b/src/routes/api/generate-message/+server.ts index 731363b..8359ed0 100644 --- a/src/routes/api/generate-message/+server.ts +++ b/src/routes/api/generate-message/+server.ts @@ -356,55 +356,17 @@ async function generateAIResponse({ log(`Background: ${attachedRules.length} rules attached`, startTime); const formattedMessages = messages.map((m) => { - // Handle attachments format if (m.attachments && m.attachments.length > 0 && m.role === 'user') { - const contentParts: Array<{ - type: string; - text?: string; - imageUrl?: string; - videoUrl?: string; - audioUrl?: string; - documentUrl?: string; - mimeType?: string; - }> = [{ type: 'text', text: m.content }]; + const contentParts = [ + { type: 'text', text: m.content }, + ...m.attachments.map(attachment => ({ + type: attachment.type, + [`${attachment.type}Url`]: attachment.url, + mimeType: attachment.mimeType, + })) + ]; - for (const attachment of m.attachments) { - switch (attachment.type) { - case 'image': - contentParts.push({ - type: 'image', - imageUrl: attachment.url, - mimeType: attachment.mimeType, - }); - break; - case 'video': - contentParts.push({ - type: 'video', - videoUrl: attachment.url, - mimeType: attachment.mimeType, - }); - break; - case 'audio': - contentParts.push({ - type: 'audio', - audioUrl: attachment.url, - mimeType: attachment.mimeType, - }); - break; - case 'document': - contentParts.push({ - type: 'document', - documentUrl: attachment.url, - mimeType: attachment.mimeType, - }); - break; - } - } - - return { - role: 'user' as const, - content: contentParts, - }; + return { role: 'user' as const, content: contentParts }; } return { diff --git a/src/routes/chat/+layout.svelte b/src/routes/chat/+layout.svelte index e2de67d..e82fe62 100644 --- a/src/routes/chat/+layout.svelte +++ b/src/routes/chat/+layout.svelte @@ -20,8 +20,8 @@ import { settings } from '$lib/state/settings.svelte.js'; import { Provider } from '$lib/types'; import { compressImage } from '$lib/utils/image-compression'; - import { supportsImages, supportsReasoning, supportsVideo, supportsAudio, supportsDocuments } from '$lib/utils/model-capabilities'; - import { AttachmentManager, type AttachmentType, type ProcessedAttachment } from '$lib/utils/attachment-manager'; + import { supportsImages, supportsReasoning } from '$lib/utils/model-capabilities'; + import { getSupportedAttachmentTypes, getFileType, getAcceptString, type ProcessedAttachment } from '$lib/utils/attachment-manager'; import { omit, pick } from '$lib/utils/object.js'; import { cn } from '$lib/utils/utils.js'; import { useConvexClient } from 'convex-svelte'; @@ -218,19 +218,12 @@ const currentModel = $derived.by(() => { if (!settings.modelId) return null; - const allModels = models.all(); - return allModels.find((m) => m.id === settings.modelId) || null; + return models.all().find((m) => m.id === settings.modelId) || null; }); - const currentModelSupportsImages = $derived(currentModel ? supportsImages(currentModel) : false); - const currentModelSupportsVideo = $derived(currentModel ? supportsVideo(currentModel) : false); - const currentModelSupportsAudio = $derived(currentModel ? supportsAudio(currentModel) : false); - const currentModelSupportsDocuments = $derived(currentModel ? supportsDocuments(currentModel) : false); - - const supportedAttachmentTypes = $derived.by(() => { - if (!currentModel) return []; - return AttachmentManager.getSupportedAttachmentTypes(currentModel); - }); + const supportedAttachmentTypes = $derived( + currentModel ? getSupportedAttachmentTypes(currentModel) : [] + ); const currentModelSupportsReasoning = $derived.by(() => { if (!settings.modelId) return false; @@ -245,11 +238,10 @@ maxSize: 100 * 1024 * 1024, // 100MB max for any file type }); - // Update the file input accept attribute reactively + // Update file input accept attribute reactively $effect(() => { if (fileInput) { - const acceptString = AttachmentManager.getAcceptString(supportedAttachmentTypes); - fileInput.accept = acceptString; + fileInput.accept = getAcceptString(supportedAttachmentTypes); } }); @@ -261,18 +253,16 @@ try { for (const file of files) { - // Validate file against current model capabilities - const validation = AttachmentManager.validateFile(file, currentModel); - if (!validation.valid) { - console.warn(`Skipping invalid file ${file.name}: ${validation.error}`); - // TODO: Show error toast to user + // Simple file validation + const fileType = getFileType(file); + if (!fileType || !supportedAttachmentTypes.includes(fileType)) { + console.warn(`Unsupported file type: ${file.name}`); continue; } + // Compress images for better performance let fileToUpload = file; - - // Compress images to max 1MB for better performance - if (validation.type === 'image') { + if (fileType === 'image') { fileToUpload = await compressImage(file, 1024 * 1024); } @@ -301,7 +291,7 @@ if (url) { uploadedFiles.push({ - type: validation.type!, + type: fileType, url, storage_id: storageId, fileName: file.name, @@ -838,7 +828,7 @@ {/if}