feat: Add support for audio, video and document file types
This commit is contained in:
parent
31d72543b3
commit
12b4fef96d
9 changed files with 532 additions and 88 deletions
|
|
@ -96,12 +96,15 @@ export const createAndAddMessage = mutation({
|
|||
role: messageRoleValidator,
|
||||
session_token: v.string(),
|
||||
web_search_enabled: v.optional(v.boolean()),
|
||||
images: v.optional(
|
||||
attachments: v.optional(
|
||||
v.array(
|
||||
v.object({
|
||||
type: v.union(v.literal('image'), v.literal('video'), v.literal('audio'), v.literal('document')),
|
||||
url: v.string(),
|
||||
storage_id: v.string(),
|
||||
fileName: v.optional(v.string()),
|
||||
fileName: v.string(),
|
||||
mimeType: v.string(),
|
||||
size: v.number(),
|
||||
})
|
||||
)
|
||||
),
|
||||
|
|
@ -137,7 +140,7 @@ export const createAndAddMessage = mutation({
|
|||
conversation_id: conversationId,
|
||||
session_token: args.session_token,
|
||||
web_search_enabled: args.web_search_enabled,
|
||||
images: args.images,
|
||||
attachments: args.attachments,
|
||||
});
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -48,13 +48,16 @@ export const create = mutation({
|
|||
token_count: v.optional(v.number()),
|
||||
web_search_enabled: v.optional(v.boolean()),
|
||||
reasoning_effort: v.optional(reasoningEffortValidator),
|
||||
// Optional image attachments
|
||||
images: v.optional(
|
||||
// Optional attachments
|
||||
attachments: v.optional(
|
||||
v.array(
|
||||
v.object({
|
||||
type: v.union(v.literal('image'), v.literal('video'), v.literal('audio'), v.literal('document')),
|
||||
url: v.string(),
|
||||
storage_id: v.string(),
|
||||
fileName: v.optional(v.string()),
|
||||
fileName: v.string(),
|
||||
mimeType: v.string(),
|
||||
size: v.number(),
|
||||
})
|
||||
)
|
||||
),
|
||||
|
|
@ -96,8 +99,8 @@ export const create = mutation({
|
|||
token_count: args.token_count,
|
||||
web_search_enabled: args.web_search_enabled,
|
||||
reasoning_effort: args.reasoning_effort,
|
||||
// Optional image attachments
|
||||
images: args.images,
|
||||
// Optional attachments
|
||||
attachments: args.attachments,
|
||||
}),
|
||||
ctx.db.patch(args.conversation_id as Id<'conversations'>, {
|
||||
generating: true,
|
||||
|
|
|
|||
|
|
@ -72,13 +72,16 @@ export default defineSchema({
|
|||
model_id: v.optional(v.string()),
|
||||
provider: v.optional(providerValidator),
|
||||
token_count: v.optional(v.number()),
|
||||
// Optional image attachments
|
||||
images: v.optional(
|
||||
// Optional attachments
|
||||
attachments: v.optional(
|
||||
v.array(
|
||||
v.object({
|
||||
type: v.union(v.literal('image'), v.literal('video'), v.literal('audio'), v.literal('document')),
|
||||
url: v.string(),
|
||||
storage_id: v.string(),
|
||||
fileName: v.optional(v.string()),
|
||||
fileName: v.string(),
|
||||
mimeType: v.string(),
|
||||
size: v.number(),
|
||||
})
|
||||
)
|
||||
),
|
||||
|
|
|
|||
|
|
@ -10,7 +10,15 @@
|
|||
import { settings } from '$lib/state/settings.svelte';
|
||||
import { Provider, PROVIDER_META } from '$lib/types';
|
||||
import { fuzzysearch } from '$lib/utils/fuzzy-search';
|
||||
import { supportsImages, supportsReasoning, supportsStreaming, supportsToolCalls } from '$lib/utils/model-capabilities';
|
||||
import {
|
||||
supportsImages,
|
||||
supportsReasoning,
|
||||
supportsStreaming,
|
||||
supportsToolCalls,
|
||||
supportsVideo,
|
||||
supportsAudio,
|
||||
supportsDocuments,
|
||||
} from '$lib/utils/model-capabilities';
|
||||
import { capitalize } from '$lib/utils/strings';
|
||||
import { cn } from '$lib/utils/utils';
|
||||
import { type Component } from 'svelte';
|
||||
|
|
@ -45,14 +53,33 @@
|
|||
|
||||
type Props = {
|
||||
class?: string;
|
||||
/* When images are attached, we should not select models that don't support images */
|
||||
onlyImageModels?: boolean;
|
||||
/* When attachments are present, restrict to models that support all attachment types */
|
||||
requiredCapabilities?: Array<'vision' | 'audio' | 'video' | 'documents'>;
|
||||
};
|
||||
|
||||
let { class: className, onlyImageModels }: Props = $props();
|
||||
let { class: className, requiredCapabilities = [] }: Props = $props();
|
||||
|
||||
const client = useConvexClient();
|
||||
|
||||
function meetsRequiredCapabilities(model: any): boolean {
|
||||
if (requiredCapabilities.length === 0) return true;
|
||||
|
||||
return requiredCapabilities.every((capability) => {
|
||||
switch (capability) {
|
||||
case 'vision':
|
||||
return supportsImages(model);
|
||||
case 'video':
|
||||
return supportsVideo(model);
|
||||
case 'audio':
|
||||
return supportsAudio(model);
|
||||
case 'documents':
|
||||
return supportsDocuments(model);
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const enabledModelsQuery = useCachedQuery(api.user_enabled_models.get_enabled, {
|
||||
session_token: session.current?.session.token ?? '',
|
||||
});
|
||||
|
|
@ -60,9 +87,9 @@
|
|||
// Get enabled models from our models state with ModelInfo data
|
||||
const enabledArr = $derived.by(() => {
|
||||
const enabledModelIds = Object.keys(enabledModelsQuery.data ?? {});
|
||||
const enabledModels = modelsState.all().filter(model =>
|
||||
enabledModelIds.some(id => id.includes(model.id))
|
||||
);
|
||||
const enabledModels = modelsState
|
||||
.all()
|
||||
.filter((model) => enabledModelIds.some((id) => id.includes(model.id)));
|
||||
return enabledModels;
|
||||
});
|
||||
|
||||
|
|
@ -146,7 +173,10 @@
|
|||
|
||||
// Group models by provider
|
||||
const groupedModels = $derived.by(() => {
|
||||
const groups: Record<Provider, typeof filteredModels> = {} as Record<Provider, typeof filteredModels>;
|
||||
const groups: Record<Provider, typeof filteredModels> = {} as Record<
|
||||
Provider,
|
||||
typeof filteredModels
|
||||
>;
|
||||
|
||||
filteredModels.forEach((model) => {
|
||||
const provider = model.provider as Provider;
|
||||
|
|
@ -159,10 +189,13 @@
|
|||
// Sort by provider order and name
|
||||
const result = Object.entries(groups)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([provider, models]) => [
|
||||
provider,
|
||||
models.sort((a, b) => a.name.localeCompare(b.name))
|
||||
] as [Provider, typeof models]);
|
||||
.map(
|
||||
([provider, models]) =>
|
||||
[provider, models.sort((a, b) => a.name.localeCompare(b.name))] as [
|
||||
Provider,
|
||||
typeof models,
|
||||
]
|
||||
);
|
||||
|
||||
return result;
|
||||
});
|
||||
|
|
@ -338,8 +371,10 @@
|
|||
{#if view === 'favorites' && pinnedModels.length > 0}
|
||||
{#each pinnedModels as model (model._id)}
|
||||
{@const modelInfo = enabledArr.find((m) => m.id === model.model_id)}
|
||||
{@const formatted = modelInfo ? formatModelName(modelInfo) : { full: model.model_id, primary: model.model_id, secondary: '' }}
|
||||
{@const disabled = onlyImageModels && modelInfo && !supportsImages(modelInfo)}
|
||||
{@const formatted = modelInfo
|
||||
? formatModelName(modelInfo)
|
||||
: { full: model.model_id, primary: model.model_id, secondary: '' }}
|
||||
{@const disabled = modelInfo && !meetsRequiredCapabilities(modelInfo)}
|
||||
|
||||
<Command.Item
|
||||
value={model.model_id}
|
||||
|
|
@ -499,8 +534,8 @@
|
|||
|
||||
{#snippet modelCard(model: (typeof enabledArr)[number])}
|
||||
{@const formatted = formatModelName(model)}
|
||||
{@const disabled = onlyImageModels && !supportsImages(model)}
|
||||
{@const enabledModelData = enabledModelsData.find(m => m.model_id === model.id)}
|
||||
{@const disabled = !meetsRequiredCapabilities(model)}
|
||||
{@const enabledModelData = enabledModelsData.find((m) => m.model_id === model.id)}
|
||||
|
||||
<Command.Item
|
||||
value={model.id}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,9 @@ export type ProviderMeta = {
|
|||
supportsStreaming: boolean;
|
||||
supportsTools: boolean;
|
||||
supportsVision: boolean;
|
||||
supportsVideo: boolean;
|
||||
supportsAudio: boolean;
|
||||
supportsDocuments: boolean;
|
||||
supportsEmbeddings: boolean;
|
||||
};
|
||||
|
||||
|
|
@ -53,6 +56,9 @@ export const PROVIDER_META: Record<Provider, ProviderMeta> = {
|
|||
supportsStreaming: true,
|
||||
supportsTools: true,
|
||||
supportsVision: true,
|
||||
supportsVideo: false,
|
||||
supportsAudio: true,
|
||||
supportsDocuments: false,
|
||||
supportsEmbeddings: true,
|
||||
},
|
||||
[Provider.Anthropic]: {
|
||||
|
|
@ -65,6 +71,9 @@ export const PROVIDER_META: Record<Provider, ProviderMeta> = {
|
|||
supportsStreaming: true,
|
||||
supportsTools: true,
|
||||
supportsVision: true,
|
||||
supportsVideo: false,
|
||||
supportsAudio: false,
|
||||
supportsDocuments: true,
|
||||
supportsEmbeddings: false,
|
||||
},
|
||||
[Provider.Gemini]: {
|
||||
|
|
@ -77,6 +86,9 @@ export const PROVIDER_META: Record<Provider, ProviderMeta> = {
|
|||
supportsStreaming: true,
|
||||
supportsTools: true,
|
||||
supportsVision: true,
|
||||
supportsVideo: true,
|
||||
supportsAudio: true,
|
||||
supportsDocuments: true,
|
||||
supportsEmbeddings: true,
|
||||
},
|
||||
[Provider.Mistral]: {
|
||||
|
|
@ -89,6 +101,9 @@ export const PROVIDER_META: Record<Provider, ProviderMeta> = {
|
|||
supportsStreaming: true,
|
||||
supportsTools: true,
|
||||
supportsVision: false,
|
||||
supportsVideo: false,
|
||||
supportsAudio: false,
|
||||
supportsDocuments: false,
|
||||
supportsEmbeddings: true,
|
||||
},
|
||||
[Provider.Cohere]: {
|
||||
|
|
@ -101,6 +116,9 @@ export const PROVIDER_META: Record<Provider, ProviderMeta> = {
|
|||
supportsStreaming: true,
|
||||
supportsTools: true,
|
||||
supportsVision: false,
|
||||
supportsVideo: false,
|
||||
supportsAudio: false,
|
||||
supportsDocuments: false,
|
||||
supportsEmbeddings: true,
|
||||
},
|
||||
[Provider.OpenRouter]: {
|
||||
|
|
@ -113,6 +131,9 @@ export const PROVIDER_META: Record<Provider, ProviderMeta> = {
|
|||
supportsStreaming: true,
|
||||
supportsTools: true,
|
||||
supportsVision: true,
|
||||
supportsVideo: false,
|
||||
supportsAudio: false,
|
||||
supportsDocuments: false,
|
||||
supportsEmbeddings: false,
|
||||
},
|
||||
};
|
||||
|
|
|
|||
223
src/lib/utils/attachment-manager.ts
Normal file
223
src/lib/utils/attachment-manager.ts
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
import type { ModelInfo } from '@keplersystems/kepler-ai-sdk';
|
||||
import { supportsImages, supportsVideo, supportsAudio, supportsDocuments } from './model-capabilities';
|
||||
|
||||
export type AttachmentType = 'image' | 'video' | 'audio' | 'document';
|
||||
|
||||
export interface AttachmentInfo {
|
||||
type: AttachmentType;
|
||||
mimeType: string;
|
||||
maxSize: number; // in bytes
|
||||
extensions: string[];
|
||||
}
|
||||
|
||||
export interface ProcessedAttachment {
|
||||
type: AttachmentType;
|
||||
url: string;
|
||||
storage_id: string;
|
||||
fileName: string;
|
||||
mimeType: string;
|
||||
size: number;
|
||||
}
|
||||
|
||||
export class AttachmentManager {
|
||||
private static readonly SUPPORTED_ATTACHMENTS: Record<AttachmentType, AttachmentInfo> = {
|
||||
image: {
|
||||
type: 'image',
|
||||
mimeType: 'image/*',
|
||||
maxSize: 10 * 1024 * 1024, // 10MB
|
||||
extensions: ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.svg']
|
||||
},
|
||||
video: {
|
||||
type: 'video',
|
||||
mimeType: 'video/*',
|
||||
maxSize: 100 * 1024 * 1024, // 100MB
|
||||
extensions: ['.mp4', '.webm', '.ogg', '.avi', '.mov', '.wmv', '.flv', '.mkv']
|
||||
},
|
||||
audio: {
|
||||
type: 'audio',
|
||||
mimeType: 'audio/*',
|
||||
maxSize: 50 * 1024 * 1024, // 50MB
|
||||
extensions: ['.mp3', '.wav', '.ogg', '.m4a', '.aac', '.flac', '.wma']
|
||||
},
|
||||
document: {
|
||||
type: 'document',
|
||||
mimeType: 'application/pdf,text/*',
|
||||
maxSize: 20 * 1024 * 1024, // 20MB
|
||||
extensions: ['.pdf', '.txt', '.md', '.doc', '.docx', '.rtf', '.csv', '.json', '.xml', '.html']
|
||||
}
|
||||
};
|
||||
|
||||
private static readonly MIME_TYPE_MAPPING: Record<string, AttachmentType> = {
|
||||
// Images
|
||||
'image/jpeg': 'image',
|
||||
'image/jpg': 'image',
|
||||
'image/png': 'image',
|
||||
'image/gif': 'image',
|
||||
'image/webp': 'image',
|
||||
'image/bmp': 'image',
|
||||
'image/svg+xml': 'image',
|
||||
|
||||
// Videos
|
||||
'video/mp4': 'video',
|
||||
'video/webm': 'video',
|
||||
'video/ogg': 'video',
|
||||
'video/avi': 'video',
|
||||
'video/quicktime': 'video',
|
||||
'video/x-msvideo': 'video',
|
||||
'video/x-flv': 'video',
|
||||
'video/x-matroska': 'video',
|
||||
|
||||
// Audio
|
||||
'audio/mpeg': 'audio',
|
||||
'audio/mp3': 'audio',
|
||||
'audio/wav': 'audio',
|
||||
'audio/ogg': 'audio',
|
||||
'audio/mp4': 'audio',
|
||||
'audio/aac': 'audio',
|
||||
'audio/flac': 'audio',
|
||||
'audio/x-ms-wma': 'audio',
|
||||
|
||||
// Documents
|
||||
'application/pdf': 'document',
|
||||
'text/plain': 'document',
|
||||
'text/markdown': 'document',
|
||||
'text/csv': 'document',
|
||||
'text/html': 'document',
|
||||
'text/xml': 'document',
|
||||
'application/json': 'document',
|
||||
'application/msword': 'document',
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'document',
|
||||
'application/rtf': 'document'
|
||||
};
|
||||
|
||||
/**
|
||||
* Get supported attachment types for a given model
|
||||
*/
|
||||
static getSupportedAttachmentTypes(model: ModelInfo): AttachmentType[] {
|
||||
const types: AttachmentType[] = [];
|
||||
|
||||
if (supportsImages(model)) types.push('image');
|
||||
if (supportsVideo(model)) types.push('video');
|
||||
if (supportsAudio(model)) types.push('audio');
|
||||
if (supportsDocuments(model)) types.push('document');
|
||||
|
||||
return types;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get attachment info for supported types
|
||||
*/
|
||||
static getAttachmentInfo(types: AttachmentType[]): AttachmentInfo[] {
|
||||
return types.map(type => this.SUPPORTED_ATTACHMENTS[type]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate if a file is supported by the given model
|
||||
*/
|
||||
static validateFile(file: File, model: ModelInfo): { valid: boolean; error?: string; type?: AttachmentType } {
|
||||
const supportedTypes = this.getSupportedAttachmentTypes(model);
|
||||
const fileType = this.getFileType(file);
|
||||
|
||||
if (!fileType) {
|
||||
return { valid: false, error: `Unsupported file type: ${file.type}` };
|
||||
}
|
||||
|
||||
if (!supportedTypes.includes(fileType)) {
|
||||
return { valid: false, error: `${fileType} files are not supported by this model` };
|
||||
}
|
||||
|
||||
const attachmentInfo = this.SUPPORTED_ATTACHMENTS[fileType];
|
||||
if (file.size > attachmentInfo.maxSize) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `File size (${this.formatFileSize(file.size)}) exceeds maximum allowed size (${this.formatFileSize(attachmentInfo.maxSize)}) for ${fileType} files`
|
||||
};
|
||||
}
|
||||
|
||||
return { valid: true, type: fileType };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file type from File object
|
||||
*/
|
||||
static getFileType(file: File): AttachmentType | null {
|
||||
if (this.MIME_TYPE_MAPPING[file.type]) {
|
||||
return this.MIME_TYPE_MAPPING[file.type];
|
||||
}
|
||||
|
||||
// Fallback to extension-based detection
|
||||
const extension = this.getFileExtension(file.name);
|
||||
for (const [type, info] of Object.entries(this.SUPPORTED_ATTACHMENTS)) {
|
||||
if (info.extensions.includes(extension)) {
|
||||
return type as AttachmentType;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file extension from filename
|
||||
*/
|
||||
private static getFileExtension(filename: string): string {
|
||||
return '.' + filename.split('.').pop()?.toLowerCase() || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Format file size for human reading
|
||||
*/
|
||||
static formatFileSize(bytes: number): string {
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB'];
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(1024));
|
||||
return Math.round((bytes / Math.pow(1024, i)) * 100) / 100 + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get accept string for HTML input element
|
||||
*/
|
||||
static getAcceptString(types: AttachmentType[]): string {
|
||||
const mimeTypes = types.map(type => this.SUPPORTED_ATTACHMENTS[type].mimeType);
|
||||
return mimeTypes.join(',');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get icon name for attachment type
|
||||
*/
|
||||
static getTypeIcon(type: AttachmentType): string {
|
||||
const icons = {
|
||||
image: 'image',
|
||||
video: 'video',
|
||||
audio: 'music',
|
||||
document: 'file-text'
|
||||
};
|
||||
return icons[type];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get display name for attachment type
|
||||
*/
|
||||
static getTypeDisplayName(type: AttachmentType): string {
|
||||
const names = {
|
||||
image: 'Image',
|
||||
video: 'Video',
|
||||
audio: 'Audio',
|
||||
document: 'Document'
|
||||
};
|
||||
return names[type];
|
||||
}
|
||||
|
||||
/**
|
||||
* Create accept string for multiple types
|
||||
*/
|
||||
static createFileInputAccept(supportedTypes: AttachmentType[]): string {
|
||||
if (supportedTypes.length === 0) return '';
|
||||
|
||||
const extensions: string[] = [];
|
||||
supportedTypes.forEach(type => {
|
||||
extensions.push(...this.SUPPORTED_ATTACHMENTS[type].extensions);
|
||||
});
|
||||
|
||||
return extensions.join(',');
|
||||
}
|
||||
}
|
||||
|
|
@ -13,13 +13,37 @@ export function supportsStreaming(model: ModelInfo): boolean {
|
|||
}
|
||||
|
||||
export function supportsToolCalls(model: ModelInfo): boolean {
|
||||
return model.capabilities.toolCalls;
|
||||
return model.capabilities.functionCalling;
|
||||
}
|
||||
|
||||
export function supportsVideo(model: ModelInfo): boolean {
|
||||
return model.capabilities.video ?? false;
|
||||
}
|
||||
|
||||
export function supportsAudio(model: ModelInfo): boolean {
|
||||
return model.capabilities.audio;
|
||||
}
|
||||
|
||||
export function supportsDocuments(model: ModelInfo): boolean {
|
||||
return model.capabilities.documents ?? false;
|
||||
}
|
||||
|
||||
export function getImageSupportedModels(models: ModelInfo[]): ModelInfo[] {
|
||||
return models.filter(supportsImages);
|
||||
}
|
||||
|
||||
export function getVideoSupportedModels(models: ModelInfo[]): ModelInfo[] {
|
||||
return models.filter(supportsVideo);
|
||||
}
|
||||
|
||||
export function getAudioSupportedModels(models: ModelInfo[]): ModelInfo[] {
|
||||
return models.filter(supportsAudio);
|
||||
}
|
||||
|
||||
export function getDocumentSupportedModels(models: ModelInfo[]): ModelInfo[] {
|
||||
return models.filter(supportsDocuments);
|
||||
}
|
||||
|
||||
export function getReasoningModels(models: ModelInfo[]): ModelInfo[] {
|
||||
return models.filter(supportsReasoning);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,12 +25,15 @@ const reqBodySchema = z
|
|||
session_token: z.string(),
|
||||
conversation_id: z.string().optional(),
|
||||
web_search_enabled: z.boolean().optional(),
|
||||
images: z
|
||||
attachments: z
|
||||
.array(
|
||||
z.object({
|
||||
type: z.enum(['image', 'video', 'audio', 'document']),
|
||||
url: z.string(),
|
||||
storage_id: z.string(),
|
||||
fileName: z.string().optional(),
|
||||
fileName: z.string(),
|
||||
mimeType: z.string(),
|
||||
size: z.number(),
|
||||
})
|
||||
)
|
||||
.optional(),
|
||||
|
|
@ -353,18 +356,57 @@ async function generateAIResponse({
|
|||
log(`Background: ${attachedRules.length} rules attached`, startTime);
|
||||
|
||||
const formattedMessages = messages.map((m) => {
|
||||
if (m.images && m.images.length > 0 && m.role === 'user') {
|
||||
// Handle attachments format
|
||||
if (m.attachments && m.attachments.length > 0 && m.role === 'user') {
|
||||
const contentParts: Array<{
|
||||
type: string;
|
||||
text?: string;
|
||||
imageUrl?: string;
|
||||
videoUrl?: string;
|
||||
audioUrl?: string;
|
||||
documentUrl?: string;
|
||||
mimeType?: string;
|
||||
}> = [{ type: 'text', text: m.content }];
|
||||
|
||||
for (const attachment of m.attachments) {
|
||||
switch (attachment.type) {
|
||||
case 'image':
|
||||
contentParts.push({
|
||||
type: 'image',
|
||||
imageUrl: attachment.url,
|
||||
mimeType: attachment.mimeType,
|
||||
});
|
||||
break;
|
||||
case 'video':
|
||||
contentParts.push({
|
||||
type: 'video',
|
||||
videoUrl: attachment.url,
|
||||
mimeType: attachment.mimeType,
|
||||
});
|
||||
break;
|
||||
case 'audio':
|
||||
contentParts.push({
|
||||
type: 'audio',
|
||||
audioUrl: attachment.url,
|
||||
mimeType: attachment.mimeType,
|
||||
});
|
||||
break;
|
||||
case 'document':
|
||||
contentParts.push({
|
||||
type: 'document',
|
||||
documentUrl: attachment.url,
|
||||
mimeType: attachment.mimeType,
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
role: 'user' as const,
|
||||
content: [
|
||||
{ type: 'text', text: m.content },
|
||||
...m.images.map((img) => ({
|
||||
type: 'image_url',
|
||||
image_url: { url: img.url },
|
||||
})),
|
||||
],
|
||||
content: contentParts,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
role: m.role as 'user' | 'assistant' | 'system',
|
||||
content: m.content,
|
||||
|
|
@ -618,7 +660,7 @@ export const POST: RequestHandler = async ({ request }) => {
|
|||
content: args.message,
|
||||
content_html: '',
|
||||
role: 'user',
|
||||
images: args.images,
|
||||
attachments: args.attachments,
|
||||
web_search_enabled: args.web_search_enabled,
|
||||
session_token: sessionToken,
|
||||
}),
|
||||
|
|
@ -657,7 +699,7 @@ export const POST: RequestHandler = async ({ request }) => {
|
|||
model_id: args.model_id,
|
||||
reasoning_effort: args.reasoning_effort,
|
||||
role: 'user',
|
||||
images: args.images,
|
||||
attachments: args.attachments,
|
||||
web_search_enabled: args.web_search_enabled,
|
||||
}),
|
||||
(e) => `Failed to create user message: ${e}`
|
||||
|
|
|
|||
|
|
@ -20,7 +20,8 @@
|
|||
import { settings } from '$lib/state/settings.svelte.js';
|
||||
import { Provider } from '$lib/types';
|
||||
import { compressImage } from '$lib/utils/image-compression';
|
||||
import { supportsImages, supportsReasoning } from '$lib/utils/model-capabilities';
|
||||
import { supportsImages, supportsReasoning, supportsVideo, supportsAudio, supportsDocuments } from '$lib/utils/model-capabilities';
|
||||
import { AttachmentManager, type AttachmentType, type ProcessedAttachment } from '$lib/utils/attachment-manager';
|
||||
import { omit, pick } from '$lib/utils/object.js';
|
||||
import { cn } from '$lib/utils/utils.js';
|
||||
import { useConvexClient } from 'convex-svelte';
|
||||
|
|
@ -30,6 +31,10 @@
|
|||
import SendIcon from '~icons/lucide/arrow-up';
|
||||
import ChevronDownIcon from '~icons/lucide/chevron-down';
|
||||
import ImageIcon from '~icons/lucide/image';
|
||||
import VideoIcon from '~icons/lucide/video';
|
||||
import AudioIcon from '~icons/lucide/music';
|
||||
import FileTextIcon from '~icons/lucide/file-text';
|
||||
import PaperclipIcon from '~icons/lucide/paperclip';
|
||||
import PanelLeftIcon from '~icons/lucide/panel-left';
|
||||
import SearchIcon from '~icons/lucide/search';
|
||||
import Settings2Icon from '~icons/lucide/settings-2';
|
||||
|
|
@ -119,8 +124,8 @@
|
|||
|
||||
loading = true;
|
||||
|
||||
const imagesCopy = [...selectedImages];
|
||||
selectedImages = [];
|
||||
const attachmentsCopy = [...selectedAttachments];
|
||||
selectedAttachments = [];
|
||||
|
||||
try {
|
||||
const res = await callGenerateMessage({
|
||||
|
|
@ -128,7 +133,7 @@
|
|||
session_token: session.current?.session.token,
|
||||
conversation_id: page.params.id ?? undefined,
|
||||
model_id: settings.modelId,
|
||||
images: imagesCopy.length > 0 ? imagesCopy : undefined,
|
||||
attachments: attachmentsCopy.length > 0 ? attachmentsCopy : undefined,
|
||||
web_search_enabled: settings.webSearchEnabled,
|
||||
reasoning_effort: currentModelSupportsReasoning ? settings.reasoningEffort : undefined,
|
||||
});
|
||||
|
|
@ -195,7 +200,7 @@
|
|||
const autosize = new TextareaAutosize();
|
||||
|
||||
const message = new PersistedState('prompt', '');
|
||||
let selectedImages = $state<{ url: string; storage_id: string; fileName?: string }[]>([]);
|
||||
let selectedAttachments = $state<ProcessedAttachment[]>([]);
|
||||
let isUploading = $state(false);
|
||||
let fileInput = $state<HTMLInputElement>();
|
||||
let imageModal = $state<{ open: boolean; imageUrl: string; fileName: string }>({
|
||||
|
|
@ -211,11 +216,20 @@
|
|||
|
||||
models.init();
|
||||
|
||||
const currentModelSupportsImages = $derived.by(() => {
|
||||
if (!settings.modelId) return false;
|
||||
const currentModel = $derived.by(() => {
|
||||
if (!settings.modelId) return null;
|
||||
const allModels = models.all();
|
||||
const currentModel = allModels.find((m) => m.id === settings.modelId);
|
||||
return currentModel ? supportsImages(currentModel) : false;
|
||||
return allModels.find((m) => m.id === settings.modelId) || null;
|
||||
});
|
||||
|
||||
const currentModelSupportsImages = $derived(currentModel ? supportsImages(currentModel) : false);
|
||||
const currentModelSupportsVideo = $derived(currentModel ? supportsVideo(currentModel) : false);
|
||||
const currentModelSupportsAudio = $derived(currentModel ? supportsAudio(currentModel) : false);
|
||||
const currentModelSupportsDocuments = $derived(currentModel ? supportsDocuments(currentModel) : false);
|
||||
|
||||
const supportedAttachmentTypes = $derived.by(() => {
|
||||
if (!currentModel) return [];
|
||||
return AttachmentManager.getSupportedAttachmentTypes(currentModel);
|
||||
});
|
||||
|
||||
const currentModelSupportsReasoning = $derived.by(() => {
|
||||
|
|
@ -228,36 +242,49 @@
|
|||
|
||||
const fileUpload = new FileUpload({
|
||||
multiple: true,
|
||||
accept: 'image/*',
|
||||
maxSize: 10 * 1024 * 1024, // 10MB
|
||||
maxSize: 100 * 1024 * 1024, // 100MB max for any file type
|
||||
});
|
||||
|
||||
// Update the file input accept attribute reactively
|
||||
$effect(() => {
|
||||
if (fileInput) {
|
||||
const acceptString = AttachmentManager.getAcceptString(supportedAttachmentTypes);
|
||||
fileInput.accept = acceptString;
|
||||
}
|
||||
});
|
||||
|
||||
async function handleFileChange(files: File[]) {
|
||||
if (!files.length || !session.current?.session.token) return;
|
||||
if (!files.length || !session.current?.session.token || !currentModel) return;
|
||||
|
||||
isUploading = true;
|
||||
const uploadedFiles: { url: string; storage_id: string; fileName?: string }[] = [];
|
||||
const uploadedFiles: ProcessedAttachment[] = [];
|
||||
|
||||
try {
|
||||
for (const file of files) {
|
||||
// Skip non-image files
|
||||
if (!file.type.startsWith('image/')) {
|
||||
console.warn('Skipping non-image file:', file.name);
|
||||
// Validate file against current model capabilities
|
||||
const validation = AttachmentManager.validateFile(file, currentModel);
|
||||
if (!validation.valid) {
|
||||
console.warn(`Skipping invalid file ${file.name}: ${validation.error}`);
|
||||
// TODO: Show error toast to user
|
||||
continue;
|
||||
}
|
||||
|
||||
// Compress image to max 1MB
|
||||
const compressedFile = await compressImage(file, 1024 * 1024);
|
||||
let fileToUpload = file;
|
||||
|
||||
// Compress images to max 1MB for better performance
|
||||
if (validation.type === 'image') {
|
||||
fileToUpload = await compressImage(file, 1024 * 1024);
|
||||
}
|
||||
|
||||
// Generate upload URL
|
||||
const uploadUrl = await client.mutation(api.storage.generateUploadUrl, {
|
||||
session_token: session.current.session.token,
|
||||
});
|
||||
|
||||
// Upload compressed file
|
||||
// Upload file
|
||||
const result = await fetch(uploadUrl, {
|
||||
method: 'POST',
|
||||
body: compressedFile,
|
||||
body: fileToUpload,
|
||||
});
|
||||
|
||||
if (!result.ok) {
|
||||
|
|
@ -273,11 +300,18 @@
|
|||
});
|
||||
|
||||
if (url) {
|
||||
uploadedFiles.push({ url, storage_id: storageId, fileName: file.name });
|
||||
uploadedFiles.push({
|
||||
type: validation.type!,
|
||||
url,
|
||||
storage_id: storageId,
|
||||
fileName: file.name,
|
||||
mimeType: file.type,
|
||||
size: file.size
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
selectedImages = [...selectedImages, ...uploadedFiles];
|
||||
selectedAttachments = [...selectedAttachments, ...uploadedFiles];
|
||||
} catch (error) {
|
||||
console.error('Upload failed:', error);
|
||||
} finally {
|
||||
|
|
@ -285,8 +319,31 @@
|
|||
}
|
||||
}
|
||||
|
||||
function removeImage(index: number) {
|
||||
selectedImages = selectedImages.filter((_, i) => i !== index);
|
||||
function removeAttachment(index: number) {
|
||||
selectedAttachments = selectedAttachments.filter((_, i) => i !== index);
|
||||
}
|
||||
|
||||
function getRequiredCapabilities(attachments: ProcessedAttachment[]): Array<'vision' | 'audio' | 'video' | 'documents'> {
|
||||
const capabilities: Array<'vision' | 'audio' | 'video' | 'documents'> = [];
|
||||
|
||||
attachments.forEach(attachment => {
|
||||
switch (attachment.type) {
|
||||
case 'image':
|
||||
if (!capabilities.includes('vision')) capabilities.push('vision');
|
||||
break;
|
||||
case 'video':
|
||||
if (!capabilities.includes('video')) capabilities.push('video');
|
||||
break;
|
||||
case 'audio':
|
||||
if (!capabilities.includes('audio')) capabilities.push('audio');
|
||||
break;
|
||||
case 'document':
|
||||
if (!capabilities.includes('documents')) capabilities.push('documents');
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
return capabilities;
|
||||
}
|
||||
|
||||
function openImageModal(imageUrl: string, fileName: string) {
|
||||
|
|
@ -446,7 +503,7 @@
|
|||
<Sidebar.Root
|
||||
bind:open={sidebarOpen}
|
||||
class="fill-device-height overflow-clip"
|
||||
{...currentModelSupportsImages ? omit(fileUpload.dropzone, ['onclick']) : {}}
|
||||
{...supportedAttachmentTypes.length > 0 ? omit(fileUpload.dropzone, ['onclick']) : {}}
|
||||
>
|
||||
<AppSidebar bind:searchModalOpen />
|
||||
|
||||
|
|
@ -608,26 +665,45 @@
|
|||
</div>
|
||||
{/if}
|
||||
<div class="flex flex-grow flex-col">
|
||||
{#if selectedImages.length > 0}
|
||||
{#if selectedAttachments.length > 0}
|
||||
<div class="mb-2 flex flex-wrap gap-2">
|
||||
{#each selectedImages as image, index (image.storage_id)}
|
||||
{#each selectedAttachments as attachment, index (attachment.storage_id)}
|
||||
<div
|
||||
class="group border-secondary-foreground/[0.08] bg-secondary-foreground/[0.02] hover:bg-secondary-foreground/10 relative flex h-12 w-12 max-w-full shrink-0 items-center justify-center gap-2 rounded-xl border border-solid p-0 transition-[width,height] duration-500"
|
||||
class="group border-secondary-foreground/[0.08] bg-secondary-foreground/[0.02] hover:bg-secondary-foreground/10 relative flex h-12 max-w-full shrink-0 items-center justify-center gap-2 rounded-xl border border-solid p-2 transition-[width,height] duration-500"
|
||||
class:w-12={attachment.type === 'image'}
|
||||
class:w-auto={attachment.type !== 'image'}
|
||||
>
|
||||
{#if attachment.type === 'image'}
|
||||
<button
|
||||
type="button"
|
||||
onclick={() => openImageModal(image.url, image.fileName || 'image')}
|
||||
onclick={() => openImageModal(attachment.url, attachment.fileName)}
|
||||
class="rounded-lg"
|
||||
>
|
||||
<img
|
||||
src={image.url}
|
||||
src={attachment.url}
|
||||
alt="Uploaded"
|
||||
class="size-10 rounded-lg object-cover opacity-100 transition-opacity"
|
||||
class="size-8 rounded-lg object-cover opacity-100 transition-opacity"
|
||||
/>
|
||||
</button>
|
||||
{:else if attachment.type === 'video'}
|
||||
<div class="flex items-center gap-2">
|
||||
<VideoIcon class="size-4 text-blue-500" />
|
||||
<span class="text-xs text-muted-foreground truncate max-w-20">{attachment.fileName}</span>
|
||||
</div>
|
||||
{:else if attachment.type === 'audio'}
|
||||
<div class="flex items-center gap-2">
|
||||
<AudioIcon class="size-4 text-green-500" />
|
||||
<span class="text-xs text-muted-foreground truncate max-w-20">{attachment.fileName}</span>
|
||||
</div>
|
||||
{:else if attachment.type === 'document'}
|
||||
<div class="flex items-center gap-2">
|
||||
<FileTextIcon class="size-4 text-orange-500" />
|
||||
<span class="text-xs text-muted-foreground truncate max-w-20">{attachment.fileName}</span>
|
||||
</div>
|
||||
{/if}
|
||||
<button
|
||||
type="button"
|
||||
onclick={() => removeImage(index)}
|
||||
onclick={() => removeAttachment(index)}
|
||||
class="bg-secondary hover:bg-muted absolute -top-1 -right-1 cursor-pointer rounded-full p-1 opacity-0 transition group-hover:opacity-100"
|
||||
>
|
||||
<XIcon class="h-3 w-3" />
|
||||
|
|
@ -707,7 +783,7 @@
|
|||
</Tooltip>
|
||||
</div>
|
||||
<div class="flex flex-wrap items-center gap-2 pr-2">
|
||||
<ModelPicker onlyImageModels={selectedImages.length > 0} />
|
||||
<ModelPicker requiredCapabilities={selectedAttachments.length > 0 ? getRequiredCapabilities(selectedAttachments) : []} />
|
||||
<div class="flex items-center gap-2">
|
||||
{#if currentModelSupportsReasoning}
|
||||
<DropdownMenu.Root>
|
||||
|
|
@ -746,7 +822,7 @@
|
|||
<SearchIcon class="!size-3" />
|
||||
<span class="hidden whitespace-nowrap sm:inline">Web search</span>
|
||||
</button>
|
||||
{#if currentModelSupportsImages}
|
||||
{#if supportedAttachmentTypes.length > 0}
|
||||
<button
|
||||
type="button"
|
||||
class="border-border hover:bg-accent/20 flex items-center gap-1 rounded-full border px-1 py-1 text-xs transition-colors disabled:opacity-50 sm:px-2"
|
||||
|
|
@ -758,9 +834,15 @@
|
|||
class="size-3 animate-spin rounded-full border-2 border-current border-t-transparent"
|
||||
></div>
|
||||
{:else}
|
||||
<ImageIcon class="!size-3" />
|
||||
<PaperclipIcon class="!size-3" />
|
||||
{/if}
|
||||
<span class="hidden whitespace-nowrap sm:inline">Attach image</span>
|
||||
<span class="hidden whitespace-nowrap sm:inline">
|
||||
{#if supportedAttachmentTypes.length === 1 && supportedAttachmentTypes[0]}
|
||||
Attach {AttachmentManager.getTypeDisplayName(supportedAttachmentTypes[0]).toLowerCase()}
|
||||
{:else}
|
||||
Attach files
|
||||
{/if}
|
||||
</span>
|
||||
</button>
|
||||
{/if}
|
||||
{#if session.current !== null && message.current.trim() !== ''}
|
||||
|
|
@ -811,12 +893,20 @@
|
|||
</div>
|
||||
</Sidebar.Inset>
|
||||
|
||||
{#if fileUpload.isDragging && currentModelSupportsImages}
|
||||
{#if fileUpload.isDragging && supportedAttachmentTypes.length > 0}
|
||||
<div class="fixed inset-0 z-50 flex items-center justify-center bg-black/10 backdrop-blur-sm">
|
||||
<div class="text-center">
|
||||
<UploadIcon class="text-primary mx-auto mb-4 h-16 w-16" />
|
||||
<p class="text-xl font-semibold">Add image</p>
|
||||
<p class="mt-2 text-sm opacity-75">Drop an image here to attach it to your message.</p>
|
||||
<p class="text-xl font-semibold">
|
||||
{#if supportedAttachmentTypes.length === 1 && supportedAttachmentTypes[0]}
|
||||
Add {AttachmentManager.getTypeDisplayName(supportedAttachmentTypes[0]).toLowerCase()}
|
||||
{:else}
|
||||
Add files
|
||||
{/if}
|
||||
</p>
|
||||
<p class="mt-2 text-sm opacity-75">
|
||||
Drop {supportedAttachmentTypes.length === 1 ? `${supportedAttachmentTypes[0]}s` : 'supported files'} here to attach to your message.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue