Merge pull request #465 from SuanmoSuanyangTechnology/feature/model_zy

Feature/model zy

web/src/components/AudioRecorder/index.tsx

@@ -1,16 +1,21 @@
 import { type FC, useRef, useState } from 'react'
 import RecordRTC from 'recordrtc'
 
-import { fileUpload } from '@/api/fileStorage'
+import { fileUploadUrlWithoutApiPrefix } from '@/api/fileStorage'
+import { request } from '@/utils/request'
 
 interface AudioRecorderProps {
-  onRecordingComplete?: (file: { file_id: string; file_key: string; }, blob: Blob) => void
+  onRecordingComplete?: (file: { file_id: string; file_key: string; url: string; type?: string; }, blob?: Blob) => void
-  className?: string
+  className?: string;
+  action?: string;
+  requestConfig?: Record<string, any>;
 }
 
 const AudioRecorder: FC<AudioRecorderProps> = ({
   onRecordingComplete,
   className = '',
+  action = fileUploadUrlWithoutApiPrefix,
+  requestConfig = {}
 }) => {
   const [isRecording, setIsRecording] = useState(false)
   const recorderRef = useRef<RecordRTC | null>(null)
@@ -33,11 +38,17 @@ const AudioRecorder: FC<AudioRecorderProps> = ({
     if (recorderRef.current) {
       recorderRef.current.stopRecording(() => {
         const blob = recorderRef.current!.getBlob()
+        const url = recorderRef.current!.toURL()
         const formData = new FormData()
         formData.append('file', blob, `recording_${Date.now()}.webm`)
-        fileUpload(formData)
+        request
+          .uploadFile(action, formData, requestConfig)
           .then(res => {
-            onRecordingComplete?.(res as { file_id: string; file_key: string; }, blob)
+            onRecordingComplete?.({
+              ...(res as { file_id: string; file_key: string }),
+              type: blob.type,
+              url
+            }, blob)
             recorderRef.current?.destroy()
             recorderRef.current = null
           })

web/src/components/Chat/ChatInput.tsx

@@ -2,10 +2,11 @@
  * @Author: ZhaoYing 
  * @Date: 2025-12-10 16:46:14 
  * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-10 12:13:52
+ * @Last Modified time: 2026-03-04 18:42:49
  */
 import { type FC, useEffect, useMemo } from 'react'
 import { Flex, Input, Form } from 'antd'
+
 import SendIcon from '@/assets/images/conversation/send.svg'
 import SendDisabledIcon from '@/assets/images/conversation/sendDisabled.svg'
 import LoadingIcon from '@/assets/images/conversation/loading.svg'
@@ -80,9 +81,31 @@ const ChatInput: FC<ChatInputProps> = ({
                 </div>
               )
             }
+            if (file.type.includes('video')) {
+              return (
+                <div key={file.uid} className="rb:w-45 rb:h-16 rb:inline-block rb:group rb:relative rb:rounded-lg">
+                  <video src={file.url} controls className="rb:w-45 rb:h-16 rb:rounded-lg rb:object-cover rb:cursor-pointer" />
+                  <div
+                    className="rb:hidden rb:group-hover:block rb:absolute rb:-right-1 rb:-top-1 rb:size-3.5 rb:cursor-pointer rb:bg-cover rb:bg-[url('@/assets/images/conversation/delete.svg')] rb:hover:bg-[url('@/assets/images/conversation/delete_hover.svg')]"
+                    onClick={() => handleDelete(file)}
+                  ></div>
+                </div>
+              )
+            }
+            if (file.type.includes('audio')) {
+              return (
+                <div key={file.uid} className="rb:w-45 rb:h-16 rb:inline-flex rb:items-center rb:group rb:relative rb:rounded-lg rb:bg-[#F0F3F8] rb:py-2 rb:px-2.5 rb:gap-2">
+                  <audio src={file.url} controls className="rb:w-45 rb:h-16" />
+                  <div
+                    className="rb:hidden rb:group-hover:block rb:absolute rb:-right-1 rb:-top-1 rb:size-3.5 rb:cursor-pointer rb:bg-cover rb:bg-[url('@/assets/images/conversation/delete.svg')] rb:hover:bg-[url('@/assets/images/conversation/delete_hover.svg')]"
+                    onClick={() => handleDelete(file)}
+                  ></div>
+                </div>
+              )
+            }
             return (
               <div key={file.uid} className="rb:w-45 rb:text-[12px] rb:gap-2.5 rb:flex rb:items-center rb:group rb:relative rb:rounded-lg rb:bg-[#F0F3F8] rb:py-2 rb:px-2.5">
-                {(file.type.includes('word') || file.type.includes('wordprocessingml.document')) && <div
+                {(file.type.includes('doc') || file.type.includes('docx') || file.type.includes('word') || file.type.includes('wordprocessingml.document')) && <div
                   className="rb:size-5 rb:cursor-pointer rb:bg-cover rb:bg-[url('@/assets/images/conversation/word_disabled.svg')] rb:hover:bg-[url('@/assets/images/conversation/word.svg')]"
                 ></div>}
                 {(file.type.includes('pdf')) && <div

web/src/i18n/en.ts

@@ -605,7 +605,13 @@ export const en = {
       ollama: "Ollama",
       xinference: "Xinference",
       gpustack: "Gpustack",
-      bedrock: "Bedrock"
+      bedrock: "Bedrock",
+
+      is_vision: 'Vision Support',
+      is_omni: 'Omni Support',
+      vision: 'Vision',
+      audio: 'Audio',
+      video: 'Video',
     },
     knowledgeBase: {
       home: 'Home',
@@ -1686,6 +1692,8 @@ Memory Bear: After the rebellion, regional warlordism intensified for several re
       uploadFile: 'Upload File',
       fileType: 'File Type',
       image: 'Image',
+      video: 'Video',
+      audio: 'Audio',
       fileUrl: 'File URL',
       addRemoteFile: 'Add Remote File',
       variableConfig: 'Variable Configuration',

web/src/i18n/zh.ts

@@ -1186,6 +1186,12 @@ export const zh = {
       xinference: "Xinference",
       gpustack: "Gpustack",
       bedrock: "Bedrock",
+
+      is_vision: '支持视觉',
+      is_omni: '支持全模态',
+      vision: '视觉',
+      audio: '音频',
+      video: '视频',
     },
     timezones: {
       'Asia/Shanghai': '中国标准时间 (UTC+8)',
@@ -1683,6 +1689,8 @@ export const zh = {
       uploadFile: '上传文件',
       fileType: '文件类型',
       image: '图片',
+      video: '视频',
+      audio: '音频',
       fileUrl: '文件链接',
       addRemoteFile: '添加远程文件',
       variableConfig: '变量配置',

web/src/utils/stream.ts

@@ -1,8 +1,8 @@
 /*
  * @Author: ZhaoYing 
  * @Date: 2026-02-02 16:35:43 
- * @Last Modified by:   ZhaoYing 
+ * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-02 16:35:43 
+ * @Last Modified time: 2026-03-04 18:19:24
  */
 /**
  * Server-Sent Events (SSE) Stream Utility Module
@@ -176,17 +176,17 @@ export const handleSSE = async (url: string, data: any, onMessage?: (data: SSEMe
       case 500:
       case 502:
         const errorData = await response.json();
-        errorData.error || i18n.t('common.serviceUpgrading');
+        let errorInfo = errorData.error || i18n.t('common.serviceUpgrading')
-        message.warning(errorData.error || i18n.t('common.serviceUpgrading'));
+        message.warning(errorInfo);
-        return;
+        throw errorInfo;
       case 400:
         const error = await response.json();
         message.warning(error.error);
-        throw error || 'Bad Request';
+        throw error.error || 'Bad Request';
       case 504:
         const errorJson = await response.json();
         message.warning(errorJson.error || i18n.t('common.serverError'));
-        return;
+        throw errorData.error;
       case 401:
         if (url?.includes('/public')) {
           return message.warning(i18n.t('common.publicApiCannotRefreshToken'));

web/src/views/ApplicationConfig/components/Chat.tsx

@@ -2,7 +2,7 @@
  * @Author: ZhaoYing 
  * @Date: 2026-02-03 16:27:39 
  * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-03-04 18:05:36
+ * @Last Modified time: 2026-03-04 18:51:20
  */
 /**
  * Chat debugging component for application testing
@@ -13,7 +13,7 @@
 import { type FC, useEffect, useState, useRef } from 'react';
 import { useTranslation } from 'react-i18next';
 import clsx from 'clsx'
-import { Flex, Dropdown, type MenuProps, App } from 'antd'
+import { Flex, Dropdown, type MenuProps, App, Divider } from 'antd'
 
 import ChatIcon from '@/assets/images/application/chat.png'
 import DebuggingEmpty from '@/assets/images/application/debuggingEmpty.png'
@@ -25,7 +25,7 @@ import type { ChatItem } from '@/components/Chat/types'
 import { type SSEMessage } from '@/utils/stream'
 import ChatInput from '@/components/Chat/ChatInput'
 import UploadFiles from '@/views/Conversation/components/FileUpload'
-// import AudioRecorder from '@/components/AudioRecorder'
+import AudioRecorder from '@/components/AudioRecorder'
 import UploadFileListModal from '@/views/Conversation/components/UploadFileListModal'
 import type { UploadFileListModalRef } from '@/views/Conversation/types'
 import type { Variable } from './VariableList/types'
@@ -88,7 +88,7 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
       content: '',
       created_at: Date.now(),
     };
-    
+
     if (isCluster) {
       updateChatList(prev => prev.map(item => ({
         ...item,
@@ -134,7 +134,7 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
     })
   }
   /** Update assistant message when error occurs */
-  const updateErrorAssistantMessage  = (message_length: number, model_config_id?: string) => {
+  const updateErrorAssistantMessage = (message_length: number, model_config_id?: string) => {
     if (message_length > 0 || !model_config_id) return
 
     updateChatList(prev => {
@@ -245,7 +245,15 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
             "stream": true,
             "timeout": 60,
           }, handleStreamMessage)
-            .finally(() => setLoading(false));
+            .catch(() => {
+              setLoading(false)
+              setCompareLoading(false)
+              updateClusterErrorAssistantMessage(0)
+            })
+            .finally(() => {
+              setLoading(false)
+              setCompareLoading(false)
+            })
         }, 0)
       })
       .catch(() => {
@@ -290,7 +298,7 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
     })
   }
   /** Update cluster message when error occurs */
-  const updateClusterErrorAssistantMessage  = (message_length: number) => {
+  const updateClusterErrorAssistantMessage = (message_length: number) => {
     if (message_length > 0) return
 
     updateChatList(prev => {
@@ -333,7 +341,7 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
           data.map(item => {
             const { conversation_id, content, message_length } = item.data as { conversation_id: string, content: string, message_length: number };
 
-            switch(item.event) {
+            switch (item.event) {
               case 'start':
                 if (conversation_id && conversationId !== conversation_id) {
                   setConversationId(conversation_id);
@@ -356,27 +364,35 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
         };
 
         setTimeout(() => {
-            draftRun(
+          draftRun(
-              data.app_id,
+            data.app_id,
-              { 
+            {
-                message, 
+              message,
-                conversation_id: conversationId, 
+              conversation_id: conversationId,
-                stream: true,
+              stream: true,
-                files: fileList.map(file => {
+              files: fileList.map(file => {
-                  if (file.url) {
+                if (file.url) {
-                    return file
+                  return file
-                  } else {
+                } else {
-                    return {
+                  return {
-                      type: file.type,
+                    type: file.type,
-                      transfer_method: 'local_file',
+                    transfer_method: 'local_file',
-                      upload_file_id: file.response.data.file_id
+                    upload_file_id: file.response.data.file_id
-                    }
                   }
-                }),
+                }
-              }, 
+              }),
-              handleStreamMessage
+            },
-            )
+            handleStreamMessage
-              .finally(() => setLoading(false))
+          )
+            .catch(() => {
+              setLoading(false)
+              setCompareLoading(false)
+              updateClusterErrorAssistantMessage(0)
+            })
+            .finally(() => {
+              setLoading(false)
+              setCompareLoading(false)
+            })
         }, 0)
       })
       .catch(() => {
@@ -395,12 +411,17 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
   const fileChange = (file?: any) => {
     setFileList([...fileList, file])
   }
-  // const handleRecordingComplete = async (file: any) => {
+  const handleRecordingComplete = async (file: any) => {
-  //   console.log('file', file)
+    setFileList([...fileList, {
-  // }
+      uid: file.file_id,
+      response: { data: file },
+      thumbUrl: file.url,
+      type: file.type
+    }])
+  }
 
   const handleShowUpload: MenuProps['onClick'] = ({ key }) => {
-    switch(key) {
+    switch (key) {
       case 'define':
         uploadFileListModalRef.current?.handleOpen()
         break
@@ -417,99 +438,98 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
   return (
     <div className="rb:relative rb:h-full rb:flex rb:flex-col">
       {chatList.length === 0
-        ? <Empty 
+        ? <Empty
-          url={DebuggingEmpty} 
+          url={DebuggingEmpty}
           size={[300, 200]}
-          title={t('application.debuggingEmpty')} 
+          title={t('application.debuggingEmpty')}
-          subTitle={t('application.debuggingEmptyDesc')} 
+          subTitle={t('application.debuggingEmptyDesc')}
           className="rb:h-full"
         />
-      : <>
+        : <>
-        <div className={clsx(`rb:relative rb:grid rb:grid-cols-${chatList.length} rb:overflow-hidden rb:w-full rb:flex-1 rb:min-h-0`)}>
+          <div className={clsx(`rb:relative rb:grid rb:grid-cols-${chatList.length} rb:overflow-hidden rb:w-full rb:flex-1 rb:min-h-0`)}>
-          {chatList.map((chat, index) => (
+            {chatList.map((chat, index) => (
-            <div key={index} className={clsx('rb:flex rb:flex-col', {
+              <div key={index} className={clsx('rb:flex rb:flex-col', {
-              "rb:border-r rb:border-[#DFE4ED]": index !== chatList.length - 1 && chatList.length > 1,
+                "rb:border-r rb:border-[#DFE4ED]": index !== chatList.length - 1 && chatList.length > 1,
-            })}>
+              })}>
-              {chat.label &&
+                {chat.label &&
-                <div className={clsx(
+                  <div className={clsx(
-                  "rb:grid rb:bg-[#F0F3F8] rb:text-center rb:flex-[0_0_auto]",
+                    "rb:grid rb:bg-[#F0F3F8] rb:text-center rb:flex-[0_0_auto]",
-                  {
+                    {
-                    'rb:rounded-tr-xl': index === chatList.length - 1,
+                      'rb:rounded-tr-xl': index === chatList.length - 1,
-                    'rb:rounded-tl-xl': index === 0,
+                      'rb:rounded-tl-xl': index === 0,
-                  }
+                    }
-                )}>
+                  )}>
-                  <div className='rb:relative rb:p-[10px_12px] rb:overflow-hidden'>
+                    <div className='rb:relative rb:p-[10px_12px] rb:overflow-hidden'>
-                    <div className="rb:text-ellipsis rb:overflow-hidden rb:whitespace-nowrap rb:w-[calc(100%-24px)]">{chat.label}</div>
+                      <div className="rb:text-ellipsis rb:overflow-hidden rb:whitespace-nowrap rb:w-[calc(100%-24px)]">{chat.label}</div>
-                    <div 
+                      <div
-                      className="rb:w-4 rb:h-4 rb:cursor-pointer rb:absolute rb:top-3 rb:right-3 rb:bg-cover rb:bg-[url('@/assets/images/close.svg')] rb:hover:bg-[url('@/assets/images/close_hover.svg')]" 
+                        className="rb:w-4 rb:h-4 rb:cursor-pointer rb:absolute rb:top-3 rb:right-3 rb:bg-cover rb:bg-[url('@/assets/images/close.svg')] rb:hover:bg-[url('@/assets/images/close_hover.svg')]"
-                      onClick={() => handleDelete(index)}
+                        onClick={() => handleDelete(index)}
-                    ></div>
+                      ></div>
+                    </div>
                   </div>
-                </div>
+                }
-              }
+                <ChatContent
-              <ChatContent
+                  classNames={{
-                classNames={{
+                    'rb:mx-[16px] rb:mt-6': true,
-                  'rb:mx-[16px] rb:mt-6': true,
+                    'rb:h-[calc(100vh-282px)]': isCluster,
-                  'rb:h-[calc(100vh-282px)]': isCluster,
+                    'rb:h-[calc(100vh-380px)]': !isCluster,
-                  'rb:h-[calc(100vh-380px)]': !isCluster,
-                }} 
-                contentClassNames={{
-                  'rb:max-w-[400px]!': chatList.length === 1,
-                  'rb:max-w-[260px]!': chatList.length === 2,
-                  'rb:max-w-[150px]!': chatList.length === 3,
-                  'rb:max-w-[108px]!': chatList.length === 4,
-                }}
-                empty={<Empty url={ChatIcon} title={t('application.chatEmpty')} isNeedSubTitle={false} size={[240, 200]} className="rb:h-full" />}
-                data={chat.list || []}
-                streamLoading={compareLoading}
-                labelPosition="top"
-                labelFormat={(item) => item.role === 'user' ? t('application.you') : chat.label}
-                errorDesc={t('application.ReplyException')}
-              />
-            </div>
-          ))}
-        </div>
-        <div className="rb:relative rb:flex rb:items-center rb:gap-2.5 rb:m-4 rb:mb-1">
-          <ChatInput
-            message={message}
-            className="rb:relative!"
-            loading={loading}
-            fileChange={updateFileList}
-            fileList={fileList}
-            onSend={isCluster ? handleClusterSend : handleSend}
-            onChange={handleMessageChange}
-          >
-            <Flex justify="space-between" className="rb:flex-1">
-              <Flex gap={8} align="center">
-                <Dropdown
-                  menu={{
-                    items: [
-                      { key: 'define', label: t('memoryConversation.addRemoteFile') },
-                      {
-                        key: 'upload', label: (
-                          <UploadFiles
-                            fileType={['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'svg']}
-                            onChange={fileChange}
-                          />
-                        )
-                      },
-                    ],
-                    onClick: handleShowUpload
                   }}
-                >
+                  contentClassNames={{
-                  <div
+                    'rb:max-w-[400px]!': chatList.length === 1,
-                    className="rb:size-6 rb:cursor-pointer rb:bg-cover rb:bg-[url('@/assets/images/conversation/link.svg')] rb:hover:bg-[url('@/assets/images/conversation/link_hover.svg')]"
+                    'rb:max-w-[260px]!': chatList.length === 2,
-                  ></div>
+                    'rb:max-w-[150px]!': chatList.length === 3,
-                </Dropdown>
+                    'rb:max-w-[108px]!': chatList.length === 4,
+                  }}
+                  empty={<Empty url={ChatIcon} title={t('application.chatEmpty')} isNeedSubTitle={false} size={[240, 200]} className="rb:h-full" />}
+                  data={chat.list || []}
+                  streamLoading={compareLoading}
+                  labelPosition="top"
+                  labelFormat={(item) => item.role === 'user' ? t('application.you') : chat.label}
+                  errorDesc={t('application.ReplyException')}
+                />
+              </div>
+            ))}
+          </div>
+          <div className="rb:relative rb:flex rb:items-center rb:gap-2.5 rb:m-4 rb:mb-1">
+            <ChatInput
+              message={message}
+              className="rb:relative!"
+              loading={loading}
+              fileChange={updateFileList}
+              fileList={fileList}
+              onSend={isCluster ? handleClusterSend : handleSend}
+              onChange={handleMessageChange}
+            >
+              <Flex justify="space-between" className="rb:flex-1">
+                <Flex gap={8} align="center">
+                  <Dropdown
+                    menu={{
+                      items: [
+                        { key: 'define', label: t('memoryConversation.addRemoteFile') },
+                        {
+                          key: 'upload', label: (
+                            <UploadFiles
+                              onChange={fileChange}
+                            />
+                          )
+                        },
+                      ],
+                      onClick: handleShowUpload
+                    }}
+                  >
+                    <div
+                      className="rb:size-6 rb:cursor-pointer rb:bg-cover rb:bg-[url('@/assets/images/conversation/link.svg')] rb:hover:bg-[url('@/assets/images/conversation/link_hover.svg')]"
+                    ></div>
+                  </Dropdown>
+                </Flex>
+                <Flex align="center">
+                  <AudioRecorder onRecordingComplete={handleRecordingComplete} />
+                  <Divider type="vertical" className="rb:ml-1.5! rb:mr-3!" />
+                </Flex>
               </Flex>
-              {/* <Flex align="center">
+            </ChatInput>
-                <AudioRecorder onRecordingComplete={handleRecordingComplete} />
+          </div>
-                <Divider type="vertical" className="rb:ml-1.5! rb:mr-3!" />
+        </>
-              </Flex> */}
-            </Flex>
-          </ChatInput>
-        </div>
-      </>
       }
 
       <UploadFileListModal

web/src/views/Conversation/components/FileUpload.tsx

@@ -2,7 +2,7 @@
  * @Author: ZhaoYing 
  * @Date: 2026-02-06 21:09:42 
  * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-11 11:32:48
+ * @Last Modified time: 2026-03-04 18:54:47
  */
 /**
  * File Upload Component
@@ -25,6 +25,7 @@ import { Upload, Progress, App } from 'antd';
 import type { UploadProps, UploadFile } from 'antd';
 import type { UploadProps as RcUploadProps } from 'antd/es/upload/interface';
 import { useTranslation } from 'react-i18next';
+
 import { request } from '@/utils/request'
 import { fileUploadUrlWithoutApiPrefix } from '@/api/fileStorage'
 
@@ -56,27 +57,36 @@ interface UploadFilesProps extends Omit<UploadProps, 'onChange'> {
   /** Custom file removal callback */
   onRemove?: (file: UploadFile) => boolean | void | Promise<boolean | void>;
 }
+
+const transform_file_type = {
+  'text/plain': 'document/text',
+  'text/markdown': 'document/markdown',
+  'text/x-markdown': 'document/x-markdown',
+
+  'application/pdf': 'document/pdf',
+
+  'application/msword': 'document/doc',
+  'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'document/docx',
+
+  'application/vnd.ms-powerpoint': 'document/ppt',
+  'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'document/pptx',
+}
 // Mapping of file extensions to MIME types
 const ALL_FILE_TYPE: {
   [key: string]: string;
 } = {
-  // txt: 'text/plain',
+  txt: 'text/plain',
+  md: 'text/markdown',
+  xmd: 'text/x-markdown',
+
   pdf: 'application/pdf',
 
   doc: 'application/msword',
   docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
-  
-  xls: 'application/vnd.ms-excel',
-  xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
-  csv: 'text/csv',
 
   ppt: 'application/vnd.ms-powerpoint',
   pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
-  
+
-  // md: 'text/markdown',
-  // htm: 'text/html',
-  // html: 'text/html',
-  // json: 'application/json',
   jpg: 'image/jpeg',
   jpeg: 'image/jpeg',
   png: 'image/png',
@@ -84,6 +94,23 @@ const ALL_FILE_TYPE: {
   bmp: 'image/bmp',
   webp: 'image/webp',
   svg: 'image/svg+xml',
+
+  mp4: 'video/mp4',
+  mov: 'video/quicktime',
+  avi: 'video/x-msvideo',
+  mkv: 'video/x-matroska',
+  webm: 'video/webm',
+  flv: 'video/x-flv',
+  wmv: 'video/x-ms-wmv',
+
+  mp3: 'audio/mpeg',
+  wav: 'audio/wav',
+  ogg: 'audio/ogg',
+  aac: 'audio/aac',
+  flac: 'audio/flac',
+  m4a: 'audio/mp4',
+  wma: 'audio/x-ms-wma',
+  xm4a: 'audio/x-m4a',
 }
 export interface UploadFilesRef {
   /** Current file list */
@@ -178,6 +205,10 @@ const UploadFiles = forwardRef<UploadFilesRef, UploadFilesProps>(({
    * Handles upload state changes
    */
   const handleChange: UploadProps['onChange'] = ({ fileList: newFileList }) => {
+    newFileList.map(file => {
+      const type = (file.type && transform_file_type[file.type as keyof typeof transform_file_type]) || file.type
+      file.type = type
+    })
     setFileList(newFileList);
     if (onChange) {
       onChange(maxCount === 1 ? newFileList[newFileList.length - 1] : newFileList);

web/src/views/Conversation/components/UploadFileListModal.tsx

@@ -2,7 +2,7 @@
  * @Author: ZhaoYing 
  * @Date: 2026-02-06 21:09:47 
  * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-09 10:17:54
+ * @Last Modified time: 2026-03-04 17:47:09
  */
 /**
  * Upload File List Modal Component
@@ -104,7 +104,9 @@ const UploadFileListModal = forwardRef<UploadFileListModalRef, UploadFileListMod
                     <Select
                       placeholder={t('memoryConversation.fileType')}
                       options={[
-                        { label: t('memoryConversation.image'), value: 'image' }
+                        { label: t('memoryConversation.image'), value: 'image' },
+                        { label: t('memoryConversation.audio'), value: 'audio' },
+                        { label: t('memoryConversation.video'), value: 'video' },
                       ]}
                       className="rb:w-30"
                     />

web/src/views/Conversation/index.tsx

@@ -14,7 +14,7 @@ import { type FC, useState, useEffect, useRef } from 'react'
 import { useParams, useLocation } from 'react-router-dom'
 import { useTranslation } from 'react-i18next'
 import InfiniteScroll from 'react-infinite-scroll-component';
-import { Flex, Skeleton, Form, Dropdown, type MenuProps, App } from 'antd'
+import { Flex, Skeleton, Form, Dropdown, type MenuProps, App, Divider } from 'antd'
 import { SettingOutlined } from '@ant-design/icons'
 import clsx from 'clsx'
 import dayjs from 'dayjs'
@@ -35,7 +35,7 @@ import OnlineCheckedIcon from '@/assets/images/conversation/onlineChecked.svg'
 import MemoryFunctionCheckedIcon from '@/assets/images/conversation/memoryFunctionChecked.svg'
 import { type SSEMessage } from '@/utils/stream'
 import UploadFiles from './components/FileUpload'
-// import AudioRecorder from '@/components/AudioRecorder'
+import AudioRecorder from '@/components/AudioRecorder'
 import { shareFileUploadUrlWithoutApiPrefix } from '@/api/fileStorage'
 import UploadFileListModal from './components/UploadFileListModal'
 import type { VariableConfigModalRef } from '@/views/Workflow/types'
@@ -305,17 +305,27 @@ const Conversation: FC = () => {
       }),
       variables: params
     }, handleStreamMessage, shareToken)
+      .catch(() => {
+        setLoading(false)
+        setStreamLoading(false)
+      })
       .finally(() => {
         setLoading(false)
+        setStreamLoading(false)
       })
   }
 
   const fileChange = (file?: any) => {
     form.setFieldValue('files', [...(queryValues.files || []), file])
   }
-  // const handleRecordingComplete = async (file: any) => {
+  const handleRecordingComplete = async (file: any) => {
-  //   console.log('file', file)
+    form.setFieldValue('files', [...(queryValues.files || []), {
-  // }
+      uid: file.file_id,
+      response: { data: file },
+      thumbUrl: file.url,
+      type: file.type
+    }])
+  }
 
   const handleShowUpload: MenuProps['onClick'] = ({ key }) => {
     switch(key) {
@@ -329,6 +339,7 @@ const Conversation: FC = () => {
     form.setFieldValue('files', [...(queryValues.files || []), ...fileList])
   }
   const updateFileList = (fileList?: any[]) => {
+    console.log('fileList', fileList)
     form.setFieldValue('files', [...(fileList || [])])
   }
 
@@ -383,7 +394,7 @@ const Conversation: FC = () => {
         <div className='rb:w-190  rb:h-screen rb:mx-auto rb:pt-10'>
         <Chat
           empty={<Empty url={ChatEmpty} className="rb:h-full" size={[320,180]} title={t('memoryConversation.chatEmpty')} subTitle={t('memoryConversation.emptyDesc')} />}
-          contentClassName="rb:h-[calc(100%-180px)]"
+          contentClassName={!queryValues?.files?.length ? "rb:h-[calc(100%-144px)]" : "rb:h-[calc(100%-208px)]"}
           data={chatList}
           streamLoading={streamLoading}
           loading={loading}
@@ -405,13 +416,12 @@ const Conversation: FC = () => {
                           key: 'upload', label: (
                             <UploadFiles
                               action={shareFileUploadUrlWithoutApiPrefix}
-                              fileType={['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'svg']}
                               onChange={fileChange}
                               requestConfig={{
                                 headers: {
                                   'Content-Type': 'multipart/form-data',
                                   Authorization: `Bearer ${shareToken || ''}`,
-                              } }}
+                              }}}
                             />
                           )
                         },
@@ -455,10 +465,19 @@ const Conversation: FC = () => {
                   </Form.Item>
                 )}
               </Flex>
-              {/* <Flex align="center">
+              <Flex align="center">
-                <AudioRecorder onRecordingComplete={handleRecordingComplete} />
+                <AudioRecorder
+                  action={shareFileUploadUrlWithoutApiPrefix}
+                  requestConfig={{
+                    headers: {
+                      'Content-Type': 'multipart/form-data',
+                      Authorization: `Bearer ${shareToken || ''}`,
+                    }
+                  }}
+                  onRecordingComplete={handleRecordingComplete}
+                />
                 <Divider type="vertical" className="rb:ml-1.5! rb:mr-3!" />
-              </Flex> */}
+              </Flex>
             </Flex>
           </Form>
         </Chat>

web/src/views/ModelManagement/components/CustomModelModal.tsx

@@ -2,7 +2,7 @@
  * @Author: ZhaoYing 
  * @Date: 2026-02-03 16:49:28 
  * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-28 17:24:05
+ * @Last Modified time: 2026-03-04 11:31:43
  */
 /**
  * Custom Model Modal
@@ -10,8 +10,8 @@
  * Supports logo upload, type/provider selection, and tagging
  */
 
-import { forwardRef, useImperativeHandle, useState } from 'react';
+import { forwardRef, useEffect, useImperativeHandle, useState } from 'react';
-import { Form, Input, App } from 'antd';
+import { Form, Input, App, Checkbox } from 'antd';
 import { useTranslation } from 'react-i18next';
 
 import type { CustomModelForm, ModelListItem, CustomModelModalRef, CustomModelModalProps } from '../types';
@@ -35,6 +35,14 @@ const CustomModelModal = forwardRef<CustomModelModalRef, CustomModelModalProps>(
   const [isEdit, setIsEdit] = useState(false);
   const [form] = Form.useForm<CustomModelForm>();
   const [loading, setLoading] = useState(false)
+  const modelType = Form.useWatch(['type'], form);
+  const isOmni = Form.useWatch(['is_omni'], form);
+
+  useEffect(() => {
+    if (isOmni) {
+      form.setFieldsValue({ is_vision: true })
+    }
+  }, [isOmni])
 
   /** Close modal and reset state */
   const handleClose = () => {
@@ -49,9 +57,12 @@ const CustomModelModal = forwardRef<CustomModelModalRef, CustomModelModalProps>(
     if (model) {
       setIsEdit(true);
       setModel(model);
+      const { capability, is_omni, ...rest} = model
       form.setFieldsValue({
-        ...model,
+        ...rest,
-        logo: model.logo && model.logo.startsWith('http') ? { url: model.logo, uid: model.logo, status: 'done', name: 'logo' } : undefined
+        logo: model.logo && model.logo.startsWith('http') ? { url: model.logo, uid: model.logo, status: 'done', name: 'logo' } : undefined,
+        is_omni,
+        is_vision: capability?.includes('vision') || false,
       });
     } else {
       setIsEdit(false);
@@ -79,9 +90,14 @@ const CustomModelModal = forwardRef<CustomModelModalRef, CustomModelModalProps>(
     form
       .validateFields()
       .then((values) => {
-        const { logo, ...rest } = values;
+        const { logo, type, is_vision, is_omni, ...rest } = values;
         const formData: CustomModelForm = {
-          ...rest
+          ...rest,
+          type,
+        }
+        if (!['embedding', 'rerank'].includes(type as string)) {
+          formData.capability = is_omni ? ["vision", "audio"] : is_vision ? ['vision'] : []
+          formData.is_omni = is_omni
         }
 
         if (typeof logo === 'object' && logo?.response?.data.file_id) {
@@ -108,7 +124,7 @@ const CustomModelModal = forwardRef<CustomModelModalRef, CustomModelModalProps>(
   useImperativeHandle(ref, () => ({
     handleOpen,
   }));
-
+  console.log('modelType', modelType)
   return (
     <RbModal
       title={isEdit ? `${model.name} - ${t('modelNew.modelConfiguration')}` : t('modelNew.createCustomModel')}
@@ -180,7 +196,6 @@ const CustomModelModal = forwardRef<CustomModelModalRef, CustomModelModalProps>(
           <Input.TextArea placeholder={t('common.pleaseEnter')} />
         </Form.Item>
 
-
         <Form.Item
           name={["api_keys", 0, "api_key"]}
           label={t('modelNew.api_key')}
@@ -196,6 +211,17 @@ const CustomModelModal = forwardRef<CustomModelModalRef, CustomModelModalProps>(
         >
           <Input placeholder="https://api.example.com/v1" />
         </Form.Item>
+
+        {!['embedding', 'rerank'].includes(modelType as string) &&
+          <>
+            <Form.Item name="is_omni" valuePropName="checked" className="rb:mb-2!">
+              <Checkbox>{t('modelNew.is_omni')}</Checkbox>
+            </Form.Item>
+            <Form.Item name="is_vision" valuePropName="checked" className="rb:mb-0!">
+              <Checkbox disabled={isOmni}>{t('modelNew.is_vision')}</Checkbox>
+            </Form.Item>
+          </>
+        }
       </Form>
     </RbModal>
   );

web/src/views/ModelManagement/components/ModelImplement/SubModelModal.tsx

@@ -2,7 +2,7 @@
  * @Author: ZhaoYing 
  * @Date: 2026-02-03 16:49:20 
  * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-03 16:54:54
+ * @Last Modified time: 2026-03-04 11:51:01
  */
 /**
  * Sub-Model Modal
@@ -10,8 +10,8 @@
  * Uses cascader for hierarchical selection
  */
 
-import { forwardRef, useImperativeHandle, useState, useEffect } from 'react';
+import { type ReactNode, forwardRef, useImperativeHandle, useState, useEffect } from 'react';
-import { Form, Cascader, App, type CascaderProps } from 'antd';
+import { Form, Cascader, App, type CascaderProps, Space } from 'antd';
 import { useTranslation } from 'react-i18next';
 
 import type { SubModelModalForm, SubModelModalRef, SubModelModalProps } from './types';
@@ -19,6 +19,7 @@ import RbModal from '@/components/RbModal'
 import CustomSelect from '@/components/CustomSelect'
 import { modelProviderUrl, getModelNewList } from '@/api/models'
 import type { ProviderModelItem } from '../../types'
+import Tag from '@/components/Tag';
 
 const { SHOW_CHILD } = Cascader;
 
@@ -27,7 +28,7 @@ const { SHOW_CHILD } = Cascader;
  */
 interface Option {
   value: string | number;
-  label: string;
+  label: string | ReactNode;
   children?: Option[];
   [key: string]: any;
 }
@@ -116,7 +117,11 @@ const SubModelModal = forwardRef<SubModelModalRef, SubModelModalProps>(({
             }))
             return {
               ...vo,
-              label: vo.name,
+              label: <Space>
+                {vo.name}
+                <Tag>{t(`modelNew.${vo.type}`)}</Tag>
+                {vo.capability?.filter(item => item !== 'video').map(vo => <Tag key={vo}>{t(`modelNew.${vo}`)}</Tag>)}
+              </Space>,
               value: vo.id,
               children: children
             }

web/src/views/ModelManagement/components/ModelListDetail.tsx

@@ -1,8 +1,8 @@
 /*
  * @Author: ZhaoYing 
  * @Date: 2026-02-03 16:49:45 
- * @Last Modified by:   ZhaoYing 
+ * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-03 16:49:45 
+ * @Last Modified time: 2026-03-04 11:50:47
  */
 /**
  * Model List Detail Drawer
@@ -133,9 +133,10 @@ const ModelListDetail = forwardRef<ModelListDetailRef, ModelListDetailProps>(({
             <RbCard
               key={item.id}
               title={item.name}
-              subTitle={<Space className="rb:mt-1!">
+              subTitle={<Space size={8} className="rb:mt-1!">
                 <Tag>{t(`modelNew.${item.type}`)}</Tag>
                 <Tag color="warning">{item.api_keys.length}{t('modelNew.apiKeyNum')}</Tag>
+                {item.capability?.filter(item => item !=='video').map(vo => <Tag key={vo}>{t(`modelNew.${vo}`)}</Tag>)}
               </Space>}
               avatarUrl={getLogoUrl(item.logo)}
               avatar={

web/src/views/ModelManagement/components/ModelSquareDetail.tsx

@@ -2,7 +2,7 @@
  * @Author: ZhaoYing 
  * @Date: 2026-02-03 16:49:49 
  * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-03 16:54:26
+ * @Last Modified time: 2026-03-04 11:50:31
  */
 /**
  * Model Square Detail Drawer
@@ -89,9 +89,10 @@ const ModelSquareDetail = forwardRef<ModelSquareDetailRef, ModelSquareDetailProp
               <RbCard
                 key={item.id}
                 title={item.name}
-                subTitle={<Space size={8}>
+                subTitle={<Space size={8} className="rb:mt-1!">
-                  <Tag className="rb:mt-1">{t(`modelNew.${item.type}`)}</Tag>
+                  <Tag>{t(`modelNew.${item.type}`)}</Tag>
-                  {item.is_official && <Tag color="success" className="rb:mt-1">{t(`modelNew.official`)}</Tag>}
+                  {item.is_official && <Tag color="success">{t(`modelNew.official`)}</Tag>}
+                  {item.capability?.filter(item => item !== 'video').map(vo => <Tag key={vo}>{t(`modelNew.${vo}`)}</Tag>)}
                 </Space>}
                 avatarUrl={getLogoUrl(item.logo)}
                 avatar={

web/src/views/ModelManagement/types.ts

@@ -1,8 +1,8 @@
 /*
  * @Author: ZhaoYing 
  * @Date: 2026-02-03 16:50:18 
- * @Last Modified by:   ZhaoYing 
+ * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-03 16:50:18 
+ * @Last Modified time: 2026-03-04 11:39:20
  */
 /**
  * Type definitions for Model Management
@@ -148,7 +148,9 @@ export interface ModelListItem {
   /** Update timestamp */
   updated_at: number;
   /** Associated API keys */
-  api_keys: ModelApiKey[]
+  api_keys: ModelApiKey[];
+  capability?: string[];
+  is_omni?: boolean;
 }
 
 /**
@@ -261,6 +263,8 @@ export interface ModelPlazaItem {
   add_count: number;
   /** Whether user has added this model */
   is_added: boolean;
+  capability?: string[];
+  is_omni?: boolean;
 }
 
 /**
@@ -291,6 +295,9 @@ export interface CustomModelForm {
     /** API base URL */
     api_base: string;
   }>
+  is_vision?: boolean;
+  is_omni?: boolean;
+  capability?: string[];
 }
 
 /**

web/src/views/Workflow/components/Chat/Chat.tsx

@@ -2,7 +2,7 @@
  * @Author: ZhaoYing 
  * @Date: 2026-02-06 21:10:56 
  * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-03-04 12:10:17
+ * @Last Modified time: 2026-03-04 18:51:48
  */
 /**
  * Workflow Chat Component
@@ -23,7 +23,7 @@
  */
 import { forwardRef, useImperativeHandle, useState, useRef } from 'react'
 import { useTranslation } from 'react-i18next'
-import { App, Space, Button, Flex, Dropdown, type MenuProps } from 'antd'
+import { App, Space, Button, Flex, Dropdown, type MenuProps, Divider } from 'antd'
 
 import ChatIcon from '@/assets/images/application/chat.png'
 import RbDrawer from '@/components/RbDrawer';
@@ -38,7 +38,7 @@ import { type SSEMessage } from '@/utils/stream'
 import type { Variable } from '../Properties/VariableList/types'
 import ChatInput from '@/components/Chat/ChatInput'
 import UploadFiles from '@/views/Conversation/components/FileUpload'
-// import AudioRecorder from '@/components/AudioRecorder'
+import AudioRecorder from '@/components/AudioRecorder'
 import UploadFileListModal from '@/views/Conversation/components/UploadFileListModal'
 import type { UploadFileListModalRef } from '@/views/Conversation/types'
 import Runtime from './Runtime';
@@ -359,6 +359,7 @@ const Chat = forwardRef<ChatRef, { appId: string; graphRef: GraphRef }>(({ appId
     setStreamLoading(true)
     draftRun(appId, data, handleStreamMessage)
       .catch((error) => {
+        console.log('draftRun error', error)
         setChatList(prev => {
           const newList = [...prev]
           const lastIndex = newList.length - 1
@@ -390,9 +391,13 @@ const Chat = forwardRef<ChatRef, { appId: string; graphRef: GraphRef }>(({ appId
   const fileChange = (file?: any) => {
     setFileList([...fileList, file])
   }
-  // const handleRecordingComplete = async (file: any) => {
+  const handleRecordingComplete = async (file: any) => {
-  //   console.log('file', file)
+    setFileList([...fileList, {
-  // }
+      response: { data: file },
+      thumbUrl: file.url,
+      type: file.type
+    }])
+  }
 
   /**
    * Handles dropdown menu actions for file upload
@@ -424,6 +429,8 @@ const Chat = forwardRef<ChatRef, { appId: string; graphRef: GraphRef }>(({ appId
     handleClose
   }));
 
+  console.log('fileList', fileList)
+
   return (
     <RbDrawer
       title={<div className="rb:flex rb:items-center rb:gap-2.5">
@@ -470,7 +477,6 @@ const Chat = forwardRef<ChatRef, { appId: string; graphRef: GraphRef }>(({ appId
                     {
                       key: 'upload', label: (
                         <UploadFiles
-                          fileType={['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'svg']}
                           onChange={fileChange}
                         />
                       )
@@ -484,10 +490,10 @@ const Chat = forwardRef<ChatRef, { appId: string; graphRef: GraphRef }>(({ appId
                 ></div>
               </Dropdown>
             </Flex>
-            {/* <Flex align="center">
+            <Flex align="center">
               <AudioRecorder onRecordingComplete={handleRecordingComplete} />
               <Divider type="vertical" className="rb:ml-1.5! rb:mr-3!" />
-            </Flex> */}
+            </Flex>
           </Flex>
         </ChatInput>
       </div>