Merge pull request #465 from SuanmoSuanyangTechnology/feature/model_zy

Feature/model zy
2026-03-05 12:29:45 +08:00
parent 5322b0c4a3 f27fc51801
commit 6ed542b007
15 changed files with 359 additions and 191 deletions
--- a/web/src/components/AudioRecorder/index.tsx
+++ b/web/src/components/AudioRecorder/index.tsx
@@ -1,16 +1,21 @@
 import { type FC, useRef, useState } from 'react'
 import RecordRTC from 'recordrtc'

-import { fileUpload } from '@/api/fileStorage'
+import { fileUploadUrlWithoutApiPrefix } from '@/api/fileStorage'
+import { request } from '@/utils/request'

 interface AudioRecorderProps {
-  onRecordingComplete?: (file: { file_id: string; file_key: string; }, blob: Blob) => void
-  className?: string
+  onRecordingComplete?: (file: { file_id: string; file_key: string; url: string; type?: string; }, blob?: Blob) => void
+  className?: string;
+  action?: string;
+  requestConfig?: Record<string, any>;
 }

 const AudioRecorder: FC<AudioRecorderProps> = ({
  onRecordingComplete,
  className = '',
+  action = fileUploadUrlWithoutApiPrefix,
+  requestConfig = {}
 }) => {
  const [isRecording, setIsRecording] = useState(false)
  const recorderRef = useRef<RecordRTC | null>(null)
@@ -33,11 +38,17 @@ const AudioRecorder: FC<AudioRecorderProps> = ({
    if (recorderRef.current) {
      recorderRef.current.stopRecording(() => {
        const blob = recorderRef.current!.getBlob()
+        const url = recorderRef.current!.toURL()
        const formData = new FormData()
        formData.append('file', blob, `recording_${Date.now()}.webm`)
-        fileUpload(formData)
+        request
+          .uploadFile(action, formData, requestConfig)
          .then(res => {
-            onRecordingComplete?.(res as { file_id: string; file_key: string; }, blob)
+            onRecordingComplete?.({
+              ...(res as { file_id: string; file_key: string }),
+              type: blob.type,
+              url
+            }, blob)
            recorderRef.current?.destroy()
            recorderRef.current = null
          })
--- a/web/src/components/Chat/ChatInput.tsx
+++ b/web/src/components/Chat/ChatInput.tsx
@@ -2,10 +2,11 @@
 * @Author: ZhaoYing 
 * @Date: 2025-12-10 16:46:14 
 * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-10 12:13:52
+ * @Last Modified time: 2026-03-04 18:42:49
 */
 import { type FC, useEffect, useMemo } from 'react'
 import { Flex, Input, Form } from 'antd'
+
 import SendIcon from '@/assets/images/conversation/send.svg'
 import SendDisabledIcon from '@/assets/images/conversation/sendDisabled.svg'
 import LoadingIcon from '@/assets/images/conversation/loading.svg'
@@ -80,9 +81,31 @@ const ChatInput: FC<ChatInputProps> = ({
                </div>
              )
            }
+            if (file.type.includes('video')) {
+              return (
+                <div key={file.uid} className="rb:w-45 rb:h-16 rb:inline-block rb:group rb:relative rb:rounded-lg">
+                  <video src={file.url} controls className="rb:w-45 rb:h-16 rb:rounded-lg rb:object-cover rb:cursor-pointer" />
+                  <div
+                    className="rb:hidden rb:group-hover:block rb:absolute rb:-right-1 rb:-top-1 rb:size-3.5 rb:cursor-pointer rb:bg-cover rb:bg-[url('@/assets/images/conversation/delete.svg')] rb:hover:bg-[url('@/assets/images/conversation/delete_hover.svg')]"
+                    onClick={() => handleDelete(file)}
+                  ></div>
+                </div>
+              )
+            }
+            if (file.type.includes('audio')) {
+              return (
+                <div key={file.uid} className="rb:w-45 rb:h-16 rb:inline-flex rb:items-center rb:group rb:relative rb:rounded-lg rb:bg-[#F0F3F8] rb:py-2 rb:px-2.5 rb:gap-2">
+                  <audio src={file.url} controls className="rb:w-45 rb:h-16" />
+                  <div
+                    className="rb:hidden rb:group-hover:block rb:absolute rb:-right-1 rb:-top-1 rb:size-3.5 rb:cursor-pointer rb:bg-cover rb:bg-[url('@/assets/images/conversation/delete.svg')] rb:hover:bg-[url('@/assets/images/conversation/delete_hover.svg')]"
+                    onClick={() => handleDelete(file)}
+                  ></div>
+                </div>
+              )
+            }
            return (
              <div key={file.uid} className="rb:w-45 rb:text-[12px] rb:gap-2.5 rb:flex rb:items-center rb:group rb:relative rb:rounded-lg rb:bg-[#F0F3F8] rb:py-2 rb:px-2.5">
-                {(file.type.includes('word') || file.type.includes('wordprocessingml.document')) && <div
+                {(file.type.includes('doc') || file.type.includes('docx') || file.type.includes('word') || file.type.includes('wordprocessingml.document')) && <div
                  className="rb:size-5 rb:cursor-pointer rb:bg-cover rb:bg-[url('@/assets/images/conversation/word_disabled.svg')] rb:hover:bg-[url('@/assets/images/conversation/word.svg')]"
                ></div>}
                {(file.type.includes('pdf')) && <div
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -605,7 +605,13 @@ export const en = {
      ollama: "Ollama",
      xinference: "Xinference",
      gpustack: "Gpustack",
-      bedrock: "Bedrock"
+      bedrock: "Bedrock",
+
+      is_vision: 'Vision Support',
+      is_omni: 'Omni Support',
+      vision: 'Vision',
+      audio: 'Audio',
+      video: 'Video',
    },
    knowledgeBase: {
      home: 'Home',
@@ -1686,6 +1692,8 @@ Memory Bear: After the rebellion, regional warlordism intensified for several re
      uploadFile: 'Upload File',
      fileType: 'File Type',
      image: 'Image',
+      video: 'Video',
+      audio: 'Audio',
      fileUrl: 'File URL',
      addRemoteFile: 'Add Remote File',
      variableConfig: 'Variable Configuration',
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -1186,6 +1186,12 @@ export const zh = {
      xinference: "Xinference",
      gpustack: "Gpustack",
      bedrock: "Bedrock",
+
+      is_vision: '支持视觉',
+      is_omni: '支持全模态',
+      vision: '视觉',
+      audio: '音频',
+      video: '视频',
    },
    timezones: {
      'Asia/Shanghai': '中国标准时间 (UTC+8)',
@@ -1683,6 +1689,8 @@ export const zh = {
      uploadFile: '上传文件',
      fileType: '文件类型',
      image: '图片',
+      video: '视频',
+      audio: '音频',
      fileUrl: '文件链接',
      addRemoteFile: '添加远程文件',
      variableConfig: '变量配置',
--- a/web/src/utils/stream.ts
+++ b/web/src/utils/stream.ts
@@ -1,8 +1,8 @@
 /*
 * @Author: ZhaoYing 
 * @Date: 2026-02-02 16:35:43 
- * @Last Modified by:   ZhaoYing 
- * @Last Modified time: 2026-02-02 16:35:43 
+ * @Last Modified by: ZhaoYing
+ * @Last Modified time: 2026-03-04 18:19:24
 */
 /**
 * Server-Sent Events (SSE) Stream Utility Module
@@ -176,17 +176,17 @@ export const handleSSE = async (url: string, data: any, onMessage?: (data: SSEMe
      case 500:
      case 502:
        const errorData = await response.json();
-        errorData.error || i18n.t('common.serviceUpgrading');
-        message.warning(errorData.error || i18n.t('common.serviceUpgrading'));
-        return;
+        let errorInfo = errorData.error || i18n.t('common.serviceUpgrading')
+        message.warning(errorInfo);
+        throw errorInfo;
      case 400:
        const error = await response.json();
        message.warning(error.error);
-        throw error || 'Bad Request';
+        throw error.error || 'Bad Request';
      case 504:
        const errorJson = await response.json();
        message.warning(errorJson.error || i18n.t('common.serverError'));
-        return;
+        throw errorData.error;
      case 401:
        if (url?.includes('/public')) {
          return message.warning(i18n.t('common.publicApiCannotRefreshToken'));
--- a/web/src/views/ApplicationConfig/components/Chat.tsx
+++ b/web/src/views/ApplicationConfig/components/Chat.tsx
@@ -2,7 +2,7 @@
 * @Author: ZhaoYing 
 * @Date: 2026-02-03 16:27:39 
 * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-03-04 18:05:36
+ * @Last Modified time: 2026-03-04 18:51:20
 */
 /**
 * Chat debugging component for application testing
@@ -13,7 +13,7 @@
 import { type FC, useEffect, useState, useRef } from 'react';
 import { useTranslation } from 'react-i18next';
 import clsx from 'clsx'
-import { Flex, Dropdown, type MenuProps, App } from 'antd'
+import { Flex, Dropdown, type MenuProps, App, Divider } from 'antd'

 import ChatIcon from '@/assets/images/application/chat.png'
 import DebuggingEmpty from '@/assets/images/application/debuggingEmpty.png'
@@ -25,7 +25,7 @@ import type { ChatItem } from '@/components/Chat/types'
 import { type SSEMessage } from '@/utils/stream'
 import ChatInput from '@/components/Chat/ChatInput'
 import UploadFiles from '@/views/Conversation/components/FileUpload'
-// import AudioRecorder from '@/components/AudioRecorder'
+import AudioRecorder from '@/components/AudioRecorder'
 import UploadFileListModal from '@/views/Conversation/components/UploadFileListModal'
 import type { UploadFileListModalRef } from '@/views/Conversation/types'
 import type { Variable } from './VariableList/types'
@@ -88,7 +88,7 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
      content: '',
      created_at: Date.now(),
    };
-    
+
    if (isCluster) {
      updateChatList(prev => prev.map(item => ({
        ...item,
@@ -134,7 +134,7 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
    })
  }
  /** Update assistant message when error occurs */
-  const updateErrorAssistantMessage  = (message_length: number, model_config_id?: string) => {
+  const updateErrorAssistantMessage = (message_length: number, model_config_id?: string) => {
    if (message_length > 0 || !model_config_id) return

    updateChatList(prev => {
@@ -245,7 +245,15 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
            "stream": true,
            "timeout": 60,
          }, handleStreamMessage)
-            .finally(() => setLoading(false));
+            .catch(() => {
+              setLoading(false)
+              setCompareLoading(false)
+              updateClusterErrorAssistantMessage(0)
+            })
+            .finally(() => {
+              setLoading(false)
+              setCompareLoading(false)
+            })
        }, 0)
      })
      .catch(() => {
@@ -290,7 +298,7 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
    })
  }
  /** Update cluster message when error occurs */
-  const updateClusterErrorAssistantMessage  = (message_length: number) => {
+  const updateClusterErrorAssistantMessage = (message_length: number) => {
    if (message_length > 0) return

    updateChatList(prev => {
@@ -333,7 +341,7 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
          data.map(item => {
            const { conversation_id, content, message_length } = item.data as { conversation_id: string, content: string, message_length: number };

-            switch(item.event) {
+            switch (item.event) {
              case 'start':
                if (conversation_id && conversationId !== conversation_id) {
                  setConversationId(conversation_id);
@@ -356,27 +364,35 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
        };

        setTimeout(() => {
-            draftRun(
-              data.app_id,
-              { 
-                message, 
-                conversation_id: conversationId, 
-                stream: true,
-                files: fileList.map(file => {
-                  if (file.url) {
-                    return file
-                  } else {
-                    return {
-                      type: file.type,
-                      transfer_method: 'local_file',
-                      upload_file_id: file.response.data.file_id
-                    }
+          draftRun(
+            data.app_id,
+            {
+              message,
+              conversation_id: conversationId,
+              stream: true,
+              files: fileList.map(file => {
+                if (file.url) {
+                  return file
+                } else {
+                  return {
+                    type: file.type,
+                    transfer_method: 'local_file',
+                    upload_file_id: file.response.data.file_id
                  }
-                }),
-              }, 
-              handleStreamMessage
-            )
-              .finally(() => setLoading(false))
+                }
+              }),
+            },
+            handleStreamMessage
+          )
+            .catch(() => {
+              setLoading(false)
+              setCompareLoading(false)
+              updateClusterErrorAssistantMessage(0)
+            })
+            .finally(() => {
+              setLoading(false)
+              setCompareLoading(false)
+            })
        }, 0)
      })
      .catch(() => {
@@ -395,12 +411,17 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
  const fileChange = (file?: any) => {
    setFileList([...fileList, file])
  }
-  // const handleRecordingComplete = async (file: any) => {
-  //   console.log('file', file)
-  // }
+  const handleRecordingComplete = async (file: any) => {
+    setFileList([...fileList, {
+      uid: file.file_id,
+      response: { data: file },
+      thumbUrl: file.url,
+      type: file.type
+    }])
+  }

  const handleShowUpload: MenuProps['onClick'] = ({ key }) => {
-    switch(key) {
+    switch (key) {
      case 'define':
        uploadFileListModalRef.current?.handleOpen()
        break
@@ -417,99 +438,98 @@ const Chat: FC<ChatProps> = ({ chatList, data, updateChatList, handleSave, sourc
  return (
    <div className="rb:relative rb:h-full rb:flex rb:flex-col">
      {chatList.length === 0
-        ? <Empty 
-          url={DebuggingEmpty} 
+        ? <Empty
+          url={DebuggingEmpty}
          size={[300, 200]}
-          title={t('application.debuggingEmpty')} 
-          subTitle={t('application.debuggingEmptyDesc')} 
+          title={t('application.debuggingEmpty')}
+          subTitle={t('application.debuggingEmptyDesc')}
          className="rb:h-full"
        />
-      : <>
-        <div className={clsx(`rb:relative rb:grid rb:grid-cols-${chatList.length} rb:overflow-hidden rb:w-full rb:flex-1 rb:min-h-0`)}>
-          {chatList.map((chat, index) => (
-            <div key={index} className={clsx('rb:flex rb:flex-col', {
-              "rb:border-r rb:border-[#DFE4ED]": index !== chatList.length - 1 && chatList.length > 1,
-            })}>
-              {chat.label &&
-                <div className={clsx(
-                  "rb:grid rb:bg-[#F0F3F8] rb:text-center rb:flex-[0_0_auto]",
-                  {
-                    'rb:rounded-tr-xl': index === chatList.length - 1,
-                    'rb:rounded-tl-xl': index === 0,
-                  }
-                )}>
-                  <div className='rb:relative rb:p-[10px_12px] rb:overflow-hidden'>
-                    <div className="rb:text-ellipsis rb:overflow-hidden rb:whitespace-nowrap rb:w-[calc(100%-24px)]">{chat.label}</div>
-                    <div 
-                      className="rb:w-4 rb:h-4 rb:cursor-pointer rb:absolute rb:top-3 rb:right-3 rb:bg-cover rb:bg-[url('@/assets/images/close.svg')] rb:hover:bg-[url('@/assets/images/close_hover.svg')]" 
-                      onClick={() => handleDelete(index)}
-                    ></div>
+        : <>
+          <div className={clsx(`rb:relative rb:grid rb:grid-cols-${chatList.length} rb:overflow-hidden rb:w-full rb:flex-1 rb:min-h-0`)}>
+            {chatList.map((chat, index) => (
+              <div key={index} className={clsx('rb:flex rb:flex-col', {
+                "rb:border-r rb:border-[#DFE4ED]": index !== chatList.length - 1 && chatList.length > 1,
+              })}>
+                {chat.label &&
+                  <div className={clsx(
+                    "rb:grid rb:bg-[#F0F3F8] rb:text-center rb:flex-[0_0_auto]",
+                    {
+                      'rb:rounded-tr-xl': index === chatList.length - 1,
+                      'rb:rounded-tl-xl': index === 0,
+                    }
+                  )}>
+                    <div className='rb:relative rb:p-[10px_12px] rb:overflow-hidden'>
+                      <div className="rb:text-ellipsis rb:overflow-hidden rb:whitespace-nowrap rb:w-[calc(100%-24px)]">{chat.label}</div>
+                      <div
+                        className="rb:w-4 rb:h-4 rb:cursor-pointer rb:absolute rb:top-3 rb:right-3 rb:bg-cover rb:bg-[url('@/assets/images/close.svg')] rb:hover:bg-[url('@/assets/images/close_hover.svg')]"
+                        onClick={() => handleDelete(index)}
+                      ></div>
+                    </div>
                  </div>
-                </div>
-              }
-              <ChatContent
-                classNames={{
-                  'rb:mx-[16px] rb:mt-6': true,
-                  'rb:h-[calc(100vh-282px)]': isCluster,
-                  'rb:h-[calc(100vh-380px)]': !isCluster,
-                }} 
-                contentClassNames={{
-                  'rb:max-w-[400px]!': chatList.length === 1,
-                  'rb:max-w-[260px]!': chatList.length === 2,
-                  'rb:max-w-[150px]!': chatList.length === 3,
-                  'rb:max-w-[108px]!': chatList.length === 4,
-                }}
-                empty={<Empty url={ChatIcon} title={t('application.chatEmpty')} isNeedSubTitle={false} size={[240, 200]} className="rb:h-full" />}
-                data={chat.list || []}
-                streamLoading={compareLoading}
-                labelPosition="top"
-                labelFormat={(item) => item.role === 'user' ? t('application.you') : chat.label}
-                errorDesc={t('application.ReplyException')}
-              />
-            </div>
-          ))}
-        </div>
-        <div className="rb:relative rb:flex rb:items-center rb:gap-2.5 rb:m-4 rb:mb-1">
-          <ChatInput
-            message={message}
-            className="rb:relative!"
-            loading={loading}
-            fileChange={updateFileList}
-            fileList={fileList}
-            onSend={isCluster ? handleClusterSend : handleSend}
-            onChange={handleMessageChange}
-          >
-            <Flex justify="space-between" className="rb:flex-1">
-              <Flex gap={8} align="center">
-                <Dropdown
-                  menu={{
-                    items: [
-                      { key: 'define', label: t('memoryConversation.addRemoteFile') },
-                      {
-                        key: 'upload', label: (
-                          <UploadFiles
-                            fileType={['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'svg']}
-                            onChange={fileChange}
-                          />
-                        )
-                      },
-                    ],
-                    onClick: handleShowUpload
+                }
+                <ChatContent
+                  classNames={{
+                    'rb:mx-[16px] rb:mt-6': true,
+                    'rb:h-[calc(100vh-282px)]': isCluster,
+                    'rb:h-[calc(100vh-380px)]': !isCluster,
                  }}
-                >
-                  <div
-                    className="rb:size-6 rb:cursor-pointer rb:bg-cover rb:bg-[url('@/assets/images/conversation/link.svg')] rb:hover:bg-[url('@/assets/images/conversation/link_hover.svg')]"
-                  ></div>
-                </Dropdown>
+                  contentClassNames={{
+                    'rb:max-w-[400px]!': chatList.length === 1,
+                    'rb:max-w-[260px]!': chatList.length === 2,
+                    'rb:max-w-[150px]!': chatList.length === 3,
+                    'rb:max-w-[108px]!': chatList.length === 4,
+                  }}
+                  empty={<Empty url={ChatIcon} title={t('application.chatEmpty')} isNeedSubTitle={false} size={[240, 200]} className="rb:h-full" />}
+                  data={chat.list || []}
+                  streamLoading={compareLoading}
+                  labelPosition="top"
+                  labelFormat={(item) => item.role === 'user' ? t('application.you') : chat.label}
+                  errorDesc={t('application.ReplyException')}
+                />
+              </div>
+            ))}
+          </div>
+          <div className="rb:relative rb:flex rb:items-center rb:gap-2.5 rb:m-4 rb:mb-1">
+            <ChatInput
+              message={message}
+              className="rb:relative!"
+              loading={loading}
+              fileChange={updateFileList}
+              fileList={fileList}
+              onSend={isCluster ? handleClusterSend : handleSend}
+              onChange={handleMessageChange}
+            >
+              <Flex justify="space-between" className="rb:flex-1">
+                <Flex gap={8} align="center">
+                  <Dropdown
+                    menu={{
+                      items: [
+                        { key: 'define', label: t('memoryConversation.addRemoteFile') },
+                        {
+                          key: 'upload', label: (
+                            <UploadFiles
+                              onChange={fileChange}
+                            />
+                          )
+                        },
+                      ],
+                      onClick: handleShowUpload
+                    }}
+                  >
+                    <div
+                      className="rb:size-6 rb:cursor-pointer rb:bg-cover rb:bg-[url('@/assets/images/conversation/link.svg')] rb:hover:bg-[url('@/assets/images/conversation/link_hover.svg')]"
+                    ></div>
+                  </Dropdown>
+                </Flex>
+                <Flex align="center">
+                  <AudioRecorder onRecordingComplete={handleRecordingComplete} />
+                  <Divider type="vertical" className="rb:ml-1.5! rb:mr-3!" />
+                </Flex>
              </Flex>
-              {/* <Flex align="center">
-                <AudioRecorder onRecordingComplete={handleRecordingComplete} />
-                <Divider type="vertical" className="rb:ml-1.5! rb:mr-3!" />
-              </Flex> */}
-            </Flex>
-          </ChatInput>
-        </div>
-      </>
+            </ChatInput>
+          </div>
+        </>
      }

      <UploadFileListModal
--- a/web/src/views/Conversation/components/FileUpload.tsx
+++ b/web/src/views/Conversation/components/FileUpload.tsx
@@ -2,7 +2,7 @@
 * @Author: ZhaoYing 
 * @Date: 2026-02-06 21:09:42 
 * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-11 11:32:48
+ * @Last Modified time: 2026-03-04 18:54:47
 */
 /**
 * File Upload Component
@@ -25,6 +25,7 @@ import { Upload, Progress, App } from 'antd';
 import type { UploadProps, UploadFile } from 'antd';
 import type { UploadProps as RcUploadProps } from 'antd/es/upload/interface';
 import { useTranslation } from 'react-i18next';
+
 import { request } from '@/utils/request'
 import { fileUploadUrlWithoutApiPrefix } from '@/api/fileStorage'

@@ -56,27 +57,36 @@ interface UploadFilesProps extends Omit<UploadProps, 'onChange'> {
  /** Custom file removal callback */
  onRemove?: (file: UploadFile) => boolean | void | Promise<boolean | void>;
 }
+
+const transform_file_type = {
+  'text/plain': 'document/text',
+  'text/markdown': 'document/markdown',
+  'text/x-markdown': 'document/x-markdown',
+
+  'application/pdf': 'document/pdf',
+
+  'application/msword': 'document/doc',
+  'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'document/docx',
+
+  'application/vnd.ms-powerpoint': 'document/ppt',
+  'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'document/pptx',
+}
 // Mapping of file extensions to MIME types
 const ALL_FILE_TYPE: {
  [key: string]: string;
 } = {
-  // txt: 'text/plain',
+  txt: 'text/plain',
+  md: 'text/markdown',
+  xmd: 'text/x-markdown',
+
  pdf: 'application/pdf',

  doc: 'application/msword',
  docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
-  
-  xls: 'application/vnd.ms-excel',
-  xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
-  csv: 'text/csv',

  ppt: 'application/vnd.ms-powerpoint',
  pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
-  
-  // md: 'text/markdown',
-  // htm: 'text/html',
-  // html: 'text/html',
-  // json: 'application/json',
+
  jpg: 'image/jpeg',
  jpeg: 'image/jpeg',
  png: 'image/png',
@@ -84,6 +94,23 @@ const ALL_FILE_TYPE: {
  bmp: 'image/bmp',
  webp: 'image/webp',
  svg: 'image/svg+xml',
+
+  mp4: 'video/mp4',
+  mov: 'video/quicktime',
+  avi: 'video/x-msvideo',
+  mkv: 'video/x-matroska',
+  webm: 'video/webm',
+  flv: 'video/x-flv',
+  wmv: 'video/x-ms-wmv',
+
+  mp3: 'audio/mpeg',
+  wav: 'audio/wav',
+  ogg: 'audio/ogg',
+  aac: 'audio/aac',
+  flac: 'audio/flac',
+  m4a: 'audio/mp4',
+  wma: 'audio/x-ms-wma',
+  xm4a: 'audio/x-m4a',
 }
 export interface UploadFilesRef {
  /** Current file list */
@@ -178,6 +205,10 @@ const UploadFiles = forwardRef<UploadFilesRef, UploadFilesProps>(({
   * Handles upload state changes
   */
  const handleChange: UploadProps['onChange'] = ({ fileList: newFileList }) => {
+    newFileList.map(file => {
+      const type = (file.type && transform_file_type[file.type as keyof typeof transform_file_type]) || file.type
+      file.type = type
+    })
    setFileList(newFileList);
    if (onChange) {
      onChange(maxCount === 1 ? newFileList[newFileList.length - 1] : newFileList);
--- a/web/src/views/Conversation/components/UploadFileListModal.tsx
+++ b/web/src/views/Conversation/components/UploadFileListModal.tsx
@@ -2,7 +2,7 @@
 * @Author: ZhaoYing 
 * @Date: 2026-02-06 21:09:47 
 * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-09 10:17:54
+ * @Last Modified time: 2026-03-04 17:47:09
 */
 /**
 * Upload File List Modal Component
@@ -104,7 +104,9 @@ const UploadFileListModal = forwardRef<UploadFileListModalRef, UploadFileListMod
                    <Select
                      placeholder={t('memoryConversation.fileType')}
                      options={[
-                        { label: t('memoryConversation.image'), value: 'image' }
+                        { label: t('memoryConversation.image'), value: 'image' },
+                        { label: t('memoryConversation.audio'), value: 'audio' },
+                        { label: t('memoryConversation.video'), value: 'video' },
                      ]}
                      className="rb:w-30"
                    />
--- a/web/src/views/Conversation/index.tsx
+++ b/web/src/views/Conversation/index.tsx
@@ -14,7 +14,7 @@ import { type FC, useState, useEffect, useRef } from 'react'
 import { useParams, useLocation } from 'react-router-dom'
 import { useTranslation } from 'react-i18next'
 import InfiniteScroll from 'react-infinite-scroll-component';
-import { Flex, Skeleton, Form, Dropdown, type MenuProps, App } from 'antd'
+import { Flex, Skeleton, Form, Dropdown, type MenuProps, App, Divider } from 'antd'
 import { SettingOutlined } from '@ant-design/icons'
 import clsx from 'clsx'
 import dayjs from 'dayjs'
@@ -35,7 +35,7 @@ import OnlineCheckedIcon from '@/assets/images/conversation/onlineChecked.svg'
 import MemoryFunctionCheckedIcon from '@/assets/images/conversation/memoryFunctionChecked.svg'
 import { type SSEMessage } from '@/utils/stream'
 import UploadFiles from './components/FileUpload'
-// import AudioRecorder from '@/components/AudioRecorder'
+import AudioRecorder from '@/components/AudioRecorder'
 import { shareFileUploadUrlWithoutApiPrefix } from '@/api/fileStorage'
 import UploadFileListModal from './components/UploadFileListModal'
 import type { VariableConfigModalRef } from '@/views/Workflow/types'
@@ -305,17 +305,27 @@ const Conversation: FC = () => {
      }),
      variables: params
    }, handleStreamMessage, shareToken)
+      .catch(() => {
+        setLoading(false)
+        setStreamLoading(false)
+      })
      .finally(() => {
        setLoading(false)
+        setStreamLoading(false)
      })
  }

  const fileChange = (file?: any) => {
    form.setFieldValue('files', [...(queryValues.files || []), file])
  }
-  // const handleRecordingComplete = async (file: any) => {
-  //   console.log('file', file)
-  // }
+  const handleRecordingComplete = async (file: any) => {
+    form.setFieldValue('files', [...(queryValues.files || []), {
+      uid: file.file_id,
+      response: { data: file },
+      thumbUrl: file.url,
+      type: file.type
+    }])
+  }

  const handleShowUpload: MenuProps['onClick'] = ({ key }) => {
    switch(key) {
@@ -329,6 +339,7 @@ const Conversation: FC = () => {
    form.setFieldValue('files', [...(queryValues.files || []), ...fileList])
  }
  const updateFileList = (fileList?: any[]) => {
+    console.log('fileList', fileList)
    form.setFieldValue('files', [...(fileList || [])])
  }

@@ -383,7 +394,7 @@ const Conversation: FC = () => {
        <div className='rb:w-190  rb:h-screen rb:mx-auto rb:pt-10'>
        <Chat
          empty={<Empty url={ChatEmpty} className="rb:h-full" size={[320,180]} title={t('memoryConversation.chatEmpty')} subTitle={t('memoryConversation.emptyDesc')} />}
-          contentClassName="rb:h-[calc(100%-180px)]"
+          contentClassName={!queryValues?.files?.length ? "rb:h-[calc(100%-144px)]" : "rb:h-[calc(100%-208px)]"}
          data={chatList}
          streamLoading={streamLoading}
          loading={loading}
@@ -405,13 +416,12 @@ const Conversation: FC = () => {
                          key: 'upload', label: (
                            <UploadFiles
                              action={shareFileUploadUrlWithoutApiPrefix}
-                              fileType={['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'svg']}
                              onChange={fileChange}
                              requestConfig={{
                                headers: {
                                  'Content-Type': 'multipart/form-data',
                                  Authorization: `Bearer ${shareToken || ''}`,
-                              } }}
+                              }}}
                            />
                          )
                        },
@@ -455,10 +465,19 @@ const Conversation: FC = () => {
                  </Form.Item>
                )}
              </Flex>
-              {/* <Flex align="center">
-                <AudioRecorder onRecordingComplete={handleRecordingComplete} />
+              <Flex align="center">
+                <AudioRecorder
+                  action={shareFileUploadUrlWithoutApiPrefix}
+                  requestConfig={{
+                    headers: {
+                      'Content-Type': 'multipart/form-data',
+                      Authorization: `Bearer ${shareToken || ''}`,
+                    }
+                  }}
+                  onRecordingComplete={handleRecordingComplete}
+                />
                <Divider type="vertical" className="rb:ml-1.5! rb:mr-3!" />
-              </Flex> */}
+              </Flex>
            </Flex>
          </Form>
        </Chat>
--- a/web/src/views/ModelManagement/components/CustomModelModal.tsx
+++ b/web/src/views/ModelManagement/components/CustomModelModal.tsx
@@ -2,7 +2,7 @@
 * @Author: ZhaoYing 
 * @Date: 2026-02-03 16:49:28 
 * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-28 17:24:05
+ * @Last Modified time: 2026-03-04 11:31:43
 */
 /**
 * Custom Model Modal
@@ -10,8 +10,8 @@
 * Supports logo upload, type/provider selection, and tagging
 */

-import { forwardRef, useImperativeHandle, useState } from 'react';
-import { Form, Input, App } from 'antd';
+import { forwardRef, useEffect, useImperativeHandle, useState } from 'react';
+import { Form, Input, App, Checkbox } from 'antd';
 import { useTranslation } from 'react-i18next';

 import type { CustomModelForm, ModelListItem, CustomModelModalRef, CustomModelModalProps } from '../types';
@@ -35,6 +35,14 @@ const CustomModelModal = forwardRef<CustomModelModalRef, CustomModelModalProps>(
  const [isEdit, setIsEdit] = useState(false);
  const [form] = Form.useForm<CustomModelForm>();
  const [loading, setLoading] = useState(false)
+  const modelType = Form.useWatch(['type'], form);
+  const isOmni = Form.useWatch(['is_omni'], form);
+
+  useEffect(() => {
+    if (isOmni) {
+      form.setFieldsValue({ is_vision: true })
+    }
+  }, [isOmni])

  /** Close modal and reset state */
  const handleClose = () => {
@@ -49,9 +57,12 @@ const CustomModelModal = forwardRef<CustomModelModalRef, CustomModelModalProps>(
    if (model) {
      setIsEdit(true);
      setModel(model);
+      const { capability, is_omni, ...rest} = model
      form.setFieldsValue({
-        ...model,
-        logo: model.logo && model.logo.startsWith('http') ? { url: model.logo, uid: model.logo, status: 'done', name: 'logo' } : undefined
+        ...rest,
+        logo: model.logo && model.logo.startsWith('http') ? { url: model.logo, uid: model.logo, status: 'done', name: 'logo' } : undefined,
+        is_omni,
+        is_vision: capability?.includes('vision') || false,
      });
    } else {
      setIsEdit(false);
@@ -79,9 +90,14 @@ const CustomModelModal = forwardRef<CustomModelModalRef, CustomModelModalProps>(
    form
      .validateFields()
      .then((values) => {
-        const { logo, ...rest } = values;
+        const { logo, type, is_vision, is_omni, ...rest } = values;
        const formData: CustomModelForm = {
-          ...rest
+          ...rest,
+          type,
+        }
+        if (!['embedding', 'rerank'].includes(type as string)) {
+          formData.capability = is_omni ? ["vision", "audio"] : is_vision ? ['vision'] : []
+          formData.is_omni = is_omni
        }

        if (typeof logo === 'object' && logo?.response?.data.file_id) {
@@ -108,7 +124,7 @@ const CustomModelModal = forwardRef<CustomModelModalRef, CustomModelModalProps>(
  useImperativeHandle(ref, () => ({
    handleOpen,
  }));
-
+  console.log('modelType', modelType)
  return (
    <RbModal
      title={isEdit ? `${model.name} - ${t('modelNew.modelConfiguration')}` : t('modelNew.createCustomModel')}
@@ -180,7 +196,6 @@ const CustomModelModal = forwardRef<CustomModelModalRef, CustomModelModalProps>(
          <Input.TextArea placeholder={t('common.pleaseEnter')} />
        </Form.Item>

-
        <Form.Item
          name={["api_keys", 0, "api_key"]}
          label={t('modelNew.api_key')}
@@ -196,6 +211,17 @@ const CustomModelModal = forwardRef<CustomModelModalRef, CustomModelModalProps>(
        >
          <Input placeholder="https://api.example.com/v1" />
        </Form.Item>
+
+        {!['embedding', 'rerank'].includes(modelType as string) &&
+          <>
+            <Form.Item name="is_omni" valuePropName="checked" className="rb:mb-2!">
+              <Checkbox>{t('modelNew.is_omni')}</Checkbox>
+            </Form.Item>
+            <Form.Item name="is_vision" valuePropName="checked" className="rb:mb-0!">
+              <Checkbox disabled={isOmni}>{t('modelNew.is_vision')}</Checkbox>
+            </Form.Item>
+          </>
+        }
      </Form>
    </RbModal>
  );
--- a/web/src/views/ModelManagement/components/ModelImplement/SubModelModal.tsx
+++ b/web/src/views/ModelManagement/components/ModelImplement/SubModelModal.tsx
@@ -2,7 +2,7 @@
 * @Author: ZhaoYing 
 * @Date: 2026-02-03 16:49:20 
 * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-03 16:54:54
+ * @Last Modified time: 2026-03-04 11:51:01
 */
 /**
 * Sub-Model Modal
@@ -10,8 +10,8 @@
 * Uses cascader for hierarchical selection
 */

-import { forwardRef, useImperativeHandle, useState, useEffect } from 'react';
-import { Form, Cascader, App, type CascaderProps } from 'antd';
+import { type ReactNode, forwardRef, useImperativeHandle, useState, useEffect } from 'react';
+import { Form, Cascader, App, type CascaderProps, Space } from 'antd';
 import { useTranslation } from 'react-i18next';

 import type { SubModelModalForm, SubModelModalRef, SubModelModalProps } from './types';
@@ -19,6 +19,7 @@ import RbModal from '@/components/RbModal'
 import CustomSelect from '@/components/CustomSelect'
 import { modelProviderUrl, getModelNewList } from '@/api/models'
 import type { ProviderModelItem } from '../../types'
+import Tag from '@/components/Tag';

 const { SHOW_CHILD } = Cascader;

@@ -27,7 +28,7 @@ const { SHOW_CHILD } = Cascader;
 */
 interface Option {
  value: string | number;
-  label: string;
+  label: string | ReactNode;
  children?: Option[];
  [key: string]: any;
 }
@@ -116,7 +117,11 @@ const SubModelModal = forwardRef<SubModelModalRef, SubModelModalProps>(({
            }))
            return {
              ...vo,
-              label: vo.name,
+              label: <Space>
+                {vo.name}
+                <Tag>{t(`modelNew.${vo.type}`)}</Tag>
+                {vo.capability?.filter(item => item !== 'video').map(vo => <Tag key={vo}>{t(`modelNew.${vo}`)}</Tag>)}
+              </Space>,
              value: vo.id,
              children: children
            }
--- a/web/src/views/ModelManagement/components/ModelListDetail.tsx
+++ b/web/src/views/ModelManagement/components/ModelListDetail.tsx
@@ -1,8 +1,8 @@
 /*
 * @Author: ZhaoYing 
 * @Date: 2026-02-03 16:49:45 
- * @Last Modified by:   ZhaoYing 
- * @Last Modified time: 2026-02-03 16:49:45 
+ * @Last Modified by: ZhaoYing
+ * @Last Modified time: 2026-03-04 11:50:47
 */
 /**
 * Model List Detail Drawer
@@ -133,9 +133,10 @@ const ModelListDetail = forwardRef<ModelListDetailRef, ModelListDetailProps>(({
            <RbCard
              key={item.id}
              title={item.name}
-              subTitle={<Space className="rb:mt-1!">
+              subTitle={<Space size={8} className="rb:mt-1!">
                <Tag>{t(`modelNew.${item.type}`)}</Tag>
                <Tag color="warning">{item.api_keys.length}{t('modelNew.apiKeyNum')}</Tag>
+                {item.capability?.filter(item => item !=='video').map(vo => <Tag key={vo}>{t(`modelNew.${vo}`)}</Tag>)}
              </Space>}
              avatarUrl={getLogoUrl(item.logo)}
              avatar={
--- a/web/src/views/ModelManagement/components/ModelSquareDetail.tsx
+++ b/web/src/views/ModelManagement/components/ModelSquareDetail.tsx
@@ -2,7 +2,7 @@
 * @Author: ZhaoYing 
 * @Date: 2026-02-03 16:49:49 
 * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-03 16:54:26
+ * @Last Modified time: 2026-03-04 11:50:31
 */
 /**
 * Model Square Detail Drawer
@@ -89,9 +89,10 @@ const ModelSquareDetail = forwardRef<ModelSquareDetailRef, ModelSquareDetailProp
              <RbCard
                key={item.id}
                title={item.name}
-                subTitle={<Space size={8}>
-                  <Tag className="rb:mt-1">{t(`modelNew.${item.type}`)}</Tag>
-                  {item.is_official && <Tag color="success" className="rb:mt-1">{t(`modelNew.official`)}</Tag>}
+                subTitle={<Space size={8} className="rb:mt-1!">
+                  <Tag>{t(`modelNew.${item.type}`)}</Tag>
+                  {item.is_official && <Tag color="success">{t(`modelNew.official`)}</Tag>}
+                  {item.capability?.filter(item => item !== 'video').map(vo => <Tag key={vo}>{t(`modelNew.${vo}`)}</Tag>)}
                </Space>}
                avatarUrl={getLogoUrl(item.logo)}
                avatar={
--- a/web/src/views/ModelManagement/types.ts
+++ b/web/src/views/ModelManagement/types.ts
@@ -1,8 +1,8 @@
 /*
 * @Author: ZhaoYing 
 * @Date: 2026-02-03 16:50:18 
- * @Last Modified by:   ZhaoYing 
- * @Last Modified time: 2026-02-03 16:50:18 
+ * @Last Modified by: ZhaoYing
+ * @Last Modified time: 2026-03-04 11:39:20
 */
 /**
 * Type definitions for Model Management
@@ -148,7 +148,9 @@ export interface ModelListItem {
  /** Update timestamp */
  updated_at: number;
  /** Associated API keys */
-  api_keys: ModelApiKey[]
+  api_keys: ModelApiKey[];
+  capability?: string[];
+  is_omni?: boolean;
 }

 /**
@@ -261,6 +263,8 @@ export interface ModelPlazaItem {
  add_count: number;
  /** Whether user has added this model */
  is_added: boolean;
+  capability?: string[];
+  is_omni?: boolean;
 }

 /**
@@ -291,6 +295,9 @@ export interface CustomModelForm {
    /** API base URL */
    api_base: string;
  }>
+  is_vision?: boolean;
+  is_omni?: boolean;
+  capability?: string[];
 }

 /**
--- a/web/src/views/Workflow/components/Chat/Chat.tsx
+++ b/web/src/views/Workflow/components/Chat/Chat.tsx
@@ -2,7 +2,7 @@
 * @Author: ZhaoYing 
 * @Date: 2026-02-06 21:10:56 
 * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-03-04 12:10:17
+ * @Last Modified time: 2026-03-04 18:51:48
 */
 /**
 * Workflow Chat Component
@@ -23,7 +23,7 @@
 */
 import { forwardRef, useImperativeHandle, useState, useRef } from 'react'
 import { useTranslation } from 'react-i18next'
-import { App, Space, Button, Flex, Dropdown, type MenuProps } from 'antd'
+import { App, Space, Button, Flex, Dropdown, type MenuProps, Divider } from 'antd'

 import ChatIcon from '@/assets/images/application/chat.png'
 import RbDrawer from '@/components/RbDrawer';
@@ -38,7 +38,7 @@ import { type SSEMessage } from '@/utils/stream'
 import type { Variable } from '../Properties/VariableList/types'
 import ChatInput from '@/components/Chat/ChatInput'
 import UploadFiles from '@/views/Conversation/components/FileUpload'
-// import AudioRecorder from '@/components/AudioRecorder'
+import AudioRecorder from '@/components/AudioRecorder'
 import UploadFileListModal from '@/views/Conversation/components/UploadFileListModal'
 import type { UploadFileListModalRef } from '@/views/Conversation/types'
 import Runtime from './Runtime';
@@ -359,6 +359,7 @@ const Chat = forwardRef<ChatRef, { appId: string; graphRef: GraphRef }>(({ appId
    setStreamLoading(true)
    draftRun(appId, data, handleStreamMessage)
      .catch((error) => {
+        console.log('draftRun error', error)
        setChatList(prev => {
          const newList = [...prev]
          const lastIndex = newList.length - 1
@@ -390,9 +391,13 @@ const Chat = forwardRef<ChatRef, { appId: string; graphRef: GraphRef }>(({ appId
  const fileChange = (file?: any) => {
    setFileList([...fileList, file])
  }
-  // const handleRecordingComplete = async (file: any) => {
-  //   console.log('file', file)
-  // }
+  const handleRecordingComplete = async (file: any) => {
+    setFileList([...fileList, {
+      response: { data: file },
+      thumbUrl: file.url,
+      type: file.type
+    }])
+  }

  /**
   * Handles dropdown menu actions for file upload
@@ -424,6 +429,8 @@ const Chat = forwardRef<ChatRef, { appId: string; graphRef: GraphRef }>(({ appId
    handleClose
  }));

+  console.log('fileList', fileList)
+
  return (
    <RbDrawer
      title={<div className="rb:flex rb:items-center rb:gap-2.5">
@@ -470,7 +477,6 @@ const Chat = forwardRef<ChatRef, { appId: string; graphRef: GraphRef }>(({ appId
                    {
                      key: 'upload', label: (
                        <UploadFiles
-                          fileType={['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'svg']}
                          onChange={fileChange}
                        />
                      )
@@ -484,10 +490,10 @@ const Chat = forwardRef<ChatRef, { appId: string; graphRef: GraphRef }>(({ appId
                ></div>
              </Dropdown>
            </Flex>
-            {/* <Flex align="center">
+            <Flex align="center">
              <AudioRecorder onRecordingComplete={handleRecordingComplete} />
              <Divider type="vertical" className="rb:ml-1.5! rb:mr-3!" />
-            </Flex> */}
+            </Flex>
          </Flex>
        </ChatInput>
      </div>