feat(web): app's chat support audio/video/document file

2026-03-05 10:30:35 +08:00
parent 31bee889d7
commit 0def474cc2
10 changed files with 284 additions and 166 deletions
--- a/web/src/views/Conversation/components/FileUpload.tsx
+++ b/web/src/views/Conversation/components/FileUpload.tsx
@@ -2,7 +2,7 @@
 * @Author: ZhaoYing 
 * @Date: 2026-02-06 21:09:42 
 * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-11 11:32:48
+ * @Last Modified time: 2026-03-04 18:54:47
 */
 /**
 * File Upload Component
@@ -25,6 +25,7 @@ import { Upload, Progress, App } from 'antd';
 import type { UploadProps, UploadFile } from 'antd';
 import type { UploadProps as RcUploadProps } from 'antd/es/upload/interface';
 import { useTranslation } from 'react-i18next';
+
 import { request } from '@/utils/request'
 import { fileUploadUrlWithoutApiPrefix } from '@/api/fileStorage'

@@ -56,27 +57,36 @@ interface UploadFilesProps extends Omit<UploadProps, 'onChange'> {
  /** Custom file removal callback */
  onRemove?: (file: UploadFile) => boolean | void | Promise<boolean | void>;
 }
+
+const transform_file_type = {
+  'text/plain': 'document/text',
+  'text/markdown': 'document/markdown',
+  'text/x-markdown': 'document/x-markdown',
+
+  'application/pdf': 'document/pdf',
+
+  'application/msword': 'document/doc',
+  'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'document/docx',
+
+  'application/vnd.ms-powerpoint': 'document/ppt',
+  'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'document/pptx',
+}
 // Mapping of file extensions to MIME types
 const ALL_FILE_TYPE: {
  [key: string]: string;
 } = {
-  // txt: 'text/plain',
+  txt: 'text/plain',
+  md: 'text/markdown',
+  xmd: 'text/x-markdown',
+
  pdf: 'application/pdf',

  doc: 'application/msword',
  docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
-  
-  xls: 'application/vnd.ms-excel',
-  xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
-  csv: 'text/csv',

  ppt: 'application/vnd.ms-powerpoint',
  pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
-  
-  // md: 'text/markdown',
-  // htm: 'text/html',
-  // html: 'text/html',
-  // json: 'application/json',
+
  jpg: 'image/jpeg',
  jpeg: 'image/jpeg',
  png: 'image/png',
@@ -84,6 +94,23 @@ const ALL_FILE_TYPE: {
  bmp: 'image/bmp',
  webp: 'image/webp',
  svg: 'image/svg+xml',
+
+  mp4: 'video/mp4',
+  mov: 'video/quicktime',
+  avi: 'video/x-msvideo',
+  mkv: 'video/x-matroska',
+  webm: 'video/webm',
+  flv: 'video/x-flv',
+  wmv: 'video/x-ms-wmv',
+
+  mp3: 'audio/mpeg',
+  wav: 'audio/wav',
+  ogg: 'audio/ogg',
+  aac: 'audio/aac',
+  flac: 'audio/flac',
+  m4a: 'audio/mp4',
+  wma: 'audio/x-ms-wma',
+  xm4a: 'audio/x-m4a',
 }
 export interface UploadFilesRef {
  /** Current file list */
@@ -178,6 +205,10 @@ const UploadFiles = forwardRef<UploadFilesRef, UploadFilesProps>(({
   * Handles upload state changes
   */
  const handleChange: UploadProps['onChange'] = ({ fileList: newFileList }) => {
+    newFileList.map(file => {
+      const type = (file.type && transform_file_type[file.type as keyof typeof transform_file_type]) || file.type
+      file.type = type
+    })
    setFileList(newFileList);
    if (onChange) {
      onChange(maxCount === 1 ? newFileList[newFileList.length - 1] : newFileList);
--- a/web/src/views/Conversation/components/UploadFileListModal.tsx
+++ b/web/src/views/Conversation/components/UploadFileListModal.tsx
@@ -2,7 +2,7 @@
 * @Author: ZhaoYing 
 * @Date: 2026-02-06 21:09:47 
 * @Last Modified by: ZhaoYing
- * @Last Modified time: 2026-02-09 10:17:54
+ * @Last Modified time: 2026-03-04 17:47:09
 */
 /**
 * Upload File List Modal Component
@@ -104,7 +104,9 @@ const UploadFileListModal = forwardRef<UploadFileListModalRef, UploadFileListMod
                    <Select
                      placeholder={t('memoryConversation.fileType')}
                      options={[
-                        { label: t('memoryConversation.image'), value: 'image' }
+                        { label: t('memoryConversation.image'), value: 'image' },
+                        { label: t('memoryConversation.audio'), value: 'audio' },
+                        { label: t('memoryConversation.video'), value: 'video' },
                      ]}
                      className="rb:w-30"
                    />
--- a/web/src/views/Conversation/index.tsx
+++ b/web/src/views/Conversation/index.tsx
@@ -14,7 +14,7 @@ import { type FC, useState, useEffect, useRef } from 'react'
 import { useParams, useLocation } from 'react-router-dom'
 import { useTranslation } from 'react-i18next'
 import InfiniteScroll from 'react-infinite-scroll-component';
-import { Flex, Skeleton, Form, Dropdown, type MenuProps, App } from 'antd'
+import { Flex, Skeleton, Form, Dropdown, type MenuProps, App, Divider } from 'antd'
 import { SettingOutlined } from '@ant-design/icons'
 import clsx from 'clsx'
 import dayjs from 'dayjs'
@@ -35,7 +35,7 @@ import OnlineCheckedIcon from '@/assets/images/conversation/onlineChecked.svg'
 import MemoryFunctionCheckedIcon from '@/assets/images/conversation/memoryFunctionChecked.svg'
 import { type SSEMessage } from '@/utils/stream'
 import UploadFiles from './components/FileUpload'
-// import AudioRecorder from '@/components/AudioRecorder'
+import AudioRecorder from '@/components/AudioRecorder'
 import { shareFileUploadUrlWithoutApiPrefix } from '@/api/fileStorage'
 import UploadFileListModal from './components/UploadFileListModal'
 import type { VariableConfigModalRef } from '@/views/Workflow/types'
@@ -305,17 +305,27 @@ const Conversation: FC = () => {
      }),
      variables: params
    }, handleStreamMessage, shareToken)
+      .catch(() => {
+        setLoading(false)
+        setStreamLoading(false)
+      })
      .finally(() => {
        setLoading(false)
+        setStreamLoading(false)
      })
  }

  const fileChange = (file?: any) => {
    form.setFieldValue('files', [...(queryValues.files || []), file])
  }
-  // const handleRecordingComplete = async (file: any) => {
-  //   console.log('file', file)
-  // }
+  const handleRecordingComplete = async (file: any) => {
+    form.setFieldValue('files', [...(queryValues.files || []), {
+      uid: file.file_id,
+      response: { data: file },
+      thumbUrl: file.url,
+      type: file.type
+    }])
+  }

  const handleShowUpload: MenuProps['onClick'] = ({ key }) => {
    switch(key) {
@@ -329,6 +339,7 @@ const Conversation: FC = () => {
    form.setFieldValue('files', [...(queryValues.files || []), ...fileList])
  }
  const updateFileList = (fileList?: any[]) => {
+    console.log('fileList', fileList)
    form.setFieldValue('files', [...(fileList || [])])
  }

@@ -383,7 +394,7 @@ const Conversation: FC = () => {
        <div className='rb:w-190  rb:h-screen rb:mx-auto rb:pt-10'>
        <Chat
          empty={<Empty url={ChatEmpty} className="rb:h-full" size={[320,180]} title={t('memoryConversation.chatEmpty')} subTitle={t('memoryConversation.emptyDesc')} />}
-          contentClassName="rb:h-[calc(100%-180px)]"
+          contentClassName={!queryValues?.files?.length ? "rb:h-[calc(100%-144px)]" : "rb:h-[calc(100%-208px)]"}
          data={chatList}
          streamLoading={streamLoading}
          loading={loading}
@@ -405,13 +416,12 @@ const Conversation: FC = () => {
                          key: 'upload', label: (
                            <UploadFiles
                              action={shareFileUploadUrlWithoutApiPrefix}
-                              fileType={['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'svg']}
                              onChange={fileChange}
                              requestConfig={{
                                headers: {
                                  'Content-Type': 'multipart/form-data',
                                  Authorization: `Bearer ${shareToken || ''}`,
-                              } }}
+                              }}}
                            />
                          )
                        },
@@ -455,10 +465,19 @@ const Conversation: FC = () => {
                  </Form.Item>
                )}
              </Flex>
-              {/* <Flex align="center">
-                <AudioRecorder onRecordingComplete={handleRecordingComplete} />
+              <Flex align="center">
+                <AudioRecorder
+                  action={shareFileUploadUrlWithoutApiPrefix}
+                  requestConfig={{
+                    headers: {
+                      'Content-Type': 'multipart/form-data',
+                      Authorization: `Bearer ${shareToken || ''}`,
+                    }
+                  }}
+                  onRecordingComplete={handleRecordingComplete}
+                />
                <Divider type="vertical" className="rb:ml-1.5! rb:mr-3!" />
-              </Flex> */}
+              </Flex>
            </Flex>
          </Form>
        </Chat>