From dfcc85a466d76d8a74f803d26d430f42300e52f7 Mon Sep 17 00:00:00 2001 From: Timebomb2018 <18868801967@163.com> Date: Tue, 17 Mar 2026 14:58:28 +0800 Subject: [PATCH 01/23] fix(app): Experience sharing: Adding 'features' to agent_config parameters --- api/app/services/multimodal_service.py | 18 +++++++++--------- api/app/utils/app_config_utils.py | 3 ++- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/api/app/services/multimodal_service.py b/api/app/services/multimodal_service.py index b30b48b2..208f6ec0 100644 --- a/api/app/services/multimodal_service.py +++ b/api/app/services/multimodal_service.py @@ -425,8 +425,8 @@ class MultimodalService: Dict: 根据 provider 返回不同格式的图片内容 """ try: - url = await self.get_file_url(file) - return await strategy.format_image(url, content=file.get_content()) + # url = await self.get_file_url(file) + return await strategy.format_image(file.url, content=file.get_content()) except Exception as e: logger.error(f"处理图片失败: {e}", exc_info=True) return { @@ -476,20 +476,20 @@ class MultimodalService: Dict: 根据 provider 返回不同格式的音频内容 """ try: - url = await self.get_file_url(file) + # url = await self.get_file_url(file) # 如果启用音频转文本且有 API Key transcription = None if self.enable_audio_transcription and self.audio_api_key: - logger.info(f"开始音频转文本: {url}") + logger.info(f"开始音频转文本: {file.url}") if self.provider == "dashscope": - transcription = await AudioTranscriptionService.transcribe_dashscope(url, self.audio_api_key) + transcription = await AudioTranscriptionService.transcribe_dashscope(file.url, self.audio_api_key) elif self.provider == "openai": - transcription = await AudioTranscriptionService.transcribe_openai(url, self.audio_api_key) + transcription = await AudioTranscriptionService.transcribe_openai(file.url, self.audio_api_key) else: logger.warning(f"Provider {self.provider} 不支持音频转文本") - return await strategy.format_audio(file.file_type, url, file.get_content(), transcription) + return await strategy.format_audio(file.file_type, file.url, file.get_content(), transcription) except Exception as e: logger.error(f"处理音频失败: {e}", exc_info=True) return { @@ -509,8 +509,8 @@ class MultimodalService: Dict: 根据 provider 返回不同格式的视频内容 """ try: - url = await self.get_file_url(file) - return await strategy.format_video(url) + # url = await self.get_file_url(file) + return await strategy.format_video(file.url) except Exception as e: logger.error(f"处理视频失败: {e}", exc_info=True) return { diff --git a/api/app/utils/app_config_utils.py b/api/app/utils/app_config_utils.py index afa18417..bc03bb28 100644 --- a/api/app/utils/app_config_utils.py +++ b/api/app/utils/app_config_utils.py @@ -100,7 +100,8 @@ def agent_config_4_app_release(release: AppRelease) -> AgentConfig: memory=config_dict.get("memory"), variables=config_dict.get("variables", []), tools=config_dict.get("tools", []), - skills=config_dict.get("skills", {}) + skills=config_dict.get("skills", {}), + features=config_dict.get("features", {}) ) return agent_config From 130490c0225c072b93ba418fd1805530a9356402 Mon Sep 17 00:00:00 2001 From: zhaoying Date: Tue, 17 Mar 2026 15:55:04 +0800 Subject: [PATCH 02/23] feat(web): app support features --- web/src/components/AudioRecorder/index.tsx | 10 +- web/src/components/ButtonCheckbox/index.tsx | 6 +- web/src/components/Chat/ChatContent.tsx | 40 +- web/src/components/Chat/ChatToolbar.tsx | 205 ++++++++++ web/src/components/Chat/types.ts | 3 +- web/src/i18n/en.ts | 13 +- web/src/i18n/zh.ts | 13 +- web/src/views/ApplicationConfig/Agent.tsx | 18 +- web/src/views/ApplicationConfig/Cluster.tsx | 18 +- .../ApplicationConfig/TestChat/index.tsx | 369 +++++------------- .../ApplicationConfig/components/Chat.tsx | 193 ++++----- .../components/ConfigHeader.tsx | 16 +- .../FeaturesConfig/FeaturesConfigModal.tsx | 151 +++++++ .../FeaturesConfig/FileUploadSettingModal.tsx | 180 +++++++++ .../{FunConfig => FeaturesConfig}/index.tsx | 29 +- .../FunConfig/FileUploadSettingModal.tsx | 182 --------- .../components/FunConfig/FunConfigModal.tsx | 140 ------- .../components/ToolList/ToolList.tsx | 22 +- .../components/ToolList/types.ts | 5 +- web/src/views/ApplicationConfig/types.ts | 66 +++- .../Conversation/components/FileUpload.tsx | 44 ++- .../components/UploadFileListModal.tsx | 29 +- web/src/views/Conversation/index.tsx | 328 ++++++---------- .../views/Workflow/components/Chat/Chat.tsx | 165 +++----- web/src/views/Workflow/index.tsx | 3 +- web/src/views/Workflow/types.ts | 4 +- 26 files changed, 1106 insertions(+), 1146 deletions(-) create mode 100644 web/src/components/Chat/ChatToolbar.tsx create mode 100644 web/src/views/ApplicationConfig/components/FeaturesConfig/FeaturesConfigModal.tsx create mode 100644 web/src/views/ApplicationConfig/components/FeaturesConfig/FileUploadSettingModal.tsx rename web/src/views/ApplicationConfig/components/{FunConfig => FeaturesConfig}/index.tsx (54%) delete mode 100644 web/src/views/ApplicationConfig/components/FunConfig/FileUploadSettingModal.tsx delete mode 100644 web/src/views/ApplicationConfig/components/FunConfig/FunConfigModal.tsx diff --git a/web/src/components/AudioRecorder/index.tsx b/web/src/components/AudioRecorder/index.tsx index 10b8eca9..b3b87130 100644 --- a/web/src/components/AudioRecorder/index.tsx +++ b/web/src/components/AudioRecorder/index.tsx @@ -2,7 +2,7 @@ * @Author: ZhaoYing * @Date: 2026-02-06 21:11:51 * @Last Modified by: ZhaoYing - * @Last Modified time: 2026-03-13 17:11:14 + * @Last Modified time: 2026-03-16 18:06:00 */ import { type FC, useRef, useState } from 'react' import RecordRTC from 'recordrtc' @@ -19,13 +19,15 @@ interface AudioRecorderProps { action?: string; /** Additional config passed to the upload request */ requestConfig?: Record; + disabled?: boolean; } const AudioRecorder: FC = ({ onRecordingComplete, className = '', action = fileUploadUrlWithoutApiPrefix, - requestConfig = {} + requestConfig = {}, + disabled = false }) => { // Whether the recorder is currently capturing audio const [isRecording, setIsRecording] = useState(false) @@ -34,6 +36,7 @@ const AudioRecorder: FC = ({ /** Request microphone access and start recording */ const startRecording = async () => { + if (disabled) return try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }) recorderRef.current = new RecordRTC(stream, { @@ -49,6 +52,7 @@ const AudioRecorder: FC = ({ /** Stop recording, upload the audio blob, then invoke the completion callback */ const stopRecording = () => { + if (disabled) return if (recorderRef.current) { recorderRef.current.stopRecording(() => { const blob = recorderRef.current!.getBlob() @@ -76,7 +80,7 @@ const AudioRecorder: FC = ({ // swap background image to reflect current state return (
= ({ align="center" justify={cicle ? 'center' : 'start'} gap={4} - className={clsx("rb:flex rb:items-center rb:cursor-pointer rb:border rb:hover:bg-[#F6F6F6]", { + className={clsx("rb:flex rb:items-center rb:cursor-pointer rb:px-2! rb:border rb:hover:bg-[#F6F6F6]", { 'rb:size-7 rb:rounded-[14px] rb:border-[0.5px] rb:border-[#EBEBEB]': cicle, - 'rb:rounded-lg rb:px-2 rb:text-[12px] rb:h-6': !cicle, + 'rb:rounded-lg rb:text-[12px] rb:h-6': !cicle, // Checked state: blue background and border "rb:bg-[rgba(21,94,239,0.06)] rb:border-[rgba(21,94,239,0.25)] rb:hover:bg-[rgba(21,94,239,0.06)] rb:text-[#155EEF]": checked, // Unchecked state: gray border and dark text diff --git a/web/src/components/Chat/ChatContent.tsx b/web/src/components/Chat/ChatContent.tsx index c1f5223c..15dcd496 100644 --- a/web/src/components/Chat/ChatContent.tsx +++ b/web/src/components/Chat/ChatContent.tsx @@ -2,13 +2,14 @@ * @Author: ZhaoYing * @Date: 2025-12-10 16:46:17 * @Last Modified by: ZhaoYing - * @Last Modified time: 2026-02-06 21:05:52 + * @Last Modified time: 2026-03-17 14:11:24 */ -import { type FC, useRef, useEffect } from 'react' +import { type FC, useRef, useEffect, useState } from 'react' import clsx from 'clsx' import Markdown from '@/components/Markdown' import type { ChatContentProps } from './types' -import { Spin } from 'antd' +import { Spin, Divider, Space } from 'antd' +import { SoundOutlined } from '@ant-design/icons' /** * Chat Content Display Component @@ -28,7 +29,25 @@ const ChatContent: FC = ({ // Scroll container reference for controlling auto-scroll to bottom const scrollContainerRef = useRef<(HTMLDivElement | null)>(null) const prevDataLengthRef = useRef(data.length); - const isScrolledToBottomRef = useRef(true); // Track if user is scrolled to bottom + const isScrolledToBottomRef = useRef(true); + const audioRef = useRef(null) + const [playingIndex, setPlayingIndex] = useState(null) + + const handlePlay = (index: number, audioUrl: string) => { + if (playingIndex === index) { + audioRef.current?.pause() + setPlayingIndex(null) + return + } + if (audioRef.current) { + audioRef.current.pause() + } + const audio = new Audio(audioUrl) + audioRef.current = audio + audio.play() + setPlayingIndex(index) + audio.onended = () => setPlayingIndex(null) + } // Track scroll position to determine if user is at bottom useEffect(() => { @@ -101,6 +120,19 @@ const ChatContent: FC = ({ {item.subContent && renderRuntime && renderRuntime(item, index)} {/* Render message content using Markdown component */} + + {item.audioUrl && <> + + + {playingIndex !== index + ? handlePlay(index, item.audioUrl!)} /> + :
handlePlay(index, item.audioUrl!)} + /> + } + + }
{/* Bottom label (such as timestamp, username, etc.) */} {labelPosition === 'bottom' && diff --git a/web/src/components/Chat/ChatToolbar.tsx b/web/src/components/Chat/ChatToolbar.tsx new file mode 100644 index 00000000..1d368c30 --- /dev/null +++ b/web/src/components/Chat/ChatToolbar.tsx @@ -0,0 +1,205 @@ +/* + * @Author: ZhaoYing + * @Date: 2026-03-17 14:22:25 + * @Last Modified by: ZhaoYing + * @Last Modified time: 2026-03-17 14:22:25 + */ +// Toolbar component for chat input area, supporting file upload, audio recording, and variable configuration +import { useRef, forwardRef, useImperativeHandle, type ReactNode, useEffect } from 'react' +import { Flex, Dropdown, Divider, App, Form, type MenuProps } from 'antd' +import { SettingOutlined } from '@ant-design/icons' +import { useTranslation } from 'react-i18next' +import clsx from 'clsx' + +import AudioRecorder from '@/components/AudioRecorder' +import UploadFiles from '@/views/Conversation/components/FileUpload' +import UploadFileListModal from '@/views/Conversation/components/UploadFileListModal' +import VariableConfigModal from '@/views/Workflow/components/Chat/VariableConfigModal' +import type { FeaturesConfigForm } from '@/views/ApplicationConfig/types' +import type { UploadFileListModalRef } from '@/views/Conversation/types' +import type { VariableConfigModalRef } from '@/views/Workflow/types' +import type { Variable } from '@/views/Workflow/components/Properties/VariableList/types' + +// Exposed methods via ref for parent components to access/set form state +export interface ChatToolbarRef { + getFiles: () => any[] + getVariables: () => Variable[] + setFiles: (files: any[]) => void + setVariables: (variables: Variable[]) => void +} + +// Props for configuring toolbar features, upload settings, and event callbacks +export interface ChatToolbarProps { + features: FeaturesConfigForm + extra?: ReactNode + uploadAction?: string + uploadRequestConfig?: { + data?: Record + headers?: Record + } + onFilesChange?: (files: any[]) => void + onVariablesChange?: (variables: Variable[]) => void + onRecordingComplete?: (file: any) => void; + defaultValue?: { memory: boolean } +} + +interface FormValues { + files: any[] + variables: Variable[]; + memory?: boolean; +} + +const ChatToolbar = forwardRef(({ + features, + extra, + uploadAction, + uploadRequestConfig, + onFilesChange, + onVariablesChange, + onRecordingComplete, + defaultValue, +}, ref) => { + const { t } = useTranslation() + const { message: messageApi } = App.useApp() + const uploadFileListModalRef = useRef(null) + const variableConfigModalRef = useRef(null) + const [form] = Form.useForm() + const queryValues = Form.useWatch([], form) + + useEffect(() => { + if (!defaultValue) return + form.setFieldsValue(defaultValue) + }, [defaultValue]) + + useImperativeHandle(ref, () => ({ + getFiles: () => form.getFieldValue('files') || [], + getVariables: () => form.getFieldValue('variables') || [], + setFiles: (files) => form.setFieldValue('files', files), + setVariables: (variables) => { + console.log('variables', variables) + form.setFieldValue('variables', variables) + }, + })) + + const { file_upload } = features || {} + + // Append newly uploaded file to the file list when upload is complete + const fileChange = (file?: any) => { + if (file?.status !== 'done') return + const files = [...(queryValues?.files || []), file] + form.setFieldValue('files', files) + onFilesChange?.(files) + } + + // Append recorded audio file to the file list and notify parent + const handleRecordingComplete = (file: any) => { + const files = [...(queryValues?.files || []), file] + form.setFieldValue('files', files) + onFilesChange?.(files) + onRecordingComplete?.(file) + } + + // Merge a batch of files (e.g. from remote URL modal) into the file list + const addFileList = (list?: any[]) => { + if (!list?.length) return + const files = [...(queryValues?.files || []), ...list] + form.setFieldValue('files', files) + onFilesChange?.(files) + } + + // Persist variable values from the config modal and notify parent + const handleVariablesSave = (values: Variable[]) => { + form.setFieldValue('variables', values) + onVariablesChange?.(values) + } + + // True when any required variable is missing a value, used to highlight the config button + const isNeedVariableConfig = queryValues?.variables?.some( + vo => vo.required && (vo.value === null || vo.value === undefined || vo.value === '') + ) + + // Build dropdown menu items based on allowed transfer methods + const fileMenus: MenuProps['items'] = [] + if (file_upload?.allowed_transfer_methods?.includes('remote_url')) { + fileMenus.push({ + key: 'url', + label: t('memoryConversation.addRemoteFile'), + onClick: () => { + if ((queryValues?.files?.length || 0) >= file_upload.max_file_count) { + messageApi.warning(t('common.fileNumTip', { num: file_upload.max_file_count })) + return + } + uploadFileListModalRef.current?.handleOpen() + } + }) + } + const enabledTypes = ['image', 'document', 'video', 'audio'].filter( + type => file_upload?.[`${type}_enabled` as keyof FeaturesConfigForm['file_upload']] + ) + if (file_upload?.allowed_transfer_methods?.includes('local_file') && enabledTypes.length > 0) { + fileMenus.push({ + key: 'upload', + label: ( + = file_upload.max_file_count} + /> + ) + }) + } + + console.log('queryValues', queryValues) + + return ( +
+ + +