import { useMemo,useRef, useState, useEffect } from 'react'; import { Button, Flex, Radio, Steps, Modal, Input, Checkbox, Select, Form, Progress, App } from 'antd'; import { useTranslation } from 'react-i18next'; import { useLocation, useNavigate, useParams } from 'react-router-dom'; import './Private.css'; import Table, { type TableRef } from '@/components/Table' import type { AnyObject } from 'antd/es/_util/type'; import type { UploadFileResponse,KnowledgeBaseDocumentData } from '@/views/KnowledgeBase/types'; import type { ColumnsType } from 'antd/es/table'; import type { UploadFile } from 'antd'; import UploadFiles from '@/components/Upload/UploadFiles'; import type { UploadRequestOption } from 'rc-upload/lib/interface'; import { uploadFile, uploadQaFile, getDocumentList, parseDocument, updateDocument, deleteDocument, createDocumentAndUpload } from '@/api/knowledgeBase'; import exitIcon from '@/assets/images/knowledgeBase/exit.png'; import SliderInput from '@/components/SliderInput'; import DelimiterSelector from '../components/DelimiterSelector'; const { TextArea } = Input; const style: React.CSSProperties = { display: 'flex', gap: 16, }; const radioWrapperBaseStyle: React.CSSProperties = { display: 'flex', alignItems: 'flex-start', columnGap: 14, // Wider gap between dot and text width: '100%', border: '1px solid #E5E5E5', borderRadius: 12, padding: 16, }; const getActiveRadioStyle = (active: boolean): React.CSSProperties => ({ ...radioWrapperBaseStyle, border: active ? '1px solid #171719' : radioWrapperBaseStyle.border, backgroundColor: active ? '#FAFAFA' : 'transparent', }); type SourceType = 'local' | 'link' | 'text' | 'csv'; type ProcessingMethod = 'directBlock' | 'qaExtract'; type ParameterSettings = 'defaultSettings' | 'customSettings'; const stepKeys = ['selectFile', 'parameterSettings', 'dataPreview', 'confirmUpload'] as const; type StepKey = typeof stepKeys[number]; const stepIndexMap: Record = { selectFile: 0, parameterSettings: 1, dataPreview: 2, confirmUpload: 3, }; interface CreateDatasetLocationState { source?: SourceType; knowledgeBaseId?: string; parentId?: string; startStep?: StepKey; fileId?: string | string[]; fileIds?: string | string[]; } interface ContentFormData { title: string; content: string; } const fileType = ['pdf', 'doc', 'docx', 'xls', 'xlsx', 'csv', 'md', 'htm', 'html', 'json', 'ppt', 'pptx', 'txt', 'png', 'jpg', 'mp3', 'mp4', 'mov', 'wav'] const csvFileType = ['csv'] const CreateDataset = () => { const { t } = useTranslation(); const navigate = useNavigate(); const { modal, message: messageApi } = App.useApp() const { knowledgeBaseId: routeKnowledgeBaseId } = useParams<{ knowledgeBaseId: string }>(); const location = useLocation(); const locationState = (location.state ?? {}) as CreateDatasetLocationState; const source = (locationState.source ?? 'local') as SourceType; const knowledgeBaseId = locationState.knowledgeBaseId || routeKnowledgeBaseId; const parentId = locationState.parentId; const initialStepKey = locationState.startStep ?? 'selectFile'; const initialFileIds = (() => { const fileIds = locationState.fileIds || locationState.fileId; if (!fileIds) return []; return Array.isArray(fileIds) ? fileIds : [fileIds]; })(); const [current, setCurrent] = useState(stepIndexMap[initialStepKey]); const tableRef = useRef(null); const [form] = Form.useForm(); const [data, setData] = useState([]); const [rechunkFileIds, setRechunkFileIds] = useState(initialFileIds); const [textFormValid, setTextFormValid] = useState(false); const [pollingLoading, setPollingLoading] = useState(false); const pollingTimerRef = useRef | null>(null); const [delimiter, setDelimiter] = useState(undefined); const [blockSize, setBlockSize] = useState(130); const [qaPrompt, setQaPrompt] = useState() console.log('qaPrompt', qaPrompt) const [processingMethod, setProcessingMethod] = useState('directBlock'); const [parameterSettings, setParameterSettings] = useState('defaultSettings'); const [pdfEnhancementEnabled, setPdfEnhancementEnabled] = useState(true); const [pdfEnhancementMethod, setPdfEnhancementMethod] = useState('mineru'); const steps = useMemo( () => [ { title: t('knowledgeBase.selectFile') }, { title: t('knowledgeBase.parameterSettings') }, // { title: t('knowledgeBase.dataPreview') }, // Temporarily hide step 3 { title: t('knowledgeBase.confirmUpload') }, ], [t], ); // 存储每个文件的 AbortController,用于取消上传 const abortControllersRef = useRef>(new Map()); const uploadRef = useRef<{ fileList: UploadFile[]; clearFiles: () => void }>(null); console.log('Upload files', uploadRef.current?.fileList.length) const handleNext = async () => { // Temporarily hide step 3: adjust step index (0->1->2 corresponds to select file->parameter settings->confirm upload) let nextStep = current + 1; if (current === 0 && source === 'csv') { return } if((nextStep === 1 && source === 'local') || (nextStep === 2 && source === 'csv')) { // Check if files have been uploaded if (rechunkFileIds.length === 0) { // If no files, prompt user to upload first Modal.warning({ title: t('common.warning') || 'Warning', content: t('knowledgeBase.pleaseUploadFileFirst') || 'Please upload files first', }); return; // Don't proceed to next step } }else if(nextStep === 1 && source === 'text'){ try { const values = await form.validateFields(); // setLoading(true); // TODO: Need to call corresponding API to save content here const params = { // ...values, kb_id: knowledgeBaseId, parent_id: parentId, }; const response = await createDocumentAndUpload(values, params) if(response) { setRechunkFileIds([response.id]) } } catch (err) { messageApi.error(t('knowledgeBase.createContentError')); } finally { // setLoading(false); } } // 从参数设置进入确认上传时的处理 if(current === 1 && nextStep === 2) { // debugger // handlePreview(data[0],0) if(parameterSettings === 'customSettings' || processingMethod === 'qaExtract' || pdfEnhancementEnabled){ rechunkFileIds.map((id) => { const params = { progress: 0, parser_config: { layout_recognize: pdfEnhancementMethod || 'DeepDOC', delimiter: delimiter, chunk_token_num: blockSize, auto_questions: processingMethod === 'directBlock' ? 0 : 1, qa_prompt: qaPrompt } } updateDocument(id, params) }) } // Execute once immediately to load document list for preview (don't auto-return) pollDocumentStatus(false); } // Limit max step to 2 (confirm upload) setCurrent(Math.min(nextStep, 2)); }; const handlePrev = () => setCurrent((c) => Math.max(c - 1, 0)); // Start upload: trigger document parsing and start polling const handleStartUpload = () => { if (rechunkFileIds.length === 0) { Modal.warning({ title: t('common.warning') || 'Warning', content: t('knowledgeBase.pleaseUploadFileFirst') || 'Please upload files first', }); return; } // 显示确认弹框 modal.confirm({ title: t('knowledgeBase.startUploadConfirmTitle') || 'Start processing documents', content: t('knowledgeBase.startUploadConfirmContent') || 'Document processing will proceed in the background. You can choose to return to the list page immediately or stay on this page to view processing progress.', okText: t('knowledgeBase.returnToList') || 'Return to list', cancelText: t('knowledgeBase.stayOnPage') || 'Stay on this page', onOk: () => { // User chose to return to list - don't show loading, navigate directly startProcessing(true); }, onCancel: () => { // User chose to stay on current page - show loading and start polling console.log('User chose to stay, starting to show loading'); setPollingLoading(true); // Delay a bit to let user see loading effect, then start processing setTimeout(() => { startProcessing(false); }, 100); }, }); }; // Function to actually start processing const startProcessing = (autoReturnToList: boolean) => { // Trigger document parsing rechunkFileIds.map((id) => { parseDocument(id, {}); }); if (autoReturnToList) { // User chose to return immediately, navigate directly (no loading shown) console.log('User chose to return to list page immediately'); handleBack(); } else { // User chose to stay, start polling to view progress (loading already set in onCancel) console.log('User chose to stay and view progress'); // Execute polling once immediately (enable auto-return) pollDocumentStatus(true); // Then execute every 3 seconds (enable auto-return) pollingTimerRef.current = setInterval(() => { pollDocumentStatus(true); }, 3000); } }; const handleDelete = (record: AnyObject) => { modal.confirm({ title: t('common.deleteWarning'), content: t('common.deleteWarningContent', { content: record.name }), onOk: async () => { await deleteDocument(record.id); // 删除成功,从 rechunkFileIds 中移除该 id setRechunkFileIds((prev) => prev.filter((id) => id !== record.id)); // 刷新列表 messageApi.success(t('common.deleteSuccess')); tableRef.current?.loadData(); }, onCancel: () => { console.log('Delete cancelled'); }, }); } // Table column configuration const columns: ColumnsType = [ { title: t('knowledgeBase.name'), dataIndex: 'file_name', key: 'file_name' }, { title: t('knowledgeBase.status'), dataIndex: 'progress', key: 'progress', render: (value: number, record: any) => { // When value >= 1 it's complete, when 0~1 show progress bar if (value >= 1) { return ( {t('knowledgeBase.completed')} ); } else if (value >= 0 && value < 1) { // Processing, show progress bar return (
); } else { // value = 0 or other cases, show pending return ( {t('knowledgeBase.pending')} ); } } }, { title: t('common.operation'), key: 'action', render: (_, record) => ( ), }, ]; // Helper function to check media file duration const checkMediaDuration = (file: File): Promise => { return new Promise((resolve, reject) => { const url = URL.createObjectURL(file); const media = document.createElement(file.type.startsWith('video/') ? 'video' : 'audio'); media.onloadedmetadata = () => { URL.revokeObjectURL(url); resolve(media.duration); }; media.onerror = () => { URL.revokeObjectURL(url); reject(new Error(`${t('knowledgeBase.unableReadFile')}`)); }; media.src = url; }); }; // Upload file const handleUpload = async (options: UploadRequestOption) => { const { file, onSuccess, onError, onProgress, filename = 'file' } = options; // Create AbortController for cancelling upload const abortController = new AbortController(); const fileUid = (file as any).uid; abortControllersRef.current.set(fileUid, abortController); // Get file extension const fileExtension = (file as File).name.split('.').pop()?.toLowerCase(); const mediaExtensions = ['mp3', 'mp4', 'mov', 'wav']; // If media file, check size and duration if (fileExtension && mediaExtensions.includes(fileExtension)) { const fileSizeInMB = (file as File).size / (1024 * 1024); // 检查文件大小(50MB限制) if (fileSizeInMB > 100) { messageApi.error(`${t('knowledgeBase.sizeLimitError')}: ${fileSizeInMB.toFixed(2)}MB`); onError?.(new Error(`${t('knowledgeBase.fileSizeExceeds')}`)); abortControllersRef.current.delete(fileUid); return; } try { // Check media duration (150 second limit) const duration = await checkMediaDuration(file as File); if (duration > 150) { messageApi.error(`${t('knowledgeBase.fileDurationLimitError')}: ${Math.round(duration)}s`); onError?.(new Error(`${t('knowledgeBase.fileDurationExceeds')}`)); abortControllersRef.current.delete(fileUid); return; } } catch (error) { messageApi.error(`${t('knowledgeBase.unableReadFile')}`); onError?.(error as Error); abortControllersRef.current.delete(fileUid); return; } } const formData = new FormData(); formData.append(filename, file as File); if (knowledgeBaseId) { formData.append('kb_id', knowledgeBaseId); } if (parentId) { formData.append('parent_id', parentId); } if (source === 'csv') { uploadQaFile(formData, { kb_id: knowledgeBaseId, parent_id: parentId, signal: abortController.signal, }) .then((res: UploadFileResponse) => { // Upload successful, remove AbortController abortControllersRef.current.delete(fileUid); onSuccess?.(res, new XMLHttpRequest()); messageApi.success(t('knowledgeBase.uploadSuccess')) handleBack() }) .catch((error) => { // Remove AbortController abortControllersRef.current.delete(fileUid); // If user actively cancelled, don't show error message if (error.name === 'AbortError' || error.code === 'ERR_CANCELED') { console.log('Upload cancelled:', (file as File).name); return; } onError?.(error as Error); }); } else { uploadFile(formData, { kb_id: knowledgeBaseId, parent_id: parentId, signal: abortController.signal, onUploadProgress: (event) => { if (!event.total) return; const percent = Math.round((event.loaded / event.total) * 100); onProgress?.({ percent }, file); }, }) .then((res: UploadFileResponse) => { // Upload successful, remove AbortController abortControllersRef.current.delete(fileUid); onSuccess?.(res, new XMLHttpRequest()); if (res?.id) { setRechunkFileIds((prev) => { if (prev.includes(res.id)) return prev; const next = [...prev, res.id]; return next; }); } }) .catch((error) => { // Remove AbortController abortControllersRef.current.delete(fileUid); // If user actively cancelled, don't show error message if (error.name === 'AbortError' || error.code === 'ERR_CANCELED') { console.log('Upload cancelled:', (file as File).name); return; } onError?.(error as Error); }); } }; // 轮询检查文档处理状态 // autoReturn: whether to automatically return to list page when all documents are completed const pollDocumentStatus = (autoReturn: boolean = false) => { console.log('Start polling document status, current pollingLoading:', pollingLoading); if (!knowledgeBaseId || !parentId || rechunkFileIds.length === 0) { console.log('Polling conditions not met, exiting'); return; } // 获取文档列表检查是否全部完成,并刷新表格数据 getDocumentList(knowledgeBaseId, { document_ids: rechunkFileIds.join(','), }) .then((res: any) => { const documents = res.items || []; setData(documents); // 只在 confirmUpload 步骤刷新表格数据 if (current === 2) { tableRef.current?.loadData(); } console.log('documents', documents); // Check if all documents have progress of 1 const allCompleted = documents.every((doc: KnowledgeBaseDocumentData) => doc.progress === 1); console.log('Polling status:', allCompleted); // 检查是否所有文档都完成了 // debugger if (allCompleted) { // 清除定时器和 loading 状态 if (pollingTimerRef.current) { clearInterval(pollingTimerRef.current); pollingTimerRef.current = null; } // 延迟清除 loading,让用户看到完成状态 setTimeout(() => { setPollingLoading(false); }, 1000); // Only auto-return when autoReturn is true if (autoReturn) { // Delay 2 seconds before navigating to let user see completion status console.log('All documents processed, returning to list page in 2 seconds'); setTimeout(() => { handleBack(); }, 2000); } else { console.log('All documents processed, user can operate manually'); } } else { // If documents are still processing, keep loading state console.log('Documents still processing, maintaining loading state'); } }) .catch((error) => { console.error('Failed to poll document status:', error); setPollingLoading(false); }); }; const handleBack = () => { if (knowledgeBaseId) { navigate(`/knowledge-base/${knowledgeBaseId}/private`, { state: { refresh: true, timestamp: Date.now(), // 添加时间戳确保每次都是新的 state // 保持返回到原来的文档文件夹位置 navigateToDocumentFolder: parentId !== knowledgeBaseId ? parentId : undefined, }, }); } else { console.warn('Missing route parameters, unable to return'); } }; const handleChange = (value: number | null) =>{ if (value !== null) { setBlockSize(value); } } // 删除已上传的文件 const handleDeleteFile = async (fileId: string) => { try { await deleteDocument(fileId); // Delete successful, remove the id from rechunkFileIds setRechunkFileIds((prev) => prev.filter((id) => id !== fileId)); console.log(`${t('common.deleteSuccess')}`); } catch (error) { messageApi.error(`${t('common.deleteFailed')}`); } }; // When navigating from other pages with fileIds, load corresponding document data // useEffect(() => { // if (initialFileIds.length > 0 && initialStepKey !== 'selectFile' && knowledgeBaseId && parentId) { // // Load document list data // getDocumentList(knowledgeBaseId,{ // document_ids: initialFileIds.join(','), // }) // .then((res: any) => { // const documents = res.items || []; // setData(documents); // }) // .catch((error) => { // console.error('Failed to load document list:', error); // }); // } // }, []); // Cleanup function: clear timer and loading state when component unmounts useEffect(() => { return () => { if (pollingTimerRef.current) { clearInterval(pollingTimerRef.current); pollingTimerRef.current = null; } setPollingLoading(false); }; }, []); // Watch for route changes, ensure state is cleaned up when page switches useEffect(() => { return () => { // Clean up state when page unmounts if (pollingTimerRef.current) { clearInterval(pollingTimerRef.current); pollingTimerRef.current = null; } setPollingLoading(false); }; }, [location.pathname]); return (<>
{/* {t('knowledgeBase.createA') + ' ' + t('knowledgeBase.dataset')} */}
exit {t('common.exit')}
{source !== 'csv' &&
}
{current === 0 && (
{source && (source === 'local' || source === 'csv') && ( { console.log('File list changed:', fileList); }} onRemove={async (file) => { // 如果文件正在上传,取消上传 const fileUid = file.uid; const abortController = abortControllersRef.current.get(fileUid); if (abortController) { abortController.abort(); abortControllersRef.current.delete(fileUid); console.log('Upload cancelled:', (file as any).name); // 取消上传后直接返回 true,允许移除文件 return true; } // Only delete server file when file upload was successful (has response.id) if (file.response?.id) { try { await deleteDocument(file.response.id); setRechunkFileIds(prev => prev.filter(id => id !== file.response.id)); console.log('Server file deleted:', file.response.id); return true; } catch (error) { console.error('Failed to delete file:', error); messageApi.error(t('common.deleteFailed') || 'Failed to delete file'); return false; // Don't remove file when deletion fails } } // Also allow removal in other cases (such as failed uploads) return true; }} /> )} {source && source === 'link' && (
{t('knowledgeBase.webLink')}