Merge branch 'feature/knowledge_lxc' into release/v0.2.4

api/app/controllers/knowledge_controller.py

@@ -509,7 +509,7 @@ async def check_yuque_auth(
         async with api_client as client:
             repos = await client.get_user_repos()
             if repos:
-                return success(data=repos, msg="Successfully auth yuque info")
+                return success(msg="Successfully auth yuque info")
         return fail(BizCode.UNAUTHORIZED, msg="auth yuque info failed", error="user_id or token is incorrect")
     except HTTPException:
         raise
@@ -519,7 +519,7 @@ async def check_yuque_auth(
 
 
 @router.get("/check/feishu/auth", response_model=ApiResponse)
-async def check_yuque_auth(
+async def check_feishu_auth(
         feishu_app_id: str,
         feishu_app_secret: str,
         feishu_folder_token: str,
@@ -539,7 +539,7 @@ async def check_yuque_auth(
         async with api_client as client:
             files = await client.list_all_folder_files(feishu_folder_token, recursive=True)
             if files:
-                return success(data=files, msg="Successfully auth feishu info")
+                return success(msg="Successfully auth feishu info")
         return fail(BizCode.UNAUTHORIZED, msg="auth feishu info failed", error="app_id or app_secret or feishu_folder_token is incorrect")
     except HTTPException:
         raise

api/app/controllers/service/rag_api_knowledge_controller.py

@@ -246,3 +246,73 @@ async def rebuild_knowledge_graph(
                                                               db=db,
                                                               current_user=current_user)
 
+
+@router.get("/check/yuque/auth", response_model=ApiResponse)
+@require_api_key(scopes=["rag"])
+async def check_yuque_auth(
+    yuque_user_id: str,
+    yuque_token: str,
+    request: Request,
+    api_key_auth: ApiKeyAuth = None,
+    db: Session = Depends(get_db),
+):
+    """
+    check yuque auth info
+    """
+    api_key = api_key_service.ApiKeyService.get_api_key(db, api_key_auth.api_key_id, api_key_auth.workspace_id)
+    current_user = api_key.creator
+    current_user.current_workspace_id = api_key_auth.workspace_id
+
+    api_logger.info(f"check yuque auth info, username: {current_user.username}")
+
+    return await knowledge_controller.check_yuque_auth(yuque_user_id=yuque_user_id,
+                                                       yuque_token=yuque_token,
+                                                       db=db,
+                                                       current_user=current_user)
+
+
+@router.get("/check/feishu/auth", response_model=ApiResponse)
+@require_api_key(scopes=["rag"])
+async def check_feishu_auth(
+    feishu_app_id: str,
+    feishu_app_secret: str,
+    feishu_folder_token: str,
+    request: Request,
+    api_key_auth: ApiKeyAuth = None,
+    db: Session = Depends(get_db),
+):
+    """
+    check feishu auth info
+    """
+    api_key = api_key_service.ApiKeyService.get_api_key(db, api_key_auth.api_key_id, api_key_auth.workspace_id)
+    current_user = api_key.creator
+    current_user.current_workspace_id = api_key_auth.workspace_id
+
+    api_logger.info(f"check feishu auth info, username: {current_user.username}")
+
+    return await knowledge_controller.check_feishu_auth(feishu_app_id=feishu_app_id,
+                                                        feishu_app_secret=feishu_app_secret,
+                                                        feishu_folder_token=feishu_folder_token,
+                                                        db=db,
+                                                        current_user=current_user)
+
+
+@router.post("/{knowledge_id}/sync", response_model=ApiResponse)
+@require_api_key(scopes=["rag"])
+async def sync_knowledge(
+    knowledge_id: uuid.UUID,
+    request: Request,
+    api_key_auth: ApiKeyAuth = None,
+    db: Session = Depends(get_db),
+):
+    """
+    sync knowledge base information based on knowledge_id
+    """
+    api_key = api_key_service.ApiKeyService.get_api_key(db, api_key_auth.api_key_id, api_key_auth.workspace_id)
+    current_user = api_key.creator
+    current_user.current_workspace_id = api_key_auth.workspace_id
+
+    return await knowledge_controller.sync_knowledge(knowledge_id=knowledge_id,
+                                                     db=db,
+                                                     current_user=current_user)
+

api/app/core/rag/app/naive.py

@@ -670,7 +670,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
             with open(filename, "rb") as f:
                 binary = f.read()
         excel_parser = ExcelParser()
-        if parser_config.get("html4excel"):
+        if parser_config.get("html4excel") and parser_config.get("html4excel").lower() == "true":
             sections = [(_, "") for _ in excel_parser.html(binary, 12) if _]
             parser_config["chunk_token_num"] = 0
         else:

api/app/models/document_model.py

@@ -19,7 +19,7 @@ class Document(Base):
     parser_config = Column(JSON, nullable=False,
                            default={
                                "layout_recognize": "DeepDOC",
-                               "chunk_token_num": 128,
+                               "chunk_token_num": 130,
                                "delimiter": "\n",
                                "auto_keywords": 0,
                                "auto_questions": 0,

api/app/tasks.py

@@ -441,7 +441,7 @@ def sync_knowledge_for_kb(kb_id: uuid.UUID):
                                     db.refresh(db_file)
                                     # Construct a save path：/files/{kb_id}/{parent_id}/{file.id}{file_extension}
                                     save_dir = os.path.join(settings.FILE_PATH, str(db_knowledge.id),
-                                                            str(db_knowledge.parent_id))
+                                                            str(db_knowledge.id))
                                     Path(save_dir).mkdir(parents=True,
                                                          exist_ok=True)  # Ensure that the directory exists
                                     save_path = os.path.join(save_dir, f"{db_file.id}{db_file.file_ext}")
@@ -498,7 +498,7 @@ def sync_knowledge_for_kb(kb_id: uuid.UUID):
                                     parser_id="naive",
                                     parser_config={
                                         "layout_recognize": "DeepDOC",
-                                        "chunk_token_num": 128,
+                                        "chunk_token_num": 130,
                                         "delimiter": "\n",
                                         "auto_keywords": 0,
                                         "auto_questions": 0,
@@ -576,7 +576,7 @@ def sync_knowledge_for_kb(kb_id: uuid.UUID):
                                     # 1. update file
                                     # Construct a save path：/files/{kb_id}/{parent_id}/{file.id}{file_extension}
                                     save_dir = os.path.join(settings.FILE_PATH, str(db_knowledge.id),
-                                                            str(db_knowledge.parent_id))
+                                                            str(db_knowledge.id))
                                     Path(save_dir).mkdir(parents=True,
                                                          exist_ok=True)  # Ensure that the directory exists
 
@@ -621,7 +621,7 @@ def sync_knowledge_for_kb(kb_id: uuid.UUID):
                                 # 1. update file
                                 # Construct a save path：/files/{kb_id}/{parent_id}/{file.id}{file_extension}
                                 save_dir = os.path.join(settings.FILE_PATH, str(db_knowledge.id),
-                                                        str(db_knowledge.parent_id))
+                                                        str(db_knowledge.id))
                                 Path(save_dir).mkdir(parents=True, exist_ok=True)  # Ensure that the directory exists
 
                                 # download document from Feishu FileInfo
@@ -667,7 +667,7 @@ def sync_knowledge_for_kb(kb_id: uuid.UUID):
                                     parser_id="naive",
                                     parser_config={
                                         "layout_recognize": "DeepDOC",
-                                        "chunk_token_num": 128,
+                                        "chunk_token_num": 130,
                                         "delimiter": "\n",
                                         "auto_keywords": 0,
                                         "auto_questions": 0,
@@ -738,7 +738,7 @@ def sync_knowledge_for_kb(kb_id: uuid.UUID):
                                     # 1. update file
                                     # Construct a save path：/files/{kb_id}/{parent_id}/{file.id}{file_extension}
                                     save_dir = os.path.join(settings.FILE_PATH, str(db_knowledge.id),
-                                                            str(db_knowledge.parent_id))
+                                                            str(db_knowledge.id))
                                     Path(save_dir).mkdir(parents=True,
                                                          exist_ok=True)  # Ensure that the directory exists
 
@@ -783,7 +783,7 @@ def sync_knowledge_for_kb(kb_id: uuid.UUID):
                                 # 1. update file
                                 # Construct a save path：/files/{kb_id}/{parent_id}/{file.id}{file_extension}
                                 save_dir = os.path.join(settings.FILE_PATH, str(db_knowledge.id),
-                                                        str(db_knowledge.parent_id))
+                                                        str(db_knowledge.id))
                                 Path(save_dir).mkdir(parents=True, exist_ok=True)  # Ensure that the directory exists
 
                                 # download document from Feishu FileInfo
@@ -829,7 +829,7 @@ def sync_knowledge_for_kb(kb_id: uuid.UUID):
                                     parser_id="naive",
                                     parser_config={
                                         "layout_recognize": "DeepDOC",
-                                        "chunk_token_num": 128,
+                                        "chunk_token_num": 130,
                                         "delimiter": "\n",
                                         "auto_keywords": 0,
                                         "auto_questions": 0,