adai/MemoryBear
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'refs/heads/release/v0.3.1' into fix/Timebomb_031

Browse Source
This commit is contained in:
Timebomb2018
2026-04-22 14:13:04 +08:00
parent 21b25bfef7 a61acbef93
commit 398aebd0c5
28 changed files with 403 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
api/app/celery_worker.py
View File

@@ -2,6 +2,8 @@
Celery Worker 入口点
用于启动 Celery Worker: celery -A app.celery_worker worker --loglevel=info
"""
from celery.signals import worker_process_init
from app.celery_app import celery_app
from app.core.logging_config import LoggingConfig, get_logger
@@ -13,4 +15,39 @@ logger.info("Celery worker logging initialized")
# 导入任务模块以注册任务
import app.tasks
@worker_process_init.connect
def _reinit_db_pool(**kwargs):
"""
prefork 子进程启动时重建被 fork 污染的资源。
fork() 后子进程继承了父进程的：
1. SQLAlchemy 连接池 — 多进程共享 TCP socket 导致 DB 连接损坏
2. ThreadPoolExecutor — fork 后线程状态不确定，第二个任务会死锁
"""
# 重建 DB 连接池
from app.db import engine
engine.dispose()
logger.info("DB connection pool disposed for forked worker process")
# 重建模块级 ThreadPoolExecutor（fork 后线程池不可用）
try:
from app.core.rag.deepdoc.parser import figure_parser
from concurrent.futures import ThreadPoolExecutor
figure_parser.shared_executor = ThreadPoolExecutor(max_workers=10)
logger.info("figure_parser.shared_executor recreated")
except Exception as e:
logger.warning(f"Failed to recreate figure_parser.shared_executor: {e}")
try:
from app.core.rag.utils import libre_office
from concurrent.futures import ThreadPoolExecutor
import os
max_workers = os.cpu_count() * 2 if os.cpu_count() else 4
libre_office.executor = ThreadPoolExecutor(max_workers=max_workers)
logger.info("libre_office.executor recreated")
except Exception as e:
logger.warning(f"Failed to recreate libre_office.executor: {e}")
__all__ = ['celery_app']

2
api/app/controllers/api_key_controller.py
View File

@@ -167,6 +167,8 @@ def update_api_key(
return success(data=api_key_schema.ApiKey.model_validate(api_key), msg="API Key 更新成功")
except BusinessException:
raise
except Exception as e:
logger.error(f"未知错误: {str(e)}", extra={
"api_key_id": str(api_key_id),

6
api/app/controllers/app_controller.py
View File

@@ -219,6 +219,7 @@ def delete_app(
@router.post("/{app_id}/copy", summary="复制应用")
@cur_workspace_access_guard()
@check_app_quota
def copy_app(
app_id: uuid.UUID,
new_name: Optional[str] = None,
@@ -1144,6 +1145,7 @@ async def import_workflow_config(
@router.post("/workflow/import/save")
@cur_workspace_access_guard()
@check_app_quota
async def save_workflow_import(
data: WorkflowImportSave,
db: Session = Depends(get_db),
@@ -1281,6 +1283,10 @@ async def import_app(
return fail(msg="YAML 格式无效，缺少 app 字段", code=BizCode.BAD_REQUEST)
target_app_id = uuid.UUID(app_id) if app_id else None
# 仅新建应用时检查配额，覆盖已有应用时跳过
if target_app_id is None:
from app.core.quota_manager import _check_quota
_check_quota(db, current_user.tenant_id, "app_quota", "app")
result_app, warnings = AppDslService(db).import_dsl(
dsl=dsl,
workspace_id=current_user.current_workspace_id,

2
api/app/controllers/chunk_controller.py
View File

@@ -457,7 +457,7 @@ async def retrieve_chunks(
if doc.metadata["doc_id"] not in seen_ids:
seen_ids.add(doc.metadata["doc_id"])
unique_rs.append(doc)
rs = vector_service.rerank(query=retrieve_data.query, docs=unique_rs, top_k=retrieve_data.top_k)
rs = vector_service.rerank(query=retrieve_data.query, docs=unique_rs, top_k=retrieve_data.top_k) if unique_rs else []
if retrieve_data.retrieve_type == chunk_schema.RetrieveType.Graph:
kb_ids = [str(kb_id) for kb_id in private_kb_ids]
workspace_ids = [str(workspace_id) for workspace_id in private_workspace_ids]

26
api/app/controllers/public_share_controller.py
View File

@@ -219,9 +219,20 @@ def list_conversations(
end_user_repo = EndUserRepository(db)
app_service = AppService(db)
app = app_service._get_app_or_404(share.app_id)
workspace_id = app.workspace_id
# 仅在新建终端用户时检查配额
existing_end_user = end_user_repo.get_end_user_by_other_id(workspace_id=workspace_id, other_id=other_id)
if existing_end_user is None:
from app.core.quota_manager import _check_quota
from app.models.workspace_model import Workspace
ws = db.query(Workspace).filter(Workspace.id == workspace_id).first()
if ws:
_check_quota(db, ws.tenant_id, "end_user_quota", "end_user")
new_end_user = end_user_repo.get_or_create_end_user(
app_id=share.app_id,
workspace_id=app.workspace_id,
workspace_id=workspace_id,
other_id=other_id
)
logger.debug(new_end_user.id)
@@ -309,7 +320,6 @@ def get_conversation(
"/chat",
summary="发送消息（支持流式和非流式）"
)
@check_end_user_quota
async def chat(
payload: conversation_schema.ChatRequest,
share_data: ShareTokenData = Depends(get_share_user_id),
@@ -350,6 +360,18 @@ async def chat(
app_service = AppService(db)
app = app_service._get_app_or_404(share.app_id)
workspace_id = app.workspace_id
# 仅在新建终端用户时检查配额，已有用户复用不受限制
existing_end_user = end_user_repo.get_end_user_by_other_id(workspace_id=workspace_id, other_id=other_id)
logger.info(f"终端用户配额检查: workspace_id={workspace_id}, other_id={other_id}, existing={existing_end_user is not None}")
if existing_end_user is None:
from app.core.quota_manager import _check_quota
from app.models.workspace_model import Workspace
ws = db.query(Workspace).filter(Workspace.id == workspace_id).first()
if ws:
logger.info(f"新终端用户，执行配额检查: tenant_id={ws.tenant_id}")
_check_quota(db, ws.tenant_id, "end_user_quota", "end_user")
new_end_user = end_user_repo.get_or_create_end_user(
app_id=share.app_id,
workspace_id=workspace_id,

10
api/app/controllers/service/app_api_controller.py
View File

@@ -106,6 +106,16 @@ async def chat(
other_id = payload.user_id
workspace_id = api_key_auth.workspace_id
end_user_repo = EndUserRepository(db)
# 仅在新建终端用户时检查配额，已有用户复用不受限制
existing_end_user = end_user_repo.get_end_user_by_other_id(workspace_id=workspace_id, other_id=other_id)
if existing_end_user is None:
from app.core.quota_manager import _check_quota
from app.models.workspace_model import Workspace
ws = db.query(Workspace).filter(Workspace.id == workspace_id).first()
if ws:
_check_quota(db, ws.tenant_id, "end_user_quota", "end_user")
new_end_user = end_user_repo.get_or_create_end_user(
app_id=app.id,
workspace_id=workspace_id,

22
api/app/core/error_codes.py
View File

@@ -32,6 +32,8 @@ class BizCode(IntEnum):
API_KEY_DAILY_LIMIT_EXCEEDED = 3015
API_KEY_QUOTA_EXCEEDED = 3016
API_KEY_RATE_LIMIT_EXCEEDED = 3017
QUOTA_EXCEEDED = 3018
RATE_LIMIT_EXCEEDED = 3019
# 资源（4xxx）
NOT_FOUND = 4000
USER_NOT_FOUND = 4001
@@ -156,7 +158,8 @@ HTTP_MAPPING = {
BizCode.API_KEY_QPS_LIMIT_EXCEEDED: 429,
BizCode.API_KEY_DAILY_LIMIT_EXCEEDED: 429,
BizCode.API_KEY_QUOTA_EXCEEDED: 429,
BizCode.QUOTA_EXCEEDED: 402,
BizCode.MODEL_CONFIG_INVALID: 400,
BizCode.API_KEY_MISSING: 400,
BizCode.PROVIDER_NOT_SUPPORTED: 400,
@@ -185,4 +188,21 @@ HTTP_MAPPING = {
BizCode.DB_ERROR: 500,
BizCode.SERVICE_UNAVAILABLE: 503,
BizCode.RATE_LIMITED: 429,
BizCode.RATE_LIMIT_EXCEEDED: 429,
}
ERROR_CODE_TO_BIZ_CODE = {
"QUOTA_EXCEEDED": BizCode.QUOTA_EXCEEDED,
"RATE_LIMIT_EXCEEDED": BizCode.RATE_LIMIT_EXCEEDED,
"API_KEY_NOT_FOUND": BizCode.API_KEY_NOT_FOUND,
"API_KEY_INVALID": BizCode.API_KEY_INVALID,
"API_KEY_EXPIRED": BizCode.API_KEY_EXPIRED,
"WORKSPACE_NOT_FOUND": BizCode.WORKSPACE_NOT_FOUND,
"WORKSPACE_NO_ACCESS": BizCode.WORKSPACE_NO_ACCESS,
"PERMISSION_DENIED": BizCode.PERMISSION_DENIED,
"TOKEN_EXPIRED": BizCode.TOKEN_EXPIRED,
"TOKEN_INVALID": BizCode.TOKEN_INVALID,
"VALIDATION_FAILED": BizCode.VALIDATION_FAILED,
"INVALID_PARAMETER": BizCode.INVALID_PARAMETER,
"MISSING_PARAMETER": BizCode.MISSING_PARAMETER,
}

8
api/app/core/rag/common/connection_utils.py
View File

@@ -33,18 +33,16 @@ def timeout(seconds: float | int | str = None, attempts: int = 2, *, exception:
thread.daemon = True
thread.start()
effective_timeout = seconds if seconds else 120 # 默认 120 秒超时
for a in range(attempts):
try:
if os.environ.get("ENABLE_TIMEOUT_ASSERTION"):
result = result_queue.get(timeout=seconds)
else:
result = result_queue.get()
result = result_queue.get(timeout=effective_timeout)
if isinstance(result, Exception):
raise result
return result
except queue.Empty:
pass
raise TimeoutError(f"Function '{func.__name__}' timed out after {seconds} seconds and {attempts} attempts.")
raise TimeoutError(f"Function '{func.__name__}' timed out after {effective_timeout} seconds and {attempts} attempts.")
@wraps(func)
async def async_wrapper(*args, **kwargs) -> Any:

2
api/app/core/rag/nlp/search.py
View File

@@ -113,7 +113,7 @@ def knowledge_retrieval(
continue
# Use the specified reranker for re-ranking
if reranker_id:
if reranker_id and all_results:
try:
all_results = rerank(db=db, reranker_id=reranker_id, query=query, docs=all_results, top_k=reranker_top_k)
except Exception as rerank_error:

4
api/app/core/rag/utils/es_conn.py
View File

@@ -68,9 +68,9 @@ class ESConnection(DocStoreConnection):
client_config = {
"hosts": [hosts],
"basic_auth": (os.getenv("ELASTICSEARCH_USERNAME", "elastic"), os.getenv("ELASTICSEARCH_PASSWORD", "elastic")),
"request_timeout": int(os.getenv("ELASTICSEARCH_REQUEST_TIMEOUT", 100000)),
"request_timeout": int(os.getenv("ELASTICSEARCH_REQUEST_TIMEOUT", 30)),
"retry_on_timeout": os.getenv("ELASTICSEARCH_RETRY_ON_TIMEOUT", True) == "true",
"max_retries": int(os.getenv("ELASTICSEARCH_MAX_RETRIES", 10000)),
"max_retries": int(os.getenv("ELASTICSEARCH_MAX_RETRIES", 3)),
}
# Only add SSL settings if using HTTPS

193
api/app/core/rag/vdb/elasticsearch/elasticsearch_vector.py
View File

@@ -1,25 +1,22 @@
import os
import logging
from typing import Any, cast
import threading
from typing import Any
from urllib.parse import urlparse
import uuid
import requests
from elasticsearch import Elasticsearch, helpers
from elasticsearch.helpers import BulkIndexError
from packaging.version import parse as parse_version
from pydantic import BaseModel, model_validator
from abc import ABC
# langchain-community
# langchain-xinference
# from langchain_community.embeddings import XinferenceEmbeddings
# from langchain_xinference import XinferenceRerank
from langchain_core.documents import Document
from app.core.models.base import RedBearModelConfig
from app.core.models import RedBearLLM, RedBearRerank
from app.core.models import RedBearRerank
from app.core.models.embedding import RedBearEmbeddings
from app.models.models_model import ModelConfig, ModelApiKey
from app.services.model_service import ModelConfigService
from app.models.models_model import ModelApiKey
from app.models.knowledge_model import Knowledge
from app.core.rag.vdb.field import Field
@@ -29,37 +26,9 @@ from app.core.rag.models.chunk import DocumentChunk
logger = logging.getLogger(__name__)
class ElasticSearchConfig(BaseModel):
# Regular Elasticsearch config
host: str | None = None
port: int | None = None
username: str | None = None
password: str | None = None
# Common config
ca_certs: str | None = None
verify_certs: bool = False
request_timeout: int = 100000
retry_on_timeout: bool = True
max_retries: int = 10000
@model_validator(mode="before")
@classmethod
def validate_config(cls, values: dict):
# Regular Elasticsearch validation
if not values.get("host"):
raise ValueError("config HOST is required for regular Elasticsearch")
if not values.get("port"):
raise ValueError("config PORT is required for regular Elasticsearch")
if not values.get("username"):
raise ValueError("config USERNAME is required for regular Elasticsearch")
if not values.get("password"):
raise ValueError("config PASSWORD is required for regular Elasticsearch")
return values
class ElasticSearchVector(BaseVector):
def __init__(self, index_name: str, config: ElasticSearchConfig, embedding_config: ModelApiKey, reranker_config: ModelApiKey):
def __init__(self, index_name: str, client: Elasticsearch,
embedding_config: ModelApiKey, reranker_config: ModelApiKey):
super().__init__(index_name.lower())
# 初始化 Embedding 模型（自动支持火山引擎多模态）
@@ -77,58 +46,8 @@ class ElasticSearchVector(BaseVector):
api_key=reranker_config.api_key,
base_url=reranker_config.api_base
))
self._client = self._init_client(config)
self._version = self._get_version()
self._check_version()
def _init_client(self, config: ElasticSearchConfig) -> Elasticsearch:
"""
Initialize Elasticsearch client for regular Elasticsearch.
"""
try:
# Regular Elasticsearch configuration
parsed_url = urlparse(config.host or "")
if parsed_url.scheme in {"http", "https"}:
hosts = f"{config.host}:{config.port}"
use_https = parsed_url.scheme == "https"
else:
hosts = f"https://{config.host}:{config.port}"
use_https = False
client_config = {
"hosts": [hosts],
"basic_auth": (config.username, config.password),
"request_timeout": config.request_timeout,
"retry_on_timeout": config.retry_on_timeout,
"max_retries": config.max_retries,
}
# Only add SSL settings if using HTTPS
if use_https:
client_config["verify_certs"] = config.verify_certs
if config.ca_certs:
client_config["ca_certs"] = config.ca_certs
client = Elasticsearch(**client_config)
# Test connection
if not client.ping():
raise ConnectionError("Failed to connect to Elasticsearch")
except requests.ConnectionError as e:
raise ConnectionError(f"Vector database connection error: {str(e)}")
except Exception as e:
raise ConnectionError(f"Elasticsearch client initialization failed: {str(e)}")
return client
def _get_version(self) -> str:
info = self._client.info()
return cast(str, info["version"]["number"])
def _check_version(self):
if parse_version(self._version) < parse_version("8.0.0"):
raise ValueError("Elasticsearch vector database version must be greater than 8.0.0")
# 使用外部传入的共享客户端
self._client = client
def get_type(self) -> str:
return "elasticsearch"
@@ -745,29 +664,79 @@ class ElasticSearchVector(BaseVector):
class ElasticSearchVectorFactory:
@staticmethod
def init_vector(knowledge: Knowledge) -> ElasticSearchVector:
"""ES 向量服务工厂 - 单例共享连接"""
_client: Elasticsearch | None = None
_lock = threading.Lock()
_version_checked = False
@classmethod
def _get_shared_client(cls) -> Elasticsearch:
"""获取共享的 ES 客户端（线程安全的懒加载单例）"""
if cls._client is not None:
return cls._client
with cls._lock:
# 双重检查，防止并发时重复创建
if cls._client is not None:
return cls._client
try:
parsed_url = urlparse(os.getenv("ELASTICSEARCH_HOST", "127.0.0.1") or "")
if parsed_url.scheme in {"http", "https"}:
hosts = f'{os.getenv("ELASTICSEARCH_HOST")}:{os.getenv("ELASTICSEARCH_PORT", 9200)}'
use_https = parsed_url.scheme == "https"
else:
hosts = f'https://{os.getenv("ELASTICSEARCH_HOST", "127.0.0.1")}:{os.getenv("ELASTICSEARCH_PORT", 9200)}'
use_https = False
client_config = {
"hosts": [hosts],
"basic_auth": (
os.getenv("ELASTICSEARCH_USERNAME", "elastic"),
os.getenv("ELASTICSEARCH_PASSWORD", "elastic"),
),
"request_timeout": int(os.getenv("ELASTICSEARCH_REQUEST_TIMEOUT", 30)),
"retry_on_timeout": True,
"max_retries": int(os.getenv("ELASTICSEARCH_MAX_RETRIES", 3)),
"connections_per_node": int(os.getenv("ELASTICSEARCH_CONNECTIONS_PER_NODE", 10)),
}
if use_https:
client_config["verify_certs"] = os.getenv("ELASTICSEARCH_VERIFY_CERTS", "false") == "true"
ca_certs = os.getenv("ELASTICSEARCH_CA_CERTS")
if ca_certs:
client_config["ca_certs"] = str(ca_certs)
client = Elasticsearch(**client_config)
if not client.ping():
raise ConnectionError("Failed to connect to Elasticsearch")
# 版本检查只做一次
if not cls._version_checked:
info = client.info()
version = info["version"]["number"]
if parse_version(version) < parse_version("8.0.0"):
raise ValueError(f"Elasticsearch version must be >= 8.0.0, got {version}")
cls._version_checked = True
logger.info(f"Elasticsearch shared client initialized, version: {version}")
cls._client = client
except requests.ConnectionError as e:
raise ConnectionError(f"Vector database connection error: {str(e)}")
except Exception as e:
raise ConnectionError(f"Elasticsearch client initialization failed: {str(e)}")
return cls._client
@classmethod
def init_vector(cls, knowledge: Knowledge) -> ElasticSearchVector:
"""创建向量服务实例（共享 ES 连接）"""
client = cls._get_shared_client()
collection_name = f"Vector_index_{knowledge.id}_Node"
# Use regular Elasticsearch with config values
config_dict = {
"host": os.getenv("ELASTICSEARCH_HOST", "127.0.0.1"),
"port": os.getenv("ELASTICSEARCH_PORT", 9200),
"username": os.getenv("ELASTICSEARCH_USERNAME", "elastic"),
"password": os.getenv("ELASTICSEARCH_PASSWORD", "elastic"),
}
# Common configuration
config_dict.update(
{
"ca_certs": str(os.getenv("ELASTICSEARCH_CA_CERTS")) if os.getenv("ELASTICSEARCH_CA_CERTS") else None,
"verify_certs": os.getenv("ELASTICSEARCH_VERIFY_CERTS", False) == "true",
"request_timeout": int(os.getenv("ELASTICSEARCH_REQUEST_TIMEOUT", 100000)),
"retry_on_timeout": os.getenv("ELASTICSEARCH_RETRY_ON_TIMEOUT", True) == "true",
"max_retries": int(os.getenv("ELASTICSEARCH_MAX_RETRIES", 10000)),
}
)
if knowledge.embedding is None:
raise ValueError(f"embedding_id config error: {str(knowledge.embedding_id)}")
if knowledge.reranker is None:
@@ -775,9 +744,9 @@ class ElasticSearchVectorFactory:
return ElasticSearchVector(
index_name=collection_name,
config=ElasticSearchConfig(**config_dict),
client=client,
embedding_config=knowledge.embedding.api_keys[0],
reranker_config=knowledge.reranker.api_keys[0]
reranker_config=knowledge.reranker.api_keys[0],
)

27
api/app/i18n/exceptions.py
View File

@@ -6,12 +6,14 @@ error messages based on the current request's language.
"""
import logging
import time
from contextvars import ContextVar
from typing import Any, Dict, Optional
from fastapi import HTTPException, Request
from app.i18n.service import get_translation_service
from app.core.error_codes import ERROR_CODE_TO_BIZ_CODE, BizCode
logger = logging.getLogger(__name__)
@@ -118,15 +120,24 @@ class I18nException(HTTPException):
**params
)
# Build error detail
detail = {
"error_code": self.error_code,
"message": message,
}
# Convert error_code string to BizCode value
biz_code = ERROR_CODE_TO_BIZ_CODE.get(
self.error_code,
BizCode.BAD_REQUEST
)
# Add parameters to detail if provided
if params:
detail["params"] = params
# Build error detail in standard format for compatibility
# main.py handler expects "message" and "error_code" fields for filtering
# but we also include standard format fields
detail = {
"code": biz_code.value,
"msg": message,
"message": message,
"error_code": self.error_code,
"data": params if params else {},
"error": message,
"time": int(time.time() * 1000),
}
# Initialize HTTPException
super().__init__(

11
api/app/repositories/end_user_repository.py
View File

@@ -66,6 +66,17 @@ class EndUserRepository:
db_logger.error(f"查询宿主 {end_user_id} 时出错: {str(e)}")
raise
def get_end_user_by_other_id(self, workspace_id: uuid.UUID, other_id: str) -> Optional["EndUser"]:
"""按 workspace_id + other_id 查找终端用户，不存在返回 None"""
return (
self.db.query(EndUser)
.filter(
EndUser.workspace_id == workspace_id,
EndUser.other_id == other_id
)
.first()
)
def get_or_create_end_user(
self,
app_id: uuid.UUID,

14
api/app/services/api_key_service.py
View File

@@ -51,7 +51,7 @@ class ApiKeyService:
if existing:
raise BusinessException(f"API Key 名称 {data.name} 已存在", BizCode.API_KEY_DUPLICATE_NAME)
# 若 rate_limit 超过租户套餐的 api_ops_rate_limit，自动截断到套餐上限
# 若 rate_limit 超过租户套餐的 api_ops_rate_limit，直接报错
from app.models.workspace_model import Workspace
from app.core.quota_manager import get_api_ops_rate_limit
@@ -59,7 +59,10 @@ class ApiKeyService:
if workspace:
tenant_api_ops_limit = get_api_ops_rate_limit(db, workspace.tenant_id)
if tenant_api_ops_limit and data.rate_limit > tenant_api_ops_limit:
data.rate_limit = tenant_api_ops_limit
raise BusinessException(
f"API Key QPS 不能超过套餐上限 {tenant_api_ops_limit}",
BizCode.BAD_REQUEST
)
# 生成 API Key
api_key = generate_api_key(data.type)
@@ -162,7 +165,7 @@ class ApiKeyService:
if existing:
raise BusinessException(f"API Key 名称 {data.name} 已存在", BizCode.API_KEY_DUPLICATE_NAME)
# 若 rate_limit 超过租户套餐的 api_ops_rate_limit，自动截断到套餐上限
# 若 rate_limit 超过租户套餐的 api_ops_rate_limit，直接报错
if data.rate_limit is not None:
from app.models.workspace_model import Workspace
from app.core.quota_manager import get_api_ops_rate_limit
@@ -171,7 +174,10 @@ class ApiKeyService:
if workspace:
tenant_api_ops_limit = get_api_ops_rate_limit(db, workspace.tenant_id)
if tenant_api_ops_limit and data.rate_limit > tenant_api_ops_limit:
data.rate_limit = tenant_api_ops_limit
raise BusinessException(
f"API Key QPS 不能超过套餐上限 {tenant_api_ops_limit}",
BizCode.BAD_REQUEST
)
update_data = data.model_dump(exclude_unset=True)
ApiKeyRepository.update(db, api_key_id, update_data)

73
api/app/services/app_dsl_service.py
View File

@@ -434,19 +434,37 @@ class AppDslService:
def _resolve_model(self, ref: Optional[dict], tenant_id: uuid.UUID, warnings: list) -> Optional[uuid.UUID]:
if not ref:
return None
q = self.db.query(ModelConfig).filter(
ModelConfig.tenant_id == tenant_id,
ModelConfig.name == ref.get("name"),
ModelConfig.is_active.is_(True)
)
if ref.get("provider"):
q = q.filter(ModelConfig.provider == ref["provider"])
if ref.get("type"):
q = q.filter(ModelConfig.type == ref["type"])
m = q.first()
if not m:
warnings.append(f"模型 '{ref.get('name')}' 未匹配，已置空，请导入后手动配置")
return m.id if m else None
model_id = ref.get("id")
if model_id:
try:
model_uuid = uuid.UUID(str(model_id))
m = self.db.query(ModelConfig).filter(
ModelConfig.id == model_uuid,
ModelConfig.tenant_id == tenant_id,
ModelConfig.is_active.is_(True)
).first()
if m:
return str(m.id)
except (ValueError, AttributeError):
pass
model_name = ref.get("name")
if model_name:
q = self.db.query(ModelConfig).filter(
ModelConfig.tenant_id == tenant_id,
ModelConfig.name == model_name,
ModelConfig.is_active.is_(True)
)
if ref.get("provider"):
q = q.filter(ModelConfig.provider == ref["provider"])
if ref.get("type"):
q = q.filter(ModelConfig.type == ref["type"])
m = q.first()
if m:
return str(m.id)
warnings.append(f"模型 '{model_name}' 未匹配，已置空，请导入后手动配置")
else:
warnings.append(f"模型 ID '{model_id}' 未匹配，已置空，请导入后手动配置")
return None
def _resolve_kb(self, ref: Optional[dict], workspace_id: uuid.UUID, warnings: list) -> Optional[str]:
if not ref:
@@ -587,7 +605,7 @@ class AppDslService:
if not kb_id:
continue
kb_ref = {}
if isinstance(kb_id, str) and len(kb_id) >= 36:
if isinstance(kb_id, str):
try:
uuid.UUID(kb_id)
kb_ref["id"] = kb_id
@@ -601,6 +619,33 @@ class AppDslService:
else:
warnings.append(f"[{node_label}] 知识库 '{kb_id}' 未匹配，已移除，请导入后手动配置")
config["knowledge_bases"] = resolved_kbs
elif node_type in (NodeType.LLM.value, NodeType.QUESTION_CLASSIFIER.value, NodeType.PARAMETER_EXTRACTOR.value):
model_ref = config.get("model_id")
if model_ref:
ref_dict = None
if isinstance(model_ref, dict):
ref_id = model_ref.get("id")
ref_name = model_ref.get("name")
if ref_id:
ref_dict = {"id": ref_id}
elif ref_name is not None:
ref_dict = {"name": ref_name, "provider": model_ref.get("provider"), "type": model_ref.get("type")}
elif isinstance(model_ref, str):
try:
uuid.UUID(model_ref)
ref_dict = {"id": model_ref}
except ValueError:
ref_dict = {"name": model_ref}
if ref_dict:
resolved_model_id = self._resolve_model(ref_dict, tenant_id, warnings)
if resolved_model_id:
config["model_id"] = resolved_model_id
else:
warnings.append(f"[{node_label}] 模型未匹配，已置空，请导入后手动配置")
config["model_id"] = None
else:
warnings.append(f"[{node_label}] 模型未匹配，已置空，请导入后手动配置")
config["model_id"] = None
resolved_nodes.append({**node, "config": config})
return resolved_nodes

52
api/app/services/knowledge_service.py
View File

@@ -7,7 +7,6 @@ from app.models.models_model import ModelConfig
from app.schemas.knowledge_schema import KnowledgeCreate, KnowledgeUpdate
from app.repositories import knowledge_repository
from app.core.logging_config import get_business_logger
from app.repositories.model_repository import ModelConfigRepository
from app.models.models_model import ModelType
business_logger = get_business_logger()
@@ -77,53 +76,32 @@ def create_knowledge(
tenant_id = workspace.tenant_id
def _get_model_by_name_or_fallback(model_name: str | None, model_types: list, label: str):
"""优先按 workspace 指定的 model name 查，找不到再 fallback 到 tenant 下第一个"""
if model_name:
model = db.query(ModelConfig).filter(
ModelConfig.tenant_id == tenant_id,
ModelConfig.name == model_name,
ModelConfig.type.in_([t.value for t in model_types]),
ModelConfig.is_active == True,
ModelConfig.is_composite == False
).first()
if model:
business_logger.debug(f"Auto-bind {label} model from workspace default: {model.id} ({model_name})")
return model
business_logger.debug(f"Workspace default {label} model '{model_name}' not found, falling back to tenant")
models = ModelConfigRepository.get_by_type(db=db, model_types=model_types, tenant_id=tenant_id, is_active=True)
if models:
business_logger.debug(f"Auto-bind {label} model from tenant fallback: {models[0].id}")
return models[0]
return None
if not knowledge.embedding_id:
model = _get_model_by_name_or_fallback(workspace.embedding, [ModelType.EMBEDDING], "embedding")
if model:
knowledge.embedding_id = model.id
if not workspace.embedding:
raise Exception("工作空间未配置 Embedding 模型，请先完善工作空间配置后重试")
knowledge.embedding_id = workspace.embedding
if not knowledge.reranker_id:
model = _get_model_by_name_or_fallback(workspace.rerank, [ModelType.RERANK], "rerank")
if model:
knowledge.reranker_id = model.id
if not workspace.rerank:
raise Exception("工作空间未配置 Rerank 模型，请先完善工作空间配置后重试")
knowledge.reranker_id = workspace.rerank
if not knowledge.llm_id:
model = _get_model_by_name_or_fallback(workspace.llm, [ModelType.LLM, ModelType.CHAT], "llm")
if model:
knowledge.llm_id = model.id
if not workspace.llm:
raise Exception("工作空间未配置 LLM 模型，请先完善工作空间配置后重试")
knowledge.llm_id = workspace.llm
if not knowledge.image2text_id:
image2text_models = db.query(ModelConfig).filter(
model = db.query(ModelConfig).filter(
ModelConfig.tenant_id == tenant_id,
ModelConfig.type.in_([ModelType.CHAT.value]),
ModelConfig.type.in_([ModelType.CHAT.value, ModelType.LLM.value]),
ModelConfig.capability.contains(["vision"]),
ModelConfig.is_active == True,
ModelConfig.is_composite == False
).order_by(ModelConfig.created_at.desc()).all()
if not image2text_models:
).order_by(ModelConfig.created_at.desc()).first()
if not model:
raise Exception("租户下没有可用的视觉模型，创建知识库失败")
knowledge.image2text_id = image2text_models[0].id
business_logger.debug(f"Auto-bind image2text model: {image2text_models[0].id}")
knowledge.image2text_id = model.id
business_logger.debug(f"Auto-bind image2text model: {model.id}")
business_logger.debug(f"Start creating the knowledge base: {knowledge.name}")
db_knowledge = knowledge_repository.create_knowledge(

32
api/app/tasks.py
View File

@@ -251,8 +251,40 @@ def parse_document(file_path: str, document_id: uuid.UUID):
# Prepare vision_model for parsing
vision_model = _build_vision_model(file_path, db_knowledge)
# 先将文件读入内存，避免解析过程中依赖 NFS 文件持续可访问
# python-docx 等库在 binary=None 时会用路径直接打开文件，
# 在 NFS/共享存储上可能因缓存失效导致 "Package not found"
max_wait_seconds = 30
wait_interval = 2
waited = 0
file_binary = None
while waited <= max_wait_seconds:
# os.listdir 强制 NFS 客户端刷新目录缓存
parent_dir = os.path.dirname(file_path)
try:
os.listdir(parent_dir)
except OSError:
pass
try:
with open(file_path, "rb") as f:
file_binary = f.read()
if not file_binary:
# NFS 上文件存在但内容为空（可能还在同步中）
raise IOError(f"File is empty (0 bytes), NFS may still be syncing: {file_path}")
break
except (FileNotFoundError, IOError) as e:
if waited >= max_wait_seconds:
raise type(e)(
f"File not accessible at '{file_path}' after waiting {max_wait_seconds}s: {e}"
)
logger.warning(f"File not ready on this node, retrying in {wait_interval}s: {file_path} ({e})")
time.sleep(wait_interval)
waited += wait_interval
from app.core.rag.app.naive import chunk
logger.info(f"[ParseDoc] file_binary size={len(file_binary)} bytes, type={type(file_binary).__name__}, bool={bool(file_binary)}")
res = chunk(filename=file_path,
binary=file_binary,
from_page=0,
to_page=DEFAULT_PARSE_TO_PAGE,
callback=progress_callback,