Compare commits

..

3 Commits

Author SHA1 Message Date
Mark
a7eaf563d7 Merge pull request #352 from SuanmoSuanyangTechnology/fix/model_base
fix(model)
2026-02-06 17:55:03 +08:00
Timebomb2018
4c7809ce4a fix(model): change the "vl" model type of dashscope to "chat" 2026-02-06 17:51:52 +08:00
Timebomb2018
51847955cd fix(model): change the "vl" model type of dashscope to "chat" 2026-02-06 17:48:01 +08:00
721 changed files with 14802 additions and 82212 deletions

5
.gitignore vendored
View File

@@ -21,7 +21,6 @@ examples/
# Temporary outputs
.DS_Store
.hypothesis/
time.log
celerybeat-schedule.db
search_results.json
@@ -29,7 +28,6 @@ search_results.json
api/migrations/versions
tmp
files
powers/
# Exclude dep files
huggingface.co/
@@ -38,4 +36,5 @@ tika-server*.jar*
cl100k_base.tiktoken
libssl*.deb
sandbox/lib/seccomp_redbear/target
sandbox/lib/seccomp_python/target
sandbox/lib/seccomp_nodejs/target

View File

@@ -226,8 +226,8 @@ REDIS_PORT=6379
REDIS_DB=1
# Celery (Using Redis as broker)
REDIS_DB_CELERY_BROKER=1
REDIS_DB_CELERY_BACKEND=2
BROKER_URL=redis://127.0.0.1:6379/0
RESULT_BACKEND=redis://127.0.0.1:6379/0
# JWT Secret Key (Formation method: openssl rand -hex 32)
SECRET_KEY=your-secret-key-here

View File

@@ -201,8 +201,8 @@ REDIS_PORT=6379
REDIS_DB=1
# Celery (使用Redis作为broker)
REDIS_DB_CELERY_BROKER=1
REDIS_DB_CELERY_BACKEND=2
BROKER_URL=redis://127.0.0.1:6379/0
RESULT_BACKEND=redis://127.0.0.1:6379/0
# JWT密钥 (生成方式: openssl rand -hex 32)
SECRET_KEY=your-secret-key-here

View File

@@ -45,8 +45,7 @@ RUN --mount=type=cache,id=mem_apt,target=/var/cache/apt,sharing=locked \
apt install -y libpython3-dev libgtk-4-1 libnss3 xdg-utils libgbm-dev && \
apt install -y libjemalloc-dev && \
apt install -y python3-pip pipx nginx unzip curl wget git vim less && \
apt install -y ghostscript && \
apt install -y libmagic1
apt install -y ghostscript
RUN if [ "$NEED_MIRROR" == "1" ]; then \
pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \

View File

@@ -10,6 +10,7 @@ from app.core.config import settings
# 设置日志记录器
logger = logging.getLogger(__name__)
# 创建连接池
pool = ConnectionPool.from_url(
f"redis://{settings.REDIS_HOST}:{settings.REDIS_PORT}",
@@ -20,7 +21,6 @@ pool = ConnectionPool.from_url(
)
aio_redis = redis.StrictRedis(connection_pool=pool)
async def get_redis_connection():
"""获取Redis连接"""
try:
@@ -29,8 +29,7 @@ async def get_redis_connection():
logger.error(f"Redis连接失败: {str(e)}")
return None
async def aio_redis_set(key: str, val: str | dict, expire: int = None):
async def aio_redis_set(key: str, val: str|dict, expire: int = None):
"""设置Redis键值
Args:
@@ -41,7 +40,7 @@ async def aio_redis_set(key: str, val: str | dict, expire: int = None):
try:
if isinstance(val, dict):
val = json.dumps(val, ensure_ascii=False)
if expire is not None:
# 设置带过期时间的键值
await aio_redis.set(key, val, ex=expire)
@@ -51,7 +50,6 @@ async def aio_redis_set(key: str, val: str | dict, expire: int = None):
except Exception as e:
logger.error(f"Redis set错误: {str(e)}")
async def aio_redis_get(key: str):
"""获取Redis键值"""
try:
@@ -60,7 +58,6 @@ async def aio_redis_get(key: str):
logger.error(f"Redis get错误: {str(e)}")
return None
async def aio_redis_delete(key: str):
"""删除Redis键"""
try:
@@ -69,7 +66,6 @@ async def aio_redis_delete(key: str):
logger.error(f"Redis delete错误: {str(e)}")
return None
async def aio_redis_publish(channel: str, message: Dict[str, Any]) -> bool:
"""发布消息到Redis频道"""
try:
@@ -82,10 +78,9 @@ async def aio_redis_publish(channel: str, message: Dict[str, Any]) -> bool:
logger.error(f"Redis发布错误: {str(e)}")
return False
class RedisSubscriber:
"""Redis订阅器"""
def __init__(self, channel: str):
self.channel = channel
self.conn = None
@@ -93,25 +88,25 @@ class RedisSubscriber:
self.is_closed = False
self._queue = asyncio.Queue()
self._task = None
async def start(self):
"""开始订阅"""
if self.is_closed or self._task:
return
self._task = asyncio.create_task(self._receive_messages())
logger.info(f"开始订阅: {self.channel}")
async def _receive_messages(self):
"""接收消息"""
try:
self.conn = await get_redis_connection()
if not self.conn:
return
self.pubsub = self.conn.pubsub()
await self.pubsub.subscribe(self.channel)
while not self.is_closed:
try:
message = await self.pubsub.get_message(ignore_subscribe_messages=True, timeout=0.01)
@@ -132,7 +127,7 @@ class RedisSubscriber:
finally:
await self._queue.put(None)
await self._cleanup()
async def _cleanup(self):
"""清理资源"""
if self.pubsub:
@@ -146,7 +141,7 @@ class RedisSubscriber:
await self.conn.close()
except Exception:
pass
async def get_message(self) -> Optional[Dict[str, Any]]:
"""获取消息"""
if self.is_closed:
@@ -158,7 +153,7 @@ class RedisSubscriber:
except Exception as e:
logger.error(f"获取消息错误: {str(e)}")
return None
async def close(self):
"""关闭订阅器"""
if self.is_closed:
@@ -168,33 +163,32 @@ class RedisSubscriber:
self._task.cancel()
await self._cleanup()
class RedisPubSubManager:
"""Redis发布订阅管理器"""
def __init__(self):
self.subscribers = {}
async def publish(self, channel: str, message: Dict[str, Any]) -> bool:
return await aio_redis_publish(channel, message)
def get_subscriber(self, channel: str) -> RedisSubscriber:
if channel in self.subscribers:
subscriber = self.subscribers[channel]
if not subscriber.is_closed:
return subscriber
subscriber = RedisSubscriber(channel)
self.subscribers[channel] = subscriber
return subscriber
def cancel_subscription(self, channel: str) -> bool:
if channel in self.subscribers:
asyncio.create_task(self.subscribers[channel].close())
del self.subscribers[channel]
return True
return False
def cancel_all_subscriptions(self) -> int:
count = len(self.subscribers)
for subscriber in self.subscribers.values():
@@ -202,6 +196,6 @@ class RedisPubSubManager:
self.subscribers.clear()
return count
# 全局实例
pubsub_manager = RedisPubSubManager()

View File

@@ -3,8 +3,9 @@ Cache 缓存模块
提供各种缓存功能的统一入口
"""
from .memory import InterestMemoryCache
from .memory import EmotionMemoryCache, ImplicitMemoryCache
__all__ = [
"InterestMemoryCache",
"EmotionMemoryCache",
"ImplicitMemoryCache",
]

View File

@@ -3,10 +3,10 @@ Memory 缓存模块
提供记忆系统相关的缓存功能
"""
from .interest_memory import InterestMemoryCache
from .activity_stats_cache import ActivityStatsCache
from .emotion_memory import EmotionMemoryCache
from .implicit_memory import ImplicitMemoryCache
__all__ = [
"InterestMemoryCache",
"ActivityStatsCache",
"EmotionMemoryCache",
"ImplicitMemoryCache",
]

View File

@@ -1,124 +0,0 @@
"""
Recent Activity Stats Cache
记忆提取活动统计缓存模块
用于缓存每次记忆提取流程的统计数据,按 workspace_id 存储24小时后释放
查询命令cache:memory:activity_stats:by_workspace:7de31a97-40a6-4fc0-b8d3-15c89f523843
"""
import json
import logging
from typing import Optional, Dict, Any
from datetime import datetime
from app.aioRedis import aio_redis
logger = logging.getLogger(__name__)
# 缓存过期时间24小时
ACTIVITY_STATS_CACHE_EXPIRE = 86400
class ActivityStatsCache:
"""记忆提取活动统计缓存类"""
PREFIX = "cache:memory:activity_stats"
@classmethod
def _get_key(cls, workspace_id: str) -> str:
"""生成 Redis key
Args:
workspace_id: 工作空间ID
Returns:
完整的 Redis key
"""
return f"{cls.PREFIX}:by_workspace:{workspace_id}"
@classmethod
async def set_activity_stats(
cls,
workspace_id: str,
stats: Dict[str, Any],
expire: int = ACTIVITY_STATS_CACHE_EXPIRE,
) -> bool:
"""设置记忆提取活动统计缓存
Args:
workspace_id: 工作空间ID
stats: 统计数据,格式:
{
"chunk_count": int,
"statements_count": int,
"triplet_entities_count": int,
"triplet_relations_count": int,
"temporal_count": int,
}
expire: 过期时间默认24小时
Returns:
是否设置成功
"""
try:
key = cls._get_key(workspace_id)
payload = {
"stats": stats,
"generated_at": datetime.now().isoformat(),
"workspace_id": workspace_id,
"cached": True,
}
value = json.dumps(payload, ensure_ascii=False)
await aio_redis.set(key, value, ex=expire)
logger.info(f"设置活动统计缓存成功: {key}, 过期时间: {expire}")
return True
except Exception as e:
logger.error(f"设置活动统计缓存失败: {e}", exc_info=True)
return False
@classmethod
async def get_activity_stats(
cls,
workspace_id: str,
) -> Optional[Dict[str, Any]]:
"""获取记忆提取活动统计缓存
Args:
workspace_id: 工作空间ID
Returns:
统计数据字典,缓存不存在或已过期返回 None
"""
try:
key = cls._get_key(workspace_id)
value = await aio_redis.get(key)
if value:
payload = json.loads(value)
logger.info(f"命中活动统计缓存: {key}")
return payload
logger.info(f"活动统计缓存不存在或已过期: {key}")
return None
except Exception as e:
logger.error(f"获取活动统计缓存失败: {e}", exc_info=True)
return None
@classmethod
async def delete_activity_stats(
cls,
workspace_id: str,
) -> bool:
"""删除记忆提取活动统计缓存
Args:
workspace_id: 工作空间ID
Returns:
是否删除成功
"""
try:
key = cls._get_key(workspace_id)
result = await aio_redis.delete(key)
logger.info(f"删除活动统计缓存: {key}, 结果: {result}")
return result > 0
except Exception as e:
logger.error(f"删除活动统计缓存失败: {e}", exc_info=True)
return False

134
api/app/cache/memory/emotion_memory.py vendored Normal file
View File

@@ -0,0 +1,134 @@
"""
Emotion Suggestions Cache
情绪个性化建议缓存模块
用于缓存用户的情绪个性化建议数据
"""
import json
import logging
from typing import Optional, Dict, Any
from datetime import datetime
from app.aioRedis import aio_redis
logger = logging.getLogger(__name__)
class EmotionMemoryCache:
"""情绪建议缓存类"""
# Key 前缀
PREFIX = "cache:memory:emotion_memory"
@classmethod
def _get_key(cls, *parts: str) -> str:
"""生成 Redis key
Args:
*parts: key 的各个部分
Returns:
完整的 Redis key
"""
return ":".join([cls.PREFIX] + list(parts))
@classmethod
async def set_emotion_suggestions(
cls,
user_id: str,
suggestions_data: Dict[str, Any],
expire: int = 86400
) -> bool:
"""设置用户情绪建议缓存
Args:
user_id: 用户IDend_user_id
suggestions_data: 建议数据字典,包含:
- health_summary: 健康状态摘要
- suggestions: 建议列表
- generated_at: 生成时间(可选)
expire: 过期时间默认24小时86400秒
Returns:
是否设置成功
"""
try:
key = cls._get_key("suggestions", user_id)
# 添加生成时间戳
if "generated_at" not in suggestions_data:
suggestions_data["generated_at"] = datetime.now().isoformat()
# 添加缓存标记
suggestions_data["cached"] = True
value = json.dumps(suggestions_data, ensure_ascii=False)
await aio_redis.set(key, value, ex=expire)
logger.info(f"设置情绪建议缓存成功: {key}, 过期时间: {expire}")
return True
except Exception as e:
logger.error(f"设置情绪建议缓存失败: {e}", exc_info=True)
return False
@classmethod
async def get_emotion_suggestions(cls, user_id: str) -> Optional[Dict[str, Any]]:
"""获取用户情绪建议缓存
Args:
user_id: 用户IDend_user_id
Returns:
建议数据字典,如果不存在或已过期返回 None
"""
try:
key = cls._get_key("suggestions", user_id)
value = await aio_redis.get(key)
if value:
data = json.loads(value)
logger.info(f"成功获取情绪建议缓存: {key}")
return data
logger.info(f"情绪建议缓存不存在或已过期: {key}")
return None
except Exception as e:
logger.error(f"获取情绪建议缓存失败: {e}", exc_info=True)
return None
@classmethod
async def delete_emotion_suggestions(cls, user_id: str) -> bool:
"""删除用户情绪建议缓存
Args:
user_id: 用户IDend_user_id
Returns:
是否删除成功
"""
try:
key = cls._get_key("suggestions", user_id)
result = await aio_redis.delete(key)
logger.info(f"删除情绪建议缓存: {key}, 结果: {result}")
return result > 0
except Exception as e:
logger.error(f"删除情绪建议缓存失败: {e}", exc_info=True)
return False
@classmethod
async def get_suggestions_ttl(cls, user_id: str) -> int:
"""获取情绪建议缓存的剩余过期时间
Args:
user_id: 用户IDend_user_id
Returns:
剩余秒数,-1表示永不过期-2表示key不存在
"""
try:
key = cls._get_key("suggestions", user_id)
ttl = await aio_redis.ttl(key)
logger.debug(f"情绪建议缓存TTL: {key} = {ttl}")
return ttl
except Exception as e:
logger.error(f"获取情绪建议缓存TTL失败: {e}")
return -2

136
api/app/cache/memory/implicit_memory.py vendored Normal file
View File

@@ -0,0 +1,136 @@
"""
Implicit Memory Profile Cache
隐式记忆用户画像缓存模块
用于缓存用户的完整画像数据(偏好标签、四维画像、兴趣领域、行为习惯)
"""
import json
import logging
from typing import Optional, Dict, Any
from datetime import datetime
from app.aioRedis import aio_redis
logger = logging.getLogger(__name__)
class ImplicitMemoryCache:
"""隐式记忆用户画像缓存类"""
# Key 前缀
PREFIX = "cache:memory:implicit_memory"
@classmethod
def _get_key(cls, *parts: str) -> str:
"""生成 Redis key
Args:
*parts: key 的各个部分
Returns:
完整的 Redis key
"""
return ":".join([cls.PREFIX] + list(parts))
@classmethod
async def set_user_profile(
cls,
user_id: str,
profile_data: Dict[str, Any],
expire: int = 86400
) -> bool:
"""设置用户完整画像缓存
Args:
user_id: 用户IDend_user_id
profile_data: 画像数据字典,包含:
- preferences: 偏好标签列表
- portrait: 四维画像对象
- interest_areas: 兴趣领域分布对象
- habits: 行为习惯列表
- generated_at: 生成时间(可选)
expire: 过期时间默认24小时86400秒
Returns:
是否设置成功
"""
try:
key = cls._get_key("profile", user_id)
# 添加生成时间戳
if "generated_at" not in profile_data:
profile_data["generated_at"] = datetime.now().isoformat()
# 添加缓存标记
profile_data["cached"] = True
value = json.dumps(profile_data, ensure_ascii=False)
await aio_redis.set(key, value, ex=expire)
logger.info(f"设置用户画像缓存成功: {key}, 过期时间: {expire}")
return True
except Exception as e:
logger.error(f"设置用户画像缓存失败: {e}", exc_info=True)
return False
@classmethod
async def get_user_profile(cls, user_id: str) -> Optional[Dict[str, Any]]:
"""获取用户完整画像缓存
Args:
user_id: 用户IDend_user_id
Returns:
画像数据字典,如果不存在或已过期返回 None
"""
try:
key = cls._get_key("profile", user_id)
value = await aio_redis.get(key)
if value:
data = json.loads(value)
logger.info(f"成功获取用户画像缓存: {key}")
return data
logger.info(f"用户画像缓存不存在或已过期: {key}")
return None
except Exception as e:
logger.error(f"获取用户画像缓存失败: {e}", exc_info=True)
return None
@classmethod
async def delete_user_profile(cls, user_id: str) -> bool:
"""删除用户完整画像缓存
Args:
user_id: 用户IDend_user_id
Returns:
是否删除成功
"""
try:
key = cls._get_key("profile", user_id)
result = await aio_redis.delete(key)
logger.info(f"删除用户画像缓存: {key}, 结果: {result}")
return result > 0
except Exception as e:
logger.error(f"删除用户画像缓存失败: {e}", exc_info=True)
return False
@classmethod
async def get_profile_ttl(cls, user_id: str) -> int:
"""获取用户画像缓存的剩余过期时间
Args:
user_id: 用户IDend_user_id
Returns:
剩余秒数,-1表示永不过期-2表示key不存在
"""
try:
key = cls._get_key("profile", user_id)
ttl = await aio_redis.ttl(key)
logger.debug(f"用户画像缓存TTL: {key} = {ttl}")
return ttl
except Exception as e:
logger.error(f"获取用户画像缓存TTL失败: {e}")
return -2

View File

@@ -1,122 +0,0 @@
"""
Interest Distribution Cache
兴趣分布缓存模块
用于缓存用户的兴趣分布标签数据,避免重复调用模型生成
"""
import json
import logging
from typing import Optional, List, Dict, Any
from datetime import datetime
from app.aioRedis import aio_redis
logger = logging.getLogger(__name__)
# 缓存过期时间24小时
INTEREST_CACHE_EXPIRE = 86400
class InterestMemoryCache:
"""兴趣分布缓存类"""
PREFIX = "cache:memory:interest_distribution"
@classmethod
def _get_key(cls, end_user_id: str, language: str) -> str:
"""生成 Redis key
Args:
end_user_id: 用户ID
language: 语言类型
Returns:
完整的 Redis key
"""
return f"{cls.PREFIX}:by_user:{end_user_id}:{language}"
@classmethod
async def set_interest_distribution(
cls,
end_user_id: str,
language: str,
data: List[Dict[str, Any]],
expire: int = INTEREST_CACHE_EXPIRE,
) -> bool:
"""设置用户兴趣分布缓存
Args:
end_user_id: 用户ID
language: 语言类型
data: 兴趣分布列表,格式 [{"name": "...", "frequency": ...}, ...]
expire: 过期时间默认24小时
Returns:
是否设置成功
"""
try:
key = cls._get_key(end_user_id, language)
payload = {
"data": data,
"generated_at": datetime.now().isoformat(),
"cached": True,
}
value = json.dumps(payload, ensure_ascii=False)
await aio_redis.set(key, value, ex=expire)
logger.info(f"设置兴趣分布缓存成功: {key}, 过期时间: {expire}")
return True
except Exception as e:
logger.error(f"设置兴趣分布缓存失败: {e}", exc_info=True)
return False
@classmethod
async def get_interest_distribution(
cls,
end_user_id: str,
language: str,
) -> Optional[List[Dict[str, Any]]]:
"""获取用户兴趣分布缓存
Args:
end_user_id: 用户ID
language: 语言类型
Returns:
兴趣分布列表,缓存不存在或已过期返回 None
"""
try:
key = cls._get_key(end_user_id, language)
value = await aio_redis.get(key)
if value:
payload = json.loads(value)
logger.info(f"命中兴趣分布缓存: {key}")
return payload.get("data")
logger.info(f"兴趣分布缓存不存在或已过期: {key}")
return None
except Exception as e:
logger.error(f"获取兴趣分布缓存失败: {e}", exc_info=True)
return None
@classmethod
async def delete_interest_distribution(
cls,
end_user_id: str,
language: str,
) -> bool:
"""删除用户兴趣分布缓存
Args:
end_user_id: 用户ID
language: 语言类型
Returns:
是否删除成功
"""
try:
key = cls._get_key(end_user_id, language)
result = await aio_redis.delete(key)
logger.info(f"删除兴趣分布缓存: {key}, 结果: {result}")
return result > 0
except Exception as e:
logger.error(f"删除兴趣分布缓存失败: {e}", exc_info=True)
return False

View File

@@ -4,51 +4,22 @@ from datetime import timedelta
from urllib.parse import quote
from celery import Celery
from celery.schedules import crontab
from app.core.config import settings
from app.core.logging_config import get_logger
logger = get_logger(__name__)
# macOS fork() safety - must be set before any Celery initialization
if platform.system() == 'Darwin':
os.environ.setdefault('OBJC_DISABLE_INITIALIZE_FORK_SAFETY', 'YES')
# 创建 Celery 应用实例
# broker: 任务队列(使用 Redis DB,由 CELERY_BROKER_DB 指定
# backend: 结果存储(使用 Redis DB,由 CELERY_BACKEND_DB 指定
# NOTE: 不要在 .env 中设置 BROKER_URL / RESULT_BACKEND / CELERY_BROKER / CELERY_BACKEND
# 这些名称会被 Celery CLI 的 Click 框架劫持,详见 docs/celery-env-bug-report.md
# Build canonical broker/backend URLs and force them into os.environ so that
# Celery's Settings.broker_url property (which checks CELERY_BROKER_URL first)
# cannot be overridden by stray env vars.
# See: https://github.com/celery/celery/issues/4284
_broker_url = f"redis://:{quote(settings.REDIS_PASSWORD)}@{settings.REDIS_HOST}:{settings.REDIS_PORT}/{settings.REDIS_DB_CELERY_BROKER}"
_backend_url = f"redis://:{quote(settings.REDIS_PASSWORD)}@{settings.REDIS_HOST}:{settings.REDIS_PORT}/{settings.REDIS_DB_CELERY_BACKEND}"
os.environ["CELERY_BROKER_URL"] = _broker_url
os.environ["CELERY_RESULT_BACKEND"] = _backend_url
# Neutralize legacy Celery env vars that can be hijacked by Celery's CLI/Click
# integration and accidentally override our canonical URLs.
os.environ.pop("BROKER_URL", None)
os.environ.pop("RESULT_BACKEND", None)
os.environ.pop("CELERY_BROKER", None)
os.environ.pop("CELERY_BACKEND", None)
# broker: 任务队列(使用 Redis DB 0
# backend: 结果存储(使用 Redis DB 10
celery_app = Celery(
"redbear_tasks",
broker=_broker_url,
backend=_backend_url,
broker=f"redis://:{quote(settings.REDIS_PASSWORD)}@{settings.REDIS_HOST}:{settings.REDIS_PORT}/{settings.CELERY_BROKER}",
backend=f"redis://:{quote(settings.REDIS_PASSWORD)}@{settings.REDIS_HOST}:{settings.REDIS_PORT}/{settings.CELERY_BACKEND}",
)
logger.info(
"Celery app initialized",
extra={
"broker": _broker_url.replace(quote(settings.REDIS_PASSWORD), "***"),
"backend": _backend_url.replace(quote(settings.REDIS_PASSWORD), "***"),
},
)
# Default queue for unrouted tasks
celery_app.conf.task_default_queue = 'memory_tasks'
@@ -63,17 +34,17 @@ celery_app.conf.update(
accept_content=['json'],
result_serializer='json',
# # 时区
# timezone='Asia/Shanghai',
# enable_utc=False,
# 时区
timezone='Asia/Shanghai',
enable_utc=True,
# 任务追踪
task_track_started=True,
task_ignore_result=False,
# 超时设置
task_time_limit=3600, # 60分钟硬超时
task_soft_time_limit=3000, # 50分钟软超时
task_time_limit=1800, # 30分钟硬超时
task_soft_time_limit=1500, # 25分钟软超时
# Worker 设置 (per-worker settings are in docker-compose command line)
worker_prefetch_multiplier=1, # Don't hoard tasks, fairer distribution
@@ -105,16 +76,12 @@ celery_app.conf.update(
# Document tasks → document_tasks queue (prefork worker)
'app.core.rag.tasks.parse_document': {'queue': 'document_tasks'},
'app.core.rag.tasks.build_graphrag_for_kb': {'queue': 'document_tasks'},
'app.core.rag.tasks.sync_knowledge_for_kb': {'queue': 'document_tasks'},
# Beat/periodic tasks → periodic_tasks queue (dedicated periodic worker)
'app.tasks.workspace_reflection_task': {'queue': 'periodic_tasks'},
'app.tasks.regenerate_memory_cache': {'queue': 'periodic_tasks'},
'app.tasks.run_forgetting_cycle_task': {'queue': 'periodic_tasks'},
'app.tasks.write_all_workspaces_memory_task': {'queue': 'periodic_tasks'},
'app.tasks.update_implicit_emotions_storage': {'queue': 'periodic_tasks'},
'app.tasks.init_implicit_emotions_for_users': {'queue': 'periodic_tasks'},
'app.tasks.init_interest_distribution_for_users': {'queue': 'periodic_tasks'},
'app.controllers.memory_storage_controller.search_all': {'queue': 'periodic_tasks'},
},
)
@@ -122,44 +89,40 @@ celery_app.conf.update(
celery_app.autodiscover_tasks(['app'])
# Celery Beat schedule for periodic tasks
memory_increment_schedule = crontab(hour=settings.MEMORY_INCREMENT_HOUR, minute=settings.MEMORY_INCREMENT_MINUTE)
memory_cache_regeneration_schedule = timedelta(hours=settings.MEMORY_CACHE_REGENERATION_HOURS)
workspace_reflection_schedule = timedelta(seconds=settings.WORKSPACE_REFLECTION_INTERVAL_SECONDS)
forgetting_cycle_schedule = timedelta(hours=settings.FORGETTING_CYCLE_INTERVAL_HOURS)
implicit_emotions_update_schedule = crontab(
hour=settings.IMPLICIT_EMOTIONS_UPDATE_HOUR,
minute=settings.IMPLICIT_EMOTIONS_UPDATE_MINUTE,
)
# memory_increment_schedule = timedelta(hours=settings.MEMORY_INCREMENT_INTERVAL_HOURS)
# memory_cache_regeneration_schedule = timedelta(hours=settings.MEMORY_CACHE_REGENERATION_HOURS)
# workspace_reflection_schedule = timedelta(seconds=30) # 每30秒运行一次settings.REFLECTION_INTERVAL_TIME
# forgetting_cycle_schedule = timedelta(hours=24) # 每24小时运行一次遗忘周期
#构建定时任务配置
beat_schedule_config = {
"run-workspace-reflection": {
"task": "app.tasks.workspace_reflection_task",
"schedule": workspace_reflection_schedule,
"args": (),
},
"regenerate-memory-cache": {
"task": "app.tasks.regenerate_memory_cache",
"schedule": memory_cache_regeneration_schedule,
"args": (),
},
"run-forgetting-cycle": {
"task": "app.tasks.run_forgetting_cycle_task",
"schedule": forgetting_cycle_schedule,
"kwargs": {
"config_id": None, # 使用默认配置,可以通过环境变量配置
},
},
"write-all-workspaces-memory": {
"task": "app.tasks.write_all_workspaces_memory_task",
"schedule": memory_increment_schedule,
"args": (),
},
"update-implicit-emotions-storage": {
"task": "app.tasks.update_implicit_emotions_storage",
"schedule": implicit_emotions_update_schedule,
"args": (),
},
}
# 构建定时任务配置
# beat_schedule_config = {
# "run-workspace-reflection": {
# "task": "app.tasks.workspace_reflection_task",
# "schedule": workspace_reflection_schedule,
# "args": (),
# },
# "regenerate-memory-cache": {
# "task": "app.tasks.regenerate_memory_cache",
# "schedule": memory_cache_regeneration_schedule,
# "args": (),
# },
# "run-forgetting-cycle": {
# "task": "app.tasks.run_forgetting_cycle_task",
# "schedule": forgetting_cycle_schedule,
# "kwargs": {
# "config_id": None, # 使用默认配置,可以通过环境变量配置
# },
# },
# }
celery_app.conf.beat_schedule = beat_schedule_config
# 如果配置了默认工作空间ID则添加记忆总量统计任务
# if settings.DEFAULT_WORKSPACE_ID:
# beat_schedule_config["write-total-memory"] = {
# "task": "app.controllers.memory_storage_controller.search_all",
# "schedule": memory_increment_schedule,
# "kwargs": {
# "workspace_id": settings.DEFAULT_WORKSPACE_ID,
# },
# }
# celery_app.conf.beat_schedule = beat_schedule_config

View File

@@ -1 +0,0 @@
"""Configuration module for application settings."""

View File

@@ -1,239 +0,0 @@
"""默认本体场景配置
本模块定义系统预设的本体场景和实体类型配置。
这些配置用于在工作空间创建时自动初始化默认场景。
支持中英文双语配置,根据用户语言偏好创建对应语言的场景。
"""
# 在线教育场景配置
ONLINE_EDUCATION_SCENE = {
"name_chinese": "在线教育",
"name_english": "Online Education",
"description_chinese": "适用于在线教育平台的本体建模,包含学生、教师、课程等核心实体类型",
"description_english": "Ontology modeling for online education platforms, including core entity types such as students, teachers, and courses",
"types": [
{
"name_chinese": "学生",
"name_english": "Student",
"description_chinese": "在教育系统中接受教育的个体,包含姓名、学号、年级、班级等属性",
"description_english": "Individuals receiving education in the education system, including attributes such as name, student ID, grade, and class"
},
{
"name_chinese": "教师",
"name_english": "Teacher",
"description_chinese": "在教育系统中提供教学服务的个体,包含姓名、工号、任教学科、职称等属性",
"description_english": "Individuals providing teaching services in the education system, including attributes such as name, employee ID, teaching subject, and title"
},
{
"name_chinese": "课程",
"name_english": "Course",
"description_chinese": "教育系统中的教学内容单元,包含课程名称、课程代码、学分、学时等属性",
"description_english": "Teaching content units in the education system, including attributes such as course name, course code, credits, and class hours"
},
{
"name_chinese": "作业",
"name_english": "Assignment",
"description_chinese": "课程中布置的学习任务,包含作业标题、截止日期、所属课程、提交状态等属性",
"description_english": "Learning tasks assigned in courses, including attributes such as assignment title, deadline, course, and submission status"
},
{
"name_chinese": "成绩",
"name_english": "Grade",
"description_chinese": "学生学习成果的评价结果,包含分数、评级、考试类型、所属课程等属性",
"description_english": "Evaluation results of student learning outcomes, including attributes such as score, rating, exam type, and course"
},
{
"name_chinese": "考试",
"name_english": "Exam",
"description_chinese": "评估学生学习成果的测试活动,包含考试名称、时间、地点、科目等属性",
"description_english": "Test activities to assess student learning outcomes, including attributes such as exam name, time, location, and subject"
},
{
"name_chinese": "教室",
"name_english": "Classroom",
"description_chinese": "进行教学活动的物理或虚拟空间,包含教室编号、容量、设备等属性",
"description_english": "Physical or virtual spaces for teaching activities, including attributes such as classroom number, capacity, and equipment"
},
{
"name_chinese": "学科",
"name_english": "Subject",
"description_chinese": "知识的分类领域,包含学科名称、代码、所属院系等属性",
"description_english": "Classification domains of knowledge, including attributes such as subject name, code, and department"
},
{
"name_chinese": "教材",
"name_english": "Textbook",
"description_chinese": "教学使用的书籍或资料包含书名、作者、出版社、ISBN等属性",
"description_english": "Books or materials used for teaching, including attributes such as title, author, publisher, and ISBN"
},
{
"name_chinese": "班级",
"name_english": "Class",
"description_chinese": "学生的组织单位,包含班级名称、年级、人数、班主任等属性",
"description_english": "Organizational units of students, including attributes such as class name, grade, number of students, and class teacher"
},
{
"name_chinese": "学期",
"name_english": "Semester",
"description_chinese": "教学时间的划分单位,包含学期名称、开始时间、结束时间等属性",
"description_english": "Time division units for teaching, including attributes such as semester name, start time, and end time"
},
{
"name_chinese": "课时",
"name_english": "Class Hour",
"description_chinese": "课程的时间单位,包含上课时间、地点、教师、课程等属性",
"description_english": "Time units of courses, including attributes such as class time, location, teacher, and course"
},
{
"name_chinese": "教学计划",
"name_english": "Teaching Plan",
"description_chinese": "课程的教学安排,包含教学目标、内容安排、进度计划等属性",
"description_english": "Teaching arrangements for courses, including attributes such as teaching objectives, content arrangement, and progress plan"
}
]
}
# 情感陪伴场景配置
EMOTIONAL_COMPANION_SCENE = {
"name_chinese": "情感陪伴",
"name_english": "Emotional Companion",
"description_chinese": "适用于情感陪伴应用的本体建模,包含用户、情绪、活动等核心实体类型",
"description_english": "Ontology modeling for emotional companion applications, including core entity types such as users, emotions, and activities",
"types": [
{
"name_chinese": "用户",
"name_english": "User",
"description_chinese": "使用情感陪伴服务的个体,包含姓名、昵称、性格特征、偏好等属性",
"description_english": "Individuals using emotional companion services, including attributes such as name, nickname, personality traits, and preferences"
},
{
"name_chinese": "情绪",
"name_english": "Emotion",
"description_chinese": "用户的情感状态,包含情绪类型、强度、触发原因、持续时间等属性",
"description_english": "Emotional states of users, including attributes such as emotion type, intensity, trigger cause, and duration"
},
{
"name_chinese": "活动",
"name_english": "Activity",
"description_chinese": "用户参与的各类活动,包含活动名称、类型、参与者、时间地点等属性",
"description_english": "Various activities users participate in, including attributes such as activity name, type, participants, time, and location"
},
{
"name_chinese": "对话",
"name_english": "Conversation",
"description_chinese": "用户之间的交流记录,包含对话主题、参与者、时间、关键内容等属性",
"description_english": "Communication records between users, including attributes such as conversation topic, participants, time, and key content"
},
{
"name_chinese": "兴趣爱好",
"name_english": "Hobby",
"description_chinese": "用户的兴趣和爱好,包含爱好名称、类别、熟练程度、相关活动等属性",
"description_english": "User interests and hobbies, including attributes such as hobby name, category, proficiency level, and related activities"
},
{
"name_chinese": "日常事件",
"name_english": "Daily Event",
"description_chinese": "用户日常生活中的事件,包含事件描述、时间、地点、相关人物等属性",
"description_english": "Events in users' daily lives, including attributes such as event description, time, location, and related people"
},
{
"name_chinese": "关系",
"name_english": "Relationship",
"description_chinese": "用户之间的社会关系,包含关系类型、亲密度、建立时间等属性",
"description_english": "Social relationships between users, including attributes such as relationship type, intimacy, and establishment time"
},
{
"name_chinese": "回忆",
"name_english": "Memory",
"description_chinese": "用户的重要记忆片段,包含回忆内容、时间、地点、相关人物等属性",
"description_english": "Important memory fragments of users, including attributes such as memory content, time, location, and related people"
},
{
"name_chinese": "地点",
"name_english": "Location",
"description_chinese": "用户活动的地理位置,包含地点名称、地址、类型、相关事件等属性",
"description_english": "Geographic locations of user activities, including attributes such as location name, address, type, and related events"
},
{
"name_chinese": "时间节点",
"name_english": "Time Point",
"description_chinese": "重要的时间标记,包含日期、事件、意义等属性",
"description_english": "Important time markers, including attributes such as date, event, and significance"
},
{
"name_chinese": "目标",
"name_english": "Goal",
"description_chinese": "用户设定的目标,包含目标描述、截止时间、完成状态、相关活动等属性",
"description_english": "Goals set by users, including attributes such as goal description, deadline, completion status, and related activities"
},
{
"name_chinese": "成就",
"name_english": "Achievement",
"description_chinese": "用户获得的成就,包含成就名称、获得时间、描述、相关目标等属性",
"description_english": "Achievements obtained by users, including attributes such as achievement name, acquisition time, description, and related goals"
}
]
}
# 导出默认场景列表
DEFAULT_SCENES = [ONLINE_EDUCATION_SCENE, EMOTIONAL_COMPANION_SCENE]
def get_scene_name(scene_config: dict, language: str = "zh") -> str:
"""获取场景名称(根据语言)
Args:
scene_config: 场景配置字典
language: 语言类型 ("zh""en")
Returns:
对应语言的场景名称
"""
if language == "en":
return scene_config.get("name_english", scene_config.get("name_chinese"))
return scene_config.get("name_chinese")
def get_scene_description(scene_config: dict, language: str = "zh") -> str:
"""获取场景描述(根据语言)
Args:
scene_config: 场景配置字典
language: 语言类型 ("zh""en")
Returns:
对应语言的场景描述
"""
if language == "en":
return scene_config.get("description_english", scene_config.get("description_chinese"))
return scene_config.get("description_chinese")
def get_type_name(type_config: dict, language: str = "zh") -> str:
"""获取类型名称(根据语言)
Args:
type_config: 类型配置字典
language: 语言类型 ("zh""en")
Returns:
对应语言的类型名称
"""
if language == "en":
return type_config.get("name_english", type_config.get("name_chinese"))
return type_config.get("name_chinese")
def get_type_description(type_config: dict, language: str = "zh") -> str:
"""获取类型描述(根据语言)
Args:
type_config: 类型配置字典
language: 语言类型 ("zh""en")
Returns:
对应语言的类型描述
"""
if language == "en":
return type_config.get("description_english", type_config.get("description_chinese"))
return type_config.get("description_chinese")

View File

@@ -1,249 +0,0 @@
# -*- coding: utf-8 -*-
"""默认本体场景初始化器
本模块提供默认本体场景和类型的自动初始化功能。
在工作空间创建时,自动添加预设的本体场景和实体类型。
Classes:
DefaultOntologyInitializer: 默认本体场景初始化器
"""
import logging
from typing import List, Optional, Tuple
from uuid import UUID
from sqlalchemy.orm import Session
from app.config.default_ontology_config import (
DEFAULT_SCENES,
get_scene_name,
get_scene_description,
get_type_name,
get_type_description,
)
from app.core.logging_config import get_business_logger
from app.repositories.ontology_scene_repository import OntologySceneRepository
from app.repositories.ontology_class_repository import OntologyClassRepository
class DefaultOntologyInitializer:
"""默认本体场景初始化器
负责在工作空间创建时自动初始化默认的本体场景和类型。
遵循最小侵入原则,确保初始化失败不阻止工作空间创建。
Attributes:
db: 数据库会话
scene_repo: 场景Repository
class_repo: 类型Repository
logger: 业务日志记录器
"""
def __init__(self, db: Session):
"""初始化
Args:
db: 数据库会话
"""
self.db = db
self.scene_repo = OntologySceneRepository(db)
self.class_repo = OntologyClassRepository(db)
self.logger = get_business_logger()
def initialize_default_scenes(
self,
workspace_id: UUID,
language: str = "zh"
) -> Tuple[bool, str]:
"""为工作空间初始化默认场景
创建两个默认场景(在线教育、情感陪伴)及其对应的实体类型。
如果创建失败,记录错误日志但不抛出异常。
Args:
workspace_id: 工作空间ID
language: 语言类型 ("zh""en"),默认为 "zh"
Returns:
Tuple[bool, str]: (是否成功, 错误信息)
"""
try:
self.logger.info(
f"开始初始化默认本体场景 - workspace_id={workspace_id}, language={language}"
)
scenes_created = 0
total_types_created = 0
# 遍历默认场景配置
for scene_config in DEFAULT_SCENES:
scene_name = get_scene_name(scene_config, language)
# 创建场景及其类型
scene_id = self._create_scene_with_types(workspace_id, scene_config, language)
if scene_id:
scenes_created += 1
# 统计类型数量
types_count = len(scene_config.get("types", []))
total_types_created += types_count
self.logger.info(
f"场景创建成功 - scene_name={scene_name}, "
f"scene_id={scene_id}, types_count={types_count}, language={language}"
)
else:
self.logger.warning(
f"场景创建失败 - scene_name={scene_name}, "
f"workspace_id={workspace_id}, language={language}"
)
# 记录总体结果
self.logger.info(
f"默认场景初始化完成 - workspace_id={workspace_id}, "
f"language={language}, scenes_created={scenes_created}, "
f"total_types_created={total_types_created}"
)
# 如果至少创建了一个场景,视为成功
if scenes_created > 0:
return True, ""
else:
error_msg = "所有默认场景创建失败"
self.logger.error(
f"默认场景初始化失败 - workspace_id={workspace_id}, "
f"language={language}, error={error_msg}"
)
return False, error_msg
except Exception as e:
error_msg = f"默认场景初始化异常: {str(e)}"
self.logger.error(
f"默认场景初始化异常 - workspace_id={workspace_id}, "
f"language={language}, error={str(e)}",
exc_info=True
)
return False, error_msg
def _create_scene_with_types(
self,
workspace_id: UUID,
scene_config: dict,
language: str = "zh"
) -> Optional[UUID]:
"""创建场景及其类型
Args:
workspace_id: 工作空间ID
scene_config: 场景配置字典
language: 语言类型 ("zh""en")
Returns:
Optional[UUID]: 创建的场景ID失败返回None
"""
try:
scene_name = get_scene_name(scene_config, language)
scene_description = get_scene_description(scene_config, language)
# 检查是否已存在同名场景(支持向后兼容)
existing_scene = self.scene_repo.get_by_name(scene_name, workspace_id)
if existing_scene:
self.logger.info(
f"场景已存在,跳过创建 - scene_name={scene_name}, "
f"workspace_id={workspace_id}, scene_id={existing_scene.scene_id}, "
f"language={language}"
)
return None
# 创建场景记录,设置 is_system_default=true
scene_data = {
"scene_name": scene_name,
"scene_description": scene_description
}
scene = self.scene_repo.create(scene_data, workspace_id)
# 设置系统默认标识
scene.is_system_default = True
self.db.flush()
self.logger.info(
f"场景创建成功 - scene_name={scene_name}, "
f"scene_id={scene.scene_id}, is_system_default=True, language={language}"
)
# 批量创建类型
types_config = scene_config.get("types", [])
types_created = self._batch_create_types(scene.scene_id, types_config, language)
self.logger.info(
f"场景类型创建完成 - scene_id={scene.scene_id}, "
f"types_created={types_created}/{len(types_config)}, language={language}"
)
return scene.scene_id
except Exception as e:
scene_name = get_scene_name(scene_config, language)
self.logger.error(
f"场景创建失败 - scene_name={scene_name}, "
f"workspace_id={workspace_id}, language={language}, error={str(e)}",
exc_info=True
)
return None
def _batch_create_types(
self,
scene_id: UUID,
types_config: List[dict],
language: str = "zh"
) -> int:
"""批量创建实体类型
Args:
scene_id: 场景ID
types_config: 类型配置列表
language: 语言类型 ("zh""en")
Returns:
int: 成功创建的类型数量
"""
created_count = 0
for type_config in types_config:
try:
type_name = get_type_name(type_config, language)
type_description = get_type_description(type_config, language)
# 创建类型数据
class_data = {
"class_name": type_name,
"class_description": type_description
}
# 创建类型
ontology_class = self.class_repo.create(class_data, scene_id)
# 设置系统默认标识
ontology_class.is_system_default = True
self.db.flush()
created_count += 1
self.logger.debug(
f"类型创建成功 - class_name={type_name}, "
f"class_id={ontology_class.class_id}, "
f"scene_id={scene_id}, is_system_default=True, language={language}"
)
except Exception as e:
type_name = get_type_name(type_config, language)
self.logger.warning(
f"单个类型创建失败,继续创建其他类型 - "
f"class_name={type_name}, scene_id={scene_id}, "
f"language={language}, error={str(e)}"
)
# 继续创建其他类型
continue
return created_count

View File

@@ -19,18 +19,14 @@ from . import (
implicit_memory_controller,
knowledge_controller,
knowledgeshare_controller,
mcp_market_controller,
mcp_market_config_controller,
memory_agent_controller,
memory_dashboard_controller,
memory_episodic_controller,
memory_explicit_controller,
memory_forget_controller,
memory_perceptual_controller,
memory_reflection_controller,
memory_short_term_controller,
memory_storage_controller,
memory_working_controller,
model_controller,
multi_agent_controller,
prompt_optimizer_controller,
@@ -43,9 +39,13 @@ from . import (
upload_controller,
user_controller,
user_memory_controllers,
workflow_controller,
workspace_controller,
memory_forget_controller,
home_page_controller,
memory_perceptual_controller,
memory_working_controller,
ontology_controller,
skill_controller
)
# 创建管理端 API 路由器
@@ -62,8 +62,6 @@ manager_router.include_router(model_controller.router)
manager_router.include_router(file_controller.router)
manager_router.include_router(document_controller.router)
manager_router.include_router(knowledge_controller.router)
manager_router.include_router(mcp_market_controller.router)
manager_router.include_router(mcp_market_config_controller.router)
manager_router.include_router(chunk_controller.router)
manager_router.include_router(test_controller.router)
manager_router.include_router(knowledgeshare_controller.router)
@@ -80,6 +78,7 @@ manager_router.include_router(release_share_controller.router)
manager_router.include_router(public_share_controller.router) # 公开路由(无需认证)
manager_router.include_router(memory_dashboard_controller.router)
manager_router.include_router(multi_agent_controller.router)
manager_router.include_router(workflow_controller.router)
manager_router.include_router(emotion_controller.router)
manager_router.include_router(emotion_config_controller.router)
manager_router.include_router(prompt_optimizer_controller.router)
@@ -93,6 +92,5 @@ manager_router.include_router(memory_perceptual_controller.router)
manager_router.include_router(memory_working_controller.router)
manager_router.include_router(file_storage_controller.router)
manager_router.include_router(ontology_controller.router)
manager_router.include_router(skill_controller.router)
__all__ = ["manager_router"]

View File

@@ -1,12 +1,9 @@
import uuid
import io
from typing import Optional, Annotated
import yaml
from fastapi import APIRouter, Depends, Path, Form, UploadFile, File
from fastapi import APIRouter, Depends, Path
from fastapi.responses import StreamingResponse
from sqlalchemy.orm import Session
from urllib.parse import quote
from app.core.error_codes import BizCode
from app.core.logging_config import get_business_logger
@@ -20,14 +17,11 @@ from app.repositories.end_user_repository import EndUserRepository
from app.schemas import app_schema
from app.schemas.response_schema import PageData, PageMeta
from app.schemas.workflow_schema import WorkflowConfig as WorkflowConfigSchema
from app.schemas.workflow_schema import WorkflowConfigUpdate, WorkflowImportSave
from app.schemas.workflow_schema import WorkflowConfigUpdate
from app.services import app_service, workspace_service
from app.services.agent_config_helper import enrich_agent_config
from app.services.app_service import AppService
from app.services.app_statistics_service import AppStatisticsService
from app.services.workflow_import_service import WorkflowImportService
from app.services.workflow_service import WorkflowService, get_workflow_service
from app.services.app_dsl_service import AppDslService
router = APIRouter(prefix="/apps", tags=["Apps"])
logger = get_business_logger()
@@ -70,7 +64,7 @@ def list_apps(
# 当 ids 存在且不为 None 时,根据 ids 获取应用
if ids is not None:
app_ids = [app_id.strip() for app_id in ids.split(',') if app_id.strip()]
app_ids = [id.strip() for id in ids.split(',') if id.strip()]
items_orm = app_service.get_apps_by_ids(db, app_ids, workspace_id)
items = [service._convert_to_schema(app, workspace_id) for app in items_orm]
return success(data=items)
@@ -399,10 +393,10 @@ async def draft_run(
from app.models import AgentConfig, ModelConfig
from sqlalchemy import select
from app.core.exceptions import BusinessException
from app.services.draft_run_service import AgentRunService
from app.services.draft_run_service import DraftRunService
service = AppService(db)
draft_service = AgentRunService(db)
draft_service = DraftRunService(db)
# 1. 验证应用
app = service._get_app_or_404(app_id)
@@ -460,8 +454,7 @@ async def draft_run(
user_id=payload.user_id or str(current_user.id),
variables=payload.variables,
storage_type=storage_type,
user_rag_memory_id=user_rag_memory_id,
files=payload.files # 传递多模态文件
user_rag_memory_id=user_rag_memory_id
):
yield event
@@ -482,13 +475,12 @@ async def draft_run(
"app_id": str(app_id),
"message_length": len(payload.message),
"has_conversation_id": bool(payload.conversation_id),
"has_variables": bool(payload.variables),
"has_files": bool(payload.files)
"has_variables": bool(payload.variables)
}
)
from app.services.draft_run_service import AgentRunService
draft_service = AgentRunService(db)
from app.services.draft_run_service import DraftRunService
draft_service = DraftRunService(db)
result = await draft_service.run(
agent_config=agent_cfg,
model_config=model_config,
@@ -498,8 +490,7 @@ async def draft_run(
user_id=payload.user_id or str(current_user.id),
variables=payload.variables,
storage_type=storage_type,
user_rag_memory_id=user_rag_memory_id,
files=payload.files # 传递多模态文件
user_rag_memory_id=user_rag_memory_id
)
logger.debug(
@@ -792,8 +783,8 @@ async def draft_run_compare(
# 流式返回
if payload.stream:
async def event_generator():
from app.services.draft_run_service import AgentRunService
draft_service = AgentRunService(db)
from app.services.draft_run_service import DraftRunService
draft_service = DraftRunService(db)
async for event in draft_service.run_compare_stream(
agent_config=agent_cfg,
models=model_configs,
@@ -807,8 +798,7 @@ async def draft_run_compare(
web_search=True,
memory=True,
parallel=payload.parallel,
timeout=payload.timeout or 60,
files=payload.files
timeout=payload.timeout or 60
):
yield event
@@ -823,8 +813,8 @@ async def draft_run_compare(
)
# 非流式返回
from app.services.draft_run_service import AgentRunService
draft_service = AgentRunService(db)
from app.services.draft_run_service import DraftRunService
draft_service = DraftRunService(db)
result = await draft_service.run_compare(
agent_config=agent_cfg,
models=model_configs,
@@ -838,8 +828,7 @@ async def draft_run_compare(
web_search=True,
memory=True,
parallel=payload.parallel,
timeout=payload.timeout or 60,
files=payload.files
timeout=payload.timeout or 60
)
logger.info(
@@ -885,60 +874,6 @@ async def update_workflow_config(
return success(data=WorkflowConfigSchema.model_validate(cfg))
@router.get("/{app_id}/workflow/export")
@cur_workspace_access_guard()
async def export_workflow_config(
app_id: uuid.UUID,
db: Annotated[Session, Depends(get_db)],
current_user: Annotated[User, Depends(get_current_user)]
):
"""导出工作流配置为YAML文件"""
workflow_service = WorkflowService(db)
return success(data={
"content": workflow_service.export_workflow_dsl(app_id=app_id),
})
@router.post("/workflow/import")
@cur_workspace_access_guard()
async def import_workflow_config(
file: UploadFile = File(...),
platform: str = Form(...),
app_id: str = Form(None),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""从YAML内容导入工作流配置"""
if not file.filename.lower().endswith((".yaml", ".yml")):
return fail(msg="Only yaml file is allowed", code=BizCode.BAD_REQUEST)
raw_text = (await file.read()).decode("utf-8")
import_service = WorkflowImportService(db)
config = yaml.safe_load(raw_text)
result = await import_service.upload_config(platform, config)
return success(data=result)
@router.post("/workflow/import/save")
@cur_workspace_access_guard()
async def save_workflow_import(
data: WorkflowImportSave,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
import_service = WorkflowImportService(db)
app = await import_service.save_workflow(
user_id=current_user.id,
workspace_id=current_user.current_workspace_id,
temp_id=data.temp_id,
name=data.name,
description=data.description,
)
return success(data=app_schema.App.model_validate(app))
@router.get("/{app_id}/statistics", summary="应用统计数据")
@cur_workspace_access_guard()
def get_app_statistics(
@@ -949,14 +884,12 @@ def get_app_statistics(
current_user=Depends(get_current_user),
):
"""获取应用统计数据
Args:
app_id: 应用ID
start_date: 开始时间戳(毫秒)
end_date: 结束时间戳(毫秒)
db: 数据库连接
current_user: 当前用户
Returns:
- daily_conversations: 每日会话数统计
- total_conversations: 总会话数
@@ -968,102 +901,15 @@ def get_app_statistics(
- total_tokens: 总token消耗
"""
workspace_id = current_user.current_workspace_id
from app.services.app_statistics_service import AppStatisticsService
stats_service = AppStatisticsService(db)
result = stats_service.get_app_statistics(
app_id=app_id,
workspace_id=workspace_id,
start_date=start_date,
end_date=end_date
)
return success(data=result)
@router.get("/workspace/api-statistics", summary="工作空间API调用统计")
@cur_workspace_access_guard()
def get_workspace_api_statistics(
start_date: int,
end_date: int,
db: Session = Depends(get_db),
current_user=Depends(get_current_user),
):
"""获取工作空间API调用统计
Args:
start_date: 开始时间戳(毫秒)
end_date: 结束时间戳(毫秒)
db: 数据库连接
current_user: 当前用户
Returns:
每日统计数据列表,每项包含:
- date: 日期
- total_calls: 当日总调用次数
- app_calls: 当日应用调用次数
- service_calls: 当日服务调用次数
"""
workspace_id = current_user.current_workspace_id
stats_service = AppStatisticsService(db)
result = stats_service.get_workspace_api_statistics(
workspace_id=workspace_id,
start_date=start_date,
end_date=end_date
)
return success(data=result)
@router.get("/{app_id}/export", summary="导出应用配置为 YAML 文件")
@cur_workspace_access_guard()
async def export_app(
app_id: uuid.UUID,
db: Annotated[Session, Depends(get_db)],
current_user: Annotated[User, Depends(get_current_user)],
release_id: Optional[uuid.UUID] = None
):
"""导出 agent / multi_agent / workflow 应用配置为 YAML 文件流。
release_id: 指定发布版本id不传则导出当前草稿配置。
"""
yaml_str, filename = AppDslService(db).export_dsl(app_id, release_id)
encoded = quote(filename, safe=".")
yaml_bytes = yaml_str.encode("utf-8")
file_stream = io.BytesIO(yaml_bytes)
file_stream.seek(0)
return StreamingResponse(
file_stream,
media_type="application/octet-stream; charset=utf-8",
headers={"Content-Disposition": f"attachment; filename={encoded}",
"Content-Length": str(len(yaml_bytes))}
)
@router.post("/import", summary="从 YAML 文件导入应用")
@cur_workspace_access_guard()
async def import_app(
file: UploadFile = File(...),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""从 YAML 文件导入 agent / multi_agent / workflow 应用。
跨空间/跨租户导入时,模型/工具/知识库会按名称匹配,匹配不到则置空并返回 warnings。
"""
if not file.filename.lower().endswith((".yaml", ".yml")):
return fail(msg="仅支持 YAML 文件", code=BizCode.BAD_REQUEST)
raw = (await file.read()).decode("utf-8")
dsl = yaml.safe_load(raw)
if not dsl or "app" not in dsl:
return fail(msg="YAML 格式无效,缺少 app 字段", code=BizCode.BAD_REQUEST)
new_app, warnings = AppDslService(db).import_dsl(
dsl=dsl,
workspace_id=current_user.current_workspace_id,
tenant_id=current_user.tenant_id,
user_id=current_user.id,
)
return success(
data={"app": app_schema.App.model_validate(new_app), "warnings": warnings},
msg="应用导入成功" + (",但部分资源需手动配置" if warnings else "")
)

View File

@@ -61,7 +61,6 @@ async def login_for_access_token(
user = auth_service.register_user_with_invite(
db=db,
email=form_data.email,
username=form_data.username,
password=form_data.password,
invite_token=form_data.invite,
workspace_id=invite_info.workspace_id

View File

@@ -441,14 +441,14 @@ async def retrieve_chunks(
# 1 participle search, 2 semantic search, 3 hybrid search
match retrieve_data.retrieve_type:
case chunk_schema.RetrieveType.PARTICIPLE:
rs = vector_service.search_by_full_text(query=retrieve_data.query, top_k=retrieve_data.top_k, indices=indices, score_threshold=retrieve_data.similarity_threshold, file_names_filter=retrieve_data.file_names_filter)
rs = vector_service.search_by_full_text(query=retrieve_data.query, top_k=retrieve_data.top_k, indices=indices, score_threshold=retrieve_data.similarity_threshold)
return success(data=rs, msg="retrieval successful")
case chunk_schema.RetrieveType.SEMANTIC:
rs = vector_service.search_by_vector(query=retrieve_data.query, top_k=retrieve_data.top_k, indices=indices, score_threshold=retrieve_data.vector_similarity_weight, file_names_filter=retrieve_data.file_names_filter)
rs = vector_service.search_by_vector(query=retrieve_data.query, top_k=retrieve_data.top_k, indices=indices, score_threshold=retrieve_data.vector_similarity_weight)
return success(data=rs, msg="retrieval successful")
case _:
rs1 = vector_service.search_by_vector(query=retrieve_data.query, top_k=retrieve_data.top_k, indices=indices, score_threshold=retrieve_data.vector_similarity_weight, file_names_filter=retrieve_data.file_names_filter)
rs2 = vector_service.search_by_full_text(query=retrieve_data.query, top_k=retrieve_data.top_k, indices=indices, score_threshold=retrieve_data.similarity_threshold, file_names_filter=retrieve_data.file_names_filter)
rs1 = vector_service.search_by_vector(query=retrieve_data.query, top_k=retrieve_data.top_k, indices=indices, score_threshold=retrieve_data.vector_similarity_weight)
rs2 = vector_service.search_by_full_text(query=retrieve_data.query, top_k=retrieve_data.top_k, indices=indices, score_threshold=retrieve_data.similarity_threshold)
# Efficient deduplication
seen_ids = set()
unique_rs = []

View File

@@ -11,7 +11,6 @@ Routes:
"""
from app.core.error_codes import BizCode
from app.core.language_utils import get_language_from_header
from app.core.logging_config import get_api_logger
from app.core.response_utils import fail, success
from app.dependencies import get_current_user, get_db
@@ -46,14 +45,11 @@ emotion_service = EmotionAnalyticsService()
@router.post("/tags", response_model=ApiResponse)
async def get_emotion_tags(
request: EmotionTagsRequest,
language_type: str = Header(default=None, alias="X-Language-Type"),
language_type: str = Header(default="zh", alias="X-Language-Type"),
current_user: User = Depends(get_current_user),
):
try:
# 使用集中化的语言校验
language = get_language_from_header(language_type)
api_logger.info(
f"用户 {current_user.username} 请求获取情绪标签统计",
extra={
@@ -61,8 +57,7 @@ async def get_emotion_tags(
"emotion_type": request.emotion_type,
"start_date": request.start_date,
"end_date": request.end_date,
"limit": request.limit,
"language_type": language
"limit": request.limit
}
)
@@ -72,8 +67,7 @@ async def get_emotion_tags(
emotion_type=request.emotion_type,
start_date=request.start_date,
end_date=request.end_date,
limit=request.limit,
language=language
limit=request.limit
)
api_logger.info(
@@ -103,14 +97,11 @@ async def get_emotion_tags(
@router.post("/wordcloud", response_model=ApiResponse)
async def get_emotion_wordcloud(
request: EmotionWordcloudRequest,
language_type: str = Header(default=None, alias="X-Language-Type"),
language_type: str = Header(default="zh", alias="X-Language-Type"),
current_user: User = Depends(get_current_user),
):
try:
# 使用集中化的语言校验
language = get_language_from_header(language_type)
api_logger.info(
f"用户 {current_user.username} 请求获取情绪词云数据",
extra={
@@ -153,14 +144,11 @@ async def get_emotion_wordcloud(
@router.post("/health", response_model=ApiResponse)
async def get_emotion_health(
request: EmotionHealthRequest,
language_type: str = Header(default=None, alias="X-Language-Type"),
language_type: str = Header(default="zh", alias="X-Language-Type"),
current_user: User = Depends(get_current_user),
):
try:
# 使用集中化的语言校验
language = get_language_from_header(language_type)
# 验证时间范围参数
if request.time_range not in ["7d", "30d", "90d"]:
raise HTTPException(
@@ -186,7 +174,7 @@ async def get_emotion_health(
"情绪健康指数获取成功",
extra={
"end_user_id": request.end_user_id,
"health_score": data.get("health_score") or 0,
"health_score": data.get("health_score", 0),
"level": data.get("level", "未知")
}
)
@@ -208,64 +196,14 @@ async def get_emotion_health(
# @router.post("/check-data", response_model=ApiResponse)
# async def check_emotion_data_exists(
# request: EmotionSuggestionsRequest,
# db: Session = Depends(get_db),
# current_user: User = Depends(get_current_user),
# ):
# """检查用户情绪建议数据是否存在
# Args:
# request: 包含 end_user_id
# db: 数据库会话
# current_user: 当前用户
# Returns:
# 数据存在状态
# """
# try:
# api_logger.info(
# f"检查用户情绪建议数据是否存在: {request.end_user_id}",
# extra={"end_user_id": request.end_user_id}
# )
# # 从数据库获取建议
# data = await emotion_service.get_cached_suggestions(
# end_user_id=request.end_user_id,
# db=db
# )
# if data is None:
# api_logger.info(f"用户 {request.end_user_id} 的情绪建议数据不存在")
# return fail(
# BizCode.NOT_FOUND,
# "情绪建议数据不存在,请点击右上角刷新进行初始化",
# {"exists": False}
# )
# api_logger.info(f"用户 {request.end_user_id} 的情绪建议数据存在")
# return success(data={"exists": True}, msg="情绪建议数据已存在")
# except Exception as e:
# api_logger.error(
# f"检查情绪建议数据失败: {str(e)}",
# extra={"end_user_id": request.end_user_id},
# exc_info=True
# )
# raise HTTPException(
# status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
# detail=f"检查情绪建议数据失败: {str(e)}"
# )
@router.post("/suggestions", response_model=ApiResponse)
async def get_emotion_suggestions(
request: EmotionSuggestionsRequest,
language_type: str = Header(default="zh", alias="X-Language-Type"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
):
"""获取个性化情绪建议(从数据库读取)
"""获取个性化情绪建议(从缓存读取)
Args:
request: 包含 end_user_id 和可选的 config_id
@@ -273,42 +211,44 @@ async def get_emotion_suggestions(
current_user: 当前用户
Returns:
的个性化情绪建议响应
存的个性化情绪建议响应
"""
try:
api_logger.info(
f"用户 {current_user.username} 请求获取个性化情绪建议",
f"用户 {current_user.username} 请求获取个性化情绪建议(缓存)",
extra={
"end_user_id": request.end_user_id,
"config_id": request.config_id
}
)
# 从数据库获取建议
# 从缓存获取建议
data = await emotion_service.get_cached_suggestions(
end_user_id=request.end_user_id,
db=db
)
if data is None:
# 缓存不存在或已过期
api_logger.info(
f"用户 {request.end_user_id} 的建议数据不存在",
f"用户 {request.end_user_id} 的建议缓存不存在或已过期",
extra={"end_user_id": request.end_user_id}
)
return success(
data={"exists": False},
msg="情绪建议数据不存在,请点击右上角刷新进行初始化"
return fail(
BizCode.NOT_FOUND,
"建议缓存不存在或已过期,请右上角刷新生成新建议",
""
)
api_logger.info(
"个性化建议获取成功",
"个性化建议获取成功(缓存)",
extra={
"end_user_id": request.end_user_id,
"suggestions_count": len(data.get("suggestions", []))
}
)
return success(data=data, msg="个性化建议获取成功")
return success(data=data, msg="个性化建议获取成功(缓存)")
except Exception as e:
api_logger.error(
@@ -325,11 +265,11 @@ async def get_emotion_suggestions(
@router.post("/generate_suggestions", response_model=ApiResponse)
async def generate_emotion_suggestions(
request: EmotionGenerateSuggestionsRequest,
language_type: str = Header(default=None, alias="X-Language-Type"),
language_type: str = Header(default="zh", alias="X-Language-Type"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
):
"""生成个性化情绪建议调用LLM并保存到数据库
"""生成个性化情绪建议调用LLM并缓存
Args:
request: 包含 end_user_id
@@ -340,9 +280,6 @@ async def generate_emotion_suggestions(
新生成的个性化情绪建议响应
"""
try:
# 使用集中化的语言校验
language = get_language_from_header(language_type)
api_logger.info(
f"用户 {current_user.username} 请求生成个性化情绪建议",
extra={
@@ -353,15 +290,15 @@ async def generate_emotion_suggestions(
# 调用服务层生成建议
data = await emotion_service.generate_emotion_suggestions(
end_user_id=request.end_user_id,
db=db,
language=language
db=db
)
# 保存到数据库
# 保存到缓存
await emotion_service.save_suggestions_cache(
end_user_id=request.end_user_id,
suggestions_data=data,
db=db
db=db,
expires_hours=24
)
api_logger.info(
@@ -383,4 +320,4 @@ async def generate_emotion_suggestions(
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"生成个性化建议失败: {str(e)}"
)
)

View File

@@ -29,7 +29,7 @@ from app.core.storage_exceptions import (
StorageUploadError,
)
from app.db import get_db
from app.dependencies import get_current_user, get_share_user_id, ShareTokenData
from app.dependencies import get_current_user
from app.models.file_metadata_model import FileMetadata
from app.models.user_model import User
from app.schemas.response_schema import ApiResponse
@@ -143,141 +143,6 @@ async def upload_file(
)
@router.post("/share/files", response_model=ApiResponse)
async def upload_file_with_share_token(
file: UploadFile = File(...),
db: Session = Depends(get_db),
share_data: ShareTokenData = Depends(get_share_user_id),
storage_service: FileStorageService = Depends(get_file_storage_service),
):
"""
Upload a file to the configured storage backend using share_token authentication.
"""
from app.services.release_share_service import ReleaseShareService
from app.models.app_model import App
from app.models.workspace_model import Workspace
# Get share and release info from share_token
service = ReleaseShareService(db)
share_info = service.get_shared_release_info(share_token=share_data.share_token)
# Get share object to access app_id
share = service.repo.get_by_share_token(share_data.share_token)
if not share:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Shared app not found"
)
# Get app to access workspace_id
app = db.query(App).filter(
App.id == share.app_id,
App.is_active.is_(True)
).first()
if not app:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="App not found"
)
# Get workspace to access tenant_id
workspace = db.query(Workspace).filter(
Workspace.id == app.workspace_id
).first()
if not workspace:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Workspace not found"
)
tenant_id = workspace.tenant_id
workspace_id = app.workspace_id
api_logger.info(
f"Storage upload request (share): tenant_id={tenant_id}, workspace_id={workspace_id}, "
f"filename={file.filename}, share_token={share_data.share_token}"
)
# Read file contents
contents = await file.read()
file_size = len(contents)
# Validate file size
if file_size == 0:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="The file is empty."
)
if file_size > settings.MAX_FILE_SIZE:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"The file size exceeds the {settings.MAX_FILE_SIZE} byte limit"
)
# Extract file extension
_, file_extension = os.path.splitext(file.filename)
file_ext = file_extension.lower()
# Generate file_id and file_key
file_id = uuid.uuid4()
file_key = generate_file_key(
tenant_id=tenant_id,
workspace_id=workspace_id,
file_id=file_id,
file_ext=file_ext,
)
# Create file metadata record with pending status
file_metadata = FileMetadata(
id=file_id,
tenant_id=tenant_id,
workspace_id=workspace_id,
file_key=file_key,
file_name=file.filename,
file_ext=file_ext,
file_size=file_size,
content_type=file.content_type,
status="pending",
)
db.add(file_metadata)
db.commit()
db.refresh(file_metadata)
# Upload file to storage backend
try:
await storage_service.upload_file(
tenant_id=tenant_id,
workspace_id=workspace_id,
file_id=file_id,
file_ext=file_ext,
content=contents,
content_type=file.content_type,
)
# Update status to completed
file_metadata.status = "completed"
db.commit()
api_logger.info(f"File uploaded to storage (share): file_key={file_key}")
except StorageUploadError as e:
# Update status to failed
file_metadata.status = "failed"
db.commit()
api_logger.error(f"Storage upload failed (share): {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"File storage failed: {str(e)}"
)
api_logger.info(f"File upload successful (share): {file.filename} (file_id: {file_id})")
return success(
data={"file_id": str(file_id), "file_key": file_key},
msg="File upload successful"
)
@router.get("/files/{file_id}", response_model=Any)
async def download_file(
file_id: uuid.UUID,

View File

@@ -122,48 +122,6 @@ def validate_confidence_threshold(threshold: float) -> None:
raise ValueError("confidence_threshold must be between 0.0 and 1.0")
@router.get("/check-data/{end_user_id}", response_model=ApiResponse)
@cur_workspace_access_guard()
async def check_user_data_exists(
end_user_id: str,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
) -> ApiResponse:
"""
检查用户画像数据是否存在
Args:
end_user_id: 目标用户ID
Returns:
数据存在状态
"""
api_logger.info(f"检查用户画像数据是否存在: {end_user_id}")
try:
# Validate inputs
validate_user_id(end_user_id)
# Create service with user-specific config
service = ImplicitMemoryService(db=db, end_user_id=end_user_id)
# Get cached profile
cached_profile = await service.get_cached_profile(end_user_id=end_user_id, db=db)
if cached_profile is None:
api_logger.info(f"用户 {end_user_id} 的画像数据不存在")
return success(
data={"exists": False},
msg="画像数据不存在,请点击右上角刷新进行初始化"
)
api_logger.info(f"用户 {end_user_id} 的画像数据存在")
return success(data={"exists": True}, msg="画像数据已存在")
except Exception as e:
return handle_implicit_memory_error(e, "检查画像数据", end_user_id)
@router.get("/preferences/{end_user_id}", response_model=ApiResponse)
@cur_workspace_access_guard()
async def get_preference_tags(
@@ -201,8 +159,12 @@ async def get_preference_tags(
cached_profile = await service.get_cached_profile(end_user_id=end_user_id, db=db)
if cached_profile is None:
api_logger.info(f"用户 {end_user_id} 的画像数据不存在")
return fail(BizCode.NOT_FOUND, "", "")
api_logger.info(f"用户 {end_user_id} 的画像缓存不存在或已过期")
return fail(
BizCode.NOT_FOUND,
"画像缓存不存在或已过期,请右上角刷新生成新画像",
""
)
# Extract preferences from cache
preferences = cached_profile.get("preferences", [])
@@ -268,8 +230,12 @@ async def get_dimension_portrait(
cached_profile = await service.get_cached_profile(end_user_id=end_user_id, db=db)
if cached_profile is None:
api_logger.info(f"用户 {end_user_id} 的画像数据不存在")
return fail(BizCode.NOT_FOUND, "", "")
api_logger.info(f"用户 {end_user_id} 的画像缓存不存在或已过期")
return fail(
BizCode.NOT_FOUND,
"画像缓存不存在或已过期,请右上角刷新生成新画像",
""
)
# Extract portrait from cache
portrait = cached_profile.get("portrait", {})
@@ -312,8 +278,12 @@ async def get_interest_area_distribution(
cached_profile = await service.get_cached_profile(end_user_id=end_user_id, db=db)
if cached_profile is None:
api_logger.info(f"用户 {end_user_id} 的画像数据不存在")
return fail(BizCode.NOT_FOUND, "", "")
api_logger.info(f"用户 {end_user_id} 的画像缓存不存在或已过期")
return fail(
BizCode.NOT_FOUND,
"画像缓存不存在或已过期,请右上角刷新生成新画像",
""
)
# Extract interest areas from cache
interest_areas = cached_profile.get("interest_areas", {})
@@ -360,8 +330,12 @@ async def get_behavior_habits(
cached_profile = await service.get_cached_profile(end_user_id=end_user_id, db=db)
if cached_profile is None:
api_logger.info(f"用户 {end_user_id} 的画像数据不存在")
return fail(BizCode.NOT_FOUND, "", "")
api_logger.info(f"用户 {end_user_id} 的画像缓存不存在或已过期")
return fail(
BizCode.NOT_FOUND,
"画像缓存不存在或已过期,请右上角刷新生成新画像",
""
)
# Extract habits from cache
habits = cached_profile.get("habits", [])

View File

@@ -9,16 +9,13 @@ from sqlalchemy import or_
from sqlalchemy.orm import Session
from app.celery_app import celery_app
from app.core.error_codes import BizCode
from app.core.logging_config import get_api_logger
from app.core.rag.common import settings
from app.core.rag.integrations.feishu.client import FeishuAPIClient
from app.core.rag.integrations.yuque.client import YuqueAPIClient
from app.core.rag.llm.chat_model import Base
from app.core.rag.nlp import rag_tokenizer, search
from app.core.rag.prompts.generator import graph_entity_types
from app.core.rag.vdb.elasticsearch.elasticsearch_vector import ElasticSearchVectorFactory
from app.core.response_utils import success, fail
from app.core.response_utils import success
from app.db import get_db
from app.dependencies import get_current_user
from app.models import knowledge_model
@@ -487,99 +484,3 @@ async def rebuild_knowledge_graph(
except Exception as e:
api_logger.error(f"Failed to rebuild knowledge graph: knowledge_id={knowledge_id} - {str(e)}")
raise
@router.get("/check/yuque/auth", response_model=ApiResponse)
async def check_yuque_auth(
yuque_user_id: str,
yuque_token: str,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
check yuque auth info
"""
api_logger.info(f"check yuque auth info, username: {current_user.username}")
try:
api_client = YuqueAPIClient(
user_id=yuque_user_id,
token=yuque_token
)
async with api_client as client:
repos = await client.get_user_repos()
if repos:
return success(msg="Successfully auth yuque info")
return fail(BizCode.UNAUTHORIZED, msg="auth yuque info failed", error="user_id or token is incorrect")
except HTTPException:
raise
except Exception as e:
api_logger.error(f"auth yuque info failed: {str(e)}")
raise
@router.get("/check/feishu/auth", response_model=ApiResponse)
async def check_feishu_auth(
feishu_app_id: str,
feishu_app_secret: str,
feishu_folder_token: str,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
check feishu auth info
"""
api_logger.info(f"check feishu auth info, username: {current_user.username}")
try:
api_client = FeishuAPIClient(
app_id=feishu_app_id,
app_secret=feishu_app_secret
)
async with api_client as client:
files = await client.list_all_folder_files(feishu_folder_token, recursive=True)
if files:
return success(msg="Successfully auth feishu info")
return fail(BizCode.UNAUTHORIZED, msg="auth feishu info failed", error="app_id or app_secret or feishu_folder_token is incorrect")
except HTTPException:
raise
except Exception as e:
api_logger.error(f"auth feishu info failed: {str(e)}")
raise
@router.post("/{knowledge_id}/sync", response_model=ApiResponse)
async def sync_knowledge(
knowledge_id: uuid.UUID,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
sync knowledge base information based on knowledge_id
"""
api_logger.info(f"Obtain details of the knowledge base: knowledge_id={knowledge_id}, username: {current_user.username}")
try:
# 1. Query knowledge base information from the database
api_logger.debug(f"Query knowledge base: {knowledge_id}")
db_knowledge = knowledge_service.get_knowledge_by_id(db, knowledge_id=knowledge_id, current_user=current_user)
if not db_knowledge:
api_logger.warning(f"The knowledge base does not exist or access is denied: knowledge_id={knowledge_id}")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="The knowledge base does not exist or access is denied"
)
# 2. sync knowledge
# from app.tasks import sync_knowledge_for_kb
# sync_knowledge_for_kb(kb_id)
task = celery_app.send_task("app.core.rag.tasks.sync_knowledge_for_kb", args=[knowledge_id])
result = {
"task_id": task.id
}
return success(data=result, msg="Task accepted. sync knowledge is being processed in the background.")
except HTTPException:
raise
except Exception as e:
api_logger.error(f"Failed to sync knowledge: knowledge_id={knowledge_id} - {str(e)}")
raise

View File

@@ -1,430 +0,0 @@
import datetime
import json
from typing import Optional
import uuid
from fastapi import APIRouter, Depends, HTTPException, status, Query
from fastapi.encoders import jsonable_encoder
import requests
from sqlalchemy import or_
from sqlalchemy.orm import Session
from modelscope.hub.errors import raise_for_http_status
from modelscope.hub.mcp_api import MCPApi
from app.core.logging_config import get_api_logger
from app.core.response_utils import success, fail
from app.db import get_db
from app.dependencies import get_current_user
from app.models import mcp_market_config_model
from app.models.user_model import User
from app.schemas import mcp_market_config_schema
from app.schemas.response_schema import ApiResponse
from app.services import mcp_market_config_service
# Obtain a dedicated API logger
api_logger = get_api_logger()
router = APIRouter(
prefix="/mcp_market_configs",
tags=["mcp_market_configs"],
dependencies=[Depends(get_current_user)] # Apply auth to all routes in this controller
)
@router.get("/mcp_servers", response_model=ApiResponse)
async def get_mcp_servers(
mcp_market_config_id: uuid.UUID,
page: int = Query(1, gt=0), # Default: 1, which must be greater than 0
pagesize: int = Query(20, gt=0, le=100), # Default: 20 items per page, maximum: 100 items
keywords: Optional[str] = Query(None, description="Search keywords (Optional search query string,e.g. Chinese service name, English service name, author/owner username)"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
Query the mcp servers list in pages
- Support keyword search for name,author,owner
- Return paging metadata + mcp server list
"""
api_logger.info(
f"Query mcp server list: tenant_id={current_user.tenant_id}, page={page}, pagesize={pagesize}, keywords={keywords}, username: {current_user.username}")
# 1. parameter validation
if page < 1 or pagesize < 1:
api_logger.warning(f"Error in paging parameters: page={page}, pagesize={pagesize}")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="The paging parameter must be greater than 0"
)
if page * pagesize > 100:
api_logger.warning(f"Paging parameters exceed ModelScope limit: page={page}, pagesize={pagesize}")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"The maximum number of MCP services can view is 100. Please visit the ModelScope MCP Plaza."
)
# 2. Query mcp market config information from the database
api_logger.debug(f"Query mcp market config: {mcp_market_config_id}")
db_mcp_market_config = mcp_market_config_service.get_mcp_market_config_by_id(db,
mcp_market_config_id=mcp_market_config_id,
current_user=current_user)
if not db_mcp_market_config:
api_logger.warning(
f"The mcp market config does not exist or access is denied: mcp_market_config_id={mcp_market_config_id}")
return success(msg='The mcp market config does not exist or access is denied')
# 3. Execute paged query
token = db_mcp_market_config.token
if not token:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="MCP market config token is not configured"
)
api = MCPApi()
api.login(token)
body = {
'filter': {},
'page_number': page,
'page_size': pagesize,
'search': keywords
}
try:
cookies = api.get_cookies(token)
r = api.session.put(
url=api.mcp_base_url,
headers=api.builder_headers(api.headers),
json=body,
cookies=cookies)
raise_for_http_status(r)
except requests.exceptions.RequestException as e:
api_logger.error(f"Failed to get MCP servers: {str(e)}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to get MCP servers: {str(e)}"
)
data = api._handle_response(r)
total = data.get('total_count', 0)
mcp_server_list = data.get('mcp_server_list', [])
# items = [{
# 'name': item.get('name', ''),
# 'id': item.get('id', ''),
# 'description': item.get('description', '')
# } for item in mcp_server_list]
# 4. Return structured response
result = {
"items": mcp_server_list,
"page": {
"page": page,
"pagesize": pagesize,
"total": total,
"has_next": True if page * pagesize < total else False
}
}
return success(data=result, msg="Query of mcp servers list successful")
@router.get("/operational_mcp_servers", response_model=ApiResponse)
async def get_operational_mcp_servers(
mcp_market_config_id: uuid.UUID,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
Query the operational mcp servers list in pages
- Support keyword search for name,author,owner
- Return paging metadata + operational mcp server list
"""
api_logger.info(
f"Query operational mcp server list: tenant_id={current_user.tenant_id}, username: {current_user.username}")
# 1. Query mcp market config information from the database
api_logger.debug(f"Query mcp market config: {mcp_market_config_id}")
db_mcp_market_config = mcp_market_config_service.get_mcp_market_config_by_id(db,
mcp_market_config_id=mcp_market_config_id,
current_user=current_user)
if not db_mcp_market_config:
api_logger.warning(
f"The mcp market config does not exist or access is denied: mcp_market_config_id={mcp_market_config_id}")
return success(msg='The mcp market config does not exist or access is denied')
# 2. Execute paged query
token = db_mcp_market_config.token
if not token:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="MCP market config token is not configured"
)
api = MCPApi()
api.login(token)
url = f'{api.mcp_base_url}/operational'
headers = api.builder_headers(api.headers)
try:
cookies = api.get_cookies(access_token=token, cookies_required=True)
r = api.session.get(url, headers=headers, cookies=cookies)
raise_for_http_status(r)
except requests.exceptions.RequestException as e:
api_logger.error(f"Failed to get operational MCP servers: {str(e)}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to get operational MCP servers: {str(e)}"
)
data = api._handle_response(r)
total = data.get('total_count', 0)
mcp_server_list = data.get('mcp_server_list', [])
# items = [{
# 'name': item.get('name', ''),
# 'id': item.get('id', ''),
# 'description': item.get('description', '')
# } for item in mcp_server_list]
# 3. Return structured response
return success(data=mcp_server_list, msg="Query of operational mcp servers list successful")
@router.get("/mcp_server", response_model=ApiResponse)
async def get_mcp_server(
mcp_market_config_id: uuid.UUID,
server_id: str,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
Get detailed information for a specific MCP Server
"""
api_logger.info(
f"Query mcp server: tenant_id={current_user.tenant_id}, mcp_market_config_id={mcp_market_config_id}, server_id={server_id}, username: {current_user.username}")
# 1. Query mcp market config information from the database
api_logger.debug(f"Query mcp market config: {mcp_market_config_id}")
db_mcp_market_config = mcp_market_config_service.get_mcp_market_config_by_id(db,
mcp_market_config_id=mcp_market_config_id,
current_user=current_user)
if not db_mcp_market_config:
api_logger.warning(
f"The mcp market config does not exist or access is denied: mcp_market_config_id={mcp_market_config_id}")
return success(msg='The mcp market config does not exist or access is denied')
# 2. Get detailed information for a specific MCP Server
token = db_mcp_market_config.token
if not token:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="MCP market config token is not configured"
)
api = MCPApi()
api.login(token)
result = api.get_mcp_server(server_id=server_id)
return success(data=result, msg="Query of mcp servers list successful")
@router.post("/mcp_market_config", response_model=ApiResponse)
async def create_mcp_market_config(
create_data: mcp_market_config_schema.McpMarketConfigCreate,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
create mcp market config
"""
api_logger.info(
f"Request to create a mcp market config: mcp_market_id={create_data.mcp_market_id}, tenant_id={current_user.tenant_id}, username: {current_user.username}")
try:
api_logger.debug(f"Start creating the mcp market config: {create_data.mcp_market_id}")
# 1. Validate token can access ModelScope MCP market
if not create_data.token:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Token is required to access ModelScope MCP market"
)
try:
api = MCPApi()
api.login(create_data.token)
body = {'filter': {}, 'page_number': 1, 'page_size': 1, 'search': None}
cookies = api.get_cookies(create_data.token)
r = api.session.put(url=api.mcp_base_url, headers=api.builder_headers(api.headers), json=body, cookies=cookies)
raise_for_http_status(r)
except Exception as e:
api_logger.warning(f"Token validation failed for ModelScope MCP market: {str(e)}")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Unable to access ModelScope MCP market with the provided token: {str(e)}"
)
# 2. Check if the mcp market name already exists
db_mcp_market_config_exist = mcp_market_config_service.get_mcp_market_config_by_mcp_market_id(db, mcp_market_id=create_data.mcp_market_id, current_user=current_user)
if db_mcp_market_config_exist:
api_logger.warning(f"The mcp market id already exists: {create_data.mcp_market_id}")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"The mcp market id already exists: {create_data.mcp_market_id}"
)
db_mcp_market_config = mcp_market_config_service.create_mcp_market_config(db=db, mcp_market_config=create_data, current_user=current_user)
api_logger.info(
f"The mcp market config has been successfully created: (ID: {db_mcp_market_config.id})")
return success(data=jsonable_encoder(mcp_market_config_schema.McpMarketConfig.model_validate(db_mcp_market_config)),
msg="The mcp market config has been successfully created")
except Exception as e:
api_logger.error(f"The creation of the mcp market config failed: {create_data.mcp_market_id} - {str(e)}")
raise
@router.get("/{mcp_market_config_id}", response_model=ApiResponse)
async def get_mcp_market_config(
mcp_market_config_id: uuid.UUID,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
Retrieve mcp market config information based on mcp_market_config_id
"""
api_logger.info(
f"Obtain details of the mcp market config: mcp_market_config_id={mcp_market_config_id}, username: {current_user.username}")
try:
# 1. Query mcp market config information from the database
api_logger.debug(f"Query mcp market config: {mcp_market_config_id}")
db_mcp_market_config = mcp_market_config_service.get_mcp_market_config_by_id(db, mcp_market_config_id=mcp_market_config_id, current_user=current_user)
if not db_mcp_market_config:
api_logger.warning(f"The mcp market config does not exist or access is denied: mcp_market_config_id={mcp_market_config_id}")
return success(msg='The mcp market config does not exist or access is denied')
api_logger.info(f"mcp market config query successful: (ID: {db_mcp_market_config.id})")
return success(data=jsonable_encoder(mcp_market_config_schema.McpMarketConfig.model_validate(db_mcp_market_config)),
msg="Successfully obtained mcp market config information")
except HTTPException:
raise
except Exception as e:
api_logger.error(f"mcp market config query failed: mcp_market_config_id={mcp_market_config_id} - {str(e)}")
raise
@router.get("/mcp_market_id/{mcp_market_id}", response_model=ApiResponse)
async def get_mcp_market_config_by_mcp_market_id(
mcp_market_id: uuid.UUID,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
Retrieve mcp market config information based on mcp_market_id
"""
api_logger.info(
f"Request to create a mcp market config: mcp_market_id={mcp_market_id}, tenant_id={current_user.tenant_id}, username: {current_user.username}")
try:
# 1. Query mcp market config information from the database
api_logger.debug(f"Query mcp market config: mcp_market_id={mcp_market_id}")
db_mcp_market_config = mcp_market_config_service.get_mcp_market_config_by_mcp_market_id(db, mcp_market_id=mcp_market_id, current_user=current_user)
if not db_mcp_market_config:
api_logger.warning(f"The mcp market config does not exist or access is denied: mcp_market_id={mcp_market_id}")
return success(msg='The mcp market config does not exist or access is denied')
api_logger.info(f"mcp market config query successful: (ID: {db_mcp_market_config.id})")
return success(data=jsonable_encoder(mcp_market_config_schema.McpMarketConfig.model_validate(db_mcp_market_config)),
msg="Successfully obtained mcp market config information")
except HTTPException:
raise
except Exception as e:
api_logger.error(f"mcp market config query failed: mcp_market_id={mcp_market_id} - {str(e)}")
raise
@router.put("/{mcp_market_config_id}", response_model=ApiResponse)
async def update_mcp_market_config(
mcp_market_config_id: uuid.UUID,
update_data: mcp_market_config_schema.McpMarketConfigUpdate,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
# 1. Check if the mcp market config exists
api_logger.debug(f"Query the mcp market config to be updated: {mcp_market_config_id}")
db_mcp_market_config = mcp_market_config_service.get_mcp_market_config_by_id(db, mcp_market_config_id=mcp_market_config_id, current_user=current_user)
if not db_mcp_market_config:
api_logger.warning(
f"The mcp market config does not exist or you do not have permission to access it: mcp_market_config_id={mcp_market_config_id}")
return success(msg='The mcp market config does not exist or access is denied')
# 2. Validate new token if provided
if update_data.token is not None:
try:
api = MCPApi()
api.login(update_data.token)
body = {'filter': {}, 'page_number': 1, 'page_size': 1, 'search': None}
cookies = api.get_cookies(update_data.token)
r = api.session.put(url=api.mcp_base_url, headers=api.builder_headers(api.headers), json=body, cookies=cookies)
raise_for_http_status(r)
except Exception as e:
api_logger.warning(f"Token validation failed for ModelScope MCP market: {str(e)}")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Unable to access ModelScope MCP market with the provided token: {str(e)}"
)
# 3. Update fields (only update non-null fields)
api_logger.debug(f"Start updating the mcp market config fields: {mcp_market_config_id}")
update_dict = update_data.dict(exclude_unset=True)
updated_fields = []
for field, value in update_dict.items():
if hasattr(db_mcp_market_config, field):
old_value = getattr(db_mcp_market_config, field)
if old_value != value:
# update value
setattr(db_mcp_market_config, field, value)
updated_fields.append(f"{field}: {old_value} -> {value}")
if updated_fields:
api_logger.debug(f"updated fields: {', '.join(updated_fields)}")
# 4. Save to database
try:
db.commit()
db.refresh(db_mcp_market_config)
api_logger.info(f"The mcp market config has been successfully updated: (ID: {db_mcp_market_config.id})")
except Exception as e:
db.rollback()
api_logger.error(f"The mcp market config update failed: mcp_market_config_id={mcp_market_config_id} - {str(e)}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"The mcp market config update failed: {str(e)}"
)
# 4. Return the updated mcp market config
return success(data=jsonable_encoder(mcp_market_config_schema.McpMarketConfig.model_validate(db_mcp_market_config)),
msg="The mcp market config information updated successfully")
@router.delete("/{mcp_market_config_id}", response_model=ApiResponse)
async def delete_mcp_market_config(
mcp_market_config_id: uuid.UUID,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
delete mcp market config
"""
api_logger.info(f"Request to delete mcp market config: mcp_market_config_id={mcp_market_config_id}, username: {current_user.username}")
try:
# 1. Check whether the mcp market config exists
api_logger.debug(f"Check whether the mcp market config exists: {mcp_market_config_id}")
db_mcp_market_config = mcp_market_config_service.get_mcp_market_config_by_id(db, mcp_market_config_id=mcp_market_config_id, current_user=current_user)
if not db_mcp_market_config:
api_logger.warning(
f"The mcp market config does not exist or you do not have permission to access it: mcp_market_config_id={mcp_market_config_id}")
return success(msg='The mcp market config does not exist or access is denied')
# 2. Deleting mcp market config
mcp_market_config_service.delete_mcp_market_config_by_id(db, mcp_market_config_id=mcp_market_config_id, current_user=current_user)
api_logger.info(f"The mcp market config has been successfully deleted: (ID: {mcp_market_config_id})")
return success(msg="The mcp market config has been successfully deleted")
except Exception as e:
api_logger.error(f"Failed to delete from the mcp market config: mcp_market_config_id={mcp_market_config_id} - {str(e)}")
raise

View File

@@ -1,262 +0,0 @@
import datetime
import json
from typing import Optional
import uuid
from fastapi import APIRouter, Depends, HTTPException, status, Query
from fastapi.encoders import jsonable_encoder
from sqlalchemy import or_
from sqlalchemy.orm import Session
from app.core.logging_config import get_api_logger
from app.core.response_utils import success, fail
from app.db import get_db
from app.dependencies import get_current_user
from app.models import mcp_market_model
from app.models.user_model import User
from app.schemas import mcp_market_schema
from app.schemas.response_schema import ApiResponse
from app.services import mcp_market_service
# Obtain a dedicated API logger
api_logger = get_api_logger()
router = APIRouter(
prefix="/mcp_markets",
tags=["mcp_markets"],
dependencies=[Depends(get_current_user)] # Apply auth to all routes in this controller
)
@router.get("/mcp_markets", response_model=ApiResponse)
async def get_mcp_markets(
page: int = Query(1, gt=0), # Default: 1, which must be greater than 0
pagesize: int = Query(20, gt=0, le=100), # Default: 20 items per page, maximum: 100 items
orderby: Optional[str] = Query(None, description="Sort fields, such as: category, created_at"),
desc: Optional[bool] = Query(False, description="Is it descending order"),
keywords: Optional[str] = Query(None, description="Search keywords (mcp_market base name)"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
Query the mcp markets list in pages
- Support keyword search for name,description
- Support dynamic sorting
- Return paging metadata + mcp_market list
"""
api_logger.info(
f"Query mcp market list: tenant_id={current_user.tenant_id}, page={page}, pagesize={pagesize}, keywords={keywords}, username: {current_user.username}")
# 1. parameter validation
if page < 1 or pagesize < 1:
api_logger.warning(f"Error in paging parameters: page={page}, pagesize={pagesize}")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="The paging parameter must be greater than 0"
)
# 2. Construct query conditions
filters = []
# Keyword search (fuzzy matching of mcp market name,description)
if keywords:
api_logger.debug(f"Add keyword search criteria: {keywords}")
filters.append(
or_(
mcp_market_model.McpMarket.name.ilike(f"%{keywords}%"),
mcp_market_model.McpMarket.description.ilike(f"%{keywords}%")
)
)
# 3. Execute paged query
try:
api_logger.debug("Start executing mcp market paging query")
total, items = mcp_market_service.get_mcp_markets_paginated(
db=db,
filters=filters,
page=page,
pagesize=pagesize,
orderby=orderby,
desc=desc,
current_user=current_user
)
api_logger.info(f"mcp market query successful: total={total}, returned={len(items)} records")
except Exception as e:
api_logger.error(f"mcp market query failed: {str(e)}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Query failed: {str(e)}"
)
# 4. Return structured response
result = {
"items": items,
"page": {
"page": page,
"pagesize": pagesize,
"total": total,
"has_next": True if page * pagesize < total else False
}
}
return success(data=jsonable_encoder(result), msg="Query of mcp market list successful")
@router.post("/mcp_market", response_model=ApiResponse)
async def create_mcp_market(
create_data: mcp_market_schema.McpMarketCreate,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
create mcp market
"""
api_logger.info(
f"Request to create a mcp market: name={create_data.name}, tenant_id={current_user.tenant_id}, username: {current_user.username}")
try:
api_logger.debug(f"Start creating the mcp market: {create_data.name}")
# 1. Check if the mcp market name already exists
db_mcp_market_exist = mcp_market_service.get_mcp_market_by_name(db, name=create_data.name, current_user=current_user)
if db_mcp_market_exist:
api_logger.warning(f"The mcp market name already exists: {create_data.name}")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"The mcp market name already exists: {create_data.name}"
)
db_mcp_market = mcp_market_service.create_mcp_market(db=db, mcp_market=create_data, current_user=current_user)
api_logger.info(
f"The mcp market has been successfully created: {db_mcp_market.name} (ID: {db_mcp_market.id})")
return success(data=jsonable_encoder(mcp_market_schema.McpMarket.model_validate(db_mcp_market)),
msg="The mcp market has been successfully created")
except Exception as e:
api_logger.error(f"The creation of the mcp market failed: {create_data.name} - {str(e)}")
raise
@router.get("/{mcp_market_id}", response_model=ApiResponse)
async def get_mcp_market(
mcp_market_id: uuid.UUID,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
Retrieve mcp market information based on mcp_market_id
"""
api_logger.info(
f"Obtain details of the mcp market: mcp_market_id={mcp_market_id}, username: {current_user.username}")
try:
# 1. Query mcp market information from the database
api_logger.debug(f"Query mcp market: {mcp_market_id}")
db_mcp_market = mcp_market_service.get_mcp_market_by_id(db, mcp_market_id=mcp_market_id, current_user=current_user)
if not db_mcp_market:
api_logger.warning(f"The mcp market does not exist or access is denied: mcp_market_id={mcp_market_id}")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="The mcp market does not exist or access is denied"
)
api_logger.info(f"mcp market query successful: {db_mcp_market.name} (ID: {db_mcp_market.id})")
return success(data=jsonable_encoder(mcp_market_schema.McpMarket.model_validate(db_mcp_market)),
msg="Successfully obtained mcp market information")
except HTTPException:
raise
except Exception as e:
api_logger.error(f"mcp market query failed: mcp_market_id={mcp_market_id} - {str(e)}")
raise
@router.put("/{mcp_market_id}", response_model=ApiResponse)
async def update_mcp_market(
mcp_market_id: uuid.UUID,
update_data: mcp_market_schema.McpMarketUpdate,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
# 1. Check if the mcp market exists
api_logger.debug(f"Query the mcp market to be updated: {mcp_market_id}")
db_mcp_market = mcp_market_service.get_mcp_market_by_id(db, mcp_market_id=mcp_market_id, current_user=current_user)
if not db_mcp_market:
api_logger.warning(
f"The mcp market does not exist or you do not have permission to access it: mcp_market_id={mcp_market_id}")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="The mcp market does not exist or you do not have permission to access it"
)
# 2. not updating the name (name already exists)
update_dict = update_data.dict(exclude_unset=True)
if "name" in update_dict:
name = update_dict["name"]
if name != db_mcp_market.name:
# Check if the mcp market name already exists
db_mcp_market_exist = mcp_market_service.get_mcp_market_by_name(db, name=name, current_user=current_user)
if db_mcp_market_exist:
api_logger.warning(f"The mcp market name already exists: {name}")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"The mcp market name already exists: {name}"
)
# 3. Update fields (only update non-null fields)
api_logger.debug(f"Start updating the mcp market fields: {mcp_market_id}")
updated_fields = []
for field, value in update_dict.items():
if hasattr(db_mcp_market, field):
old_value = getattr(db_mcp_market, field)
if old_value != value:
# update value
setattr(db_mcp_market, field, value)
updated_fields.append(f"{field}: {old_value} -> {value}")
if updated_fields:
api_logger.debug(f"updated fields: {', '.join(updated_fields)}")
# 4. Save to database
try:
db.commit()
db.refresh(db_mcp_market)
api_logger.info(f"The mcp market has been successfully updated: {db_mcp_market.name} (ID: {db_mcp_market.id})")
except Exception as e:
db.rollback()
api_logger.error(f"The mcp market update failed: mcp_market_id={mcp_market_id} - {str(e)}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"The mcp market update failed: {str(e)}"
)
# 5. Return the updated mcp market
return success(data=jsonable_encoder(mcp_market_schema.McpMarket.model_validate(db_mcp_market)),
msg="The mcp market information updated successfully")
@router.delete("/{mcp_market_id}", response_model=ApiResponse)
async def delete_mcp_market(
mcp_market_id: uuid.UUID,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
delete mcp market
"""
api_logger.info(f"Request to delete mcp market: mcp_market_id={mcp_market_id}, username: {current_user.username}")
try:
# 1. Check whether the mcp market exists
api_logger.debug(f"Check whether the mcp market exists: {mcp_market_id}")
db_mcp_market = mcp_market_service.get_mcp_market_by_id(db, mcp_market_id=mcp_market_id, current_user=current_user)
if not db_mcp_market:
api_logger.warning(
f"The mcp market does not exist or you do not have permission to access it: mcp_market_id={mcp_market_id}")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="The mcp market does not exist or you do not have permission to access it"
)
# 2. Deleting mcp market
mcp_market_service.delete_mcp_market_by_id(db, mcp_market_id=mcp_market_id, current_user=current_user)
api_logger.info(f"The mcp market has been successfully deleted: (ID: {mcp_market_id})")
return success(msg="The mcp market has been successfully deleted")
except Exception as e:
api_logger.error(f"Failed to delete from the mcp market: mcp_market_id={mcp_market_id} - {str(e)}")
raise

View File

@@ -1,29 +1,26 @@
from typing import List, Optional
from dotenv import load_dotenv
from fastapi import APIRouter, Depends, File, Form, Query, UploadFile, Header
from sqlalchemy.orm import Session
from starlette.responses import StreamingResponse
from app.cache.memory.interest_memory import InterestMemoryCache
from app.celery_app import celery_app
from app.core.error_codes import BizCode
from app.core.language_utils import get_language_from_header
from app.core.logging_config import get_api_logger
from app.core.memory.agent.utils.redis_tool import store
from app.core.memory.agent.utils.session_tools import SessionService
from app.core.rag.llm.cv_model import QWenCV
from app.core.response_utils import fail, success
from app.db import get_db
from app.dependencies import cur_workspace_access_guard, get_current_user
from app.models import ModelApiKey
from app.models.user_model import User
from app.repositories import knowledge_repository
from app.core.memory.agent.utils.session_tools import SessionService
from app.core.memory.agent.utils.redis_tool import store
from app.repositories import knowledge_repository, WorkspaceRepository
from app.schemas.memory_agent_schema import UserInput, Write_UserInput
from app.schemas.response_schema import ApiResponse
from app.services import task_service, workspace_service
from app.services.memory_agent_service import MemoryAgentService
from app.services.model_service import ModelConfigService
from dotenv import load_dotenv
from fastapi import APIRouter, Depends, File, Form, Query, UploadFile,Header
from sqlalchemy.orm import Session
from starlette.responses import StreamingResponse
load_dotenv()
api_logger = get_api_logger()
@@ -38,7 +35,7 @@ router = APIRouter(
@router.get("/health/status", response_model=ApiResponse)
async def get_health_status(
current_user: User = Depends(get_current_user)
current_user: User = Depends(get_current_user)
):
"""
Get latest health status written by Celery periodic task
@@ -56,9 +53,8 @@ async def get_health_status(
@router.get("/download_log")
async def download_log(
log_type: str = Query("file", regex="^(file|transmission)$",
description="日志类型: file=完整文件, transmission=实时流式传输"),
current_user: User = Depends(get_current_user)
log_type: str = Query("file", regex="^(file|transmission)$", description="日志类型: file=完整文件, transmission=实时流式传输"),
current_user: User = Depends(get_current_user)
):
"""
Download or stream agent service log file
@@ -77,16 +73,16 @@ async def download_log(
- transmission mode: StreamingResponse with SSE
"""
api_logger.info(f"Log download requested with log_type={log_type}")
# Validate log_type parameter (FastAPI Query regex already validates, but explicit check for clarity)
if log_type not in ["file", "transmission"]:
api_logger.warning(f"Invalid log_type parameter: {log_type}")
return fail(
BizCode.BAD_REQUEST,
"无效的log_type参数",
BizCode.BAD_REQUEST,
"无效的log_type参数",
"log_type必须是'file''transmission'"
)
# Route to appropriate mode
if log_type == "file":
# File mode: Return complete log file content
@@ -121,28 +117,23 @@ async def download_log(
@router.post("/writer_service", response_model=ApiResponse)
@cur_workspace_access_guard()
async def write_server(
user_input: Write_UserInput,
language_type: str = Header(default=None, alias="X-Language-Type"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
user_input: Write_UserInput,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
Write service endpoint - processes write operations synchronously
Args:
user_input: Write request containing message and end_user_id
language_type: 语言类型 ("zh" 中文, "en" 英文),通过 X-Language-Type Header 传递
Returns:
Response with write operation status
"""
# 使用集中化的语言校验
language = get_language_from_header(language_type)
config_id = user_input.config_id
workspace_id = current_user.current_workspace_id
api_logger.info(f"Write service: workspace_id={workspace_id}, config_id={config_id}, language_type={language}")
api_logger.info(f"Write service: workspace_id={workspace_id}, config_id={config_id}")
# 获取 storage_type如果为 None 则使用默认值
storage_type = workspace_service.get_workspace_storage_type(
db=db,
@@ -151,7 +142,7 @@ async def write_server(
)
if storage_type is None: storage_type = 'neo4j'
user_rag_memory_id = ''
# 如果 storage_type 是 rag必须确保有有效的 user_rag_memory_id
if storage_type == 'rag':
if workspace_id:
@@ -163,15 +154,13 @@ async def write_server(
if knowledge:
user_rag_memory_id = str(knowledge.id)
else:
api_logger.warning(
f"未找到名为 'USER_RAG_MERORY' 的知识库workspace_id: {workspace_id},将使用 neo4j 存储")
api_logger.warning(f"未找到名为 'USER_RAG_MERORY' 的知识库workspace_id: {workspace_id},将使用 neo4j 存储")
storage_type = 'neo4j'
else:
api_logger.warning("workspace_id 为空,无法使用 rag 存储,将使用 neo4j 存储")
storage_type = 'neo4j'
api_logger.info(
f"Write service requested for group {user_input.end_user_id}, storage_type: {storage_type}, user_rag_memory_id: {user_rag_memory_id}")
api_logger.info(f"Write service requested for group {user_input.end_user_id}, storage_type: {storage_type}, user_rag_memory_id: {user_rag_memory_id}")
try:
messages_list = memory_agent_service.get_messages_list(user_input)
result = await memory_agent_service.write_memory(
@@ -179,9 +168,8 @@ async def write_server(
messages_list,
config_id,
db,
storage_type,
user_rag_memory_id,
language
storage_type,
user_rag_memory_id
)
return success(data=result, msg="写入成功")
@@ -199,29 +187,23 @@ async def write_server(
@router.post("/writer_service_async", response_model=ApiResponse)
@cur_workspace_access_guard()
async def write_server_async(
user_input: Write_UserInput,
language_type: str = Header(default=None, alias="X-Language-Type"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
user_input: Write_UserInput,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
Async write service endpoint - enqueues write processing to Celery
Args:
user_input: Write request containing message and end_user_id
language_type: 语言类型 ("zh" 中文, "en" 英文),通过 X-Language-Type Header 传递
Returns:
Task ID for tracking async operation
Use GET /memory/write_result/{task_id} to check task status and get result
"""
# 使用集中化的语言校验
language = get_language_from_header(language_type)
config_id = user_input.config_id
workspace_id = current_user.current_workspace_id
api_logger.info(
f"Async write service: workspace_id={workspace_id}, config_id={config_id}, language_type={language}")
api_logger.info(f"Async write service: workspace_id={workspace_id}, config_id={config_id}")
# 获取 storage_type如果为 None 则使用默认值
storage_type = workspace_service.get_workspace_storage_type(
@@ -246,10 +228,10 @@ async def write_server_async(
task = celery_app.send_task(
"app.core.memory.agent.write_message",
args=[user_input.end_user_id, messages_list, config_id, storage_type, user_rag_memory_id, language]
args=[user_input.end_user_id, messages_list, config_id, storage_type, user_rag_memory_id]
)
api_logger.info(f"Write task queued: {task.id}")
return success(data={"task_id": task.id}, msg="写入任务已提交")
except Exception as e:
api_logger.error(f"Async write operation failed: {str(e)}")
@@ -259,9 +241,9 @@ async def write_server_async(
@router.post("/read_service", response_model=ApiResponse)
@cur_workspace_access_guard()
async def read_server(
user_input: UserInput,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
user_input: UserInput,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
Read service endpoint - processes read operations synchronously
@@ -296,9 +278,8 @@ async def read_server(
)
if knowledge:
user_rag_memory_id = str(knowledge.id)
api_logger.info(
f"Read service: group={user_input.end_user_id}, storage_type={storage_type}, user_rag_memory_id={user_rag_memory_id}, workspace_id={workspace_id}")
api_logger.info(f"Read service: group={user_input.end_user_id}, storage_type={storage_type}, user_rag_memory_id={user_rag_memory_id}, workspace_id={workspace_id}")
try:
result = await memory_agent_service.read_memory(
user_input.end_user_id,
@@ -312,8 +293,7 @@ async def read_server(
)
if str(user_input.search_switch) == "2":
retrieve_info = result['answer']
history = await SessionService(store).get_history(user_input.end_user_id, user_input.end_user_id,
user_input.end_user_id)
history = await SessionService(store).get_history(user_input.end_user_id, user_input.end_user_id, user_input.end_user_id)
query = user_input.message
# 调用 memory_agent_service 的方法生成最终答案
@@ -326,7 +306,7 @@ async def read_server(
db=db
)
if "信息不足,无法回答" in result['answer']:
result['answer'] = retrieve_info
result['answer']=retrieve_info
return success(data=result, msg="回复对话消息成功")
except BaseException as e:
# Handle ExceptionGroup from TaskGroup (Python 3.11+) or BaseExceptionGroup
@@ -342,10 +322,9 @@ async def read_server(
@router.post("/file", response_model=ApiResponse)
async def file_update(
files: List[UploadFile] = File(..., description="要上传的文件"),
model_id: str = Form(..., description="模型ID"),
model_id:str = Form(..., description="模型ID"),
metadata: Optional[str] = Form(None, description="文件元数据 (JSON格式)"),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
文件上传接口 - 支持图片识别
@@ -358,6 +337,9 @@ async def file_update(
Returns:
文件处理结果
"""
db_gen = get_db() # get_db 通常是一个生成器
db = next(db_gen)
api_logger.info(f"File upload requested, file count: {len(files)}")
config = ModelConfigService.get_model_by_id(db=db, model_id=model_id)
apiConfig: ModelApiKey = config.api_keys[0]
@@ -366,7 +348,7 @@ async def file_update(
for file in files:
api_logger.debug(f"Processing file: {file.filename}, content_type: {file.content_type}")
content = await file.read()
if file.content_type and file.content_type.startswith("image/"):
vision_model = QWenCV(
key=apiConfig.api_key,
@@ -380,12 +362,12 @@ async def file_update(
else:
api_logger.warning(f"Unsupported file type: {file.content_type}")
file_content.append(f"[不支持的文件类型: {file.content_type}]")
result_text = ';'.join(file_content)
api_logger.info(f"File processing completed, result length: {len(result_text)}")
return success(data=result_text, msg="转换文本成功")
except Exception as e:
api_logger.error(f"File processing failed: {str(e)}", exc_info=True)
return fail(BizCode.INTERNAL_ERROR, "转换文本失败", str(e))
@@ -435,8 +417,8 @@ async def read_server_async(
@router.get("/read_result/", response_model=ApiResponse)
async def get_read_task_result(
task_id: str,
current_user: User = Depends(get_current_user)
task_id: str,
current_user: User = Depends(get_current_user)
):
"""
Get the status and result of an async read task
@@ -457,7 +439,7 @@ async def get_read_task_result(
try:
result = task_service.get_task_memory_read_result(task_id)
status = result.get("status")
if status == "SUCCESS":
# 任务成功完成
task_result = result.get("result", {})
@@ -475,7 +457,7 @@ async def get_read_task_result(
else:
# 旧格式:直接返回结果
return success(data=task_result, msg="查询任务已完成")
elif status == "FAILURE":
# 任务失败
error_info = result.get("result", "Unknown error")
@@ -484,7 +466,7 @@ async def get_read_task_result(
else:
error_msg = str(error_info)
return fail(BizCode.INTERNAL_ERROR, "查询任务失败", error_msg)
elif status in ["PENDING", "STARTED"]:
# 任务进行中
return success(
@@ -504,7 +486,7 @@ async def get_read_task_result(
},
msg=f"任务状态: {status}"
)
except Exception as e:
api_logger.error(f"Read task status check failed: {str(e)}", exc_info=True)
return fail(BizCode.INTERNAL_ERROR, "任务状态查询失败", str(e))
@@ -512,8 +494,8 @@ async def get_read_task_result(
@router.get("/write_result/", response_model=ApiResponse)
async def get_write_task_result(
task_id: str,
current_user: User = Depends(get_current_user)
task_id: str,
current_user: User = Depends(get_current_user)
):
"""
Get the status and result of an async write task
@@ -534,7 +516,7 @@ async def get_write_task_result(
try:
result = task_service.get_task_memory_write_result(task_id)
status = result.get("status")
if status == "SUCCESS":
# 任务成功完成
task_result = result.get("result", {})
@@ -552,7 +534,7 @@ async def get_write_task_result(
else:
# 旧格式:直接返回结果
return success(data=task_result, msg="写入任务已完成")
elif status == "FAILURE":
# 任务失败
error_info = result.get("result", "Unknown error")
@@ -561,7 +543,7 @@ async def get_write_task_result(
else:
error_msg = str(error_info)
return fail(BizCode.INTERNAL_ERROR, "写入任务失败", error_msg)
elif status in ["PENDING", "STARTED"]:
# 任务进行中
return success(
@@ -581,7 +563,7 @@ async def get_write_task_result(
},
msg=f"任务状态: {status}"
)
except Exception as e:
api_logger.error(f"Write task status check failed: {str(e)}", exc_info=True)
return fail(BizCode.INTERNAL_ERROR, "任务状态查询失败", str(e))
@@ -589,9 +571,9 @@ async def get_write_task_result(
@router.post("/status_type", response_model=ApiResponse)
async def status_type(
user_input: Write_UserInput,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
user_input: Write_UserInput,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
Determine the type of user message (read or write)
@@ -634,21 +616,26 @@ async def status_type(
@router.get("/stats/types", response_model=ApiResponse)
async def get_knowledge_type_stats_api(
end_user_id: Optional[str] = Query(None, description="用户ID可选"),
only_active: bool = Query(True, description="仅统计有效记录(status=1)"),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
end_user_id: Optional[str] = Query(None, description="用户ID可选"),
only_active: bool = Query(True, description="仅统计有效记录(status=1)"),
current_user: User = Depends(get_current_user)
):
"""
统计当前空间下各知识库类型的数量,包含 General | Web | Third-party | Folder。
统计当前空间下各知识库类型的数量,包含 General | Web | Third-party | Folder | memory
会对缺失类型补 0返回字典形式。
可选按状态过滤。
- 知识库类型根据当前用户的 current_workspace_id 过滤
- 如果用户没有当前工作空间,对应的统计返回 0
- memory 是 Neo4j 中 Chunk 的数量,根据 end_user_id (end_user_id) 过滤
- 如果用户没有当前工作空间或未提供 end_user_id对应的统计返回 0
"""
api_logger.info(
f"Knowledge type stats requested for workspace_id: {current_user.current_workspace_id}, end_user_id: {end_user_id}")
api_logger.info(f"Knowledge type stats requested for workspace_id: {current_user.current_workspace_id}, end_user_id: {end_user_id}")
try:
from app.db import get_db
# 获取数据库会话
db_gen = get_db()
db = next(db_gen)
# 调用service层函数
result = await memory_agent_service.get_knowledge_type_stats(
end_user_id=end_user_id,
@@ -656,70 +643,59 @@ async def get_knowledge_type_stats_api(
current_workspace_id=current_user.current_workspace_id,
db=db
)
return success(data=result, msg="获取知识库类型统计成功")
except Exception as e:
api_logger.error(f"Knowledge type stats failed: {str(e)}")
return fail(BizCode.INTERNAL_ERROR, "获取知识库类型统计失败", str(e))
@router.get("/analytics/interest_distribution/by_user", response_model=ApiResponse)
async def get_interest_distribution_by_user_api(
end_user_id: str = Query(..., description="用户ID必填"),
limit: int = Query(5, le=5, description="返回兴趣标签数量限制最多5个"),
language_type: str = Header(default=None, alias="X-Language-Type"),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
@router.get("/analytics/hot_memory_tags/by_user", response_model=ApiResponse)
async def get_hot_memory_tags_by_user_api(
end_user_id: Optional[str] = Query(None, description="用户ID可选"),
language_type: str = Header(default="zh", alias="X-Language-Type"),
limit: int = Query(20, description="返回标签数量限制"),
current_user: User = Depends(get_current_user),
db: Session=Depends(get_db),
):
"""
获取指定用户的兴趣分布标签
与热门标签不同,此接口专注于识别用户的兴趣活动(运动、爱好、学习、创作等),
过滤掉纯物品、工具、地点等不代表用户主动参与活动的名词。
获取指定用户的热门记忆标签
返回格式:
[
{"name": "兴趣活动", "frequency": 频次},
{"name": "标签", "frequency": 频次},
...
]
"""
language = get_language_from_header(language_type)
api_logger.info(f"Interest distribution by user requested: end_user_id={end_user_id}, language={language}")
workspace_id=current_user.current_workspace_id
workspace_repo = WorkspaceRepository(db)
workspace_models = workspace_repo.get_workspace_models_configs(workspace_id)
if workspace_models:
model_id = workspace_models.get("llm", None)
else:
model_id = None
api_logger.info(f"Hot memory tags by user requested: end_user_id={end_user_id}")
try:
# 优先读取缓存
cached = await InterestMemoryCache.get_interest_distribution(
result = await memory_agent_service.get_hot_memory_tags_by_user(
end_user_id=end_user_id,
language=language,
language_type=language_type,
model_id=model_id,
limit=limit
)
if cached is not None:
api_logger.info(f"Interest distribution cache hit: end_user_id={end_user_id}")
return success(data=cached, msg="获取兴趣分布标签成功")
# 缓存未命中,调用模型生成
result = await memory_agent_service.get_interest_distribution_by_user(
end_user_id=end_user_id,
limit=limit,
language=language
)
# 写入缓存24小时过期
await InterestMemoryCache.set_interest_distribution(
end_user_id=end_user_id,
language=language,
data=result,
)
return success(data=result, msg="获取兴趣分布标签成功")
return success(data=result, msg="获取热门记忆标签成功")
except Exception as e:
api_logger.error(f"Interest distribution by user failed: {str(e)}")
return fail(BizCode.INTERNAL_ERROR, "获取兴趣分布标签失败", str(e))
api_logger.error(f"Hot memory tags by user failed: {str(e)}")
return fail(BizCode.INTERNAL_ERROR, "获取热门记忆标签失败", str(e))
@router.get("/analytics/user_profile", response_model=ApiResponse)
async def get_user_profile_api(
end_user_id: Optional[str] = Query(None, description="用户ID可选"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
end_user_id: Optional[str] = Query(None, description="用户ID可选"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
获取用户详情,包含:
@@ -757,17 +733,17 @@ async def get_user_profile_api(
# ):
# """
# Get parsed API documentation (Public endpoint - no authentication required)
# Args:
# file_path: Optional path to API docs file. If None, uses default path.
# Returns:
# Parsed API documentation including title, meta info, and sections
# """
# api_logger.info(f"API docs requested, file_path: {file_path or 'default'}")
# try:
# result = await memory_agent_service.get_api_docs(file_path)
# if result.get("success"):
# return success(msg=result["msg"], data=result["data"])
# else:
@@ -783,9 +759,9 @@ async def get_user_profile_api(
@router.get("/end_user/{end_user_id}/connected_config", response_model=ApiResponse)
async def get_end_user_connected_config(
end_user_id: str,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
end_user_id: str,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""
获取终端用户关联的记忆配置
@@ -804,9 +780,9 @@ async def get_end_user_connected_config(
from app.services.memory_agent_service import (
get_end_user_connected_config as get_config,
)
api_logger.info(f"Getting connected config for end_user: {end_user_id}")
try:
result = get_config(end_user_id, db)
return success(data=result, msg="获取终端用户关联配置成功")
@@ -815,4 +791,4 @@ async def get_end_user_connected_config(
return fail(BizCode.NOT_FOUND, str(e))
except Exception as e:
api_logger.error(f"Failed to get end user connected config: {str(e)}", exc_info=True)
return fail(BizCode.INTERNAL_ERROR, "获取终端用户关联配置失败", str(e))
return fail(BizCode.INTERNAL_ERROR, "获取终端用户关联配置失败", str(e))

View File

@@ -1,5 +1,4 @@
from fastapi import APIRouter, Depends, HTTPException, status, Query
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from typing import Optional
from app.core.response_utils import success
@@ -10,7 +9,6 @@ from app.schemas.response_schema import ApiResponse
from app.services import memory_dashboard_service, memory_storage_service, workspace_service
from app.services.memory_agent_service import get_end_users_connected_configs_batch
from app.services.app_statistics_service import AppStatisticsService
from app.core.logging_config import get_api_logger
# 获取API专用日志器
@@ -150,21 +148,6 @@ async def get_workspace_end_users(
return {uid: {"total": 0} for uid in end_user_ids}
# 触发按需初始化:为 implicit_emotions_storage 中没有记录的用户异步生成数据
try:
from app.celery_app import celery_app as _celery_app
_celery_app.send_task(
"app.tasks.init_implicit_emotions_for_users",
kwargs={"end_user_ids": end_user_ids},
)
_celery_app.send_task(
"app.tasks.init_interest_distribution_for_users",
kwargs={"end_user_ids": end_user_ids},
)
api_logger.info(f"已触发按需初始化任务,候选用户数: {len(end_user_ids)}")
except Exception as e:
api_logger.warning(f"触发按需初始化任务失败(不影响主流程): {e}")
# 并发执行配置查询和记忆数量查询
memory_configs_map, memory_nums_map = await asyncio.gather(
get_memory_configs(),
@@ -403,15 +386,14 @@ def get_current_user_rag_total_num(
@router.get("/rag_content", response_model=ApiResponse)
def get_rag_content(
end_user_id: str = Query(..., description="宿主ID"),
page: int = Query(1, gt=0, description="页码从1开始"),
pagesize: int = Query(15, gt=0, le=100, description="每页返回记录数"),
limit: int = Query(15, description="返回记录数"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
):
"""
获取当前宿主知识库中的chunk内容(分页)
获取当前宿主知识库中的chunk内容
"""
data = memory_dashboard_service.get_rag_content(end_user_id, page, pagesize, db, current_user)
data = memory_dashboard_service.get_rag_content(end_user_id, limit, db, current_user)
return success(data=data, msg="宿主RAGchunk数据获取成功")
@@ -424,18 +406,26 @@ async def get_chunk_summary_tag(
current_user: User = Depends(get_current_user),
):
"""
读取RAG摘要、标签和人物形象纯读库不触发生成
获取chunk总结、提取的标签和人物形象
返回格式:
{
"summary": "用户摘要",
"tags": [{"tag": "标签1", "frequency": 5}, ...],
"personas": ["产品设计师", ...],
"generated": true/false // false表示尚未生产请调用 /generate_rag_profile
"summary": "chunk内容的总结",
"tags": [
{"tag": "标签1", "frequency": 5},
{"tag": "标签2", "frequency": 3},
...
],
"personas": [
"产品设计师",
"旅行爱好者",
"摄影发烧友",
...
]
}
"""
api_logger.info(f"用户 {current_user.username} 取宿主 {end_user_id}RAG摘要/标签/人物形象")
api_logger.info(f"用户 {current_user.username} 请求获取宿主 {end_user_id}chunk摘要标签人物形象")
data = await memory_dashboard_service.get_chunk_summary_and_tags(
end_user_id=end_user_id,
limit=limit,
@@ -443,8 +433,9 @@ async def get_chunk_summary_tag(
db=db,
current_user=current_user
)
return success(data=data, msg="获取成功")
api_logger.info(f"成功获取chunk摘要、{len(data.get('tags', []))} 个标签和 {len(data.get('personas', []))} 个人物形象")
return success(data=data, msg="chunk摘要、标签和人物形象获取成功")
@router.get("/chunk_insight", response_model=ApiResponse)
@@ -455,64 +446,29 @@ async def get_chunk_insight(
current_user: User = Depends(get_current_user),
):
"""
读取RAG洞察报告纯读库不触发生成
获取chunk的洞察内容
返回格式:
{
"insight": "总体概述",
"behavior_pattern": "行为模式",
"key_findings": "关键发现",
"growth_trajectory": "成长轨迹",
"generated": true/false // false表示尚未生产请调用 /generate_rag_profile
"insight": "对chunk内容的深度洞察分析"
}
"""
api_logger.info(f"用户 {current_user.username} 取宿主 {end_user_id}RAG洞察")
api_logger.info(f"用户 {current_user.username} 请求获取宿主 {end_user_id}chunk洞察")
data = await memory_dashboard_service.get_chunk_insight(
end_user_id=end_user_id,
limit=limit,
db=db,
current_user=current_user
)
return success(data=data, msg="获取成功")
class GenerateRagProfileRequest(BaseModel):
end_user_id: str = Field(..., description="宿主ID")
limit: int = Field(15, description="参与生成的chunk数量上限")
max_tags: int = Field(10, description="最大标签数量")
@router.post("/generate_rag_profile", response_model=ApiResponse)
async def generate_rag_profile(
body: GenerateRagProfileRequest,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
):
"""
生产接口为RAG存储模式的宿主全量重新生成完整画像并持久化到end_user表。
每次请求都会重新生成,覆盖已有数据。
"""
api_logger.info(f"用户 {current_user.username} 触发RAG画像生产: end_user_id={body.end_user_id}")
data = await memory_dashboard_service.generate_rag_profile(
end_user_id=body.end_user_id,
limit=body.limit,
max_tags=body.max_tags,
db=db,
current_user=current_user,
)
api_logger.info(f"RAG画像生产完成: {data}")
return success(data=data, msg="RAG画像生产完成")
api_logger.info("成功获取chunk洞察")
return success(data=data, msg="chunk洞察获取成功")
@router.get("/dashboard_data", response_model=ApiResponse)
async def dashboard_data(
end_user_id: Optional[str] = Query(None, description="可选的用户ID"),
start_date: Optional[int] = Query(None, description="开始时间戳(毫秒)"),
end_date: Optional[int] = Query(None, description="结束时间戳(毫秒)"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
):
@@ -547,15 +503,6 @@ async def dashboard_data(
workspace_id = current_user.current_workspace_id
api_logger.info(f"用户 {current_user.username} 请求获取工作空间 {workspace_id} 的dashboard整合数据")
# 如果没有提供时间范围默认使用最近30天
if start_date is None or end_date is None:
from datetime import datetime, timedelta
end_dt = datetime.now()
start_dt = end_dt - timedelta(days=30)
end_date = int(end_dt.timestamp() * 1000)
start_date = int(start_dt.timestamp() * 1000)
api_logger.info(f"使用默认时间范围: {start_dt}{end_dt}")
# 获取 storage_type如果为 None 则使用默认值
storage_type = workspace_service.get_workspace_storage_type(
db=db,
@@ -616,22 +563,17 @@ async def dashboard_data(
except Exception as e:
api_logger.warning(f"获取知识库类型统计失败: {str(e)}")
# 3. 获取API调用统计total_api_call
# 3. 获取API调用增量total_api_call,转换为整数
try:
# 使用 AppStatisticsService 获取真实的API调用统计
app_stats_service = AppStatisticsService(db)
api_stats = app_stats_service.get_workspace_api_statistics(
api_increment = memory_dashboard_service.get_workspace_api_increment(
db=db,
workspace_id=workspace_id,
start_date=start_date,
end_date=end_date
current_user=current_user
)
# 计算总调用次数
total_api_calls = sum(item.get("total_calls", 0) for item in api_stats)
neo4j_data["total_api_call"] = total_api_calls
api_logger.info(f"成功获取API调用统计: {neo4j_data['total_api_call']}")
neo4j_data["total_api_call"] = api_increment
api_logger.info(f"成功获取API调用增量: {neo4j_data['total_api_call']}")
except Exception as e:
api_logger.error(f"获取API调用统计失败: {str(e)}")
neo4j_data["total_api_call"] = 0
api_logger.warning(f"获取API调用增量失败: {str(e)}")
result["neo4j_data"] = neo4j_data
api_logger.info("成功获取neo4j_data")
@@ -647,8 +589,8 @@ async def dashboard_data(
# 获取RAG相关数据
try:
# total_memory: 只统计用户知识库permission_id='Memory')的chunk数
total_chunk = memory_dashboard_service.get_rag_user_kb_total_chunk(db, current_user)
# total_memory: 使用 total_chunkchunk数
total_chunk = memory_dashboard_service.get_rag_total_chunk(db, current_user)
rag_data["total_memory"] = total_chunk
# total_app: 统计当前空间下的所有app数量
@@ -660,23 +602,10 @@ async def dashboard_data(
total_kb = memory_dashboard_service.get_rag_total_kb(db, current_user)
rag_data["total_knowledge"] = total_kb
# total_api_call: 使用 AppStatisticsService 获取真实的API调用统计
try:
app_stats_service = AppStatisticsService(db)
api_stats = app_stats_service.get_workspace_api_statistics(
workspace_id=workspace_id,
start_date=start_date,
end_date=end_date
)
# 计算总调用次数
total_api_calls = sum(item.get("total_calls", 0) for item in api_stats)
rag_data["total_api_call"] = total_api_calls
api_logger.info(f"成功获取RAG模式API调用统计: {rag_data['total_api_call']}")
except Exception as e:
api_logger.warning(f"获取RAG模式API调用统计失败使用默认值: {str(e)}")
rag_data["total_api_call"] = 0
# total_api_call: 固定值
rag_data["total_api_call"] = 1024
api_logger.info(f"成功获取RAG相关数据: memory={total_chunk}, app={len(apps_orm)}, knowledge={total_kb}, api_calls={rag_data['total_api_call']}")
api_logger.info(f"成功获取RAG相关数据: memory={total_chunk}, app={len(apps_orm)}, knowledge={total_kb}")
except Exception as e:
api_logger.warning(f"获取RAG相关数据失败: {str(e)}")

View File

@@ -3,10 +3,9 @@
包含情景记忆总览和详情查询接口
"""
from fastapi import APIRouter, Depends, Header
from fastapi import APIRouter, Depends
from app.core.error_codes import BizCode
from app.core.language_utils import get_language_from_header
from app.core.logging_config import get_api_logger
from app.core.response_utils import fail, success
from app.dependencies import get_current_user
@@ -15,7 +14,6 @@ from app.schemas.response_schema import ApiResponse
from app.schemas.memory_episodic_schema import (
EpisodicMemoryOverviewRequest,
EpisodicMemoryDetailsRequest,
translate_episodic_type,
)
from app.services.memory_episodic_service import memory_episodic_service
@@ -86,7 +84,6 @@ async def get_episodic_memory_overview_api(
@router.post("/details", response_model=ApiResponse)
async def get_episodic_memory_details_api(
request: EpisodicMemoryDetailsRequest,
language_type: str = Header(default=None, alias="X-Language-Type"),
current_user: User = Depends(get_current_user),
) -> dict:
"""
@@ -114,11 +111,6 @@ async def get_episodic_memory_details_api(
summary_id=request.summary_id
)
# 根据语言参数翻译 episodic_type
language = get_language_from_header(language_type)
if "episodic_type" in result:
result["episodic_type"] = translate_episodic_type(result["episodic_type"], language)
api_logger.info(
f"成功获取情景记忆详情: end_user_id={request.end_user_id}, summary_id={request.summary_id}"
)

View File

@@ -3,7 +3,6 @@ import time
import uuid
from uuid import UUID
from app.core.language_utils import get_language_from_header
from app.core.logging_config import get_api_logger
from app.core.memory.storage_services.reflection_engine.self_reflexion import (
ReflectionConfig,
@@ -104,18 +103,14 @@ async def start_workspace_reflection(
) -> dict:
"""启动工作空间中所有匹配应用的反思功能"""
workspace_id = current_user.current_workspace_id
reflection_service = MemoryReflectionService(db)
try:
api_logger.info(f"用户 {current_user.username} 启动workspace反思workspace_id: {workspace_id}")
# 使用独立的数据库会话来获取工作空间应用详情,避免事务失败
from app.db import get_db_context
with get_db_context() as query_db:
service = WorkspaceAppService(query_db)
result = service.get_workspace_apps_detailed(workspace_id)
service = WorkspaceAppService(db)
result = service.get_workspace_apps_detailed(workspace_id)
reflection_results = []
for data in result['apps_detailed_info']:
# 跳过没有配置的应用
if not data['memory_configs']:
@@ -137,36 +132,33 @@ async def start_workspace_reflection(
api_logger.debug(f"配置 {config_id_str} 没有匹配的release")
continue
# 为每个用户执行反思 - 使用独立的数据库会话
# 为每个用户执行反思
for user in end_users:
api_logger.info(f"为用户 {user['id']} 启动反思config_id: {config_id_str}")
# 为每个用户创建独立的数据库会话,避免事务失败影响其他用户
with get_db_context() as user_db:
try:
reflection_service = MemoryReflectionService(user_db)
reflection_result = await reflection_service.start_text_reflection(
config_data=config,
end_user_id=user['id']
)
try:
reflection_result = await reflection_service.start_text_reflection(
config_data=config,
end_user_id=user['id']
)
reflection_results.append({
"app_id": data['id'],
"config_id": config_id_str,
"end_user_id": user['id'],
"reflection_result": reflection_result
})
except Exception as e:
api_logger.error(f"用户 {user['id']} 反思失败: {str(e)}")
reflection_results.append({
"app_id": data['id'],
"config_id": config_id_str,
"end_user_id": user['id'],
"reflection_result": {
"status": "错误",
"message": f"反思失败: {str(e)}"
}
})
reflection_results.append({
"app_id": data['id'],
"config_id": config_id_str,
"end_user_id": user['id'],
"reflection_result": reflection_result
})
except Exception as e:
api_logger.error(f"用户 {user['id']} 反思失败: {str(e)}")
reflection_results.append({
"app_id": data['id'],
"config_id": config_id_str,
"end_user_id": user['id'],
"reflection_result": {
"status": "错误",
"message": f"反思失败: {str(e)}"
}
})
return success(data=reflection_results, msg="反思配置成功")
@@ -219,13 +211,11 @@ async def start_reflection_configs(
@router.get("/reflection/run")
async def reflection_run(
config_id: UUID|int,
language_type: str = Header(default=None, alias="X-Language-Type"),
language_type: str = Header(default="zh", alias="X-Language-Type"),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
) -> dict:
"""Activate the reflection function for all matching applications in the workspace"""
# 使用集中化的语言校验
language = get_language_from_header(language_type)
api_logger.info(f"用户 {current_user.username} 查询反思配置config_id: {config_id}")
config_id = resolve_config_id(config_id, db)

View File

@@ -1,18 +1,15 @@
from typing import Optional
from dotenv import load_dotenv
from fastapi import APIRouter, Depends, Header, HTTPException, status
from sqlalchemy.orm import Session
from app.core.language_utils import get_language_from_header
from fastapi import APIRouter, Depends, HTTPException, status,Header
from app.core.logging_config import get_api_logger
from app.core.response_utils import success
from app.db import get_db
from app.dependencies import get_current_user
from app.models.user_model import User
from app.services.memory_short_service import LongService, ShortService
from app.services.memory_storage_service import search_entity
from app.services.memory_storage_service import search_entity
from app.services.memory_short_service import ShortService,LongService
from dotenv import load_dotenv
from sqlalchemy.orm import Session
from typing import Optional
load_dotenv()
api_logger = get_api_logger()
@@ -23,19 +20,16 @@ router = APIRouter(
@router.get("/short_term")
async def short_term_configs(
end_user_id: str,
language_type:str = Header(default=None, alias="X-Language-Type"),
language_type:str = Header(default="zh", alias="X-Language-Type"),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
# 使用集中化的语言校验
language = get_language_from_header(language_type)
# 获取短期记忆数据
short_term=ShortService(end_user_id, db)
short_term=ShortService(end_user_id)
short_result=short_term.get_short_databasets()
short_count=short_term.get_short_count()
long_term=LongService(end_user_id, db)
long_term=LongService(end_user_id)
long_result=long_term.get_long_databasets()
entity_result = await search_entity(end_user_id)

View File

@@ -1,12 +1,8 @@
import os
from typing import Optional
from uuid import UUID
from fastapi import APIRouter, Depends, Query
from fastapi.responses import StreamingResponse, JSONResponse
from sqlalchemy.orm import Session
from app.core.error_codes import BizCode
from app.core.language_utils import get_language_from_header
from app.core.logging_config import get_api_logger
from app.core.response_utils import fail, success
from app.db import get_db
@@ -15,6 +11,7 @@ from app.models.user_model import User
from app.schemas.memory_storage_schema import (
ConfigKey,
ConfigParamsCreate,
ConfigParamsDelete,
ConfigPilotRun,
ConfigUpdate,
ConfigUpdateExtracted,
@@ -34,7 +31,7 @@ from app.services.memory_storage_service import (
search_entity,
search_statement,
)
from fastapi import APIRouter, Depends, Header
from fastapi import APIRouter, Depends
from fastapi.responses import StreamingResponse
from sqlalchemy.orm import Session
@@ -75,9 +72,68 @@ async def get_storage_info(
return fail(BizCode.INTERNAL_ERROR, "存储信息获取失败", str(e))
# --- DB connection dependency ---
_CONN: Optional[object] = None
"""PostgreSQL 连接生成与管理(使用 psycopg2"""
# 这个可以转移,可能是已经有的
# PostgreSQL 数据库连接
def _make_pgsql_conn() -> Optional[object]: # 创建 PostgreSQL 数据库连接
host = os.getenv("DB_HOST")
user = os.getenv("DB_USER")
password = os.getenv("DB_PASSWORD")
database = os.getenv("DB_NAME")
port_str = os.getenv("DB_PORT")
try:
import psycopg2 # type: ignore
port = int(port_str) if port_str else 5432
conn = psycopg2.connect(
host=host or "localhost",
port=port,
user=user,
password=password,
dbname=database,
)
# 设置自动提交,避免显式事务管理
conn.autocommit = True
# 设置会话时区为中国标准时间Asia/Shanghai便于直接以本地时区展示
try:
cur = conn.cursor()
cur.execute("SET TIME ZONE 'Asia/Shanghai'")
cur.close()
except Exception:
# 时区设置失败不影响连接,仅记录但不抛出
pass
return conn
except Exception as e:
try:
print(f"[PostgreSQL] 连接失败: {e}")
except Exception:
pass
return None
def get_db_conn() -> Optional[object]: # 获取 PostgreSQL 数据库连接
global _CONN
if _CONN is None:
_CONN = _make_pgsql_conn()
return _CONN
def reset_db_conn() -> bool: # 重置 PostgreSQL 数据库连接
"""Close and recreate the global DB connection."""
global _CONN
try:
if _CONN:
try:
_CONN.close()
except Exception:
pass
_CONN = _make_pgsql_conn()
return _CONN is not None
except Exception:
_CONN = None
return False
@router.post("/create_config", response_model=ApiResponse) # 创建配置文件,其他参数默认
@@ -85,8 +141,7 @@ def create_config(
payload: ConfigParamsCreate,
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
x_language_type: Optional[str] = Header(None, alias="X-Language-Type"),
) -> dict:
) -> dict:
workspace_id = current_user.current_workspace_id
# 检查用户是否已选择工作空间
if workspace_id is None:
@@ -100,29 +155,7 @@ def create_config(
svc = DataConfigService(db)
result = svc.create(payload)
return success(data=result, msg="创建成功")
except ValueError as e:
err_str = str(e)
if err_str.startswith("DUPLICATE_CONFIG_NAME:"):
config_name = err_str.split(":", 1)[1]
api_logger.warning(f"重复的配置名称 '{config_name}' 在工作空间 {workspace_id}")
lang = get_language_from_header(x_language_type)
if lang == "en":
msg = fail(BizCode.BAD_REQUEST, "Config name already exists", f"A config named \"{config_name}\" already exists in the current workspace. Please use a different name.")
else:
msg = fail(BizCode.BAD_REQUEST, "配置名称已存在", f"当前工作空间下已存在名为「{config_name}」的记忆配置,请使用其他名称")
return JSONResponse(status_code=400, content=msg)
api_logger.error(f"Create config failed: {err_str}")
return fail(BizCode.INTERNAL_ERROR, "创建配置失败", err_str)
except Exception as e:
from sqlalchemy.exc import IntegrityError
if isinstance(e, IntegrityError) and "uq_workspace_config_name" in str(getattr(e, 'orig', '')):
api_logger.warning(f"重复的配置名称 '{payload.config_name}' 在工作空间 {workspace_id}")
lang = get_language_from_header(x_language_type)
if lang == "en":
msg = fail(BizCode.BAD_REQUEST, "Config name already exists", f"A config named \"{payload.config_name}\" already exists in the current workspace. Please use a different name.")
else:
msg = fail(BizCode.BAD_REQUEST, "配置名称已存在", f"当前工作空间下已存在名为「{payload.config_name}」的记忆配置,请使用其他名称")
return JSONResponse(status_code=400, content=msg)
api_logger.error(f"Create config failed: {str(e)}")
return fail(BizCode.INTERNAL_ERROR, "创建配置失败", str(e))
@@ -130,20 +163,9 @@ def create_config(
@router.delete("/delete_config", response_model=ApiResponse) # 删除数据库中的内容(按配置名称)
def delete_config(
config_id: UUID|int,
force: bool = Query(False, description="是否强制删除(即使有终端用户正在使用)"),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
) -> dict:
"""删除记忆配置(带终端用户保护)
- 检查是否为默认配置,默认配置不允许删除
- 检查是否有终端用户连接到该配置
- 如果有连接且 force=False返回警告
- 如果 force=True清除终端用户引用后删除配置
Query Parameters:
force: 设置为 true 可强制删除(即使有终端用户正在使用)
"""
) -> dict:
workspace_id = current_user.current_workspace_id
config_id=resolve_config_id(config_id, db)
# 检查用户是否已选择工作空间
@@ -151,62 +173,21 @@ def delete_config(
api_logger.warning(f"用户 {current_user.username} 尝试删除配置但未选择工作空间")
return fail(BizCode.INVALID_PARAMETER, "请先切换到一个工作空间", "current_workspace_id is None")
api_logger.info(
f"用户 {current_user.username} 在工作空间 {workspace_id} 请求删除配置: "
f"config_id={config_id}, force={force}"
)
api_logger.info(f"用户 {current_user.username} 在工作空间 {workspace_id} 请求删除配置: {config_id}")
try:
# 使用带保护的删除服务
from app.services.memory_config_service import MemoryConfigService
config_service = MemoryConfigService(db)
result = config_service.delete_config(config_id=config_id, force=force)
if result["status"] == "error":
api_logger.warning(
f"记忆配置删除被拒绝: config_id={config_id}, reason={result['message']}"
)
return fail(
code=BizCode.FORBIDDEN,
msg=result["message"],
data={"config_id": str(config_id), "is_default": result.get("is_default", False)}
)
if result["status"] == "warning":
api_logger.warning(
f"记忆配置正在使用,无法删除: config_id={config_id}, "
f"connected_count={result['connected_count']}"
)
return fail(
code=BizCode.RESOURCE_IN_USE,
msg=result["message"],
data={
"connected_count": result["connected_count"],
"force_required": result["force_required"]
}
)
api_logger.info(
f"记忆配置删除成功: config_id={config_id}, "
f"affected_users={result['affected_users']}"
)
return success(
msg=result["message"],
data={"affected_users": result["affected_users"]}
)
svc = DataConfigService(db)
result = svc.delete(ConfigParamsDelete(config_id=config_id))
return success(data=result, msg="删除成功")
except Exception as e:
api_logger.error(f"Delete config failed: {str(e)}", exc_info=True)
api_logger.error(f"Delete config failed: {str(e)}")
return fail(BizCode.INTERNAL_ERROR, "删除配置失败", str(e))
@router.post("/update_config", response_model=ApiResponse) # 更新配置文件中name和desc
def update_config(
payload: ConfigUpdate,
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
) -> dict:
) -> dict:
workspace_id = current_user.current_workspace_id
payload.config_id = resolve_config_id(payload.config_id, db)
# 检查用户是否已选择工作空间
@@ -234,7 +215,7 @@ def update_config_extracted(
payload: ConfigUpdateExtracted,
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
) -> dict:
) -> dict:
workspace_id = current_user.current_workspace_id
payload.config_id = resolve_config_id(payload.config_id, db)
# 检查用户是否已选择工作空间
@@ -261,7 +242,7 @@ def read_config_extracted(
config_id: UUID | int,
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
) -> dict:
) -> dict:
workspace_id = current_user.current_workspace_id
config_id = resolve_config_id(config_id, db)
# 检查用户是否已选择工作空间
@@ -282,7 +263,7 @@ def read_config_extracted(
def read_all_config(
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
) -> dict:
) -> dict:
workspace_id = current_user.current_workspace_id
# 检查用户是否已选择工作空间
@@ -304,22 +285,17 @@ def read_all_config(
@router.post("/pilot_run", response_model=None)
async def pilot_run(
payload: ConfigPilotRun,
language_type: str = Header(default=None, alias="X-Language-Type"),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
) -> StreamingResponse:
# 使用集中化的语言校验
language = get_language_from_header(language_type)
api_logger.info(
f"Pilot run requested: config_id={payload.config_id}, "
f"dialogue_text_length={len(payload.dialogue_text)}, "
f"custom_text_length={len(payload.custom_text) if payload.custom_text else 0}"
f"dialogue_text_length={len(payload.dialogue_text)}"
)
payload.config_id = resolve_config_id(payload.config_id, db)
svc = DataConfigService(db)
return StreamingResponse(
svc.pilot_run_stream(payload, language=language),
svc.pilot_run_stream(payload),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
@@ -328,8 +304,9 @@ async def pilot_run(
},
)
# ==================== Search & Analytics ====================
"""
以下为搜索与分析接口,直接挂载到同一 router统一响应为 ApiResponse。
"""
@router.get("/search/kb_type_distribution", response_model=ApiResponse)
async def get_kb_type_distribution(
@@ -469,9 +446,8 @@ async def get_hot_memory_tags_api(
try:
# 尝试从Redis缓存获取
import json
from app.aioRedis import aio_redis_get, aio_redis_set
import json
cached_result = await aio_redis_get(cache_key)
if cached_result:
@@ -544,11 +520,10 @@ async def clear_hot_memory_tags_cache(
@router.get("/analytics/recent_activity_stats", response_model=ApiResponse)
async def get_recent_activity_stats_api(
current_user: User = Depends(get_current_user),
) -> dict:
workspace_id = str(current_user.current_workspace_id) if current_user.current_workspace_id else None
api_logger.info(f"Recent activity stats requested: workspace_id={workspace_id}")
) -> dict:
api_logger.info("Recent activity stats requested")
try:
result = await analytics_recent_activity_stats(workspace_id=workspace_id)
result = await analytics_recent_activity_stats()
return success(data=result, msg="查询成功")
except Exception as e:
api_logger.error(f"Recent activity stats failed: {str(e)}")

View File

@@ -328,7 +328,7 @@ async def update_composite_model(
try:
if model_data.type is not None:
raise BusinessException("不允许更改模型类型", BizCode.INVALID_PARAMETER)
raise BusinessException("不允许更改模型类型和供应商", BizCode.INVALID_PARAMETER)
result_orm = await ModelConfigService.update_composite_model(db=db, model_id=model_id, model_data=model_data, tenant_id=current_user.tenant_id)
api_logger.info(f"组合模型更新成功: {result_orm.name} (ID: {model_id})")
@@ -368,14 +368,6 @@ def update_model(
更新模型配置
"""
api_logger.info(f"更新模型配置请求: model_id={model_id}, 用户: {current_user.username}, tenant_id={current_user.tenant_id}")
if model_data.type is not None or model_data.provider is not None:
raise BusinessException("不允许更改模型类型和供应商", BizCode.INVALID_PARAMETER)
if model_data.is_active:
active_keys = ModelApiKeyService.get_api_keys_by_model(db=db, model_config_id=model_id, is_active=model_data.is_active)
if not active_keys:
raise BusinessException("请先为该模型配置可用的 API Key", BizCode.INVALID_PARAMETER)
try:
api_logger.debug(f"开始更新模型配置: model_id={model_id}")
@@ -474,9 +466,7 @@ async def create_model_api_key_by_provider(
config=api_key_data.config,
is_active=api_key_data.is_active,
priority=api_key_data.priority,
model_config_ids=model_config_ids,
capability=api_key_data.capability,
is_omni=api_key_data.is_omni
model_config_ids=model_config_ids
)
created_keys, failed_models = await ModelApiKeyService.create_api_key_by_provider(db=db, data=create_data)

View File

@@ -4,14 +4,13 @@
Endpoints:
POST /api/memory/ontology/extract - 提取本体类
POST /api/memory/ontology/export - 按场景导出OWL文件
POST /api/memory/ontology/import - 导入OWL文件到指定场景
POST /api/memory/ontology/export - 导出OWL文件
POST /api/memory/ontology/scene - 创建本体场景
PUT /api/memory/ontology/scene/{scene_id} - 更新本体场景
DELETE /api/memory/ontology/scene/{scene_id} - 删除本体场景
GET /api/memory/ontology/scene/{scene_id} - 获取单个场景
GET /api/memory/ontology/scenes - 获取场景列表
POST /api/memory/ontology/class - 创建本体类型(支持批量)
POST /api/memory/ontology/class - 创建本体类型
PUT /api/memory/ontology/class/{class_id} - 更新本体类型
DELETE /api/memory/ontology/class/{class_id} - 删除本体类型
GET /api/memory/ontology/class/{class_id} - 获取单个类型
@@ -20,26 +19,23 @@ Endpoints:
import logging
import tempfile
import io
from typing import Dict, Optional, List
from urllib.parse import quote
from typing import Dict, Optional
from fastapi import APIRouter, Depends, HTTPException, File, UploadFile, Form, Header
from fastapi.responses import StreamingResponse, JSONResponse
from fastapi import APIRouter, Depends, HTTPException, Header
from sqlalchemy.orm import Session
from app.core.config import settings
from app.core.error_codes import BizCode
from app.core.language_utils import get_language_from_header
from app.core.logging_config import get_api_logger, get_business_logger
from app.core.logging_config import get_api_logger
from app.core.response_utils import fail, success
from app.db import get_db
from app.dependencies import get_current_user
from app.models.user_model import User
from app.core.memory.models.ontology_scenario_models import OntologyClass
from app.services.memory_base_service import Translation_English
from app.core.memory.models.ontology_models import OntologyClass
from typing import List
from app.schemas.ontology_schemas import (
ExportBySceneRequest,
ExportBySceneResponse,
ExportRequest,
ExportResponse,
ExtractionRequest,
ExtractionResponse,
SceneCreateRequest,
@@ -50,7 +46,6 @@ from app.schemas.ontology_schemas import (
ClassUpdateRequest,
ClassResponse,
ClassListResponse,
ImportOwlResponse,
)
from app.schemas.response_schema import ApiResponse
from app.services.ontology_service import OntologyService
@@ -61,7 +56,6 @@ from app.repositories.ontology_scene_repository import OntologySceneRepository
api_logger = get_api_logger()
business_logger = get_business_logger()
logger = logging.getLogger(__name__)
router = APIRouter(
@@ -70,6 +64,72 @@ router = APIRouter(
)
async def translate_ontology_classes(
classes: List[OntologyClass],
model_id: str
) -> List[OntologyClass]:
"""翻译本体类列表
将本体类的中文字段翻译为英文,包括:
- name_chinese: 中文名称
- description: 描述
- examples: 示例列表
Args:
classes: 本体类列表
model_id: LLM模型ID用于翻译
Returns:
List[OntologyClass]: 翻译后的本体类列表
"""
translated_classes = []
for ontology_class in classes:
# 创建类的副本,避免修改原对象
translated_class = ontology_class.model_copy(deep=True)
# 翻译 name_chinese 字段
if translated_class.name_chinese:
try:
translated_class.name_chinese = await Translation_English(
model_id,
translated_class.name_chinese
)
except Exception as e:
logger.warning(f"Failed to translate name_chinese: {e}")
# 保留原文
# 翻译 description 字段
if translated_class.description:
try:
translated_class.description = await Translation_English(
model_id,
translated_class.description
)
except Exception as e:
logger.warning(f"Failed to translate description: {e}")
# 保留原文
# 翻译 examples 列表
if translated_class.examples:
translated_examples = []
for example in translated_class.examples:
try:
translated_example = await Translation_English(
model_id,
example
)
translated_examples.append(translated_example)
except Exception as e:
logger.warning(f"Failed to translate example: {e}")
translated_examples.append(example) # 保留原文
translated_class.examples = translated_examples
translated_classes.append(translated_class)
return translated_classes
def _get_ontology_service(
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
@@ -124,23 +184,15 @@ def _get_ontology_service(
)
# 通过 Repository 获取可用的 API Key负载均衡逻辑由 Repository 处理)
# from app.repositories.model_repository import ModelApiKeyRepository
from app.services.model_service import ModelApiKeyService
api_key_config = ModelApiKeyService.get_available_api_key(db, model_config.id)
if not api_key_config:
from app.repositories.model_repository import ModelApiKeyRepository
api_keys = ModelApiKeyRepository.get_by_model_config(db, model_config.id)
if not api_keys:
logger.error(f"Model {llm_id} has no active API key")
raise HTTPException(
status_code=400,
detail="指定的LLM模型没有可用的API密钥"
)
# api_keys = ModelApiKeyRepository.get_by_model_config(db, model_config.id)
# if not api_keys:
# logger.error(f"Model {llm_id} has no active API key")
# raise HTTPException(
# status_code=400,
# detail="指定的LLM模型没有可用的API密钥"
# )
# api_key_config = api_keys[0]
api_key_config = api_keys[0]
is_composite = getattr(model_config, 'is_composite', False)
logger.info(
@@ -162,7 +214,6 @@ def _get_ontology_service(
provider=actual_provider,
api_key=api_key_config.api_key,
base_url=api_key_config.api_base,
is_omni=api_key_config.is_omni,
max_retries=3,
timeout=60.0
)
@@ -193,7 +244,7 @@ def _get_ontology_service(
@router.post("/extract", response_model=ApiResponse)
async def extract_ontology(
request: ExtractionRequest,
language_type: str = Header(default=None, alias="X-Language-Type"),
language_type: str = Header(default="zh", alias="X-Language-Type"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
@@ -202,25 +253,50 @@ async def extract_ontology(
从场景描述中提取符合OWL规范的本体类。
提取结果仅返回给前端,不会自动保存到数据库。
前端可以从返回结果中选择需要的类型,然后调用 /class 接口创建类型。
支持中英文切换,通过 X-Language-Type Header 指定语言。
Args:
request: 提取请求,包含scenario、domain、llm_id和scene_id
language_type: 语言类型 Header (zh/en)
language_type: 语言类型'zh'(中文)或 'en'(英文),默认 'zh'
db: 数据库会话
current_user: 当前用户
Returns:
ApiResponse: 包含提取结果的响应
Response format:
{
"code": 200,
"msg": "本体提取成功",
"data": {
"classes": [
{
"id": "147d9db50b524a9e909e01a753d3acdd",
"name": "Patient",
"name_chinese": "患者",
"description": "在医疗机构中接受诊疗、护理或健康管理的个体",
"examples": ["糖尿病患者", "术后康复患者", "门诊初诊患者"],
"parent_class": null,
"entity_type": "Person",
"domain": "Healthcare"
},
...
],
"domain": "Healthcare",
"extracted_count": 7
}
}
"""
api_logger.info(
f"Ontology extraction requested by user {current_user.id}, "
f"scenario_length={len(request.scenario)}, "
f"domain={request.domain}, "
f"llm_id={request.llm_id}, "
f"scene_id={request.scene_id}"
f"scene_id={request.scene_id}, "
f"language_type={language_type}"
)
try:
# 使用集中化的语言校验
language = get_language_from_header(language_type)
# 获取当前工作空间ID
workspace_id = current_user.current_workspace_id
if not workspace_id:
@@ -234,22 +310,36 @@ async def extract_ontology(
llm_id=request.llm_id
)
# 调用服务层执行提取
# 调用服务层执行提取传入scene_id和workspace_id
result = await service.extract_ontology(
scenario=request.scenario,
domain=request.domain,
scene_id=request.scene_id,
workspace_id=workspace_id,
language=language
workspace_id=workspace_id
)
# 根据语言类型统一 name 字段
# zh: name 使用 name_chinese中文名
# en: name 保持原值(英文 PascalCase
if language == "zh":
for cls in result.classes:
if cls.name_chinese:
cls.name = cls.name_chinese
# ===== 新增:翻译逻辑 =====
# 如果需要英文,则翻译数据
if language_type != 'zh':
api_logger.info(f"Translating extraction result to English")
# 翻译 classes 列表
result.classes = await translate_ontology_classes(
result.classes,
request.llm_id
)
# 翻译 domain 字段
if result.domain:
try:
result.domain = await Translation_English(
request.llm_id,
result.domain
)
except Exception as e:
logger.warning(f"Failed to translate domain: {e}")
# 保留原文
# ===== 翻译逻辑结束 =====
# 构建响应
response = ExtractionResponse(
@@ -260,7 +350,7 @@ async def extract_ontology(
api_logger.info(
f"Ontology extraction completed, extracted {len(result.classes)} classes, "
f"scene_id={request.scene_id}, language={language}"
f"saved to scene {request.scene_id}, language={language_type}"
)
return success(data=response.model_dump(), msg="本体提取成功")
@@ -281,6 +371,146 @@ async def extract_ontology(
return fail(BizCode.INTERNAL_ERROR, "本体提取失败", str(e))
@router.post("/export", response_model=ApiResponse)
async def export_owl(
request: ExportRequest,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""导出OWL文件
将提取的本体类导出为OWL文件,支持多种格式。
导出操作不需要LLM,只使用OWL验证器和Owlready2库。
Args:
request: 导出请求,包含classes、format和include_metadata
db: 数据库会话
current_user: 当前用户
Returns:
ApiResponse: 包含OWL文件内容的响应
Supported formats:
- rdfxml: 标准OWL RDF/XML格式(完整)
- turtle: Turtle格式(可读性好)
- ntriples: N-Triples格式(简单)
- json: JSON格式(简化,只包含类信息)
Response format:
{
"code": 200,
"msg": "OWL文件导出成功",
"data": {
"owl_content": "...",
"format": "rdfxml",
"classes_count": 7
}
}
"""
api_logger.info(
f"OWL export requested by user {current_user.id}, "
f"classes_count={len(request.classes)}, "
f"format={request.format}, "
f"include_metadata={request.include_metadata}"
)
try:
# 验证格式
valid_formats = ["rdfxml", "turtle", "ntriples", "json"]
if request.format not in valid_formats:
api_logger.warning(f"Invalid export format: {request.format}")
return fail(
BizCode.BAD_REQUEST,
"不支持的导出格式",
f"format必须是以下之一: {', '.join(valid_formats)}"
)
# JSON格式直接导出,不需要OWL验证
if request.format == "json":
owl_validator = OWLValidator()
owl_content = owl_validator.export_to_owl(
world=None,
format="json",
classes=request.classes
)
response = ExportResponse(
owl_content=owl_content,
format=request.format,
classes_count=len(request.classes)
)
api_logger.info(
f"JSON export completed, content_length={len(owl_content)}"
)
return success(data=response.model_dump(), msg="OWL文件导出成功")
# 创建临时文件路径
with tempfile.NamedTemporaryFile(
mode='w',
suffix='.owl',
delete=False
) as tmp_file:
output_path = tmp_file.name
# 导出操作不需要LLM,直接使用OWL验证器
owl_validator = OWLValidator()
# 验证本体类
logger.debug("Validating ontology classes")
is_valid, errors, world = owl_validator.validate_ontology_classes(
classes=request.classes,
)
if not is_valid:
logger.warning(
f"OWL validation found {len(errors)} issues during export: {errors}"
)
# 继续导出,但记录警告
if not world:
error_msg = "Failed to create OWL world for export"
logger.error(error_msg)
return fail(BizCode.INTERNAL_ERROR, "创建OWL世界失败", error_msg)
# 导出OWL文件
logger.info(f"Exporting to {request.format} format")
owl_content = owl_validator.export_to_owl(
world=world,
output_path=output_path,
format=request.format,
classes=request.classes
)
# 构建响应
response = ExportResponse(
owl_content=owl_content,
format=request.format,
classes_count=len(request.classes)
)
api_logger.info(
f"OWL export completed, format={request.format}, "
f"content_length={len(owl_content)}"
)
return success(data=response.model_dump(), msg="OWL文件导出成功")
except ValueError as e:
# 验证错误 (400)
api_logger.warning(f"Validation error in export: {str(e)}")
return fail(BizCode.BAD_REQUEST, "请求参数无效", str(e))
except RuntimeError as e:
# 运行时错误 (500)
api_logger.error(f"Runtime error in export: {str(e)}", exc_info=True)
return fail(BizCode.INTERNAL_ERROR, "OWL文件导出失败", str(e))
except Exception as e:
# 未知错误 (500)
api_logger.error(f"Unexpected error in export: {str(e)}", exc_info=True)
return fail(BizCode.INTERNAL_ERROR, "OWL文件导出失败", str(e))
# ==================== 本体场景管理接口 ====================
@@ -289,8 +519,7 @@ async def extract_ontology(
async def create_scene(
request: SceneCreateRequest,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
x_language_type: Optional[str] = Header(None, alias="X-Language-Type")
current_user: User = Depends(get_current_user)
):
"""创建本体场景
@@ -361,18 +590,8 @@ async def create_scene(
return fail(BizCode.BAD_REQUEST, "请求参数无效", str(e))
except RuntimeError as e:
err_str = str(e)
if "UniqueViolation" in err_str or "uq_workspace_scene_name" in err_str:
api_logger.warning(f"Duplicate scene name '{request.scene_name}' in workspace {current_user.current_workspace_id}")
from app.core.language_utils import get_language_from_header
lang = get_language_from_header(x_language_type)
if lang == "en":
msg = fail(BizCode.BAD_REQUEST, "Scene name already exists", f"A scene named \"{request.scene_name}\" already exists in the current workspace. Please use a different name.")
else:
msg = fail(BizCode.BAD_REQUEST, "场景名称已存在", f"当前工作空间下已存在名为「{request.scene_name}」的场景,请使用其他名称")
return JSONResponse(status_code=400, content=msg)
api_logger.error(f"Runtime error in scene creation: {err_str}", exc_info=True)
return fail(BizCode.INTERNAL_ERROR, "场景创建失败", err_str)
api_logger.error(f"Runtime error in scene creation: {str(e)}", exc_info=True)
return fail(BizCode.INTERNAL_ERROR, "场景创建失败", str(e))
except Exception as e:
api_logger.error(f"Unexpected error in scene creation: {str(e)}", exc_info=True)
@@ -420,20 +639,6 @@ async def update_scene(
api_logger.warning(f"User {current_user.id} has no current workspace")
return fail(BizCode.BAD_REQUEST, "请求参数无效", "当前用户没有工作空间")
# 检查是否为系统默认场景
scene_repo = OntologySceneRepository(db)
scene = scene_repo.get_by_id(scene_uuid)
if scene and scene.is_system_default:
business_logger.warning(
f"尝试修改系统默认场景: user_id={current_user.id}, "
f"scene_id={scene_id}, scene_name={scene.scene_name}"
)
return fail(
BizCode.BAD_REQUEST,
"系统默认场景不可修改",
"该场景为系统预设场景,不允许修改"
)
# 创建OntologyService实例
from app.core.memory.llm_tools.openai_client import OpenAIClient
from app.core.models.base import RedBearModelConfig
@@ -526,19 +731,6 @@ async def delete_scene(
api_logger.warning(f"User {current_user.id} has no current workspace")
return fail(BizCode.BAD_REQUEST, "请求参数无效", "当前用户没有工作空间")
# 检查是否为系统默认场景
scene_repo = OntologySceneRepository(db)
scene = scene_repo.get_by_id(scene_uuid)
if scene and scene.is_system_default:
business_logger.warning(
f"尝试删除系统默认场景: user_id={current_user.id}, "
f"scene_id={scene_id}, scene_name={scene.scene_name}"
)
raise HTTPException(
status_code=400,
detail="SYSTEM_DEFAULT_SCENE_CANNOT_DELETE"
)
# 创建OntologyService实例
from app.core.memory.llm_tools.openai_client import OpenAIClient
from app.core.models.base import RedBearModelConfig
@@ -562,9 +754,6 @@ async def delete_scene(
return success(data={"deleted": success_flag}, msg="场景删除成功")
except HTTPException:
raise
except ValueError as e:
api_logger.warning(f"Validation error in scene deletion: {str(e)}")
return fail(BizCode.BAD_REQUEST, "请求参数无效", str(e))
@@ -672,8 +861,7 @@ async def get_scenes(
async def create_class(
request: ClassCreateRequest,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
x_language_type: Optional[str] = Header(None, alias="X-Language-Type")
current_user: User = Depends(get_current_user)
):
"""创建本体类型
@@ -688,7 +876,7 @@ async def create_class(
ApiResponse: 包含创建的类型信息
"""
from app.controllers.ontology_secondary_routes import create_class_handler
return await create_class_handler(request, db, current_user, x_language_type)
return await create_class_handler(request, db, current_user)
@router.put("/class/{class_id}", response_model=ApiResponse)
@@ -815,370 +1003,3 @@ async def get_class(
"""
from app.controllers.ontology_secondary_routes import get_class_handler
return await get_class_handler(class_id, db, current_user)
# ==================== OWL 导入接口 ====================
@router.post("/import", response_model=ApiResponse)
async def import_owl_file(
scene_name: str = Form(..., description="场景名称"),
scene_description: Optional[str] = Form(None, description="场景描述(可选)"),
file: UploadFile = File(..., description="OWL/TTL文件"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""导入 OWL/TTL 文件并创建新场景
上传 OWL 或 TTL 文件解析其中定义的本体类型owl:Class
解析成功后创建新场景,并将类型保存到该场景的 ontology_class 表中。
文件格式根据文件扩展名自动识别:
- .owl, .rdf, .xml -> rdfxml 格式
- .ttl -> turtle 格式
Args:
scene_name: 场景名称(表单字段)
scene_description: 场景描述(表单字段,可选)
file: 上传的文件(支持 .owl, .ttl, .rdf, .xml
db: 数据库会话
current_user: 当前用户
Returns:
ApiResponse: 包含导入结果
"""
from app.repositories.ontology_scene_repository import OntologySceneRepository
from app.repositories.ontology_class_repository import OntologyClassRepository
# 根据文件扩展名确定格式
filename = file.filename.lower() if file.filename else ""
if filename.endswith('.ttl'):
owl_format = "turtle"
file_type = "ttl"
elif filename.endswith(('.owl', '.rdf', '.xml')):
owl_format = "rdfxml"
file_type = "owl"
else:
return fail(
BizCode.BAD_REQUEST,
"文件格式不支持",
f"不支持的文件格式: {filename},支持的格式: .owl, .ttl, .rdf, .xml"
)
api_logger.info(
f"OWL import requested by user {current_user.id}, "
f"scene_name={scene_name}, "
f"filename={file.filename}, "
f"format={owl_format}"
)
try:
# 获取当前工作空间ID
workspace_id = current_user.current_workspace_id
if not workspace_id:
api_logger.warning(f"User {current_user.id} has no current workspace")
return fail(BizCode.BAD_REQUEST, "请求参数无效", "当前用户没有工作空间")
# 1. 验证场景名称不为空
if not scene_name or not scene_name.strip():
return fail(BizCode.BAD_REQUEST, "请求参数无效", "场景名称不能为空")
scene_name = scene_name.strip()
# 2. 检查场景名称是否已存在
scene_repo = OntologySceneRepository(db)
existing_scene = scene_repo.get_by_name(scene_name, workspace_id)
if existing_scene:
api_logger.warning(f"Scene name already exists: {scene_name}")
return fail(
BizCode.BAD_REQUEST,
"场景名称已存在",
f"工作空间下已存在名为 '{scene_name}' 的场景"
)
# 3. 读取文件内容
try:
content = await file.read()
owl_content = content.decode('utf-8')
except UnicodeDecodeError:
return fail(
BizCode.BAD_REQUEST,
f"{file_type}文件导入失败",
"文件编码错误,请确保文件使用 UTF-8 编码"
)
# 4. 解析 OWL 内容(先解析,成功后再创建场景)
owl_validator = OWLValidator()
parsed_classes = owl_validator.parse_owl_content(
owl_content=owl_content,
format=owl_format
)
if not parsed_classes:
api_logger.warning("No classes found in OWL content")
return fail(
BizCode.BAD_REQUEST,
"未找到本体类型",
"文件中没有定义任何本体类型owl:Class"
)
# 5. 文件解析成功,创建场景
scene = scene_repo.create(
scene_data={
"scene_name": scene_name,
"scene_description": scene_description
},
workspace_id=workspace_id
)
scene_uuid = scene.scene_id
api_logger.info(f"Scene created for import: {scene_uuid}")
# 6. 批量创建类型(去重同一批次内的重复类型)
class_repo = OntologyClassRepository(db)
created_items = []
existing_names = set()
skipped_count = 0
for cls in parsed_classes:
class_name = cls["name"]
class_description = cls.get("description")
# 检查同一批次内是否重复
if class_name in existing_names:
skipped_count += 1
api_logger.debug(f"Skipping duplicate class in batch: {class_name}")
continue
# 创建类型
ontology_class = class_repo.create(
class_data={
"class_name": class_name,
"class_description": class_description
},
scene_id=scene_uuid
)
# 添加到已存在集合,防止同一批次内重复
existing_names.add(class_name)
created_items.append(ClassResponse(
class_id=ontology_class.class_id,
class_name=ontology_class.class_name,
class_description=ontology_class.class_description,
scene_id=ontology_class.scene_id,
created_at=ontology_class.created_at,
updated_at=ontology_class.updated_at
))
# 7. 提交事务
db.commit()
# 8. 构建响应
response = ImportOwlResponse(
scene_id=scene_uuid,
scene_name=scene.scene_name,
imported_count=len(created_items),
skipped_count=skipped_count,
items=created_items
)
api_logger.info(
f"{file_type} import completed, "
f"scene_id={scene_uuid}, "
f"scene_name={scene_name}, "
f"format={owl_format}, "
f"imported={len(created_items)}, "
f"skipped={skipped_count}"
)
return success(data=response.model_dump(), msg=f"{file_type}文件导入成功")
except ValueError as e:
db.rollback()
api_logger.warning(f"Validation error in import: {str(e)}")
return fail(BizCode.BAD_REQUEST, f"{file_type}文件导入失败", str(e))
except Exception as e:
db.rollback()
api_logger.error(f"Unexpected error in import: {str(e)}", exc_info=True)
return fail(BizCode.INTERNAL_ERROR, f"{file_type}文件导入失败", str(e))
# ==================== OWL 导出接口 ====================
@router.post("/export")
async def export_owl_by_scene(
request: ExportBySceneRequest,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""按场景导出OWL/TTL文件
根据scene_id从数据库查询该场景下的所有本体类型并导出为文件下载。
Args:
request: 导出请求,包含 scene_id 和 format
db: 数据库会话
current_user: 当前用户
Returns:
StreamingResponse: 文件流响应,浏览器会直接下载文件
"""
from uuid import UUID
from app.repositories.ontology_scene_repository import OntologySceneRepository
from app.repositories.ontology_class_repository import OntologyClassRepository
api_logger.info(
f"OWL export by scene requested by user {current_user.id}, "
f"scene_id={request.scene_id}, "
f"format={request.format}"
)
try:
# 验证格式参数
valid_formats = ["rdfxml", "turtle"]
owl_format = request.format.lower() if request.format else "rdfxml"
if owl_format not in valid_formats:
api_logger.warning(f"Invalid format: {request.format}")
return fail(
BizCode.BAD_REQUEST,
"格式参数无效",
f"不支持的格式: {request.format},支持的格式: rdfxml, turtle"
)
# 获取当前工作空间ID
workspace_id = current_user.current_workspace_id
if not workspace_id:
api_logger.warning(f"User {current_user.id} has no current workspace")
return fail(BizCode.BAD_REQUEST, "请求参数无效", "当前用户没有工作空间")
# 1. 查询场景信息
scene_repo = OntologySceneRepository(db)
scene = scene_repo.get_by_id(request.scene_id)
if not scene:
api_logger.warning(f"Scene not found: {request.scene_id}")
return fail(BizCode.NOT_FOUND, "场景不存在", f"找不到场景: {request.scene_id}")
# 验证场景属于当前工作空间
if scene.workspace_id != workspace_id:
api_logger.warning(
f"Scene {request.scene_id} does not belong to workspace {workspace_id}"
)
return fail(BizCode.FORBIDDEN, "无权访问", "该场景不属于当前工作空间")
# 2. 查询场景下的所有本体类型
class_repo = OntologyClassRepository(db)
ontology_classes_db = class_repo.get_classes_by_scene(request.scene_id)
if not ontology_classes_db:
api_logger.warning(f"No classes found in scene: {request.scene_id}")
return fail(BizCode.BAD_REQUEST, "场景为空", "该场景下没有定义任何本体类型")
# 3. 将数据库模型转换为OWL导出所需的OntologyClass格式
ontology_classes: List[OntologyClass] = []
for db_class in ontology_classes_db:
owl_class = OntologyClass(
id=str(db_class.class_id),
name=db_class.class_name,
name_chinese=db_class.class_name if _is_chinese(db_class.class_name) else None,
description=db_class.class_description or "",
examples=[],
parent_class=None,
entity_type="Concept",
domain=scene.scene_name
)
ontology_classes.append(owl_class)
# 4. 确定文件名、扩展名和 MIME 类型
file_ext = ".ttl" if owl_format == "turtle" else ".owl"
filename = _sanitize_filename(scene.scene_name) + file_ext
media_type = "text/turtle" if owl_format == "turtle" else "application/rdf+xml"
file_type = "ttl" if owl_format == "turtle" else "owl"
# 5. 导出OWL文件
with tempfile.NamedTemporaryFile(
mode='w',
suffix='.owl',
delete=False
) as tmp_file:
output_path = tmp_file.name
owl_validator = OWLValidator()
# 验证本体类
is_valid, errors, world = owl_validator.validate_ontology_classes(
classes=ontology_classes,
)
if not is_valid:
logger.warning(
f"OWL validation found {len(errors)} issues during export: {errors}"
)
if not world:
error_msg = "Failed to create OWL world for export"
logger.error(error_msg)
return fail(BizCode.INTERNAL_ERROR, "创建OWL世界失败", error_msg)
# 导出OWL文件使用请求指定的格式
owl_content = owl_validator.export_to_owl(
world=world,
output_path=output_path,
format=owl_format,
classes=ontology_classes
)
api_logger.info(
f"{file_type} export by scene completed, "
f"scene={scene.scene_name}, "
f"filename={filename}, "
f"format={owl_format}, "
f"classes_count={len(ontology_classes)}"
)
# 6. 返回文件流响应
# filename 使用 ASCII 安全的默认名filename* 使用 UTF-8 编码的实际名称
ascii_filename = f"ontology{file_ext}"
encoded_filename = quote(filename)
return StreamingResponse(
io.BytesIO(owl_content.encode('utf-8')),
media_type=media_type,
headers={
"Content-Disposition": f"attachment; filename=\"{ascii_filename}\"; filename*=UTF-8''{encoded_filename}"
}
)
except ValueError as e:
api_logger.warning(f"Validation error in export by scene: {str(e)}")
file_type = "ttl" if (request.format and request.format.lower() == "turtle") else "owl"
return fail(BizCode.BAD_REQUEST, "请求参数无效", str(e))
except RuntimeError as e:
api_logger.error(f"Runtime error in export by scene: {str(e)}", exc_info=True)
file_type = "ttl" if (request.format and request.format.lower() == "turtle") else "owl"
return fail(BizCode.INTERNAL_ERROR, f"{file_type}文件导出失败", str(e))
except Exception as e:
api_logger.error(f"Unexpected error in export by scene: {str(e)}", exc_info=True)
file_type = "ttl" if (request.format and request.format.lower() == "turtle") else "owl"
return fail(BizCode.INTERNAL_ERROR, f"{file_type}文件导出失败", str(e))
def _is_chinese(text: str) -> bool:
"""检查文本是否包含中文字符"""
for char in text:
if '\u4e00' <= char <= '\u9fff':
return True
return False
def _sanitize_filename(name: str) -> str:
"""清理文件名,移除不合法字符"""
import re
# 移除或替换不合法的文件名字符
sanitized = re.sub(r'[<>:"/\\|?*]', '_', name)
# 移除前后空格
sanitized = sanitized.strip()
# 如果为空,使用默认名称
if not sanitized:
sanitized = "ontology_export"
return sanitized

View File

@@ -7,11 +7,11 @@
from uuid import UUID
from typing import Optional
from fastapi import Depends, Header
from fastapi import Depends
from sqlalchemy.orm import Session
from app.core.error_codes import BizCode
from app.core.logging_config import get_api_logger, get_business_logger
from app.core.logging_config import get_api_logger
from app.core.response_utils import fail, success
from app.db import get_db
from app.dependencies import get_current_user
@@ -30,11 +30,9 @@ from app.schemas.response_schema import ApiResponse
from app.services.ontology_service import OntologyService
from app.core.memory.llm_tools.openai_client import OpenAIClient
from app.core.models.base import RedBearModelConfig
from app.repositories.ontology_class_repository import OntologyClassRepository
api_logger = get_api_logger()
business_logger = get_business_logger()
def _get_dummy_ontology_service(db: Session) -> OntologyService:
@@ -58,7 +56,7 @@ async def scenes_handler(
workspace_id: Optional[str] = None,
scene_name: Optional[str] = None,
page: Optional[int] = None,
pagesize: Optional[int] = None,
page_size: Optional[int] = None,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
@@ -71,14 +69,14 @@ async def scenes_handler(
workspace_id: 工作空间ID可选默认当前用户工作空间
scene_name: 场景名称关键词(可选,支持模糊匹配)
page: 页码可选从1开始仅在全量查询时有效
pagesize: 每页数量(可选,仅在全量查询时有效)
page_size: 每页数量(可选,仅在全量查询时有效)
db: 数据库会话
current_user: 当前用户
"""
operation = "search" if scene_name else "list"
api_logger.info(
f"Scene {operation} requested by user {current_user.id}, "
f"workspace_id={workspace_id}, keyword={scene_name}, page={page}, pagesize={pagesize}"
f"workspace_id={workspace_id}, keyword={scene_name}, page={page}, page_size={page_size}"
)
try:
@@ -105,13 +103,13 @@ async def scenes_handler(
api_logger.warning(f"Invalid page number: {page}")
return fail(BizCode.BAD_REQUEST, "请求参数无效", "页码必须大于0")
if pagesize is not None and pagesize < 1:
api_logger.warning(f"Invalid pagesize: {pagesize}")
if page_size is not None and page_size < 1:
api_logger.warning(f"Invalid page_size: {page_size}")
return fail(BizCode.BAD_REQUEST, "请求参数无效", "每页数量必须大于0")
# 如果只提供了page或pagesize中的一个返回错误
if (page is not None and pagesize is None) or (page is None and pagesize is not None):
api_logger.warning(f"Incomplete pagination params: page={page}, pagesize={pagesize}")
# 如果只提供了page或page_size中的一个返回错误
if (page is not None and page_size is None) or (page is None and page_size is not None):
api_logger.warning(f"Incomplete pagination params: page={page}, page_size={page_size}")
return fail(BizCode.BAD_REQUEST, "请求参数无效", "分页参数page和pagesize必须同时提供")
# 模糊搜索场景(支持分页)
@@ -119,15 +117,17 @@ async def scenes_handler(
total = len(scenes)
# 如果提供了分页参数,进行分页处理
if page is not None and pagesize is not None:
start_idx = (page - 1) * pagesize
end_idx = start_idx + pagesize
if page is not None and page_size is not None:
start_idx = (page - 1) * page_size
end_idx = start_idx + page_size
scenes = scenes[start_idx:end_idx]
# 构建响应
items = []
for scene in scenes:
# 获取前3个class_name作为entity_type
entity_type = [cls.class_name for cls in scene.classes[:3]] if scene.classes else None
# 动态计算 type_num
type_num = len(scene.classes) if scene.classes else 0
items.append(SceneResponse(
@@ -139,16 +139,17 @@ async def scenes_handler(
workspace_id=scene.workspace_id,
created_at=scene.created_at,
updated_at=scene.updated_at,
classes_count=type_num,
is_system_default=scene.is_system_default
classes_count=type_num
))
# 构建响应(包含分页信息)
if page is not None and pagesize is not None:
hasnext = (page * pagesize) < total
if page is not None and page_size is not None:
# 计算是否有下一页
hasnext = (page * page_size) < total
pagination_info = PaginationInfo(
page=page,
pagesize=pagesize,
pagesize=page_size,
total=total,
hasnext=hasnext
)
@@ -162,25 +163,28 @@ async def scenes_handler(
)
else:
# 获取所有场景(支持分页)
# 验证分页参数
if page is not None and page < 1:
api_logger.warning(f"Invalid page number: {page}")
return fail(BizCode.BAD_REQUEST, "请求参数无效", "页码必须大于0")
if pagesize is not None and pagesize < 1:
api_logger.warning(f"Invalid pagesize: {pagesize}")
if page_size is not None and page_size < 1:
api_logger.warning(f"Invalid page_size: {page_size}")
return fail(BizCode.BAD_REQUEST, "请求参数无效", "每页数量必须大于0")
# 如果只提供了page或pagesize中的一个返回错误
if (page is not None and pagesize is None) or (page is None and pagesize is not None):
api_logger.warning(f"Incomplete pagination params: page={page}, pagesize={pagesize}")
# 如果只提供了page或page_size中的一个返回错误
if (page is not None and page_size is None) or (page is None and page_size is not None):
api_logger.warning(f"Incomplete pagination params: page={page}, page_size={page_size}")
return fail(BizCode.BAD_REQUEST, "请求参数无效", "分页参数page和pagesize必须同时提供")
scenes, total = service.list_scenes(ws_uuid, page, pagesize)
scenes, total = service.list_scenes(ws_uuid, page, page_size)
# 构建响应
items = []
for scene in scenes:
# 获取前3个class_name作为entity_type
entity_type = [cls.class_name for cls in scene.classes[:3]] if scene.classes else None
# 动态计算 type_num
type_num = len(scene.classes) if scene.classes else 0
items.append(SceneResponse(
@@ -192,16 +196,17 @@ async def scenes_handler(
workspace_id=scene.workspace_id,
created_at=scene.created_at,
updated_at=scene.updated_at,
classes_count=type_num,
is_system_default=scene.is_system_default
classes_count=type_num
))
# 构建响应(包含分页信息)
if page is not None and pagesize is not None:
hasnext = (page * pagesize) < total
if page is not None and page_size is not None:
# 计算是否有下一页
hasnext = (page * page_size) < total
pagination_info = PaginationInfo(
page=page,
pagesize=pagesize,
pagesize=page_size,
total=total,
hasnext=hasnext
)
@@ -231,8 +236,7 @@ async def scenes_handler(
async def create_class_handler(
request: ClassCreateRequest,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
x_language_type: Optional[str] = None
current_user: User = Depends(get_current_user)
):
"""创建本体类型(统一使用列表形式,支持单个或批量)"""
@@ -265,11 +269,8 @@ async def create_class_handler(
]
if count == 1:
# 单个创建 - 先检查重名
# 单个创建
class_data = classes_data[0]
existing = OntologyClassRepository(db).get_by_name(class_data["class_name"], request.scene_id)
if existing:
raise ValueError(f"DUPLICATE_CLASS_NAME:{class_data['class_name']}")
ontology_class = service.create_class(
scene_id=request.scene_id,
class_name=class_data["class_name"],
@@ -327,36 +328,12 @@ async def create_class_handler(
return success(data=response.model_dump(mode='json'), msg="批量创建完成")
except ValueError as e:
err_str = str(e)
if err_str.startswith("DUPLICATE_CLASS_NAME:"):
class_name = err_str.split(":", 1)[1]
api_logger.warning(f"Duplicate class name '{class_name}' in scene {request.scene_id}")
from app.core.language_utils import get_language_from_header
from fastapi.responses import JSONResponse
lang = get_language_from_header(x_language_type)
if lang == "en":
msg = fail(BizCode.BAD_REQUEST, "Class name already exists", f"A class named \"{class_name}\" already exists in this scene. Please use a different name.")
else:
msg = fail(BizCode.BAD_REQUEST, "类型名称已存在", f"当前场景下已存在名为「{class_name}」的类型,请使用其他名称")
return JSONResponse(status_code=400, content=msg)
api_logger.warning(f"Validation error in class creation: {err_str}")
return fail(BizCode.BAD_REQUEST, "请求参数无效", err_str)
api_logger.warning(f"Validation error in class creation: {str(e)}")
return fail(BizCode.BAD_REQUEST, "请求参数无效", str(e))
except RuntimeError as e:
err_str = str(e)
if "UniqueViolation" in err_str or "uq_scene_class_name" in err_str:
api_logger.warning(f"Duplicate class name in scene {request.scene_id}")
from app.core.language_utils import get_language_from_header
from fastapi.responses import JSONResponse
lang = get_language_from_header(x_language_type)
class_name = request.classes[0].class_name if request.classes else ""
if lang == "en":
msg = fail(BizCode.BAD_REQUEST, "Class name already exists", f"A class named \"{class_name}\" already exists in this scene. Please use a different name.")
else:
msg = fail(BizCode.BAD_REQUEST, "类型名称已存在", f"当前场景下已存在名为「{class_name}」的类型,请使用其他名称")
return JSONResponse(status_code=400, content=msg)
api_logger.error(f"Runtime error in class creation: {err_str}", exc_info=True)
return fail(BizCode.INTERNAL_ERROR, "类型创建失败", err_str)
api_logger.error(f"Runtime error in class creation: {str(e)}", exc_info=True)
return fail(BizCode.INTERNAL_ERROR, "类型创建失败", str(e))
except Exception as e:
api_logger.error(f"Unexpected error in class creation: {str(e)}", exc_info=True)
@@ -389,20 +366,6 @@ async def update_class_handler(
api_logger.warning(f"User {current_user.id} has no current workspace")
return fail(BizCode.BAD_REQUEST, "请求参数无效", "当前用户没有工作空间")
# 检查是否为系统默认类型
class_repo = OntologyClassRepository(db)
ontology_class = class_repo.get_by_id(class_uuid)
if ontology_class and ontology_class.is_system_default:
business_logger.warning(
f"尝试修改系统默认类型: user_id={current_user.id}, "
f"class_id={class_id}, class_name={ontology_class.class_name}"
)
return fail(
BizCode.BAD_REQUEST,
"系统默认类型不可修改",
"该类型为系统预设类型,不允许修改"
)
# 创建Service
service = _get_dummy_ontology_service(db)
@@ -466,20 +429,6 @@ async def delete_class_handler(
api_logger.warning(f"User {current_user.id} has no current workspace")
return fail(BizCode.BAD_REQUEST, "请求参数无效", "当前用户没有工作空间")
# 检查是否为系统默认类型
class_repo = OntologyClassRepository(db)
ontology_class = class_repo.get_by_id(class_uuid)
if ontology_class and ontology_class.is_system_default:
business_logger.warning(
f"尝试删除系统默认类型: user_id={current_user.id}, "
f"class_id={class_id}, class_name={ontology_class.class_name}"
)
return fail(
BizCode.BAD_REQUEST,
"系统默认类型不可删除",
"该类型为系统预设类型,不允许删除"
)
# 创建Service
service = _get_dummy_ontology_service(db)
@@ -636,7 +585,6 @@ async def classes_handler(
scene_id=scene_uuid,
scene_name=scene.scene_name,
scene_description=scene.scene_description,
is_system_default=scene.is_system_default,
items=items
)

View File

@@ -120,8 +120,7 @@ async def get_prompt_opt(
session_id=session_id,
user_id=current_user.id,
current_prompt=data.current_prompt,
user_require=data.message,
skill=data.skill
user_require=data.message
):
# chunk 是 prompt 的增量内容
yield f"event:message\ndata: {json.dumps(chunk)}\n\n"

View File

@@ -2,32 +2,25 @@ import hashlib
import json
import uuid
from typing import Annotated
from fastapi import APIRouter, Depends, Query, Request
from fastapi.responses import StreamingResponse
from sqlalchemy.orm import Session
from app.core.error_codes import BizCode
from app.core.exceptions import BusinessException
from app.core.logging_config import get_business_logger
from app.core.response_utils import success, fail
from app.core.response_utils import success
from app.db import get_db, get_db_read
from app.dependencies import get_share_user_id, ShareTokenData
from app.models.app_model import App
from app.models.app_model import AppType
from app.repositories import knowledge_repository
from app.repositories.end_user_repository import EndUserRepository
from app.repositories.workflow_repository import WorkflowConfigRepository
from app.schemas import release_share_schema, conversation_schema
from app.schemas.response_schema import PageData, PageMeta
from app.services import workspace_service
from app.services.app_chat_service import AppChatService, get_app_chat_service
from app.services.auth_service import create_access_token
from app.services.conversation_service import ConversationService
from app.services.release_share_service import ReleaseShareService
from app.services.shared_chat_service import SharedChatService
from app.services.workflow_service import WorkflowService
from app.utils.app_config_utils import workflow_config_4_app_release, \
from app.services.app_chat_service import AppChatService, get_app_chat_service
from app.utils.app_config_utils import dict_to_multi_agent_config, workflow_config_4_app_release, \
agent_config_4_app_release, multi_agent_config_4_app_release
router = APIRouter(prefix="/public/share", tags=["Public Share"])
@@ -213,13 +206,15 @@ def list_conversations(
logger.debug(f"share_data:{share_data.user_id}")
other_id = share_data.user_id
service = SharedChatService(db)
share, release = service.get_release_by_share_token(share_data.share_token, password)
share, release = service._get_release_by_share_token(share_data.share_token, password)
from app.repositories.end_user_repository import EndUserRepository
end_user_repo = EndUserRepository(db)
new_end_user = end_user_repo.get_or_create_end_user(
app_id=share.app_id,
other_id=other_id
)
logger.debug(new_end_user.id)
service = SharedChatService(db)
conversations, total = service.list_conversations(
share_token=share_data.share_token,
user_id=str(new_end_user.id),
@@ -298,15 +293,19 @@ async def chat(
# 提前验证和准备(在流式响应开始前完成)
# 这样可以确保错误能正确返回,而不是在流式响应中间出错
from app.models.app_model import AppType
try:
from app.core.exceptions import BusinessException
from app.core.error_codes import BizCode
from app.services.app_service import AppService
# 验证分享链接和密码
share, release = service.get_release_by_share_token(share_token, password)
share, release = service._get_release_by_share_token(share_token, password)
# # Create end_user_id by concatenating app_id with user_id
# end_user_id = f"{share.app_id}_{user_id}"
# Store end_user_id in database with original user_id
from app.repositories.end_user_repository import EndUserRepository
end_user_repo = EndUserRepository(db)
new_end_user = end_user_repo.get_or_create_end_user(
app_id=share.app_id,
@@ -319,6 +318,7 @@ async def chat(
"""获取存储类型和工作空间的ID"""
# 直接通过 SQLAlchemy 查询 app仅查询未删除的应用
from app.models.app_model import App
app = db.query(App).filter(
App.id == appid,
App.is_active.is_(True)
@@ -359,12 +359,12 @@ async def chat(
app_type = release.app.type if release.app else None
# 根据应用类型验证配置
if app_type == AppType.AGENT:
if app_type == "agent":
# Agent 类型:验证模型配置
model_config_id = release.default_model_config_id
if not model_config_id:
raise BusinessException("Agent 应用未配置模型", BizCode.AGENT_CONFIG_MISSING)
elif app_type == AppType.MULTI_AGENT:
elif app_type == "multi_agent":
# Multi-Agent 类型:验证多 Agent 配置
config = release.config or {}
if not config.get("sub_agents"):
@@ -438,8 +438,7 @@ async def chat(
memory=payload.memory,
storage_type=storage_type,
user_rag_memory_id=user_rag_memory_id,
workspace_id=workspace_id,
files=payload.files # 传递多模态文件
workspace_id=workspace_id
):
yield event
@@ -476,8 +475,7 @@ async def chat(
memory=payload.memory,
storage_type=storage_type,
user_rag_memory_id=user_rag_memory_id,
workspace_id=workspace_id,
files=payload.files # 传递多模态文件
workspace_id=workspace_id
)
return success(data=conversation_schema.ChatResponse(**result).model_dump(mode="json"))
elif app_type == AppType.MULTI_AGENT:
@@ -580,7 +578,6 @@ async def chat(
conversation_id=conversation.id, # 使用已创建的会话 ID
user_id=end_user_id, # 转换为字符串
variables=payload.variables,
files=payload.files,
config=config,
web_search=payload.web_search,
memory=payload.memory,
@@ -588,8 +585,7 @@ async def chat(
user_rag_memory_id=user_rag_memory_id,
app_id=release.app_id,
workspace_id=workspace_id,
release_id=release.id,
public=True
release_id=release.id
):
event_type = event.get("event", "message")
event_data = event.get("data", {})
@@ -638,34 +634,6 @@ async def chat(
# return success(data=conversation_schema.ChatResponse(**result).model_dump(mode="json"))
else:
from app.core.exceptions import BusinessException
from app.core.error_codes import BizCode
raise BusinessException(f"不支持的应用类型: {app_type}", BizCode.APP_TYPE_NOT_SUPPORTED)
@router.get("/config", summary="获取应用启动配置")
async def config_query(
password: str = Query(None, description="访问密码"),
share_data: ShareTokenData = Depends(get_share_user_id),
db: Session = Depends(get_db),
):
share_service = SharedChatService(db)
share_token = share_data.share_token
share, release = share_service.get_release_by_share_token(share_token, password)
if release.app.type == AppType.WORKFLOW:
workflow_service = WorkflowService(db)
content = {
"app_type": release.app.type,
"variables": workflow_service.get_start_node_variables(release.config)
}
elif release.app.type == AppType.AGENT:
content = {
"app_type": release.app.type,
"variables": release.config.get("variables")
}
elif release.app.type == AppType.MULTI_AGENT:
content = {
"app_type": release.app.type,
"variables": []
}
else:
return fail(msg="Unsupported app type", code=BizCode.APP_TYPE_NOT_SUPPORTED)
return success(data=content)

View File

@@ -12,6 +12,7 @@ from app.core.exceptions import BusinessException
from app.core.logging_config import get_business_logger
from app.core.response_utils import success
from app.db import get_db
from app.dependencies import get_app_or_workspace
from app.models.app_model import App
from app.models.app_model import AppType
from app.repositories import knowledge_repository
@@ -20,10 +21,9 @@ from app.schemas import AppChatRequest, conversation_schema
from app.schemas.api_key_schema import ApiKeyAuth
from app.services import workspace_service
from app.services.app_chat_service import AppChatService, get_app_chat_service
from app.services.app_service import get_app_service, AppService
from app.services.conversation_service import ConversationService, get_conversation_service
from app.utils.app_config_utils import workflow_config_4_app_release, \
agent_config_4_app_release, multi_agent_config_4_app_release
from app.utils.app_config_utils import dict_to_multi_agent_config, workflow_config_4_app_release, agent_config_4_app_release, multi_agent_config_4_app_release
from app.services.app_service import get_app_service, AppService
router = APIRouter(prefix="/app", tags=["V1 - App API"])
logger = get_business_logger()
@@ -34,7 +34,6 @@ async def list_apps():
"""列出可访问的应用(占位)"""
return success(data=[], msg="App API - Coming Soon")
# /v1/app/chat
# @router.post("/chat")
@@ -74,21 +73,21 @@ def _checkAppConfig(app: App):
else:
raise BusinessException("不支持的应用类型", BizCode.AGENT_CONFIG_MISSING)
@router.post("/chat")
@require_api_key(scopes=["app"])
async def chat(
request: Request,
api_key_auth: ApiKeyAuth = None,
db: Session = Depends(get_db),
conversation_service: Annotated[ConversationService, Depends(get_conversation_service)] = None,
app_chat_service: Annotated[AppChatService, Depends(get_app_chat_service)] = None,
app_service: Annotated[AppService, Depends(get_app_service)] = None,
message: str = Body(..., description="聊天消息内容"),
request:Request,
api_key_auth: ApiKeyAuth = None,
db: Session = Depends(get_db),
conversation_service: Annotated[ConversationService, Depends(get_conversation_service)] = None,
app_chat_service: Annotated[AppChatService, Depends(get_app_chat_service)] = None,
app_service: Annotated[AppService, Depends(get_app_service)] = None,
message: str = Body(..., description="聊天消息内容"),
):
body = await request.json()
payload = AppChatRequest(**body)
other_id = payload.user_id
app = app_service.get_app(api_key_auth.resource_id, api_key_auth.workspace_id)
other_id = payload.user_id
workspace_id = app.workspace_id
@@ -99,8 +98,8 @@ async def chat(
original_user_id=other_id # Save original user_id to other_id
)
end_user_id = str(new_end_user.id)
web_search = True
memory = True
web_search=True
memory=True
# 提前验证和准备(在流式响应开始前完成)
storage_type = workspace_service.get_workspace_storage_type_without_auth(
db=db,
@@ -134,8 +133,7 @@ async def chat(
app_id=app.id,
workspace_id=workspace_id,
user_id=end_user_id,
is_draft=False,
conversation_id=payload.conversation_id
is_draft=False
)
if app_type == AppType.AGENT:
@@ -148,17 +146,16 @@ async def chat(
if payload.stream:
async def event_generator():
async for event in app_chat_service.agnet_chat_stream(
message=payload.message,
conversation_id=conversation.id, # 使用已创建的会话 ID
user_id=end_user_id, # 转换为字符串
variables=payload.variables,
web_search=web_search,
config=agent_config,
memory=memory,
storage_type=storage_type,
user_rag_memory_id=user_rag_memory_id,
workspace_id=workspace_id,
files=payload.files # 传递多模态文件
message=payload.message,
conversation_id=conversation.id, # 使用已创建的会话 ID
user_id= end_user_id, # 转换为字符串
variables=payload.variables,
web_search=web_search,
config=agent_config,
memory=memory,
storage_type=storage_type,
user_rag_memory_id=user_rag_memory_id,
workspace_id=workspace_id
):
yield event
@@ -178,13 +175,12 @@ async def chat(
conversation_id=conversation.id, # 使用已创建的会话 ID
user_id=end_user_id, # 转换为字符串
variables=payload.variables,
config=agent_config,
config= agent_config,
web_search=web_search,
memory=memory,
storage_type=storage_type,
user_rag_memory_id=user_rag_memory_id,
workspace_id=workspace_id,
files=payload.files # 传递多模态文件
workspace_id=workspace_id
)
return success(data=conversation_schema.ChatResponse(**result).model_dump(mode="json"))
elif app_type == AppType.MULTI_AGENT:
@@ -194,15 +190,15 @@ async def chat(
async def event_generator():
async for event in app_chat_service.multi_agent_chat_stream(
message=payload.message,
conversation_id=conversation.id, # 使用已创建的会话 ID
user_id=end_user_id, # 转换为字符串
variables=payload.variables,
config=config,
web_search=web_search,
memory=memory,
storage_type=storage_type,
user_rag_memory_id=user_rag_memory_id
message=payload.message,
conversation_id=conversation.id, # 使用已创建的会话 ID
user_id=end_user_id, # 转换为字符串
variables=payload.variables,
config=config,
web_search=web_search,
memory=memory,
storage_type=storage_type,
user_rag_memory_id=user_rag_memory_id
):
yield event
@@ -236,20 +232,19 @@ async def chat(
if payload.stream:
async def event_generator():
async for event in app_chat_service.workflow_chat_stream(
message=payload.message,
conversation_id=conversation.id, # 使用已创建的会话 ID
user_id=end_user_id, # 转换为字符串
variables=payload.variables,
files=payload.files,
config=config,
web_search=web_search,
memory=memory,
storage_type=storage_type,
user_rag_memory_id=user_rag_memory_id,
app_id=app.id,
workspace_id=workspace_id,
release_id=app.current_release.id,
public=True
message=payload.message,
conversation_id=conversation.id, # 使用已创建的会话 ID
user_id=end_user_id, # 转换为字符串
variables=payload.variables,
config=config,
web_search=web_search,
memory=memory,
storage_type=storage_type,
user_rag_memory_id=user_rag_memory_id,
app_id=app.id,
workspace_id=workspace_id,
release_id=app.current_release.id,
):
event_type = event.get("event", "message")
event_data = event.get("data", {})
@@ -299,3 +294,4 @@ async def chat(
from app.core.exceptions import BusinessException
from app.core.error_codes import BizCode
raise BusinessException(f"不支持的应用类型: {app_type}", BizCode.APP_TYPE_NOT_SUPPORTED)

View File

@@ -39,7 +39,7 @@ async def write_memory_api_service(
Stores memory content for the specified end user using the Memory API Service.
"""
logger.info(f"Memory write request - end_user_id: {payload.end_user_id}, workspace_id: {api_key_auth.workspace_id}")
logger.info(f"Memory write request - end_user_id: {payload.end_user_id}, tenant_id: {api_key_auth.tenant_id}")
memory_api_service = MemoryAPIService(db)

View File

@@ -246,73 +246,3 @@ async def rebuild_knowledge_graph(
db=db,
current_user=current_user)
@router.get("/check/yuque/auth", response_model=ApiResponse)
@require_api_key(scopes=["rag"])
async def check_yuque_auth(
yuque_user_id: str,
yuque_token: str,
request: Request,
api_key_auth: ApiKeyAuth = None,
db: Session = Depends(get_db),
):
"""
check yuque auth info
"""
api_key = api_key_service.ApiKeyService.get_api_key(db, api_key_auth.api_key_id, api_key_auth.workspace_id)
current_user = api_key.creator
current_user.current_workspace_id = api_key_auth.workspace_id
api_logger.info(f"check yuque auth info, username: {current_user.username}")
return await knowledge_controller.check_yuque_auth(yuque_user_id=yuque_user_id,
yuque_token=yuque_token,
db=db,
current_user=current_user)
@router.get("/check/feishu/auth", response_model=ApiResponse)
@require_api_key(scopes=["rag"])
async def check_feishu_auth(
feishu_app_id: str,
feishu_app_secret: str,
feishu_folder_token: str,
request: Request,
api_key_auth: ApiKeyAuth = None,
db: Session = Depends(get_db),
):
"""
check feishu auth info
"""
api_key = api_key_service.ApiKeyService.get_api_key(db, api_key_auth.api_key_id, api_key_auth.workspace_id)
current_user = api_key.creator
current_user.current_workspace_id = api_key_auth.workspace_id
api_logger.info(f"check feishu auth info, username: {current_user.username}")
return await knowledge_controller.check_feishu_auth(feishu_app_id=feishu_app_id,
feishu_app_secret=feishu_app_secret,
feishu_folder_token=feishu_folder_token,
db=db,
current_user=current_user)
@router.post("/{knowledge_id}/sync", response_model=ApiResponse)
@require_api_key(scopes=["rag"])
async def sync_knowledge(
knowledge_id: uuid.UUID,
request: Request,
api_key_auth: ApiKeyAuth = None,
db: Session = Depends(get_db),
):
"""
sync knowledge base information based on knowledge_id
"""
api_key = api_key_service.ApiKeyService.get_api_key(db, api_key_auth.api_key_id, api_key_auth.workspace_id)
current_user = api_key.creator
current_user.current_workspace_id = api_key_auth.workspace_id
return await knowledge_controller.sync_knowledge(knowledge_id=knowledge_id,
db=db,
current_user=current_user)

View File

@@ -1,85 +0,0 @@
"""Skill Controller - 技能市场管理"""
from fastapi import APIRouter, Depends, Query
from sqlalchemy.orm import Session
from typing import Optional
import uuid
from app.db import get_db
from app.dependencies import get_current_user
from app.models import User
from app.schemas import skill_schema
from app.schemas.response_schema import PageData, PageMeta
from app.services.skill_service import SkillService
from app.core.response_utils import success
router = APIRouter(prefix="/skills", tags=["Skills"])
@router.post("", summary="创建技能")
def create_skill(
data: skill_schema.SkillCreate,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""创建技能 - 可以关联现有工具内置、MCP、自定义"""
tenant_id = current_user.tenant_id
skill = SkillService.create_skill(db, data, tenant_id)
return success(data=skill_schema.Skill.model_validate(skill), msg="技能创建成功")
@router.get("", summary="技能列表")
def list_skills(
search: Optional[str] = Query(None, description="搜索关键词"),
is_active: Optional[bool] = Query(None, description="是否激活"),
is_public: Optional[bool] = Query(None, description="是否公开"),
page: int = Query(1, ge=1, description="页码"),
pagesize: int = Query(10, ge=1, le=100, description="每页数量"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""技能市场列表 - 包含本工作空间和公开的技能"""
tenant_id = current_user.tenant_id
skills, total = SkillService.list_skills(
db, tenant_id, search, is_active, is_public, page, pagesize
)
items = [skill_schema.Skill.model_validate(s) for s in skills]
meta = PageMeta(page=page, pagesize=pagesize, total=total, hasnext=(page * pagesize) < total)
return success(data=PageData(page=meta, items=items), msg="技能市场列表获取成功")
@router.get("/{skill_id}", summary="获取技能详情")
def get_skill(
skill_id: uuid.UUID,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""获取技能详情"""
tenant_id = current_user.tenant_id
skill = SkillService.get_skill(db, skill_id, tenant_id)
return success(data=skill_schema.Skill.model_validate(skill), msg="获取技能详情成功")
@router.put("/{skill_id}", summary="更新技能")
def update_skill(
skill_id: uuid.UUID,
data: skill_schema.SkillUpdate,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""更新技能"""
tenant_id = current_user.tenant_id
skill = SkillService.update_skill(db, skill_id, data, tenant_id)
return success(data=skill_schema.Skill.model_validate(skill), msg="技能更新成功")
@router.delete("/{skill_id}", summary="删除技能")
def delete_skill(
skill_id: uuid.UUID,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""删除技能"""
tenant_id = current_user.tenant_id
SkillService.delete_skill(db, skill_id, tenant_id)
return success(msg="技能删除成功")

View File

@@ -14,7 +14,6 @@ from app.models import User
from app.models.tool_model import ToolType, ToolStatus, AuthType
from app.services.tool_service import ToolService
from app.schemas.response_schema import ApiResponse
from app.core.exceptions import BusinessException
router = APIRouter(prefix="/tools", tags=["Tool System"])
@@ -98,13 +97,7 @@ async def create_tool(
):
"""创建工具"""
try:
# 将 MCP 来源字段合并进 config
if request.tool_type == ToolType.MCP:
for key in ("source_channel", "market_id", "market_config_id", "mcp_service_id"):
val = getattr(request, key, None)
if val is not None:
request.config[key] = val
tool_id = await service.create_tool(
tool_id = service.create_tool(
name=request.name,
tool_type=request.tool_type,
tenant_id=current_user.tenant_id,
@@ -114,8 +107,6 @@ async def create_tool(
tags=request.tags
)
return success(data={"tool_id": tool_id}, msg="工具创建成功")
except BusinessException as e:
raise HTTPException(status_code=400, detail=e.message)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:

View File

@@ -2,23 +2,15 @@ from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
import uuid
from app.core.error_codes import BizCode
from app.core.exceptions import BusinessException
from app.db import get_db
from app.dependencies import get_current_user, get_current_superuser
from app.models.user_model import User
from app.schemas import user_schema
from app.schemas.user_schema import (
ChangePasswordRequest,
AdminChangePasswordRequest,
SendEmailCodeRequest,
VerifyEmailCodeRequest,
VerifyPasswordRequest)
from app.schemas.user_schema import ChangePasswordRequest, AdminChangePasswordRequest
from app.schemas.response_schema import ApiResponse
from app.services import user_service
from app.core.logging_config import get_api_logger
from app.core.response_utils import success
from app.core.security import verify_password
# 获取API专用日志器
api_logger = get_api_logger()
@@ -100,7 +92,7 @@ def get_current_user_info(
result_schema.current_workspace_name = current_workspace.name
for ws in result.workspaces:
if ws.workspace_id == current_user.current_workspace_id and ws.is_active:
if ws.workspace_id == current_user.current_workspace_id:
result_schema.role = ws.role
break
@@ -128,7 +120,6 @@ def get_tenant_superusers(
return success(data=superusers_schema, msg="租户超管列表获取成功")
@router.get("/{user_id}", response_model=ApiResponse)
def get_user_info_by_id(
user_id: uuid.UUID,
@@ -189,54 +180,4 @@ async def admin_change_password(
return success(msg="密码修改成功")
else:
api_logger.info(f"管理员密码重置成功: 用户 {request.user_id}, 随机密码已生成")
return success(data=generated_password, msg="密码重置成功")
@router.post("/verify_pwd", response_model=ApiResponse)
def verify_pwd(
request: VerifyPasswordRequest,
current_user: User = Depends(get_current_user),
):
"""验证当前用户密码"""
api_logger.info(f"用户验证密码请求: {current_user.username}")
is_valid = verify_password(request.password, current_user.hashed_password)
api_logger.info(f"用户密码验证结果: {current_user.username}, valid={is_valid}")
if not is_valid:
raise BusinessException("密码验证失败", code=BizCode.VALIDATION_FAILED)
return success(data={"valid": is_valid}, msg="验证完成")
@router.post("/send-email-code", response_model=ApiResponse)
async def send_email_code(
request: SendEmailCodeRequest,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
):
"""发送邮箱验证码"""
api_logger.info(f"用户请求发送邮箱验证码: {current_user.username}, email={request.email}")
await user_service.send_email_code_method(db=db, email=request.email, user_id=current_user.id)
api_logger.info(f"邮箱验证码已发送: {current_user.username}")
return success(msg="验证码已发送到您的邮箱,请查收")
@router.put("/change-email", response_model=ApiResponse)
async def change_email(
request: VerifyEmailCodeRequest,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user),
):
"""验证验证码并修改邮箱"""
api_logger.info(f"用户修改邮箱: {current_user.username}, new_email={request.new_email}")
await user_service.verify_and_change_email(
db=db,
user_id=current_user.id,
new_email=request.new_email,
code=request.code
)
api_logger.info(f"用户邮箱修改成功: {current_user.username}")
return success(msg="邮箱修改成功")
return success(data=generated_password, msg="密码重置成功")

View File

@@ -8,11 +8,11 @@ from sqlalchemy.orm import Session
from fastapi import APIRouter, Depends,Header
from app.db import get_db
from app.core.language_utils import get_language_from_header
from app.core.logging_config import get_api_logger
from app.core.response_utils import success, fail
from app.core.error_codes import BizCode
from app.core.api_key_utils import timestamp_to_datetime
from app.services.memory_base_service import Translation_English
from app.services.user_memory_service import (
UserMemoryService,
analytics_memory_types,
@@ -45,6 +45,7 @@ router = APIRouter(
@router.get("/analytics/memory_insight/report", response_model=ApiResponse)
async def get_memory_insight_report_api(
end_user_id: str,
language_type: str = Header(default="zh", alias="X-Language-Type"),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
) -> dict:
@@ -54,10 +55,18 @@ async def get_memory_insight_report_api(
此接口仅查询数据库中已缓存的记忆洞察数据,不执行生成操作。
如需生成新的洞察报告,请使用专门的生成接口。
"""
workspace_id = current_user.current_workspace_id
workspace_repo = WorkspaceRepository(db)
workspace_models = workspace_repo.get_workspace_models_configs(workspace_id)
if workspace_models:
model_id = workspace_models.get("llm", None)
else:
model_id = None
api_logger.info(f"记忆洞察报告查询请求: end_user_id={end_user_id}, user={current_user.username}")
try:
# 调用服务层获取缓存数据
result = await user_memory_service.get_cached_memory_insight(db, end_user_id)
result = await user_memory_service.get_cached_memory_insight(db, end_user_id,model_id,language_type)
if result["is_cached"]:
api_logger.info(f"成功返回缓存的记忆洞察报告: end_user_id={end_user_id}")
@@ -73,7 +82,7 @@ async def get_memory_insight_report_api(
@router.get("/analytics/user_summary", response_model=ApiResponse)
async def get_user_summary_api(
end_user_id: str,
language_type: str = Header(default=None, alias="X-Language-Type"),
language_type: str = Header(default="zh", alias="X-Language-Type"),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
) -> dict:
@@ -82,14 +91,7 @@ async def get_user_summary_api(
此接口仅查询数据库中已缓存的用户摘要数据,不执行生成操作。
如需生成新的用户摘要,请使用专门的生成接口。
语言控制:
- 使用 X-Language-Type Header 指定语言
- 如果未传 Header默认使用中文 (zh)
"""
# 使用集中化的语言校验
language = get_language_from_header(language_type)
workspace_id = current_user.current_workspace_id
workspace_repo = WorkspaceRepository(db)
workspace_models = workspace_repo.get_workspace_models_configs(workspace_id)
@@ -101,7 +103,7 @@ async def get_user_summary_api(
api_logger.info(f"用户摘要查询请求: end_user_id={end_user_id}, user={current_user.username}")
try:
# 调用服务层获取缓存数据
result = await user_memory_service.get_cached_user_summary(db, end_user_id,model_id,language)
result = await user_memory_service.get_cached_user_summary(db, end_user_id,model_id,language_type)
if result["is_cached"]:
api_logger.info(f"成功返回缓存的用户摘要: end_user_id={end_user_id}")
@@ -117,7 +119,6 @@ async def get_user_summary_api(
@router.post("/analytics/generate_cache", response_model=ApiResponse)
async def generate_cache_api(
request: GenerateCacheRequest,
language_type: str = Header(default=None, alias="X-Language-Type"),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
) -> dict:
@@ -126,14 +127,7 @@ async def generate_cache_api(
- 如果提供 end_user_id只为该用户生成
- 如果不提供,为当前工作空间的所有用户生成
语言控制:
- 使用 X-Language-Type Header 指定语言 ("zh" 中文, "en" 英文)
- 如果未传 Header默认使用中文 (zh)
"""
# 使用集中化的语言校验
language = get_language_from_header(language_type)
workspace_id = current_user.current_workspace_id
# 检查用户是否已选择工作空间
@@ -145,7 +139,7 @@ async def generate_cache_api(
api_logger.info(
f"缓存生成请求: user={current_user.username}, workspace={workspace_id}, "
f"end_user_id={end_user_id if end_user_id else '全部用户'}, language={language}"
f"end_user_id={end_user_id if end_user_id else '全部用户'}"
)
try:
@@ -154,10 +148,10 @@ async def generate_cache_api(
api_logger.info(f"开始为单个用户生成缓存: end_user_id={end_user_id}")
# 生成记忆洞察
insight_result = await user_memory_service.generate_and_cache_insight(db, end_user_id, workspace_id, language=language)
insight_result = await user_memory_service.generate_and_cache_insight(db, end_user_id, workspace_id)
# 生成用户摘要
summary_result = await user_memory_service.generate_and_cache_summary(db, end_user_id, workspace_id, language=language)
summary_result = await user_memory_service.generate_and_cache_summary(db, end_user_id, workspace_id)
# 构建响应
result = {
@@ -191,7 +185,7 @@ async def generate_cache_api(
# 为整个工作空间生成
api_logger.info(f"开始为工作空间 {workspace_id} 批量生成缓存")
result = await user_memory_service.generate_cache_for_workspace(db, workspace_id, language=language)
result = await user_memory_service.generate_cache_for_workspace(db, workspace_id)
# 记录统计信息
api_logger.info(
@@ -391,13 +385,10 @@ async def update_end_user_profile(
return fail(BizCode.INTERNAL_ERROR, "用户信息更新失败", error_msg)
@router.get("/memory_space/timeline_memories", response_model=ApiResponse)
async def memory_space_timeline_of_shared_memories(id: str, label: str,language_type: str = Header(default=None, alias="X-Language-Type"),
async def memory_space_timeline_of_shared_memories(id: str, label: str,language_type: str = Header(default="zh", alias="X-Language-Type"),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
# 使用集中化的语言校验
language = get_language_from_header(language_type)
workspace_id=current_user.current_workspace_id
workspace_repo = WorkspaceRepository(db)
workspace_models = workspace_repo.get_workspace_models_configs(workspace_id)
@@ -407,7 +398,7 @@ async def memory_space_timeline_of_shared_memories(id: str, label: str,language_
else:
model_id = None
MemoryEntity = MemoryEntityService(id, label)
timeline_memories_result = await MemoryEntity.get_timeline_memories_server(model_id, language)
timeline_memories_result = await MemoryEntity.get_timeline_memories_server(model_id, language_type)
return success(data=timeline_memories_result, msg="共同记忆时间线")
@router.get("/memory_space/relationship_evolution", response_model=ApiResponse)

View File

@@ -0,0 +1,610 @@
"""
工作流 API 控制器
"""
import logging
import uuid
from typing import Annotated
from fastapi import APIRouter, Depends, Path, Query
from sqlalchemy.orm import Session
from app.db import get_db
from app.dependencies import get_current_user, cur_workspace_access_guard
from app.models.user_model import User
from app.models.app_model import App
from app.services.workflow_service import WorkflowService, get_workflow_service
from app.schemas.workflow_schema import (
WorkflowConfigCreate,
WorkflowConfigUpdate,
WorkflowConfig,
WorkflowValidationResponse,
WorkflowExecution,
WorkflowNodeExecution,
WorkflowExecutionRequest,
WorkflowExecutionResponse
)
from app.core.response_utils import success, fail
from app.core.exceptions import BusinessException
from app.core.error_codes import BizCode
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/apps", tags=["workflow"])
# ==================== 工作流配置管理 ====================
@router.post("/{app_id}/workflow")
@cur_workspace_access_guard()
async def create_workflow_config(
app_id: Annotated[uuid.UUID, Path(description="应用 ID")],
config: WorkflowConfigCreate,
db: Annotated[Session, Depends(get_db)],
current_user: Annotated[User, Depends(get_current_user)],
service: Annotated[WorkflowService, Depends(get_workflow_service)]
):
"""创建工作流配置
创建或更新应用的工作流配置。配置会进行基础验证,但允许保存不完整的配置(草稿)。
"""
try:
# 验证应用是否存在且属于当前工作空间
app = db.query(App).filter(
App.id == app_id,
App.workspace_id == current_user.current_workspace_id,
App.is_active.is_(True)
).first()
if not app:
return fail(
code=BizCode.NOT_FOUND,
msg="应用不存在或无权访问"
)
# 验证应用类型
if app.type != "workflow":
return fail(
code=BizCode.INVALID_PARAMETER,
msg=f"应用类型必须为 workflow当前为 {app.type}"
)
# 创建工作流配置
workflow_config = service.create_workflow_config(
app_id=app_id,
nodes=[node.model_dump() for node in config.nodes],
edges=[edge.model_dump() for edge in config.edges],
variables=[var.model_dump() for var in config.variables],
execution_config=config.execution_config.model_dump(),
triggers=[trigger.model_dump() for trigger in config.triggers],
validate=True # 进行基础验证
)
return success(
data=WorkflowConfig.model_validate(workflow_config),
msg="工作流配置创建成功"
)
except BusinessException as e:
logger.warning(f"创建工作流配置失败: {e.message}")
return fail(code=e.error_code, msg=e.message)
except Exception as e:
logger.error(f"创建工作流配置异常: {e}", exc_info=True)
return fail(
code=BizCode.INTERNAL_ERROR,
msg=f"创建工作流配置失败: {str(e)}"
)
#
# @router.get("/{app_id}/workflow")
# async def get_workflow_config(
# app_id: Annotated[uuid.UUID, Path(description="应用 ID")],
# db: Annotated[Session, Depends(get_db)],
# current_user: Annotated[User, Depends(get_current_user)]
#
# ):
# """获取工作流配置
#
# 获取应用的工作流配置详情。
# """
# try:
# # 验证应用是否存在且属于当前工作空间
# app = db.query(App).filter(
# App.id == app_id,
# App.workspace_id == current_user.current_workspace_id,
# App.is_active == True
# ).first()
#
# if not app:
# return fail(
# code=BizCode.NOT_FOUND,
# msg="应用不存在或无权访问"
# )
#
# # 获取工作流配置
# service = WorkflowService(db)
# workflow_config = service.get_workflow_config(app_id)
#
# if not workflow_config:
# return fail(
# code=BizCode.NOT_FOUND,
# msg="工作流配置不存在"
# )
#
# return success(
# data=WorkflowConfig.model_validate(workflow_config)
# )
#
# except Exception as e:
# logger.error(f"获取工作流配置异常: {e}", exc_info=True)
# return fail(
# code=BizCode.INTERNAL_ERROR,
# msg=f"获取工作流配置失败: {str(e)}"
# )
# @router.put("/{app_id}/workflow")
# async def update_workflow_config(
# app_id: Annotated[uuid.UUID, Path(description="应用 ID")],
# config: WorkflowConfigUpdate,
# db: Annotated[Session, Depends(get_db)],
# current_user: Annotated[User, Depends(get_current_user)],
# service: Annotated[WorkflowService, Depends(get_workflow_service)]
# ):
# """更新工作流配置
# 更新应用的工作流配置。可以部分更新,未提供的字段保持不变。
# """
# try:
# # 验证应用是否存在且属于当前工作空间
# app = db.query(App).filter(
# App.id == app_id,
# App.workspace_id == current_user.current_workspace_id,
# App.is_active == True
# ).first()
# if not app:
# return fail(
# code=BizCode.NOT_FOUND,
# msg="应用不存在或无权访问"
# )
# # 更新工作流配置
# workflow_config = service.update_workflow_config(
# app_id=app_id,
# nodes=[node.model_dump() for node in config.nodes] if config.nodes else None,
# edges=[edge.model_dump() for edge in config.edges] if config.edges else None,
# variables=[var.model_dump() for var in config.variables] if config.variables else None,
# execution_config=config.execution_config.model_dump() if config.execution_config else None,
# triggers=[trigger.model_dump() for trigger in config.triggers] if config.triggers else None,
# validate=True
# )
# return success(
# data=WorkflowConfig.model_validate(workflow_config),
# msg="工作流配置更新成功"
# )
# except BusinessException as e:
# logger.warning(f"更新工作流配置失败: {e.message}")
# return fail(code=e.error_code, msg=e.message)
# except Exception as e:
# logger.error(f"更新工作流配置异常: {e}", exc_info=True)
# return fail(
# code=BizCode.INTERNAL_ERROR,
# msg=f"更新工作流配置失败: {str(e)}"
# )
@router.delete("/{app_id}/workflow")
async def delete_workflow_config(
app_id: Annotated[uuid.UUID, Path(description="应用 ID")],
db: Annotated[Session, Depends(get_db)],
current_user: Annotated[User, Depends(get_current_user)],
service: Annotated[WorkflowService, Depends(get_workflow_service)]
):
"""删除工作流配置
删除应用的工作流配置。
"""
try:
# 验证应用是否存在且属于当前工作空间
app = db.query(App).filter(
App.id == app_id,
App.workspace_id == current_user.current_workspace_id,
App.is_active.is_(True)
).first()
if not app:
return fail(
code=BizCode.NOT_FOUND,
msg="应用不存在或无权访问"
)
# 删除工作流配置
deleted = service.delete_workflow_config(app_id)
if not deleted:
return fail(
code=BizCode.NOT_FOUND,
msg="工作流配置不存在"
)
return success(msg="工作流配置删除成功")
except Exception as e:
logger.error(f"删除工作流配置异常: {e}", exc_info=True)
return fail(
code=BizCode.INTERNAL_ERROR,
msg=f"删除工作流配置失败: {str(e)}"
)
@router.post("/{app_id}/workflow/validate")
async def validate_workflow_config(
app_id: Annotated[uuid.UUID, Path(description="应用 ID")],
db: Annotated[Session, Depends(get_db)],
current_user: Annotated[User, Depends(get_current_user)],
service: Annotated[WorkflowService, Depends(get_workflow_service)],
for_publish: Annotated[bool, Query(description="是否为发布验证")] = False
):
"""验证工作流配置
验证工作流配置是否有效。可以选择是否进行发布级别的严格验证。
"""
try:
# 验证应用是否存在且属于当前工作空间
app = db.query(App).filter(
App.id == app_id,
App.workspace_id == current_user.current_workspace_id,
App.is_active.is_(True)
).first()
if not app:
return fail(
code=BizCode.NOT_FOUND,
msg="应用不存在或无权访问"
)
# 验证工作流配置
if for_publish:
is_valid, errors = service.validate_workflow_config_for_publish(app_id)
else:
workflow_config = service.get_workflow_config(app_id)
if not workflow_config:
return fail(
code=BizCode.NOT_FOUND,
msg="工作流配置不存在"
)
from app.core.workflow.validator import validate_workflow_config as validate_config
config_dict = {
"nodes": workflow_config.nodes,
"edges": workflow_config.edges,
"variables": workflow_config.variables,
"execution_config": workflow_config.execution_config,
"triggers": workflow_config.triggers
}
is_valid, errors = validate_config(config_dict, for_publish=False)
return success(
data=WorkflowValidationResponse(
is_valid=is_valid,
errors=errors,
warnings=[]
)
)
except BusinessException as e:
logger.warning(f"验证工作流配置失败: {e.message}")
return fail(code=e.error_code, msg=e.message)
except Exception as e:
logger.error(f"验证工作流配置异常: {e}", exc_info=True)
return fail(
code=BizCode.INTERNAL_ERROR,
msg=f"验证工作流配置失败: {str(e)}"
)
# ==================== 工作流执行管理 ====================
@router.get("/{app_id}/workflow/executions")
async def get_workflow_executions(
app_id: Annotated[uuid.UUID, Path(description="应用 ID")],
db: Annotated[Session, Depends(get_db)],
current_user: Annotated[User, Depends(get_current_user)],
service: Annotated[WorkflowService, Depends(get_workflow_service)],
limit: Annotated[int, Query(ge=1, le=100)] = 50,
offset: Annotated[int, Query(ge=0)] = 0
):
"""获取工作流执行记录列表
获取应用的工作流执行历史记录。
"""
try:
# 验证应用是否存在且属于当前工作空间
app = db.query(App).filter(
App.id == app_id,
App.workspace_id == current_user.current_workspace_id,
App.is_active.is_(True)
).first()
if not app:
return fail(
code=BizCode.NOT_FOUND,
msg="应用不存在或无权访问"
)
# 获取执行记录
executions = service.get_executions_by_app(app_id, limit, offset)
# 获取统计信息
statistics = service.get_execution_statistics(app_id)
return success(
data={
"executions": [WorkflowExecution.model_validate(e) for e in executions],
"statistics": statistics,
"pagination": {
"limit": limit,
"offset": offset,
"total": statistics["total"]
}
}
)
except Exception as e:
logger.error(f"获取工作流执行记录异常: {e}", exc_info=True)
return fail(
code=BizCode.INTERNAL_ERROR,
msg=f"获取工作流执行记录失败: {str(e)}"
)
@router.get("/workflow/executions/{execution_id}")
async def get_workflow_execution(
execution_id: Annotated[str, Path(description="执行 ID")],
db: Annotated[Session, Depends(get_db)],
current_user: Annotated[User, Depends(get_current_user)],
service: Annotated[WorkflowService, Depends(get_workflow_service)]
):
"""获取工作流执行详情
获取单个工作流执行的详细信息,包括所有节点的执行记录。
"""
try:
# 获取执行记录
execution = service.get_execution(execution_id)
if not execution:
return fail(
code=BizCode.NOT_FOUND,
msg="执行记录不存在"
)
# 验证应用是否属于当前工作空间
app = db.query(App).filter(
App.id == execution.app_id,
App.workspace_id == current_user.current_workspace_id,
App.is_active.is_(True)
).first()
if not app:
return fail(
code=BizCode.NOT_FOUND,
msg="无权访问该执行记录"
)
# 获取节点执行记录
node_executions = service.node_execution_repo.get_by_execution_id(execution.id)
return success(
data={
"execution": WorkflowExecution.model_validate(execution),
"node_executions": [
WorkflowNodeExecution.model_validate(ne) for ne in node_executions
]
}
)
except Exception as e:
logger.error(f"获取工作流执行详情异常: {e}", exc_info=True)
return fail(
code=BizCode.INTERNAL_ERROR,
msg=f"获取工作流执行详情失败: {str(e)}"
)
# ==================== 工作流执行 ====================
@router.post("/{app_id}/workflow/run")
async def run_workflow(
app_id: Annotated[uuid.UUID, Path(description="应用 ID")],
request: WorkflowExecutionRequest,
db: Annotated[Session, Depends(get_db)],
current_user: Annotated[User, Depends(get_current_user)],
service: Annotated[WorkflowService, Depends(get_workflow_service)]
):
"""执行工作流
执行工作流并返回结果。支持流式和非流式两种模式。
**非流式模式**:等待工作流执行完成后返回完整结果。
**流式模式**:实时返回执行过程中的事件(节点开始、节点完成、工作流完成等)。
"""
try:
# 验证应用是否存在且属于当前工作空间
app = db.query(App).filter(
App.id == app_id,
App.workspace_id == current_user.current_workspace_id,
App.is_active.is_(True)
).first()
if not app:
return fail(
code=BizCode.NOT_FOUND,
msg="应用不存在或无权访问"
)
# 验证应用类型
if app.type != "workflow":
return fail(
code=BizCode.INVALID_PARAMETER,
msg=f"应用类型必须为 workflow当前为 {app.type}"
)
# 准备输入数据
input_data = {
"message": request.message or "",
"variables": request.variables
}
# 执行工作流
if request.stream:
# 流式执行
from fastapi.responses import StreamingResponse
import json
async def event_generator():
"""生成 SSE 事件
SSE 格式:
event: <event_type>
data: <json_data>
支持的事件类型:
- workflow_start: 工作流开始
- workflow_end: 工作流结束
- node_start: 节点开始执行
- node_end: 节点执行完成
- node_chunk: 中间节点的流式输出
- message: 最终消息的流式输出End 节点及其相邻节点)
"""
try:
async for event in await service.run_workflow(
app_id=app_id,
input_data=input_data,
triggered_by=current_user.id,
conversation_id=uuid.UUID(request.conversation_id) if request.conversation_id else None,
stream=True
):
# 提取事件类型和数据
event_type = event.get("event", "message")
event_data = event.get("data", {})
# 转换为标准 SSE 格式(字符串)
# event: <type>
# data: <json>
sse_message = f"event: {event_type}\ndata: {json.dumps(event_data)}\n\n"
yield sse_message
except Exception as e:
logger.error(f"流式执行异常: {e}", exc_info=True)
# 发送错误事件
sse_error = f"event: error\ndata: {json.dumps({'error': str(e)})}\n\n"
yield sse_error
return StreamingResponse(
event_generator(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"X-Accel-Buffering": "no" # 禁用 nginx 缓冲
}
)
else:
# 非流式执行
result = await service.run_workflow(
app_id=app_id,
input_data=input_data,
triggered_by=current_user.id,
conversation_id=uuid.UUID(request.conversation_id) if request.conversation_id else None,
stream=False
)
return success(
data=WorkflowExecutionResponse(
execution_id=result["execution_id"],
status=result["status"],
output=result.get("output"),
output_data=result.get("output_data"),
error_message=result.get("error_message"),
elapsed_time=result.get("elapsed_time"),
token_usage=result.get("token_usage")
),
msg="工作流执行完成"
)
except BusinessException as e:
logger.warning(f"执行工作流失败: {e.message}")
return fail(code=e.error_code, msg=e.message)
except Exception as e:
logger.error(f"执行工作流异常: {e}", exc_info=True)
return fail(
code=BizCode.INTERNAL_ERROR,
msg=f"执行工作流失败: {str(e)}"
)
@router.post("/workflow/executions/{execution_id}/cancel")
async def cancel_workflow_execution(
execution_id: Annotated[str, Path(description="执行 ID")],
db: Annotated[Session, Depends(get_db)],
current_user: Annotated[User, Depends(get_current_user)],
service: Annotated[WorkflowService, Depends(get_workflow_service)]
):
"""取消工作流执行
取消正在运行的工作流执行。
**注意**:当前版本仅更新状态为 cancelled实际的执行取消功能待实现。
"""
try:
# 获取执行记录
execution = service.get_execution(execution_id)
if not execution:
return fail(
code=BizCode.NOT_FOUND,
msg="执行记录不存在"
)
# 验证应用是否属于当前工作空间
app = db.query(App).filter(
App.id == execution.app_id,
App.workspace_id == current_user.current_workspace_id,
App.is_active.is_(True)
).first()
if not app:
return fail(
code=BizCode.NOT_FOUND,
msg="无权访问该执行记录"
)
# 检查执行状态
if execution.status not in ["pending", "running"]:
return fail(
code=BizCode.INVALID_PARAMETER,
msg=f"无法取消状态为 {execution.status} 的执行"
)
# 更新状态为 cancelled
service.update_execution_status(execution_id, "cancelled")
return success(msg="工作流执行已取消")
except BusinessException as e:
logger.warning(f"取消工作流执行失败: {e.message}")
return fail(code=e.code, msg=e.message)
except Exception as e:
logger.error(f"取消工作流执行异常: {e}", exc_info=True)
return fail(
code=BizCode.INTERNAL_ERROR,
msg=f"取消工作流执行失败: {str(e)}"
)

View File

@@ -1,7 +1,7 @@
import uuid
from typing import List, Optional
from fastapi import APIRouter, Depends, Header, HTTPException, Query, status
from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy.orm import Session
from app.core.logging_config import get_api_logger
@@ -95,29 +95,16 @@ def get_workspaces(
@router.post("", response_model=ApiResponse)
def create_workspace(
workspace: WorkspaceCreate,
language_type: str = Header(default="zh", alias="X-Language-Type"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_superuser),
):
"""创建新的工作空间"""
from app.core.language_utils import get_language_from_header
# 验证并获取语言参数
language = get_language_from_header(language_type)
api_logger.info(
f"用户 {current_user.username} 请求创建工作空间: {workspace.name}, "
f"language={language}"
)
api_logger.info(f"用户 {current_user.username} 请求创建工作空间: {workspace.name}")
result = workspace_service.create_workspace(
db=db, workspace=workspace, user=current_user, language=language
)
db=db, workspace=workspace, user=current_user)
api_logger.info(
f"工作空间创建成功 - 名称: {workspace.name}, ID: {result.id}, "
f"创建者: {current_user.username}, language={language}"
)
api_logger.info(f"工作空间创建成功 - 名称: {workspace.name}, ID: {result.id}, 创建者: {current_user.username}")
result_schema = WorkspaceResponse.model_validate(result)
return success(data=result_schema, msg="工作空间创建成功")

View File

@@ -1,4 +0,0 @@
# -*- coding: UTF-8 -*-
# Author: Eternity
# @Email: 1533512157@qq.com
# @Time : 2026/2/9 16:24

View File

@@ -1,162 +0,0 @@
"""Agent Middleware - 动态技能过滤"""
import uuid
from typing import List, Dict, Any, Optional
from langchain_core.runnables import RunnablePassthrough
from app.services.skill_service import SkillService
from app.repositories.skill_repository import SkillRepository
class AgentMiddleware:
"""Agent 中间件 - 用于动态过滤和加载技能"""
def __init__(self, skills: Optional[dict] = None):
"""
初始化中间件
Args:
skills: 技能配置字典 {"enabled": bool, "all_skills": bool, "skill_ids": [...]}
"""
self.skills = skills or {}
self.enabled = self.skills.get('enabled', False)
self.all_skills = self.skills.get('all_skills', False)
self.skill_ids = self.skills.get('skill_ids', [])
@staticmethod
def filter_tools(
tools: List,
message: str = "",
skill_configs: Dict[str, Any] = None,
tool_to_skill_map: Dict[str, str] = None
) -> tuple[List, List[str]]:
"""
根据消息内容和技能配置动态过滤工具
Args:
tools: 所有可用工具列表
message: 用户消息(可用于智能过滤)
skill_configs: 技能配置字典 {skill_id: {"keywords": [...], "enabled": True, "prompt": "..."}}
tool_to_skill_map: 工具到技能的映射 {tool_name: skill_id}
Returns:
(过滤后的工具列表, 激活的技能ID列表)
"""
if not tools:
return [], []
# 如果没有技能配置,返回所有工具
if not skill_configs:
return tools, []
# 基于关键词匹配激活技能
activated_skill_ids = []
message_lower = message.lower()
for skill_id, config in skill_configs.items():
if not config.get('enabled', True):
continue
keywords = config.get('keywords', [])
# 如果没有关键词限制,或消息包含关键词,则激活该技能
if not keywords or any(kw.lower() in message_lower for kw in keywords):
activated_skill_ids.append(skill_id)
# 如果没有工具映射关系,返回所有工具
if not tool_to_skill_map:
return tools, activated_skill_ids
# 根据激活的技能过滤工具
filtered_tools = []
for tool in tools:
tool_name = getattr(tool, 'name', str(id(tool)))
# 如果工具不属于任何skillbase_tools或者工具所属的skill被激活则保留
if tool_name not in tool_to_skill_map or tool_to_skill_map[tool_name] in activated_skill_ids:
filtered_tools.append(tool)
return filtered_tools, activated_skill_ids
def load_skill_tools(self, db, tenant_id: uuid.UUID, base_tools: List = None) -> tuple[List, Dict[str, Any], Dict[str, str]]:
"""
加载技能关联的工具
Args:
db: 数据库会话
tenant_id: 租户id
base_tools: 基础工具列表
Returns:
(工具列表, 技能配置字典, 工具到技能的映射 {tool_name: skill_id})
"""
tools_dict = {}
tool_to_skill_map = {} # 工具名称到技能ID的映射
if base_tools:
for tool in base_tools:
tool_name = getattr(tool, 'name', str(id(tool)))
tools_dict[tool_name] = tool
# base_tools 不属于任何 skill不加入映射
skill_configs = {}
skill_ids_to_load = []
# 如果启用技能且 all_skills 为 True加载租户下所有激活的技能
if self.enabled and self.all_skills:
skills, _ = SkillRepository.list_skills(db, tenant_id, is_active=True, page=1, pagesize=1000)
skill_ids_to_load = [str(skill.id) for skill in skills]
elif self.enabled and self.skill_ids:
skill_ids_to_load = self.skill_ids
if skill_ids_to_load:
for skill_id in skill_ids_to_load:
try:
skill = SkillRepository.get_by_id(db, uuid.UUID(skill_id), tenant_id)
if skill and skill.is_active:
# 保存技能配置包含prompt
config = skill.config or {}
config['prompt'] = skill.prompt
config['name'] = skill.name
skill_configs[skill_id] = config
except Exception:
continue
# 加载技能工具并获取映射关系
skill_tools, skill_tool_map = SkillService.load_skill_tools(db, skill_ids_to_load, tenant_id)
# 只添加不冲突的 skill_tools
for tool in skill_tools:
tool_name = getattr(tool, 'name', str(id(tool)))
if tool_name not in tools_dict:
tools_dict[tool_name] = tool
# 复制映射关系
if tool_name in skill_tool_map:
tool_to_skill_map[tool_name] = skill_tool_map[tool_name]
return list(tools_dict.values()), skill_configs, tool_to_skill_map
@staticmethod
def get_active_prompts(activated_skill_ids: List[str], skill_configs: Dict[str, Any]) -> str:
"""
根据激活的技能ID获取对应的提示词
Args:
activated_skill_ids: 被激活的技能ID列表
skill_configs: 技能配置字典
Returns:
合并后的提示词
"""
prompts = []
for skill_id in activated_skill_ids:
config = skill_configs.get(skill_id, {})
prompt = config.get('prompt')
name = config.get('name', 'Skill')
if prompt:
prompts.append(f"# {name}\n{prompt}")
return "\n\n".join(prompts) if prompts else ""
@staticmethod
def create_runnable():
"""创建可运行的中间件"""
return RunnablePassthrough()

View File

@@ -11,37 +11,33 @@ LangChain Agent 封装
import time
from typing import Any, AsyncGenerator, Dict, List, Optional, Sequence
from app.core.memory.agent.langgraph_graph.write_graph import write_long_term
from app.core.memory.agent.langgraph_graph.write_graph import write_long_term
from app.db import get_db
from app.core.logging_config import get_business_logger
from app.core.models import RedBearLLM, RedBearModelConfig
from app.models.models_model import ModelType, ModelProvider
from app.models.models_model import ModelType
from app.services.memory_agent_service import (
get_end_user_connected_config,
)
from langchain.agents import create_agent
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
from langchain_core.tools import BaseTool
logger = get_business_logger()
class LangChainAgent:
def __init__(
self,
model_name: str,
api_key: str,
provider: str = "openai",
api_base: Optional[str] = None,
is_omni: bool = False,
temperature: float = 0.7,
max_tokens: int = 2000,
system_prompt: Optional[str] = None,
tools: Optional[Sequence[BaseTool]] = None,
streaming: bool = False,
max_iterations: Optional[int] = None, # 最大迭代次数None 表示自动计算)
max_tool_consecutive_calls: int = 3 # 单个工具最大连续调用次数
self,
model_name: str,
api_key: str,
provider: str = "openai",
api_base: Optional[str] = None,
temperature: float = 0.7,
max_tokens: int = 2000,
system_prompt: Optional[str] = None,
tools: Optional[Sequence[BaseTool]] = None,
streaming: bool = False
):
"""初始化 LangChain Agent
@@ -54,37 +50,13 @@ class LangChainAgent:
max_tokens: 最大 token 数
system_prompt: 系统提示词
tools: 工具列表(可选,框架自动走 ReAct 循环)
streaming: 是否启用流式输出
max_iterations: 最大迭代次数None 表示自动计算:基础 5 次 + 每个工具 2 次)
max_tool_consecutive_calls: 单个工具最大连续调用次数(默认 3 次)
streaming: 是否启用流式输出(默认 True
"""
self.model_name = model_name
self.provider = provider
self.system_prompt = system_prompt or "你是一个专业的AI助手"
self.tools = tools or []
self.streaming = streaming
self.is_omni = is_omni
self.max_tool_consecutive_calls = max_tool_consecutive_calls
# 工具调用计数器:记录每个工具的连续调用次数
self.tool_call_counter: Dict[str, int] = {}
self.last_tool_called: Optional[str] = None
# 根据工具数量动态调整最大迭代次数
# 基础值 + 每个工具额外的调用机会
if max_iterations is None:
# 自动计算:基础 5 次 + 每个工具 2 次额外机会
self.max_iterations = 5 + len(self.tools) * 2
else:
self.max_iterations = max_iterations
self.system_prompt = system_prompt or "你是一个专业的AI助手"
logger.debug(
f"Agent 迭代次数配置: max_iterations={self.max_iterations}, "
f"tool_count={len(self.tools)}, "
f"max_tool_consecutive_calls={self.max_tool_consecutive_calls}, "
f"auto_calculated={max_iterations is None}"
)
# 创建 RedBearLLM支持多提供商
model_config = RedBearModelConfig(
@@ -92,7 +64,6 @@ class LangChainAgent:
provider=provider,
api_key=api_key,
base_url=api_base,
is_omni=is_omni,
extra_params={
"temperature": temperature,
"max_tokens": max_tokens,
@@ -109,14 +80,11 @@ class LangChainAgent:
if streaming and hasattr(self._underlying_llm, 'streaming'):
self._underlying_llm.streaming = True
# 包装工具以跟踪连续调用次数
wrapped_tools = self._wrap_tools_with_tracking(self.tools) if self.tools else None
# 使用 create_agent 创建 agent graphLangChain 1.x 标准方式)
# 无论是否有工具,都使用 agent 统一处理
self.agent = create_agent(
model=self.llm,
tools=wrapped_tools,
tools=self.tools if self.tools else None,
system_prompt=self.system_prompt
)
@@ -128,92 +96,17 @@ class LangChainAgent:
"has_api_base": bool(api_base),
"temperature": temperature,
"streaming": streaming,
"max_iterations": self.max_iterations,
"max_tool_consecutive_calls": self.max_tool_consecutive_calls,
"tool_count": len(self.tools),
"tool_names": [tool.name for tool in self.tools] if self.tools else [],
# "tool_count": len(self.tools)
}
)
def _wrap_tools_with_tracking(self, tools: Sequence[BaseTool]) -> List[BaseTool]:
"""包装工具以跟踪连续调用次数
Args:
tools: 原始工具列表
Returns:
List[BaseTool]: 包装后的工具列表
"""
from langchain_core.tools import StructuredTool
from functools import wraps
wrapped_tools = []
for original_tool in tools:
tool_name = original_tool.name
original_func = original_tool.func if hasattr(original_tool, 'func') else None
if not original_func:
# 如果无法获取原始函数,直接使用原工具
wrapped_tools.append(original_tool)
continue
# 创建包装函数
def make_wrapped_func(tool_name, original_func):
"""创建包装函数的工厂函数,避免闭包问题"""
@wraps(original_func)
def wrapped_func(*args, **kwargs):
"""包装后的工具函数,跟踪连续调用次数"""
# 检查是否是连续调用同一个工具
if self.last_tool_called == tool_name:
self.tool_call_counter[tool_name] = self.tool_call_counter.get(tool_name, 0) + 1
else:
# 切换到新工具,重置计数器
self.tool_call_counter[tool_name] = 1
self.last_tool_called = tool_name
current_count = self.tool_call_counter[tool_name]
logger.debug(
f"工具调用: {tool_name}, 连续调用次数: {current_count}/{self.max_tool_consecutive_calls}"
)
# 检查是否超过最大连续调用次数
if current_count > self.max_tool_consecutive_calls:
logger.warning(
f"工具 '{tool_name}' 连续调用次数已达上限 ({self.max_tool_consecutive_calls})"
f"返回提示信息"
)
return (
f"工具 '{tool_name}' 已连续调用 {self.max_tool_consecutive_calls} 次,"
f"未找到有效结果。请尝试其他方法或直接回答用户的问题。"
)
# 调用原始工具函数
return original_func(*args, **kwargs)
return wrapped_func
# 使用 StructuredTool 创建新工具
wrapped_tool = StructuredTool(
name=original_tool.name,
description=original_tool.description,
func=make_wrapped_func(tool_name, original_func),
args_schema=original_tool.args_schema if hasattr(original_tool, 'args_schema') else None
)
wrapped_tools.append(wrapped_tool)
return wrapped_tools
def _prepare_messages(
self,
message: str,
history: Optional[List[Dict[str, str]]] = None,
context: Optional[str] = None,
files: Optional[List[Dict[str, Any]]] = None
self,
message: str,
history: Optional[List[Dict[str, str]]] = None,
context: Optional[str] = None
) -> List[BaseMessage]:
"""准备消息列表
@@ -221,7 +114,6 @@ class LangChainAgent:
message: 用户消息
history: 历史消息列表
context: 上下文信息
files: 多模态文件内容列表(已处理)
Returns:
List[BaseMessage]: 消息列表
@@ -244,50 +136,9 @@ class LangChainAgent:
if context:
user_content = f"参考信息:\n{context}\n\n用户问题:\n{user_content}"
# 构建用户消息(支持多模态)
if files and len(files) > 0:
content_parts = self._build_multimodal_content(user_content, files)
messages.append(HumanMessage(content=content_parts))
else:
# 纯文本消息
messages.append(HumanMessage(content=user_content))
messages.append(HumanMessage(content=user_content))
return messages
def _build_multimodal_content(self, text: str, files: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
构建多模态消息内容
Args:
text: 文本内容
files: 文件列表(已由 MultimodalService 处理为对应 provider 的格式)
Returns:
List[Dict]: 消息内容列表
"""
# 根据 provider 使用不同的文本格式
# if (self.provider.lower() in [ModelProvider.BEDROCK, ModelProvider.OPENAI, ModelProvider.XINFERENCE,
# ModelProvider.GPUSTACK] or (
# self.provider.lower() == ModelProvider.DASHSCOPE and self.is_omni)):
# # Anthropic/Bedrock/Xinference/Gpustack/Openai: {"type": "text", "text": "..."}
# content_parts = [{"type": "text", "text": text}]
# else:
# # 通义千问等: {"text": "..."}
# content_parts = [{"type": "text", "text": text}]
content_parts = [{"type": "text", "text": text}]
# 添加文件内容
# MultimodalService 已经根据 provider 返回了正确格式,直接使用
content_parts.extend(files)
logger.debug(
f"构建多模态消息: provider={self.provider}, "
f"parts={len(content_parts)}, "
f"files={len(files)}"
)
return content_parts
async def chat(
self,
message: str,
@@ -297,8 +148,7 @@ class LangChainAgent:
config_id: Optional[str] = None, # 添加这个参数
storage_type: Optional[str] = None,
user_rag_memory_id: Optional[str] = None,
memory_flag: Optional[bool] = True,
files: Optional[List[Dict[str, Any]]] = None # 新增:多模态文件
memory_flag: Optional[bool] = True
) -> Dict[str, Any]:
"""执行对话
@@ -310,7 +160,7 @@ class LangChainAgent:
Returns:
Dict: 包含 content 和元数据的字典
"""
message_chat = message
message_chat= message
start_time = time.time()
actual_config_id = config_id
# If config_id is None, try to get from end_user's connected config
@@ -330,11 +180,11 @@ class LangChainAgent:
except Exception as e:
logger.warning(f"Failed to get db session: {e}")
actual_end_user_id = end_user_id if end_user_id is not None else "unknown"
logger.info(f'写入类型{storage_type, str(end_user_id), message, str(user_rag_memory_id)}')
print(f'写入类型{storage_type, str(end_user_id), message, str(user_rag_memory_id)}')
logger.info(f'写入类型{storage_type,str(end_user_id), message, str(user_rag_memory_id)}')
print(f'写入类型{storage_type,str(end_user_id), message, str(user_rag_memory_id)}')
try:
# 准备消息列表(支持多模态)
messages = self._prepare_messages(message, history, context, files)
# 准备消息列表
messages = self._prepare_messages(message, history, context)
logger.debug(
"准备调用 LangChain Agent",
@@ -342,86 +192,27 @@ class LangChainAgent:
"has_context": bool(context),
"has_history": bool(history),
"has_tools": bool(self.tools),
"has_files": bool(files),
"message_count": len(messages),
"max_iterations": self.max_iterations
"message_count": len(messages)
}
)
# 统一使用 agent.invoke 调用
# 通过 recursion_limit 限制最大迭代次数,防止工具调用死循环
try:
result = await self.agent.ainvoke(
{"messages": messages},
config={"recursion_limit": self.max_iterations}
)
except RecursionError as e:
logger.warning(
f"Agent 达到最大迭代次数限制 ({self.max_iterations}),可能存在工具调用循环",
extra={"error": str(e)}
)
# 返回一个友好的错误提示
return {
"content": f"抱歉,我在处理您的请求时遇到了问题。已达到最大处理步骤限制({self.max_iterations}次)。请尝试简化您的问题或稍后再试。",
"model": self.model_name,
"elapsed_time": time.time() - start_time,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
}
}
result = await self.agent.ainvoke({"messages": messages})
# 获取最后的 AI 消息
output_messages = result.get("messages", [])
content = ""
logger.debug(f"输出消息数量: {len(output_messages)}")
total_tokens = 0
for msg in reversed(output_messages):
if isinstance(msg, AIMessage):
logger.debug(f"找到 AI 消息content 类型: {type(msg.content)}")
logger.debug(f"AI 消息内容: {msg.content}")
# 处理多模态响应content 可能是字符串或列表
if isinstance(msg.content, str):
content = msg.content
logger.debug(f"提取字符串内容,长度: {len(content)}")
elif isinstance(msg.content, list):
# 多模态响应:提取文本部分
logger.debug(f"多模态响应,列表长度: {len(msg.content)}")
text_parts = []
for item in msg.content:
logger.debug(f"处理项: {item}")
if isinstance(item, dict):
# 通义千问格式: {"text": "..."}
if "text" in item:
text = item.get("text", "")
text_parts.append(text)
logger.debug(f"提取文本: {text[:100]}...")
# OpenAI 格式: {"type": "text", "text": "..."}
elif item.get("type") == "text":
text = item.get("text", "")
text_parts.append(text)
logger.debug(f"提取文本: {text[:100]}...")
elif isinstance(item, str):
text_parts.append(item)
logger.debug(f"提取字符串: {item[:100]}...")
content = "".join(text_parts)
logger.debug(f"合并后内容长度: {len(content)}")
else:
content = str(msg.content)
logger.debug(f"转换为字符串: {content[:100]}...")
content = msg.content
response_meta = msg.response_metadata if hasattr(msg, 'response_metadata') else None
total_tokens = response_meta.get("token_usage", {}).get("total_tokens", 0) if response_meta else 0
break
logger.info(f"最终提取的内容长度: {len(content)}")
elapsed_time = time.time() - start_time
if memory_flag:
await write_long_term(storage_type, end_user_id, message_chat, content, user_rag_memory_id,
actual_config_id)
await write_long_term(storage_type, end_user_id, message_chat, content, user_rag_memory_id, actual_config_id)
response = {
"content": content,
"model": self.model_name,
@@ -448,16 +239,15 @@ class LangChainAgent:
raise
async def chat_stream(
self,
message: str,
history: Optional[List[Dict[str, str]]] = None,
context: Optional[str] = None,
end_user_id: Optional[str] = None,
config_id: Optional[str] = None,
storage_type: Optional[str] = None,
user_rag_memory_id: Optional[str] = None,
memory_flag: Optional[bool] = True,
files: Optional[List[Dict[str, Any]]] = None # 新增:多模态文件
self,
message: str,
history: Optional[List[Dict[str, str]]] = None,
context: Optional[str] = None,
end_user_id:Optional[str] = None,
config_id: Optional[str] = None,
storage_type:Optional[str] = None,
user_rag_memory_id:Optional[str] = None,
memory_flag: Optional[bool] = True
) -> AsyncGenerator[str, None]:
"""执行流式对话
@@ -491,13 +281,14 @@ class LangChainAgent:
except Exception as e:
logger.warning(f"Failed to get db session: {e}")
# 注意:不在这里写入用户消息,等 AI 回复后一起写入
try:
# 准备消息列表(支持多模态)
messages = self._prepare_messages(message, history, context, files)
# 准备消息列表
messages = self._prepare_messages(message, history, context)
logger.debug(
f"准备流式调用has_tools={bool(self.tools)}, has_files={bool(files)}, message_count={len(messages)}"
f"准备流式调用has_tools={bool(self.tools)}, message_count={len(messages)}"
)
chunk_count = 0
@@ -508,90 +299,39 @@ class LangChainAgent:
full_content = ''
try:
async for event in self.agent.astream_events(
{"messages": messages},
version="v2",
config={"recursion_limit": self.max_iterations}
{"messages": messages},
version="v2"
):
chunk_count += 1
kind = event.get("event")
# 处理所有可能的流式事件
if kind == "on_chat_model_stream":
# LLM 流式输出
chunk = event.get("data", {}).get("chunk")
if chunk and hasattr(chunk, "content"):
# 处理多模态响应content 可能是字符串或列表
chunk_content = chunk.content
if isinstance(chunk_content, str) and chunk_content:
full_content += chunk_content
yield chunk_content
yielded_content = True
elif isinstance(chunk_content, list):
# 多模态响应:提取文本部分
for item in chunk_content:
if isinstance(item, dict):
# 通义千问格式: {"text": "..."}
if "text" in item:
text = item.get("text", "")
if text:
full_content += text
yield text
yielded_content = True
# OpenAI 格式: {"type": "text", "text": "..."}
elif item.get("type") == "text":
text = item.get("text", "")
if text:
full_content += text
yield text
yielded_content = True
elif isinstance(item, str):
full_content += item
yield item
yielded_content = True
full_content+=chunk.content
if chunk and hasattr(chunk, "content") and chunk.content:
yield chunk.content
yielded_content = True
elif kind == "on_llm_stream":
# 另一种 LLM 流式事件
chunk = event.get("data", {}).get("chunk")
if chunk:
if hasattr(chunk, "content"):
chunk_content = chunk.content
if isinstance(chunk_content, str) and chunk_content:
full_content += chunk_content
yield chunk_content
yielded_content = True
elif isinstance(chunk_content, list):
# 多模态响应:提取文本部分
for item in chunk_content:
if isinstance(item, dict):
# 通义千问格式: {"text": "..."}
if "text" in item:
text = item.get("text", "")
if text:
full_content += text
yield text
yielded_content = True
# OpenAI 格式: {"type": "text", "text": "..."}
elif item.get("type") == "text":
text = item.get("text", "")
if text:
full_content += text
yield text
yielded_content = True
elif isinstance(item, str):
full_content += item
yield item
yielded_content = True
if hasattr(chunk, "content") and chunk.content:
full_content+=chunk.content
yield chunk.content
yielded_content = True
elif isinstance(chunk, str):
full_content += chunk
yield chunk
yielded_content = True
# 记录工具调用(可选)
elif kind == "on_tool_start":
logger.debug(f"工具调用开始: {event.get('name')}")
elif kind == "on_tool_end":
logger.debug(f"工具调用结束: {event.get('name')}")
logger.debug(f"Agent 流式完成,共 {chunk_count} 个事件")
# 统计token消耗
output_messages = event.get("data", {}).get("output", {}).get("messages", [])
@@ -603,8 +343,7 @@ class LangChainAgent:
yield total_tokens
break
if memory_flag:
await write_long_term(storage_type, end_user_id, message_chat, full_content, user_rag_memory_id,
actual_config_id)
await write_long_term(storage_type, end_user_id, message_chat, full_content, user_rag_memory_id, actual_config_id)
except Exception as e:
logger.error(f"Agent astream_events 失败: {str(e)}", exc_info=True)
raise
@@ -618,3 +357,5 @@ class LangChainAgent:
logger.info("=" * 80)
logger.info("chat_stream 方法执行结束")
logger.info("=" * 80)

View File

@@ -1,9 +1,9 @@
import json
import os
from pathlib import Path
from typing import Annotated, Optional
from typing import Any, Dict, Optional
from dotenv import load_dotenv
from pydantic import Field, TypeAdapter
load_dotenv()
@@ -16,18 +16,18 @@ class Settings:
# cloud: SaaS 云服务版(全功能,按量计费)
# enterprise: 企业私有化版License 控制)
DEPLOYMENT_MODE: str = os.getenv("DEPLOYMENT_MODE", "community")
# License 配置(企业版)
LICENSE_FILE: str = os.getenv("LICENSE_FILE", "/etc/app/license.json")
LICENSE_SERVER_URL: str = os.getenv("LICENSE_SERVER_URL", "https://license.yourcompany.com")
# 计费服务配置SaaS 版)
BILLING_SERVICE_URL: str = os.getenv("BILLING_SERVICE_URL", "")
# 基础 URL用于 SSO 回调等)
BASE_URL: str = os.getenv("BASE_URL", "http://localhost:8000")
FRONTEND_URL: str = os.getenv("FRONTEND_URL", "http://localhost:3000")
ENABLE_SINGLE_WORKSPACE: bool = os.getenv("ENABLE_SINGLE_WORKSPACE", "true").lower() == "true"
# API Keys Configuration
OPENAI_API_KEY: str = os.getenv("OPENAI_API_KEY", "")
@@ -57,6 +57,7 @@ class Settings:
REDIS_PORT: int = int(os.getenv("REDIS_PORT", "6379"))
REDIS_DB: int = int(os.getenv("REDIS_DB", "1"))
REDIS_PASSWORD: str = os.getenv("REDIS_PASSWORD", "")
# ElasticSearch configuration
ELASTICSEARCH_HOST: str = os.getenv("ELASTICSEARCH_HOST", "https://127.0.0.1")
@@ -90,7 +91,7 @@ class Settings:
# Single Sign-On configuration
ENABLE_SINGLE_SESSION: bool = os.getenv("ENABLE_SINGLE_SESSION", "false").lower() == "true"
# SSO 免登配置
SSO_TOKEN_EXPIRE_SECONDS: int = int(os.getenv("SSO_TOKEN_EXPIRE_SECONDS", "300"))
SSO_TRUSTED_SOURCES_CONFIG: str = os.getenv("SSO_TRUSTED_SOURCES_CONFIG", "{}")
@@ -114,7 +115,6 @@ class Settings:
S3_ACCESS_KEY_ID: str = os.getenv("S3_ACCESS_KEY_ID", "")
S3_SECRET_ACCESS_KEY: str = os.getenv("S3_SECRET_ACCESS_KEY", "")
S3_BUCKET_NAME: str = os.getenv("S3_BUCKET_NAME", "")
S3_ENDPOINT_URL: str = os.getenv("S3_ENDPOINT_URL", "")
# VOLC ASR settings
VOLC_APP_KEY: str = os.getenv("VOLC_APP_KEY", "")
@@ -130,7 +130,7 @@ class Settings:
# Server Configuration
SERVER_IP: str = os.getenv("SERVER_IP", "127.0.0.1")
FILE_LOCAL_SERVER_URL: str = os.getenv("FILE_LOCAL_SERVER_URL", "http://localhost:8000/api")
FILE_LOCAL_SERVER_URL : str = os.getenv("FILE_LOCAL_SERVER_URL", "http://localhost:8000/api")
# ========================================================================
# Internal Configuration (not in .env, used by application code)
@@ -190,45 +190,19 @@ class Settings:
LOG_FILE_MAX_SIZE_MB: int = int(os.getenv("LOG_FILE_MAX_SIZE_MB", "10")) # 10MB
# Celery configuration (internal)
# NOTE: 变量名不以 CELERY_ 开头,避免被 Celery CLI 的前缀匹配机制劫持
# 详见 docs/celery-env-bug-report.md
# 默认使用 Redis DB 3 (broker) 和 DB 4 (backend),与业务缓存 (DB 1/2) 隔离
# 多人共用同一 Redis 时,每位开发者应在 .env 中配置不同的 DB 编号避免任务互相干扰
REDIS_DB_CELERY_BROKER: int = int(os.getenv("REDIS_DB_CELERY_BROKER", "3"))
REDIS_DB_CELERY_BACKEND: int = int(os.getenv("REDIS_DB_CELERY_BACKEND", "4"))
# SMTP Email Configuration
SMTP_SERVER: str = os.getenv("SMTP_SERVER", "smtp.gmail.com")
SMTP_PORT: int = int(os.getenv("SMTP_PORT", "587"))
SMTP_USER: str = os.getenv("SMTP_USER", "")
SMTP_PASSWORD: str = os.getenv("SMTP_PASSWORD", "")
CELERY_BROKER: int = int(os.getenv("CELERY_BROKER", "1"))
CELERY_BACKEND: int = int(os.getenv("CELERY_BACKEND", "2"))
REFLECTION_INTERVAL_SECONDS: float = float(os.getenv("REFLECTION_INTERVAL_SECONDS", "300"))
HEALTH_CHECK_SECONDS: float = float(os.getenv("HEALTH_CHECK_SECONDS", "600"))
MEMORY_INCREMENT_INTERVAL_HOURS: float = float(os.getenv("MEMORY_INCREMENT_INTERVAL_HOURS", "24"))
DEFAULT_WORKSPACE_ID: Optional[str] = os.getenv("DEFAULT_WORKSPACE_ID", None)
REFLECTION_INTERVAL_TIME: Optional[str] = int(os.getenv("REFLECTION_INTERVAL_TIME", 30))
# Memory Cache Regeneration Configuration
MEMORY_CACHE_REGENERATION_HOURS: int = int(os.getenv("MEMORY_CACHE_REGENERATION_HOURS", "24"))
# Celery Beat Schedule Configuration (定时任务执行频率)
MEMORY_INCREMENT_HOUR: int = TypeAdapter(
Annotated[int, Field(ge=0, le=23, description="cron hour [0, 23]")]
).validate_python(int(os.getenv("MEMORY_INCREMENT_HOUR", "2")))
MEMORY_INCREMENT_MINUTE: int = TypeAdapter(
Annotated[int, Field(ge=0, le=59, description="cron minute [0, 59]")]
).validate_python(int(os.getenv("MEMORY_INCREMENT_MINUTE", "0")))
WORKSPACE_REFLECTION_INTERVAL_SECONDS: int = TypeAdapter(
Annotated[int, Field(ge=1, description="reflection interval in seconds, must be >= 1")]
).validate_python(int(os.getenv("WORKSPACE_REFLECTION_INTERVAL_SECONDS", "30")))
FORGETTING_CYCLE_INTERVAL_HOURS: int = TypeAdapter(
Annotated[int, Field(ge=1, description="forgetting cycle interval in hours, must be >= 1")]
).validate_python(int(os.getenv("FORGETTING_CYCLE_INTERVAL_HOURS", "24")))
IMPLICIT_EMOTIONS_UPDATE_HOUR: int = int(os.getenv("IMPLICIT_EMOTIONS_UPDATE_HOUR", "2"))
# implicit_emotions_update: 每天几分执行分钟0-59
IMPLICIT_EMOTIONS_UPDATE_MINUTE: int = int(os.getenv("IMPLICIT_EMOTIONS_UPDATE_MINUTE", "0"))
# Memory Module Configuration (internal)
MEMORY_OUTPUT_DIR: str = os.getenv("MEMORY_OUTPUT_DIR", "logs/memory-output")
MEMORY_CONFIG_DIR: str = os.getenv("MEMORY_CONFIG_DIR", "app/core/memory")
@@ -241,35 +215,9 @@ class Settings:
# official environment system version
SYSTEM_VERSION: str = os.getenv("SYSTEM_VERSION", "v0.2.1")
# model square loading
LOAD_MODEL: bool = os.getenv("LOAD_MODEL", "false").lower() == "true"
# workflow config
WORKFLOW_IMPORT_CACHE_TIMEOUT: int = int(os.getenv("WORKFLOW_IMPORT_CACHE_TIMEOUT", 1800))
WORKFLOW_NODE_TIMEOUT: int = int(os.getenv("WORKFLOW_NODE_TIMEOUT", 600))
# ========================================================================
# General Ontology Type Configuration
# ========================================================================
# 通用本体文件路径列表(逗号分隔)
GENERAL_ONTOLOGY_FILES: str = os.getenv("GENERAL_ONTOLOGY_FILES", "api/app/core/memory/ontology_services/General_purpose_entity.ttl")
# 是否启用通用本体类型功能
ENABLE_GENERAL_ONTOLOGY_TYPES: bool = os.getenv("ENABLE_GENERAL_ONTOLOGY_TYPES", "true").lower() == "true"
# Prompt 中最大类型数量
MAX_ONTOLOGY_TYPES_IN_PROMPT: int = int(os.getenv("MAX_ONTOLOGY_TYPES_IN_PROMPT", "50"))
# 核心通用类型列表(逗号分隔)
CORE_GENERAL_TYPES: str = os.getenv(
"CORE_GENERAL_TYPES",
"Person,Organization,Company,GovernmentAgency,Place,Location,City,Country,Building,"
"Event,SportsEvent,SocialEvent,Work,Book,Film,Software,Concept,TopicalConcept,AcademicSubject"
)
# 实验模式开关(允许通过 API 动态切换本体配置)
ONTOLOGY_EXPERIMENT_MODE: bool = os.getenv("ONTOLOGY_EXPERIMENT_MODE", "true").lower() == "true"
def get_memory_output_path(self, filename: str = "") -> str:
"""
Get the full path for memory module output files.

View File

@@ -46,7 +46,6 @@ class BizCode(IntEnum):
RESOURCE_ALREADY_EXISTS = 5002
VERSION_ALREADY_EXISTS = 5003
STATE_CONFLICT = 5004
RESOURCE_IN_USE = 5005
# 应用发布6xxx
PUBLISH_FAILED = 6001
@@ -126,7 +125,6 @@ HTTP_MAPPING = {
BizCode.RESOURCE_ALREADY_EXISTS: 409,
BizCode.VERSION_ALREADY_EXISTS: 409,
BizCode.STATE_CONFLICT: 409,
BizCode.RESOURCE_IN_USE: 409,
BizCode.PUBLISH_FAILED: 500,
BizCode.NO_DRAFT_TO_PUBLISH: 400,
BizCode.ROLLBACK_TARGET_NOT_FOUND: 400,

View File

@@ -1,82 +0,0 @@
# -*- coding: utf-8 -*-
"""语言处理工具模块
本模块提供集中化的语言校验和处理功能,确保整个应用中语言参数的一致性。
Functions:
validate_language: 校验语言参数,确保其为有效值
get_language_from_header: 从请求头获取并校验语言参数
"""
from typing import Optional
from app.core.logging_config import get_logger
logger = get_logger(__name__)
# 支持的语言列表
SUPPORTED_LANGUAGES = {"zh", "en"}
# 默认回退语言
DEFAULT_LANGUAGE = "zh"
def validate_language(language: Optional[str]) -> str:
"""
校验语言参数,确保其为有效值。
Args:
language: 待校验的语言代码,可以是 None、"zh""en" 或其他值
Returns:
有效的语言代码("zh""en"
Examples:
>>> validate_language("zh")
'zh'
>>> validate_language("en")
'en'
>>> validate_language("EN") # 大小写不敏感
'en'
>>> validate_language(None) # None 回退到默认值
'zh'
>>> validate_language("fr") # 不支持的语言回退到默认值
'zh'
"""
if language is None:
return DEFAULT_LANGUAGE
# 标准化:转小写并去除空白
lang = str(language).lower().strip()
if lang in SUPPORTED_LANGUAGES:
return lang
logger.warning(
f"无效的语言参数 '{language}',已回退到默认值 '{DEFAULT_LANGUAGE}'"
f"支持的语言: {SUPPORTED_LANGUAGES}"
)
return DEFAULT_LANGUAGE
def get_language_from_header(language_type: Optional[str]) -> str:
"""
从请求头获取并校验语言参数。
这是一个便捷函数,用于在 controller 层统一处理 X-Language-Type Header。
Args:
language_type: 从 X-Language-Type Header 获取的语言值
Returns:
有效的语言代码("zh""en"
Examples:
>>> get_language_from_header(None) # Header 未传递
'zh'
>>> get_language_from_header("en")
'en'
>>> get_language_from_header("invalid") # 无效值回退
'zh'
"""
return validate_language(language_type)

View File

@@ -38,56 +38,6 @@ class SensitiveDataLoggingFilter(logging.Filter):
return True
class Neo4jSuccessNotificationFilter(logging.Filter):
"""Neo4j 日志过滤器:过滤成功/信息性状态的通知,保留真正的警告和错误
Neo4j 驱动会以 WARNING 级别记录所有数据库通知,包括成功的操作。
这个过滤器会过滤掉以下 GQL 状态码的通知,只保留真正的警告和错误:
- 00000: 成功完成 (successful completion)
- 00N00: 无数据 (no data)
- 00NA0: 无数据,信息性通知 (no data, informational notification)
使用正则表达式进行更严格的匹配,避免误过滤无关的警告。
"""
import re
# 编译正则表达式以提高性能
# 匹配所有"成功/信息性"的 GQL 状态码:
# 00000 = 成功完成, 00N00 = 无数据, 00NA0 = 无数据信息性通知
GQL_STATUS_PATTERN = re.compile(r"gql_status=['\"](00000|00N00|00NA0)['\"]")
# 匹配 status_description 中的成功完成或信息性通知消息
SUCCESS_DESC_PATTERN = re.compile(r"status_description=['\"]note:\s*(successful\s+completion|no\s+data)['\"]", re.IGNORECASE)
def filter(self, record: logging.LogRecord) -> bool:
"""
过滤 Neo4j 成功通知
Args:
record: 日志记录
Returns:
True表示允许记录False表示拒绝过滤掉
"""
# 只处理 INFO 和 WARNING 级别的日志
# Neo4j 驱动对 severity='INFORMATION' 的通知使用 INFO 级别,
# 对 severity='WARNING' 的通知使用 WARNING 级别
if record.levelno not in (logging.INFO, logging.WARNING):
return True
# 检查是否是 Neo4j 的成功通知
message = str(record.msg)
# 使用正则表达式进行更严格的匹配
# 这样可以避免误过滤包含这些子字符串但不是 Neo4j 通知的日志
if self.GQL_STATUS_PATTERN.search(message) or self.SUCCESS_DESC_PATTERN.search(message):
return False # 过滤掉这条日志
# 保留其他所有日志(包括真正的警告和错误)
return True
class LoggingConfig:
"""全局日志配置类"""
@@ -115,22 +65,6 @@ class LoggingConfig:
# 清除现有处理器
root_logger.handlers.clear()
# Neo4j 通知过滤器 - 挂在 handler 上确保所有传播上来的日志都能被过滤
neo4j_filter = Neo4jSuccessNotificationFilter()
# 抑制 Neo4j 通知日志
# Neo4j 驱动内部会给 neo4j.notifications logger 配置自己的 handler
# 导致日志绕过根 logger 的 filter 直接输出。
# 多管齐下确保过滤生效:
# 1. 设置 neo4j.notifications 级别为 WARNING过滤 INFO 级别的 00NA0 通知)
# 2. 在所有 neo4j logger 上添加 filter过滤 WARNING 级别的成功通知)
# 3. 在根 handler 上也添加 filter兜底
neo4j_notifications_logger = logging.getLogger("neo4j.notifications")
neo4j_notifications_logger.setLevel(logging.WARNING)
for neo4j_logger_name in ["neo4j", "neo4j.io", "neo4j.pool", "neo4j.notifications"]:
neo4j_logger = logging.getLogger(neo4j_logger_name)
neo4j_logger.addFilter(neo4j_filter)
# 创建格式化器
formatter = logging.Formatter(
fmt=settings.LOG_FORMAT,
@@ -146,7 +80,6 @@ class LoggingConfig:
console_handler.setFormatter(formatter)
console_handler.setLevel(getattr(logging, settings.LOG_LEVEL.upper()))
console_handler.addFilter(sensitive_filter)
console_handler.addFilter(neo4j_filter)
root_logger.addHandler(console_handler)
# 文件处理器(带轮转)
@@ -160,7 +93,6 @@ class LoggingConfig:
file_handler.setFormatter(formatter)
file_handler.setLevel(getattr(logging, settings.LOG_LEVEL.upper()))
file_handler.addFilter(sensitive_filter)
file_handler.addFilter(neo4j_filter)
root_logger.addHandler(file_handler)
cls._initialized = True

View File

@@ -1,10 +1,10 @@
import json
import os
import json
import time
from app.core.logging_config import get_agent_logger
from app.db import get_db
from app.core.memory.agent.models.problem_models import ProblemExtensionResponse
from app.core.memory.agent.services.optimized_llm_service import LLMServiceMixin
from app.core.memory.agent.utils.llm_tools import (
PROJECT_ROOT_,
ReadState,
@@ -12,9 +12,10 @@ from app.core.memory.agent.utils.llm_tools import (
from app.core.memory.agent.utils.redis_tool import store
from app.core.memory.agent.utils.session_tools import SessionService
from app.core.memory.agent.utils.template_tools import TemplateService
from app.db import get_db_context
from app.core.memory.agent.services.optimized_llm_service import LLMServiceMixin
template_root = os.path.join(PROJECT_ROOT_, 'memory', 'agent', 'utils', 'prompt')
db_session = next(get_db())
logger = get_agent_logger(__name__)
@@ -52,14 +53,13 @@ async def Split_The_Problem(state: ReadState) -> ReadState:
try:
# 使用优化的LLM服务
with get_db_context() as db_session:
structured = await problem_service.call_llm_structured(
state=state,
db_session=db_session,
system_prompt=system_prompt,
response_model=ProblemExtensionResponse,
fallback_value=[]
)
structured = await problem_service.call_llm_structured(
state=state,
db_session=db_session,
system_prompt=system_prompt,
response_model=ProblemExtensionResponse,
fallback_value=[]
)
# 添加更详细的日志记录
logger.info(f"Split_The_Problem: 开始处理问题分解,内容长度: {len(content)}")
@@ -111,7 +111,7 @@ async def Split_The_Problem(state: ReadState) -> ReadState:
"error_type": type(e).__name__,
"error_message": str(e),
"content_length": len(content),
"llm_model_id": str(memory_config.llm_model_id) if memory_config else None
"llm_model_id": memory_config.llm_model_id if memory_config else None
}
logger.error(f"Split_The_Problem error details: {error_details}")
@@ -171,14 +171,13 @@ async def Problem_Extension(state: ReadState) -> ReadState:
try:
# 使用优化的LLM服务
with get_db_context() as db_session:
response_content = await problem_service.call_llm_structured(
state=state,
db_session=db_session,
system_prompt=system_prompt,
response_model=ProblemExtensionResponse,
fallback_value=[]
)
response_content = await problem_service.call_llm_structured(
state=state,
db_session=db_session,
system_prompt=system_prompt,
response_model=ProblemExtensionResponse,
fallback_value=[]
)
logger.info(f"Problem_Extension: 开始处理问题扩展,问题数量: {len(databasets)}")
@@ -221,7 +220,7 @@ async def Problem_Extension(state: ReadState) -> ReadState:
"error_type": type(e).__name__,
"error_message": str(e),
"questions_count": len(databasets),
"llm_model_id": str(memory_config.llm_model_id) if memory_config else None
"llm_model_id": memory_config.llm_model_id if memory_config else None
}
logger.error(f"Problem_Extension error details: {error_details}")

View File

@@ -6,26 +6,31 @@ import os
# ===== 第三方库 =====
from langchain.agents import create_agent
from langchain_openai import ChatOpenAI
from app.core.logging_config import get_agent_logger
from app.db import get_db, get_db_context
from app.schemas import model_schema
from app.services.memory_config_service import MemoryConfigService
from app.services.model_service import ModelConfigService
from app.core.memory.agent.services.search_service import SearchService
from app.core.memory.agent.utils.llm_tools import (
COUNTState,
ReadState,
deduplicate_entries,
merge_to_key_value_pairs,
)
from app.core.memory.agent.langgraph_graph.tools.tool import (
create_hybrid_retrieval_tool_sync,
create_time_retrieval_tool,
extract_tool_message_content,
)
from app.core.memory.agent.services.search_service import SearchService
from app.core.memory.agent.utils.llm_tools import (
ReadState,
deduplicate_entries,
merge_to_key_value_pairs,
)
from app.core.rag.nlp.search import knowledge_retrieval
from app.db import get_db_context
from app.schemas import model_schema
from app.services.memory_config_service import MemoryConfigService
from app.services.model_service import ModelConfigService
logger = get_agent_logger(__name__)
db = next(get_db())
async def rag_config(state):
@@ -45,12 +50,10 @@ async def rag_config(state):
"reranker_top_k": 10
}
return kb_config
async def rag_knowledge(state, question):
async def rag_knowledge(state,question):
kb_config = await rag_config(state)
end_user_id = state.get('end_user_id', '')
user_rag_memory_id = state.get("user_rag_memory_id", '')
user_rag_memory_id=state.get("user_rag_memory_id",'')
retrieve_chunks_result = knowledge_retrieval(question, kb_config, [str(end_user_id)])
try:
retrieval_knowledge = [i.page_content for i in retrieve_chunks_result]
@@ -58,13 +61,13 @@ async def rag_knowledge(state, question):
cleaned_query = question
raw_results = clean_content
logger.info(f" Using RAG storage with memory_id={user_rag_memory_id}")
except Exception:
retrieval_knowledge = []
except Exception :
retrieval_knowledge=[]
clean_content = ''
raw_results = ''
cleaned_query = question
logger.info(f"No content retrieved from knowledge base: {user_rag_memory_id}")
return retrieval_knowledge, clean_content, cleaned_query, raw_results
return retrieval_knowledge,clean_content,cleaned_query,raw_results
async def llm_infomation(state: ReadState) -> ReadState:
@@ -110,7 +113,7 @@ async def clean_databases(data) -> str:
# 收集所有内容
content_list = []
# 处理重排序结果
reranked = results.get('reranked_results', {})
if reranked:
@@ -138,6 +141,7 @@ async def clean_databases(data) -> str:
elif isinstance(item, str):
text_parts.append(item)
return '\n'.join(text_parts).strip()
except Exception as e:
@@ -146,23 +150,23 @@ async def clean_databases(data) -> str:
async def retrieve_nodes(state: ReadState) -> ReadState:
'''
模型信息
'''
problem_extension = state.get('problem_extension', '')['context']
storage_type = state.get('storage_type', '')
user_rag_memory_id = state.get('user_rag_memory_id', '')
end_user_id = state.get('end_user_id', '')
problem_extension=state.get('problem_extension', '')['context']
storage_type=state.get('storage_type', '')
user_rag_memory_id=state.get('user_rag_memory_id', '')
end_user_id=state.get('end_user_id', '')
memory_config = state.get('memory_config', None)
original = state.get('data', '')
problem_list = []
for key, values in problem_extension.items():
original=state.get('data', '')
problem_list=[]
for key,values in problem_extension.items():
for data in values:
problem_list.append(data)
logger.info(f"Retrieve: storage_type={storage_type}, user_rag_memory_id={user_rag_memory_id}")
# 创建异步任务处理单个问题
async def process_question_nodes(idx, question):
try:
@@ -240,7 +244,7 @@ async def retrieve_nodes(state: ReadState) -> ReadState:
send_verify = []
for i, j in zip(keys, val, strict=False):
if j != ['']:
if j!=['']:
send_verify.append({
"Query_small": i,
"Answer_Small": j
@@ -253,13 +257,15 @@ async def retrieve_nodes(state: ReadState) -> ReadState:
}
logger.info(f"Collected {len(intermediate_outputs)} intermediate outputs from search results")
return {'retrieve': dup_databases}
return {'retrieve':dup_databases}
async def retrieve(state: ReadState) -> ReadState:
# 从state中获取end_user_id
import time
start = time.time()
start=time.time()
problem_extension = state.get('problem_extension', '')['context']
storage_type = state.get('storage_type', '')
user_rag_memory_id = state.get('user_rag_memory_id', '')
@@ -277,7 +283,6 @@ async def retrieve(state: ReadState) -> ReadState:
with get_db_context() as db: # 使用同步数据库上下文管理器
config_service = MemoryConfigService(db)
return await llm_infomation(state)
llm_config = await get_llm_info()
api_key_obj = llm_config.api_keys[0]
api_key = api_key_obj.api_key
@@ -291,11 +296,11 @@ async def retrieve(state: ReadState) -> ReadState:
)
time_retrieval_tool = create_time_retrieval_tool(end_user_id)
search_params = {"end_user_id": end_user_id, "return_raw_results": True}
hybrid_retrieval = create_hybrid_retrieval_tool_sync(memory_config, **search_params)
search_params = { "end_user_id": end_user_id, "return_raw_results": True }
hybrid_retrieval=create_hybrid_retrieval_tool_sync(memory_config, **search_params)
agent = create_agent(
llm,
tools=[time_retrieval_tool, hybrid_retrieval],
tools=[time_retrieval_tool,hybrid_retrieval],
system_prompt=f"我是检索专家可以根据适合的工具进行检索。当前使用的end_user_id是: {end_user_id}"
)
@@ -309,8 +314,7 @@ async def retrieve(state: ReadState) -> ReadState:
async with SEMAPHORE: # 限制并发
try:
if storage_type == "rag" and user_rag_memory_id:
retrieval_knowledge, clean_content, cleaned_query, raw_results = await rag_knowledge(state,
question)
retrieval_knowledge, clean_content, cleaned_query, raw_results = await rag_knowledge(state, question)
else:
cleaned_query = question
# 使用 asyncio 在线程池中运行同步的 agent.invoke
@@ -409,3 +413,5 @@ async def retrieve(state: ReadState) -> ReadState:
# json.dump(dup_databases, f, indent=4)
logger.info(f"Collected {len(intermediate_outputs)} intermediate outputs from search results")
return {'retrieve': dup_databases}

View File

@@ -1,3 +1,5 @@
import os
import time
@@ -15,77 +17,33 @@ from app.core.memory.agent.utils.llm_tools import (
from app.core.memory.agent.utils.redis_tool import store
from app.core.memory.agent.utils.session_tools import SessionService
from app.core.memory.agent.utils.template_tools import TemplateService
from app.core.rag.nlp.search import knowledge_retrieval
from app.db import get_db_context
from app.db import get_db
template_root = os.path.join(PROJECT_ROOT_, 'memory', 'agent', 'utils', 'prompt')
logger = get_agent_logger(__name__)
db_session = next(get_db())
class SummaryNodeService(LLMServiceMixin):
"""总结节点服务类"""
def __init__(self):
super().__init__()
self.template_service = TemplateService(template_root)
# 创建全局服务实例
summary_service = SummaryNodeService()
async def rag_config(state):
user_rag_memory_id = state.get('user_rag_memory_id', '')
kb_config = {
"knowledge_bases": [
{
"kb_id": user_rag_memory_id,
"similarity_threshold": 0.7,
"vector_similarity_weight": 0.5,
"top_k": 10,
"retrieve_type": "participle"
}
],
"merge_strategy": "weight",
"reranker_id": os.getenv('reranker_id'),
"reranker_top_k": 10
}
return kb_config
async def rag_knowledge(state, question):
kb_config = await rag_config(state)
end_user_id = state.get('end_user_id', '')
user_rag_memory_id = state.get("user_rag_memory_id", '')
retrieve_chunks_result = knowledge_retrieval(question, kb_config, [str(end_user_id)])
try:
retrieval_knowledge = [i.page_content for i in retrieve_chunks_result]
clean_content = '\n\n'.join(retrieval_knowledge)
cleaned_query = question
raw_results = clean_content
logger.info(f" Using RAG storage with memory_id={user_rag_memory_id}")
except Exception:
retrieval_knowledge = []
clean_content = ''
raw_results = ''
cleaned_query = question
logger.info(f"No content retrieved from knowledge base: {user_rag_memory_id}")
return retrieval_knowledge, clean_content, cleaned_query, raw_results
async def summary_history(state: ReadState) -> ReadState:
end_user_id = state.get("end_user_id", '')
history = await SessionService(store).get_history(end_user_id, end_user_id, end_user_id)
return history
async def summary_llm(state: ReadState, history, retrieve_info, template_name, operation_name, response_model,
search_mode) -> str:
async def summary_llm(state: ReadState, history, retrieve_info, template_name, operation_name, response_model,search_mode) -> str:
"""
增强的summary_llm函数包含更好的错误处理和数据验证
"""
data = state.get("data", '')
# 构建系统提示词
if str(search_mode) == "0":
system_prompt = await summary_service.template_service.render_template(
@@ -104,19 +62,18 @@ async def summary_llm(state: ReadState, history, retrieve_info, template_name, o
)
try:
# 使用优化的LLM服务进行结构化输出
with get_db_context() as db_session:
structured = await summary_service.call_llm_structured(
state=state,
db_session=db_session,
system_prompt=system_prompt,
response_model=response_model,
fallback_value=None
)
structured = await summary_service.call_llm_structured(
state=state,
db_session=db_session,
system_prompt=system_prompt,
response_model=response_model,
fallback_value=None
)
# 验证结构化响应
if structured is None:
logger.warning("LLM返回None使用默认回答")
logger.warning(f"LLM返回None使用默认回答")
return "信息不足,无法回答"
# 根据操作类型提取答案
if operation_name == "summary":
aimessages = getattr(structured, 'query_answer', None) or "信息不足,无法回答"
@@ -125,18 +82,18 @@ async def summary_llm(state: ReadState, history, retrieve_info, template_name, o
if hasattr(structured, 'data') and structured.data:
aimessages = getattr(structured.data, 'query_answer', None) or "信息不足,无法回答"
else:
logger.warning("结构化响应缺少data字段")
logger.warning(f"结构化响应缺少data字段")
aimessages = "信息不足,无法回答"
# 验证答案不为空
if not aimessages or aimessages.strip() == "":
aimessages = "信息不足,无法回答"
return aimessages
except Exception as e:
logger.error(f"结构化输出失败: {e}", exc_info=True)
# 尝试非结构化输出作为fallback
try:
logger.info("尝试非结构化输出作为fallback")
@@ -146,7 +103,7 @@ async def summary_llm(state: ReadState, history, retrieve_info, template_name, o
system_prompt=system_prompt,
fallback_message="信息不足,无法回答"
)
if response and response.strip():
# 简单清理响应
cleaned_response = response.strip()
@@ -154,17 +111,16 @@ async def summary_llm(state: ReadState, history, retrieve_info, template_name, o
if cleaned_response.startswith('```'):
lines = cleaned_response.split('\n')
cleaned_response = '\n'.join(lines[1:-1])
return cleaned_response
else:
return "信息不足,无法回答"
except Exception as fallback_error:
logger.error(f"Fallback也失败: {fallback_error}")
return "信息不足,无法回答"
async def summary_redis_save(state: ReadState, aimessages) -> ReadState:
async def summary_redis_save(state: ReadState,aimessages) -> ReadState:
data = state.get("data", '')
end_user_id = state.get("end_user_id", '')
await SessionService(store).save_session(
@@ -176,12 +132,10 @@ async def summary_redis_save(state: ReadState, aimessages) -> ReadState:
)
await SessionService(store).cleanup_duplicates()
logger.info(f"sessionid: {aimessages} 写入成功")
async def summary_prompt(state: ReadState, aimessages, raw_results) -> ReadState:
storage_type = state.get("storage_type", '')
user_rag_memory_id = state.get("user_rag_memory_id", '')
data = state.get("data", '')
async def summary_prompt(state: ReadState,aimessages,raw_results) -> ReadState:
storage_type=state.get("storage_type",'')
user_rag_memory_id=state.get("user_rag_memory_id",'')
data=state.get("data", '')
input_summary = {
"status": "success",
"summary_result": aimessages,
@@ -198,14 +152,14 @@ async def summary_prompt(state: ReadState, aimessages, raw_results) -> ReadState
"user_rag_memory_id": user_rag_memory_id
}
}
retrieve = {
retrieve={
"status": "success",
"summary_result": aimessages,
"storage_type": storage_type,
"user_rag_memory_id": user_rag_memory_id,
"_intermediate": {
"type": "retrieval_summary",
"title": "快速检索",
"title":"快速检索",
"summary": aimessages,
"query": data,
"storage_type": storage_type,
@@ -213,18 +167,17 @@ async def summary_prompt(state: ReadState, aimessages, raw_results) -> ReadState
}
}
return input_summary, retrieve
return input_summary,retrieve
async def Input_Summary(state: ReadState) -> ReadState:
start = time.time()
storage_type = state.get("storage_type", '')
start=time.time()
storage_type=state.get("storage_type",'')
memory_config = state.get('memory_config', None)
user_rag_memory_id = state.get("user_rag_memory_id", '')
data = state.get("data", '')
end_user_id = state.get("end_user_id", '')
user_rag_memory_id=state.get("user_rag_memory_id",'')
data=state.get("data", '')
end_user_id=state.get("end_user_id", '')
logger.info(f"Input_Summary: storage_type={storage_type}, user_rag_memory_id={user_rag_memory_id}")
history = await summary_history(state)
history = await summary_history( state)
search_params = {
"end_user_id": end_user_id,
"question": data,
@@ -233,14 +186,12 @@ async def Input_Summary(state: ReadState) -> ReadState:
}
try:
if storage_type != "rag":
retrieve_info, question, raw_results = await SearchService().execute_hybrid_search(**search_params,
memory_config=memory_config)
else:
retrieval_knowledge, retrieve_info, question, raw_results = await rag_knowledge(state, data)
retrieve_info, question, raw_results = await SearchService().execute_hybrid_search(**search_params, memory_config=memory_config)
except Exception as e:
logger.error(f"Input_Summary: hybrid_search failed, using empty results: {e}", exc_info=True)
logger.error( f"Input_Summary: hybrid_search failed, using empty results: {e}", exc_info=True )
retrieve_info, question, raw_results = "", data, []
try:
# aimessages=await summary_llm(state,history,retrieve_info,'Retrieve_Summary_prompt.jinja2',
# 'input_summary',RetrieveSummaryResponse)
@@ -248,8 +199,8 @@ async def Input_Summary(state: ReadState) -> ReadState:
summary_result = await summary_prompt(state, retrieve_info, retrieve_info)
summary = summary_result[0]
except Exception as e:
logger.error(f"Input_Summary failed: {e}", exc_info=True)
summary = {
logger.error( f"Input_Summary failed: {e}", exc_info=True )
summary= {
"status": "fail",
"summary_result": "信息不足,无法回答",
"storage_type": storage_type,
@@ -262,31 +213,30 @@ async def Input_Summary(state: ReadState) -> ReadState:
except Exception:
duration = 0.0
log_time('检索', duration)
return {"summary": summary}
return {"summary":summary}
async def Retrieve_Summary(state: ReadState) -> ReadState:
retrieve = state.get("retrieve", '')
history = await summary_history(state)
async def Retrieve_Summary(state: ReadState)-> ReadState:
retrieve=state.get("retrieve", '')
history = await summary_history( state)
import json
with open("检索.json", "w", encoding='utf-8') as f:
with open("检索.json","w",encoding='utf-8') as f:
f.write(json.dumps(retrieve, indent=4, ensure_ascii=False))
retrieve = retrieve.get("Expansion_issue", [])
start = time.time()
retrieve_info_str = []
retrieve=retrieve.get("Expansion_issue", [])
start=time.time()
retrieve_info_str=[]
for data in retrieve:
if data == '':
retrieve_info_str = ''
if data=='':
retrieve_info_str=''
else:
for key, value in data.items():
if key == 'Answer_Small':
if key=='Answer_Small':
for i in value:
retrieve_info_str.append(i)
retrieve_info_str = list(set(retrieve_info_str))
retrieve_info_str = '\n'.join(retrieve_info_str)
retrieve_info_str=list(set(retrieve_info_str))
retrieve_info_str='\n'.join(retrieve_info_str)
aimessages = await summary_llm(state, history, retrieve_info_str,
'direct_summary_prompt.jinja2', 'retrieve_summary', RetrieveSummaryResponse, "1")
aimessages=await summary_llm(state,history,retrieve_info_str,
'direct_summary_prompt.jinja2','retrieve_summary',RetrieveSummaryResponse,"1")
if '信息不足,无法回答' not in str(aimessages) or str(aimessages) != "":
await summary_redis_save(state, aimessages)
if aimessages == '':
@@ -298,33 +248,33 @@ async def Retrieve_Summary(state: ReadState) -> ReadState:
except Exception:
duration = 0.0
log_time('Retrieval summary', duration)
# 修复协程调用 - 先await然后访问返回值
summary_result = await summary_prompt(state, aimessages, retrieve_info_str)
summary = summary_result[1]
return {"summary": summary}
return {"summary":summary}
async def Summary(state: ReadState) -> ReadState:
start = time.time()
async def Summary(state: ReadState)-> ReadState:
start=time.time()
query = state.get("data", '')
verify = state.get("verify", '')
verify_expansion_issue = verify.get("verified_data", '')
retrieve_info_str = ''
verify=state.get("verify", '')
verify_expansion_issue=verify.get("verified_data", '')
retrieve_info_str=''
for data in verify_expansion_issue:
for key, value in data.items():
if key == 'answer_small':
if key=='answer_small':
for i in value:
retrieve_info_str += i + '\n'
history = await summary_history(state)
retrieve_info_str+=i+'\n'
history=await summary_history(state)
data = {
"query": query,
"history": history,
"retrieve_info": retrieve_info_str
}
aimessages = await summary_llm(state, history, data,
'summary_prompt.jinja2', 'summary', SummaryResponse, 0)
aimessages=await summary_llm(state,history,data,
'summary_prompt.jinja2','summary',SummaryResponse,0)
if '信息不足,无法回答' not in str(aimessages) or str(aimessages) != "":
await summary_redis_save(state, aimessages)
@@ -339,12 +289,11 @@ async def Summary(state: ReadState) -> ReadState:
# 修复协程调用 - 先await然后访问返回值
summary_result = await summary_prompt(state, aimessages, retrieve_info_str)
summary = summary_result[1]
return {"summary": summary}
return {"summary":summary}
async def Summary_fails(state: ReadState) -> ReadState:
storage_type = state.get("storage_type", '')
user_rag_memory_id = state.get("user_rag_memory_id", '')
async def Summary_fails(state: ReadState)-> ReadState:
storage_type=state.get("storage_type", '')
user_rag_memory_id=state.get("user_rag_memory_id", '')
history = await summary_history(state)
query = state.get("data", '')
verify = state.get("verify", '')
@@ -360,12 +309,12 @@ async def Summary_fails(state: ReadState) -> ReadState:
"history": history,
"retrieve_info": retrieve_info_str
}
aimessages = await summary_llm(state, history, data,
'fail_summary_prompt.jinja2', 'summary', SummaryResponse, 0)
result = {
aimessages = await summary_llm(state, history, data,
'fail_summary_prompt.jinja2', 'summary', SummaryResponse, 0)
result= {
"status": "success",
"summary_result": aimessages,
"storage_type": storage_type,
"user_rag_memory_id": user_rag_memory_id
}
return {"summary": result}
return {"summary":result}

View File

@@ -1,9 +1,8 @@
import asyncio
import os
from app.core.logging_config import get_agent_logger
from app.db import get_db
from app.core.memory.agent.models.verification_models import VerificationResult
from app.core.memory.agent.services.optimized_llm_service import LLMServiceMixin
from app.core.memory.agent.utils.llm_tools import (
PROJECT_ROOT_,
ReadState,
@@ -11,30 +10,28 @@ from app.core.memory.agent.utils.llm_tools import (
from app.core.memory.agent.utils.redis_tool import store
from app.core.memory.agent.utils.session_tools import SessionService
from app.core.memory.agent.utils.template_tools import TemplateService
from app.db import get_db_context
from app.core.memory.agent.services.optimized_llm_service import LLMServiceMixin
template_root = os.path.join(PROJECT_ROOT_, 'memory', 'agent', 'utils', 'prompt')
db_session = next(get_db())
logger = get_agent_logger(__name__)
class VerificationNodeService(LLMServiceMixin):
"""验证节点服务类"""
def __init__(self):
super().__init__()
self.template_service = TemplateService(template_root)
# 创建全局服务实例
verification_service = VerificationNodeService()
async def Verify_prompt(state: ReadState, messages_deal: VerificationResult):
"""处理验证结果并生成输出格式"""
storage_type = state.get('storage_type', '')
user_rag_memory_id = state.get('user_rag_memory_id', '')
data = state.get('data', '')
# 将 VerificationItem 对象转换为字典列表
verified_data = []
if messages_deal.expansion_issue:
@@ -43,7 +40,7 @@ async def Verify_prompt(state: ReadState, messages_deal: VerificationResult):
verified_data.append(item.model_dump())
elif isinstance(item, dict):
verified_data.append(item)
Verify_result = {
"status": messages_deal.split_result,
"verified_data": verified_data,
@@ -61,37 +58,34 @@ async def Verify_prompt(state: ReadState, messages_deal: VerificationResult):
}
}
return Verify_result
async def Verify(state: ReadState):
logger.info("=== Verify 节点开始执行 ===")
try:
content = state.get('data', '')
end_user_id = state.get('end_user_id', '')
memory_config = state.get('memory_config', None)
logger.info(f"Verify: content={content[:50] if content else 'empty'}..., end_user_id={end_user_id}")
history = await SessionService(store).get_history(end_user_id, end_user_id, end_user_id)
logger.info(f"Verify: 获取历史记录完成history length={len(history)}")
retrieve = state.get("retrieve", {})
logger.info(
f"Verify: retrieve data type={type(retrieve)}, keys={retrieve.keys() if isinstance(retrieve, dict) else 'N/A'}")
logger.info(f"Verify: retrieve data type={type(retrieve)}, keys={retrieve.keys() if isinstance(retrieve, dict) else 'N/A'}")
retrieve_expansion = retrieve.get("Expansion_issue", []) if isinstance(retrieve, dict) else []
logger.info(f"Verify: Expansion_issue length={len(retrieve_expansion)}")
messages = {
"Query": content,
"Expansion_issue": retrieve_expansion
}
logger.info("Verify: 开始渲染模板")
# 生成 JSON schema 以指导 LLM 输出正确格式
json_schema = VerificationResult.model_json_schema()
system_prompt = await verification_service.template_service.render_template(
template_name='split_verify_prompt.jinja2',
operation_name='split_verify_prompt',
@@ -100,30 +94,29 @@ async def Verify(state: ReadState):
json_schema=json_schema
)
logger.info(f"Verify: 模板渲染完成prompt length={len(system_prompt)}")
# 使用优化的LLM服务添加超时保护
logger.info("Verify: 开始调用 LLM")
try:
# 添加 asyncio.wait_for 超时包裹,防止无限等待
# 超时时间设置为 150 秒(比 LLM 配置的 120 秒稍长)
with get_db_context() as db_session:
structured = await asyncio.wait_for(
verification_service.call_llm_structured(
state=state,
db_session=db_session,
system_prompt=system_prompt,
response_model=VerificationResult,
fallback_value={
"query": content,
"history": history if isinstance(history, list) else [],
"expansion_issue": [],
"split_result": "failed",
"reason": "验证失败或超时"
}
),
timeout=150.0 # 150秒超时
)
import asyncio
structured = await asyncio.wait_for(
verification_service.call_llm_structured(
state=state,
db_session=db_session,
system_prompt=system_prompt,
response_model=VerificationResult,
fallback_value={
"query": content,
"history": history if isinstance(history, list) else [],
"expansion_issue": [],
"split_result": "failed",
"reason": "验证失败或超时"
}
),
timeout=150.0 # 150秒超时
)
logger.info(f"Verify: LLM 调用完成result={structured}")
except asyncio.TimeoutError:
logger.error("Verify: LLM 调用超时150秒使用 fallback 值")
@@ -134,11 +127,11 @@ async def Verify(state: ReadState):
split_result="failed",
reason="LLM调用超时"
)
result = await Verify_prompt(state, structured)
logger.info("=== Verify 节点执行完成 ===")
return {"verify": result}
except Exception as e:
logger.error(f"Verify 节点执行失败: {e}", exc_info=True)
# 返回失败的验证结果
@@ -159,4 +152,4 @@ async def Verify(state: ReadState):
"user_rag_memory_id": state.get('user_rag_memory_id', '')
}
}
}
}

View File

@@ -1,4 +1,3 @@
from app.cache.memory.interest_memory import InterestMemoryCache
from app.core.memory.agent.utils.llm_tools import WriteState
from app.core.memory.agent.utils.write_tools import write
from app.core.logging_config import get_agent_logger
@@ -11,7 +10,7 @@ async def write_node(state: WriteState) -> WriteState:
Write data to the database/file system.
Args:
state: WriteState containing messages, end_user_id, memory_config, and language
state: WriteState containing messages, end_user_id, and memory_config
Returns:
dict: Contains 'write_result' with status and data fields
@@ -19,7 +18,6 @@ async def write_node(state: WriteState) -> WriteState:
messages = state.get('messages', [])
end_user_id = state.get('end_user_id', '')
memory_config = state.get('memory_config', '')
language = state.get('language', 'zh') # 默认中文
# Convert LangChain messages to structured format expected by write()
structured_messages = []
@@ -37,19 +35,9 @@ async def write_node(state: WriteState) -> WriteState:
messages=structured_messages,
end_user_id=end_user_id,
memory_config=memory_config,
language=language,
)
logger.info(f"Write completed successfully! Config: {memory_config.config_name}")
# 写入 neo4j 成功后,删除该用户的兴趣分布缓存,确保下次请求重新生成
for lang in ["zh", "en"]:
deleted = await InterestMemoryCache.delete_interest_distribution(
end_user_id=end_user_id,
language=lang,
)
if deleted:
logger.info(f"Invalidated interest distribution cache: end_user_id={end_user_id}, language={lang}")
write_result = {
"status": "success",
"data": structured_messages,

View File

@@ -5,6 +5,7 @@ from langchain_core.messages import HumanMessage
from langgraph.constants import START, END
from langgraph.graph import StateGraph
from app.db import get_db
from app.services.memory_config_service import MemoryConfigService
@@ -31,6 +32,7 @@ from app.core.memory.agent.langgraph_graph.routing.routers import (
)
@asynccontextmanager
async def make_read_graph():
"""创建并返回 LangGraph 工作流"""
@@ -47,7 +49,7 @@ async def make_read_graph():
workflow.add_node("Retrieve_Summary", Retrieve_Summary)
workflow.add_node("Summary", Summary)
workflow.add_node("Summary_fails", Summary_fails)
# 添加边
workflow.add_edge(START, "content_input")
workflow.add_conditional_edges("content_input", Split_continue)
@@ -60,20 +62,20 @@ async def make_read_graph():
workflow.add_edge("Summary_fails", END)
workflow.add_edge("Summary", END)
'''-----'''
# workflow.add_edge("Retrieve", END)
# 编译工作流
graph = workflow.compile()
yield graph
except Exception as e:
print(f"创建工作流失败: {e}")
raise
finally:
print("工作流创建完成")
async def main():
"""主函数 - 运行工作流"""
message = "昨天有什么好看的电影"
@@ -90,19 +92,17 @@ async def main():
service_name="MemoryAgentService"
)
import time
start = time.time()
start=time.time()
try:
async with make_read_graph() as graph:
config = {"configurable": {"thread_id": end_user_id}}
# 初始状态 - 包含所有必要字段
initial_state = {"messages": [HumanMessage(content=message)], "search_switch": search_switch,
"end_user_id": end_user_id
, "storage_type": storage_type, "user_rag_memory_id": user_rag_memory_id,
"memory_config": memory_config}
initial_state = {"messages": [HumanMessage(content=message)] ,"search_switch":search_switch,"end_user_id":end_user_id
,"storage_type":storage_type,"user_rag_memory_id":user_rag_memory_id,"memory_config":memory_config}
# 获取节点更新信息
_intermediate_outputs = []
summary = ''
async for update_event in graph.astream(
initial_state,
stream_mode="updates",
@@ -110,7 +110,7 @@ async def main():
):
for node_name, node_data in update_event.items():
print(f"处理节点: {node_name}")
# 处理不同Summary节点的返回结构
if 'Summary' in node_name:
if 'InputSummary' in node_data and 'summary_result' in node_data['InputSummary']:
@@ -125,22 +125,23 @@ async def main():
spit_data = node_data.get('spit_data', {}).get('_intermediate', None)
if spit_data and spit_data != [] and spit_data != {}:
_intermediate_outputs.append(spit_data)
# Problem_Extension 节点
problem_extension = node_data.get('problem_extension', {}).get('_intermediate', None)
if problem_extension and problem_extension != [] and problem_extension != {}:
_intermediate_outputs.append(problem_extension)
# Retrieve 节点
retrieve_node = node_data.get('retrieve', {}).get('_intermediate_outputs', None)
if retrieve_node and retrieve_node != [] and retrieve_node != {}:
_intermediate_outputs.extend(retrieve_node)
# Verify 节点
verify_n = node_data.get('verify', {}).get('_intermediate', None)
if verify_n and verify_n != [] and verify_n != {}:
_intermediate_outputs.append(verify_n)
# Summary 节点
summary_n = node_data.get('summary', {}).get('_intermediate', None)
if summary_n and summary_n != [] and summary_n != {}:
@@ -160,20 +161,17 @@ async def main():
#
print(f"=== 最终摘要 ===")
print(summary)
except Exception as e:
import traceback
traceback.print_exc()
finally:
db_session.close()
end = time.time()
print(100 * 'y')
print(f"总耗时: {end - start}s")
print(100 * 'y')
end=time.time()
print(100*'y')
print(f"总耗时: {end-start}s")
print(100*'y')
if __name__ == "__main__":
import asyncio
asyncio.run(main())

View File

@@ -1,3 +1,4 @@
import asyncio
import json
import sys
@@ -19,8 +20,6 @@ logger = get_agent_logger(__name__)
if sys.platform.startswith("win"):
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
@asynccontextmanager
async def make_write_graph():
"""

View File

@@ -21,7 +21,7 @@ async def get_chunked_dialogs(
end_user_id: Group identifier
messages: Structured message list [{"role": "user", "content": "..."}, ...]
ref_id: Reference identifier
config_id: Configuration ID for processing (used to load pruning config)
config_id: Configuration ID for processing
Returns:
List of DialogData objects with generated chunks
@@ -57,63 +57,6 @@ async def get_chunked_dialogs(
end_user_id=end_user_id,
config_id=config_id
)
# 语义剪枝步骤(在分块之前)
try:
from app.core.memory.storage_services.extraction_engine.data_preprocessing.data_pruning import SemanticPruner
from app.core.memory.models.config_models import PruningConfig
from app.db import get_db_context
from app.services.memory_config_service import MemoryConfigService
from app.core.memory.utils.llm.llm_utils import MemoryClientFactory
# 加载剪枝配置
pruning_config = None
if config_id:
try:
with get_db_context() as db:
# 使用 MemoryConfigService 加载完整的 MemoryConfig 对象
config_service = MemoryConfigService(db)
memory_config = config_service.load_memory_config(
config_id=config_id,
service_name="semantic_pruning"
)
if memory_config:
pruning_config = PruningConfig(
pruning_switch=memory_config.pruning_enabled,
pruning_scene=memory_config.pruning_scene or "education",
pruning_threshold=memory_config.pruning_threshold,
scene_id=str(memory_config.scene_id) if memory_config.scene_id else None,
ontology_classes=memory_config.ontology_classes,
)
logger.info(f"[剪枝] 加载配置: switch={pruning_config.pruning_switch}, scene={pruning_config.pruning_scene}, threshold={pruning_config.pruning_threshold}")
# 获取LLM客户端用于剪枝
if pruning_config.pruning_switch:
factory = MemoryClientFactory(db)
llm_client = factory.get_llm_client_from_config(memory_config)
# 执行剪枝 - 使用 prune_dataset 支持消息级剪枝
pruner = SemanticPruner(config=pruning_config, llm_client=llm_client)
original_msg_count = len(dialog_data.context.msgs)
# 使用 prune_dataset 而不是 prune_dialog
# prune_dataset 会进行消息级剪枝,即使对话整体相关也会删除不重要消息
pruned_dialogs = await pruner.prune_dataset([dialog_data])
if pruned_dialogs:
dialog_data = pruned_dialogs[0]
remaining_msg_count = len(dialog_data.context.msgs)
deleted_count = original_msg_count - remaining_msg_count
logger.info(f"[剪枝] 完成: 原始{original_msg_count}条 -> 保留{remaining_msg_count}条 (删除{deleted_count}条)")
else:
logger.warning("[剪枝] prune_dataset 返回空列表")
else:
logger.info("[剪枝] 配置中剪枝开关关闭,跳过剪枝")
except Exception as e:
logger.warning(f"[剪枝] 加载配置失败,跳过剪枝: {e}", exc_info=True)
except Exception as e:
logger.warning(f"[剪枝] 执行失败,跳过剪枝: {e}", exc_info=True)
chunker = DialogueChunker(chunker_strategy)
extracted_chunks = await chunker.process_dialogue(dialog_data)

View File

@@ -0,0 +1,56 @@
import asyncio
from typing import Dict, Optional
from app.core.memory.utils.llm.llm_utils import get_llm_client_fast
from app.db import get_db
from app.core.logging_config import get_agent_logger
logger = get_agent_logger(__name__)
class LLMClientPool:
"""LLM客户端连接池"""
def __init__(self, max_size: int = 5):
self.max_size = max_size
self.pools: Dict[str, asyncio.Queue] = {}
self.active_clients: Dict[str, int] = {}
async def get_client(self, llm_model_id: str):
"""获取LLM客户端"""
if llm_model_id not in self.pools:
self.pools[llm_model_id] = asyncio.Queue(maxsize=self.max_size)
self.active_clients[llm_model_id] = 0
pool = self.pools[llm_model_id]
try:
# 尝试从池中获取客户端
client = pool.get_nowait()
logger.debug(f"从池中获取LLM客户端: {llm_model_id}")
return client
except asyncio.QueueEmpty:
# 池为空,创建新客户端
if self.active_clients[llm_model_id] < self.max_size:
db_session = next(get_db())
client = get_llm_client_fast(llm_model_id, db_session)
self.active_clients[llm_model_id] += 1
logger.debug(f"创建新LLM客户端: {llm_model_id}")
return client
else:
# 等待可用客户端
logger.debug(f"等待LLM客户端可用: {llm_model_id}")
return await pool.get()
async def return_client(self, llm_model_id: str, client):
"""归还LLM客户端到池中"""
if llm_model_id in self.pools:
try:
self.pools[llm_model_id].put_nowait(client)
logger.debug(f"归还LLM客户端到池: {llm_model_id}")
except asyncio.QueueFull:
# 池已满,丢弃客户端
self.active_clients[llm_model_id] -= 1
logger.debug(f"池已满丢弃LLM客户端: {llm_model_id}")
# 全局客户端池
llm_client_pool = LLMClientPool()

View File

@@ -18,7 +18,6 @@ class WriteState(TypedDict):
memory_config: object
write_result: dict
data: str
language: str # 语言类型 ("zh" 中文, "en" 英文)
class ReadState(TypedDict):
"""

View File

@@ -34,17 +34,17 @@ async def write(
memory_config: MemoryConfig,
messages: list,
ref_id: str = "wyl20251027",
language: str = "zh",
) -> None:
"""
Execute the complete knowledge extraction pipeline.
Args:
user_id: User identifier
apply_id: Application identifier
end_user_id: Group identifier
memory_config: MemoryConfig object containing all configuration
messages: Structured message list [{"role": "user", "content": "..."}, ...]
ref_id: Reference ID, defaults to "wyl20251027"
language: 语言类型 ("zh" 中文, "en" 英文),默认中文
"""
# Extract config values
embedding_model_id = str(memory_config.embedding_model_id)
@@ -94,39 +94,12 @@ async def write(
from app.core.memory.utils.config.config_utils import get_pipeline_config
pipeline_config = get_pipeline_config(memory_config)
# Fetch ontology types if scene_id is configured
ontology_types = None
if memory_config.scene_id:
try:
from app.core.memory.ontology_services.ontology_type_loader import load_ontology_types_for_scene
with get_db_context() as db:
ontology_types = load_ontology_types_for_scene(
scene_id=memory_config.scene_id,
workspace_id=memory_config.workspace_id,
db=db
)
if ontology_types:
logger.info(
f"Loaded {len(ontology_types.types)} ontology types for scene_id: {memory_config.scene_id}"
)
else:
logger.info(f"No ontology classes found for scene_id: {memory_config.scene_id}")
except Exception as e:
logger.warning(
f"Failed to fetch ontology types for scene_id {memory_config.scene_id}: {e}",
exc_info=True
)
orchestrator = ExtractionOrchestrator(
llm_client=llm_client,
embedder_client=embedder_client,
connector=neo4j_connector,
config=pipeline_config,
embedding_id=embedding_model_id,
language=language,
ontology_types=ontology_types,
)
# Run the complete extraction pipeline
@@ -200,7 +173,7 @@ async def write(
step_start = time.time()
try:
summaries = await memory_summary_generation(
chunked_dialogs, llm_client=llm_client, embedder_client=embedder_client, language=language
chunked_dialogs, llm_client=llm_client, embedder_client=embedder_client
)
try:
@@ -225,24 +198,5 @@ async def write(
with open(log_file, "a", encoding="utf-8") as f:
f.write(f"=== Pipeline Run Completed: {timestamp} ===\n\n")
# 将提取统计写入 Redis按 workspace_id 存储
try:
from app.cache.memory.activity_stats_cache import ActivityStatsCache
stats_to_cache = {
"chunk_count": len(all_chunk_nodes) if all_chunk_nodes else 0,
"statements_count": len(all_statement_nodes) if all_statement_nodes else 0,
"triplet_entities_count": len(all_entity_nodes) if all_entity_nodes else 0,
"triplet_relations_count": len(all_entity_entity_edges) if all_entity_entity_edges else 0,
"temporal_count": 0,
}
await ActivityStatsCache.set_activity_stats(
workspace_id=str(memory_config.workspace_id),
stats=stats_to_cache,
)
logger.info(f"[WRITE] 活动统计已写入 Redis: workspace_id={memory_config.workspace_id}")
except Exception as cache_err:
logger.warning(f"[WRITE] 写入活动统计缓存失败(不影响主流程): {cache_err}", exc_info=True)
logger.info("=== Pipeline Complete ===")
logger.info(f"Total execution time: {total_time:.2f} seconds")
logger.info(f"Total execution time: {total_time:.2f} seconds")

View File

@@ -1,12 +1,9 @@
import asyncio
import json
import logging
import os
from typing import List, Tuple
from app.core.config import settings
logger = logging.getLogger(__name__)
from app.core.memory.utils.llm.llm_utils import MemoryClientFactory
from app.db import get_db_context
from app.repositories.neo4j.neo4j_connector import Neo4jConnector
@@ -19,10 +16,6 @@ class FilteredTags(BaseModel):
"""用于接收LLM筛选后的核心标签列表的模型。"""
meaningful_tags: List[str] = Field(..., description="从原始列表中筛选出的具有核心代表意义的名词列表。")
class InterestTags(BaseModel):
"""用于接收LLM筛选后的兴趣活动标签列表的模型。"""
interest_tags: List[str] = Field(..., description="从原始列表中筛选出的代表用户兴趣活动的标签列表。")
async def filter_tags_with_llm(tags: List[str], end_user_id: str) -> List[str]:
"""
使用LLM筛选标签列表仅保留具有代表性的核心名词。
@@ -46,20 +39,16 @@ async def filter_tags_with_llm(tags: List[str], end_user_id: str) -> List[str]:
connected_config = get_end_user_connected_config(end_user_id, db)
config_id = connected_config.get("memory_config_id")
workspace_id = connected_config.get("workspace_id")
if not config_id and not workspace_id:
if not config_id:
raise ValueError(
f"No memory_config_id found for end_user_id: {end_user_id}. "
"Please ensure the user has a valid memory configuration."
)
# Use the config_id to get the proper LLM client with workspace fallback
# Use the config_id to get the proper LLM client
config_service = MemoryConfigService(db)
memory_config = config_service.load_memory_config(
config_id=config_id,
workspace_id=workspace_id
)
memory_config = config_service.load_memory_config(config_id)
if not memory_config.llm_model_id:
raise ValueError(
@@ -92,74 +81,10 @@ async def filter_tags_with_llm(tags: List[str], end_user_id: str) -> List[str]:
return structured_response.meaningful_tags
except Exception as e:
logger.error(f"LLM筛选过程中发生错误: {e}", exc_info=True)
print(f"LLM筛选过程中发生错误: {e}")
# 在LLM失败时返回原始标签确保流程继续
return tags
async def filter_interests_with_llm(tags: List[str], end_user_id: str, language: str = "zh") -> List[str]:
"""
使用LLM从标签列表中筛选出代表用户兴趣活动的标签。
与 filter_tags_with_llm 不同,此函数专注于识别"活动/行为"类兴趣,
过滤掉纯物品、工具、地点等不代表用户主动参与活动的名词。
Args:
tags: 原始标签列表
end_user_id: 用户ID用于获取LLM配置
Returns:
筛选后的兴趣活动标签列表
"""
try:
with get_db_context() as db:
from app.services.memory_agent_service import (
get_end_user_connected_config,
)
connected_config = get_end_user_connected_config(end_user_id, db)
config_id = connected_config.get("memory_config_id")
workspace_id = connected_config.get("workspace_id")
if not config_id and not workspace_id:
raise ValueError(
f"No memory_config_id found for end_user_id: {end_user_id}."
)
config_service = MemoryConfigService(db)
memory_config = config_service.load_memory_config(
config_id=config_id,
workspace_id=workspace_id
)
if not memory_config.llm_model_id:
raise ValueError(
f"No llm_model_id found in memory config {config_id}."
)
factory = MemoryClientFactory(db)
llm_client = factory.get_llm_client(memory_config.llm_model_id)
tag_list_str = ", ".join(tags)
from app.core.memory.utils.prompt.prompt_utils import render_interest_filter_prompt
rendered_prompt = render_interest_filter_prompt(tag_list_str, language=language)
messages = [
{
"role": "user",
"content": rendered_prompt
}
]
structured_response = await llm_client.response_structured(
messages=messages,
response_model=InterestTags
)
return structured_response.interest_tags
except Exception as e:
logger.error(f"兴趣标签LLM筛选过程中发生错误: {e}", exc_info=True)
return tags
async def get_raw_tags_from_db(
connector: Neo4jConnector,
end_user_id: str,
@@ -210,14 +135,14 @@ async def get_raw_tags_from_db(
return [(record["name"], record["frequency"]) for record in results]
async def get_hot_memory_tags(end_user_id: str, limit: int = 10, by_user: bool = False) -> List[Tuple[str, int]]:
async def get_hot_memory_tags(end_user_id: str, limit: int = 40, by_user: bool = False) -> List[Tuple[str, int]]:
"""
获取原始标签然后使用LLM进行筛选返回最终的热门标签列表。
查询更多的标签(40)给LLM提供更丰富的上下文进行筛选但最终返回数量由limit参数控制
查询更多的标签(limit=40)给LLM提供更丰富的上下文进行筛选。
Args:
end_user_id: 必需参数。如果by_user=False则为end_user_id如果by_user=True则为user_id
limit: 最终返回的标签数量限制默认10
limit: 返回的标签数量限制
by_user: 是否按user_id查询默认False按end_user_id查询
Raises:
@@ -232,9 +157,8 @@ async def get_hot_memory_tags(end_user_id: str, limit: int = 10, by_user: bool =
# 使用项目的Neo4jConnector
connector = Neo4jConnector()
try:
# 1. 从数据库获取原始排名靠前的标签查询40条给LLM提供更丰富的上下文
query_limit = 40
raw_tags_with_freq = await get_raw_tags_from_db(connector, end_user_id, query_limit, by_user=by_user)
# 1. 从数据库获取原始排名靠前的标签
raw_tags_with_freq = await get_raw_tags_from_db(connector, end_user_id, limit, by_user=by_user)
if not raw_tags_with_freq:
return []
@@ -249,61 +173,7 @@ async def get_hot_memory_tags(end_user_id: str, limit: int = 10, by_user: bool =
if tag in meaningful_tag_names:
final_tags.append((tag, freq))
# 4. 限制返回的标签数量
return final_tags[:limit]
return final_tags
finally:
# 确保关闭连接
await connector.close()
async def get_interest_distribution(end_user_id: str, limit: int = 10, by_user: bool = False, language: str = "zh") -> List[Tuple[str, int]]:
"""
获取用户的兴趣分布标签。
与 get_hot_memory_tags 不同,此函数使用专门针对"活动/行为"的LLM prompt
过滤掉纯物品、工具、地点等,只保留能代表用户兴趣爱好的活动类标签。
Args:
end_user_id: 必需参数。如果by_user=False则为end_user_id如果by_user=True则为user_id
limit: 最终返回的标签数量限制默认10
by_user: 是否按user_id查询默认False按end_user_id查询
Raises:
ValueError: 如果end_user_id未提供或为空
"""
if not end_user_id or not end_user_id.strip():
raise ValueError(
"end_user_id is required. Please provide a valid end_user_id or user_id."
)
connector = Neo4jConnector()
try:
# 查询更多原始标签给LLM提供充足上下文
query_limit = 40
raw_tags_with_freq = await get_raw_tags_from_db(connector, end_user_id, query_limit, by_user=by_user)
if not raw_tags_with_freq:
return []
raw_tag_names = [tag for tag, freq in raw_tags_with_freq]
raw_freq_map = {tag: freq for tag, freq in raw_tags_with_freq}
# 使用兴趣活动专用prompt进行筛选支持语义推断出新标签
interest_tag_names = await filter_interests_with_llm(raw_tag_names, end_user_id, language=language)
# 构建最终标签列表:
# - 原始标签中存在的,保留原始频率
# - LLM推断出的新标签不在原始列表中赋予默认频率1
final_tags = []
seen = set()
for tag in interest_tag_names:
if tag in seen:
continue
seen.add(tag)
freq = raw_freq_map.get(tag, 1)
final_tags.append((tag, freq))
# 按频率降序排列
final_tags.sort(key=lambda x: x[1], reverse=True)
return final_tags[:limit]
finally:
await connector.close()

View File

@@ -108,6 +108,7 @@ class DimensionAnalyzer:
# Create dimension portrait
portrait = DimensionPortrait(
user_id=user_id,
creativity=dimension_scores["creativity"],
aesthetic=dimension_scores["aesthetic"],
technology=dimension_scores["technology"],
@@ -219,7 +220,7 @@ class DimensionAnalyzer:
"""Create an empty dimension portrait when no data is available.
Args:
user_id: Target user ID (used for logging only)
user_id: Target user ID
Returns:
Empty DimensionPortrait
@@ -227,6 +228,7 @@ class DimensionAnalyzer:
current_time = datetime.now()
return DimensionPortrait(
user_id=user_id,
creativity=self._create_default_dimension_score("creativity"),
aesthetic=self._create_default_dimension_score("aesthetic"),
technology=self._create_default_dimension_score("technology"),

View File

@@ -7,7 +7,7 @@ providing percentage distribution that totals 100%.
import logging
from datetime import datetime
from typing import Dict, List, Optional
from typing import Any, Dict, List, Optional
from app.core.memory.analytics.implicit_memory.llm_client import ImplicitMemoryLLMClient
from app.core.memory.llm_tools.llm_client import LLMClientException
@@ -133,6 +133,7 @@ class InterestAnalyzer:
# Create interest area distribution
distribution = InterestAreaDistribution(
user_id=user_id,
tech=interest_categories["tech"],
lifestyle=interest_categories["lifestyle"],
music=interest_categories["music"],
@@ -250,7 +251,7 @@ class InterestAnalyzer:
"""Create an empty interest distribution when no data is available.
Args:
user_id: Target user ID (used for logging only)
user_id: Target user ID
Returns:
Empty InterestAreaDistribution with equal percentages
@@ -258,15 +259,15 @@ class InterestAnalyzer:
current_time = datetime.now()
equal_percentage = 25.0 # 100% / 4 categories
def default_category(name: str) -> InterestCategory:
return InterestCategory(
category_name=name,
percentage=equal_percentage,
evidence=["Insufficient data for analysis"],
trending_direction=None
)
default_category = lambda name: InterestCategory(
category_name=name,
percentage=equal_percentage,
evidence=["Insufficient data for analysis"],
trending_direction=None
)
return InterestAreaDistribution(
user_id=user_id,
tech=default_category("tech"),
lifestyle=default_category("lifestyle"),
music=default_category("music"),

View File

@@ -16,7 +16,6 @@ Summary {{ loop.index }}:
3. DO NOT use long phrases - use short nouns or noun phrases
4. Only include preferences with confidence_score >= 0.3
5. **IMPORTANT: Output language MUST match the input language. If summaries are in Chinese, output in Chinese. If in English, output in English.**
6. **CRITICAL: supporting_evidence must be DIRECT QUOTES or paraphrases from the user's actual statements. DO NOT reference summary numbers (e.g., "Summary 1", "摘要1"). DO NOT describe what the summary contains. Extract the actual user behavior or statement as evidence.**
## Output Format
{
@@ -39,16 +38,6 @@ Summary {{ loop.index }}:
]
}
## BAD supporting_evidence examples (DO NOT do this):
- "Summary 1西湖为核心景区" ❌
- "摘要2中提到喜欢咖啡" ❌
- "Based on Summary 3" ❌
## GOOD supporting_evidence examples:
- "去过西湖断桥、苏堤" ✓
- "每天早上喝咖啡" ✓
- "mentioned visiting the lake twice" ✓
## Example (English input → English output)
{
"preferences": [

View File

@@ -58,25 +58,12 @@ from app.core.memory.models.triplet_models import (
TripletExtractionResponse,
)
# Ontology scenario models (LLM extracted from scenarios)
from app.core.memory.models.ontology_scenario_models import (
# Ontology models
from app.core.memory.models.ontology_models import (
OntologyClass,
OntologyExtractionResponse,
)
# Ontology extraction models (for extraction flow)
from app.core.memory.models.ontology_extraction_models import (
OntologyTypeInfo,
OntologyTypeList,
)
# Ontology general models (loaded from external ontology files)
from app.core.memory.models.ontology_general_models import (
OntologyFileFormat,
GeneralOntologyType,
GeneralOntologyTypeRegistry,
)
# Variable configuration models
from app.core.memory.models.variate_config import (
StatementExtractionConfig,
@@ -127,13 +114,6 @@ __all__ = [
# Ontology models
"OntologyClass",
"OntologyExtractionResponse",
# Ontology type models for extraction flow
"OntologyTypeInfo",
"OntologyTypeList",
# General ontology type models
"OntologyFileFormat",
"GeneralOntologyType",
"GeneralOntologyTypeRegistry",
# Variable configuration
"StatementExtractionConfig",
"ForgettingEngineConfig",

View File

@@ -10,7 +10,7 @@ Classes:
TemporalSearchParams: Parameters for temporal search queries
"""
from typing import Optional, List
from typing import Optional
from pydantic import BaseModel, Field
@@ -55,26 +55,17 @@ class PruningConfig(BaseModel):
Attributes:
pruning_switch: Enable or disable semantic pruning
pruning_scene: Scene name for pruning, either a built-in key
('education', 'online_service', 'outbound') or a custom scene_name
from ontology_scene table
pruning_scene: Scene type for pruning ('education', 'online_service', 'outbound')
pruning_threshold: Pruning ratio (0-0.9, max 0.9 to avoid complete removal)
scene_id: Optional ontology scene UUID, used to load custom ontology classes
ontology_classes: List of class_name strings from ontology_class table,
injected into the prompt when pruning_scene is not a built-in scene
"""
pruning_switch: bool = Field(False, description="Enable semantic pruning when True.")
pruning_scene: str = Field(
"education",
description="Scene for pruning: built-in key or custom scene_name from ontology_scene.",
description="Scene for pruning: one of 'education', 'online_service', 'outbound'.",
)
pruning_threshold: float = Field(
0.5, ge=0.0, le=0.9,
description="Pruning ratio within 0-0.9 (max 0.9 to avoid termination).")
scene_id: Optional[str] = Field(None, description="Ontology scene UUID (optional).")
ontology_classes: Optional[List[str]] = Field(
None, description="Class names from ontology_class table for custom scenes."
)
class TemporalSearchParams(BaseModel):

View File

@@ -1,105 +0,0 @@
# -*- coding: utf-8 -*-
"""本体类型数据结构模块
本模块定义用于在萃取流程中传递本体类型信息的轻量级数据类。
Classes:
OntologyTypeInfo: 单个本体类型信息
OntologyTypeList: 本体类型列表
"""
from dataclasses import dataclass
from typing import List
@dataclass
class OntologyTypeInfo:
"""本体类型信息,用于萃取流程中传递。
Attributes:
class_name: 类型名称
class_description: 类型描述
"""
class_name: str
class_description: str
def to_prompt_format(self) -> str:
"""转换为提示词格式。
Returns:
格式化的字符串,如 "- TypeName: Description"
"""
return f"- {self.class_name}: {self.class_description}"
@dataclass
class OntologyTypeList:
"""本体类型列表。
Attributes:
types: 本体类型信息列表
"""
types: List[OntologyTypeInfo]
@classmethod
def from_db_models(cls, ontology_classes: list) -> "OntologyTypeList":
"""从数据库模型转换创建 OntologyTypeList。
Args:
ontology_classes: OntologyClass 数据库模型列表,
每个对象应包含 class_name 和 class_description 属性
Returns:
包含转换后类型信息的 OntologyTypeList 实例
"""
types = [
OntologyTypeInfo(
class_name=oc.class_name,
class_description=oc.class_description or ""
)
for oc in ontology_classes
]
return cls(types=types)
def to_prompt_section(self) -> str:
"""转换为提示词中的类型列表部分。
Returns:
格式化的类型列表字符串,每行一个类型;
如果列表为空则返回空字符串
"""
if not self.types:
return ""
lines = [t.to_prompt_format() for t in self.types]
return "\n".join(lines)
def get_type_names(self) -> List[str]:
"""获取所有类型名称列表。
Returns:
类型名称字符串列表
"""
return [t.class_name for t in self.types]
def get_type_hierarchy_hints(self) -> List[str]:
"""获取类型层次结构提示列表。
尝试从通用本体注册表中获取每个类型的继承链信息。
Returns:
层次提示字符串列表,格式为 "类型名 → 父类1 → 父类2"
"""
hints = []
try:
from app.core.memory.ontology_services.ontology_type_merger import OntologyTypeMerger
merger = OntologyTypeMerger()
for type_info in self.types:
hint = merger.get_type_hierarchy_hint(type_info.class_name)
if hint:
hints.append(hint)
except Exception:
# 如果无法获取层次信息,返回空列表
pass
return hints

View File

@@ -1,223 +0,0 @@
# -*- coding: utf-8 -*-
"""通用本体类型数据模型模块
本模块定义用于通用本体类型管理的数据结构,包括:
- OntologyFileFormat: 本体文件格式枚举
- GeneralOntologyType: 通用本体类型数据类
- GeneralOntologyTypeRegistry: 通用本体类型注册表
Classes:
OntologyFileFormat: 本体文件格式枚举,支持 TTL、OWL/XML、RDF/XML、N-Triples、JSON-LD
GeneralOntologyType: 通用本体类型包含类名、URI、标签、描述、父类等信息
GeneralOntologyTypeRegistry: 类型注册表,管理类型集合和层次结构
"""
import logging
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Dict, List, Optional, Set
logger = logging.getLogger(__name__)
class OntologyFileFormat(Enum):
"""本体文件格式枚举
支持的格式:
- TURTLE: Turtle 格式 (.ttl 文件)
- RDF_XML: RDF/XML 格式 (.owl, .rdf 文件)
- N_TRIPLES: N-Triples 格式 (.nt 文件)
- JSON_LD: JSON-LD 格式 (.jsonld, .json 文件)
"""
TURTLE = "turtle" # .ttl 文件
RDF_XML = "xml" # .owl, .rdf (RDF/XML 格式)
N_TRIPLES = "nt" # .nt 文件
JSON_LD = "json-ld" # .jsonld 文件
@classmethod
def from_extension(cls, file_path: str) -> "OntologyFileFormat":
"""根据文件扩展名推断格式
Args:
file_path: 文件路径
Returns:
推断出的文件格式,默认返回 RDF_XML
"""
ext = file_path.lower().split('.')[-1]
format_map = {
'ttl': cls.TURTLE,
'owl': cls.RDF_XML,
'rdf': cls.RDF_XML,
'nt': cls.N_TRIPLES,
'jsonld': cls.JSON_LD,
'json': cls.JSON_LD,
}
return format_map.get(ext, cls.RDF_XML)
@dataclass
class GeneralOntologyType:
"""通用本体类型
表示从本体文件中解析出的类型定义,包含类型的基本信息和层次关系。
Attributes:
class_name: 类型名称,如 "Person"
class_uri: 完整 URI"http://dbpedia.org/ontology/Person"
labels: 多语言标签字典,键为语言代码(如 "en", "zh"),值为标签文本
description: 类型描述
parent_class: 父类名称,用于构建类型层次
source_file: 来源文件路径
"""
class_name: str # 类型名称,如 "Person"
class_uri: str # 完整 URI
labels: Dict[str, str] = field(default_factory=dict) # 多语言标签
description: Optional[str] = None # 类型描述
parent_class: Optional[str] = None # 父类名称
source_file: Optional[str] = None # 来源文件
def get_label(self, lang: str = "en") -> str:
"""获取指定语言的标签
优先返回指定语言的标签,如果不存在则尝试返回英文标签,
最后返回类型名称作为默认值。
Args:
lang: 语言代码,默认为 "en"
Returns:
指定语言的标签,或默认值
"""
return self.labels.get(lang, self.labels.get("en", self.class_name))
@dataclass
class GeneralOntologyTypeRegistry:
"""通用本体类型注册表
管理解析后的本体类型集合,提供类型查询、层次遍历、注册表合并等功能。
Attributes:
types: 类型字典,键为类型名称,值为 GeneralOntologyType 实例
hierarchy: 层次结构字典,键为父类名称,值为子类名称集合
source_files: 已加载的源文件路径列表
"""
types: Dict[str, GeneralOntologyType] = field(default_factory=dict)
hierarchy: Dict[str, Set[str]] = field(default_factory=dict) # 父类 -> 子类集合
source_files: List[str] = field(default_factory=list)
def get_type(self, name: str) -> Optional[GeneralOntologyType]:
"""根据名称获取类型
Args:
name: 类型名称
Returns:
对应的 GeneralOntologyType 实例,如果不存在则返回 None
"""
return self.types.get(name)
def get_ancestors(self, name: str) -> List[str]:
"""获取类型的所有祖先类型(防循环)
从当前类型开始,沿着父类链向上遍历,返回所有祖先类型名称。
使用 visited 集合防止循环引用导致的无限循环。
Args:
name: 类型名称
Returns:
祖先类型名称列表,按从近到远的顺序排列
"""
ancestors = []
current = name
visited = set()
while current and current not in visited:
visited.add(current)
type_info = self.types.get(current)
if type_info and type_info.parent_class:
# 检测循环引用
if type_info.parent_class in visited:
logger.warning(
f"检测到类型层次循环引用: {current} -> {type_info.parent_class}"
f"已遍历路径: {' -> '.join([name] + ancestors)}"
)
break
ancestors.append(type_info.parent_class)
current = type_info.parent_class
else:
break
return ancestors
def get_descendants(self, name: str) -> Set[str]:
"""获取类型的所有后代类型
从当前类型开始,沿着子类关系向下遍历,返回所有后代类型名称。
使用广度优先搜索,避免重复处理已访问的类型。
Args:
name: 类型名称
Returns:
后代类型名称集合
"""
descendants: Set[str] = set()
to_process = [name]
while to_process:
current = to_process.pop()
children = self.hierarchy.get(current, set())
new_children = children - descendants
descendants.update(new_children)
to_process.extend(new_children)
return descendants
def merge(self, other: "GeneralOntologyTypeRegistry") -> None:
"""合并另一个注册表(先加载的优先)
将另一个注册表的类型和层次结构合并到当前注册表。
对于同名类型,保留当前注册表中已存在的定义(先加载优先)。
层次结构会合并所有子类关系。
Args:
other: 要合并的另一个注册表
"""
for name, type_info in other.types.items():
if name not in self.types:
self.types[name] = type_info
for parent, children in other.hierarchy.items():
if parent not in self.hierarchy:
self.hierarchy[parent] = set()
self.hierarchy[parent].update(children)
self.source_files.extend(other.source_files)
def get_statistics(self) -> Dict[str, Any]:
"""获取注册表统计信息
Returns:
包含以下键的字典:
- total_types: 总类型数
- root_types: 根类型数(无父类的类型)
- max_depth: 类型层次的最大深度
- source_files: 源文件列表
"""
return {
"total_types": len(self.types),
"root_types": len([t for t in self.types.values() if not t.parent_class]),
"max_depth": self._calculate_max_depth(),
"source_files": self.source_files,
}
def _calculate_max_depth(self) -> int:
"""计算类型层次的最大深度
遍历所有类型,计算每个类型到根的深度,返回最大值。
Returns:
类型层次的最大深度
"""
max_depth = 0
for type_name in self.types:
depth = len(self.get_ancestors(type_name))
max_depth = max(max_depth, depth)
return max_depth

View File

@@ -74,7 +74,7 @@ class OntologyClass(BaseModel):
"""Validate that the class name follows PascalCase convention.
PascalCase rules:
- Must start with an uppercase letter (for English) or any character (for Chinese/Unicode)
- Must start with an uppercase letter
- Cannot contain spaces
- Should not contain special characters except underscores
@@ -90,10 +90,7 @@ class OntologyClass(BaseModel):
if not v:
raise ValueError("Class name cannot be empty")
# For Chinese/Unicode characters, skip the uppercase check
# Only check uppercase for ASCII letters
first_char = v[0]
if first_char.isascii() and first_char.isalpha() and not first_char.isupper():
if not v[0].isupper():
raise ValueError(
f"Class name '{v}' must start with an uppercase letter (PascalCase)"
)
@@ -103,11 +100,11 @@ class OntologyClass(BaseModel):
f"Class name '{v}' cannot contain spaces (PascalCase)"
)
# Check for invalid characters (allow alphanumeric, underscore, and Unicode characters)
if not all(c.isalnum() or c == '_' or ord(c) > 127 for c in v):
# Check for invalid characters (allow alphanumeric and underscore only)
if not all(c.isalnum() or c == '_' for c in v):
raise ValueError(
f"Class name '{v}' contains invalid characters. "
"Only alphanumeric characters, underscores, and Unicode characters are allowed"
"Only alphanumeric characters and underscores are allowed"
)
return v

File diff suppressed because it is too large Load Diff

View File

@@ -1,39 +0,0 @@
# -*- coding: utf-8 -*-
"""本体类型服务模块
本模块提供本体类型相关的服务,包括:
- OntologyTypeMerger: 本体类型合并服务
- get_general_ontology_registry: 获取通用本体类型注册表(单例,懒加载)
- get_ontology_type_merger: 获取类型合并服务实例
- reload_ontology_registry: 重新加载本体注册表(实验模式)
- clear_ontology_cache: 清除本体缓存
- is_general_ontology_enabled: 检查通用本体类型功能是否启用
- load_ontology_types_for_scene: 从数据库加载场景的本体类型
- create_empty_ontology_type_list: 创建空的本体类型列表
- load_ontology_types_with_fallback: 加载本体类型(带通用类型回退)
"""
from .ontology_type_merger import OntologyTypeMerger, DEFAULT_CORE_GENERAL_TYPES
from .ontology_type_loader import (
get_general_ontology_registry,
get_ontology_type_merger,
reload_ontology_registry,
clear_ontology_cache,
is_general_ontology_enabled,
load_ontology_types_for_scene,
create_empty_ontology_type_list,
load_ontology_types_with_fallback,
)
__all__ = [
"OntologyTypeMerger",
"DEFAULT_CORE_GENERAL_TYPES",
"get_general_ontology_registry",
"get_ontology_type_merger",
"reload_ontology_registry",
"clear_ontology_cache",
"is_general_ontology_enabled",
"load_ontology_types_for_scene",
"create_empty_ontology_type_list",
"load_ontology_types_with_fallback",
]

View File

@@ -1,270 +0,0 @@
"""本体类型加载器
提供统一的本体类型加载逻辑,避免代码重复。
Functions:
load_ontology_types_for_scene: 从数据库加载场景的本体类型
is_general_ontology_enabled: 检查是否启用通用本体
get_general_ontology_registry: 获取通用本体类型注册表(单例,懒加载)
get_ontology_type_merger: 获取类型合并服务实例
reload_ontology_registry: 重新加载本体注册表
clear_ontology_cache: 清除本体缓存
"""
import logging
import os
from typing import Optional
from uuid import UUID
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
# 模块级缓存(单例)
_general_registry_cache = None
_ontology_type_merger_cache = None
def load_ontology_types_for_scene(
scene_id: Optional[UUID],
workspace_id: UUID,
db: Session
) -> Optional["OntologyTypeList"]:
"""从数据库加载场景的本体类型
统一的本体类型加载逻辑,用于替代各处重复的加载代码。
Args:
scene_id: 场景ID如果为 None 则返回 None
workspace_id: 工作空间ID
db: 数据库会话
Returns:
OntologyTypeList 如果场景有类型定义,否则返回 None
Examples:
>>> ontology_types = load_ontology_types_for_scene(
... scene_id=scene_uuid,
... workspace_id=workspace_uuid,
... db=db_session
... )
>>> if ontology_types:
... print(f"Loaded {len(ontology_types.types)} types")
"""
if not scene_id:
return None
try:
from app.core.memory.models.ontology_extraction_models import OntologyTypeList
from app.repositories.ontology_class_repository import OntologyClassRepository
# 查询场景的本体类型
ontology_repo = OntologyClassRepository(db)
ontology_classes = ontology_repo.get_classes_by_scene(
scene_id=scene_id
)
if not ontology_classes:
logger.info(f"No ontology types found for scene_id: {scene_id}")
return None
# 转换为 OntologyTypeList
ontology_types = OntologyTypeList.from_db_models(ontology_classes)
logger.info(
f"Loaded {len(ontology_types.types)} ontology types for scene_id: {scene_id}"
)
return ontology_types
except Exception as e:
logger.error(f"Failed to load ontology types for scene_id {scene_id}: {e}", exc_info=True)
return None
def create_empty_ontology_type_list() -> Optional["OntologyTypeList"]:
"""创建空的本体类型列表(用于仅使用通用类型的场景)
Returns:
空的 OntologyTypeList 如果通用本体已启用,否则返回 None
"""
try:
from app.core.memory.models.ontology_extraction_models import OntologyTypeList
if is_general_ontology_enabled():
logger.info("Creating empty OntologyTypeList for general types only")
return OntologyTypeList(types=[])
return None
except Exception as e:
logger.warning(f"Failed to create empty OntologyTypeList: {e}")
return None
def is_general_ontology_enabled() -> bool:
"""检查是否启用了通用本体
通过配置开关和注册表是否可用来判断。
Returns:
True 如果通用本体已启用,否则 False
"""
try:
from app.core.config import settings
if not settings.ENABLE_GENERAL_ONTOLOGY_TYPES:
return False
registry = get_general_ontology_registry()
return registry is not None and len(registry.types) > 0
except Exception as e:
logger.warning(f"Failed to check general ontology status: {e}")
return False
def get_general_ontology_registry():
"""获取通用本体类型注册表(单例,懒加载)
从配置的本体文件中解析并缓存注册表。
Returns:
GeneralOntologyTypeRegistry 实例,如果加载失败则返回 None
"""
global _general_registry_cache
if _general_registry_cache is not None:
return _general_registry_cache
try:
from app.core.config import settings
if not settings.ENABLE_GENERAL_ONTOLOGY_TYPES:
logger.info("通用本体类型功能已禁用")
return None
# 解析本体文件路径
file_names = [f.strip() for f in settings.GENERAL_ONTOLOGY_FILES.split(",") if f.strip()]
if not file_names:
logger.warning("未配置通用本体文件")
return None
# 构建完整路径(相对于项目根目录)
base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
file_paths = []
for name in file_names:
full_path = os.path.join(base_dir, name)
if os.path.exists(full_path):
file_paths.append(full_path)
else:
logger.warning(f"本体文件不存在: {full_path}")
if not file_paths:
logger.warning("没有找到可用的通用本体文件")
return None
# 解析本体文件
from app.core.memory.utils.ontology.ontology_parser import MultiOntologyParser
parser = MultiOntologyParser(file_paths)
_general_registry_cache = parser.parse_all()
logger.info(f"通用本体注册表加载完成: {len(_general_registry_cache.types)} 个类型")
return _general_registry_cache
except Exception as e:
logger.error(f"加载通用本体注册表失败: {e}", exc_info=True)
return None
def get_ontology_type_merger():
"""获取类型合并服务实例(单例,懒加载)
Returns:
OntologyTypeMerger 实例,如果通用本体未启用则返回 None
"""
global _ontology_type_merger_cache
if _ontology_type_merger_cache is not None:
return _ontology_type_merger_cache
try:
registry = get_general_ontology_registry()
if registry is None:
return None
from app.core.config import settings
from app.core.memory.ontology_services.ontology_type_merger import OntologyTypeMerger
# 从配置读取核心类型
core_types_str = settings.CORE_GENERAL_TYPES
core_types = [t.strip() for t in core_types_str.split(",") if t.strip()] if core_types_str else None
_ontology_type_merger_cache = OntologyTypeMerger(
general_registry=registry,
max_types_in_prompt=settings.MAX_ONTOLOGY_TYPES_IN_PROMPT,
core_types=core_types,
)
logger.info("OntologyTypeMerger 实例创建完成")
return _ontology_type_merger_cache
except Exception as e:
logger.error(f"创建 OntologyTypeMerger 失败: {e}", exc_info=True)
return None
def reload_ontology_registry():
"""重新加载本体注册表(清除缓存后重新加载)
用于实验模式下动态更新本体配置。
"""
clear_ontology_cache()
registry = get_general_ontology_registry()
if registry:
get_ontology_type_merger()
logger.info("本体注册表已重新加载")
return registry
def clear_ontology_cache():
"""清除本体缓存"""
global _general_registry_cache, _ontology_type_merger_cache
_general_registry_cache = None
_ontology_type_merger_cache = None
logger.info("本体缓存已清除")
def load_ontology_types_with_fallback(
scene_id: Optional[UUID],
workspace_id: UUID,
db: Session,
enable_general_fallback: bool = True
) -> Optional["OntologyTypeList"]:
"""加载本体类型,如果场景没有类型则回退到通用类型
这是一个便捷函数,组合了场景类型加载和通用类型回退逻辑。
Args:
scene_id: 场景ID
workspace_id: 工作空间ID
db: 数据库会话
enable_general_fallback: 是否在没有场景类型时启用通用类型回退
Returns:
OntologyTypeList 或 None
"""
# 首先尝试加载场景类型
ontology_types = load_ontology_types_for_scene(
scene_id=scene_id,
workspace_id=workspace_id,
db=db
)
# 如果没有场景类型且启用了回退,创建空列表以使用通用类型
if ontology_types is None and enable_general_fallback:
ontology_types = create_empty_ontology_type_list()
if ontology_types:
logger.info("No scene ontology types, will use general ontology types only")
return ontology_types

View File

@@ -1,231 +0,0 @@
# -*- coding: utf-8 -*-
"""本体类型合并服务模块
本模块实现本体类型合并服务,负责按优先级合并场景类型与通用类型。
合并优先级:
1. 场景特定类型(最高优先级)
2. 核心通用类型
3. 相关父类类型(最低优先级)
Classes:
OntologyTypeMerger: 本体类型合并服务类
Constants:
DEFAULT_CORE_GENERAL_TYPES: 默认核心通用类型集合
"""
import logging
from typing import List, Optional, Set
from app.core.memory.models.ontology_general_models import GeneralOntologyTypeRegistry
from app.core.memory.models.ontology_extraction_models import OntologyTypeInfo, OntologyTypeList
logger = logging.getLogger(__name__)
# 默认核心通用类型
DEFAULT_CORE_GENERAL_TYPES: Set[str] = {
"Person", "Organization", "Company", "GovernmentAgency",
"Place", "Location", "City", "Country", "Building",
"Event", "SportsEvent", "MusicEvent", "SocialEvent",
"Work", "Book", "Film", "Software", "Album",
"Concept", "TopicalConcept", "AcademicSubject",
"Device", "Food", "Drug", "ChemicalSubstance",
"TimePeriod", "Year",
}
class OntologyTypeMerger:
"""本体类型合并服务
负责按优先级合并场景类型与通用类型,生成用于三元组提取的类型列表。
合并优先级:
1. 场景特定类型(最高优先级)- 标记为 [场景类型]
2. 核心通用类型 - 标记为 [通用类型]
3. 相关父类类型(最低优先级)- 标记为 [通用父类]
Attributes:
general_registry: 通用本体类型注册表
max_types_in_prompt: Prompt 中最大类型数量限制
core_types: 核心通用类型集合
Example:
>>> registry = GeneralOntologyTypeRegistry()
>>> merger = OntologyTypeMerger(registry, max_types_in_prompt=50)
>>> merged = merger.merge(scene_types)
>>> print(len(merged.types))
"""
def __init__(
self,
general_registry: GeneralOntologyTypeRegistry,
max_types_in_prompt: int = 50,
core_types: Optional[List[str]] = None
):
"""初始化本体类型合并服务
Args:
general_registry: 通用本体类型注册表
max_types_in_prompt: Prompt 中最大类型数量,默认 50
core_types: 自定义核心类型列表,如果为 None 则使用默认核心类型
"""
self.general_registry = general_registry
self.max_types_in_prompt = max_types_in_prompt
self.core_types: Set[str] = set(core_types) if core_types else DEFAULT_CORE_GENERAL_TYPES.copy()
def update_core_types(self, core_types: List[str]) -> None:
"""动态更新核心类型列表
更新后立即生效,无需重启服务。
Args:
core_types: 新的核心类型列表
"""
self.core_types = set(core_types)
logger.info(f"核心类型已更新: {len(self.core_types)} 个类型")
def merge(
self,
scene_types: Optional[OntologyTypeList],
include_related_types: bool = True
) -> OntologyTypeList:
"""合并场景类型与通用类型
按优先级合并类型:
1. 场景特定类型(最高优先级)
2. 核心通用类型
3. 相关父类类型(可选)
合并后的类型总数不超过 max_types_in_prompt。
Args:
scene_types: 场景特定类型列表,可以为 None
include_related_types: 是否包含相关父类类型,默认 True
Returns:
合并后的类型列表,每个类型带有来源标记
"""
merged_types: List[OntologyTypeInfo] = []
seen_names: Set[str] = set()
# 1. 场景特定类型(最高优先级)
scene_type_count = 0
if scene_types and scene_types.types:
for scene_type in scene_types.types:
if scene_type.class_name not in seen_names:
merged_types.append(OntologyTypeInfo(
class_name=scene_type.class_name,
class_description=f"[场景类型] {scene_type.class_description}"
))
seen_names.add(scene_type.class_name)
scene_type_count += 1
# 2. 核心通用类型
remaining_slots = self.max_types_in_prompt - len(merged_types)
core_types_added: List[OntologyTypeInfo] = []
for type_name in self.core_types:
if type_name not in seen_names and remaining_slots > 0:
general_type = self.general_registry.get_type(type_name)
if general_type:
description = (
general_type.labels.get("zh") or
general_type.description or
general_type.get_label("en") or
type_name
)
core_types_added.append(OntologyTypeInfo(
class_name=type_name,
class_description=f"[通用类型] {description}"
))
seen_names.add(type_name)
remaining_slots -= 1
merged_types.extend(core_types_added)
# 3. 相关父类类型
related_types_added: List[OntologyTypeInfo] = []
if include_related_types and scene_types and scene_types.types:
for scene_type in scene_types.types:
if remaining_slots <= 0:
break
general_type = self.general_registry.get_type(scene_type.class_name)
if general_type and general_type.parent_class:
parent_name = general_type.parent_class
if parent_name not in seen_names:
parent_type = self.general_registry.get_type(parent_name)
if parent_type:
description = (
parent_type.labels.get("zh") or
parent_type.description or
parent_name
)
related_types_added.append(OntologyTypeInfo(
class_name=parent_name,
class_description=f"[通用父类] {description}"
))
seen_names.add(parent_name)
remaining_slots -= 1
merged_types.extend(related_types_added)
logger.info(
f"类型合并完成: 场景类型 {scene_type_count} 个, "
f"核心通用类型 {len(core_types_added)} 个, "
f"相关类型 {len(related_types_added)} 个, "
f"总计 {len(merged_types)}"
)
return OntologyTypeList(types=merged_types)
def get_type_hierarchy_hint(self, type_name: str) -> Optional[str]:
"""获取类型的层次提示信息(最多 3 级)
返回类型的继承链信息,格式为 "类型名 → 父类1 → 父类2 → 父类3"
Args:
type_name: 类型名称
Returns:
层次提示字符串,如果类型不存在或没有父类则返回 None
"""
general_type = self.general_registry.get_type(type_name)
if not general_type:
return None
ancestors = self.general_registry.get_ancestors(type_name)
if ancestors:
# 限制最多 3 级祖先
return f"{type_name}{''.join(ancestors[:3])}"
return None
def get_merge_statistics(self, scene_types: Optional[OntologyTypeList]) -> dict:
"""获取合并统计信息
执行合并操作并返回各类型来源的数量统计。
Args:
scene_types: 场景特定类型列表
Returns:
包含以下键的统计字典:
- total_types: 合并后总类型数
- scene_types: 场景类型数量
- general_types: 通用类型数量
- parent_types: 父类类型数量
- available_core_types: 可用核心类型数量
- registry_total_types: 注册表中总类型数
"""
merged = self.merge(scene_types)
scene_count = sum(1 for t in merged.types if "[场景类型]" in t.class_description)
general_count = sum(1 for t in merged.types if "[通用类型]" in t.class_description)
parent_count = sum(1 for t in merged.types if "[通用父类]" in t.class_description)
return {
"total_types": len(merged.types),
"scene_types": scene_count,
"general_types": general_count,
"parent_types": parent_count,
"available_core_types": len(self.core_types),
"registry_total_types": len(self.general_registry.types),
}

View File

@@ -5,27 +5,20 @@
- 对话级一次性抽取判定相关性
- 仅对"不相关对话"的消息按比例删除
- 重要信息(时间、编号、金额、联系方式、地址等)优先保留
- 改进版:增强重要性判断、智能填充消息识别、问答对保护、并发优化
"""
import asyncio
import os
import hashlib
import json
import re
from collections import OrderedDict
from datetime import datetime
from typing import List, Optional, Dict, Tuple, Set
from typing import List, Optional
from pydantic import BaseModel, Field
from app.core.memory.models.message_models import DialogData, ConversationMessage, ConversationContext
from app.core.memory.models.config_models import PruningConfig
from app.core.memory.utils.config.config_utils import get_pruning_config
from app.core.memory.utils.prompt.prompt_utils import prompt_env, log_prompt_rendering, log_template_rendering
from app.core.memory.storage_services.extraction_engine.data_preprocessing.scene_config import (
SceneConfigRegistry,
ScenePatterns
)
class DialogExtractionResponse(BaseModel):
@@ -33,7 +26,6 @@ class DialogExtractionResponse(BaseModel):
- is_related对话与场景的相关性判定。
- times / ids / amounts / contacts / addresses / keywords重要信息片段用来在不相关对话中保留关键消息。
- preserve_keywords情绪/兴趣/爱好/个人观点相关词,包含这些词的消息必须强制保留。
"""
is_related: bool = Field(...)
times: List[str] = Field(default_factory=list)
@@ -42,24 +34,6 @@ class DialogExtractionResponse(BaseModel):
contacts: List[str] = Field(default_factory=list)
addresses: List[str] = Field(default_factory=list)
keywords: List[str] = Field(default_factory=list)
preserve_keywords: List[str] = Field(default_factory=list, description="情绪/兴趣/爱好/个人观点相关词,包含这些词的消息强制保留")
class MessageImportanceResponse(BaseModel):
"""消息重要性批量判断的结构化返回用于LLM语义判断
- importance_scores: 消息索引到重要性分数的映射 (0-10分)
- reasons: 可选的判断理由
"""
importance_scores: Dict[int, int] = Field(default_factory=dict, description="消息索引到重要性分数(0-10)的映射")
reasons: Optional[Dict[int, str]] = Field(default_factory=dict, description="可选的判断理由")
class QAPair(BaseModel):
"""问答对模型,用于识别和保护对话中的问答结构。"""
question_idx: int = Field(..., description="问题消息的索引")
answer_idx: int = Field(..., description="答案消息的索引")
confidence: float = Field(default=1.0, description="问答对的置信度(0-1)")
class SemanticPruner:
@@ -69,280 +43,109 @@ class SemanticPruner:
重要信息(时间、编号、金额、联系方式、地址等)优先保留。
"""
def __init__(self, config: Optional[PruningConfig] = None, llm_client=None, language: str = "zh", max_concurrent: int = 5):
# 如果没有提供config使用默认配置
if config is None:
# 使用默认的剪枝配置
config = PruningConfig(
pruning_switch=False, # 默认关闭剪枝,保持向后兼容
pruning_scene="education",
pruning_threshold=0.5
)
self.config = config
def __init__(self, config: Optional[PruningConfig] = None, llm_client=None):
cfg_dict = get_pruning_config() if config is None else config.model_dump()
self.config = PruningConfig.model_validate(cfg_dict)
self.llm_client = llm_client
self.language = language # 保存语言配置
self.max_concurrent = max_concurrent # 新增:最大并发数
# 详细日志配置:限制逐条消息日志的数量
self._detailed_prune_logging = True # 是否启用详细日志
self._max_debug_msgs_per_dialog = 20 # 每个对话最多记录前N条消息的详细日志
# 加载统一填充词库
self.scene_config: ScenePatterns = SceneConfigRegistry.get_config(self.config.pruning_scene)
# 本体类型列表(用于注入提示词,所有场景均支持)
self._ontology_classes = getattr(self.config, "ontology_classes", None) or []
self._log(f"[剪枝-初始化] 场景={self.config.pruning_scene}")
if self._ontology_classes:
self._log(f"[剪枝-初始化] 注入本体类型: {self._ontology_classes}")
else:
self._log(f"[剪枝-初始化] 未找到本体类型,将使用通用提示词")
# Load Jinja2 template
self.template = prompt_env.get_template("extracat_Pruning.jinja2")
# 对话抽取缓存:使用 OrderedDict 实现 LRU 缓存
self._dialog_extract_cache: OrderedDict[str, DialogExtractionResponse] = OrderedDict()
self._cache_max_size = 1000 # 缓存大小限制
# 对话抽取缓存:避免同一对话重复调用 LLM / 重复渲染
self._dialog_extract_cache: dict[str, DialogExtractionResponse] = {}
# 运行日志:收集关键终端输出,便于写入 JSON
self.run_logs: List[str] = []
# 采用顺序处理,移除并发配置以简化与稳定执行
# _is_important_message 和 _importance_score 已移除:
# 重要性判断完全由 extracat_Pruning.jinja2 提示词 + LLM 的 preserve_tokens 机制承担。
# LLM 根据注入的本体工程类型语义识别需要保护的内容,无需硬编码正则规则。
def _is_important_message(self, message: ConversationMessage) -> bool:
"""基于启发式规则识别重要信息消息,优先保留。
- 含日期/时间如YYYY-MM-DD、HH:MM、2024年11月10日、上午/下午)。
- 含编号/ID/订单号/申请号/账号/电话/金额等关键字段。
- 关键词:"时间""日期""编号""订单""流水""金额""""""电话""手机号""邮箱""地址"
"""
import re
text = message.msg.strip()
if not text:
return False
patterns = [
r"\b\d{4}-\d{1,2}-\d{1,2}\b",
r"\b\d{1,2}:\d{2}\b",
r"\d{4}\d{1,2}月\d{1,2}日",
r"上午|下午|AM|PM",
r"订单号|工单|申请号|编号|ID|账号|账户",
r"电话|手机号|微信|QQ|邮箱",
r"地址|地点",
r"金额|费用|价格|¥|¥|\d+元",
r"时间|日期|有效期|截止",
]
for p in patterns:
if re.search(p, text, flags=re.IGNORECASE):
return True
return False
def _importance_score(self, message: ConversationMessage) -> int:
"""为重要消息打分,用于在保留比例内优先保留更关键的内容。
简单启发:匹配到的类别越多、越关键分值越高。
"""
import re
text = message.msg.strip()
score = 0
weights = [
(r"\b\d{4}-\d{1,2}-\d{1,2}\b", 3),
(r"\b\d{1,2}:\d{2}\b", 2),
(r"\d{4}\d{1,2}月\d{1,2}日", 3),
(r"订单号|工单|申请号|编号|ID|账号|账户", 4),
(r"电话|手机号|微信|QQ|邮箱", 3),
(r"地址|地点", 2),
(r"金额|费用|价格|¥|¥|\d+元", 4),
(r"时间|日期|有效期|截止", 2),
]
for p, w in weights:
if re.search(p, text, flags=re.IGNORECASE):
score += w
return score
def _is_filler_message(self, message: ConversationMessage) -> bool:
"""检测典型寒暄/口头禅/确认类短消息。
"""检测典型寒暄/口头禅/确认类短消息用于跳过LLM分类以加速
判断顺序
1. 空消息
2. 场景特定填充词库精确匹配
3. 常见寒暄精确匹配
4. 纯表情/标点
满足以下之一视为填充消息
- 纯标点或长度很短(<= 4 个汉字或 <= 8 个字符)且不包含数字或关键实体;
- 常见词:你好/您好/在吗/嗯/嗯嗯/哦/好的/好/行/可以/不可以/谢谢/拜拜/再见/哈哈/呵呵/哈哈哈/。。。/??。
"""
import re
t = message.msg.strip()
if not t:
return True
# 检查是否在场景特定填充词库中(精确匹配)
if t in self.scene_config.filler_phrases:
# 常见填充语
fillers = [
"你好", "您好", "在吗", "", "嗯嗯", "", "好的", "", "", "可以", "不可以", "谢谢",
"拜拜", "再见", "哈哈", "呵呵", "哈哈哈", "。。。", "??", ""
]
if t in fillers:
return True
# 常见寒暄和问候(精确匹配,避免误删)
common_greetings = {
"在吗", "在不在", "在呢", "在的",
"你好", "您好", "hello", "hi",
"拜拜", "再见", "", "88", "bye",
"好的", "", "", "可以", "", "", "",
"是的", "", "对的", "没错", "是啊",
"哈哈", "呵呵", "嘿嘿", "嗯嗯"
}
if t in common_greetings:
return True
# 检查是否为纯表情符号(方括号包裹)
if re.fullmatch(r"(\[[^\]]+\])+", t):
return True
# 纯标点符号
if re.fullmatch(r"[。!?,.!?…·\s]+", t):
return True
# 长度与字符类型判断
if len(t) <= 8:
# 非数字、无关键实体的短文本
if not re.search(r"[0-9]", t) and not self._is_important_message(message):
# 主要是标点或简单确认词
if re.fullmatch(r"[。!?,.!?…·\s]+", t) or t in fillers:
return True
return False
async def _batch_evaluate_importance_with_llm(
self,
messages: List[ConversationMessage],
context: str = ""
) -> Dict[int, int]:
"""使用LLM批量评估消息的重要性语义层面
Args:
messages: 消息列表
context: 对话上下文(可选)
Returns:
消息索引到重要性分数(0-10)的映射
"""
if not self.llm_client or not messages:
return {}
# 构建批量评估的提示词
msg_list = []
for idx, msg in enumerate(messages):
msg_list.append(f"{idx}. {msg.msg}")
msg_text = "\n".join(msg_list)
prompt = f"""请评估以下消息的重要性给每条消息打分0-10分
- 0-2分无意义的寒暄、口头禅、纯表情
- 3-5分一般性对话有一定信息量但不关键
- 6-8分包含重要信息时间、地点、人物、事件等
- 9-10分关键决策、承诺、重要数据
对话上下文:
{context if context else ""}
待评估的消息:
{msg_text}
请以JSON格式返回格式为
{{
"importance_scores": {{
"0": 分数,
"1": 分数,
...
}}
}}
"""
try:
messages_for_llm = [
{"role": "system", "content": "你是一个专业的对话分析助手,擅长评估消息的重要性。"},
{"role": "user", "content": prompt}
]
response = await self.llm_client.response_structured(
messages_for_llm,
MessageImportanceResponse
)
# 转换字符串键为整数键
return {int(k): v for k, v in response.importance_scores.items()}
except Exception as e:
self._log(f"[剪枝-LLM] 批量重要性评估失败: {str(e)[:100]}")
return {}
def _identify_qa_pairs(self, messages: List[ConversationMessage]) -> List[QAPair]:
"""识别对话中的问答对,用于保护问答结构的完整性。
改进版:使用场景特定的问句关键词,并排除寒暄类问句
Args:
messages: 消息列表
Returns:
问答对列表
"""
qa_pairs = []
# 寒暄类问句,不应该被保护(这些不是真正的问答)
greeting_questions = {
"在吗", "在不在", "你好吗", "怎么样", "好吗",
"有空吗", "忙吗", "睡了吗", "起床了吗"
}
for i in range(len(messages) - 1):
current_msg = messages[i].msg.strip()
next_msg = messages[i + 1].msg.strip()
# 排除寒暄类问句
if current_msg in greeting_questions:
continue
# 使用场景特定的问句关键词,但要求更严格
is_question = False
# 1. 以问号结尾
if current_msg.endswith("") or current_msg.endswith("?"):
is_question = True
# 2. 包含实质性问句关键词(排除"吗"这种太宽泛的)
elif any(word in current_msg for word in ["什么", "为什么", "怎么", "如何", "哪里", "哪个", "", "多少", "几点", "何时"]):
is_question = True
if is_question and next_msg:
# 检查下一条消息是否像答案(不是另一个问句,也不是寒暄)
is_answer = not (next_msg.endswith("") or next_msg.endswith("?"))
# 排除寒暄类回复
greeting_answers = {"你好", "您好", "在呢", "在的", "", "", "好的"}
if next_msg in greeting_answers:
is_answer = False
if is_answer:
qa_pairs.append(QAPair(
question_idx=i,
answer_idx=i + 1,
confidence=0.8 # 基于规则的置信度
))
return qa_pairs
def _get_protected_indices(
self,
messages: List[ConversationMessage],
qa_pairs: List[QAPair],
window_size: int = 2
) -> Set[int]:
"""获取需要保护的消息索引集合(问答对+上下文窗口)。
Args:
messages: 消息列表
qa_pairs: 问答对列表
window_size: 上下文窗口大小(前后各保留几条消息)
Returns:
需要保护的消息索引集合
"""
protected = set()
for qa_pair in qa_pairs:
# 保护问答对本身
protected.add(qa_pair.question_idx)
protected.add(qa_pair.answer_idx)
# 保护上下文窗口
for offset in range(-window_size, window_size + 1):
q_idx = qa_pair.question_idx + offset
a_idx = qa_pair.answer_idx + offset
if 0 <= q_idx < len(messages):
protected.add(q_idx)
if 0 <= a_idx < len(messages):
protected.add(a_idx)
return protected
async def _extract_dialog_important(self, dialog_text: str) -> DialogExtractionResponse:
"""对话级一次性抽取:从整段对话中提取重要信息并判定相关性。
改进版:
- LRU缓存管理
- 重试机制
- 降级策略
- 仅使用 LLM 结构化输出;
"""
# 缓存命中则直接返回(场景+内容作为键)
cache_key = f"{self.config.pruning_scene}:" + hashlib.sha1(dialog_text.encode("utf-8")).hexdigest()
# LRU缓存如果命中移到末尾最近使用
if cache_key in self._dialog_extract_cache:
self._dialog_extract_cache.move_to_end(cache_key)
return self._dialog_extract_cache[cache_key]
# LRU缓存大小限制超过限制时删除最旧的条目
if len(self._dialog_extract_cache) >= self._cache_max_size:
# 删除最旧的条目OrderedDict的第一个
oldest_key = next(iter(self._dialog_extract_cache))
del self._dialog_extract_cache[oldest_key]
self._log(f"[剪枝-缓存] LRU缓存已满删除最旧条目")
rendered = self.template.render(
pruning_scene=self.config.pruning_scene,
ontology_classes=self._ontology_classes,
dialog_text=dialog_text,
language=self.language
)
log_template_rendering("extracat_Pruning.jinja2", {
"pruning_scene": self.config.pruning_scene,
"ontology_classes_count": len(self._ontology_classes),
"language": self.language
})
rendered = self.template.render(pruning_scene=self.config.pruning_scene, dialog_text=dialog_text)
log_template_rendering("extracat_Pruning.jinja2", {"pruning_scene": self.config.pruning_scene})
log_prompt_rendering("pruning-extract", rendered)
# 强制使用 LLM
# 强制使用 LLM;移除正则回退
if not self.llm_client:
raise RuntimeError("llm_client 未配置;请配置 LLM 以进行结构化抽取。")
@@ -350,32 +153,12 @@ class SemanticPruner:
{"role": "system", "content": "你是一个严谨的场景抽取助手,只输出严格 JSON。"},
{"role": "user", "content": rendered},
]
# 重试机制
max_retries = 3
for attempt in range(max_retries):
try:
ex = await self.llm_client.response_structured(messages, DialogExtractionResponse)
self._dialog_extract_cache[cache_key] = ex
return ex
except Exception as e:
if attempt < max_retries - 1:
self._log(f"[剪枝-LLM] 第 {attempt + 1} 次尝试失败,重试中... 错误: {str(e)[:100]}")
await asyncio.sleep(0.5 * (attempt + 1)) # 指数退避
continue
else:
# 降级策略:标记为相关,避免误删
self._log(f"[剪枝-LLM] LLM 调用失败 {max_retries} 次,使用降级策略(标记为相关)")
fallback_response = DialogExtractionResponse(
is_related=True,
times=[],
ids=[],
amounts=[],
contacts=[],
addresses=[],
keywords=[]
)
return fallback_response
try:
ex = await self.llm_client.response_structured(messages, DialogExtractionResponse)
self._dialog_extract_cache[cache_key] = ex
return ex
except Exception as e:
raise RuntimeError("LLM 结构化抽取失败;请检查 LLM 配置或重试。") from e
def _msg_matches_tokens(self, message: ConversationMessage, tokens: List[str]) -> bool:
"""判断消息是否包含任意抽取到的重要片段。"""
@@ -401,56 +184,62 @@ class SemanticPruner:
# 相关对话不剪枝
return dialog
# 在不相关对话中,LLM 已通过 preserve_tokens 标记需要保护的内容
preserve_tokens = (
extraction.times + extraction.ids + extraction.amounts +
extraction.contacts + extraction.addresses + extraction.keywords +
extraction.preserve_keywords
)
# 在不相关对话中,识别重要/不重要消息
tokens = extraction.times + extraction.ids + extraction.amounts + extraction.contacts + extraction.addresses + extraction.keywords
msgs = dialog.context.msgs
# 分类:填充 / 其他可删LLM保护消息通过不加入任何桶来隐式保护
filler_ids: set = set()
deletable: List[ConversationMessage] = []
imp_unrel_msgs: List[ConversationMessage] = []
unimp_unrel_msgs: List[ConversationMessage] = []
for m in msgs:
if self._msg_matches_tokens(m, preserve_tokens):
pass # 保护消息:不加入任何桶,不会被删除
elif self._is_filler_message(m):
filler_ids.add(id(m))
if self._msg_matches_tokens(m, tokens) or self._is_important_message(m):
imp_unrel_msgs.append(m)
else:
deletable.append(m)
# 计算删除目标
unimp_unrel_msgs.append(m)
# 计算总删除目标数量
total_unrel = len(msgs)
delete_target = int(total_unrel * proportion)
if proportion > 0 and total_unrel > 0 and delete_target == 0:
delete_target = 1
max_deletable = min(len(filler_ids) + len(deletable), max(0, total_unrel - 1))
imp_del_cap = min(int(len(imp_unrel_msgs) * proportion), len(imp_unrel_msgs))
unimp_del_cap = len(unimp_unrel_msgs)
max_capacity = max(0, len(msgs) - 1)
max_deletable = min(imp_del_cap + unimp_del_cap, max_capacity)
delete_target = min(delete_target, max_deletable)
# 删除配额分配
del_unimp = min(delete_target, unimp_del_cap)
rem = delete_target - del_unimp
del_imp = min(rem, imp_del_cap)
# 优先删填充,再删其他可删消息(按出现顺序)
to_delete_ids: set = set()
# 选取删除集合
unimp_delete_ids = []
imp_delete_ids = []
if del_unimp > 0:
# 按出现顺序选取前 del_unimp 条不重要消息进行删除(确定性、可复现)
unimp_delete_ids = [id(m) for m in unimp_unrel_msgs[:del_unimp]]
if del_imp > 0:
imp_sorted = sorted(imp_unrel_msgs, key=lambda m: self._importance_score(m))
imp_delete_ids = [id(m) for m in imp_sorted[:del_imp]]
# 统计实际删除数量(重要/不重要)
actual_unimp_deleted = 0
actual_imp_deleted = 0
kept_msgs = []
delete_targets = set(unimp_delete_ids) | set(imp_delete_ids)
for m in msgs:
if len(to_delete_ids) >= delete_target:
break
if id(m) in filler_ids:
to_delete_ids.add(id(m))
for m in deletable:
if len(to_delete_ids) >= delete_target:
break
to_delete_ids.add(id(m))
kept_msgs = [m for m in msgs if id(m) not in to_delete_ids]
mid = id(m)
if mid in delete_targets:
if mid in set(unimp_delete_ids) and actual_unimp_deleted < del_unimp:
actual_unimp_deleted += 1
continue
if mid in set(imp_delete_ids) and actual_imp_deleted < del_imp:
actual_imp_deleted += 1
continue
kept_msgs.append(m)
if not kept_msgs and msgs:
kept_msgs = [msgs[0]]
deleted_total = len(msgs) - len(kept_msgs)
protected_count = len(msgs) - len(filler_ids) - len(deletable)
deleted_total = actual_unimp_deleted + actual_imp_deleted
self._log(
f"[剪枝-对话] 对话ID={dialog.id} 总消息={len(msgs)} "
f"(保护={protected_count} 填充={len(filler_ids)} 可删={len(deletable)}) "
f"删除目标={delete_target} 实删={deleted_total} 保留={len(kept_msgs)}"
f"[剪枝-对话] 对话ID={dialog.id} 总消息={len(msgs)} 删除目标={delete_target} 实删={deleted_total} 保留={len(kept_msgs)}"
)
dialog.context = ConversationContext(msgs=kept_msgs)
@@ -459,14 +248,12 @@ class SemanticPruner:
async def prune_dataset(self, dialogs: List[DialogData]) -> List[DialogData]:
"""数据集层面:全局消息级剪枝,保留所有对话。
改进版:
- 消息级独立判断,每条消息根据场景规则独立评估
- 问答对保护已注释(暂不启用,留作观察)
- 优化删除策略:填充消息 → 不重要消息 → 低分重要消息
- 只删除"不重要的不相关消息",重要信息(时间、编号等)强制保留
- 保证每段对话至少保留1条消息不会删除整段对话
- 仅在"不相关对话"的范围内执行消息剪枝;相关对话不动。
- 只删除"不重要的不相关消息",重要信息(时间、编号等)强制保留。
- 删除总量 = 阈值 * 全部不相关可删消息数,按可删容量比例分配;顺序删除。
- 保证每段对话至少保留1条消息不会删除整段对话。
"""
# 如果剪枝功能关闭,直接返回原始数据集
# 如果剪枝功能关闭,直接返回原始数据集
if not self.config.pruning_switch:
return dialogs
@@ -477,139 +264,179 @@ class SemanticPruner:
proportion = 0.9
if proportion < 0.0:
proportion = 0.0
evaluated_dialogs = [] # list of dicts: {dialog, is_related}
self._log(
f"[剪枝-数据集] 对话总数={len(dialogs)} 场景={self.config.pruning_scene} 删除比例={proportion} 开关={self.config.pruning_switch} 模式=消息级独立判断"
f"[剪枝-数据集] 对话总数={len(dialogs)} 场景={self.config.pruning_scene} 删除比例={proportion} 开关={self.config.pruning_switch}"
)
# 对话级相关性分类(一次性对整段对话文本进行判断,顺序执行并复用缓存)
evaluated_dialogs = []
for idx, dd in enumerate(dialogs):
try:
ex = await self._extract_dialog_important(dd.content)
evaluated_dialogs.append({
"dialog": dd,
"is_related": bool(ex.is_related),
"index": idx,
"extraction": ex
})
except Exception:
evaluated_dialogs.append({
"dialog": dd,
"is_related": True,
"index": idx,
"extraction": None
})
# 统计相关 / 不相关对话
not_related_dialogs = [d for d in evaluated_dialogs if not d["is_related"]]
related_dialogs = [d for d in evaluated_dialogs if d["is_related"]]
self._log(
f"[剪枝-数据集] 相关对话数={len(related_dialogs)} 不相关对话数={len(not_related_dialogs)}"
)
# 简洁打印第几段对话相关/不相关索引基于1
def _fmt_indices(items, cap: int = 10):
inds = [i["index"] + 1 for i in items]
if len(inds) <= cap:
return inds
# 超过上限时只打印前cap个并标注总数
return inds[:cap] + ["...", f"{len(inds)}"]
rel_inds = _fmt_indices(related_dialogs)
nrel_inds = _fmt_indices(not_related_dialogs)
self._log(f"[剪枝-数据集] 相关对话:第{rel_inds}段;不相关对话:第{nrel_inds}")
result: List[DialogData] = []
total_original_msgs = 0
total_deleted_msgs = 0
if not_related_dialogs:
# 为每个不相关对话进行一次性抽取,识别重要/不重要(避免逐条 LLM
per_dialog_info = {}
total_unrelated = 0
total_capacity = 0
for d in not_related_dialogs:
dd = d["dialog"]
extraction = d.get("extraction")
if extraction is None:
extraction = await self._extract_dialog_important(dd.content)
# 合并所有重要标记
tokens = extraction.times + extraction.ids + extraction.amounts + extraction.contacts + extraction.addresses + extraction.keywords
msgs = dd.context.msgs
# 分类消息
imp_unrel_msgs = [m for m in msgs if self._msg_matches_tokens(m, tokens) or self._is_important_message(m)]
unimp_unrel_msgs = [m for m in msgs if m not in imp_unrel_msgs]
# 重要消息按重要性排序
imp_sorted_ids = [id(m) for m in sorted(imp_unrel_msgs, key=lambda m: self._importance_score(m))]
info = {
"dialog": dd,
"total_msgs": len(msgs),
"unrelated_count": len(msgs),
"imp_ids_sorted": imp_sorted_ids,
"unimp_ids": [id(m) for m in unimp_unrel_msgs],
}
per_dialog_info[d["index"]] = info
total_unrelated += info["unrelated_count"]
# 全局删除配额:比例作用于全部不相关消息(重要+不重要)
global_delete = int(total_unrelated * proportion)
if proportion > 0 and total_unrelated > 0 and global_delete == 0:
global_delete = 1
# 每段的最大可删容量:不重要全部 + 重要最多删除 floor(len(重要)*比例)且至少保留1条消息
capacities = []
for d in not_related_dialogs:
idx = d["index"]
info = per_dialog_info[idx]
# 统计重要数量
imp_count = len(info["imp_ids_sorted"])
unimp_count = len(info["unimp_ids"])
imp_cap = int(imp_count * proportion)
cap = min(unimp_count + imp_cap, max(0, info["total_msgs"] - 1))
capacities.append(cap)
total_capacity = sum(capacities)
if global_delete > total_capacity:
print(f"[剪枝-数据集] 不相关消息总数={total_unrelated},目标删除={global_delete},最大可删={total_capacity}(重要消息按比例保留)。将按最大可删执行。")
global_delete = total_capacity
# 并发执行所有对话的 LLM 抽取(获取 preserve_keywords 等保护信息)
semaphore = asyncio.Semaphore(self.max_concurrent)
async def extract_with_semaphore(dd: DialogData) -> DialogExtractionResponse:
async with semaphore:
try:
return await self._extract_dialog_important(dd.content)
except Exception as e:
self._log(f"[剪枝-LLM] 对话抽取失败,使用降级策略: {str(e)[:100]}")
return DialogExtractionResponse(is_related=True)
extraction_tasks = [extract_with_semaphore(dd) for dd in dialogs]
extraction_results: List[DialogExtractionResponse] = await asyncio.gather(*extraction_tasks)
for d_idx, (dd, extraction) in enumerate(zip(dialogs, extraction_results)):
msgs = dd.context.msgs
original_count = len(msgs)
total_original_msgs += original_count
# 从 LLM 抽取结果中获取所有需要保留的 token
preserve_tokens = (
extraction.times + extraction.ids + extraction.amounts +
extraction.contacts + extraction.addresses + extraction.keywords +
extraction.preserve_keywords # 情绪/兴趣/爱好关键词
)
# 判断是否需要详细日志
should_log_details = self._detailed_prune_logging and original_count <= self._max_debug_msgs_per_dialog
if self._detailed_prune_logging and original_count > self._max_debug_msgs_per_dialog:
self._log(f" 对话[{d_idx}]消息数={original_count},仅采样前{self._max_debug_msgs_per_dialog}条进行详细日志")
if extraction.preserve_keywords:
self._log(f" 对话[{d_idx}] LLM抽取到情绪/兴趣保护词: {extraction.preserve_keywords}")
# 消息级分类LLM保护 / 填充 / 其他可删
llm_protected_msgs = [] # LLM 保护消息preserve_tokens 命中):绝对不可删除
filler_msgs = [] # 填充消息(优先删除)
deletable_msgs = [] # 其余消息(按比例删除)
for idx, m in enumerate(msgs):
msg_text = m.msg.strip()
if self._msg_matches_tokens(m, preserve_tokens):
llm_protected_msgs.append((idx, m))
if should_log_details or idx < self._max_debug_msgs_per_dialog:
self._log(f" [{idx}] '{msg_text[:30]}...' → 保护LLM不可删")
elif self._is_filler_message(m):
filler_msgs.append((idx, m))
if should_log_details or idx < self._max_debug_msgs_per_dialog:
self._log(f" [{idx}] '{msg_text[:30]}...' → 填充")
else:
deletable_msgs.append((idx, m))
if should_log_details or idx < self._max_debug_msgs_per_dialog:
self._log(f" [{idx}] '{msg_text[:30]}...' → 可删")
# important_msgs 仅用于日志统计
important_msgs = llm_protected_msgs
# 计算删除配额
delete_target = int(original_count * proportion)
if proportion > 0 and original_count > 0 and delete_target == 0:
delete_target = 1
# 确保至少保留1条消息
max_deletable = max(0, original_count - 1)
delete_target = min(delete_target, max_deletable)
# 删除策略:优先删填充消息,再按出现顺序删其余可删消息
to_delete_indices = set()
deleted_details = []
# 第一步:删除填充消息
for idx, msg in filler_msgs:
if len(to_delete_indices) >= delete_target:
# 配额分配:按不相关消息占比分配到各对话,但不超过各自容量
alloc = []
for i, d in enumerate(not_related_dialogs):
idx = d["index"]
info = per_dialog_info[idx]
share = int(global_delete * (info["unrelated_count"] / total_unrelated)) if total_unrelated > 0 else 0
alloc.append(min(share, capacities[i]))
allocated = sum(alloc)
rem = global_delete - allocated
turn = 0
while rem > 0 and turn < 100000:
progressed = False
for i in range(len(not_related_dialogs)):
if rem <= 0:
break
if alloc[i] < capacities[i]:
alloc[i] += 1
rem -= 1
progressed = True
if not progressed:
break
to_delete_indices.add(idx)
deleted_details.append(f"[{idx}] 填充: '{msg.msg[:50]}'")
turn += 1
# 第二步:如果还需要删除,按出现顺序删可删消息
for idx, msg in deletable_msgs:
if len(to_delete_indices) >= delete_target:
break
to_delete_indices.add(idx)
deleted_details.append(f"[{idx}] 可删: '{msg.msg[:50]}'")
# 执行删除
kept_msgs = []
for idx, m in enumerate(msgs):
if idx not in to_delete_indices:
kept_msgs.append(m)
# 确保至少保留1条
if not kept_msgs and msgs:
kept_msgs = [msgs[0]]
dd.context.msgs = kept_msgs
deleted_count = original_count - len(kept_msgs)
total_deleted_msgs += deleted_count
# 输出删除详情
if deleted_details:
self._log(f"[剪枝-删除详情] 对话 {d_idx+1} 删除了以下消息:")
for detail in deleted_details:
self._log(f" {detail}")
# ========== 问答对统计(已注释) ==========
# qa_info = f",问答对={len(qa_pairs)}" if qa_pairs else ""
# ========================================
self._log(
f"[剪枝-对话] 对话 {d_idx+1} 总消息={original_count} "
f"(保护={len(important_msgs)} 填充={len(filler_msgs)} 可删={len(deletable_msgs)}) "
f"删除={deleted_count} 保留={len(kept_msgs)}"
)
result.append(dd)
# 应用删除:相关对话不动;不相关按分配先删不重要,再删重要(低分优先)
total_deleted_confirm = 0
for d in evaluated_dialogs:
dd = d["dialog"]
msgs = dd.context.msgs
original = len(msgs)
if d["is_related"]:
result.append(dd)
continue
idx_in_unrel = next((k for k, x in enumerate(not_related_dialogs) if x["index"] == d["index"]), None)
if idx_in_unrel is None:
result.append(dd)
continue
quota = alloc[idx_in_unrel]
info = per_dialog_info[d["index"]]
# 计算本对话重要最多可删数量
imp_count = len(info["imp_ids_sorted"])
imp_del_cap = int(imp_count * proportion)
# 先构造顺序删除的"不重要ID集合"(按出现顺序前 quota 条)
unimp_delete_ids = set(info["unimp_ids"][:min(quota, len(info["unimp_ids"]))])
del_unimp = min(quota, len(unimp_delete_ids))
rem_quota = quota - del_unimp
# 再从重要里选低分优先的删除ID不超过 imp_del_cap
imp_delete_ids = set(info["imp_ids_sorted"][:min(rem_quota, imp_del_cap)])
deleted_here = 0
actual_unimp_deleted = 0
actual_imp_deleted = 0
kept = []
for m in msgs:
mid = id(m)
if mid in unimp_delete_ids and actual_unimp_deleted < del_unimp:
actual_unimp_deleted += 1
deleted_here += 1
continue
if mid in imp_delete_ids and actual_imp_deleted < len(imp_delete_ids):
actual_imp_deleted += 1
deleted_here += 1
continue
kept.append(m)
if not kept and msgs:
kept = [msgs[0]]
dd.context.msgs = kept
total_deleted_confirm += deleted_here
self._log(
f"[剪枝-对话] 对话 {d['index']+1} 总消息={original} 分配删除={quota} 实删={deleted_here} 保留={len(kept)}"
)
result.append(dd)
self._log(f"[剪枝-数据集] 全局消息级顺序剪枝完成,总删除 {total_deleted_confirm} 条(不相关消息,重要按比例保留)。")
else:
# 全部相关:不执行剪枝
result = [d["dialog"] for d in evaluated_dialogs]
self._log(f"[剪枝-数据集] 剩余对话数={len(result)}")
# 保存日志
# 将本次剪枝阶段的终端输出保存为 JSON 文件(仅在剪枝器内部完成)
try:
from app.core.config import settings
settings.ensure_memory_output_dir()
log_output_path = settings.get_memory_output_path("pruned_terminal.json")
# 去除日志前缀标签(如 [剪枝-数据集]、[剪枝-对话])后再解析为结构化字段保存
sanitized_logs = [self._sanitize_log_line(l) for l in self.run_logs]
payload = self._parse_logs_to_structured(sanitized_logs)
with open(log_output_path, "w", encoding="utf-8") as f:
@@ -621,7 +448,6 @@ class SemanticPruner:
if not result:
print("警告: 语义剪枝后数据集为空,已回退为未剪枝数据以避免流程中断")
return dialogs
return result
def _log(self, msg: str) -> None:

View File

@@ -1,66 +0,0 @@
"""
场景特定配置 - 统一填充词库
重要性判断已完全交由 extracat_Pruning.jinja2 提示词 + LLM preserve_tokens 机制承担。
本模块仅保留统一填充词库filler_phrases用于识别无意义寒暄/表情/口头禅。
所有场景共用同一份词库,场景差异由 LLM 语义判断处理。
"""
from typing import List, Set
from dataclasses import dataclass, field
@dataclass
class ScenePatterns:
"""场景特定的识别模式(仅保留填充词库)"""
filler_phrases: Set[str] = field(default_factory=set)
class SceneConfigRegistry:
"""场景配置注册表 - 所有场景共用统一填充词库"""
BASE_FILLERS: Set[str] = {
# 基础寒暄
"你好", "您好", "在吗", "在的", "在呢", "", "嗯嗯", "", "哦哦",
"好的", "", "", "可以", "不可以", "谢谢", "多谢", "感谢",
"拜拜", "再见", "88", "", "回见",
# 口头禅
"哈哈", "呵呵", "哈哈哈", "嘿嘿", "嘻嘻", "hiahia",
"", "", "", "", "", "", "嗯哼",
# 确认词
"是的", "", "对的", "没错", "好嘞", "收到", "明白", "了解", "知道了",
# 服务类套话
"请问", "请稍等", "稍等", "马上", "立即",
"正在查询", "正在处理", "正在为您", "帮您查一下",
"还有其他问题吗", "还需要什么帮助", "很高兴为您服务",
"感谢您的耐心等待", "抱歉让您久等了",
"已记录", "已反馈", "已转接", "已升级",
"祝您生活愉快", "欢迎下次咨询",
# 外呼套话
"", "hello", "打扰了", "不好意思",
"方便接电话吗", "现在方便吗", "占用您一点时间",
"我是", "我们是", "我们公司", "我们这边",
"了解一下", "介绍一下", "简单说一下",
"考虑考虑", "想一想", "再说", "再看看",
"不需要", "不感兴趣", "没兴趣", "不用了",
"没问题", "那就这样", "再联系", "回头聊", "有需要再说",
# 教育场景套话
"老师好", "同学们好", "上课", "下课", "起立", "坐下",
"举手", "请坐", "很好", "不错", "继续",
"下一个", "下一题", "下一位", "还有吗", "还有问题吗",
# 标点和符号
"。。。", "...", "???", "", "!!!", "",
# 表情符号
"[微笑]", "[呲牙]", "[发呆]", "[得意]", "[流泪]", "[害羞]", "[闭嘴]",
"[睡]", "[大哭]", "[尴尬]", "[发怒]", "[调皮]", "[龇牙]", "[惊讶]",
"[难过]", "[酷]", "[冷汗]", "[抓狂]", "[吐]", "[偷笑]", "[可爱]",
"[白眼]", "[傲慢]", "[饥饿]", "[困]", "[惊恐]", "[流汗]", "[憨笑]",
# 网络用语
"hhh", "hhhh", "2333", "666", "gg", "ok", "OK", "okok",
"emmm", "emm", "em", "mmp", "wtf", "omg",
}
@classmethod
def get_config(cls, scene: str = "") -> ScenePatterns:
"""所有场景统一返回同一份填充词库"""
return ScenePatterns(filler_phrases=cls.BASE_FILLERS)

View File

@@ -34,8 +34,6 @@ from app.core.memory.models.graph_models import (
StatementNode,
)
from app.core.memory.models.message_models import DialogData
from app.core.memory.models.ontology_extraction_models import OntologyTypeList
from app.core.memory.models.ontology_extraction_models import OntologyTypeList
from app.core.memory.models.variate_config import (
ExtractionPipelineConfig,
)
@@ -97,9 +95,6 @@ class ExtractionOrchestrator:
config: Optional[ExtractionPipelineConfig] = None,
progress_callback: Optional[Callable[[str, str, Optional[Dict[str, Any]]], Awaitable[None]]] = None,
embedding_id: Optional[str] = None,
ontology_types: Optional[OntologyTypeList] = None,
enable_general_types: bool = True,
language: str = "zh",
):
"""
初始化流水线编排器
@@ -113,7 +108,6 @@ class ExtractionOrchestrator:
- 接受 (stage: str, message: str, data: Optional[Dict[str, Any]]) 并返回 Awaitable[None]
- 在管线关键点调用以报告进度和结果数据
embedding_id: 嵌入模型ID如果为 None 则从全局配置获取(向后兼容)
language: 语言类型 ("zh" 中文, "en" 英文),默认中文
"""
self.llm_client = llm_client
self.embedder_client = embedder_client
@@ -122,30 +116,6 @@ class ExtractionOrchestrator:
self.is_pilot_run = False # 默认非试运行模式
self.progress_callback = progress_callback # 保存进度回调函数
self.embedding_id = embedding_id # 保存嵌入模型ID
self.language = language # 保存语言配置
# 处理本体类型配置
# 根据 enable_general_types 参数决定是否将通用本体类型与场景特定类型合并
# 如果启用合并且配置中开启了通用本体功能,则使用 OntologyTypeMerger 进行融合
if enable_general_types and ontology_types:
from app.core.memory.ontology_services.ontology_type_loader import (
get_ontology_type_merger,
is_general_ontology_enabled,
)
if is_general_ontology_enabled():
merger = get_ontology_type_merger()
self.ontology_types = merger.merge(ontology_types)
logger.info(
f"已启用通用本体类型融合: 场景类型 {len(ontology_types.types) if ontology_types.types else 0} 个 -> "
f"合并后 {len(self.ontology_types.types) if self.ontology_types.types else 0}"
)
else:
self.ontology_types = ontology_types
logger.info("通用本体类型功能已在配置中禁用,仅使用场景类型")
else:
self.ontology_types = ontology_types
if not enable_general_types and ontology_types:
logger.info("enable_general_types=False仅使用场景类型")
# 保存去重消歧的详细记录(内存中的数据结构)
self.dedup_merge_records: List[Dict[str, Any]] = [] # 实体合并记录
@@ -157,7 +127,7 @@ class ExtractionOrchestrator:
llm_client=llm_client,
config=self.config.statement_extraction,
)
self.triplet_extractor = TripletExtractor(llm_client=llm_client,ontology_types=self.ontology_types, language=language)
self.triplet_extractor = TripletExtractor(llm_client=llm_client)
self.temporal_extractor = TemporalExtractor(llm_client=llm_client)
logger.info("ExtractionOrchestrator 初始化完成")
@@ -645,25 +615,9 @@ class ExtractionOrchestrator:
logger.info(f"总陈述句: {total_statements}, 用户陈述句: {filtered_statements}, 开始全局并行提取情绪")
# 初始化情绪提取服务
# 如果 emotion_model_id 为空,回退到工作空间默认 LLM
from app.services.emotion_extraction_service import EmotionExtractionService
emotion_model_id = memory_config.emotion_model_id
if not emotion_model_id and memory_config.workspace_id:
from app.repositories.workspace_repository import get_workspace_models_configs
from app.db import SessionLocal
db = SessionLocal()
try:
workspace_models = get_workspace_models_configs(db, memory_config.workspace_id)
if workspace_models and workspace_models.get("llm"):
emotion_model_id = workspace_models["llm"]
logger.info(f"emotion_model_id 为空,使用工作空间默认 LLM: {emotion_model_id}")
finally:
db.close()
emotion_service = EmotionExtractionService(
llm_id=emotion_model_id if emotion_model_id else None
llm_id=memory_config.emotion_model_id if memory_config.emotion_model_id else None
)
# 全局并行处理所有陈述句
@@ -1932,17 +1886,17 @@ def preprocess_data(
Returns:
经过清洗转换后的 DialogData 列表
"""
logger.debug("=== 数据预处理 ===")
print("\n=== 数据预处理 ===")
from app.core.memory.storage_services.extraction_engine.data_preprocessing.data_preprocessor import (
DataPreprocessor,
)
preprocessor = DataPreprocessor()
try:
cleaned_data = preprocessor.preprocess(input_path=input_path, output_path=output_path, skip_cleaning=skip_cleaning, indices=indices)
logger.debug(f"数据预处理完成!共处理了 {len(cleaned_data)} 条对话数据")
print(f"数据预处理完成!共处理了 {len(cleaned_data)} 条对话数据")
return cleaned_data
except Exception as e:
logger.error(f"数据预处理过程中出现错误: {e}")
print(f"数据预处理过程中出现错误: {e}")
raise
@@ -1961,7 +1915,7 @@ async def get_chunked_dialogs_from_preprocessed(
Returns:
带 chunks 的 DialogData 列表
"""
logger.debug(f"=== 批量对话分块处理 (使用 {chunker_strategy}) ===")
print(f"\n=== 批量对话分块处理 (使用 {chunker_strategy}) ===")
if not data:
raise ValueError("预处理数据为空,无法进行分块")
@@ -1988,7 +1942,6 @@ async def get_chunked_dialogs_with_preprocessing(
input_data_path: Optional[str] = None,
llm_client: Optional[Any] = None,
skip_cleaning: bool = True,
pruning_config: Optional[Dict] = None,
) -> List[DialogData]:
"""包含数据预处理步骤的完整分块流程
@@ -2001,12 +1954,11 @@ async def get_chunked_dialogs_with_preprocessing(
input_data_path: 输入数据路径
llm_client: LLM 客户端
skip_cleaning: 是否跳过数据清洗步骤默认False
pruning_config: 剪枝配置字典,包含 pruning_switch, pruning_scene, pruning_threshold
Returns:
带 chunks 的 DialogData 列表
"""
logger.debug("=== 完整数据处理流程(包含预处理)===")
print("\n=== 完整数据处理流程(包含预处理)===")
if input_data_path is None:
input_data_path = os.path.join(
@@ -2032,19 +1984,7 @@ async def get_chunked_dialogs_with_preprocessing(
from app.core.memory.storage_services.extraction_engine.data_preprocessing.data_pruning import (
SemanticPruner,
)
from app.core.memory.models.config_models import PruningConfig
# 构建剪枝配置
if pruning_config:
# 使用传入的配置
config = PruningConfig(**pruning_config)
logger.debug(f"[剪枝] 使用传入配置: switch={config.pruning_switch}, scene={config.pruning_scene}, threshold={config.pruning_threshold}")
else:
# 使用默认配置(关闭剪枝)
config = None
logger.debug("[剪枝] 未提供配置,使用默认配置(剪枝关闭)")
pruner = SemanticPruner(config=config, llm_client=llm_client)
pruner = SemanticPruner(llm_client=llm_client)
# 记录单对话场景下剪枝前的消息数量
single_dialog_original_msgs = None
@@ -2057,12 +1997,12 @@ async def get_chunked_dialogs_with_preprocessing(
if len(preprocessed_data) == 1 and single_dialog_original_msgs is not None:
remaining_msgs = len(preprocessed_data[0].context.msgs) if preprocessed_data[0].context else 0
deleted_msgs = max(0, single_dialog_original_msgs - remaining_msgs)
logger.debug(
print(
f"语义剪枝完成!剩余 1 条对话!原始消息数:{single_dialog_original_msgs}"
f"保留消息数:{remaining_msgs},删除 {deleted_msgs} 条。"
)
else:
logger.debug(f"语义剪枝完成!剩余 {len(preprocessed_data)} 条对话")
print(f"语义剪枝完成!剩余 {len(preprocessed_data)} 条对话")
# 保存剪枝后的数据
try:
@@ -2073,9 +2013,9 @@ async def get_chunked_dialogs_with_preprocessing(
dp = DataPreprocessor(output_file_path=pruned_output_path)
dp.save_data(preprocessed_data, output_path=pruned_output_path)
except Exception as se:
logger.error(f"保存剪枝结果失败:{se}")
print(f"保存剪枝结果失败:{se}")
except Exception as e:
logger.error(f"语义剪枝过程中出现错误,跳过剪枝: {e}")
print(f"语义剪枝过程中出现错误,跳过剪枝: {e}")
# 步骤3: 对话分块
return await get_chunked_dialogs_from_preprocessed(

View File

@@ -1,7 +1,5 @@
import os
from typing import Optional, List, Any
from enum import Enum
from pathlib import Path
from typing import Optional
from app.core.logging_config import get_memory_logger
from app.core.memory.models.message_models import DialogData, Chunk
@@ -12,20 +10,6 @@ from app.core.memory.utils.config.config_utils import get_chunker_config
logger = get_memory_logger(__name__)
class ChunkerStrategy(Enum):
"""Supported chunking strategies."""
RECURSIVE = "RecursiveChunker"
SEMANTIC = "SemanticChunker"
LATE = "LateChunker"
NEURAL = "NeuralChunker"
LLM = "LLMChunker"
@classmethod
def get_valid_strategies(cls) -> List[str]:
"""Get list of valid strategy names."""
return [strategy.value for strategy in cls]
class DialogueChunker:
"""A class that processes dialogues and fills them with chunks based on a specified strategy.
@@ -33,51 +17,23 @@ class DialogueChunker:
of different chunking strategies to dialogue data.
"""
def __init__(self, chunker_strategy: str = "RecursiveChunker", llm_client: Optional[Any] = None):
def __init__(self, chunker_strategy: str = "RecursiveChunker", llm_client=None):
"""Initialize the DialogueChunker with a specific chunking strategy.
Args:
chunker_strategy: The chunking strategy to use (default: RecursiveChunker)
Options: SemanticChunker, RecursiveChunker, LateChunker, NeuralChunker, LLMChunker
llm_client: LLM client instance (required for LLMChunker strategy)
Raises:
ValueError: If chunker_strategy is invalid or required parameters are missing
Options: SemanticChunker, RecursiveChunker, LateChunker, NeuralChunker
"""
# Validate strategy
valid_strategies = ChunkerStrategy.get_valid_strategies()
if chunker_strategy not in valid_strategies:
raise ValueError(
f"Invalid chunker_strategy: '{chunker_strategy}'. "
f"Must be one of {valid_strategies}"
)
self.chunker_strategy = chunker_strategy
logger.info(f"Initializing DialogueChunker with strategy: {chunker_strategy}")
chunker_config_dict = get_chunker_config(chunker_strategy)
self.chunker_config = ChunkerConfig.model_validate(chunker_config_dict)
try:
# Load and validate configuration
chunker_config_dict = get_chunker_config(chunker_strategy)
if not chunker_config_dict:
raise ValueError(f"Failed to load configuration for strategy: {chunker_strategy}")
self.chunker_config = ChunkerConfig.model_validate(chunker_config_dict)
# Initialize chunker client
if self.chunker_config.chunker_strategy == "LLMChunker":
if not llm_client:
raise ValueError("llm_client is required for LLMChunker strategy")
self.chunker_client = ChunkerClient(self.chunker_config, llm_client)
else:
self.chunker_client = ChunkerClient(self.chunker_config)
logger.info(f"DialogueChunker initialized successfully with strategy: {chunker_strategy}")
except Exception as e:
logger.error(f"Failed to initialize DialogueChunker: {e}", exc_info=True)
raise
if self.chunker_config.chunker_strategy == "LLMChunker":
self.chunker_client = ChunkerClient(self.chunker_config, llm_client)
else:
self.chunker_client = ChunkerClient(self.chunker_config)
async def process_dialogue(self, dialogue: DialogData) -> List[Chunk]:
async def process_dialogue(self, dialogue: DialogData) -> list[Chunk]:
"""Process a dialogue by generating chunks and adding them to the DialogData object.
Args:
@@ -87,125 +43,54 @@ class DialogueChunker:
A list of Chunk objects
Raises:
ValueError: If dialogue is invalid or chunking fails
Exception: If chunking process encounters an error
ValueError: If chunking fails or returns empty chunks
"""
# Validate input
if not dialogue:
raise ValueError("dialogue cannot be None")
if not dialogue.context or not dialogue.context.msgs:
result_dialogue = await self.chunker_client.generate_chunks(dialogue)
chunks = result_dialogue.chunks
if not chunks or len(chunks) == 0:
raise ValueError(
f"Dialogue {dialogue.ref_id} has no messages to chunk. "
f"Context: {dialogue.context is not None}, "
f"Messages: {len(dialogue.context.msgs) if dialogue.context else 0}"
f"Chunking failed: No chunks generated for dialogue {dialogue.ref_id}. "
f"Messages: {len(dialogue.context.msgs) if dialogue.context else 0}, "
f"Strategy: {self.chunker_config.chunker_strategy}"
)
logger.info(
f"Processing dialogue {dialogue.ref_id} with {len(dialogue.context.msgs)} messages "
f"using strategy: {self.chunker_strategy}"
)
try:
# Generate chunks
result_dialogue = await self.chunker_client.generate_chunks(dialogue)
chunks = result_dialogue.chunks
# Validate results
if not chunks or len(chunks) == 0:
raise ValueError(
f"Chunking failed: No chunks generated for dialogue {dialogue.ref_id}. "
f"Messages: {len(dialogue.context.msgs)}, "
f"Content length: {len(dialogue.content) if dialogue.content else 0}, "
f"Strategy: {self.chunker_config.chunker_strategy}"
)
return chunks
logger.info(
f"Successfully generated {len(chunks)} chunks for dialogue {dialogue.ref_id}. "
f"Total characters processed: {len(dialogue.content) if dialogue.content else 0}"
)
return chunks
except ValueError:
# Re-raise validation errors
raise
except Exception as e:
logger.error(
f"Error processing dialogue {dialogue.ref_id} with strategy {self.chunker_strategy}: {e}",
exc_info=True
)
raise
def save_chunking_results(
self,
chunks: List[Chunk],
dialogue: DialogData,
output_path: Optional[str] = None,
preview_length: int = 100
) -> str:
def save_chunking_results(self, dialogue: DialogData, output_path: Optional[str] = None) -> str:
"""Save the chunking results to a file and return the output path.
Args:
chunks: List of Chunk objects to save
dialogue: The DialogData object that was processed
output_path: Optional path to save the output (defaults to current directory)
preview_length: Maximum length of content preview (default: 100)
dialogue: The processed DialogData object with chunks
output_path: Optional path to save the output
Returns:
The path where the output was saved
Raises:
ValueError: If chunks or dialogue is invalid
IOError: If file writing fails
"""
# Validate input
if not chunks:
raise ValueError("chunks list cannot be empty")
if not dialogue:
raise ValueError("dialogue cannot be None")
# Generate default output path if not provided
if not output_path:
output_dir = Path(__file__).parent.parent.parent
output_path = str(output_dir / f"chunker_output_{self.chunker_strategy.lower()}.txt")
output_path = os.path.join(
os.path.dirname(__file__), "..", "..",
f"chunker_output_{self.chunker_strategy.lower()}.txt"
)
output_lines = [
f"=== Chunking Results ({self.chunker_strategy}) ===",
f"Dialogue ID: {dialogue.ref_id}",
f"Original conversation has {len(dialogue.context.msgs)} messages",
f"Total characters: {len(dialogue.content)}",
f"Generated {len(dialogue.chunks)} chunks:"
]
logger.info(f"Saving chunking results to: {output_path}")
try:
# Prepare output content
output_lines = [
f"=== Chunking Results ({self.chunker_strategy}) ===",
f"Dialogue ID: {dialogue.ref_id}",
f"Original conversation has {len(dialogue.context.msgs) if dialogue.context else 0} messages",
f"Total characters: {len(dialogue.content) if dialogue.content else 0}",
f"Generated {len(chunks)} chunks:",
""
]
for i, chunk in enumerate(chunks, 1):
content_preview = chunk.content[:preview_length] if chunk.content else ""
if len(chunk.content) > preview_length:
content_preview += "..."
output_lines.append(f" Chunk {i}: {len(chunk.content)} characters")
output_lines.append(f" Content preview: {content_preview}")
if chunk.metadata:
output_lines.append(f" Metadata: {chunk.metadata}")
output_lines.append("")
for i, chunk in enumerate(dialogue.chunks):
output_lines.append(f" Chunk {i+1}: {len(chunk.content)} characters")
output_lines.append(f" Content preview: {chunk.content}...")
if chunk.metadata:
output_lines.append(f" Metadata: {chunk.metadata}")
# Write to file
with open(output_path, "w", encoding="utf-8") as f:
f.write("\n".join(output_lines))
with open(output_path, "w", encoding="utf-8") as f:
f.write("\n".join(output_lines))
logger.info(f"Successfully saved chunking results to: {output_path}")
return output_path
except IOError as e:
logger.error(f"Failed to write chunking results to {output_path}: {e}", exc_info=True)
raise
except Exception as e:
logger.error(f"Unexpected error saving chunking results: {e}", exc_info=True)
raise
logger.info(f"Chunking results saved to: {output_path}")
return output_path

View File

@@ -10,11 +10,38 @@ from app.core.memory.models.base_response import RobustLLMResponse
from app.core.memory.models.graph_models import MemorySummaryNode
from app.core.memory.models.message_models import DialogData
from app.core.memory.utils.prompt.prompt_utils import render_memory_summary_prompt
from app.core.language_utils import validate_language # 使用集中化的语言校验
from pydantic import Field
logger = get_memory_logger(__name__)
# 支持的语言列表和默认回退值
SUPPORTED_LANGUAGES = {"zh", "en"}
FALLBACK_LANGUAGE = "en"
def validate_language(language: Optional[str]) -> str:
"""
校验语言参数,确保其为有效值。
Args:
language: 待校验的语言代码
Returns:
有效的语言代码("zh""en"
"""
if language is None:
return FALLBACK_LANGUAGE
lang = str(language).lower().strip()
if lang in SUPPORTED_LANGUAGES:
return lang
logger.warning(
f"无效的语言参数 '{language}',已回退到默认值 '{FALLBACK_LANGUAGE}'"
f"支持的语言: {SUPPORTED_LANGUAGES}"
)
return FALLBACK_LANGUAGE
class MemorySummaryResponse(RobustLLMResponse):
"""Structured response for summary generation per chunk.
@@ -33,7 +60,7 @@ class MemorySummaryResponse(RobustLLMResponse):
async def generate_title_and_type_for_summary(
content: str,
llm_client,
language: str = "zh"
language: str = None
) -> Tuple[str, str]:
"""
为MemorySummary生成标题和类型
@@ -43,14 +70,17 @@ async def generate_title_and_type_for_summary(
Args:
content: Summary的内容文本
llm_client: LLM客户端实例
language: 生成标题使用的语言 ("zh" 中文, "en" 英文)默认中文
language: 生成标题使用的语言 ("zh" 中文, "en" 英文)如果为None则从配置读取
Returns:
(标题, 类型)元组
"""
from app.core.memory.utils.prompt.prompt_utils import render_episodic_title_and_type_prompt
from app.core.config import settings
# 验证语言参数
# 如果没有指定语言,从配置中读取,并校验有效性
if language is None:
language = settings.DEFAULT_LANGUAGE
language = validate_language(language)
# 定义有效的类型集合
@@ -158,7 +188,6 @@ async def _process_chunk_summary(
chunk,
llm_client,
embedder: OpenAIEmbedderClient,
language: str = "zh",
) -> Optional[MemorySummaryNode]:
"""Process a single chunk to generate a memory summary node."""
# Skip empty chunks
@@ -166,8 +195,9 @@ async def _process_chunk_summary(
return None
try:
# 验证语言参数
language = validate_language(language)
# 从配置中获取语言设置(只获取一次,复用),并校验有效性
from app.core.config import settings
language = validate_language(settings.DEFAULT_LANGUAGE)
# Render prompt via Jinja2 for a single chunk
prompt_content = await render_memory_summary_prompt(
@@ -237,21 +267,13 @@ async def memory_summary_generation(
chunked_dialogs: List[DialogData],
llm_client,
embedder_client: OpenAIEmbedderClient,
language: str = "zh",
) -> List[MemorySummaryNode]:
"""Generate memory summaries per chunk, embed them, and return nodes.
Args:
chunked_dialogs: 分块后的对话数据
llm_client: LLM客户端
embedder_client: 嵌入客户端
language: 语言类型 ("zh" 中文, "en" 英文),默认中文
"""
"""Generate memory summaries per chunk, embed them, and return nodes."""
# Collect all tasks for parallel processing
tasks = []
for dialog in chunked_dialogs:
for chunk in dialog.chunks:
tasks.append(_process_chunk_summary(dialog, chunk, llm_client, embedder_client, language=language))
tasks.append(_process_chunk_summary(dialog, chunk, llm_client, embedder_client))
# Process all chunks in parallel
results = await asyncio.gather(*tasks, return_exceptions=False)

View File

@@ -14,7 +14,7 @@ import time
from typing import List, Optional
from app.core.memory.llm_tools.openai_client import OpenAIClient
from app.core.memory.models.ontology_scenario_models import (
from app.core.memory.models.ontology_models import (
OntologyClass,
OntologyExtractionResponse,
)
@@ -64,7 +64,6 @@ class OntologyExtractor:
llm_max_tokens: int = 2000,
max_description_length: int = 500,
timeout: Optional[float] = None,
language: str = "zh",
) -> OntologyExtractionResponse:
"""Extract ontology classes from a scenario description.
@@ -85,7 +84,6 @@ class OntologyExtractor:
llm_max_tokens: LLM max tokens parameter (default: 2000)
max_description_length: Maximum description length (default: 500)
timeout: Optional timeout in seconds for LLM call (default: None, no timeout)
language: Language for output ("zh" for Chinese, "en" for English)
Returns:
OntologyExtractionResponse containing validated ontology classes
@@ -118,7 +116,7 @@ class OntologyExtractor:
logger.info(
f"Starting ontology extraction - scenario_length={len(scenario)}, "
f"domain={domain}, max_classes={max_classes}, min_classes={min_classes}, "
f"timeout={timeout}, language={language}"
f"timeout={timeout}"
)
try:
@@ -136,7 +134,6 @@ class OntologyExtractor:
max_classes=max_classes,
llm_temperature=llm_temperature,
llm_max_tokens=llm_max_tokens,
language=language,
),
timeout=timeout
)
@@ -159,7 +156,6 @@ class OntologyExtractor:
max_classes=max_classes,
llm_temperature=llm_temperature,
llm_max_tokens=llm_max_tokens,
language=language,
)
llm_duration = time.time() - llm_start_time
@@ -264,7 +260,6 @@ class OntologyExtractor:
max_classes: int,
llm_temperature: float,
llm_max_tokens: int,
language: str = "zh",
) -> OntologyExtractionResponse:
"""Call LLM to extract ontology classes from scenario.
@@ -277,7 +272,6 @@ class OntologyExtractor:
max_classes: Maximum number of classes to extract
llm_temperature: LLM temperature parameter
llm_max_tokens: LLM max tokens parameter
language: Language for output ("zh" for Chinese, "en" for English)
Returns:
OntologyExtractionResponse from LLM
@@ -292,7 +286,6 @@ class OntologyExtractor:
domain=domain,
max_classes=max_classes,
json_schema=OntologyExtractionResponse.model_json_schema(),
language=language,
)
logger.debug(f"Rendered prompt length: {len(prompt_content)}")

View File

@@ -1,6 +1,6 @@
import os
import asyncio
from typing import List, Dict, Optional
from typing import List, Dict
from app.core.logging_config import get_memory_logger
from app.core.memory.llm_tools.openai_client import OpenAIClient
@@ -8,7 +8,6 @@ from app.core.memory.utils.prompt.prompt_utils import render_triplet_extraction_
from app.core.memory.utils.data.ontology import PREDICATE_DEFINITIONS, Predicate # 引入枚举 Predicate 白名单过滤
from app.core.memory.models.triplet_models import TripletExtractionResponse
from app.core.memory.models.message_models import DialogData, Statement
from app.core.memory.models.ontology_extraction_models import OntologyTypeList
from app.core.memory.utils.log.logging_utils import prompt_logger
logger = get_memory_logger(__name__)
@@ -18,22 +17,13 @@ logger = get_memory_logger(__name__)
class TripletExtractor:
"""Extracts knowledge triplets and entities from statements using LLM"""
def __init__(
self,
llm_client: OpenAIClient,
ontology_types: Optional[OntologyTypeList] = None,
language: str = "zh"):
def __init__(self, llm_client: OpenAIClient):
"""Initialize the TripletExtractor with an LLM client
Args:
llm_client: OpenAIClient instance for processing
language: 语言类型 ("zh" 中文, "en" 英文),默认中文
ontology_types: Optional OntologyTypeList containing predefined ontology types
for entity classification guidance
"""
self.llm_client = llm_client
self.ontology_types = ontology_types
self.language = language
def _get_language(self) -> str:
"""Get the configured language for entity descriptions
@@ -41,7 +31,8 @@ class TripletExtractor:
Returns:
Language code ("zh" or "en")
"""
return self.language
from app.core.config import settings
return settings.DEFAULT_LANGUAGE
async def _extract_triplets(self, statement: Statement, chunk_content: str) -> TripletExtractionResponse:
"""Process a single statement and return extracted triplets and entities"""
@@ -59,8 +50,7 @@ class TripletExtractor:
chunk_content=chunk_content,
json_schema=TripletExtractionResponse.model_json_schema(),
predicate_instructions=PREDICATE_DEFINITIONS,
language=self._get_language(),
ontology_types=self.ontology_types,
language=self._get_language()
)
# Create messages for LLM

View File

@@ -462,8 +462,8 @@ class ReflectionEngine:
List[Any]: 反思数据列表
"""
print("=== 获取反思数据 ===")
print(f" 主机ID: {host_id}")
if self.config.reflexion_range == ReflectionRange.PARTIAL:
neo4j_query = neo4j_query_part.format(host_id)
neo4j_statement = neo4j_statement_part.format(host_id)

View File

@@ -1,12 +0,0 @@
# -*- coding: utf-8 -*-
"""本体解析工具模块
本模块提供本体文件解析功能,支持多种 RDF 格式的本体文件解析。
Modules:
ontology_parser: 本体文件解析器
"""
from .ontology_parser import MultiOntologyParser, OntologyParser
__all__ = ["OntologyParser", "MultiOntologyParser"]

View File

@@ -1,366 +0,0 @@
# -*- coding: utf-8 -*-
"""本体文件解析器模块
本模块提供统一的本体文件解析功能,支持多种 RDF 格式:
- Turtle (.ttl)
- OWL/XML (.owl)
- RDF/XML (.rdf)
- N-Triples (.nt)
- JSON-LD (.jsonld)
解析器会自动根据文件扩展名推断格式,并在解析失败时尝试其他格式。
解析结果包含类定义的名称、URI、多语言标签、描述和父类信息。
Classes:
OntologyParser: 统一本体文件解析器
MultiOntologyParser: 多本体文件解析器
Example:
>>> parser = OntologyParser("ontology.ttl")
>>> registry = parser.parse()
>>> print(f"解析了 {len(registry.types)} 个类型")
>>> multi_parser = MultiOntologyParser(["ontology1.ttl", "ontology2.owl"])
>>> merged_registry = multi_parser.parse_all()
>>> print(f"合并后共 {len(merged_registry.types)} 个类型")
"""
import logging
import re
from typing import List, Optional
from rdflib import OWL, RDF, RDFS, Graph, URIRef
from app.core.memory.models.ontology_general_models import (
GeneralOntologyType,
GeneralOntologyTypeRegistry,
OntologyFileFormat,
)
logger = logging.getLogger(__name__)
class OntologyParser:
"""统一本体文件解析器
解析本体文件并提取类定义,构建类型注册表。支持多种 RDF 格式,
并提供格式自动推断和回退机制。
Attributes:
file_path: 本体文件路径
file_format: 文件格式,如果未指定则根据扩展名推断
graph: rdflib Graph 实例,用于存储解析后的 RDF 数据
Example:
>>> parser = OntologyParser("dbpedia.owl")
>>> registry = parser.parse()
>>> person_type = registry.get_type("Person")
>>> if person_type:
... print(f"Person URI: {person_type.class_uri}")
"""
def __init__(
self,
file_path: str,
file_format: Optional[OntologyFileFormat] = None,
):
"""初始化解析器
Args:
file_path: 本体文件路径
file_format: 文件格式,如果未指定则根据扩展名自动推断
"""
self.file_path = file_path
self.file_format = file_format or OntologyFileFormat.from_extension(file_path)
self.graph = Graph()
def parse(self) -> GeneralOntologyTypeRegistry:
"""解析本体文件,返回类型注册表
首先尝试使用推断的格式解析文件,如果失败则尝试其他格式。
解析成功后,遍历所有 owl:Class 和 rdfs:Class 定义,
提取类信息并构建层次结构。
Returns:
GeneralOntologyTypeRegistry: 包含所有解析出的类型和层次结构的注册表
Raises:
ValueError: 当所有格式都无法解析文件时抛出
"""
logger.info(f"开始解析本体文件: {self.file_path}")
# 尝试解析,失败则尝试其他格式
self._parse_with_fallback()
registry = GeneralOntologyTypeRegistry()
registry.source_files.append(self.file_path)
# 遍历 owl:Class
for class_uri in self.graph.subjects(RDF.type, OWL.Class):
type_info = self._parse_class(class_uri)
if type_info:
registry.types[type_info.class_name] = type_info
self._update_hierarchy(registry, type_info)
# 遍历 rdfs:Class避免重复
for class_uri in self.graph.subjects(RDF.type, RDFS.Class):
uri_str = str(class_uri)
# 检查是否已经作为 owl:Class 解析过
if uri_str not in [t.class_uri for t in registry.types.values()]:
type_info = self._parse_class(class_uri)
if type_info and type_info.class_name not in registry.types:
registry.types[type_info.class_name] = type_info
self._update_hierarchy(registry, type_info)
logger.info(f"本体解析完成: {len(registry.types)} 个类型")
return registry
def _parse_with_fallback(self) -> None:
"""尝试解析文件,失败时尝试其他格式
首先使用推断的格式解析,如果失败则依次尝试 RDF_XML 和 TURTLE 格式。
Raises:
ValueError: 当所有格式都无法解析文件时抛出
"""
try:
self.graph.parse(self.file_path, format=self.file_format.value)
return
except Exception as e:
logger.warning(f"使用 {self.file_format.value} 格式解析失败: {e}")
# 尝试其他格式
fallback_formats = [
OntologyFileFormat.RDF_XML,
OntologyFileFormat.TURTLE,
OntologyFileFormat.N_TRIPLES,
OntologyFileFormat.JSON_LD,
]
for fmt in fallback_formats:
if fmt != self.file_format:
try:
self.graph.parse(self.file_path, format=fmt.value)
logger.info(f"使用回退格式 {fmt.value} 解析成功")
return
except Exception:
continue
raise ValueError(f"无法解析本体文件: {self.file_path}")
def _update_hierarchy(
self,
registry: GeneralOntologyTypeRegistry,
type_info: GeneralOntologyType
) -> None:
"""更新层次结构
如果类型有父类,将其添加到层次结构中。
Args:
registry: 类型注册表
type_info: 类型信息
"""
if type_info.parent_class:
if type_info.parent_class not in registry.hierarchy:
registry.hierarchy[type_info.parent_class] = set()
registry.hierarchy[type_info.parent_class].add(type_info.class_name)
def _parse_class(self, class_uri: URIRef) -> Optional[GeneralOntologyType]:
"""解析单个类定义
从 RDF 图中提取类的名称、URI、标签、描述和父类信息。
过滤空白节点和内置类型Thing、Resource
Args:
class_uri: 类的 URI 引用
Returns:
GeneralOntologyType 实例,如果应该跳过该类则返回 None
"""
uri_str = str(class_uri)
class_name = self._extract_local_name(uri_str)
# 过滤空白节点和内置类型
if not class_name:
return None
if class_name.startswith('_:'):
return None
if class_name in ('Thing', 'Resource'):
return None
# 过滤空白节点 URI以 _: 开头或包含空白节点标识)
if uri_str.startswith('_:'):
return None
# 提取标签
labels = self._extract_labels(class_uri)
# 提取描述
description = self._extract_description(class_uri)
# 提取父类
parent_class = self._extract_parent_class(class_uri)
return GeneralOntologyType(
class_name=class_name,
class_uri=uri_str,
labels=labels,
description=description,
parent_class=parent_class,
source_file=self.file_path
)
def _extract_labels(self, class_uri: URIRef) -> dict:
"""提取类的多语言标签
从 rdfs:label 属性中提取所有语言的标签。
如果没有标签,使用类名作为英文标签。
Args:
class_uri: 类的 URI 引用
Returns:
语言代码到标签文本的字典
"""
labels = {}
for label in self.graph.objects(class_uri, RDFS.label):
lang = getattr(label, 'language', None) or "en"
labels[lang] = str(label)
# 如果没有标签,使用类名作为默认标签
if not labels:
class_name = self._extract_local_name(str(class_uri))
if class_name:
labels["en"] = class_name
return labels
def _extract_description(self, class_uri: URIRef) -> Optional[str]:
"""提取类的描述
从 rdfs:comment 属性中提取描述,优先使用英文描述。
Args:
class_uri: 类的 URI 引用
Returns:
类的描述文本,如果没有则返回 None
"""
description = None
for comment in self.graph.objects(class_uri, RDFS.comment):
lang = getattr(comment, 'language', None)
# 优先使用英文描述
if lang == "en":
return str(comment)
# 如果还没有描述,使用无语言标记或其他语言的描述
if description is None:
description = str(comment)
return description
def _extract_parent_class(self, class_uri: URIRef) -> Optional[str]:
"""提取类的父类
从 rdfs:subClassOf 属性中提取第一个有效的父类。
过滤内置类型Thing、Resource和空白节点。
Args:
class_uri: 类的 URI 引用
Returns:
父类名称,如果没有有效父类则返回 None
"""
for parent_uri in self.graph.objects(class_uri, RDFS.subClassOf):
parent_uri_str = str(parent_uri)
# 跳过空白节点
if parent_uri_str.startswith('_:'):
continue
parent_name = self._extract_local_name(parent_uri_str)
# 过滤内置类型
if parent_name and parent_name not in ('Thing', 'Resource'):
return parent_name
return None
def _extract_local_name(self, uri: str) -> Optional[str]:
"""从 URI 中提取本地名称
支持两种常见的 URI 格式:
1. 使用 # 分隔的 URI如 http://example.org/ontology#Person
2. 使用 / 分隔的 URI如 http://dbpedia.org/ontology/Person
Args:
uri: 完整的 URI 字符串
Returns:
本地名称,如果无法提取则返回 None
"""
# 处理空白节点
if uri.startswith('_:'):
return None
# 尝试使用 # 分隔
if '#' in uri:
local_name = uri.rsplit('#', 1)[1]
if local_name:
return local_name
# 尝试使用 / 分隔
if '/' in uri:
local_name = uri.rsplit('/', 1)[1]
if local_name:
return local_name
# 使用正则表达式作为最后手段
match = re.search(r'[#/]([^#/]+)$', uri)
return match.group(1) if match else None
class MultiOntologyParser:
"""多本体文件解析器
支持加载多个本体文件并将它们合并到一个统一的类型注册表中。
先加载的文件中的类型定义优先保留(当存在同名类型时)。
Attributes:
file_paths: 本体文件路径列表
Example:
>>> parser = MultiOntologyParser([
... "app/core/memory/ontology_services/General_purpose_entity.ttl",
... "domain_specific.owl"
... ])
>>> registry = parser.parse_all()
>>> print(f"合并后共 {len(registry.types)} 个类型")
"""
def __init__(self, file_paths: List[str]):
"""初始化多文件解析器
Args:
file_paths: 本体文件路径列表
"""
self.file_paths = file_paths
def parse_all(self) -> GeneralOntologyTypeRegistry:
"""解析所有本体文件并合并
依次解析每个本体文件,并将结果合并到一个统一的注册表中。
如果某个文件解析失败,会记录警告日志并跳过该文件继续处理。
Returns:
GeneralOntologyTypeRegistry: 合并后的类型注册表
"""
merged_registry = GeneralOntologyTypeRegistry()
for file_path in self.file_paths:
try:
parser = OntologyParser(file_path)
registry = parser.parse()
merged_registry.merge(registry)
logger.info(f"已合并本体文件: {file_path}")
except Exception as e:
logger.warning(f"跳过无法解析的本体文件 {file_path}: {e}")
logger.info(f"多本体合并完成: 共 {len(merged_registry.types)} 个类型")
return merged_registry

View File

@@ -9,29 +9,22 @@ current_dir = os.path.dirname(os.path.abspath(__file__))
prompt_dir = os.path.join(current_dir, "prompts")
prompt_env = Environment(loader=FileSystemLoader(prompt_dir))
async def get_prompts(message: str, language: str = "zh") -> list[dict]:
async def get_prompts(message: str) -> list[dict]:
"""
Renders system and user prompts using Jinja2 templates.
Args:
message: The message content
language: Language for output ("zh" for Chinese, "en" for English)
Returns:
List of message dictionaries with role and content
"""
system_template = prompt_env.get_template("system.jinja2")
user_template = prompt_env.get_template("user.jinja2")
system_prompt = system_template.render(language=language)
user_prompt = user_template.render(message=message, language=language)
system_prompt = system_template.render()
user_prompt = user_template.render(message=message)
# 记录渲染结果到提示日志(与示例日志结构一致)
log_prompt_rendering('system', system_prompt)
log_prompt_rendering('user', user_prompt)
# 可选:记录模板渲染信息(仅当 prompt_templates.log 存在时生效)
log_template_rendering('system.jinja2', {'language': language})
log_template_rendering('user.jinja2', {'message': message, 'language': language})
log_template_rendering('system.jinja2', {})
log_template_rendering('user.jinja2', {'message': message})
return [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt},
@@ -45,7 +38,6 @@ async def render_statement_extraction_prompt(
include_dialogue_context: bool = False,
dialogue_content: str | None = None,
max_dialogue_chars: int | None = None,
language: str = "zh",
) -> str:
"""
Renders the statement extraction prompt using the extract_statement.jinja2 template.
@@ -54,11 +46,6 @@ async def render_statement_extraction_prompt(
chunk_content: The content of the chunk to process
definitions: Label definitions for statement classification
json_schema: JSON schema for the expected output format
granularity: Extraction granularity level (1-3)
include_dialogue_context: Whether to include full dialogue context
dialogue_content: Full dialogue content for context
max_dialogue_chars: Maximum characters for dialogue context
language: Language for output ("zh" for Chinese, "en" for English)
Returns:
Rendered prompt content as string
@@ -82,7 +69,6 @@ async def render_statement_extraction_prompt(
granularity=granularity,
include_dialogue_context=include_dialogue_context,
dialogue_context=ctx,
language=language,
)
# 记录渲染结果到提示日志(与示例日志结构一致)
log_prompt_rendering('statement extraction', rendered_prompt)
@@ -104,7 +90,6 @@ async def render_temporal_extraction_prompt(
temporal_guide: dict,
statement_guide: dict,
json_schema: dict,
language: str = "zh",
) -> str:
"""
Renders the temporal extraction prompt using the extract_temporal.jinja2 template.
@@ -115,7 +100,6 @@ async def render_temporal_extraction_prompt(
temporal_guide: Guidance on temporal types.
statement_guide: Guidance on statement types.
json_schema: JSON schema for the expected output format.
language: Language for output ("zh" for Chinese, "en" for English)
Returns:
Rendered prompt content as a string.
@@ -127,7 +111,6 @@ async def render_temporal_extraction_prompt(
temporal_guide=temporal_guide,
statement_guide=statement_guide,
json_schema=json_schema,
language=language,
)
# 记录渲染结果到提示日志(与示例日志结构一致)
log_prompt_rendering('temporal extraction', rendered_prompt)
@@ -147,7 +130,6 @@ def render_entity_dedup_prompt(
context: dict,
json_schema: dict,
disambiguation_mode: bool = False,
language: str = "zh",
) -> str:
"""
Render the entity deduplication prompt using the entity_dedup.jinja2 template.
@@ -157,8 +139,6 @@ def render_entity_dedup_prompt(
entity_b: Dict of entity B attributes
context: Dict of computed signals (group/type gate, similarities, co-occurrence, relation statements)
json_schema: JSON schema for the structured output (EntityDedupDecision)
disambiguation_mode: Whether to use disambiguation mode
language: Language for output ("zh" for Chinese, "en" for English)
Returns:
Rendered prompt content as string
@@ -177,7 +157,6 @@ def render_entity_dedup_prompt(
relation_statements=context.get("relation_statements", []),
json_schema=json_schema,
disambiguation_mode=disambiguation_mode,
language=language,
)
# prompt_logger.info("\n=== RENDERED ENTITY DEDUP PROMPT ===")
@@ -198,14 +177,7 @@ def render_entity_dedup_prompt(
# Args:
# entity_a: Dict of entity A attributes
async def render_triplet_extraction_prompt(
statement: str,
chunk_content: str,
json_schema: dict,
predicate_instructions: dict = None,
language: str = "zh",
ontology_types: "OntologyTypeList | None" = None,
) -> str:
async def render_triplet_extraction_prompt(statement: str, chunk_content: str, json_schema: dict, predicate_instructions: dict = None, language: str = "zh") -> str:
"""
Renders the triplet extraction prompt using the extract_triplet.jinja2 template.
@@ -215,31 +187,17 @@ async def render_triplet_extraction_prompt(
json_schema: JSON schema for the expected output format
predicate_instructions: Optional predicate instructions
language: The language to use for entity descriptions ("zh" for Chinese, "en" for English)
ontology_types: Optional OntologyTypeList containing predefined ontology types for entity classification
Returns:
Rendered prompt content as string
"""
template = prompt_env.get_template("extract_triplet.jinja2")
# 准备本体类型数据
ontology_type_section = ""
ontology_type_names = []
type_hierarchy_hints = []
if ontology_types and ontology_types.types:
ontology_type_section = ontology_types.to_prompt_section()
ontology_type_names = ontology_types.get_type_names()
type_hierarchy_hints = ontology_types.get_type_hierarchy_hints()
rendered_prompt = template.render(
statement=statement,
chunk_content=chunk_content,
json_schema=json_schema,
predicate_instructions=predicate_instructions,
language=language,
ontology_types=ontology_type_section,
ontology_type_names=ontology_type_names,
type_hierarchy_hints=type_hierarchy_hints,
language=language
)
# 记录渲染结果到提示日志(与示例日志结构一致)
log_prompt_rendering('triplet extraction', rendered_prompt)
@@ -249,10 +207,7 @@ async def render_triplet_extraction_prompt(
'chunk_content': 'str',
'json_schema': 'TripletExtractionResponse.schema',
'predicate_instructions': 'PREDICATE_DEFINITIONS',
'language': language,
'ontology_types': bool(ontology_type_section),
'ontology_type_count': len(ontology_type_names),
'type_hierarchy_hints_count': len(type_hierarchy_hints),
'language': language
})
return rendered_prompt
@@ -294,8 +249,7 @@ async def render_memory_summary_prompt(
async def render_emotion_extraction_prompt(
statement: str,
extract_keywords: bool,
enable_subject: bool,
language: str = "zh"
enable_subject: bool
) -> str:
"""
Renders the emotion extraction prompt using the extract_emotion.jinja2 template.
@@ -304,7 +258,6 @@ async def render_emotion_extraction_prompt(
statement: The statement to analyze
extract_keywords: Whether to extract emotion keywords
enable_subject: Whether to enable subject classification
language: Language for output ("zh" for Chinese, "en" for English)
Returns:
Rendered prompt content as string
@@ -313,8 +266,7 @@ async def render_emotion_extraction_prompt(
rendered_prompt = template.render(
statement=statement,
extract_keywords=extract_keywords,
enable_subject=enable_subject,
language=language
enable_subject=enable_subject
)
# 记录渲染结果到提示日志
@@ -331,8 +283,7 @@ async def render_emotion_extraction_prompt(
async def render_emotion_suggestions_prompt(
health_data: dict,
patterns: dict,
user_profile: dict,
language: str = "zh"
user_profile: dict
) -> str:
"""
Renders the emotion suggestions generation prompt using the generate_emotion_suggestions.jinja2 template.
@@ -341,7 +292,6 @@ async def render_emotion_suggestions_prompt(
health_data: 情绪健康数据
patterns: 情绪模式分析结果
user_profile: 用户画像数据
language: 输出语言 ("zh" 中文, "en" 英文)
Returns:
Rendered prompt content as string
@@ -349,39 +299,18 @@ async def render_emotion_suggestions_prompt(
import json
# 预处理 emotion_distribution 为 JSON 字符串
# 如果是中文,将 emotion_distribution 的 key 翻译为中文
emotion_distribution = health_data.get('emotion_distribution', {})
if language == "zh":
emotion_type_zh = {
'joy': '喜悦', 'sadness': '悲伤', 'anger': '愤怒',
'fear': '恐惧', 'surprise': '惊讶', 'neutral': '中性'
}
emotion_distribution = {
emotion_type_zh.get(k, k): v for k, v in emotion_distribution.items()
}
emotion_distribution_json = json.dumps(
emotion_distribution,
health_data.get('emotion_distribution', {}),
ensure_ascii=False,
indent=2
)
# 翻译 dominant_negative_emotion
dominant_negative_translated = None
dominant_neg = patterns.get('dominant_negative_emotion')
if dominant_neg and language == "zh":
emotion_type_zh_map = {
'sadness': '悲伤', 'anger': '愤怒', 'fear': '恐惧'
}
dominant_negative_translated = emotion_type_zh_map.get(dominant_neg, dominant_neg)
template = prompt_env.get_template("generate_emotion_suggestions.jinja2")
rendered_prompt = template.render(
health_data=health_data,
patterns=patterns,
user_profile=user_profile,
emotion_distribution_json=emotion_distribution_json,
language=language,
dominant_negative_translated=dominant_negative_translated
emotion_distribution_json=emotion_distribution_json
)
# 记录渲染结果到提示日志
@@ -399,9 +328,7 @@ async def render_emotion_suggestions_prompt(
async def render_user_summary_prompt(
user_id: str,
entities: str,
statements: str,
language: str = "zh",
user_display_name: str = None
statements: str
) -> str:
"""
Renders the user summary prompt using the user_summary.jinja2 template.
@@ -410,23 +337,15 @@ async def render_user_summary_prompt(
user_id: User identifier
entities: Core entities with frequency information
statements: Representative statement samples
language: The language to use for summary generation ("zh" for Chinese, "en" for English)
user_display_name: Display name for the user (e.g., other_name or "该用户"/"the user")
Returns:
Rendered prompt content as string
"""
# 如果没有提供 user_display_name使用默认值
if user_display_name is None:
user_display_name = "该用户" if language == "zh" else "the user"
template = prompt_env.get_template("user_summary.jinja2")
rendered_prompt = template.render(
user_id=user_id,
entities=entities,
statements=statements,
language=language,
user_display_name=user_display_name
statements=statements
)
# 记录渲染结果到提示日志
@@ -435,9 +354,7 @@ async def render_user_summary_prompt(
log_template_rendering('user_summary.jinja2', {
'user_id': user_id,
'entities_len': len(entities),
'statements_len': len(statements),
'language': language,
'user_display_name': user_display_name
'statements_len': len(statements)
})
return rendered_prompt
@@ -446,8 +363,7 @@ async def render_user_summary_prompt(
async def render_memory_insight_prompt(
domain_distribution: str = None,
active_periods: str = None,
social_connections: str = None,
language: str = "zh"
social_connections: str = None
) -> str:
"""
Renders the memory insight prompt using the memory_insight.jinja2 template.
@@ -456,7 +372,6 @@ async def render_memory_insight_prompt(
domain_distribution: 核心领域分布信息
active_periods: 活跃时段信息
social_connections: 社交关联信息
language: The language to use for report generation ("zh" for Chinese, "en" for English)
Returns:
Rendered prompt content as string
@@ -465,8 +380,7 @@ async def render_memory_insight_prompt(
rendered_prompt = template.render(
domain_distribution=domain_distribution,
active_periods=active_periods,
social_connections=social_connections,
language=language
social_connections=social_connections
)
# 记录渲染结果到提示日志
@@ -475,8 +389,7 @@ async def render_memory_insight_prompt(
log_template_rendering('memory_insight.jinja2', {
'has_domain_distribution': bool(domain_distribution),
'has_active_periods': bool(active_periods),
'has_social_connections': bool(social_connections),
'language': language
'has_social_connections': bool(social_connections)
})
return rendered_prompt
@@ -511,8 +424,7 @@ async def render_ontology_extraction_prompt(
scenario: str,
domain: str | None = None,
max_classes: int = 15,
json_schema: dict | None = None,
language: str = "zh"
json_schema: dict | None = None
) -> str:
"""
Renders the ontology extraction prompt using the extract_ontology.jinja2 template.
@@ -522,7 +434,6 @@ async def render_ontology_extraction_prompt(
domain: Optional domain hint for the scenario (e.g., "Healthcare", "Education")
max_classes: Maximum number of classes to extract (default: 15)
json_schema: JSON schema for the expected output format
language: Language for output ("zh" for Chinese, "en" for English)
Returns:
Rendered prompt content as string
@@ -532,8 +443,7 @@ async def render_ontology_extraction_prompt(
scenario=scenario,
domain=domain,
max_classes=max_classes,
json_schema=json_schema,
language=language
json_schema=json_schema
)
# 记录渲染结果到提示日志
@@ -543,25 +453,7 @@ async def render_ontology_extraction_prompt(
'scenario_len': len(scenario) if scenario else 0,
'domain': domain,
'max_classes': max_classes,
'json_schema': 'OntologyExtractionResponse.schema',
'language': language
'json_schema': 'OntologyExtractionResponse.schema'
})
return rendered_prompt
def render_interest_filter_prompt(tag_list: str, language: str = "zh") -> str:
"""
Renders the interest filter prompt using the interest_filter.jinja2 template.
Args:
tag_list: Comma-separated string of raw tags to filter
language: Output language ("zh" for Chinese, "en" for English)
Returns:
Rendered prompt content as string
"""
template = prompt_env.get_template("interest_filter.jinja2")
rendered_prompt = template.render(tag_list=tag_list, language=language)
log_prompt_rendering('interest filter', rendered_prompt)
return rendered_prompt

View File

@@ -1,16 +1,9 @@
===Task===
{% if language == "zh" %}
===任务===
你是一个实体去重/消歧判断助手。你将被提供两个实体的详细信息和上下文,请严格根据指引判断它们是否是同一真实世界实体,并在需要时进行类型消歧。
模式: {{ '消歧模式' if disambiguation_mode else '去重模式' }}
{% else %}
You are an entity deduplication/disambiguation assistant. You will be provided with detailed information and context for two entities. Please strictly follow the guidelines to determine whether they are the same real-world entity and perform type disambiguation when necessary.
Mode: {{ 'Disambiguation Mode' if disambiguation_mode else 'Deduplication Mode' }}
{% endif %}
===Input===
{% if language == "zh" %}
===输入===
实体A:
- 名称: "{{ entity_a.name | default('') }}"
- 类型: "{{ entity_a.entity_type | default('') }}"
@@ -41,41 +34,8 @@ Mode: {{ 'Disambiguation Mode' if disambiguation_mode else 'Deduplication Mode'
{% for s in relation_statements %}
- {{ s }}
{% endfor %}
{% else %}
Entity A:
- Name: "{{ entity_a.name | default('') }}"
- Type: "{{ entity_a.entity_type | default('') }}"
- Description: "{{ entity_a.description | default('') }}"
- Aliases: {{ entity_a.aliases | default([]) }}
{# TODO: fact_summary feature temporarily disabled, to be enabled after future development #}
{# - Summary: "{{ entity_a.fact_summary | default('') }}" #}
- Connection Strength: "{{ entity_a.connect_strength | default('') }}"
Entity B:
- Name: "{{ entity_b.name | default('') }}"
- Type: "{{ entity_b.entity_type | default('') }}"
- Description: "{{ entity_b.description | default('') }}"
- Aliases: {{ entity_b.aliases | default([]) }}
{# TODO: fact_summary feature temporarily disabled, to be enabled after future development #}
{# - Summary: "{{ entity_b.fact_summary | default('') }}" #}
- Connection Strength: "{{ entity_b.connect_strength | default('') }}"
Context:
- Same Group: {{ same_group | default(false) }}
- Type Consistent or Unknown: {{ type_ok | default(false) }}
- Type Similarity (0-1): {{ type_similarity | default(0.0) }}
- Name Text Similarity (0-1): {{ name_text_sim | default(0.0) }}
- Name Embedding Similarity (0-1): {{ name_embed_sim | default(0.0) }}
- Name Contains Relationship: {{ name_contains | default(false) }}
- Context Co-occurrence (same statement refers to both): {{ co_occurrence | default(false) }}
- Related Relationship Statements (from entity-entity edges):
{% for s in relation_statements %}
- {{ s }}
{% endfor %}
{% endif %}
===Guidelines===
{% if language == "zh" %}
===判定指引===
{% if disambiguation_mode %}
- 这是"同名但类型不同"的消歧场景。请判断两者是否指向同一真实世界实体。
- 综合名称文本/向量相似度、别名、描述、摘要与上下文关系(同源与关系陈述)进行判断。
@@ -108,43 +68,8 @@ Context:
- 优先保留连接强度更强(strong/both)者;其余相同则保留描述/摘要更丰富者再相同时保留实体Acanonical_idx=0
- **注意**别名aliases已在三元组提取阶段获取合并时会自动整合无需在此阶段提取。
{% endif %}
{% else %}
{% if disambiguation_mode %}
- This is a disambiguation scenario for "same name but different types". Please determine whether they refer to the same real-world entity.
- Make judgments based on name text/vector similarity, aliases, descriptions, summaries, and contextual relationships (co-occurrence and relationship statements).
- **Alias Handling (High Priority)**:
* If the alias lists of both entities have intersections, this is a strong signal of identity
* If one entity's name appears in another entity's aliases, it should be considered a high-confidence match
* If one entity's alias exactly matches another entity's name, it should be considered a high-confidence match
* Alias matching weight should be higher than pure name text similarity
- If unable to determine with sufficient confidence, handle conservatively: do not merge, and suggest blocking this pair in other fuzzy/heuristic merges (block_pair=true).
- If merging is needed (should_merge=true), select the "canonical entity" (canonical_idx) and **must** provide a suggested unified type (suggested_type).
- **Type Unification Principles (Important)**:
* Prioritize more specific and accurate types (e.g., HistoricalPeriod over Organization, MilitaryCapability over Concept)
* If both types are specific but different, choose the type that best matches the entity's core semantics
* Generic types (Concept, Phenomenon, Condition, State, Attribute, Event) have lower priority than domain-specific types
* Suggested type must be consistent with context and entity description
- Canonical entity priority: higher connection strength (strong/both); if equal, retain the one with richer description/summary; if still equal, retain Entity A (canonical_idx=0).
- **Note**: Aliases are already obtained during triplet extraction and will be automatically integrated during merging; no need to extract at this stage.
{% else %}
- If entity types are the same or either is UNKNOWN/empty, can proceed as candidates; if types clearly conflict (e.g., person vs. item), unless aliases and descriptions are highly consistent, determine as different entities.
- **Alias Matching Priority (Highest Priority)**:
* If Entity A's name exactly matches any of Entity B's aliases, it should be considered a high-confidence match
* If Entity B's name exactly matches any of Entity A's aliases, it should be considered a high-confidence match
* If any alias of Entity A exactly matches any alias of Entity B, it should be considered a high-confidence match
* When aliases match exactly, merging should be considered even if name text similarity is low
* Alias matching confidence should be higher than pure name similarity matching
- Make judgments based on name text/vector similarity, aliases, descriptions, summaries, and contextual relationships.
- When context co-occurs or there are clear relationship statements supporting identity (e.g., the same object is repeatedly mentioned or aliases correspond), the judgment threshold can be moderately lowered.
- Conservative decision: when unable to determine with sufficient confidence, do not merge (same_entity=false).
- If merging is needed, select the "canonical entity to retain" (canonical_idx) as the more appropriate one:
- Prioritize retaining the one with stronger connection strength (strong/both); if equal, retain the one with richer description/summary; if still equal, retain Entity A (canonical_idx=0).
- **Note**: Aliases are already obtained during triplet extraction and will be automatically integrated during merging; no need to extract at this stage.
{% endif %}
{% endif %}
**Output format**
{% if language == "zh" %}
{% if disambiguation_mode %}
返回JSON格式必须包含以下字段
{
@@ -178,41 +103,6 @@ Context:
- confidence: 决策的置信度范围0.0-1.0
- reason: 决策理由的简短说明
{% endif %}
{% else %}
{% if disambiguation_mode %}
Return JSON format with the following required fields:
{
"should_merge": boolean,
"canonical_idx": 0 or 1,
"confidence": float (0.0-1.0),
"block_pair": boolean,
"suggested_type": "string or null",
"reason": "string"
}
**Field Descriptions**:
- should_merge: Whether these two entities should be merged (true/false)
- canonical_idx: Index of the canonical entity, 0 for Entity A, 1 for Entity B
- confidence: Confidence level of the decision, range 0.0-1.0
- block_pair: Whether to block this pair in other fuzzy/heuristic merges (true/false)
- suggested_type: Suggested unified type (string or null)
- reason: Brief explanation of the decision
{% else %}
Return JSON format with the following required fields:
{
"same_entity": boolean,
"canonical_idx": 0 or 1,
"confidence": float (0.0-1.0),
"reason": "string"
}
**Field Descriptions**:
- same_entity: Whether the two entities refer to the same real-world entity (true/false)
- canonical_idx: Index of the canonical entity, 0 for Entity A, 1 for Entity B
- confidence: Confidence level of the decision, range 0.0-1.0
- reason: Brief explanation of the decision
{% endif %}
{% endif %}
**CRITICAL JSON FORMATTING REQUIREMENTS:**
1. Use only standard ASCII double quotes (") for JSON structure - never use Chinese quotation marks ("") or other Unicode quotes
@@ -220,9 +110,5 @@ Return JSON format with the following required fields:
3. Do not include line breaks within JSON string values
4. Test your JSON output mentally to ensure it can be parsed correctly
{% if language == "zh" %}
输出语言应始终与输入语言相同。
{% else %}
The output language should always be the same as the input language.
{% endif %}
{{ json_schema }}

View File

@@ -1,6 +1,6 @@
{#
对话级抽取与相关性判定模板(用于剪枝加速)
输入pruning_scene, ontology_classes, dialog_text, language
输入pruning_scene, dialog_text
输出:严格 JSON不要包含任何多余文本字段
- is_related: bool是否与所选场景相关
- times: [string],从对话中抽取的时间相关文本(日期、时间、时间段、有效期等)
@@ -9,71 +9,29 @@
- contacts: [string],联系方式(电话/手机号/邮箱/微信/QQ等
- addresses: [string],地址/地点相关文本
- keywords: [string],其它有助于保留的重要关键词(与场景强相关的术语)
- preserve_keywords: [string],必须保留的情绪/兴趣/爱好/个人偏好相关词或短语片段
要求:
- 必须只输出上述 JSON且键名一致不得输出解释、前后缀不得包含注释。
- times/ids/amounts/contacts/addresses/keywords/preserve_keywords 仅抽取原文片段或规范化后的简单字符串。
- times/ids/amounts/contacts/addresses/keywords 仅抽取原文片段或规范化后的简单字符串。
- 仅输出上述键;避免多余解释或字段。
#}
{# ── 确定场景说明 ── #}
{% if ontology_classes and ontology_classes | length > 0 %}
{% if language == 'en' %}
{% set custom_types_str = ontology_classes | join(', ') %}
{% set instruction = 'Scene "' ~ pruning_scene ~ '": The dialogue is related to this scene if it involves any of the following entity types: ' ~ custom_types_str ~ '.' %}
{% else %}
{% set custom_types_str = ontology_classes | join('、') %}
{% set instruction = '场景「' ~ pruning_scene ~ '」:对话涉及以下任意实体类型时视为相关:' ~ custom_types_str ~ '。' %}
{% endif %}
{% else %}
{% if language == 'en' %}
{% set custom_types_str = '' %}
{% set instruction = 'Scene "' ~ pruning_scene ~ '": Determine whether the dialogue content is relevant to this scene based on overall context.' %}
{% else %}
{% set custom_types_str = '' %}
{% set instruction = '场景「' ~ pruning_scene ~ '」:根据对话整体内容判断是否与该场景相关。' %}
{% endif %}
{% set scene_instructions = {
'education': '教育场景:教学、课程、考试、作业、老师/学生互动、学习资源、学校管理等。',
'online_service': '在线客服场景:客户咨询、问题排查、服务工单、售后支持、订单/退款、工单升级等。',
'outbound': '外呼场景:电话外呼、邀约、调研问卷、线索跟进、对话脚本、回访记录等。'
} %}
{% set scene_key = pruning_scene %}
{% if scene_key not in scene_instructions %}
{% set scene_key = 'education' %}
{% endif %}
{% if language == "zh" %}
你是一个对话内容分析助手。请对下方对话全文进行一次性分析,完成两项任务:
1. 判断对话是否与指定场景相关;
2. 从对话中抽取所有需要保留的重要信息片段。
{% set instruction = scene_instructions[scene_key] %}
请在下方对话全文基础上,按该场景进行一次性抽取并判定相关性:
场景说明:{{ instruction }}
{% if custom_types_str %}
重要提示:只要对话中出现与上述实体类型({{ custom_types_str }}相关的内容即判定为相关is_related=true
{% endif %}
---
【必须保留的内容(不可删除)】
以下类型的内容无论是否与场景直接相关,都必须保留,请将其关键词/短语抽取到对应字段:
- 时间信息:日期、时间点、时间段、有效期 → times 字段
- 编号信息学号、工号、订单号、申请号、账号、ID → ids 字段
- 金额信息:价格、费用、金额(含货币符号或单位) → amounts 字段
- 联系方式电话、手机号、邮箱、微信、QQ → contacts 字段
- 地址信息:地点、地址、位置 → addresses 字段
- 场景关键词:与场景强相关的专业术语、事件名称 → keywords 字段
- **情绪与情感**:喜悦、悲伤、愤怒、焦虑、开心、难过、委屈、兴奋、害怕、担心、压力、感动等情绪表达 → preserve_keywords 字段
- **兴趣与爱好**:喜欢、热爱、爱好、擅长、享受、沉迷、着迷、讨厌某事物等个人偏好表达 → preserve_keywords 字段
- **个人观点与态度**:对某事物的明确看法、评价、立场 → preserve_keywords 字段
【可以删除的内容】
以下类型的内容属于低价值信息,可以在剪枝时删除:
- 纯寒暄问候:如"你好"、"在吗"、"拜拜"、"嗯"、"好的"、"哦"等无实质内容的短语
- 纯表情/符号:如"[微笑]"、"😊"、"哈哈"等
- 重复确认:如"对对对"、"是的是的"、"嗯嗯嗯"等无新增信息的重复
- 无意义填充:如"啊"、"呢"、"嘛"等语气词单独成句
**注意:即使消息很短,只要包含情绪、兴趣、爱好、个人观点等有价值信息,就必须保留,不得删除。**
例如:
- "我好开心呀" → 包含情绪开心必须保留preserve_keywords 中加入"开心"
- "好喜欢打羽毛球呀" → 包含兴趣爱好喜欢打羽毛球必须保留preserve_keywords 中加入"喜欢打羽毛球"
- "我好难过" → 包含情绪难过必须保留preserve_keywords 中加入"难过"
- "太好啦!看到你开心,我也跟着心情亮起来" → 包含情绪必须保留preserve_keywords 中加入"开心"
---
对话全文:
"""
{{ dialog_text }}
@@ -87,60 +45,5 @@
"amounts": [<string>...],
"contacts": [<string>...],
"addresses": [<string>...],
"keywords": [<string>...],
"preserve_keywords": [<string>...]
}
{% else %}
You are a dialogue content analysis assistant. Please analyze the full dialogue below in one pass and complete two tasks:
1. Determine whether the dialogue is relevant to the specified scene;
2. Extract all important information fragments that must be preserved.
Scenario Description: {{ instruction }}
{% if custom_types_str %}
Important: If the dialogue contains content related to any of the entity types above ({{ custom_types_str }}), mark it as relevant (is_related=true).
{% endif %}
---
[MUST PRESERVE (cannot be deleted)]
The following types of content must always be preserved regardless of scene relevance. Extract their keywords/phrases into the corresponding fields:
- Time information: dates, time points, durations, expiry dates → times field
- ID information: student IDs, employee IDs, order numbers, application numbers, account IDs → ids field
- Amount information: prices, fees, amounts (with currency symbols or units) → amounts field
- Contact information: phone numbers, emails, WeChat, QQ → contacts field
- Address information: locations, addresses, places → addresses field
- Scene keywords: professional terms and event names strongly related to the scene → keywords field
- **Emotions and feelings**: joy, sadness, anger, anxiety, happiness, sadness, excitement, fear, worry, stress, being moved, etc. → preserve_keywords field
- **Interests and hobbies**: likes, loves, hobbies, good at, enjoys, obsessed with, hates something, personal preferences → preserve_keywords field
- **Personal opinions and attitudes**: clear views, evaluations, or stances on something → preserve_keywords field
[CAN BE DELETED]
The following types of content are low-value and can be removed during pruning:
- Pure greetings: e.g., "hello", "are you there", "bye", "ok", "yeah" — short phrases with no substantive content
- Pure emojis/symbols: e.g., "[smile]", "😊", "haha"
- Repetitive confirmations: e.g., "yes yes yes", "right right", "uh huh" — repetitions with no new information
- Meaningless fillers: standalone interjections like "ah", "well", "hmm"
**Note: Even if a message is short, if it contains emotions, interests, hobbies, or personal opinions, it MUST be preserved.**
Examples:
- "I'm so happy!" → contains emotion (happy), must preserve; add "happy" to preserve_keywords
- "I love playing badminton!" → contains interest (love playing badminton), must preserve; add "love playing badminton" to preserve_keywords
- "I feel so sad" → contains emotion (sad), must preserve; add "sad" to preserve_keywords
---
Full Dialogue:
"""
{{ dialog_text }}
"""
Output strict JSON only (fixed keys, order doesn't matter):
{
"is_related": <true or false>,
"times": [<string>...],
"ids": [<string>...],
"amounts": [<string>...],
"contacts": [<string>...],
"addresses": [<string>...],
"keywords": [<string>...],
"preserve_keywords": [<string>...]
}
{% endif %}
"keywords": [<string>...]
}

View File

@@ -1,4 +1,3 @@
{% if language == "zh" %}
你是一个专业的情绪分析专家。请分析以下陈述句的情绪信息。
陈述句:{{ statement }}
@@ -56,62 +55,3 @@
- 主体分类要准确优先识别用户本人self
请以 JSON 格式返回结果。
{% else %}
You are a professional emotion analysis expert. Please analyze the emotional information in the following statement.
Statement: {{ statement }}
Please extract the following information:
1. emotion_type (Emotion Type):
- joy: happiness, delight, pleasure, satisfaction, cheerfulness
- sadness: sorrow, grief, disappointment, depression, regret
- anger: rage, irritation, dissatisfaction, annoyance, frustration
- fear: anxiety, worry, concern, nervousness, apprehension
- surprise: astonishment, amazement, shock, wonder
- neutral: neutral, objective statement, no obvious emotion
2. emotion_intensity (Emotion Intensity):
- 0.0-0.3: weak emotion
- 0.3-0.7: moderate emotion
- 0.7-1.0: strong emotion
{% if extract_keywords %}
3. emotion_keywords (Emotion Keywords):
- Words directly expressing emotions in the original sentence
- Extract up to 3 keywords
- Return empty list if no obvious emotion words
{% else %}
3. emotion_keywords (Emotion Keywords):
- Return empty list
{% endif %}
{% if enable_subject %}
4. emotion_subject (Emotion Subject):
- self: user's own emotions (includes "I", "we", "us" and other first-person pronouns)
- other: others' emotions (includes names, "he/she" and other third-person pronouns)
- object: evaluation of things (for products, places, events, etc.)
Note:
- If multiple subjects are present, prioritize identifying the user (self)
- If the subject cannot be clearly determined, default to self
5. emotion_target (Emotion Target):
- If there is a clear emotion target, extract its name
- If there is no clear target, return null
{% else %}
4. emotion_subject (Emotion Subject):
- Default to self
5. emotion_target (Emotion Target):
- Return null
{% endif %}
Notes:
- If the statement is an objective factual statement with no obvious emotion, mark as neutral
- Emotion intensity should match the context, do not over-interpret
- Emotion keywords should be accurate, do not add words not in the original sentence
- Subject classification should be accurate, prioritize identifying the user (self)
Please return the result in JSON format.
{% endif %}

View File

@@ -1,100 +1,19 @@
===Task===
{% if language == "zh" %}
从给定的场景描述中提取本体类,遵循本体工程标准。
{% else %}
Extract ontology classes from the given scenario description following ontology engineering standards.
{% endif %}
===Role===
{% if language == "zh" %}
你是一位专业的本体工程师精通知识表示和OWLWeb本体语言标准。你的任务是从场景描述中识别抽象类和概念而不是具体实例。
{% else %}
You are a professional ontology engineer with expertise in knowledge representation and OWL (Web Ontology Language) standards. Your task is to identify abstract classes and concepts from scenario descriptions, not concrete instances.
{% endif %}
===Scenario Description===
{{ scenario }}
{% if domain -%}
===Domain Hint===
{% if language == "zh" %}
此场景属于 **{{ domain }}** 领域。提取类时请考虑领域特定的概念和术语。
{% else %}
This scenario belongs to the **{{ domain }}** domain. Consider domain-specific concepts and terminology when extracting classes.
{% endif %}
{%- endif %}
===Output Language===
{% if language == "en" -%}
**IMPORTANT: All output content MUST be in English.**
- Class names (name field): English in PascalCase format
- Chinese name (name_chinese field): Provide Chinese translation
- Descriptions: MUST be in English
- Examples: MUST be in English
- Domain: MUST be in English
{%- else -%}
**IMPORTANT: Output content language requirements:**
- Class names (name field): English in PascalCase format
- Chinese name (name_chinese field): Chinese translation
- Descriptions: MUST be in Chinese (中文)
- Examples: MUST be in Chinese (中文)
- Domain: Can be in Chinese or English
{%- endif %}
===Extraction Rules===
{% if language == "zh" %}
**1. 抽象类,而非实例:**
- 提取抽象类别和概念(如"医疗程序"、"患者"、"诊断"
- 不要提取具体实例(如"张三"、"301房间"、"2024-01-15"
- 以"事物的类型"而非"具体事物"的角度思考
**2. 命名规范:**
- "name"字段使用中文名称
- 使用清晰、描述性的中文名称
- 示例:"医疗程序"、"医疗服务提供者"、"诊断测试"
**3. 领域相关性:**
- 专注于场景领域的核心类
- 优先提取代表关键概念、实体或关系的类
- 避免过于通用的类(如"事物"、"对象"),除非它们在领域中有特定含义
**4. 类数量:**
- 提取5到{{ max_classes }}个类
- 目标是覆盖场景主要概念的平衡集合
- 质量优于数量:优先选择定义明确的类
**5. 清晰的描述:**
- 用中文提供简洁、信息丰富的描述最多500字
- 描述类代表什么,而不是具体实例
- 使用清晰、自然的中文解释类在领域中的作用
**6. 具体示例:**
- 为每个类提供2-5个中文具体实例示例
- 示例应该是该类的具体、现实的实例
- 示例有助于阐明类的范围和含义
- 示例格式:["示例1", "示例2", "示例3"]
**7. 类层次结构:**
- 在适用的情况下识别父子关系
- 使用parent_class字段指定继承关系
- 父类必须是提取的类之一或标准OWL类
- 顶级类的parent_class设为null
**8. 实体类型:**
- 为每个类分配适当的entity_type
- 常见类型:"人物"、"组织"、"地点"、"事件"、"概念"、"过程"、"对象"、"角色"
- 选择最具体的适用类型
**9. 语言一致性:**
- 所有字段内容必须使用中文
- "name"字段使用中文名称
- "description"字段使用中文描述
- "examples"字段使用中文示例
- "entity_type"字段使用中文类型名称
- "domain"字段使用中文领域名称
{% else %}
**1. Abstract Classes, Not Instances:**
- Extract abstract categories and concepts (e.g., "MedicalProcedure", "Patient", "Diagnosis")
- Do NOT extract concrete instances (e.g., "John Smith", "Room 301", "2024-01-15")
@@ -105,6 +24,8 @@ This scenario belongs to the **{{ domain }}** domain. Consider domain-specific c
- Examples: "MedicalProcedure", "HealthcareProvider", "DiagnosticTest"
- Avoid: "medical procedure", "healthcare_provider", "diagnostic-test"
- Use clear, descriptive names in English
- Avoid abbreviations unless they are standard in the domain (e.g., "API", "DNA")
- Provide Chinese translation in the "name_chinese" field (e.g., "医疗程序", "医疗服务提供者", "诊断测试")
**3. Domain Relevance:**
- Focus on classes that are central to the scenario's domain
@@ -116,31 +37,17 @@ This scenario belongs to the **{{ domain }}** domain. Consider domain-specific c
- Aim for a balanced set covering the main concepts in the scenario
- Quality over quantity: prefer well-defined classes over exhaustive lists
**5. Clear Descriptions:**
{% if language == "en" -%}
- Provide concise, informative descriptions in English (max 500 characters)
- Provide concise, informative descriptions in Chinese (max 500 characters)
- Describe what the class represents, not specific instances
- Use clear, natural English language that explains the class's role in the domain
{%- else -%}
- Provide concise, informative descriptions in English (max 500 characters)
- Describe what the class represents, not specific instances
- Use clear, natural English language
{%- endif %}
- Use clear, natural Chinese language that explains the class's role in the domain
**6. Concrete Examples:**
{% if language == "en" -%}
- Provide 2-5 concrete instance examples in English for each class
- Provide 2-5 concrete instance examples in Chinese for each class
- Examples should be specific, realistic instances of the class
- Examples help clarify the class's scope and meaning
- Use natural English language for examples
- Example format: ["Example1", "Example2", "Example3"]
{%- else -%}
- Provide 2-5 concrete instance examples in English for each class
- Examples should be specific, realistic instances of the class
- Examples help clarify the class's scope and meaning
- Example format: ["Example1", "Example2", "Example3"]
{%- endif %}
- Use natural Chinese language for examples
- Example format: ["示例1", "示例2", "示例3"]
**7. Class Hierarchy:**
- Identify parent-child relationships where applicable
@@ -153,121 +60,20 @@ This scenario belongs to the **{{ domain }}** domain. Consider domain-specific c
- Common types: "Person", "Organization", "Location", "Event", "Concept", "Process", "Object", "Role"
- Choose the most specific type that applies
**9. Language Consistency:**
- All field content must be in English
- "name" field uses English PascalCase names
- "description" field uses English descriptions
- "examples" field uses English examples
- "entity_type" field uses English type names
- "domain" field uses English domain names
{% endif %}
**9. OWL Reserved Words:**
- Do NOT use OWL reserved words as class names
- Reserved words include: "Thing", "Nothing", "Class", "Property", "ObjectProperty", "DatatypeProperty", "AnnotationProperty", "Ontology", "Individual", "Literal"
- If a reserved word is needed, add a domain-specific prefix (e.g., "MedicalClass" instead of "Class")
**10. Language Consistency:**
- Extract all class names in English (PascalCase format) for the "name" field
- Provide Chinese translation for class names in the "name_chinese" field
- Descriptions MUST be in Chinese (中文)
- Examples MUST be in Chinese (中文)
- Use clear, natural Chinese language for descriptions and examples
===Examples===
{% if language == "zh" %}
**示例1医疗领域**
场景:"一家医院管理患者记录,安排预约,并协调医疗程序。医生诊断病情并开具治疗方案。"
输出:
{
"classes": [
{
"name": "患者",
"description": "在医疗机构接受医疗护理或治疗的人",
"examples": ["张三", "李四", "患有糖尿病的老年患者"],
"parent_class": null,
"entity_type": "人物",
"domain": "医疗"
},
{
"name": "医疗程序",
"description": "为医疗诊断或治疗而执行的系统性操作流程",
"examples": ["手术", "血液检查", "X光检查", "疫苗接种"],
"parent_class": null,
"entity_type": "过程",
"domain": "医疗"
},
{
"name": "诊断",
"description": "基于症状和检查结果对疾病或状况的识别",
"examples": ["糖尿病诊断", "癌症诊断", "流感诊断"],
"parent_class": null,
"entity_type": "概念",
"domain": "医疗"
},
{
"name": "医生",
"description": "诊断和治疗患者的持证医疗专业人员",
"examples": ["全科医生", "外科医生", "心脏病专家"],
"parent_class": null,
"entity_type": "角色",
"domain": "医疗"
},
{
"name": "治疗",
"description": "为治愈或管理疾病状况而提供的医疗护理或疗法",
"examples": ["药物治疗", "物理治疗", "化疗", "手术治疗"],
"parent_class": null,
"entity_type": "过程",
"domain": "医疗"
}
],
"domain": "医疗"
}
**示例2教育领域**
场景:"一所大学提供由教授教授的课程。学生注册项目,参加讲座,并完成作业以获得学位。"
输出:
{
"classes": [
{
"name": "学生",
"description": "在教育机构注册学习的人",
"examples": ["本科生", "研究生", "在职学生"],
"parent_class": null,
"entity_type": "角色",
"domain": "教育"
},
{
"name": "课程",
"description": "涵盖特定学科或主题的结构化教育课程",
"examples": ["计算机科学导论", "微积分I", "世界历史"],
"parent_class": null,
"entity_type": "概念",
"domain": "教育"
},
{
"name": "教授",
"description": "教授课程并进行研究的学术教师",
"examples": ["助理教授", "副教授", "正教授"],
"parent_class": null,
"entity_type": "角色",
"domain": "教育"
},
{
"name": "学术项目",
"description": "通向学位或证书的结构化课程体系",
"examples": ["理学学士", "文学硕士", "博士项目"],
"parent_class": null,
"entity_type": "概念",
"domain": "教育"
},
{
"name": "作业",
"description": "分配给学生以评估学习成果的任务或项目",
"examples": ["论文", "习题集", "研究报告", "实验报告"],
"parent_class": null,
"entity_type": "对象",
"domain": "教育"
}
],
"domain": "教育"
}
{% else %}
{% if language == "en" -%}
**Example 1 (Healthcare Domain):**
Scenario: "A hospital manages patient records, schedules appointments, and coordinates medical procedures. Doctors diagnose conditions and prescribe treatments."
@@ -277,8 +83,8 @@ Output:
{
"name": "Patient",
"name_chinese": "患者",
"description": "A person who receives medical care or treatment at a healthcare facility",
"examples": ["Outpatient", "Inpatient", "Emergency patient", "Chronic disease patient"],
"description": "在医疗机构接受医疗护理或治疗的人",
"examples": ["张三", "李四", "患有糖尿病的老年患者"],
"parent_class": null,
"entity_type": "Person",
"domain": "Healthcare"
@@ -286,8 +92,8 @@ Output:
{
"name": "MedicalProcedure",
"name_chinese": "医疗程序",
"description": "A systematic operation or process performed for medical diagnosis or treatment",
"examples": ["Surgery", "Blood test", "X-ray examination", "Vaccination"],
"description": "为医疗诊断或治疗而执行的系统性操作流程",
"examples": ["手术", "血液检查", "X光检查", "疫苗接种"],
"parent_class": null,
"entity_type": "Process",
"domain": "Healthcare"
@@ -295,8 +101,8 @@ Output:
{
"name": "Diagnosis",
"name_chinese": "诊断",
"description": "The identification of a disease or condition based on symptoms and examination results",
"examples": ["Diabetes diagnosis", "Cancer diagnosis", "Flu diagnosis"],
"description": "基于症状和检查结果对疾病或状况的识别",
"examples": ["糖尿病诊断", "癌症诊断", "流感诊断"],
"parent_class": null,
"entity_type": "Concept",
"domain": "Healthcare"
@@ -304,8 +110,8 @@ Output:
{
"name": "Doctor",
"name_chinese": "医生",
"description": "A licensed medical professional who diagnoses and treats patients",
"examples": ["General practitioner", "Surgeon", "Cardiologist"],
"description": "诊断和治疗患者的持证医疗专业人员",
"examples": ["全科医生", "外科医生", "心脏病专家"],
"parent_class": null,
"entity_type": "Role",
"domain": "Healthcare"
@@ -313,8 +119,8 @@ Output:
{
"name": "Treatment",
"name_chinese": "治疗",
"description": "Medical care or therapy provided to cure or manage a disease condition",
"examples": ["Medication therapy", "Physical therapy", "Chemotherapy", "Surgical treatment"],
"description": "为治愈或管理疾病状况而提供的医疗护理或疗法",
"examples": ["药物治疗", "物理治疗", "化疗", "手术治疗"],
"parent_class": null,
"entity_type": "Process",
"domain": "Healthcare"
@@ -323,56 +129,6 @@ Output:
"domain": "Healthcare",
"namespace": "http://example.org/healthcare#"
}
{%- else -%}
**Example 1 (Healthcare Domain):**
Scenario: "A hospital manages patient records, schedules appointments, and coordinates medical procedures. Doctors diagnose conditions and prescribe treatments."
Output:
{
"classes": [
{
"name": "Patient",
"description": "A person receiving medical care or treatment at a healthcare facility",
"examples": ["John Smith", "Jane Doe", "Elderly patient with diabetes"],
"parent_class": null,
"entity_type": "Person",
"domain": "Healthcare"
},
{
"name": "MedicalProcedure",
"description": "A systematic operation performed for medical diagnosis or treatment",
"examples": ["Surgery", "Blood test", "X-ray examination", "Vaccination"],
"parent_class": null,
"entity_type": "Process",
"domain": "Healthcare"
},
{
"name": "Diagnosis",
"description": "Identification of a disease or condition based on symptoms and examination results",
"examples": ["Diabetes diagnosis", "Cancer diagnosis", "Flu diagnosis"],
"parent_class": null,
"entity_type": "Concept",
"domain": "Healthcare"
},
{
"name": "Doctor",
"description": "A licensed medical professional who diagnoses and treats patients",
"examples": ["General practitioner", "Surgeon", "Cardiologist"],
"parent_class": null,
"entity_type": "Role",
"domain": "Healthcare"
},
{
"name": "Treatment",
"description": "Medical care or therapy provided to cure or manage a disease condition",
"examples": ["Medication therapy", "Physical therapy", "Chemotherapy", "Surgical treatment"],
"parent_class": null,
"entity_type": "Process",
"domain": "Healthcare"
}
],
"domain": "Healthcare"
}
**Example 2 (Education Domain):**
Scenario: "A university offers courses taught by professors. Students enroll in programs, attend lectures, and complete assignments to earn degrees."
@@ -382,49 +138,62 @@ Output:
"classes": [
{
"name": "Student",
"description": "A person enrolled in an educational institution for learning",
"examples": ["Undergraduate student", "Graduate student", "Part-time student"],
"name_chinese": "学生",
"description": "在教育机构注册学习的人",
"examples": ["本科生", "研究生", "在职学生"],
"parent_class": null,
"entity_type": "Role",
"domain": "Education"
},
{
"name": "Course",
"description": "A structured educational program covering a specific subject or topic",
"examples": ["Introduction to Computer Science", "Calculus I", "World History"],
"name_chinese": "课程",
"description": "涵盖特定学科或主题的结构化教育课程",
"examples": ["计算机科学导论", "微积分I", "世界历史"],
"parent_class": null,
"entity_type": "Concept",
"domain": "Education"
},
{
"name": "Professor",
"description": "An academic teacher who teaches courses and conducts research",
"examples": ["Assistant professor", "Associate professor", "Full professor"],
"name_chinese": "教授",
"description": "教授课程并进行研究的学术教师",
"examples": ["助理教授", "副教授", "正教授"],
"parent_class": null,
"entity_type": "Role",
"domain": "Education"
},
{
"name": "AcademicProgram",
"description": "A structured curriculum leading to a degree or certificate",
"examples": ["Bachelor of Science", "Master of Arts", "PhD program"],
"name_chinese": "学术项目",
"description": "通向学位或证书的结构化课程体系",
"examples": ["理学学士", "文学硕士", "博士项目"],
"parent_class": null,
"entity_type": "Concept",
"domain": "Education"
},
{
"name": "Assignment",
"description": "A task or project assigned to students to assess learning outcomes",
"examples": ["Essay", "Problem set", "Research paper", "Lab report"],
"name_chinese": "作业",
"description": "分配给学生以评估学习成果的任务或项目",
"examples": ["论文", "习题集", "研究报告", "实验报告"],
"parent_class": null,
"entity_type": "Object",
"domain": "Education"
},
{
"name": "Lecture",
"name_chinese": "讲座",
"description": "由教师进行的教育性演讲或讲座",
"examples": ["入门讲座", "客座讲座", "在线讲座"],
"parent_class": null,
"entity_type": "Event",
"domain": "Education"
}
],
"domain": "Education"
"domain": "Education",
"namespace": "http://example.org/education#"
}
{% endif %}
{% endif %}
===Output Format===
@@ -434,12 +203,8 @@ Output:
- Escape quotation marks in text with backslashes (\")
- Ensure proper string closure and comma separation
- No line breaks within JSON string values
- All class names must be in PascalCase format
- All class names must be unique (case-insensitive)
- Extract between 5 and {{ max_classes }} classes
{% if language == "zh" %}
- 所有字段内容必须使用中文
{% else %}
- All field content must be in English
{% endif %}
{{ json_schema }}

View File

@@ -5,13 +5,8 @@
===Tasks===
{% if language == "zh" %}
你的任务是根据详细的提取指南,从提供的对话片段中识别和提取陈述句。
每个陈述句必须按照下面提到的标准进行标记。
{% else %}
Your task is to identify and extract declarative statements from the provided conversational chunk based on the detailed extraction guidelines.
Each statement must be labeled as per the criteria mentioned below.
{% endif %}
===Inputs===
{% if inputs %}
@@ -22,32 +17,6 @@ Each statement must be labeled as per the criteria mentioned below.
===Extraction Instructions===
{% if language == "zh" %}
{% if granularity %}
{% if granularity == 3 %}
原子化和清晰:构建陈述句以清楚地显示单一的主谓宾关系。最好有多个较小的陈述句,而不是一个复杂的陈述句。
上下文独立:陈述句必须在不需要阅读整个对话的情况下可以理解。
{% elif granularity == 2 %}
在句子级别提取陈述句。每个陈述句应对应一个单一、完整的思想(通常是来源中的一个完整句子),但要重新表述以获得最大的清晰度,删除对话填充词(例如,"嗯"、"像"、感叹词)。
{% elif granularity == 1 %}
仅提取精华句子,并将片段总结为多个独立的陈述句,每个陈述句关注事实陈述、用户偏好、关系和显著的时间上下文。
{% endif %}
{% endif %}
上下文解析要求:
- 将指示代词("那个"、"这个"、"那些"、"这些")解析为其具体指代对象
- 如果陈述句包含无法从对话上下文中解析的模糊引用,则:
a) 扩展陈述句以包含对话早期的缺失上下文
b) 标记陈述句为需要额外上下文
c) 如果陈述句在没有上下文的情况下变得无意义,则跳过提取
对话上下文和共指消解:
- 将每个陈述句归属于说出它的参与者。
- 如果参与者列表为说话者提供了名称(例如,"李雪(用户)"),请在提取的陈述句中使用具体名称("李雪"),而不是通用角色("用户")。
- 将所有代词解析为对话上下文中的具体人物或实体。
- 识别并将抽象引用解析为其具体名称(如果提到)。
- 将缩写和首字母缩略词扩展为其完整形式。
{% else %}
{% if granularity %}
{% if granularity == 3 %}
Atomic & Clear: Structure statements to clearly show a single subject-predicate-object relationship. It is better to have multiple smaller statements than one complex one.
@@ -60,7 +29,7 @@ Extract only essence sentences and summarize the chunk into multiple, standalone
{% endif %}
Context Resolution Requirements:
- Resolve demonstrative pronouns ("that," "this," "those") to their specific referents
- Resolve demonstrative pronouns ("that," "this," "those","这个", "那个") to their specific referents
- If a statement contains vague references that cannot be resolved from the conversation context, either:
a) Expand the statement to include the missing context from earlier in the conversation
b) Mark the statement as requiring additional context
@@ -72,36 +41,16 @@ Conversational Context & Co-reference Resolution:
- Resolve all pronouns to the specific person or entity from the conversation's context.
- Identify and resolve abstract references to their specific names if mentioned.
- Expand abbreviations and acronyms to their full form.
{% endif %}
{% if include_dialogue_context %}
{% if language == "zh" %}
===完整对话上下文===
以下是完整的对话上下文,以帮助您理解引用、代词和对话流程:
{% else %}
===Full Dialogue Context===
The following is the complete dialogue context to help you understand references, pronouns, and conversational flow:
{% endif %}
{{ dialogue_context }}
{% if language == "zh" %}
===对话上下文结束===
{% else %}
===End of Dialogue Context===
{% endif %}
{% endif %}
{% if language == "zh" %}
过滤和格式化:
- 仅提取陈述句。
不要提取问题、命令、问候语或对话填充词。
时间精度:
包括任何明确的日期、时间或定量限定符。
如果一个句子既描述了事件的开始(静态)又描述了其持续性质(动态),则将两者提取为单独的陈述句。
{% else %}
Filtering and Formatting:
- Extract only declarative statements.
@@ -110,114 +59,18 @@ Temporal Precision:
Include any explicit dates, times, or quantitative qualifiers.
If a sentence describes both the start of an event (static) and its ongoing nature (dynamic), extract both as separate statements.
{% endif %}
{%- if definitions %}
{%- for section_key, section_dict in definitions.items() %}
==== {{ tidy(section_key) | upper }} {% if language == "zh" %}定义和指导{% else %}DEFINITIONS & GUIDANCE{% endif %} ====
==== {{ tidy(section_key) | upper }} DEFINITIONS & GUIDANCE ====
{%- for category, details in section_dict.items() %}
{{ loop.index }}. {{ category }}
- {% if language == "zh" %}定义{% else %}Definition{% endif %}: {{ details.get("definition", "") }}
- Definition: {{ details.get("definition", "") }}
{% endfor -%}
{% endfor -%}
{% endif -%}
===Examples===
{% if language == "zh" %}
示例 1: 英文对话
示例片段: """
日期: 2024年3月15日
参与者:
- Sarah Chen (用户)
- 助手 (AI)
用户: "我最近一直在尝试水彩画,画了一些花朵。"
AI: "水彩画很有趣!水彩颜料通常由颜料与阿拉伯树胶等粘合剂混合而成。你觉得怎么样?"
用户: "我认为色彩组合可以改进,但我真的很喜欢玫瑰和百合。"
"""
示例输出: {
"statements": [
{
"statement": "Sarah Chen 最近一直在尝试水彩画。",
"statement_type": "FACT",
"temporal_type": "DYNAMIC",
"relevance": "RELEVANT"
},
{
"statement": "Sarah Chen 画了一些花朵。",
"statement_type": "FACT",
"temporal_type": "DYNAMIC",
"relevance": "RELEVANT"
},
{
"statement": "水彩颜料通常由颜料与阿拉伯树胶等粘合剂混合而成。",
"statement_type": "FACT",
"temporal_type": "ATEMPORAL",
"relevance": "IRRELEVANT"
},
{
"statement": "Sarah Chen 认为她的水彩画中的色彩组合可以改进。",
"statement_type": "OPINION",
"temporal_type": "STATIC",
"relevance": "RELEVANT"
},
{
"statement": "Sarah Chen 真的很喜欢玫瑰和百合。",
"statement_type": "FACT",
"temporal_type": "STATIC",
"relevance": "RELEVANT"
}
]
}
示例 2: 中文对话示例
示例片段: """
日期: 2024年3月15日
参与者:
- 张曼婷 (用户)
- 小助手 (AI助手)
用户: "我最近在尝试水彩画,画了一些花朵。"
AI: "水彩画很有趣!水彩颜料通常由颜料和阿拉伯树胶等粘合剂混合而成。你觉得怎么样?"
用户: "我觉得色彩搭配还有提升的空间,不过我很喜欢玫瑰和百合这两种花。"
"""
示例输出: {
"statements": [
{
"statement": "张曼婷最近在尝试水彩画。",
"statement_type": "FACT",
"temporal_type": "DYNAMIC",
"relevance": "RELEVANT"
},
{
"statement": "张曼婷画了一些花朵。",
"statement_type": "FACT",
"temporal_type": "DYNAMIC",
"relevance": "RELEVANT"
},
{
"statement": "水彩颜料通常由颜料和阿拉伯树胶等粘合剂混合而成。",
"statement_type": "FACT",
"temporal_type": "ATEMPORAL",
"relevance": "IRRELEVANT"
},
{
"statement": "张曼婷觉得水彩画的色彩搭配还有提升的空间。",
"statement_type": "OPINION",
"temporal_type": "STATIC",
"relevance": "RELEVANT"
},
{
"statement": "张曼婷很喜欢玫瑰和百合。",
"statement_type": "FACT",
"temporal_type": "STATIC",
"relevance": "RELEVANT"
}
]
}
{% else %}
Example 1: English Conversation
Example Chunk: """
Date: March 15, 2024
@@ -311,33 +164,8 @@ Example Output: {
}
]
}
{% endif %}
===End of Examples===
{% if language == "zh" %}
===反思过程===
提取陈述句后,执行以下自我审查步骤:
**步骤 1: 归属检查**
- 确认每个陈述句都正确归属于正确的说话者
- 验证说话者名称在整个过程中使用一致
- 检查 AI 助手陈述句是否正确归属
**步骤 2: 完整性审查**
- 确保没有遗漏重要的陈述句
- 检查时间信息是否保留
**步骤 3: 分类验证**
- 审查 statement_type 分类FACT/OPINION/PREDICTION/SUGGESTION
- 验证 temporal_type 分配STATIC/DYNAMIC/ATEMPORAL
- 确保分类与提供的定义一致
**步骤 4: 最终质量检查**
- 删除任何问题、命令或对话填充词
- 验证 JSON 格式合规性
- 确认输出语言与输入语言匹配
{% else %}
===Reflection Process===
After extracting statements, perform the following self-review steps:
@@ -360,7 +188,6 @@ After extracting statements, perform the following self-review steps:
- Remove any questions, commands, or conversational filler
- Verify JSON format compliance
- Confirm output language matches input language
{% endif %}
**Output format**
**CRITICAL JSON FORMATTING REQUIREMENTS:**
@@ -371,21 +198,10 @@ After extracting statements, perform the following self-review steps:
5. Example of proper escaping: "statement": "John said: \"I really like this book.\""
**LANGUAGE REQUIREMENT:**
{% if language == "zh" %}
- 输出语言应始终与输入语言匹配
- 如果输入是中文,则用中文提取陈述句
- 如果输入是英文,则用英文提取陈述句
- 保留原始语言,不要翻译
{% else %}
- The output language should ALWAYS match the input language
- If input is in English, extract statements in English
- If input is in Chinese, extract statements in Chinese
- Preserve the original language and do not translate
{% endif %}
{% if language == "zh" %}
仅返回与以下架构匹配的 JSON 对象数组中提取的标记陈述句列表:
{% else %}
Return only a list of extracted labelled statements in the JSON ARRAY of objects that match the schema below:
{% endif %}
{{ json_schema }}
{{ json_schema }}

View File

@@ -14,113 +14,68 @@
#}
# Task
{% if language == "zh" %}
从提供的陈述句中提取时间信息(日期和时间范围)。确定所描述的关系或事件何时生效以及何时结束(如果适用)。
{% else %}
Extract temporal information (dates and time ranges) from the provided statement. Determine when the relationship or event described became valid and when it ended (if applicable).
{% endif %}
# {% if language == "zh" %}输入数据{% else %}Input Data{% endif %}
# Input Data
{% if inputs %}
{% for key, val in inputs.items() %}
- {{ key }}: {{val}}
{% endfor %}
{% endif %}
# {% if language == "zh" %}时间字段{% else %}Temporal Fields{% endif %}
# Temporal Fields
{% if language == "zh" %}
- **valid_at**: 关系/事件开始或成为真实的时间ISO 8601 格式)
- **invalid_at**: 关系/事件结束或停止为真的时间ISO 8601 格式,如果正在进行则为 null
{% else %}
- **valid_at**: When the relationship/event started or became true (ISO 8601 format)
- **invalid_at**: When the relationship/event ended or stopped being true (ISO 8601 format, or null if ongoing)
{% endif %}
# {% if language == "zh" %}提取规则{% else %}Extraction Rules{% endif %}
# Extraction Rules
## {% if language == "zh" %}核心原则{% else %}Core Principles{% endif %}
{% if language == "zh" %}
1. **仅使用明确陈述的时间信息** - 不要从外部知识推断日期
2. **使用参考/发布日期作为"现在"** 解释相对时间时
3. **仅在日期与关系的有效性相关时设置日期** - 忽略偶然的时间提及
4. **对于时间点事件**,仅设置 `valid_at`
{% else %}
## Core Principles
1. **Only use explicitly stated temporal information** - do not infer dates from external knowledge
2. **Use the reference/publication date as "now"** when interpreting relative times
3. **Set dates only if they relate to the validity of the relationship** - ignore incidental time mentions
4. **For point-in-time events**, set only `valid_at`
{% endif %}
## {% if language == "zh" %}日期格式要求{% else %}Date Format Requirements{% endif %}
{% if language == "zh" %}
- 使用 ISO 8601: `YYYY-MM-DDTHH:MM:SS.SSSSSSZ`
- 如果未指定时间,使用 `00:00:00`(午夜)
- 如果仅提及年份,根据情况使用 `YYYY-01-01`(开始)或 `YYYY-12-31`(结束)
- 如果仅提及月份,使用月份的第一天或最后一天
- 始终包含时区(如果未指定,使用 `Z` 表示 UTC
- 根据参考日期将相对时间("两周前"、"去年")转换为绝对日期
{% else %}
## Date Format Requirements
- Use ISO 8601: `YYYY-MM-DDTHH:MM:SS.SSSSSSZ`
- If no time specified, use `00:00:00` (midnight)
- If only year mentioned, use `YYYY-01-01` (start) or `YYYY-12-31` (end) as appropriate
- If only month mentioned, use first or last day of month
- Always include timezone (use `Z` for UTC if unspecified)
- Convert relative times ("two weeks ago", "last year") to absolute dates based on reference date
{% endif %}
## {% if language == "zh" %}陈述句类型规则{% else %}Statement Type Rules{% endif %}
## Statement Type Rules
{{ inputs.get("statement_type") | upper }} {% if language == "zh" %}陈述句指导{% else %}Statement Guidance{% endif %}:
{{ inputs.get("statement_type") | upper }} Statement Guidance:
{%for key, guide in statement_guide.items() %}
- {{ tidy(key) | capitalize }}: {{ guide }}
{% endfor %}
**{% if language == "zh" %}特殊情况{% else %}Special Cases{% endif %}:**
{% if language == "zh" %}
- **意见陈述句**: 仅设置 `valid_at`(意见表达的时间)
- **预测陈述句**: 如果明确提及,将 `invalid_at` 设置为预测窗口的结束
{% else %}
**Special Cases:**
- **Opinion statements**: Set only `valid_at` (when opinion was expressed)
- **Prediction statements**: Set `invalid_at` to the end of the prediction window if explicitly mentioned
{% endif %}
## {% if language == "zh" %}时间类型规则{% else %}Temporal Type Rules{% endif %}
## Temporal Type Rules
{{ inputs.get("temporal_type") | upper }} {% if language == "zh" %}时间类型指导{% else %}Temporal Type Guidance{% endif %}:
{{ inputs.get("temporal_type") | upper }} Temporal Type Guidance:
{% for key, guide in temporal_guide.items() %}
- {{ tidy(key) | capitalize }}: {{ guide }}
{% endfor %}
{% if inputs.get('quarter') and inputs.get('publication_date') %}
## {% if language == "zh" %}季度参考{% else %}Quarter Reference{% endif %}
{% if language == "zh" %}
假设 {{ inputs.quarter }} 在 {{ inputs.publication_date }} 结束。从此基线计算任何季度引用Q1、Q2 等)的日期。
{% else %}
## Quarter Reference
Assume {{ inputs.quarter }} ends on {{ inputs.publication_date }}. Calculate dates for any quarter references (Q1, Q2, etc.) from this baseline.
{% endif %}
{% endif %}
# {% if language == "zh" %}输出要求{% else %}Output Requirements{% endif %}
# Output Requirements
## {% if language == "zh" %}JSON 格式化(关键){% else %}JSON Formatting (CRITICAL){% endif %}
{% if language == "zh" %}
1. 使用**仅标准 ASCII 双引号** (") - 永远不要使用中文引号("")或其他 Unicode 变体
2. 使用反斜杠转义内部引号: `\"`
3. JSON 字符串值中不要有换行符
4. 正确关闭并用逗号分隔所有字段
{% else %}
## JSON Formatting (CRITICAL)
1. Use **only standard ASCII double quotes** (") - never use Chinese quotes ("") or other Unicode variants
2. Escape internal quotes with backslash: `\"`
3. No line breaks within JSON string values
4. Properly close and comma-separate all fields
{% endif %}
## {% if language == "zh" %}语言{% else %}Language{% endif %}
{% if language == "zh" %}
输出语言必须与输入语言匹配。
{% else %}
## Language
Output language must match input language.
{% endif %}
{{ json_schema }}

View File

@@ -6,96 +6,64 @@
Extract entities and knowledge triplets from the given statement.
{% if language == "zh" %}
**重要:请使用中文生成实体名称name描述description和示例example。**
**重要请使用中文生成实体描述description和示例example。**
{% else %}
**Important: Please generate entity names, descriptions and examples in English. If the original text is in Chinese, translate entity names to English.**
**Important: Please generate entity descriptions and examples in English.**
{% endif %}
===Inputs===
**Chunk Content:** "{{ chunk_content }}"
**Statement:** "{{ statement }}"
{% if ontology_types %}
===Ontology Type Guidance===
**CRITICAL RULE: You MUST ONLY use the predefined ontology type names listed below for the entity "type" field. Do NOT use any other type names, even if they seem reasonable.**
**If no predefined type fits an entity, use the CLOSEST matching predefined type. NEVER invent new type names.**
**Type Priority (from highest to lowest):**
1. **[场景类型] Scene Types** - Domain-specific types, ALWAYS prefer these first
2. **[通用类型] General Types** - Common types from standard ontologies (DBpedia)
3. **[通用父类] Parent Types** - Provide type hierarchy context
**Type Matching Rules:**
- Entity type MUST exactly match one of the predefined type names below
- Do NOT use types like "Equipment", "Component", "Concept", "Action", "Condition", "Data", "Duration" unless they appear in the predefined list
- Do NOT modify, translate, abbreviate, or create variations of type names
- Prefer scene types (marked [场景类型]) over general types when both could apply
- If uncertain, check the type description to find the best match
**Predefined Ontology Types:**
{{ ontology_types }}
{% if type_hierarchy_hints %}
**Type Hierarchy Reference:**
The following shows type inheritance relationships (Child → Parent → Grandparent):
{% for hint in type_hierarchy_hints %}
- {{ hint }}
{% endfor %}
{% endif %}
**ALLOWED Type Names (use EXACTLY one of these, no exceptions):**
{{ ontology_type_names | join(', ') }}
{% endif %}
===Guidelines===
**Entity Extraction:**
- Extract entities with their types, context-independent descriptions, **concise examples**, aliases, and semantic memory classification
{% if language == "zh" %}
- **实体名称name必须使用中文**
- **实体描述description必须使用中文**
- **示例example必须使用中文**
{% else %}
- **Entity names must be in English** (translate if the original is in another language)
- **Entity descriptions must be in English**
- **Examples must be in English**
{% endif %}
- **Semantic Memory Classification (is_explicit_memory):**
* Set to `true` if the entity represents **explicit/semantic memory**:
- **Concepts:** "Machine Learning", "Photosynthesis", "Democracy"
- **Knowledge:** "Python Programming Language", "Theory of Relativity"
- **Definitions:** "API (Application Programming Interface)", "REST API"
- **Principles:** "SOLID Principles", "First Law of Thermodynamics"
- **Theories:** "Evolution Theory", "Quantum Mechanics"
- **Methods/Techniques:** "Agile Development", "Machine Learning Algorithm"
- **Technical Terms:** "Neural Network", "Database"
- **Concepts:** "Machine Learning", "Photosynthesis", "Democracy", "人工智能", "光合作用", "民主"
- **Knowledge:** "Python Programming Language", "Theory of Relativity", "Python编程语言", "相对论"
- **Definitions:** "API (Application Programming Interface)", "REST API", "应用程序接口"
- **Principles:** "SOLID Principles", "First Law of Thermodynamics", "SOLID原则", "热力学第一定律"
- **Theories:** "Evolution Theory", "Quantum Mechanics", "进化论", "量子力学"
- **Methods/Techniques:** "Agile Development", "Machine Learning Algorithm", "敏捷开发", "机器学习算法"
- **Technical Terms:** "Neural Network", "Database", "神经网络", "数据库"
* Set to `false` for:
- **People:** "John Smith", "Dr. Wang"
- **Organizations:** "Microsoft", "Harvard University"
- **Locations:** "Beijing", "Central Park"
- **Events:** "2024 Conference", "Project Meeting"
- **Specific objects:** "iPhone 15", "Building A"
- **People:** "John Smith", "Dr. Wang", "张明", "王博士"
- **Organizations:** "Microsoft", "Harvard University", "微软", "哈佛大学"
- **Locations:** "Beijing", "Central Park", "北京", "中央公园"
- **Events:** "2024 Conference", "Project Meeting", "2024会议", "项目会议"
- **Specific objects:** "iPhone 15", "Building A", "iPhone 15", "A栋"
- **Example Generation (IMPORTANT for semantic memory entities):**
* For entities where `is_explicit_memory=true`, generate a **concise example (around 20 characters)** to help understand the concept
* The example should be:
- **Specific and concrete**: Use real-world scenarios or applications
- **Brief**: Around 20 characters (can be slightly longer if needed for clarity)
{% if language == "zh" %}
- **使用中文**
{% else %}
- **In English**
{% endif %}
- **In the same language as the entity name**
* Examples:
- Entity: "机器学习" → example: "如:用神经网络识别图片中的猫狗"
- Entity: "SOLID Principles" → example: "e.g., Single Responsibility, Open-Closed"
- Entity: "Photosynthesis" → example: "e.g., plants convert sunlight to energy"
- Entity: "人工智能" → example: "如:智能客服、自动驾驶"
* For non-semantic entities (`is_explicit_memory=false`), the example field can be empty
- **Aliases Extraction:**
{% if language == "zh" %}
* 别名使用中文
{% else %}
* Aliases should be in English
{% endif %}
* Include common alternative names, abbreviations and full names
* If no aliases exist, use empty array: []
- **Aliases Extraction (Important):**
* **CRITICAL: Extract aliases ONLY in the SAME LANGUAGE as the input text**
* **DO NOT translate or add aliases in different languages**
* Include common alternative names in the same language (e.g., "北京" → aliases: ["北平", "京城"])
* Include abbreviations and full names in the same language (e.g., "联合国" → aliases: ["联合国组织"])
* Include nicknames and common variations in the same language (e.g., "纽约" → aliases: ["纽约市", "大苹果"])
* If no aliases exist in the same language, use empty array: []
* **Examples:**
- Chinese input "北京" → aliases: ["北平", "京城"] (NOT ["Beijing", "Peking"])
- English input "Beijing" → aliases: ["Peking"] (NOT ["北京", "北平"])
- Chinese input "苹果公司" → aliases: ["苹果"] (NOT ["Apple Inc.", "Apple"])
- Exclude lengthy quotes, calendar dates, temporal ranges, and temporal expressions
- For numeric values: extract as separate entities (instance_of: 'Numeric', name: units, numeric_value: value)
Example: £30 → name: 'GBP', numeric_value: 30, instance_of: 'Numeric'
@@ -105,11 +73,6 @@ The following shows type inheritance relationships (Child → Parent → Grandpa
- Subject: main entity performing the action or being described
- Predicate: relationship between entities (e.g., 'is', 'works at', 'believes')
- Object: entity, value, or concept affected by the predicate
{% if language == "zh" %}
- subject_name 和 object_name 必须使用中文
{% else %}
- subject_name and object_name must be in English (translate if original is in another language)
{% endif %}
- Exclude all temporal expressions from every field
- Use ONLY the predicates listed in "Predicate Instructions" (uppercase English tokens)
- Do NOT translate predicate tokens
@@ -118,7 +81,7 @@ The following shows type inheritance relationships (Child → Parent → Grandpa
**When NOT to extract triplets:**
- Non-propositional utterances (emotions, fillers, onomatopoeia)
- No clear predicate from the given definitions applies
- Standalone noun phrases or checklist items → extract as entities only
- Standalone noun phrases or checklist items (e.g., "三脚架", "备用电池") → extract as entities only
- Do NOT invent generic predicates (e.g., "IS_DOING", "FEELS", "MENTIONS")
**If no valid triplet exists:** Return triplets: [], extract entities if present, otherwise both arrays empty.
@@ -133,86 +96,248 @@ Use ONLY these predicates. If none fits, set triplets to [].
===Examples===
{% if language == "en" %}
**Example 1 (English output):** "I plan to travel to Paris next week and visit the Louvre."
**Example 1 (English):** "I plan to travel to Paris next week and visit the Louvre."
Output:
{
"triplets": [
{"subject_name": "I", "subject_id": 0, "predicate": "PLANS_TO_VISIT", "object_name": "Paris", "object_id": 1, "value": null},
{"subject_name": "I", "subject_id": 0, "predicate": "PLANS_TO_VISIT", "object_name": "Louvre", "object_id": 2, "value": null}
{
"subject_name": "I",
"subject_id": 0,
"predicate": "PLANS_TO_VISIT",
"object_name": "Paris",
"object_id": 1,
"value": null
},
{
"subject_name": "I",
"subject_id": 0,
"predicate": "PLANS_TO_VISIT",
"object_name": "Louvre",
"object_id": 2,
"value": null
}
],
"entities": [
{"entity_idx": 0, "name": "I", "type": "Person", "description": "The user", "example": "", "aliases": [], "is_explicit_memory": false},
{"entity_idx": 1, "name": "Paris", "type": "Location", "description": "Capital city of France", "example": "", "aliases": [], "is_explicit_memory": false},
{"entity_idx": 2, "name": "Louvre", "type": "Location", "description": "World-famous museum located in Paris", "example": "", "aliases": ["Louvre Museum"], "is_explicit_memory": false}
{
"entity_idx": 0,
"name": "I",
"type": "Person",
"description": "The user",
"example": "",
"aliases": [],
"is_explicit_memory": false
},
{
"entity_idx": 1,
"name": "Paris",
"type": "Location",
"description": "Capital city of France",
"example": "",
"aliases": [],
"is_explicit_memory": false
},
{
"entity_idx": 2,
"name": "Louvre",
"type": "Location",
"description": "World-famous museum located in Paris",
"example": "",
"aliases": ["Louvre Museum"],
"is_explicit_memory": false
}
]
}
**Example 2 (Chinese input → English output - IMPORTANT: translate entity names):** "张明在腾讯工作负责AI产品开发。"
**Example 2 (English):** "John Smith works at Google and is responsible for AI product development."
Output:
{
"triplets": [
{"subject_name": "Zhang Ming", "subject_id": 0, "predicate": "WORKS_AT", "object_name": "Tencent", "object_id": 1, "value": null},
{"subject_name": "Zhang Ming", "subject_id": 0, "predicate": "RESPONSIBLE_FOR", "object_name": "AI product development", "object_id": 2, "value": null}
{
"subject_name": "John Smith",
"subject_id": 0,
"predicate": "WORKS_AT",
"object_name": "Google",
"object_id": 1,
"value": null
},
{
"subject_name": "John Smith",
"subject_id": 0,
"predicate": "RESPONSIBLE_FOR",
"object_name": "AI product development",
"object_id": 2,
"value": null
}
],
"entities": [
{"entity_idx": 0, "name": "Zhang Ming", "type": "Person", "description": "Individual person name", "example": "", "aliases": [], "is_explicit_memory": false},
{"entity_idx": 1, "name": "Tencent", "type": "Organization", "description": "Chinese technology company", "example": "", "aliases": ["Tencent Holdings"], "is_explicit_memory": false},
{"entity_idx": 2, "name": "AI product development", "type": "Concept", "description": "Artificial intelligence product development work", "example": "e.g., developing chatbots", "aliases": [], "is_explicit_memory": true}
{
"entity_idx": 0,
"name": "John Smith",
"type": "Person",
"description": "Individual person name",
"example": "",
"aliases": [],
"is_explicit_memory": false
},
{
"entity_idx": 1,
"name": "Google",
"type": "Organization",
"description": "American technology company",
"example": "",
"aliases": ["Google LLC", "Alphabet Inc."],
"is_explicit_memory": false
},
{
"entity_idx": 2,
"name": "AI product development",
"type": "Concept",
"description": "Artificial intelligence product development work",
"example": "e.g., developing chatbots, recommendation systems",
"aliases": [],
"is_explicit_memory": true
}
]
}
**Example 3 (Chinese input → English output):** "三脚架"
**Example 3 (Chinese):** "我计划下周去巴黎旅行,参观卢浮宫。"
Output:
{
"triplets": [
{
"subject_name": "我",
"subject_id": 0,
"predicate": "PLANS_TO_VISIT",
"object_name": "巴黎",
"object_id": 1,
"value": null
},
{
"subject_name": "我",
"subject_id": 0,
"predicate": "PLANS_TO_VISIT",
"object_name": "卢浮宫",
"object_id": 2,
"value": null
}
],
"entities": [
{
"entity_idx": 0,
"name": "我",
"type": "Person",
"description": "用户本人",
"example": "",
"aliases": [],
"is_explicit_memory": false
},
{
"entity_idx": 1,
"name": "巴黎",
"type": "Location",
"description": "法国首都城市",
"example": "",
"aliases": [],
"is_explicit_memory": false
},
{
"entity_idx": 2,
"name": "卢浮宫",
"type": "Location",
"description": "位于巴黎的世界著名博物馆",
"example": "",
"aliases": [],
"is_explicit_memory": false
}
]
}
**Example 4 (Chinese):** "张明在腾讯工作负责AI产品开发。"
Output:
{
"triplets": [
{
"subject_name": "张明",
"subject_id": 0,
"predicate": "WORKS_AT",
"object_name": "腾讯",
"object_id": 1,
"value": null
},
{
"subject_name": "张明",
"subject_id": 0,
"predicate": "RESPONSIBLE_FOR",
"object_name": "AI产品开发",
"object_id": 2,
"value": null
}
],
"entities": [
{
"entity_idx": 0,
"name": "张明",
"type": "Person",
"description": "个人姓名",
"example": "",
"aliases": [],
"is_explicit_memory": false
},
{
"entity_idx": 1,
"name": "腾讯",
"type": "Organization",
"description": "中国科技公司",
"example": "",
"aliases": ["腾讯控股", "腾讯公司"],
"is_explicit_memory": false
},
{
"entity_idx": 2,
"name": "AI产品开发",
"type": "Concept",
"description": "人工智能产品研发工作",
"example": "如:开发智能客服机器人、推荐系统",
"aliases": [],
"is_explicit_memory": true
}
]
}
**Example 5 (Entity Only - English):** "Tripod"
Output:
{
"triplets": [],
"entities": [
{"entity_idx": 0, "name": "Tripod", "type": "Equipment", "description": "Photography equipment accessory", "example": "", "aliases": ["Camera Tripod"], "is_explicit_memory": false}
]
}
{% else %}
**Example 1 (English input → Chinese output):** "I plan to travel to Paris next week and visit the Louvre."
Output:
{
"triplets": [
{"subject_name": "我", "subject_id": 0, "predicate": "PLANS_TO_VISIT", "object_name": "巴黎", "object_id": 1, "value": null},
{"subject_name": "我", "subject_id": 0, "predicate": "PLANS_TO_VISIT", "object_name": "卢浮宫", "object_id": 2, "value": null}
],
"entities": [
{"entity_idx": 0, "name": "我", "type": "Person", "description": "用户本人", "example": "", "aliases": [], "is_explicit_memory": false},
{"entity_idx": 1, "name": "巴黎", "type": "Location", "description": "法国首都城市", "example": "", "aliases": [], "is_explicit_memory": false},
{"entity_idx": 2, "name": "卢浮宫", "type": "Location", "description": "位于巴黎的世界著名博物馆", "example": "", "aliases": [], "is_explicit_memory": false}
{
"entity_idx": 0,
"name": "Tripod",
"type": "Equipment",
"description": "Photography equipment accessory",
"example": "",
"aliases": ["Camera Tripod"],
"is_explicit_memory": false
}
]
}
**Example 2 (Chinese input → Chinese output):** "张明在腾讯工作负责AI产品开发。"
Output:
{
"triplets": [
{"subject_name": "张明", "subject_id": 0, "predicate": "WORKS_AT", "object_name": "腾讯", "object_id": 1, "value": null},
{"subject_name": "张明", "subject_id": 0, "predicate": "RESPONSIBLE_FOR", "object_name": "AI产品开发", "object_id": 2, "value": null}
],
"entities": [
{"entity_idx": 0, "name": "张明", "type": "Person", "description": "个人姓名", "example": "", "aliases": [], "is_explicit_memory": false},
{"entity_idx": 1, "name": "腾讯", "type": "Organization", "description": "中国科技公司", "example": "", "aliases": ["腾讯控股", "腾讯公司"], "is_explicit_memory": false},
{"entity_idx": 2, "name": "AI产品开发", "type": "Concept", "description": "人工智能产品研发工作", "example": "如:开发智能客服机器人", "aliases": [], "is_explicit_memory": true}
]
}
**Example 3 (Entity Only - Chinese):** "三脚架"
**Example 6 (Entity Only - Chinese):** "三脚架"
Output:
{
"triplets": [],
"entities": [
{"entity_idx": 0, "name": "三脚架", "type": "Equipment", "description": "摄影器材配件", "example": "", "aliases": ["相机三脚架"], "is_explicit_memory": false}
{
"entity_idx": 0,
"name": "三脚架",
"type": "Equipment",
"description": "摄影器材配件",
"example": "",
"aliases": ["相机三脚架"],
"is_explicit_memory": false
}
]
}
{% endif %}
===End of Examples===
{% if ontology_types %}
**⚠️ REMINDER: The examples above use generic type names for illustration only. You MUST use ONLY the predefined ontology type names from the "ALLOWED Type Names" list above. For example, use "PredictiveMaintenance" instead of "Concept", use "ProductionLine" instead of "Equipment", etc. Map each entity to the closest matching predefined type.**
{% endif %}
===Output Format===
@@ -223,10 +348,10 @@ Output:
- Ensure proper string closure and comma separation
- No line breaks within JSON string values
{% if language == "zh" %}
- **语言要求:实体名称name描述description示例example、subject_name、object_name 必须使用中文**
- **语言要求实体描述description示例example必须使用中文**
{% else %}
- **Language Requirement: Entity names, descriptions, examples, subject_name, object_name must be in English**
- **If the original text is in Chinese, translate all names to English**
- **Language Requirement: Entity descriptions and examples must be in English**
{% endif %}
- Preserve the original language and do not translate
{{ json_schema }}
{{ json_schema }}

View File

@@ -1,103 +1,9 @@
{% if language == "en" %}
You are a professional mental health consultant. Based on the following user's emotional health data and personal information, generate 3-5 personalized emotional improvement suggestions.
## Core Principle (Highest Priority)
**You must strictly base your suggestions on the emotion distribution data provided below. As long as any emotion type has a count ≥ 1, that emotion EXISTS and you must acknowledge and address it in your suggestions. You must NEVER claim an emotion is "zero" or "absent" when its count is ≥ 1.**
Specific rules:
1. Carefully check the count for each emotion type in "Emotion Distribution" — count ≥ 1 means the emotion exists
2. Even if an emotion appeared only once, you must mention it in health_summary or suggestions and provide targeted advice
3. Never state that an emotion is "zero" or "non-existent" unless its count in the distribution data is truly 0
4. If positive emotions (e.g., Joy) exist, health_summary must affirm this positive signal
5. If negative emotions (e.g., Sadness, Anger, Fear) exist even once, you must provide targeted improvement suggestions
6. A high proportion of neutral emotions does NOT mean other emotions are absent — address all non-zero emotions
## User Emotional Health Data
Health Score: {{ health_data.health_score }}/100
Health Level: {{ health_data.level }}
Total Emotion Records: {{ health_data.dimensions.positivity_rate.positive_count + health_data.dimensions.positivity_rate.negative_count + health_data.dimensions.positivity_rate.neutral_count }}
Dimension Analysis:
- Positivity Rate: {{ health_data.dimensions.positivity_rate.score }}/100
- Positive Emotions: {{ health_data.dimensions.positivity_rate.positive_count }} times
- Negative Emotions: {{ health_data.dimensions.positivity_rate.negative_count }} times
- Neutral Emotions: {{ health_data.dimensions.positivity_rate.neutral_count }} times
- Stability: {{ health_data.dimensions.stability.score }}/100
- Standard Deviation: {{ health_data.dimensions.stability.std_deviation }}
- Resilience: {{ health_data.dimensions.resilience.score }}/100
- Recovery Rate: {{ health_data.dimensions.resilience.recovery_rate }}
Emotion Distribution (check each item — every emotion with count ≥ 1 must be reflected in suggestions):
{{ emotion_distribution_json }}
## Emotion Pattern Analysis
Dominant Negative Emotion: {{ patterns.dominant_negative_emotion|default('None') }}
Emotion Volatility: {{ patterns.emotion_volatility|default('Unknown') }}
High Intensity Emotion Count: {{ patterns.high_intensity_emotions|default([])|length }}
## User Interests
{{ user_profile.interests|default(['Unknown'])|join(', ') }}
## Task Requirements
Please generate 3-5 personalized suggestions, each containing:
1. type: Suggestion type (Emotion Balance/Activity Recommendation/Social Connection/Stress Management)
2. title: Suggestion title (short and impactful)
3. content: Suggestion content (detailed explanation, 50-100 words)
4. priority: Priority level (High/Medium/Low)
5. actionable_steps: 3 specific executable steps
Also provide a health_summary (no more than 50 words) summarizing the user's overall emotional state.
**The health_summary must truthfully reflect ALL non-zero emotions from the distribution data. Do not omit any emotion type that has appeared.**
Please return in JSON format as follows:
{
"health_summary": "Your emotional health status...",
"suggestions": [
{
"type": "Emotion Balance",
"title": "Suggestion Title",
"content": "Suggestion content...",
"priority": "High",
"actionable_steps": ["Step 1", "Step 2", "Step 3"]
}
]
}
Notes:
- CRITICAL: Any emotion with count ≥ 1 in the distribution MUST be acknowledged and addressed — never ignore or claim it is zero
- Suggestions should be specific and actionable, avoid vague advice
- Provide personalized suggestions based on user's interests and hobbies
- Provide targeted suggestions for main issues (such as dominant negative emotions)
- Allocate priorities reasonably (at least 1 high, 1-2 medium, rest low)
- The 3 steps for each suggestion should be progressive and easy to implement
- All output must be in English
{% else %}
你是一位专业的心理健康顾问。请根据以下用户的情绪健康数据和个人信息生成3-5条个性化的情绪改善建议。
## 核心原则(最高优先级)
**你必须严格基于下方提供的情绪分布数据来生成建议。只要某种情绪的出现次数 ≥ 1就代表该情绪确实存在你必须在建议中承认并回应这一情绪绝对不能说"该情绪为零"或"没有该情绪"。**
具体规则:
1. 仔细查看"情绪分布"中每种情绪的出现次数,次数 ≥ 1 即表示该情绪存在
2. 即使某种情绪只出现了1次也必须在 health_summary 或建议中提及并给出针对性建议
3. 严禁在输出中声称某种情绪"为零"或"不存在"除非该情绪在分布数据中确实为0次
4. 如果正面情绪如喜悦存在health_summary 中必须肯定这一积极信号
5. 如果负面情绪如悲伤、愤怒、恐惧存在即使只有1次也必须给出针对性的改善建议
6. 中性情绪占比高不代表没有其他情绪,必须同时关注所有非零情绪
## 用户情绪健康数据
健康分数:{{ health_data.health_score }}/100
健康等级:{{ health_data.level }}
情绪记录总数:{{ health_data.dimensions.positivity_rate.positive_count + health_data.dimensions.positivity_rate.negative_count + health_data.dimensions.positivity_rate.neutral_count }}条
维度分析:
- 积极率:{{ health_data.dimensions.positivity_rate.score }}/100
@@ -111,12 +17,12 @@ Notes:
- 恢复力:{{ health_data.dimensions.resilience.score }}/100
- 恢复率:{{ health_data.dimensions.resilience.recovery_rate }}
情绪分布请逐项检查次数≥1的情绪都必须在建议中体现
情绪分布:
{{ emotion_distribution_json }}
## 情绪模式分析
主要负面情绪:{{ dominant_negative_translated|default(patterns.dominant_negative_emotion)|default('无') }}
主要负面情绪:{{ patterns.dominant_negative_emotion|default('无') }}
情绪波动性:{{ patterns.emotion_volatility|default('未知') }}
高强度情绪次数:{{ patterns.high_intensity_emotions|default([])|length }}
@@ -127,35 +33,31 @@ Notes:
## 任务要求
请生成3-5条个性化建议每条建议包含
1. type: 建议类型(情绪平衡/活动建议/社交联系/压力管理
1. type: 建议类型(emotion_balance/activity_recommendation/social_connection/stress_management
2. title: 建议标题(简短有力)
3. content: 建议内容详细说明50-100字
4. priority: 优先级(高/中/低
4. priority: 优先级(high/medium/low
5. actionable_steps: 3个可执行的具体步骤
同时提供一个health_summary不超过50字概括用户的整体情绪状态。
**health_summary 必须如实反映情绪分布中所有非零情绪的存在,不得遗漏任何已出现的情绪类型。**
请以JSON格式返回格式如下
{
"health_summary": "您的情绪健康状况...",
"suggestions": [
{
"type": "情绪平衡",
"type": "emotion_balance",
"title": "建议标题",
"content": "建议内容...",
"priority": "",
"priority": "high",
"actionable_steps": ["步骤1", "步骤2", "步骤3"]
}
]
}
注意事项:
- 所有输出内容必须完全使用中文严禁出现任何英文单词或短语包括情绪类型名称如fear、sadness、anger等必须使用对应的中文恐惧、悲伤、愤怒等
- 再次强调情绪分布中出现次数≥1的情绪必须在建议中被提及和回应绝不能忽略或声称为零
- 建议要具体、可执行,避免空泛
- 结合用户的兴趣爱好提供个性化建议
- 针对主要问题(如主要负面情绪)提供针对性建议
- 优先级要合理分配至少1个1-2个中,其余低
- 优先级要合理分配至少1个high1-2个medium其余low
- 每个建议的3个步骤要循序渐进、易于实施
{% endif %}

View File

@@ -1,67 +0,0 @@
{% if language == "zh" %}
You are a user interest analysis expert. Your task is to infer and extract the user's core hobby/interest activities from a tag list. The tags may be specific project names, tool names, or compound nouns — your job is to identify the underlying interest they represent.
**Step 1 - Infer the underlying interest from each tag**:
Look at each tag and ask: "What hobby or interest does this tag suggest the user has?"
Examples of inference:
- '攀岩', '室内攀岩馆', '攀岩者数据仪表盘', '路线解锁地图', '指力', '路线等级', '当日攀岩流畅度' → '攀岩'
- '风光摄影元数据增强器', 'EXIF数据', '.CR2文件', '.NEF文件', '日出拍摄点', '曝光补偿', '光圈', '太阳高度角', '云量预测图层' → '摄影'
- '晨间冥想坚持天数', '身心协同峰值' → '冥想'
- '川味可视化', '川菜' → '烹饪'
- '开源项目命名建议', 'climbviz', '可视化', '力量增长雷达图' → '编程' 或 '数据可视化'
- '吉他', '指弹', '琴谱' → '吉他'
- '跑步', '5公里', '跑鞋' → '跑步'
- '瑜伽垫', '瑜伽课' → '瑜伽'
**Step 2 - Consolidate and deduplicate**:
- Merge tags that point to the same interest into one representative label
- Use concise, standard hobby names (e.g., '攀岩', '摄影', '编程', '烹饪', '冥想', '吉他', '跑步')
- If multiple tags all point to '攀岩', output '攀岩' only once
**Step 3 - Filter out non-interest tags**:
Remove tags that do NOT suggest any hobby or interest:
- Generic system/assistant terms (e.g., '助手', '用户', 'AI')
- Pure abstract metrics with no clear hobby link (e.g., '完成时间', '日期', '自我评分')
- Location names with no clear hobby link (e.g., '青城山后山' alone — but if combined with photography context, infer '摄影')
**Output format**: Return a list of concise interest activity names in Chinese.
**Example**:
Input: ['攀岩', '攀岩者数据仪表盘', '路线解锁地图', '指力', '风光摄影元数据增强器', 'EXIF数据', '晨间冥想坚持天数', '川味可视化', '可视化', '助手', '完成时间']
Output: ['攀岩', '摄影', '冥想', '烹饪', '编程']
Now process the following tag list and return the inferred interest activities in Chinese: {{ tag_list }}
{% else %}
You are a user interest analysis expert. Your task is to infer and extract the user's core hobby/interest activities from a tag list. The tags may be specific project names, tool names, or compound nouns — your job is to identify the underlying interest they represent.
**Step 1 - Infer the underlying interest from each tag**:
Look at each tag and ask: "What hobby or interest does this tag suggest the user has?"
Examples of inference:
- 'rock climbing', 'indoor climbing gym', 'climber dashboard', 'route map', 'finger strength' → 'rock climbing'
- 'landscape photography metadata enhancer', 'EXIF data', 'sunrise shooting spot', 'exposure compensation' → 'photography'
- 'morning meditation streak', 'mind-body peak' → 'meditation'
- 'Sichuan cuisine visualization', 'Sichuan food' → 'cooking'
- 'open source project', 'data visualization tool', 'Python' → 'programming'
- 'guitar', 'fingerpicking', 'sheet music' → 'guitar'
- 'running', '5km', 'running shoes' → 'running'
**Step 2 - Consolidate and deduplicate**:
- Merge tags that point to the same interest into one representative label
- Use concise, standard hobby names (e.g., 'rock climbing', 'photography', 'programming', 'cooking', 'meditation')
- If multiple tags all point to 'rock climbing', output 'rock climbing' only once
**Step 3 - Filter out non-interest tags**:
Remove tags that do NOT suggest any hobby or interest:
- Generic system/assistant terms (e.g., 'assistant', 'user', 'AI')
- Pure abstract metrics with no clear hobby link (e.g., 'completion time', 'date', 'self-rating')
**Output format**: Return a list of concise interest activity names in English.
**Example**:
Input: ['rock climbing', 'climber dashboard', 'route map', 'finger strength', 'landscape photography metadata enhancer', 'EXIF data', 'morning meditation streak', 'Sichuan cuisine visualization', 'visualization', 'assistant', 'completion time']
Output: ['rock climbing', 'photography', 'meditation', 'cooking', 'programming']
Now process the following tag list and return the inferred interest activities in English: {{ tag_list }}
{% endif %}

View File

@@ -7,12 +7,6 @@
Your task is to generate a comprehensive memory insight report based on the provided data analysis. The report should include four distinct sections that capture different aspects of the user's memory patterns and characteristics.
{% if language == "zh" %}
**重要:请使用中文生成记忆洞察报告内容。**
{% else %}
**Important: Please generate the memory insight report content in English.**
{% endif %}
===Inputs===
{% if domain_distribution %}
@@ -37,105 +31,56 @@ Your task is to generate a comprehensive memory insight report based on the prov
**Section-Specific Requirements:**
{% if language == "zh" %}
1. **总体概述** (100-150字)
- 重点:基于交互日志对用户档案进行整体分析
- 描述用户的主要角色、工作网络和协作精神
- 使用专业、数据驱动的语言风格
- 示例参考:"通过对156次交互日志的深度分析系统发现张三是一位主要从事用户档案和数据分析的产品经理。他的工作网络体现出鲜明的目标导向和团队协作精神。"
1. **总体概述 (Overview)** (100-150 Chinese characters)
- Focus on: Overall analysis of user profile based on interaction logs
- Describe the user's main role, work network, and collaboration spirit
- Use professional, data-driven language style
- Example reference: "通过对156次交互日志的深度分析系统发现三层一位主要用户档案和数据分析的产品经理。他的工作网络体现出鲜明的目标导向和团队协作精神。"
2. **行为模式** (80-120字)
- 重点:工作模式、时间规律和行为特征
- 描述每周工作模式和时间偏好
- 使用客观、分析性的语言
- 示例参考:"张三的工作模式呈现出鲜明的周期性:周一通常用于规划和会议,周三周四专注于产品设计和用户研究,周五进行总结和复盘。他倾向于在上午进行头脑风暴,下午处理执行性工作。"
2. **行为模式 (Behavior Pattern)** (80-120 Chinese characters)
- Focus on: Work patterns, time regularity, and behavioral characteristics
- Describe weekly work patterns and time preferences
- Use objective, analytical language
- Example reference: "张三的工作模式呈现出鲜明的周期性:周一通常用于规划和会议,周三周四专注于产品设计和用户研究,周五进行总结和复盘。他倾向于在上午进行头脑风暴,下午处理执行性工作。"
3. **关键发现** (3-4个要点每个30-50字)
- 重点:关于用户行为和偏好的具体、有洞察力的观察
- 使用项目符号(•)格式
- 每个发现应具体且有数据支持
- 示例参考:
3. **关键发现 (Key Findings)** (3-4 bullet points, 30-50 characters each)
- Focus on: Specific, insightful observations about user behavior and preferences
- Use bullet points (•) format
- Each finding should be concrete and data-supported
- Example reference:
"• 在产品决策中张三总是优先考虑用户反应这在68%的决策记录中得到体现
• 他善于使用数据可视化工具来支持论点,这种习惯在项目管理中发挥了重要作用
• 团队成员对他的评价中,"思路清晰"和"思路敏捷"两个关键词出现频率最高
• 他对AI机器学习领域保持持续关注近3个月参加了7次相关培训"
4. **成长轨迹** (100-150字)
- 重点:用户的成长历程、关键里程碑和能力提升
- 按时间顺序组织内容
- 突出角色变化和成就
- 使用积极、鼓励的语气
- 示例参考:"从入职时的产品经理成长为高级产品经理,张三在产品规划、团队管理和技术理解三个方面都有显著提升。特别是在最近一年,他开始独立主导更复杂的项目,展现出更强的战略思维能力。他的成长轨迹显示出对新技术的持续学习和对产品思维的不断深化。"
{% else %}
1. **Overview** (100-150 words)
- Focus on: Overall analysis of user profile based on interaction logs
- Describe the user's main role, work network, and collaboration spirit
- Use professional, data-driven language style
- Example reference: "Through in-depth analysis of 156 interaction logs, the system identified Zhang San as a product manager primarily focused on user profiling and data analysis. His work network demonstrates a clear goal-oriented approach and team collaboration spirit."
2. **Behavior Pattern** (80-120 words)
- Focus on: Work patterns, time regularity, and behavioral characteristics
- Describe weekly work patterns and time preferences
- Use objective, analytical language
- Example reference: "Zhang San's work pattern shows distinct periodicity: Mondays are typically used for planning and meetings, Wednesdays and Thursdays focus on product design and user research, and Fridays are for summary and review. He tends to brainstorm in the morning and handle execution tasks in the afternoon."
3. **Key Findings** (3-4 bullet points, 30-50 words each)
- Focus on: Specific, insightful observations about user behavior and preferences
- Use bullet points (•) format
- Each finding should be concrete and data-supported
- Example reference:
"• In product decisions, Zhang San always prioritizes user feedback, as evidenced in 68% of decision records
• He excels at using data visualization tools to support arguments, a habit that plays an important role in project management
• Among team member evaluations, 'clear thinking' and 'quick thinking' are the most frequently mentioned keywords
• He maintains continuous attention to AI and machine learning, attending 7 related training sessions in the past 3 months"
4. **Growth Trajectory** (100-150 words)
4. **成长轨迹 (Growth Trajectory)** (100-150 Chinese characters)
- Focus on: User's growth journey, key milestones, and capability improvements
- Organize content chronologically
- Highlight role changes and achievements
- Use positive, encouraging tone
- Example reference: "Growing from a product manager at entry to a senior product manager, Zhang San has shown significant improvement in product planning, team management, and technical understanding. Especially in the past year, he has begun to independently lead more complex projects, demonstrating stronger strategic thinking capabilities. His growth trajectory shows continuous learning of new technologies and deepening of product thinking."
{% endif %}
- Example reference: "从入职时的产品经理成长为高级产品经理,张三在产品单独、团队管理和技术理解三个方面都有显著提升。特别是在最近一年,他开始独立主导更复杂的项目,展现出更强的战略思维能力。他的成长轨迹显示出对新技术的持续学习和对产品思维的不断深化。"
===Output Format (MUST STRICTLY FOLLOW)===
{% if language == "zh" %}
【总体概述】
[100-150字,基于交互分析描述用户整体档案和工作网络]
[100-150 characters describing overall user profile and work network based on interaction analysis]
【行为模式】
[80-120字,描述工作模式、时间规律和行为特征]
[80-120 characters describing work patterns, time regularity, and behavioral characteristics]
【关键发现】
• [第一个关键发现有数据支持30-50字]
• [第二个关键发现有数据支持30-50字]
• [第三个关键发现有数据支持30-50字]
• [第四个关键发现有数据支持30-50字]
• [First key finding with data support, 30-50 characters]
• [Second key finding with data support, 30-50 characters]
• [Third key finding with data support, 30-50 characters]
• [Fourth key finding with data support, 30-50 characters]
【成长轨迹】
[100-150字,描述成长历程、关键里程碑和能力提升]
{% else %}
【Overview】
[100-150 words describing overall user profile and work network based on interaction analysis]
【Behavior Pattern】
[80-120 words describing work patterns, time regularity, and behavioral characteristics]
【Key Findings】
• [First key finding with data support, 30-50 words]
• [Second key finding with data support, 30-50 words]
• [Third key finding with data support, 30-50 words]
• [Fourth key finding with data support, 30-50 words]
【Growth Trajectory】
[100-150 words describing growth journey, milestones, and capability improvements]
{% endif %}
[100-150 characters describing growth journey, milestones, and capability improvements]
===Example===
{% if language == "zh" %}
Example Input:
- 核心领域分布: 产品管理(38%), 数据分析(24%), 团队协作(21%)
- 活跃时段: 用户在每年的 4 和 10 月最为活跃
@@ -156,28 +101,6 @@ Example Output:
【成长轨迹】
从入职时的产品经理成长为高级产品经理张三在产品规划、团队管理和技术理解三个方面都有显著提升。特别是在最近一年他开始独立主导更复杂的项目展现出更强的战略思维能力。他与李明的47条共同记忆见证了他的成长历程。
{% else %}
Example Input:
- Core Domain Distribution: Product Management (38%), Data Analysis (24%), Team Collaboration (21%)
- Active Periods: User is most active in April and October each year
- Social Connections: Has the most shared memories (47 entries) with user "Li Ming", primarily during 2020-2023
Example Output:
【Overview】
Through in-depth analysis of 156 interaction logs, the system identified Zhang San as a product manager primarily focused on user profiling and data analysis. His work network demonstrates a clear goal-oriented approach and team collaboration spirit, with deep practical experience in product management, data analysis, and team collaboration.
【Behavior Pattern】
Zhang San's work pattern shows distinct periodicity: Mondays are typically used for planning and meetings, Wednesdays and Thursdays focus on product design and user research, and Fridays are for summary and review. He tends to brainstorm in the morning and handle execution tasks in the afternoon. April and October are his most active periods each year.
【Key Findings】
• In product decisions, Zhang San always prioritizes user feedback, as evidenced in 68% of decision records
• He excels at using data visualization tools to support arguments, a habit that plays an important role in project management
• Among team member evaluations, "clear thinking" and "quick thinking" are the most frequently mentioned keywords
• He maintains continuous attention to AI and machine learning, attending 7 related training sessions in the past 3 months
【Growth Trajectory】
Growing from a product manager at entry to a senior product manager, Zhang San has shown significant improvement in product planning, team management, and technical understanding. Especially in the past year, he has begun to independently lead more complex projects, demonstrating stronger strategic thinking capabilities. His 47 shared memories with Li Ming bear witness to his growth journey.
{% endif %}
===End of Example===
@@ -210,40 +133,20 @@ After generating the report, perform the following self-review steps:
===Output Requirements===
{% if language == "zh" %}
**语言要求:**
- 输出语言必须始终为简体中文
- 所有章节内容必须使用中文
- 章节标题必须使用指定的中文格式:【总体概述】【行为模式】【关键发现】【成长轨迹】
**格式要求:**
- 每个章节必须以标题开头,标题独占一行
- 内容紧跟标题之后
- 章节之间用空行分隔
- 关键发现章节必须使用项目符号(•)
- 严格遵守每个章节的字数限制
**内容要求:**
- 仅使用提供的数据点
- 不得捏造或推测信息
- 如果某个章节数据不足,请简要说明或跳过
- 全文保持专业、分析性的语气
{% else %}
**LANGUAGE REQUIREMENT:**
- The output language must ALWAYS be English
- All section content must be in English
- Section headers must use the specified English format: 【Overview】【Behavior Pattern】【Key Findings】【Growth Trajectory
- The output language should ALWAYS be Chinese (Simplified)
- All section content must be in Chinese
- Section headers must use the specified Chinese format: 【总体概述】【行为模式】【关键发现】【成长轨迹
**FORMAT REQUIREMENT:**
- Each section must start with its header on a new line
- Content follows immediately after the header
- Sections are separated by blank lines
- Key Findings section must use bullet points (•)
- Strictly adhere to word limits for each section
- Strictly adhere to character limits for each section
**CONTENT REQUIREMENT:**
- Only use provided data points
- Do not fabricate or speculate information
- If data is insufficient for a section, provide a brief note or skip
- Maintain professional, analytical tone throughout
{% endif %}

View File

@@ -1,7 +1,2 @@
{% if language == "zh" %}
你是一个从对话消息中提取实体节点的 AI 助手。
你的主要任务是提取和分类说话者以及对话中提到的其他重要实体。
{% else %}
You are an AI assistant that extracts entity nodes from conversational messages.
Your primary task is to extract and classify the speaker and other significant entities mentioned in the conversation.
{% endif %}
Your primary task is to extract and classify the speaker and other significant entities mentioned in the conversation.

View File

@@ -1,13 +1,5 @@
{% if language == "zh" %}
给定一个对话上下文和一个当前消息。
你的任务是提取在当前消息中**明确或隐含**提到的用户名称和年龄。
代词引用(如 he/she/they 或 this/that/those应消歧为引用实体的名称。
{{ message }}
{% else %}
You are given a conversation context and a CURRENT MESSAGE.
Your task is to extract user name and age mentioned **explicitly or implicitly** in the CURRENT MESSAGE.
Pronoun references such as he/she/they or this/that/those should be disambiguated to the names of the reference entities.
{{ message }}
{% endif %}
{{ message }}

View File

@@ -7,15 +7,10 @@
Your task is to generate a comprehensive user profile based on the provided entities and statements. The profile should include four distinct sections that capture different aspects of the user's identity and characteristics.
{% if language == "zh" %}
**重要:请使用中文生成用户画像内容。**
{% else %}
**Important: Please generate the user profile content in English.**
{% endif %}
===Inputs===
{% if user_display_name %}
- User Display Name: {{ user_display_name }}
{% if user_id %}
- User ID: {{ user_id }}
{% endif %}
{% if entities %}
- Core Entities & Frequency: {{ entities }}
@@ -33,97 +28,50 @@ Your task is to generate a comprehensive user profile based on the provided enti
3. Avoid excessive adjectives and empty phrases
4. Strictly follow the output format specified below
{% if language == "zh" %}
**【严格人称规定】**
- 在描述用户时,必须使用"{{ user_display_name }}"作为人称
- 绝对禁止使用用户ID如 {{ user_id }})来称呼用户
- 绝对禁止在摘要中出现任何形式的UUID或ID字符串
- 如果需要指代用户,只能使用"{{ user_display_name }}"或相应的代词(他/她/TA
{% else %}
**【STRICT PRONOUN RULES】**
- When describing the user, you MUST use "{{ user_display_name }}" as the reference
- It is ABSOLUTELY FORBIDDEN to use the user ID (such as {{ user_id }}) to refer to the user
- It is ABSOLUTELY FORBIDDEN to include any form of UUID or ID string in the summary
- If you need to refer to the user, you can ONLY use "{{ user_display_name }}" or appropriate pronouns (he/she/they)
{% endif %}
**Section-Specific Requirements:**
{% if language == "zh" %}
1. **基本介绍** (4-5句话最多150字)
- 重点:身份、职业、地点及其他基本人口统计信息
- 提供关于用户是谁的事实背景
2. **性格特点** (2-3句话最多80字)
- 重点:性格特征、行为习惯、沟通风格
- 描述用户互动和行为中可观察到的模式
3. **核心价值观** (1-2句话最多50字)
- 重点:价值观、信念、目标和愿望
- 捕捉对用户最重要的内容以及驱动其决策的因素
4. **一句话总结** (1句话最多40字)
- 提供对用户核心特质的高度浓缩描述
- 类似于捕捉其本质的个人标语或座右铭
{% else %}
1. **Basic Introduction** (4-5 sentences, max 150 words)
1. **Basic Introduction** (4-5 sentences, max 150 Chinese characters)
- Focus on: identity, occupation, location, and other basic demographic information
- Provide factual background about who the user is
2. **Personality Traits** (2-3 sentences, max 80 words)
2. **Personality Traits** (2-3 sentences, max 80 Chinese characters)
- Focus on: personality characteristics, behavioral habits, communication style
- Describe observable patterns in how the user interacts and behaves
3. **Core Values** (1-2 sentences, max 50 words)
3. **Core Values** (1-2 sentences, max 50 Chinese characters)
- Focus on: values, beliefs, goals, and aspirations
- Capture what matters most to the user and what drives their decisions
4. **One-Sentence Summary** (1 sentence, max 40 words)
4. **One-Sentence Summary** (1 sentence, max 40 Chinese characters)
- Provide a highly condensed characterization of the user's core traits
- Similar to a personal tagline or motto that captures their essence
{% endif %}
===Output Format (MUST STRICTLY FOLLOW)===
{% if language == "zh" %}
【基本介绍】
[4-5句话描述用户的基本身份、职业和地点]
【性格特点】
[2-3句话描述用户的性格特征、行为习惯和沟通风格]
【核心价值观】
[1-2句话描述用户的价值观、信念和目标]
【一句话总结】
[1句话提供对用户核心特质的高度浓缩总结]
{% else %}
【Basic Introduction】
[4-5 sentences describing the user's basic identity, occupation, and location]
Personality Traits
性格特点
[2-3 sentences describing the user's personality traits, behavioral habits, and communication style]
Core Values
核心价值观
[1-2 sentences describing the user's values, beliefs, and goals]
One-Sentence Summary
一句话总结
[1 sentence providing a highly condensed summary of the user's core characteristics]
{% endif %}
===Example===
{% if language == "zh" %}
Example Input:
- User Display Name: 张三
- User ID: user_12345
- Core Entities & Frequency: 产品经理 (15), AI (12), 深圳 (10), 数据分析 (8), 团队协作 (7)
- Representative Statement Samples: 我在深圳从事产品经理工作已经5年了 | 我相信好的产品源于对用户需求的深刻理解 | 我喜欢在团队中起到协调作用 | 数据驱动决策是我的工作原则
Example Output:
【基本介绍】
张三一名充满热情的高级产品经理,在深圳工作。在过去的5年里张三专注于AI和数据驱动的产品设计致力于创造能够真正改善用户生活的产品。张三相信好的产品源于对用户需求的深刻理解和对技术可能性的不断探索。
我是张三一名充满热情的高级产品经理。在过去的5年里专注于AI和数据驱动的产品设计致力于创造能够真正改善用户生活的产品。相信好的产品源于对用户需求的深刻理解和对技术可能性的不断探索。
【性格特点】
性格开朗,善于沟通,注重细节。喜欢在团队中起到协调作用,帮助大家达成共识。面对挑战时保持乐观,相信每个问题都有解决方案。
@@ -133,25 +81,6 @@ Example Output:
【一句话总结】
"让每一个产品决策都充满温度。"
{% else %}
Example Input:
- User Display Name: John
- Core Entities & Frequency: Product Manager (15), AI (12), San Francisco (10), Data Analysis (8), Team Collaboration (7)
- Representative Statement Samples: I have been working as a product manager in San Francisco for 5 years | I believe good products come from deep understanding of user needs | I enjoy playing a coordinating role in teams | Data-driven decision making is my work principle
Example Output:
【Basic Introduction】
John is a passionate senior product manager based in San Francisco. Over the past 5 years, John has focused on AI and data-driven product design, dedicated to creating products that truly improve users' lives. John believes good products stem from deep understanding of user needs and continuous exploration of technological possibilities.
【Personality Traits】
Outgoing personality with excellent communication skills and attention to detail. Enjoys playing a coordinating role in teams, helping everyone reach consensus. Maintains optimism when facing challenges, believing every problem has a solution.
【Core Values】
User-first, data-driven, continuous learning, team collaboration
【One-Sentence Summary】
"Making every product decision with warmth and purpose."
{% endif %}
===End of Example===
@@ -162,7 +91,7 @@ Before generating your final output, internally verify:
1. All content is grounded in provided data (no fabrication)
2. Format follows the specified structure with correct headers
3. Tone is objective, third-person, and neutral
4. All four sections are complete and within character/word limits
4. All four sections are complete and within character limits
**IMPORTANT: These checks are for your internal use only. DO NOT include them in your output.**
@@ -172,24 +101,14 @@ Before generating your final output, internally verify:
**CRITICAL: Your response must ONLY contain the four sections below. Do not include any reflection, self-review, or meta-commentary.**
**LANGUAGE REQUIREMENT:**
{% if language == "zh" %}
- 输出语言必须为简体中文
- 所有部分内容必须使用中文
- 部分标题必须使用指定的中文格式:【基本介绍】【性格特点】【核心价值观】【一句话总结】
{% else %}
- The output language must be English
- All section content must be in English
- Section headers must use the specified format: 【Basic Introduction】【Personality Traits】【Core Values】【One-Sentence Summary】
{% endif %}
- The output language should ALWAYS be Chinese (Simplified)
- All section content must be in Chinese
- Section headers must use the specified Chinese format: 【基本介绍】【性格特点】【核心价值观】【一句话总结】
**FORMAT REQUIREMENT:**
- Each section must start with its header on a new line
- Content follows immediately after the header
- Sections are separated by blank lines
{% if language == "zh" %}
- 严格遵守每个部分的字数限制
{% else %}
- Strictly adhere to word limits for each section
{% endif %}
- **DO NOT include any text after the final section**
- Strictly adhere to character limits for each section
- **DO NOT include any text after the 【一句话总结】 section**
- **DO NOT output reflection steps, self-review, or verification notes**

Some files were not shown because too many files have changed in this diff Show More