From e2e5c1571a1f4a25d9909e28568cafafe81fa61e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=90=E5=8A=9B=E9=BD=90?= Date: Thu, 18 Dec 2025 09:56:35 +0000 Subject: [PATCH 01/20] Merge #13 into develop from fix/stream-output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'fix/stream-output' * fix/stream-output: (17 commits squashed) - [fix]Fix the issue where the streaming output effect is not obvious. - [fix]Fix the issue where the streaming output effect is not obvious. - Merge branch 'fix/stream-output' of codeup.aliyun.com:redbearai/python/redbear-mem-open into fix/stream-output - [fix] - [fix]Skip time extraction - [fix] - [fix]Skip time extraction - Merge branch 'fix/stream-output' of codeup.aliyun.com:redbearai/python/redbear-mem-open into fix/stream-output - [fix]Remove human-induced delays - [fix]Fix the issue where the streaming output effect is not obvious. - [fix] - [fix]Skip time extraction - [fix]Fix the issue where the streaming output effect is not obvious. - [fix] - [fix]Skip time extraction - [fix]Remove human-induced delays - Merge branch 'fix/stream-output' of codeup.aliyun.com:redbearai/python/redbear-mem-open into fix/stream-output Signed-off-by: 乐力齐 Reviewed-by: aliyun6762716068 Merged-by: aliyun6762716068 CR-link: https://codeup.aliyun.com/redbearai/python/redbear-mem-open/change/13 --- .../extraction_orchestrator.py | 239 ++++++++++-------- 1 file changed, 138 insertions(+), 101 deletions(-) diff --git a/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py b/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py index 7eec1189..e00bcf0a 100644 --- a/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py +++ b/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py @@ -179,8 +179,21 @@ class ExtractionOrchestrator: all_statements_list.extend(chunk.statements) total_statements = len(all_statements_list) - # 步骤 2: 并行执行三元组提取、时间信息提取和基础嵌入生成 - logger.info("步骤 2/6: 并行执行三元组提取、时间信息提取和嵌入生成") + # 🔥 陈述句提取完成后，立即发送知识抽取完成消息 + if self.progress_callback: + extraction_stats = { + "statements_count": total_statements, + "entities_count": 0, # 暂时为0，后续会更新 + "triplets_count": 0, # 暂时为0，后续会更新 + "temporal_ranges_count": 0, # 暂时为0，后续会更新 + } + await self.progress_callback("knowledge_extraction_complete", "知识抽取完成", extraction_stats) + + # 🔥 立即发送下一阶段的开始消息，让前端知道进入了创建节点和边阶段 + await self.progress_callback("creating_nodes_edges", "正在创建节点和边...") + + # 步骤 2: 并行执行三元组提取、时间信息提取和基础嵌入生成（后台静默执行） + logger.info("步骤 2/6: 并行执行三元组提取、时间信息提取和嵌入生成（后台静默执行）") ( triplet_maps, temporal_maps, @@ -206,72 +219,6 @@ class ExtractionOrchestrator: logger.info("步骤 3/6: 生成实体嵌入") triplet_maps = await self._generate_entity_embeddings(triplet_maps) - # 进度回调：按三个阶段分别输出知识抽取结果 - if self.progress_callback: - # 第一阶段：陈述句提取结果 - for i, stmt in enumerate(all_statements_list[:10]): # 只输出前10个陈述句 - stmt_result = { - "extraction_type": "statement", - "statement_index": i + 1, - "statement": stmt.statement, - "statement_id": stmt.id - } - await self.progress_callback("knowledge_extraction_result", "陈述句提取完成", stmt_result) - - # 第二阶段：三元组提取结果 - for i, triplet in enumerate(all_triplets_list[:10]): # 只输出前10个三元组 - triplet_result = { - "extraction_type": "triplet", - "triplet_index": i + 1, - "subject": triplet.subject_name, - "predicate": triplet.predicate, - "object": triplet.object_name - } - await self.progress_callback("knowledge_extraction_result", "三元组提取完成", triplet_result) - - # 第三阶段：时间提取结果 - if total_temporal > 0: - # 收集时间信息 - temporal_results = [] - for dialog in dialog_data_list: - for chunk in dialog.chunks: - for statement in chunk.statements: - if hasattr(statement, 'temporal_validity') and statement.temporal_validity: - temporal_results.append({ - "statement_id": statement.id, - "statement": statement.statement, - "valid_at": statement.temporal_validity.valid_at, - "invalid_at": statement.temporal_validity.invalid_at - }) - - # 输出时间提取结果 - for i, temporal_result in enumerate(temporal_results[:5]): # 只输出前5个时间提取结果 - time_result = { - "extraction_type": "temporal", - "temporal_index": i + 1, - "statement": temporal_result["statement"], - "valid_at": temporal_result["valid_at"], - "invalid_at": temporal_result["invalid_at"] - } - await self.progress_callback("knowledge_extraction_result", "时间提取完成", time_result) - else: - # 如果没有时间信息，也发送一个时间提取完成的消息 - time_result = { - "extraction_type": "temporal", - "temporal_index": 0, - "message": "未发现时间信息" - } - await self.progress_callback("knowledge_extraction_result", "时间提取完成", time_result) - - # 进度回调：知识抽取完成，传递知识抽取的统计信息 - extraction_stats = { - "statements_count": total_statements, - "entities_count": total_entities, - "triplets_count": total_triplets, - "temporal_ranges_count": total_temporal, - } - await self.progress_callback("knowledge_extraction_complete", "知识抽取完成", extraction_stats) - # 步骤 4: 将提取的数据赋值到语句 logger.info("步骤 4/6: 数据赋值") dialog_data_list = await self._assign_extracted_data( @@ -285,6 +232,9 @@ class ExtractionOrchestrator: # 步骤 5: 创建节点和边 logger.info("步骤 5/6: 创建节点和边") + + # 注意：creating_nodes_edges 消息已在知识抽取完成后立即发送 + ( dialogue_nodes, chunk_nodes, @@ -304,6 +254,8 @@ class ExtractionOrchestrator: else: logger.info("步骤 6/6: 两阶段去重和消歧") + # 注意：deduplication 消息已在创建节点和边完成后立即发送 + result = await self._run_dedup_and_write_summary( dialogue_nodes, chunk_nodes, @@ -328,7 +280,7 @@ class ExtractionOrchestrator: self, dialog_data_list: List[DialogData] ) -> List[DialogData]: """ - 从对话中提取陈述句（优化版：全局分块级并行） + 从对话中提取陈述句（流式输出版本：边提取边发送进度） Args: dialog_data_list: 对话数据列表 @@ -336,7 +288,7 @@ class ExtractionOrchestrator: Returns: 更新后的对话数据列表（包含提取的陈述句） """ - logger.info("开始陈述句提取（全局分块级并行）") + logger.info("开始陈述句提取（全局分块级并行 + 流式输出）") # 收集所有分块及其元数据 all_chunks = [] @@ -349,17 +301,44 @@ class ExtractionOrchestrator: chunk_metadata.append((d_idx, c_idx)) logger.info(f"收集到 {len(all_chunks)} 个分块，开始全局并行提取") + + # 用于跟踪已完成的分块数量 + completed_chunks = 0 + total_chunks = len(all_chunks) # 全局并行处理所有分块 - async def extract_for_chunk(chunk_data): + async def extract_for_chunk(chunk_data, chunk_index): + nonlocal completed_chunks chunk, group_id, dialogue_content = chunk_data try: - return await self.statement_extractor._extract_statements(chunk, group_id, dialogue_content) + statements = await self.statement_extractor._extract_statements(chunk, group_id, dialogue_content) + + # 流式输出：每提取完一个分块的陈述句，立即发送进度 + # 注意：只在试运行模式下发送陈述句详情，正式模式不发送 + completed_chunks += 1 + if self.progress_callback and statements and self.is_pilot_run: + # 发送前3个陈述句作为示例 + for idx, stmt in enumerate(statements[:3]): + stmt_result = { + "extraction_type": "statement", + "statement": stmt.statement, + "statement_id": stmt.id, + "chunk_progress": f"{completed_chunks}/{total_chunks}", + "statement_index_in_chunk": idx + 1 + } + await self.progress_callback( + "knowledge_extraction_result", + f"陈述句提取中 ({completed_chunks}/{total_chunks})", + stmt_result + ) + + return statements except Exception as e: logger.error(f"分块 {chunk.id} 陈述句提取失败: {e}") + completed_chunks += 1 return [] - tasks = [extract_for_chunk(chunk_data) for chunk_data in all_chunks] + tasks = [extract_for_chunk(chunk_data, i) for i, chunk_data in enumerate(all_chunks)] results = await asyncio.gather(*tasks, return_exceptions=True) # 将结果分配回对话 @@ -391,7 +370,7 @@ class ExtractionOrchestrator: self, dialog_data_list: List[DialogData] ) -> List[Dict[str, Any]]: """ - 从对话中提取三元组（优化版：全局陈述句级并行） + 从对话中提取三元组（流式输出版本：边提取边发送进度） Args: dialog_data_list: 对话数据列表 @@ -399,7 +378,7 @@ class ExtractionOrchestrator: Returns: 三元组映射列表，每个对话对应一个字典 """ - logger.info("开始三元组提取（全局陈述句级并行）") + logger.info("开始三元组提取（全局陈述句级并行 + 流式输出）") # 收集所有陈述句及其元数据 all_statements = [] @@ -412,18 +391,30 @@ class ExtractionOrchestrator: statement_metadata.append((d_idx, statement.id)) logger.info(f"收集到 {len(all_statements)} 个陈述句，开始全局并行提取三元组") + + # 用于跟踪已完成的陈述句数量 + completed_statements = 0 + total_statements = len(all_statements) # 全局并行处理所有陈述句 - async def extract_for_statement(stmt_data): + async def extract_for_statement(stmt_data, stmt_index): + nonlocal completed_statements statement, chunk_content = stmt_data try: - return await self.triplet_extractor._extract_triplets(statement, chunk_content) + triplet_info = await self.triplet_extractor._extract_triplets(statement, chunk_content) + + # 注意：不再发送三元组提取的流式输出 + # 三元组提取在后台执行，但不向前端发送详细信息 + completed_statements += 1 + + return triplet_info except Exception as e: logger.error(f"陈述句 {statement.id} 三元组提取失败: {e}") + completed_statements += 1 from app.core.memory.models.triplet_models import TripletExtractionResponse return TripletExtractionResponse(triplets=[], entities=[]) - tasks = [extract_for_statement(stmt_data) for stmt_data in all_statements] + tasks = [extract_for_statement(stmt_data, i) for i, stmt_data in enumerate(all_statements)] results = await asyncio.gather(*tasks, return_exceptions=True) # 将结果组织成对话级别的映射 @@ -458,7 +449,7 @@ class ExtractionOrchestrator: self, dialog_data_list: List[DialogData] ) -> List[Dict[str, Any]]: """ - 从对话中提取时间信息（优化版：全局陈述句级并行） + 从对话中提取时间信息（流式输出版本：边提取边发送进度） Args: dialog_data_list: 对话数据列表 @@ -466,7 +457,21 @@ class ExtractionOrchestrator: Returns: 时间信息映射列表，每个对话对应一个字典 """ - logger.info("开始时间信息提取（全局陈述句级并行）") + # 试运行模式：跳过时间提取以节省时间 + if self.is_pilot_run: + logger.info("试运行模式：跳过时间信息提取（节省约 10-15 秒）") + # 为所有陈述句返回空的时间范围 + from app.core.memory.models.message_models import TemporalValidityRange + temporal_maps = [] + for dialog in dialog_data_list: + temporal_map = {} + for chunk in dialog.chunks: + for statement in chunk.statements: + temporal_map[statement.id] = TemporalValidityRange(valid_at=None, invalid_at=None) + temporal_maps.append(temporal_map) + return temporal_maps + + logger.info("开始时间信息提取（全局陈述句级并行 + 流式输出）") # 收集所有需要提取时间的陈述句 all_statements = [] @@ -494,18 +499,30 @@ class ExtractionOrchestrator: statement_metadata.append((d_idx, statement.id)) logger.info(f"收集到 {len(all_statements)} 个需要时间提取的陈述句，开始全局并行提取") + + # 用于跟踪已完成的时间提取数量 + completed_temporal = 0 + total_temporal_statements = len(all_statements) # 全局并行处理所有陈述句 - async def extract_for_statement(stmt_data): + async def extract_for_statement(stmt_data, stmt_index): + nonlocal completed_temporal statement, ref_dates = stmt_data try: - return await self.temporal_extractor._extract_temporal_ranges(statement, ref_dates) + temporal_range = await self.temporal_extractor._extract_temporal_ranges(statement, ref_dates) + + # 注意：不再发送时间提取的流式输出 + # 时间提取在后台执行，但不向前端发送详细信息 + completed_temporal += 1 + + return temporal_range except Exception as e: logger.error(f"陈述句 {statement.id} 时间信息提取失败: {e}") + completed_temporal += 1 from app.core.memory.models.message_models import TemporalValidityRange return TemporalValidityRange(valid_at=None, invalid_at=None) - tasks = [extract_for_statement(stmt_data) for stmt_data in all_statements] + tasks = [extract_for_statement(stmt_data, i) for i, stmt_data in enumerate(all_statements)] results = await asyncio.gather(*tasks, return_exceptions=True) # 将结果组织成对话级别的映射 @@ -832,9 +849,7 @@ class ExtractionOrchestrator: """ logger.info("开始创建节点和边") - # 进度回调：正在创建节点和边 - if self.progress_callback: - await self.progress_callback("creating_nodes_edges", "正在创建节点和边...") + # 注意：开始消息已在 run 方法中发送，这里不再重复发送 dialogue_nodes = [] chunk_nodes = [] @@ -846,8 +861,13 @@ class ExtractionOrchestrator: # 用于去重的集合 entity_id_set = set() + + # 用于跟踪进度 + total_dialogs = len(dialog_data_list) + processed_dialogs = 0 for dialog_data in dialog_data_list: + processed_dialogs += 1 # 创建对话节点 dialogue_node = DialogueNode( id=dialog_data.id, @@ -994,6 +1014,26 @@ class ExtractionOrchestrator: expired_at=dialog_data.expired_at, ) entity_entity_edges.append(entity_entity_edge) + + # 流式输出：每创建一个关系边，立即发送进度（限制发送数量） + if self.progress_callback and len(entity_entity_edges) <= 10: + # 获取实体名称 + source_name = triplet.subject_name + target_name = triplet.object_name + relationship_result = { + "result_type": "relationship_creation", + "relationship_index": len(entity_entity_edges), + "source_entity": source_name, + "relation_type": triplet.predicate, + "target_entity": target_name, + "relationship_text": f"{source_name} -[{triplet.predicate}]-> {target_name}", + "dialog_progress": f"{processed_dialogs}/{total_dialogs}" + } + await self.progress_callback( + "creating_nodes_edges_result", + f"关系创建中 ({processed_dialogs}/{total_dialogs})", + relationship_result + ) else: logger.warning( f"跳过三元组 - 无法找到实体ID: subject_id={triplet.subject_id}, " @@ -1008,12 +1048,9 @@ class ExtractionOrchestrator: f"实体-实体边: {len(entity_entity_edges)}" ) - # 进度回调：只输出关系创建结果 + # 进度回调：创建节点和边完成，传递结果统计 + # 注意：具体的关系创建结果已经在创建过程中实时发送了 if self.progress_callback: - # 输出关系创建结果 - await self._output_relationship_creation_results(entity_entity_edges, entity_nodes) - - # 进度回调：创建节点和边完成，传递结果统计 nodes_edges_stats = { "dialogue_nodes_count": len(dialogue_nodes), "chunk_nodes_count": len(chunk_nodes), @@ -1071,7 +1108,7 @@ class ExtractionOrchestrator: """ logger.info("开始两阶段实体去重和消歧") - # 进度回调：正在去重消歧 + # 进度回调：发送去重消歧开始消息 if self.progress_callback: await self.progress_callback("deduplication", "正在去重消歧...") @@ -1154,25 +1191,26 @@ class ExtractionOrchestrator: f"实体-实体边减少 {len(entity_entity_edges) - len(final_entity_entity_edges)}" ) - # 进度回调：输出去重消歧的具体结果 + # 流式输出：实时输出去重消歧的具体结果 if self.progress_callback: - # 分析实体合并情况 + # 分析实体合并情况（使用内存中的记录） merge_info = await self._analyze_entity_merges(entity_nodes, final_entity_nodes) - # 输出去重合并的实体示例 + # 逐个输出去重合并的实体示例 for i, merge_detail in enumerate(merge_info[:5]): # 输出前5个去重结果 dedup_result = { "result_type": "entity_merge", "merged_entity_name": merge_detail["main_entity_name"], "merged_count": merge_detail["merged_count"], + "merge_progress": f"{i + 1}/{min(len(merge_info), 5)}", "message": f"{merge_detail['main_entity_name']}合并{merge_detail['merged_count']}个：相似实体已合并" } - await self.progress_callback("dedup_disambiguation_result", "实体去重完成", dedup_result) + await self.progress_callback("dedup_disambiguation_result", "实体去重中", dedup_result) - # 分析实体消歧情况 + # 分析实体消歧情况（使用内存中的记录） disamb_info = await self._analyze_entity_disambiguation(entity_nodes, final_entity_nodes) - # 输出实体消歧的结果 + # 逐个输出实体消歧的结果 for i, disamb_detail in enumerate(disamb_info[:5]): # 输出前5个消歧结果 disamb_result = { "result_type": "entity_disambiguation", @@ -1180,11 +1218,10 @@ class ExtractionOrchestrator: "disambiguation_type": disamb_detail["disamb_type"], "confidence": disamb_detail.get("confidence", "unknown"), "reason": disamb_detail.get("reason", ""), + "disamb_progress": f"{i + 1}/{min(len(disamb_info), 5)}", "message": f"{disamb_detail['entity_name']}消歧完成：{disamb_detail['disamb_type']}" } - await self.progress_callback("dedup_disambiguation_result", "实体消歧完成", disamb_result) - - + await self.progress_callback("dedup_disambiguation_result", "实体消歧中", disamb_result) # 进度回调：去重消歧完成，传递去重和消歧的具体效果 await self._send_dedup_progress_callback( From c0d6604981225e2f15fc0fc4170445dc27a5060a Mon Sep 17 00:00:00 2001 From: lixiangcheng1 Date: Thu, 18 Dec 2025 18:51:32 +0800 Subject: [PATCH 02/20] [fix]document chunk QA --- api/app/core/rag/graphrag/utils.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/api/app/core/rag/graphrag/utils.py b/api/app/core/rag/graphrag/utils.py index 65beb31f..a2290516 100644 --- a/api/app/core/rag/graphrag/utils.py +++ b/api/app/core/rag/graphrag/utils.py @@ -1,12 +1,23 @@ import xxhash -from app.aioRedis import aio_redis_set, aio_redis_get +import redis +from app.core.config import settings + +redis_client = redis.StrictRedis( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + db=settings.REDIS_DB, + password=settings.REDIS_PASSWORD, + decode_responses=True, + max_connections=30 +) + def get_llm_cache(llmnm, txt, history, genconf): hasher = xxhash.xxh64() - hasher.update((str(llmnm)+str(txt)+str(history)+str(genconf)).encode("utf-8")) + hasher.update((str(llmnm) + str(txt) + str(history) + str(genconf)).encode("utf-8")) k = hasher.hexdigest() - bin = aio_redis_get(k) + bin = redis_client.get(k) if not bin: return None return bin @@ -14,6 +25,6 @@ def get_llm_cache(llmnm, txt, history, genconf): def set_llm_cache(llmnm, txt, v, history, genconf): hasher = xxhash.xxh64() - hasher.update((str(llmnm)+str(txt)+str(history)+str(genconf)).encode("utf-8")) + hasher.update((str(llmnm) + str(txt) + str(history) + str(genconf)).encode("utf-8")) k = hasher.hexdigest() - aio_redis_set(k, v.encode("utf-8"), 24 * 3600) + redis_client.set(k, v.encode("utf-8"), 24 * 3600) From 0503b26232e6386a68351c9777e621ad9640a307 Mon Sep 17 00:00:00 2001 From: Mark Date: Thu, 18 Dec 2025 19:46:36 +0800 Subject: [PATCH 03/20] [add] workflow support stream mode --- api/app/controllers/app_controller.py | 19 +- api/app/core/workflow/executor.py | 198 +++++++++++---------- api/app/core/workflow/nodes/base_config.py | 5 + api/app/core/workflow/nodes/end/node.py | 1 - api/app/core/workflow/nodes/llm/node.py | 6 +- api/app/services/workflow_service.py | 145 ++++++++++++++- 6 files changed, 256 insertions(+), 118 deletions(-) diff --git a/api/app/controllers/app_controller.py b/api/app/controllers/app_controller.py index 3d09f5fc..a92cfab2 100644 --- a/api/app/controllers/app_controller.py +++ b/api/app/controllers/app_controller.py @@ -421,8 +421,8 @@ async def draft_run( # 流式返回 if payload.stream: async def event_generator(): - - + + async for event in draft_service.run_stream( agent_config=agent_cfg, model_config=model_config, @@ -574,7 +574,7 @@ async def draft_run( # 3. 流式返回 if payload.stream: logger.debug( - "开始多智能体流式试运行", + "开始工作流流式试运行", extra={ "app_id": str(app_id), "message_length": len(payload.message), @@ -583,16 +583,13 @@ async def draft_run( ) async def event_generator(): - """多智能体流式事件生成器""" - multiservice = MultiAgentService(db) + """工作流事件生成器""" # 调用多智能体服务的流式方法 - async for event in multiservice.run_stream( + async for event in workflow_service.run_stream( app_id=app_id, - request=multi_agent_request, - storage_type=storage_type, - user_rag_memory_id=user_rag_memory_id - + payload=payload, + config=config ): yield event @@ -617,7 +614,7 @@ async def draft_run( ) result = await workflow_service.run(app_id, payload,config) - + logger.debug( "工作流试运行返回结果", extra={ diff --git a/api/app/core/workflow/executor.py b/api/app/core/workflow/executor.py index a945356a..80d5316a 100644 --- a/api/app/core/workflow/executor.py +++ b/api/app/core/workflow/executor.py @@ -5,27 +5,25 @@ """ import logging -import uuid import datetime from typing import Any from langchain_core.messages import HumanMessage from langgraph.graph import StateGraph, START, END +from langgraph.graph.state import CompiledStateGraph from app.core.workflow.nodes import WorkflowState, NodeFactory from app.core.workflow.expression_evaluator import evaluate_condition -from app.models.workflow_model import WorkflowExecution, WorkflowNodeExecution -from app.db import get_db logger = logging.getLogger(__name__) class WorkflowExecutor: """工作流执行器 - + 负责将工作流配置转换为 LangGraph 并执行。 """ - + def __init__( self, workflow_config: dict[str, Any], @@ -34,7 +32,7 @@ class WorkflowExecutor: user_id: str ): """初始化执行器 - + Args: workflow_config: 工作流配置 execution_id: 执行 ID @@ -48,25 +46,25 @@ class WorkflowExecutor: self.nodes = workflow_config.get("nodes", []) self.edges = workflow_config.get("edges", []) self.execution_config = workflow_config.get("execution_config", {}) - + def _prepare_initial_state(self, input_data: dict[str, Any]) -> WorkflowState: """准备初始状态（注入系统变量和会话变量） - + 变量命名空间： - sys.xxx - 系统变量（execution_id, workspace_id, user_id, message, input_variables 等） - conv.xxx - 会话变量（跨多轮对话保持） - node_id.xxx - 节点输出（执行时动态生成） - + Args: input_data: 输入数据 - + Returns: 初始化的工作流状态 """ user_message = input_data.get("message") or "" conversation_vars = input_data.get("conversation_vars") or {} input_variables = input_data.get("variables") or {} # Start 节点的自定义变量 - + # 构建分层的变量结构 variables = { "sys": { @@ -79,7 +77,7 @@ class WorkflowExecutor: }, "conv": conversation_vars # 会话级变量（跨多轮对话保持） } - + return { "messages": [HumanMessage(content=user_message)], "variables": variables, @@ -91,34 +89,34 @@ class WorkflowExecutor: "error": None, "error_node": None } - - - def build_graph(self) -> StateGraph: + + + def build_graph(self) -> CompiledStateGraph: """构建 LangGraph - + Returns: 编译后的状态图 """ logger.info(f"开始构建工作流图: execution_id={self.execution_id}") - + # 1. 创建状态图 workflow = StateGraph(WorkflowState) - + # 2. 添加所有节点（包括 start 和 end） start_node_id = None end_node_ids = [] - + for node in self.nodes: node_type = node.get("type") node_id = node.get("id") - + # 记录 start 和 end 节点 ID if node_type == "start": start_node_id = node_id elif node_type == "end": end_node_ids.append(node_id) - + # 创建节点实例（现在 start 和 end 也会被创建） node_instance = NodeFactory.create_node(node, self.workflow_config) if node_instance: @@ -128,40 +126,40 @@ class WorkflowExecutor: async def node_func(state: WorkflowState): return await inst.run(state) return node_func - + workflow.add_node(node_id, make_node_func(node_instance)) logger.debug(f"添加节点: {node_id} (type={node_type})") - + # 3. 添加边 # 从 START 连接到 start 节点 if start_node_id: workflow.add_edge(START, start_node_id) logger.debug(f"添加边: START -> {start_node_id}") - + for edge in self.edges: source = edge.get("source") target = edge.get("target") edge_type = edge.get("type") condition = edge.get("condition") - + # 跳过从 start 节点出发的边（因为已经从 START 连接到 start） if source == start_node_id: # 但要连接 start 到下一个节点 workflow.add_edge(source, target) logger.debug(f"添加边: {source} -> {target}") continue - + # 处理到 end 节点的边 if target in end_node_ids: # 连接到 end 节点 workflow.add_edge(source, target) logger.debug(f"添加边: {source} -> {target}") continue - + # 跳过错误边（在节点内部处理） if edge_type == "error": continue - + if condition: # 条件边 def router(state: WorkflowState, cond=condition, tgt=target): @@ -178,74 +176,74 @@ class WorkflowExecutor: ): return tgt return END # 条件不满足，结束 - + workflow.add_conditional_edges(source, router) logger.debug(f"添加条件边: {source} -> {target} (condition={condition})") else: # 普通边 workflow.add_edge(source, target) logger.debug(f"添加边: {source} -> {target}") - + # 从 end 节点连接到 END for end_node_id in end_node_ids: workflow.add_edge(end_node_id, END) logger.debug(f"添加边: {end_node_id} -> END") - + # 4. 编译图 graph = workflow.compile() logger.info(f"工作流图构建完成: execution_id={self.execution_id}") - + return graph - + async def execute( self, input_data: dict[str, Any] ) -> dict[str, Any]: """执行工作流（非流式） - + Args: input_data: 输入数据，包含 message 和 variables - + Returns: 执行结果，包含 status, output, node_outputs, elapsed_time, token_usage """ logger.info(f"开始执行工作流: execution_id={self.execution_id}") - + # 记录开始时间 start_time = datetime.datetime.now() - + # 1. 构建图 graph = self.build_graph() - + # 2. 初始化状态（自动注入系统变量） initial_state = self._prepare_initial_state(input_data) - + # 3. 执行工作流 try: result = await graph.ainvoke(initial_state) - + # 计算耗时 end_time = datetime.datetime.now() elapsed_time = (end_time - start_time).total_seconds() - + # 提取节点输出（现在包含 start 和 end 节点） node_outputs = result.get("node_outputs", {}) - + # 提取最终输出（从最后一个非 start/end 节点） final_output = self._extract_final_output(node_outputs) - + # 聚合 token 使用情况 token_usage = self._aggregate_token_usage(node_outputs) - + # 提取 conversation_id（从 start 节点输出） conversation_id = None for node_id, node_output in node_outputs.items(): if node_output.get("node_type") == "start": conversation_id = node_output.get("output", {}).get("conversation_id") break - + logger.info(f"工作流执行完成: execution_id={self.execution_id}, elapsed_time={elapsed_time:.2f}s") - + return { "status": "completed", "output": final_output, @@ -256,12 +254,12 @@ class WorkflowExecutor: "token_usage": token_usage, "error": result.get("error") } - + except Exception as e: # 计算耗时（即使失败也记录） end_time = datetime.datetime.now() elapsed_time = (end_time - start_time).total_seconds() - + logger.error(f"工作流执行失败: execution_id={self.execution_id}, error={e}", exc_info=True) return { "status": "failed", @@ -271,86 +269,94 @@ class WorkflowExecutor: "elapsed_time": elapsed_time, "token_usage": None } - + async def execute_stream( self, input_data: dict[str, Any] ): """执行工作流（流式） - + + 手动执行节点以支持细粒度的流式输出： + - workflow_start: 工作流开始 + - node_start: 节点开始执行 + - node_chunk: LLM 节点的流式输出片段（逐 token） + - node_complete: 节点执行完成 + - workflow_complete: 工作流完成 + Args: input_data: 输入数据 - + Yields: 流式事件 """ - logger.info(f"开始执行工作流（流式）: execution_id={self.execution_id}") - + # + logger.info(f"开始执行工作流: execution_id={self.execution_id}") + + # 记录开始时间 + start_time = datetime.datetime.now() + # 1. 构建图 graph = self.build_graph() - + # 2. 初始化状态（自动注入系统变量） initial_state = self._prepare_initial_state(input_data) - - # 3. 流式执行工作流 + + # 3. 执行工作流 try: - # 使用 astream 获取节点级别的更新 - async for event in graph.astream(initial_state, stream_mode="updates"): - for node_name, state_update in event.items(): - yield { - "type": "node_complete", - "node": node_name, - "data": state_update, - "execution_id": self.execution_id - } - - logger.info(f"工作流执行完成（流式）: execution_id={self.execution_id}") - - # 发送完成事件 - yield { - "type": "workflow_complete", - "execution_id": self.execution_id - } - + async for chunk in graph.astream( + initial_state, + # subgraphs=True, + stream_mode="updates", + ): + # print(chunk) + yield chunk + except Exception as e: - logger.error(f"工作流执行失败（流式）: execution_id={self.execution_id}, error={e}", exc_info=True) + # 计算耗时（即使失败也记录） + end_time = datetime.datetime.now() + elapsed_time = (end_time - start_time).total_seconds() + + logger.error(f"工作流执行失败: execution_id={self.execution_id}, error={e}", exc_info=True) yield { - "type": "workflow_error", - "execution_id": self.execution_id, - "error": str(e) + "status": "failed", + "error": str(e), + "output": None, + "node_outputs": {}, + "elapsed_time": elapsed_time, + "token_usage": None } - + def _extract_final_output(self, node_outputs: dict[str, Any]) -> str | None: """从节点输出中提取最终输出 - + 优先级： 1. 最后一个执行的非 start/end 节点的 output 2. 如果没有节点输出，返回 None - + Args: node_outputs: 所有节点的输出 - + Returns: 最终输出字符串或 None """ if not node_outputs: return None - + # 获取最后一个节点的输出 last_node_output = list(node_outputs.values())[-1] if node_outputs else None - + if last_node_output and isinstance(last_node_output, dict): return last_node_output.get("output") - + return None - + def _aggregate_token_usage(self, node_outputs: dict[str, Any]) -> dict[str, int] | None: """聚合所有节点的 token 使用情况 - + Args: node_outputs: 所有节点的输出 - + Returns: 聚合的 token 使用情况 {"prompt_tokens": x, "completion_tokens": y, "total_tokens": z} 如果没有 token 使用信息，返回 None @@ -359,7 +365,7 @@ class WorkflowExecutor: total_completion_tokens = 0 total_tokens = 0 has_token_info = False - + for node_output in node_outputs.values(): if isinstance(node_output, dict): token_usage = node_output.get("token_usage") @@ -368,16 +374,16 @@ class WorkflowExecutor: total_prompt_tokens += token_usage.get("prompt_tokens", 0) total_completion_tokens += token_usage.get("completion_tokens", 0) total_tokens += token_usage.get("total_tokens", 0) - + if not has_token_info: return None - + return { "prompt_tokens": total_prompt_tokens, "completion_tokens": total_completion_tokens, "total_tokens": total_tokens } - + async def execute_workflow( workflow_config: dict[str, Any], @@ -387,14 +393,14 @@ async def execute_workflow( user_id: str ) -> dict[str, Any]: """执行工作流（便捷函数） - + Args: workflow_config: 工作流配置 input_data: 输入数据 execution_id: 执行 ID workspace_id: 工作空间 ID user_id: 用户 ID - + Returns: 执行结果 """ @@ -415,14 +421,14 @@ async def execute_workflow_stream( user_id: str ): """执行工作流（流式，便捷函数） - + Args: workflow_config: 工作流配置 input_data: 输入数据 execution_id: 执行 ID workspace_id: 工作空间 ID user_id: 用户 ID - + Yields: 流式事件 """ diff --git a/api/app/core/workflow/nodes/base_config.py b/api/app/core/workflow/nodes/base_config.py index 8423f479..90d02732 100644 --- a/api/app/core/workflow/nodes/base_config.py +++ b/api/app/core/workflow/nodes/base_config.py @@ -50,6 +50,11 @@ class VariableDefinition(BaseModel): description="变量描述" ) + max_length: int = Field( + default=200, + description="只对字符串类型生效" + ) + class Config: json_schema_extra = { "examples": [ diff --git a/api/app/core/workflow/nodes/end/node.py b/api/app/core/workflow/nodes/end/node.py index 1c0e6747..ad028f31 100644 --- a/api/app/core/workflow/nodes/end/node.py +++ b/api/app/core/workflow/nodes/end/node.py @@ -5,7 +5,6 @@ End 节点实现 """ import logging -from typing import Any from app.core.workflow.nodes.base_node import BaseNode, WorkflowState diff --git a/api/app/core/workflow/nodes/llm/node.py b/api/app/core/workflow/nodes/llm/node.py index bfc7da58..cf665ff1 100644 --- a/api/app/core/workflow/nodes/llm/node.py +++ b/api/app/core/workflow/nodes/llm/node.py @@ -10,10 +10,8 @@ from langchain_core.messages import AIMessage, SystemMessage, HumanMessage from app.core.workflow.nodes.base_node import BaseNode, WorkflowState from app.core.models import RedBearLLM, RedBearModelConfig -from app.models import ModelConfig -from app.db import get_db, get_db_context -from app.models.models_model import ModelApiKey -from app.services.model_service import ModelConfigService, ModelApiKeyService +from app.db import get_db_context +from app.services.model_service import ModelConfigService from app.core.exceptions import BusinessException from app.core.error_codes import BizCode diff --git a/api/app/services/workflow_service.py b/api/app/services/workflow_service.py index c604697b..f0b71824 100644 --- a/api/app/services/workflow_service.py +++ b/api/app/services/workflow_service.py @@ -1,7 +1,7 @@ """ 工作流服务层 """ - +import json import logging import uuid import datetime @@ -438,7 +438,7 @@ class WorkflowService: message=f"工作流配置不存在: app_id={app_id}" ) input_data = {"message": payload.message, "variables": payload.variables, "conversation_id": payload.conversation_id} - + # 转换 user_id 为 UUID triggered_by_uuid = None if payload.user_id: @@ -446,7 +446,7 @@ class WorkflowService: triggered_by_uuid = uuid.UUID(payload.user_id) except (ValueError, AttributeError): logger.warning(f"无效的 user_id 格式: {payload.user_id}") - + # 转换 conversation_id 为 UUID conversation_id_uuid = None if payload.conversation_id: @@ -454,7 +454,7 @@ class WorkflowService: conversation_id_uuid = uuid.UUID(payload.conversation_id) except (ValueError, AttributeError): logger.warning(f"无效的 conversation_id 格式: {payload.conversation_id}") - + # 2. 创建执行记录 execution = self.create_execution( workflow_config_id=config.id, @@ -530,6 +530,109 @@ class WorkflowService: message=f"工作流执行失败: {str(e)}" ) + async def run_stream( + self, + app_id: uuid.UUID, + payload: DraftRunRequest, + config: WorkflowConfig + ): + """运行工作流（流式） + + Args: + app_id: 应用 ID + payload: 请求对象（包含 message, variables, conversation_id 等） + config: 存储类型（可选） + + Yields: + SSE 格式的流式事件 + + Raises: + BusinessException: 配置不存在或执行失败时抛出 + """ + # 1. 获取工作流配置 + if not config: + config = self.get_workflow_config(app_id) + if not config: + raise BusinessException( + code=BizCode.CONFIG_MISSING, + message=f"工作流配置不存在: app_id={app_id}" + ) + input_data = {"message": payload.message, "variables": payload.variables, + "conversation_id": payload.conversation_id} + + # 转换 user_id 为 UUID + triggered_by_uuid = None + if payload.user_id: + try: + triggered_by_uuid = uuid.UUID(payload.user_id) + except (ValueError, AttributeError): + logger.warning(f"无效的 user_id 格式: {payload.user_id}") + + # 转换 conversation_id 为 UUID + conversation_id_uuid = None + if payload.conversation_id: + try: + conversation_id_uuid = uuid.UUID(payload.conversation_id) + except (ValueError, AttributeError): + logger.warning(f"无效的 conversation_id 格式: {payload.conversation_id}") + + # 2. 创建执行记录 + execution = self.create_execution( + workflow_config_id=config.id, + app_id=app_id, + trigger_type="manual", + triggered_by=triggered_by_uuid, + conversation_id=conversation_id_uuid, + input_data=input_data + ) + + # 3. 构建工作流配置字典 + workflow_config_dict = { + "nodes": config.nodes, + "edges": config.edges, + "variables": config.variables, + "execution_config": config.execution_config + } + + # 4. 获取工作空间 ID（从 app 获取） + from app.models import App + + # 5. 流式执行工作流 + from app.core.workflow.executor import execute_workflow, execute_workflow_stream + + try: + # 更新状态为运行中 + self.update_execution_status(execution.execution_id, "running") + + # 发送开始事件 + yield f"data: {json.dumps({'type': 'workflow_start', 'execution_id': execution.execution_id})}\n\n" + + # 调用流式执行 + async for event in self._run_workflow_stream( + workflow_config=workflow_config_dict, + input_data=input_data, + execution_id=execution.execution_id, + workspace_id="", + user_id=payload.user_id + ): + # 清理事件数据，移除不可序列化的对象 + cleaned_event = self._clean_event_for_json(event) + # 转换为 SSE 格式 + yield f"data: {json.dumps(cleaned_event)}\n\n" + + # 发送完成事件 + yield f"data: {json.dumps({'type': 'workflow_end', 'execution_id': execution.execution_id})}\n\n" + + except Exception as e: + logger.error(f"工作流流式执行失败: execution_id={execution.execution_id}, error={e}", exc_info=True) + self.update_execution_status( + execution.execution_id, + "failed", + error_message=str(e) + ) + # 发送错误事件 + yield f"data: {json.dumps({'type': 'error', 'execution_id': execution.execution_id, 'error': str(e)})}\n\n" + async def run_workflow( self, app_id: uuid.UUID, @@ -651,14 +754,44 @@ class WorkflowService: message=f"工作流执行失败: {str(e)}" ) + def _clean_event_for_json(self, event: dict[str, Any]) -> dict[str, Any]: + """清理事件数据，移除不可序列化的对象 + + Args: + event: 原始事件数据 + + Returns: + 可序列化的事件数据 + """ + from langchain_core.messages import BaseMessage + + def clean_value(value): + """递归清理值""" + if isinstance(value, BaseMessage): + # 将 Message 对象转换为字典 + return { + "type": value.__class__.__name__, + "content": value.content, + } + elif isinstance(value, dict): + return {k: clean_value(v) for k, v in value.items()} + elif isinstance(value, list): + return [clean_value(item) for item in value] + elif isinstance(value, (str, int, float, bool, type(None))): + return value + else: + # 其他不可序列化的对象转换为字符串 + return str(value) + + return clean_value(event) + async def _run_workflow_stream( self, workflow_config: dict[str, Any], input_data: dict[str, Any], execution_id: str, workspace_id: str, - user_id: str - ): + user_id: str): """运行工作流（流式，内部方法） Args: From 406a6d1d9015780f7f8c4947ea4d20650a30e370 Mon Sep 17 00:00:00 2001 From: mengyonghao <1533512157@qq.com> Date: Thu, 18 Dec 2025 14:50:10 +0800 Subject: [PATCH 04/20] fix(workflow): fix run_workflow streaming issues Resolve exceptions during run_workflow streaming and define proper status codes for error cases. --- api/app/controllers/workflow_controller.py | 2 +- api/app/services/workflow_service.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/api/app/controllers/workflow_controller.py b/api/app/controllers/workflow_controller.py index 9ccfa858..091846f6 100644 --- a/api/app/controllers/workflow_controller.py +++ b/api/app/controllers/workflow_controller.py @@ -473,7 +473,7 @@ async def run_workflow( async def event_generator(): """生成 SSE 事件""" try: - async for event in service.run_workflow( + async for event in await service.run_workflow( app_id=app_id, input_data=input_data, triggered_by=current_user.id, diff --git a/api/app/services/workflow_service.py b/api/app/services/workflow_service.py index f0b71824..fbf09505 100644 --- a/api/app/services/workflow_service.py +++ b/api/app/services/workflow_service.py @@ -5,7 +5,7 @@ import json import logging import uuid import datetime -from typing import Any, Annotated +from typing import Any, Annotated, AsyncGenerator from sqlalchemy.orm import Session from fastapi import Depends @@ -81,7 +81,7 @@ class WorkflowService: if not is_valid: logger.warning(f"工作流配置验证失败: {errors}") raise BusinessException( - error_code=BizCode.INVALID_PARAMETER, + code=BizCode.INVALID_PARAMETER, message=f"工作流配置无效: {'; '.join(errors)}" ) @@ -140,7 +140,7 @@ class WorkflowService: config = self.get_workflow_config(app_id) if not config: raise BusinessException( - error_code=BizCode.RESOURCE_NOT_FOUND, + code=BizCode.NOT_FOUND, message=f"工作流配置不存在: app_id={app_id}" ) @@ -166,7 +166,7 @@ class WorkflowService: if not is_valid: logger.warning(f"工作流配置验证失败: {errors}") raise BusinessException( - error_code=BizCode.INVALID_PARAMETER, + code=BizCode.INVALID_PARAMETER, message=f"工作流配置无效: {'; '.join(errors)}" ) @@ -245,7 +245,7 @@ class WorkflowService: config = self.get_workflow_config(app_id) if not config: raise BusinessException( - error_code=BizCode.RESOURCE_NOT_FOUND, + code=BizCode.NOT_FOUND, message=f"工作流配置不存在: app_id={app_id}" ) @@ -359,7 +359,7 @@ class WorkflowService: execution = self.get_execution(execution_id) if not execution: raise BusinessException( - error_code=BizCode.RESOURCE_NOT_FOUND, + code=BizCode.NOT_FOUND, message=f"执行记录不存在: execution_id={execution_id}" ) @@ -640,7 +640,7 @@ class WorkflowService: triggered_by: uuid.UUID, conversation_id: uuid.UUID | None = None, stream: bool = False - ): + ) -> AsyncGenerator | dict: """运行工作流 Args: @@ -660,7 +660,7 @@ class WorkflowService: config = self.get_workflow_config(app_id) if not config: raise BusinessException( - error_code=BizCode.RESOURCE_NOT_FOUND, + code=BizCode.NOT_FOUND, message=f"工作流配置不存在: app_id={app_id}" ) @@ -687,7 +687,7 @@ class WorkflowService: app = self.db.query(App).filter(App.id == app_id).first() if not app: raise BusinessException( - error_code=BizCode.RESOURCE_NOT_FOUND, + code=BizCode.NOT_FOUND, message=f"应用不存在: app_id={app_id}" ) @@ -750,7 +750,7 @@ class WorkflowService: error_message=str(e) ) raise BusinessException( - error_code=BizCode.INTERNAL_ERROR, + code=BizCode.INTERNAL_ERROR, message=f"工作流执行失败: {str(e)}" ) From c1a412508ba6c78c5e6591f3a4296fd5959be416 Mon Sep 17 00:00:00 2001 From: mengyonghao <1533512157@qq.com> Date: Fri, 19 Dec 2025 14:08:54 +0800 Subject: [PATCH 05/20] fix(prompt-optimizer): switch to built-in system prompt - Replace the system prompt of the prompt optimization model with a built-in prompt. - Remove system prompt entries from the database. - Remove the API endpoint for managing system prompt configuration. --- .../prompt_optimizer_controller.py | 34 +--- api/app/models/__init__.py | 3 +- api/app/models/prompt_optimizer_model.py | 43 ----- .../prompt_optimizer_repository.py | 105 ----------- api/app/services/prompt_optimizer_service.py | 170 +++++++++--------- 5 files changed, 86 insertions(+), 269 deletions(-) diff --git a/api/app/controllers/prompt_optimizer_controller.py b/api/app/controllers/prompt_optimizer_controller.py index d647f0c0..d73ea0df 100644 --- a/api/app/controllers/prompt_optimizer_controller.py +++ b/api/app/controllers/prompt_optimizer_controller.py @@ -117,7 +117,7 @@ async def get_prompt_opt( session_id=session_id, user_id=current_user.id, current_prompt=data.current_prompt, - message=data.message + user_require=data.message ) service.create_message( tenant_id=current_user.tenant_id, @@ -136,35 +136,3 @@ async def get_prompt_opt( return success(data=result_schema) -@router.put( - "/model", - summary="Create or update prompt model config", - response_model=ApiResponse -) -def set_system_prompt( - data: PromptOptModelSet = ..., - db: Session = Depends(get_db), - current_user=Depends(get_current_user), -): - """ - Create or update a system prompt model configuration for the tenant. - - Args: - data (PromptOptModelSet): Model configuration data including model ID, - system prompt, and optional configuration ID - db (Session): Database session - current_user: Current user information - - Returns: - UUID: The ID of the created or updated model configuration. - """ - if data.id is None: - data.id = uuid.uuid4() - - model_config = PromptOptimizerService(db).create_update_model_config( - current_user.tenant_id, - data.id, - data.system_prompt - ) - return success(data=model_config.id) - diff --git a/api/app/models/__init__.py b/api/app/models/__init__.py index fc497215..198a788e 100644 --- a/api/app/models/__init__.py +++ b/api/app/models/__init__.py @@ -20,7 +20,7 @@ from .data_config_model import DataConfig from .multi_agent_model import MultiAgentConfig, AgentInvocation from .workflow_model import WorkflowConfig, WorkflowExecution, WorkflowNodeExecution from .retrieval_info import RetrievalInfo -from .prompt_optimizer_model import PromptOptimizerModelConfig, PromptOptimizerSession, PromptOptimizerSessionHistory +from .prompt_optimizer_model import PromptOptimizerSession, PromptOptimizerSessionHistory __all__ = [ "Tenants", @@ -56,7 +56,6 @@ __all__ = [ "WorkflowExecution", "WorkflowNodeExecution", "RetrievalInfo", - "PromptOptimizerModelConfig", "PromptOptimizerSession", "PromptOptimizerSessionHistory" ] diff --git a/api/app/models/prompt_optimizer_model.py b/api/app/models/prompt_optimizer_model.py index 5191fc2e..39845ee7 100644 --- a/api/app/models/prompt_optimizer_model.py +++ b/api/app/models/prompt_optimizer_model.py @@ -27,49 +27,6 @@ class RoleType(StrEnum): ASSISTANT = "assistant" -class PromptOptimizerModelConfig(Base): - """ - Prompt Optimization Model Configuration. - - This table stores system-level prompt configurations for each tenant. - The configuration defines the base system prompt used during prompt - optimization sessions and serves as a foundational instruction set - for the optimization process. - - Each tenant may have one or more model configurations depending on - business requirements. - - Table Name: - prompt_model_config - - Columns: - id (UUID): - Primary key. Unique identifier for the prompt model configuration. - tenant_id (UUID): - Foreign key referencing `tenants.id`. - Identifies the tenant that owns this configuration. - system_prompt (Text): - The system-level prompt used to guide prompt optimization logic. - created_at (DateTime): - Timestamp indicating when the configuration was created. - updated_at (DateTime): - Timestamp indicating the last update time of the configuration. - - Usage: - - Loaded when initializing a prompt optimization session - - Acts as the root system instruction for all subsequent prompts - """ - __tablename__ = "prompt_model_config" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, index=True) - tenant_id = Column(UUID(as_uuid=True), ForeignKey("tenants.id"), nullable=False, comment="Tenant ID") - # model_id = Column(UUID(as_uuid=True), nullable=False, comment="Model ID") - system_prompt = Column(Text, nullable=False, comment="System Prompt") - - created_at = Column(DateTime, default=datetime.datetime.now, comment="Creation Time") - updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now, comment="Update Time") - - class PromptOptimizerSession(Base): """ Prompt Optimization Session Registry. diff --git a/api/app/repositories/prompt_optimizer_repository.py b/api/app/repositories/prompt_optimizer_repository.py index ecb2af98..ba65257a 100644 --- a/api/app/repositories/prompt_optimizer_repository.py +++ b/api/app/repositories/prompt_optimizer_repository.py @@ -1,120 +1,15 @@ import uuid -from typing import Optional from sqlalchemy.orm import Session from app.core.logging_config import get_db_logger from app.models.prompt_optimizer_model import ( - PromptOptimizerModelConfig, PromptOptimizerSession, PromptOptimizerSessionHistory, RoleType ) db_logger = get_db_logger() -class PromptOptimizerModelConfigRepository: - """Repository for managing prompt optimizer model configurations.""" - - def __init__(self, db: Session): - self.db = db - - def get_by_tenant_id(self, tenant_id: uuid.UUID) -> Optional[PromptOptimizerModelConfig]: - """ - Retrieve the prompt optimizer model configuration for a specific tenant. - - Args: - tenant_id (uuid.UUID): The unique identifier of the tenant. - - Returns: - Optional[PromptOptimizerModelConfig]: The model configuration if found, else None. - """ - db_logger.debug(f"Get prompt optimization model configuration: tenant_id={tenant_id}") - - try: - config = self.db.query(PromptOptimizerModelConfig).filter( - PromptOptimizerModelConfig.tenant_id == tenant_id, - # PromptOptimizerModelConfig.model_id == model_id - ).first() - if config: - db_logger.debug(f"Prompt optimization model configuration found: (ID: {config.id})") - else: - db_logger.debug(f"Prompt optimization model configuration not found: tenant_id={tenant_id}") - return config - except Exception as e: - db_logger.error( - f"Error retrieving prompt optimization model configuration: tenant_id={tenant_id} - {str(e)}") - raise - - def get_by_config_id(self, tenant_id: uuid.UUID, config_id: uuid.UUID) -> Optional[PromptOptimizerModelConfig]: - """ - Retrieve a specific prompt optimizer model configuration by config ID and tenant ID. - - Args: - tenant_id (uuid.UUID): The unique identifier of the tenant. - config_id (uuid.UUID): The unique identifier of the model configuration. - - Returns: - Optional[PromptOptimizerModelConfig]: The model configuration if found, else None. - """ - db_logger.debug(f"Get prompt optimization model configuration: config_id={config_id}, tenant_id={tenant_id}") - try: - model = self.db.query(PromptOptimizerModelConfig).filter( - PromptOptimizerModelConfig.tenant_id == tenant_id, - PromptOptimizerModelConfig.id == config_id - ).first() - if model: - db_logger.debug(f"Prompt optimization model configuration found: (ID: {model.id})") - else: - db_logger.debug(f"Prompt optimization model configuration not found: config_id={config_id}") - return model - except Exception as e: - db_logger.error( - f"Error retrieving prompt optimization model configuration: model_id={config_id} - {str(e)}") - raise - - def create_or_update( - self, - config_id: uuid.UUID, - tenant_id: uuid.UUID, - system_prompt: str, - ) -> Optional[PromptOptimizerModelConfig]: - """ - Create a new or update an existing prompt optimizer model configuration. - - If a configuration with the given config_id exists, it updates its system_prompt. - Otherwise, it creates a new configuration record. - - Args: - config_id (uuid.UUID): The unique identifier for the configuration. - tenant_id (uuid.UUID): The tenant's unique identifier. - system_prompt (str): The system prompt content for prompt optimization. - - Returns: - Optional[PromptOptimizerModelConfig]: The created or updated model configuration. - """ - db_logger.debug(f"Create/Update prompt optimization model configuration: tenant_id={tenant_id}") - existing_config = self.get_by_config_id(tenant_id, config_id) - - if existing_config: - existing_config.system_prompt = system_prompt - self.db.commit() - self.db.refresh(existing_config) - db_logger.debug(f"Prompt optimization model configuration update: ID:{config_id}") - return existing_config - else: - config = PromptOptimizerModelConfig( - id=config_id, - # model_id=model_id, - tenant_id=tenant_id, - system_prompt=system_prompt - ) - self.db.add(config) - self.db.commit() - self.db.refresh(config) - db_logger.debug(f"Prompt optimization model configuration created: ID:{config.id}") - return config - - class PromptOptimizerSessionRepository: """Repository for managing prompt optimization sessions and session history.""" diff --git a/api/app/services/prompt_optimizer_service.py b/api/app/services/prompt_optimizer_service.py index 0cdaabf5..5355474f 100644 --- a/api/app/services/prompt_optimizer_service.py +++ b/api/app/services/prompt_optimizer_service.py @@ -1,4 +1,3 @@ -import json import re import uuid @@ -12,13 +11,11 @@ from app.core.models import RedBearModelConfig from app.core.models.llm import RedBearLLM from app.models import ModelConfig, ModelApiKey, ModelType, PromptOptimizerSessionHistory from app.models.prompt_optimizer_model import ( - PromptOptimizerModelConfig, PromptOptimizerSession, RoleType ) from app.repositories.model_repository import ModelConfigRepository from app.repositories.prompt_optimizer_repository import ( - PromptOptimizerModelConfigRepository, PromptOptimizerSessionRepository ) from app.schemas.prompt_optimizer_schema import OptimizePromptResult @@ -34,32 +31,24 @@ class PromptOptimizerService: self, tenant_id: uuid.UUID, model_id: uuid.UUID - ) -> tuple[PromptOptimizerModelConfig, ModelConfig]: + ) -> ModelConfig: """ - Retrieve the prompt optimizer model configuration and model configuration. + Retrieve the model configuration for a specific tenant. - This method retrieves the prompt optimizer model configuration associated - with the specified model ID and tenant. It also fetches the corresponding - model configuration. + This method fetches the model configuration associated with the given + tenant_id and model_id. If no configuration is found, a BusinessException + is raised. Args: tenant_id (uuid.UUID): The unique identifier of the tenant. - model_id (uuid.UUID): The unique identifier of the prompt optimization model. + model_id (uuid.UUID): The unique identifier of the model. Returns: - tuple[PromptOptimzerModelConfig, ModelConfig]: - A tuple containing the prompt optimizer model configuration - and the corresponding model configuration. + ModelConfig: The corresponding model configuration object. Raises: - BusinessException: If the prompt optimizer model configuration does not exist. BusinessException: If the model configuration does not exist. """ - prompt_config = PromptOptimizerModelConfigRepository(self.db).get_by_tenant_id( - tenant_id - ) - if not prompt_config: - raise BusinessException("提示词模型配置不存在", BizCode.NOT_FOUND) model = ModelConfigRepository.get_by_id( self.db, model_id, tenant_id=tenant_id @@ -67,35 +56,7 @@ class PromptOptimizerService: if not model: raise BusinessException("模型配置不存在", BizCode.MODEL_NOT_FOUND) - return prompt_config, model - - def create_update_model_config( - self, - tenant_id: uuid.UUID, - config_id: uuid.UUID, - system_prompt: str, - ) -> PromptOptimizerModelConfig: - """ - Create or update a prompt optimizer model configuration. - - This method creates a new prompt optimizer model configuration or updates - an existing one identified by the given configuration ID. The configuration - defines the system prompt used for prompt optimization. - - Args: - tenant_id (uuid.UUID): The unique identifier of the tenant. - config_id (uuid.UUID): The unique identifier of the configuration to create or update. - system_prompt (str): The system prompt content used for prompt optimization. - - Returns: - PromptOptimzerModelConfig: The created or updated prompt optimizer model configuration. - """ - prompt_config = PromptOptimizerModelConfigRepository(self.db).create_or_update( - config_id=config_id, - tenant_id=tenant_id, - system_prompt=system_prompt, - ) - return prompt_config + return model def create_session( self, @@ -159,37 +120,46 @@ class PromptOptimizerService: session_id: uuid.UUID, user_id: uuid.UUID, current_prompt: str, - message: str + user_require: str ) -> OptimizePromptResult: """ - Optimize a prompt using a prompt optimizer LLM. + Optimize a user-provided prompt using a configured prompt optimizer LLM. - This method uses a configured prompt optimizer model to refine an existing - prompt based on the user's requirements. The optimized prompt is generated - according to predefined system rules, including Jinja2 variable syntax and - a strict JSON output format. + This method refines the original prompt according to the user's requirements, + generating an optimized version that is directly usable by AI tools. The + optimization process follows strict rules, including: + - Wrapping user-inserted variables in double curly braces {{}}. + - Adhering to Jinja2 variable syntax if applicable. + - Ensuring a clear logic flow, explicit instructions, and strong executability. + - Producing output in a strict JSON format. + + Steps performed: + 1. Retrieve the model configuration for the given tenant and model. + 2. Fetch the session message history for context. + 3. Instantiate the LLM with the appropriate API key and model configuration. + 4. Build system messages outlining optimization rules. + 5. Format the user's original prompt and requirements as a user message. + 6. Send messages to the LLM to generate the optimized prompt. + 7. Generate a concise description summarizing the changes made during optimization. Args: - tenant_id (uuid.UUID): The unique identifier of the tenant. - model_id (uuid.UUID): The unique identifier of the prompt optimizer model. - session_id (uuid.UUID): The unique identifier of the prompt optimization session. - user_id (uuid.UUID): The unique identifier of the user associated with the session. - current_prompt (str): The original prompt to be optimized. - message (str): The user's requirements or modification instructions. + tenant_id (uuid.UUID): Tenant identifier. + model_id (uuid.UUID): Prompt optimizer model identifier. + session_id (uuid.UUID): Prompt optimization session identifier. + user_id (uuid.UUID): Identifier of the user associated with the session. + current_prompt (str): Original prompt to optimize. + user_require (str): User's requirements or instructions for optimization. Returns: - dict: A dictionary containing the optimized prompt and the description - of changes, in the following format: - { - "prompt": "", - "desc": "" - } + OptimizePromptResult: An object containing: + - prompt: The optimized prompt string. + - desc: A short description summarizing the changes. Raises: - BusinessException: If the model response cannot be parsed as valid JSON + BusinessException: If the LLM response cannot be parsed as valid JSON or does not conform to the expected output format. """ - prompt_config, model_config = self.get_model_config(tenant_id, model_id) + model_config = self.get_model_config(tenant_id, model_id) session_history = self.get_session_message_history(session_id=session_id, user_id=user_id) # Create LLM instance @@ -204,36 +174,65 @@ class PromptOptimizerService: # build message messages = [ # init system_prompt - (RoleType.SYSTEM.value, prompt_config.system_prompt), + ( + RoleType.SYSTEM.value, + "Your task is to optimize the original prompt provided by the user so that it can be directly used by AI tools," + "and the variables that the user needs to insert must be wrapped in {{}}. " + "The optimized prompt should align with the optimization direction specified by the user (if any) and ensure clear logic, explicit instructions, and strong executability. " + "Please follow these rules when optimizing: " + '1. Ensure variables are wrapped in {{}}, e.g., optimize "Please enter your question" to "Please enter your {{question}}"' + "2. Instructions must be specific and operable, avoiding vague expressions" + "3. If the original prompt lacks key elements (such as output format requirements), supplement them completely " + "4. Keep the language concise and avoid redundancy " + "5. If the user does not specify an optimization direction, the default optimization is to make the prompt structurally clear and with explicit instructions" + "Please directly output the optimized prompt without additional explanations. The optimized prompt should be directly usable with correct variable positions." + ), # base model limit (RoleType.SYSTEM.value, "Optimization Rules:\n" "1. Fully adjust the prompt content according to the user's requirements.\n" - "2. When the user requests the insertion of variables, you must use Jinja2 syntax {{variable_name}} " - "(the variable name should be determined based on the user's requirement).\n" + "When variables are required, use double curly braces {{variable_name}} as placeholders." + "Variable names must be derived from the user's requirements.\n" "3. Keep the prompt logic clear and instructions explicit.\n" - "4. Ensure that the modified prompt can be directly used.\n\n" - "Output Requirements:\n" - "Provide the result in JSON format, containing exactly two fields:\n" - " - prompt: The modified prompt (string).\n" - " - desc: A response addressing the user's optimization request (string).") + "4. Ensure that the modified prompt can be directly used.\n\n") ] messages.extend(session_history[:-1]) # last message is current message user_message_template = ChatPromptTemplate.from_messages([ - (RoleType.USER.value, "[current_prompt]\n{current_prompt}\n[user_require]\n{message}") + (RoleType.USER.value, "[original_prompt]\n{current_prompt}\n[user_require]\n{user_require}") ]) - formatted_user_message = user_message_template.format(current_prompt=current_prompt, message=message) + formatted_user_message = user_message_template.format(current_prompt=current_prompt, user_require=user_require) messages.extend([(RoleType.USER.value, formatted_user_message)]) logger.info(f"Prompt optimization message: {messages}") - result = await llm.ainvoke(messages) - try: - data_dict = json.loads(result.content) - model_resp = OptimizePromptResult.model_validate(data_dict) - except Exception as e: - logger.error(f"Failed to parse model reponse to json - Error: {str(e)}", exc_info=True) - raise BusinessException("Failed to parse model response", BizCode.PARSER_NOT_SUPPORTED) - return model_resp + optim_prompt = await llm.ainvoke(messages) + optim_desc = [ + ( + RoleType.SYSTEM.value, + "You are a prompt optimization assistant.\n" + "Compare the original prompt, the user's requirements, " + "and the optimized prompt.\n" + "Summarize the changes made during optimization.\n\n" + "Rules:\n" + "1. Output must be a single short sentence.\n" + "2. Be concise and factual.\n" + "3. Do not explain the prompts themselves.\n" + "4. Do not include any extra text." + ), + ( + "[Original Prompt]\n" + f"{current_prompt}\n\n" + "[User Requirements]\n" + f"{user_require}\n\n" + "[Optimized Prompt]\n" + f"{optim_prompt.content}" + ) + ] + optim_desc = await llm.ainvoke(optim_desc) + + return OptimizePromptResult( + prompt=optim_prompt.content, + desc=optim_desc.content + ) @staticmethod def parser_prompt_variables(prompt: str): @@ -277,4 +276,3 @@ class PromptOptimizerService: content=content ) return message - From c06a7b31ae138b47c932b01fba20a553a5889411 Mon Sep 17 00:00:00 2001 From: mengyonghao <1533512157@qq.com> Date: Fri, 19 Dec 2025 14:19:18 +0800 Subject: [PATCH 06/20] feat(workflow): add conditional branch (If-Else) node - Introduce a new conditional branch node for workflows. - Supports multiple case branches with logical operators (AND/OR). - Enables workflow routing based on evaluated conditions. --- api/app/core/workflow/executor.py | 85 +++++---- .../core/workflow/nodes/if_else/__init__.py | 5 + api/app/core/workflow/nodes/if_else/config.py | 122 +++++++++++++ api/app/core/workflow/nodes/if_else/node.py | 168 ++++++++++++++++++ 4 files changed, 345 insertions(+), 35 deletions(-) create mode 100644 api/app/core/workflow/nodes/if_else/__init__.py create mode 100644 api/app/core/workflow/nodes/if_else/config.py create mode 100644 api/app/core/workflow/nodes/if_else/node.py diff --git a/api/app/core/workflow/executor.py b/api/app/core/workflow/executor.py index 80d5316a..03cefe78 100644 --- a/api/app/core/workflow/executor.py +++ b/api/app/core/workflow/executor.py @@ -4,16 +4,17 @@ 基于 LangGraph 的工作流执行引擎。 """ -import logging import datetime +import logging from typing import Any from langchain_core.messages import HumanMessage from langgraph.graph import StateGraph, START, END from langgraph.graph.state import CompiledStateGraph -from app.core.workflow.nodes import WorkflowState, NodeFactory from app.core.workflow.expression_evaluator import evaluate_condition +from app.core.workflow.nodes import WorkflowState, NodeFactory +from app.core.workflow.nodes.enums import NodeType logger = logging.getLogger(__name__) @@ -25,11 +26,11 @@ class WorkflowExecutor: """ def __init__( - self, - workflow_config: dict[str, Any], - execution_id: str, - workspace_id: str, - user_id: str + self, + workflow_config: dict[str, Any], + execution_id: str, + workspace_id: str, + user_id: str ): """初始化执行器 @@ -90,8 +91,6 @@ class WorkflowExecutor: "error_node": None } - - def build_graph(self) -> CompiledStateGraph: """构建 LangGraph @@ -112,19 +111,36 @@ class WorkflowExecutor: node_id = node.get("id") # 记录 start 和 end 节点 ID - if node_type == "start": + if node_type == NodeType.START: start_node_id = node_id - elif node_type == "end": + elif node_type == NodeType.END: end_node_ids.append(node_id) # 创建节点实例（现在 start 和 end 也会被创建） node_instance = NodeFactory.create_node(node, self.workflow_config) + + if node_type in [NodeType.IF_ELSE]: + # Build ordered boolean expression strings for each branch. + # These expressions will be attached to outgoing edges as + # LangGraph conditional routing rules. + expressions = node_instance.build_conditional_edge_expressions() + + # Collect all outgoing edges from the current node. + # The order of edges must match the order of generated expressions. + related_edge = [edge for edge in self.edges if edge.get("source") == node_id] + + # Attach each condition expression to the corresponding edge + # based on branch priority + for idx in range(len(expressions)): + related_edge[idx]['condition'] = f"node.{node_id}.output == '{related_edge[idx]['label']}'" + if node_instance: # 包装节点的 run 方法 # 使用函数工厂避免闭包问题 def make_node_func(inst): async def node_func(state: WorkflowState): return await inst.run(state) + return node_func workflow.add_node(node_id, make_node_func(node_instance)) @@ -165,14 +181,14 @@ class WorkflowExecutor: def router(state: WorkflowState, cond=condition, tgt=target): """条件路由函数""" if evaluate_condition( - cond, - state.get("variables", {}), - state.get("node_outputs", {}), - { - "execution_id": state.get("execution_id"), - "workspace_id": state.get("workspace_id"), - "user_id": state.get("user_id") - } + cond, + state.get("variables", {}), + state.get("node_outputs", {}), + { + "execution_id": state.get("execution_id"), + "workspace_id": state.get("workspace_id"), + "user_id": state.get("user_id") + } ): return tgt return END # 条件不满足，结束 @@ -196,8 +212,8 @@ class WorkflowExecutor: return graph async def execute( - self, - input_data: dict[str, Any] + self, + input_data: dict[str, Any] ) -> dict[str, Any]: """执行工作流（非流式） @@ -271,8 +287,8 @@ class WorkflowExecutor: } async def execute_stream( - self, - input_data: dict[str, Any] + self, + input_data: dict[str, Any] ): """执行工作流（流式） @@ -305,7 +321,7 @@ class WorkflowExecutor: try: async for chunk in graph.astream( initial_state, - # subgraphs=True, + # subgraphs=True, stream_mode="updates", ): # print(chunk) @@ -326,7 +342,6 @@ class WorkflowExecutor: "token_usage": None } - def _extract_final_output(self, node_outputs: dict[str, Any]) -> str | None: """从节点输出中提取最终输出 @@ -386,11 +401,11 @@ class WorkflowExecutor: async def execute_workflow( - workflow_config: dict[str, Any], - input_data: dict[str, Any], - execution_id: str, - workspace_id: str, - user_id: str + workflow_config: dict[str, Any], + input_data: dict[str, Any], + execution_id: str, + workspace_id: str, + user_id: str ) -> dict[str, Any]: """执行工作流（便捷函数） @@ -414,11 +429,11 @@ async def execute_workflow( async def execute_workflow_stream( - workflow_config: dict[str, Any], - input_data: dict[str, Any], - execution_id: str, - workspace_id: str, - user_id: str + workflow_config: dict[str, Any], + input_data: dict[str, Any], + execution_id: str, + workspace_id: str, + user_id: str ): """执行工作流（流式，便捷函数） diff --git a/api/app/core/workflow/nodes/if_else/__init__.py b/api/app/core/workflow/nodes/if_else/__init__.py new file mode 100644 index 00000000..ffdf3b5b --- /dev/null +++ b/api/app/core/workflow/nodes/if_else/__init__.py @@ -0,0 +1,5 @@ +"""Condition Node""" +from app.core.workflow.nodes.if_else.config import IfElseNodeConfig +from app.core.workflow.nodes.if_else.node import IfElseNode + +__all__ = ["IfElseNode", "IfElseNodeConfig"] diff --git a/api/app/core/workflow/nodes/if_else/config.py b/api/app/core/workflow/nodes/if_else/config.py new file mode 100644 index 00000000..1a9adbbb --- /dev/null +++ b/api/app/core/workflow/nodes/if_else/config.py @@ -0,0 +1,122 @@ +"""Condition Configuration""" +from pydantic import Field, BaseModel, field_validator +from enum import StrEnum +from app.core.workflow.nodes.base_config import BaseNodeConfig + + +class LogicOperator(StrEnum): + AND = "and" + OR = "or" + + +class ComparisonOpeartor(StrEnum): + EMPTY = "empty" + NOT_EMPTY = "not_empty" + CONTAINS = "contains" + NOT_CONTAINS = "not_contains" + START_WITH = "startwith" + END_WITH = "endwith" + EQ = "eq" + NE = "ne" + LT = "lt" + LE = "le" + GT = "gt" + GE = "ge" + + +class ConditionDetail(BaseModel): + comparison_operator: ComparisonOpeartor = Field( + ..., + description="Comparison operator used to evaluate the condition" + ) + + left: str = Field( + ..., + description="Value to compare against" + ) + + right: str = Field( + ..., + description="Value to compare with" + ) + + +class ConditionBranchConfig(BaseModel): + """Configuration for a conditional branch""" + + logical_operator: LogicOperator = Field( + default=LogicOperator.AND.value, + description="Logical operator used to combine multiple condition expressions" + ) + + conditions: list[ConditionDetail] = Field( + ..., + description="List of condition expressions within this branch" + ) + + +class IfElseNodeConfig(BaseNodeConfig): + cases: list[ConditionBranchConfig] = Field( + ..., + description="List of branch conditions or expressions" + ) + + @field_validator("cases") + @classmethod + def validate_case_number(cls, v, info): + if len(v) < 1: + raise ValueError("At least one cases are required") + return v + + class Config: + json_schema_extra = { + "examples": [ + { + "cases": [ + # if/CASE1 + { + "logical_operator": "and", + "conditions": [ + { + "left": "sys.message", + "comparison_operator": "eq", + "right": "'test'" + } + ] + }, + ] + }, + { + "case_number": 3, + "cases": [ + # if/CASE1 + { + "logic": "or", + "conditions": [ + { + "left": "sys.message", + "comparison_operator": "eq", + "right": "'test'" + } + ] + }, + # elif/CASE2 + { + "logic": "and", + "conditions": [ + { + "left": "sys.message", + "comparison_operator": "eq", + "right": "'test'" + }, + { + "left": "sys.message", + "comparison_operator": "contains", + "right": "'test'" + } + ] + }, + ] + } + ] + } diff --git a/api/app/core/workflow/nodes/if_else/node.py b/api/app/core/workflow/nodes/if_else/node.py new file mode 100644 index 00000000..3219edae --- /dev/null +++ b/api/app/core/workflow/nodes/if_else/node.py @@ -0,0 +1,168 @@ +import logging +from typing import Any + +from simpleeval import NameNotDefined, InvalidExpression + +from app.core.workflow.nodes import BaseNode, WorkflowState +from app.core.workflow.nodes.if_else import IfElseNodeConfig +from app.core.workflow.nodes.if_else.config import LogicOperator, ConditionDetail, ComparisonOpeartor + +logger = logging.getLogger(__name__) + + +class ConditionExpressionBuilder: + """ + Build a Python boolean expression string based on a comparison operator. + + This class does not evaluate the expression. + It only generates a valid Python expression string + that can be evaluated later in a workflow context. + """ + + def __init__(self, left: str, operator: ComparisonOpeartor, right: str): + self.left = left + self.operator = operator + self.right = right + + def _empty(self): + return f"{self.left} == ''" + + def _not_empty(self): + return f"{self.left} != ''" + + def _contains(self): + return f"{self.right} in {self.left}" + + def _not_contains(self): + return f"{self.right} not in {self.left}" + + def _startwith(self): + return f'{self.left}.startswith({self.right})' + + def _endwith(self): + return f'{self.left}.endswith({self.right})' + + def _eq(self): + return f"{self.left} == {self.right}" + + def _ne(self): + return f"{self.left} != {self.right}" + + def _lt(self): + return f"{self.left} < {self.right}" + + def _le(self): + return f"{self.left} <= {self.right}" + + def _gt(self): + return f"{self.left} > {self.right}" + + def _ge(self): + return f"{self.left} >= {self.right}" + + def build(self): + match self.operator: + case ComparisonOpeartor.EMPTY: + return self._empty() + case ComparisonOpeartor.NOT_EMPTY: + return self._not_empty() + case ComparisonOpeartor.CONTAINS: + return self._contains() + case ComparisonOpeartor.NOT_CONTAINS: + return self._not_contains() + case ComparisonOpeartor.START_WITH: + return self._startwith() + case ComparisonOpeartor.END_WITH: + return self._endwith() + case ComparisonOpeartor.EQ: + return self._eq() + case ComparisonOpeartor.NE: + return self._ne() + case ComparisonOpeartor.LT: + return self._lt() + case ComparisonOpeartor.LE: + return self._le() + case ComparisonOpeartor.GT: + return self._gt() + case ComparisonOpeartor.GE: + return self._ge() + case _: + raise ValueError(f"Invalid condition: {self.operator}") + + +class IfElseNode(BaseNode): + def __init__(self, node_config: dict[str, Any], workflow_config: dict[str, Any]): + super().__init__(node_config, workflow_config) + self.typed_config = IfElseNodeConfig(**self.config) + + @staticmethod + def _build_condition_expression( + condition: ConditionDetail, + ) -> str: + """ + Build a single boolean condition expression string. + + This method does NOT evaluate the condition. + It only generates a valid Python boolean expression string + (e.g. "x > 10", "'a' in name") that can later be used + in a conditional edge or evaluated by the workflow engine. + + Args: + condition (ConditionDetail): Definition of a single comparison condition. + + Returns: + str: A Python boolean expression string. + """ + return ConditionExpressionBuilder( + left=condition.left, + operator=condition.comparison_operator, + right=condition.right + ).build() + + def build_conditional_edge_expressions(self) -> list[str]: + """ + Build conditional edge expressions for the If-Else node. + + This method does NOT evaluate any condition at runtime. + Instead, it converts each case branch into a Python boolean + expression string, which will later be attached to LangGraph + as conditional edges. + + Each returned expression corresponds to one branch and is + evaluated in order. A fallback 'True' condition is appended + to ensure a default branch when no previous conditions match. + + Returns: + list[str]: A list of Python boolean expression strings, + ordered by branch priority. + """ + branch_index = 0 + conditions = [] + + for case_branch in self.typed_config.cases: + branch_index += 1 + + branch_conditions = [ + self._build_condition_expression(condition) + for condition in case_branch.conditions + ] + if len(branch_conditions) > 1: + combined_condition = f' {case_branch.logical_operator} '.join(branch_conditions) + else: + combined_condition = branch_conditions[0] + conditions.append(combined_condition) + + # Default fallback branch + conditions.append("True") + + return conditions + + async def execute(self, state: WorkflowState) -> Any: + """ + """ + expressions = self.build_conditional_edge_expressions() + for i in range(len(expressions)): + logger.info(expressions[i]) + if self._evaluate_condition(expressions[i], state): + return f'CASE{i+1}' + return f'CASE{len(expressions)}' From 4d7a89f58ba27ce6d8566afff9583b04c8bc14b0 Mon Sep 17 00:00:00 2001 From: mengyonghao <1533512157@qq.com> Date: Fri, 19 Dec 2025 14:21:27 +0800 Subject: [PATCH 07/20] perf(types): add Union type declaration for workflow nodes - Introduce a `Nodes` type as a Union of all workflow node classes. - Improves type checking and IDE autocompletion. --- api/app/core/workflow/nodes/enums.py | 21 +++++++++++++++++++++ api/app/core/workflow/nodes/node_factory.py | 10 ++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/api/app/core/workflow/nodes/enums.py b/api/app/core/workflow/nodes/enums.py index 9cec19d2..5e586a9c 100644 --- a/api/app/core/workflow/nodes/enums.py +++ b/api/app/core/workflow/nodes/enums.py @@ -1,4 +1,14 @@ from enum import StrEnum +from typing import Union + +from app.core.workflow.nodes.base_node import BaseNode +from app.core.workflow.nodes.if_else import IfElseNode +from app.core.workflow.nodes.llm import LLMNode +from app.core.workflow.nodes.agent import AgentNode +from app.core.workflow.nodes.transform import TransformNode +from app.core.workflow.nodes.start import StartNode +from app.core.workflow.nodes.end import EndNode + class NodeType(StrEnum): START = "start" @@ -13,3 +23,14 @@ class NodeType(StrEnum): HTTP_REQUEST = "http-request" TOOL = "tool" AGENT = "agent" + + +WorkflowNode = Union[ + BaseNode, + StartNode, + EndNode, + LLMNode, + IfElseNode, + AgentNode, + TransformNode, +] diff --git a/api/app/core/workflow/nodes/node_factory.py b/api/app/core/workflow/nodes/node_factory.py index f279d13a..e1f32308 100644 --- a/api/app/core/workflow/nodes/node_factory.py +++ b/api/app/core/workflow/nodes/node_factory.py @@ -8,7 +8,8 @@ import logging from typing import Any from app.core.workflow.nodes.base_node import BaseNode -from app.core.workflow.nodes.enums import NodeType +from app.core.workflow.nodes.enums import NodeType, WorkflowNode +from app.core.workflow.nodes.if_else import IfElseNode from app.core.workflow.nodes.llm import LLMNode from app.core.workflow.nodes.agent import AgentNode from app.core.workflow.nodes.transform import TransformNode @@ -25,16 +26,17 @@ class NodeFactory: """ # 节点类型注册表 - _node_types: dict[str, type[BaseNode]] = { + _node_types: dict[str, type[WorkflowNode]] = { NodeType.START: StartNode, NodeType.END: EndNode, NodeType.LLM: LLMNode, NodeType.AGENT: AgentNode, NodeType.TRANSFORM: TransformNode, + NodeType.IF_ELSE: IfElseNode } @classmethod - def register_node_type(cls, node_type: str, node_class: type[BaseNode]): + def register_node_type(cls, node_type: str, node_class: type[WorkflowNode]): """注册新的节点类型 Args: @@ -55,7 +57,7 @@ class NodeFactory: cls, node_config: dict[str, Any], workflow_config: dict[str, Any] - ) -> BaseNode | None: + ) -> WorkflowNode | None: """创建节点实例 Args: From bf702b1b92009e162bda1516b1f6fd2172cf5c27 Mon Sep 17 00:00:00 2001 From: mengyonghao <1533512157@qq.com> Date: Fri, 19 Dec 2025 14:23:29 +0800 Subject: [PATCH 08/20] fix(expression-eval): fix variable extraction issue in Jinja2 templates - Resolve the bug where variables inside Jinja2 template expressions were not correctly extracted. - Ensure expressions containing {{ ... }} are parsed reliably. --- api/app/core/workflow/expression_evaluator.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/api/app/core/workflow/expression_evaluator.py b/api/app/core/workflow/expression_evaluator.py index c8875d79..81ab25dc 100644 --- a/api/app/core/workflow/expression_evaluator.py +++ b/api/app/core/workflow/expression_evaluator.py @@ -5,6 +5,7 @@ """ import logging +import re from typing import Any from simpleeval import simple_eval, NameNotDefined, InvalidExpression @@ -59,9 +60,10 @@ class ExpressionEvaluator: """ # 移除 Jinja2 模板语法的花括号（如果存在） expression = expression.strip() - if expression.startswith("{{") and expression.endswith("}}"): - expression = expression[2:-2].strip() - + # "{{system.message}} == {{ user.messge }}" -> "system.message == user.message" + pattern = r"\{\{\s*(.*?)\s*\}\}" + expression = re.sub(pattern, r"\1", expression).strip() + # 构建命名空间上下文 context = { "var": variables, # 用户变量 From 73ab2c7986d0f26cbb36ce3e131318bd0f56b243 Mon Sep 17 00:00:00 2001 From: mengyonghao <1533512157@qq.com> Date: Fri, 19 Dec 2025 14:34:01 +0800 Subject: [PATCH 09/20] docs(samples): add config example for If-Else node - Provide a sample configuration for the If-Else workflow node. - Helps users understand how to define conditional branches. --- api/app/core/workflow/nodes/if_else/config.py | 56 +++++++++---------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/api/app/core/workflow/nodes/if_else/config.py b/api/app/core/workflow/nodes/if_else/config.py index 1a9adbbb..1eaddc63 100644 --- a/api/app/core/workflow/nodes/if_else/config.py +++ b/api/app/core/workflow/nodes/if_else/config.py @@ -73,49 +73,43 @@ class IfElseNodeConfig(BaseNodeConfig): "examples": [ { "cases": [ - # if/CASE1 + # CASE1 / IF Branch { "logical_operator": "and", "conditions": [ { - "left": "sys.message", - "comparison_operator": "eq", - "right": "'test'" + { + "left": "node.userinput.message", + "comparison_operator": "eq", + "right": "'123'" + }, + { + "left": "node.userinput.test", + "comparison_operator": "eq", + "right": "True" + } } ] }, - ] - }, - { - "case_number": 3, - "cases": [ - # if/CASE1 + # CASE1 / ELIF Branch { - "logic": "or", + "logical_operator": "or", "conditions": [ { - "left": "sys.message", - "comparison_operator": "eq", - "right": "'test'" + { + "left": "node.userinput.test", + "comparison_operator": "eq", + "right": "False" + }, + { + "left": "node.userinput.message", + "comparison_operator": "contains", + "right": "'123'" + } } ] - }, - # elif/CASE2 - { - "logic": "and", - "conditions": [ - { - "left": "sys.message", - "comparison_operator": "eq", - "right": "'test'" - }, - { - "left": "sys.message", - "comparison_operator": "contains", - "right": "'test'" - } - ] - }, + } + # CASE3 / ELSE Branch ] } ] From d3d3c3b3ce589ce43f149c2d1de5763beb588222 Mon Sep 17 00:00:00 2001 From: mengyonghao <1533512157@qq.com> Date: Fri, 19 Dec 2025 14:43:47 +0800 Subject: [PATCH 10/20] style(workflow): update condition edge comments for conditional nodes --- api/app/core/workflow/executor.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/api/app/core/workflow/executor.py b/api/app/core/workflow/executor.py index 03cefe78..3c4b8840 100644 --- a/api/app/core/workflow/executor.py +++ b/api/app/core/workflow/executor.py @@ -120,18 +120,20 @@ class WorkflowExecutor: node_instance = NodeFactory.create_node(node, self.workflow_config) if node_type in [NodeType.IF_ELSE]: - # Build ordered boolean expression strings for each branch. - # These expressions will be attached to outgoing edges as - # LangGraph conditional routing rules. expressions = node_instance.build_conditional_edge_expressions() - # Collect all outgoing edges from the current node. - # The order of edges must match the order of generated expressions. + # Number of branches, usually matches the number of conditional expressions + branch_number = len(expressions) + + # Find all edges whose source is the current node related_edge = [edge for edge in self.edges if edge.get("source") == node_id] - # Attach each condition expression to the corresponding edge - # based on branch priority - for idx in range(len(expressions)): + # Iterate over each branch + for idx in range(branch_number): + # Generate a condition expression for each edge + # Used later to determine which branch to take based on the node's output + # Assumes node output `node..output` matches the edge's label + # For example, if node.123.output == 'CASE1', take the branch labeled 'CASE1' related_edge[idx]['condition'] = f"node.{node_id}.output == '{related_edge[idx]['label']}'" if node_instance: From 0ccb4a095ab2feeed57de0c66e79674b7759231e Mon Sep 17 00:00:00 2001 From: mengyonghao <1533512157@qq.com> Date: Fri, 19 Dec 2025 15:16:00 +0800 Subject: [PATCH 11/20] style(enums): correct enum class name spelling --- api/app/core/workflow/nodes/if_else/config.py | 4 +-- api/app/core/workflow/nodes/if_else/node.py | 28 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/api/app/core/workflow/nodes/if_else/config.py b/api/app/core/workflow/nodes/if_else/config.py index 1eaddc63..0e759569 100644 --- a/api/app/core/workflow/nodes/if_else/config.py +++ b/api/app/core/workflow/nodes/if_else/config.py @@ -9,7 +9,7 @@ class LogicOperator(StrEnum): OR = "or" -class ComparisonOpeartor(StrEnum): +class ComparisonOperator(StrEnum): EMPTY = "empty" NOT_EMPTY = "not_empty" CONTAINS = "contains" @@ -25,7 +25,7 @@ class ComparisonOpeartor(StrEnum): class ConditionDetail(BaseModel): - comparison_operator: ComparisonOpeartor = Field( + comparison_operator: ComparisonOperator = Field( ..., description="Comparison operator used to evaluate the condition" ) diff --git a/api/app/core/workflow/nodes/if_else/node.py b/api/app/core/workflow/nodes/if_else/node.py index 3219edae..fcfbd9ac 100644 --- a/api/app/core/workflow/nodes/if_else/node.py +++ b/api/app/core/workflow/nodes/if_else/node.py @@ -5,7 +5,7 @@ from simpleeval import NameNotDefined, InvalidExpression from app.core.workflow.nodes import BaseNode, WorkflowState from app.core.workflow.nodes.if_else import IfElseNodeConfig -from app.core.workflow.nodes.if_else.config import LogicOperator, ConditionDetail, ComparisonOpeartor +from app.core.workflow.nodes.if_else.config import LogicOperator, ConditionDetail, ComparisonOperator logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ class ConditionExpressionBuilder: that can be evaluated later in a workflow context. """ - def __init__(self, left: str, operator: ComparisonOpeartor, right: str): + def __init__(self, left: str, operator: ComparisonOperator, right: str): self.left = left self.operator = operator self.right = right @@ -62,29 +62,29 @@ class ConditionExpressionBuilder: def build(self): match self.operator: - case ComparisonOpeartor.EMPTY: + case ComparisonOperator.EMPTY: return self._empty() - case ComparisonOpeartor.NOT_EMPTY: + case ComparisonOperator.NOT_EMPTY: return self._not_empty() - case ComparisonOpeartor.CONTAINS: + case ComparisonOperator.CONTAINS: return self._contains() - case ComparisonOpeartor.NOT_CONTAINS: + case ComparisonOperator.NOT_CONTAINS: return self._not_contains() - case ComparisonOpeartor.START_WITH: + case ComparisonOperator.START_WITH: return self._startwith() - case ComparisonOpeartor.END_WITH: + case ComparisonOperator.END_WITH: return self._endwith() - case ComparisonOpeartor.EQ: + case ComparisonOperator.EQ: return self._eq() - case ComparisonOpeartor.NE: + case ComparisonOperator.NE: return self._ne() - case ComparisonOpeartor.LT: + case ComparisonOperator.LT: return self._lt() - case ComparisonOpeartor.LE: + case ComparisonOperator.LE: return self._le() - case ComparisonOpeartor.GT: + case ComparisonOperator.GT: return self._gt() - case ComparisonOpeartor.GE: + case ComparisonOperator.GE: return self._ge() case _: raise ValueError(f"Invalid condition: {self.operator}") From 39de537c14d20c659d5356e7ddc455d733992ec9 Mon Sep 17 00:00:00 2001 From: mengyonghao <1533512157@qq.com> Date: Fri, 19 Dec 2025 15:43:56 +0800 Subject: [PATCH 12/20] refactor(workflow): unify all enum classes in one file and restructure workflow node type definitions --- api/app/core/workflow/nodes/__init__.py | 13 ++++--- api/app/core/workflow/nodes/enums.py | 36 +++++++++---------- api/app/core/workflow/nodes/if_else/config.py | 31 ++++------------ api/app/core/workflow/nodes/if_else/node.py | 5 ++- api/app/core/workflow/nodes/node_factory.py | 26 +++++++++----- 5 files changed, 52 insertions(+), 59 deletions(-) diff --git a/api/app/core/workflow/nodes/__init__.py b/api/app/core/workflow/nodes/__init__.py index 820c9301..d143c693 100644 --- a/api/app/core/workflow/nodes/__init__.py +++ b/api/app/core/workflow/nodes/__init__.py @@ -4,13 +4,14 @@ 提供各种类型的节点实现，用于工作流执行。 """ -from app.core.workflow.nodes.base_node import BaseNode, WorkflowState -from app.core.workflow.nodes.llm import LLMNode from app.core.workflow.nodes.agent import AgentNode -from app.core.workflow.nodes.transform import TransformNode -from app.core.workflow.nodes.start import StartNode +from app.core.workflow.nodes.base_node import BaseNode, WorkflowState from app.core.workflow.nodes.end import EndNode -from app.core.workflow.nodes.node_factory import NodeFactory +from app.core.workflow.nodes.if_else import IfElseNode +from app.core.workflow.nodes.llm import LLMNode +from app.core.workflow.nodes.node_factory import NodeFactory, WorkflowNode +from app.core.workflow.nodes.start import StartNode +from app.core.workflow.nodes.transform import TransformNode __all__ = [ "BaseNode", @@ -18,7 +19,9 @@ __all__ = [ "LLMNode", "AgentNode", "TransformNode", + "IfElseNode", "StartNode", "EndNode", "NodeFactory", + "WorkflowNode" ] diff --git a/api/app/core/workflow/nodes/enums.py b/api/app/core/workflow/nodes/enums.py index 5e586a9c..af5ddbaa 100644 --- a/api/app/core/workflow/nodes/enums.py +++ b/api/app/core/workflow/nodes/enums.py @@ -1,13 +1,4 @@ from enum import StrEnum -from typing import Union - -from app.core.workflow.nodes.base_node import BaseNode -from app.core.workflow.nodes.if_else import IfElseNode -from app.core.workflow.nodes.llm import LLMNode -from app.core.workflow.nodes.agent import AgentNode -from app.core.workflow.nodes.transform import TransformNode -from app.core.workflow.nodes.start import StartNode -from app.core.workflow.nodes.end import EndNode class NodeType(StrEnum): @@ -25,12 +16,21 @@ class NodeType(StrEnum): AGENT = "agent" -WorkflowNode = Union[ - BaseNode, - StartNode, - EndNode, - LLMNode, - IfElseNode, - AgentNode, - TransformNode, -] +class ComparisonOperator(StrEnum): + EMPTY = "empty" + NOT_EMPTY = "not_empty" + CONTAINS = "contains" + NOT_CONTAINS = "not_contains" + START_WITH = "startwith" + END_WITH = "endwith" + EQ = "eq" + NE = "ne" + LT = "lt" + LE = "le" + GT = "gt" + GE = "ge" + + +class LogicOperator(StrEnum): + AND = "and" + OR = "or" diff --git a/api/app/core/workflow/nodes/if_else/config.py b/api/app/core/workflow/nodes/if_else/config.py index 0e759569..4e424b54 100644 --- a/api/app/core/workflow/nodes/if_else/config.py +++ b/api/app/core/workflow/nodes/if_else/config.py @@ -1,27 +1,8 @@ """Condition Configuration""" from pydantic import Field, BaseModel, field_validator -from enum import StrEnum + from app.core.workflow.nodes.base_config import BaseNodeConfig - - -class LogicOperator(StrEnum): - AND = "and" - OR = "or" - - -class ComparisonOperator(StrEnum): - EMPTY = "empty" - NOT_EMPTY = "not_empty" - CONTAINS = "contains" - NOT_CONTAINS = "not_contains" - START_WITH = "startwith" - END_WITH = "endwith" - EQ = "eq" - NE = "ne" - LT = "lt" - LE = "le" - GT = "gt" - GE = "ge" +from app.core.workflow.nodes.enums import ComparisonOperator, LogicOperator class ConditionDetail(BaseModel): @@ -77,7 +58,7 @@ class IfElseNodeConfig(BaseNodeConfig): { "logical_operator": "and", "conditions": [ - { + [ { "left": "node.userinput.message", "comparison_operator": "eq", @@ -88,14 +69,14 @@ class IfElseNodeConfig(BaseNodeConfig): "comparison_operator": "eq", "right": "True" } - } + ] ] }, # CASE1 / ELIF Branch { "logical_operator": "or", "conditions": [ - { + [ { "left": "node.userinput.test", "comparison_operator": "eq", @@ -106,7 +87,7 @@ class IfElseNodeConfig(BaseNodeConfig): "comparison_operator": "contains", "right": "'123'" } - } + ] ] } # CASE3 / ELSE Branch diff --git a/api/app/core/workflow/nodes/if_else/node.py b/api/app/core/workflow/nodes/if_else/node.py index fcfbd9ac..ed3dbbd6 100644 --- a/api/app/core/workflow/nodes/if_else/node.py +++ b/api/app/core/workflow/nodes/if_else/node.py @@ -1,11 +1,10 @@ import logging from typing import Any -from simpleeval import NameNotDefined, InvalidExpression - from app.core.workflow.nodes import BaseNode, WorkflowState +from app.core.workflow.nodes.enums import ComparisonOperator from app.core.workflow.nodes.if_else import IfElseNodeConfig -from app.core.workflow.nodes.if_else.config import LogicOperator, ConditionDetail, ComparisonOperator +from app.core.workflow.nodes.if_else.config import ConditionDetail logger = logging.getLogger(__name__) diff --git a/api/app/core/workflow/nodes/node_factory.py b/api/app/core/workflow/nodes/node_factory.py index e1f32308..1abace67 100644 --- a/api/app/core/workflow/nodes/node_factory.py +++ b/api/app/core/workflow/nodes/node_factory.py @@ -5,19 +5,29 @@ """ import logging -from typing import Any +from typing import Any, Union +from app.core.workflow.nodes.agent import AgentNode from app.core.workflow.nodes.base_node import BaseNode -from app.core.workflow.nodes.enums import NodeType, WorkflowNode +from app.core.workflow.nodes.end import EndNode +from app.core.workflow.nodes.enums import NodeType from app.core.workflow.nodes.if_else import IfElseNode from app.core.workflow.nodes.llm import LLMNode -from app.core.workflow.nodes.agent import AgentNode -from app.core.workflow.nodes.transform import TransformNode from app.core.workflow.nodes.start import StartNode -from app.core.workflow.nodes.end import EndNode +from app.core.workflow.nodes.transform import TransformNode logger = logging.getLogger(__name__) +WorkflowNode = Union[ + BaseNode, + StartNode, + EndNode, + LLMNode, + IfElseNode, + AgentNode, + TransformNode, +] + class NodeFactory: """节点工厂 @@ -54,9 +64,9 @@ class NodeFactory: @classmethod def create_node( - cls, - node_config: dict[str, Any], - workflow_config: dict[str, Any] + cls, + node_config: dict[str, Any], + workflow_config: dict[str, Any] ) -> WorkflowNode | None: """创建节点实例 From beb0f0f6df9ff973a8bf5bc2dd641b6d73dcc848 Mon Sep 17 00:00:00 2001 From: mengyonghao <1533512157@qq.com> Date: Fri, 19 Dec 2025 15:59:28 +0800 Subject: [PATCH 13/20] feat(workflow): add import for if-else node configuration --- api/app/core/workflow/nodes/configs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/app/core/workflow/nodes/configs.py b/api/app/core/workflow/nodes/configs.py index 99d06036..15ab0ce9 100644 --- a/api/app/core/workflow/nodes/configs.py +++ b/api/app/core/workflow/nodes/configs.py @@ -13,6 +13,7 @@ from app.core.workflow.nodes.end.config import EndNodeConfig from app.core.workflow.nodes.llm.config import LLMNodeConfig, MessageConfig from app.core.workflow.nodes.agent.config import AgentNodeConfig from app.core.workflow.nodes.transform.config import TransformNodeConfig +from app.core.workflow.nodes.if_else.config import IfElseNodeConfig __all__ = [ # 基础类 @@ -26,4 +27,5 @@ __all__ = [ "MessageConfig", "AgentNodeConfig", "TransformNodeConfig", + "IfElseNodeConfig", ] From 5c0d8b42f3854fb50a2d83f5bc25b73651e94aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=B0=E6=9C=88?= Date: Fri, 19 Dec 2025 08:04:12 +0000 Subject: [PATCH 14/20] Merge #9 into develop from fix/memory_reflection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增反思功能（功能配置接口+反思celery后台检测反思的迭代周期） * fix/memory_reflection: (24 commits squashed) - 新增反思功能（功能配置接口+反思celery后台检测反思的迭代周期） - 新增反思功能（功能配置接口+反思celery后台检测反思的迭代周期） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 Signed-off-by: aliyun8644380055 Commented-by: aliyun8644380055 Commented-by: aliyun6762716068 Reviewed-by: aliyun6762716068 Merged-by: aliyun6762716068 CR-link: https://codeup.aliyun.com/redbearai/python/redbear-mem-open/change/9 --- api/app/celery_app.py | 19 +- api/app/controllers/__init__.py | 5 +- .../memory_reflection_controller.py | 200 +++++++++ api/app/core/config.py | 1 + .../reflection_engine/example/example.json | 210 +++++++++ .../reflection_engine/self_reflexion.py | 322 ++++++++------ api/app/core/memory/utils/config/get_data.py | 62 +-- .../utils/prompt/prompts/evaluate.jinja2 | 221 +++++++++- .../utils/prompt/prompts/reflexion.jinja2 | 307 +++++++++++++- .../memory/utils/prompt/template_render.py | 28 +- api/app/models/data_config_model.py | 26 +- api/app/models/end_user_model.py | 1 + .../repositories/data_config_repository.py | 252 +++++++---- api/app/repositories/neo4j/cypher_queries.py | 54 +++ api/app/repositories/neo4j/neo4j_update.py | 227 ++++++++++ api/app/schemas/end_user_schema.py | 1 + api/app/schemas/memory_reflection_schemas.py | 54 +++ api/app/schemas/memory_storage_schema.py | 63 ++- api/app/services/memory_reflection_service.py | 397 ++++++++++++++++++ api/app/tasks.py | 163 ++++++- api/check_code.py | 108 +++++ 21 files changed, 2384 insertions(+), 337 deletions(-) create mode 100644 api/app/controllers/memory_reflection_controller.py create mode 100644 api/app/core/memory/storage_services/reflection_engine/example/example.json create mode 100644 api/app/repositories/neo4j/neo4j_update.py create mode 100644 api/app/schemas/memory_reflection_schemas.py create mode 100644 api/app/services/memory_reflection_service.py create mode 100755 api/check_code.py diff --git a/api/app/celery_app.py b/api/app/celery_app.py index d072a346..ce7e9300 100644 --- a/api/app/celery_app.py +++ b/api/app/celery_app.py @@ -83,17 +83,18 @@ celery_app.autodiscover_tasks(['app']) reflection_schedule = timedelta(seconds=settings.REFLECTION_INTERVAL_SECONDS) health_schedule = timedelta(seconds=settings.HEALTH_CHECK_SECONDS) memory_increment_schedule = timedelta(hours=settings.MEMORY_INCREMENT_INTERVAL_HOURS) - +workspace_reflection_schedule = timedelta(seconds=30) # 每30秒运行一次settings.REFLECTION_INTERVAL_TIME # 构建定时任务配置 beat_schedule_config = { - "run-reflection-engine": { - "task": "app.core.memory.agent.reflection.timer", - "schedule": reflection_schedule, - "args": (), - }, - "check-read-service": { - "task": "app.core.memory.agent.health.check_read_service", - "schedule": health_schedule, + + # "check-read-service": { + # "task": "app.core.memory.agent.health.check_read_service", + # "schedule": health_schedule, + # "args": (), + # }, + "run-workspace-reflection": { + "task": "app.tasks.workspace_reflection_task", + "schedule": workspace_reflection_schedule, "args": (), }, } diff --git a/api/app/controllers/__init__.py b/api/app/controllers/__init__.py index a3caaf4a..ddf534c6 100644 --- a/api/app/controllers/__init__.py +++ b/api/app/controllers/__init__.py @@ -23,12 +23,13 @@ from . import ( memory_dashboard_controller, memory_storage_controller, memory_dashboard_controller, + memory_reflection_controller, api_key_controller, release_share_controller, public_share_controller, multi_agent_controller, workflow_controller, - prompt_optimizer_controller + prompt_optimizer_controller, ) # 创建管理端 API 路由器 @@ -60,5 +61,5 @@ manager_router.include_router(memory_dashboard_controller.router) manager_router.include_router(multi_agent_controller.router) manager_router.include_router(workflow_controller.router) manager_router.include_router(prompt_optimizer_controller.router) - +manager_router.include_router(memory_reflection_controller.router) __all__ = ["manager_router"] diff --git a/api/app/controllers/memory_reflection_controller.py b/api/app/controllers/memory_reflection_controller.py new file mode 100644 index 00000000..759c25c5 --- /dev/null +++ b/api/app/controllers/memory_reflection_controller.py @@ -0,0 +1,200 @@ +import asyncio + +from dotenv import load_dotenv +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy.orm import Session +from sqlalchemy import text + +from app.core.logging_config import get_api_logger +from app.core.memory.storage_services.reflection_engine.self_reflexion import ReflectionConfig, ReflectionEngine +from app.dependencies import get_current_user +from app.db import get_db +from app.models.user_model import User +from app.repositories.data_config_repository import DataConfigRepository +from app.repositories.neo4j.neo4j_connector import Neo4jConnector + +from app.services.memory_reflection_service import WorkspaceAppService, MemoryReflectionService + +from app.schemas.memory_reflection_schemas import Memory_Reflection + +load_dotenv() +api_logger = get_api_logger() + +router = APIRouter( + prefix="/memory", + tags=["Memory"], +) + + +@router.post("/reflection/save") +async def save_reflection_config( + request: Memory_Reflection, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +) -> dict: + """Save reflection configuration to data_comfig table""" + + + + try: + config_id = request.config_id + if not config_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="缺少必需参数: config_id" + ) + + api_logger.info(f"用户 {current_user.username} 保存反思配置，config_id: {config_id}") + + update_params = { + "enable_self_reflexion": request.reflectionenabled, + "iteration_period": request.reflection_period_in_hours, + "reflexion_range": request.reflexion_range, + "baseline": request.baseline, + "reflection_model_id": request.reflection_model_id, + "memory_verify": request.memory_verify, + "quality_assessment": request.quality_assessment, + } + + + + query, params = DataConfigRepository.build_update_reflection(config_id, **update_params) + + result = db.execute(text(query), params) + if result.rowcount == 0: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"未找到config_id为 {config_id} 的配置" + ) + + db.commit() + + # 查询更新后的配置 + select_query, select_params = DataConfigRepository.build_select_reflection(config_id) + result = db.execute(text(select_query), select_params).fetchone() + + if not result: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"更新后未找到config_id为 {config_id} 的配置" + ) + + api_logger.info(f"成功保存反思配置到数据库，config_id: {config_id}") + + # 返回结果 + return { + "status": "成功", + "message": "反思配置已保存", + "config_id": config_id, + "database_record": { + "config_id": result.config_id, + "enable_self_reflexion": result.enable_self_reflexion, + "iteration_period": result.iteration_period, + "reflexion_range": result.reflexion_range, + "baseline": result.baseline, + "reflection_model_id": result.reflection_model_id, + "memory_verify": result.memory_verify, + "quality_assessment": result.quality_assessment, + "user_id": result.user_id + } + } + + except ValueError as ve: + api_logger.error(f"参数错误: {str(ve)}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"参数错误: {str(ve)}" + ) + except Exception as e: + api_logger.error(f"反思配置保存失败: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"反思配置保存失败: {str(e)}" + ) + + +@router.post("/reflection") +async def start_workspace_reflection( + request: dict, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +) -> dict: + """Activate the reflection function for all matching applications in the workspace""" + workspace_id = current_user.current_workspace_id + reflection_service = MemoryReflectionService(db) + + try: + api_logger.info(f"用户 {current_user.username} 启动workspace反思，workspace_id: {workspace_id}") + + service = WorkspaceAppService(db) + result = service.get_workspace_apps_detailed(workspace_id) + + reflection_results = [] + + for data in result['apps_detailed_info']: + if data['data_configs'] == []: + continue + + releases = data['releases'] + data_configs = data['data_configs'] + end_users = data['end_users'] + + for base, config, user in zip(releases, data_configs, end_users): + if int(base['config']) == int(config['config_id']) and base['app_id'] == user['app_id']: + # 调用反思服务 + api_logger.info(f"为用户 {user['id']} 启动反思，config_id: {config['config_id']}") + + reflection_result = await reflection_service.start_reflection_from_data( + config_data=config, + end_user_id=user['id'] + ) + + reflection_results.append({ + "app_id": base['app_id'], + "config_id": config['config_id'], + "end_user_id": user['id'], + "reflection_result": reflection_result + }) + + return { + "status": "完成", + "message": f"成功处理 {len(reflection_results)} 个反思任务", + "workspace_id": str(workspace_id), + "reflection_count": len(reflection_results), + "reflection_results": reflection_results + } + + except Exception as e: + api_logger.error(f"启动workspace反思失败: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"启动workspace反思失败: {str(e)}" + ) + +@router.post("/reflection/run") +async def reflection_run( + reflection: Memory_Reflection, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +) -> dict: + """Activate the reflection function for all matching applications in the workspace""" + config = ReflectionConfig( + enabled=reflection.reflectionenabled, + iteration_period=reflection.reflection_period_in_hours, + reflexion_range=reflection.reflexion_range, + baseline=reflection.baseline, + output_example='', + memory_verify=reflection.memory_verify, + quality_assessment=reflection.quality_assessment, + violation_handling_strategy="block", + model_id=reflection.reflection_model_id + ) + connector = Neo4jConnector() + engine = ReflectionEngine( + config=config, + neo4j_connector=connector, + llm_client=reflection.reflection_model_id # 传入 model_id + ) + + result=await (engine.reflection_run()) + return result diff --git a/api/app/core/config.py b/api/app/core/config.py index 48f79d5e..41e9f0cf 100644 --- a/api/app/core/config.py +++ b/api/app/core/config.py @@ -148,6 +148,7 @@ class Settings: HEALTH_CHECK_SECONDS: float = float(os.getenv("HEALTH_CHECK_SECONDS", "600")) MEMORY_INCREMENT_INTERVAL_HOURS: float = float(os.getenv("MEMORY_INCREMENT_INTERVAL_HOURS", "24")) DEFAULT_WORKSPACE_ID: Optional[str] = os.getenv("DEFAULT_WORKSPACE_ID", None) + REFLECTION_INTERVAL_TIME:Optional[str] = int(os.getenv("REFLECTION_INTERVAL_TIME", 30)) # Memory Module Configuration (internal) MEMORY_OUTPUT_DIR: str = os.getenv("MEMORY_OUTPUT_DIR", "logs/memory-output") diff --git a/api/app/core/memory/storage_services/reflection_engine/example/example.json b/api/app/core/memory/storage_services/reflection_engine/example/example.json new file mode 100644 index 00000000..6528da60 --- /dev/null +++ b/api/app/core/memory/storage_services/reflection_engine/example/example.json @@ -0,0 +1,210 @@ +{ + "memory_verify": { + "source_data": [ + { + "statement_name": "用户是2023年春天去北京工作的。", + "statement_id": "62beac695b1346f4871740a45db88782", + "statement_created_at": "2025-12-19T10:31:15.239252" + }, + { + "statement_name": "用户后来基本一直都在北京上班。", + "statement_id": "4cba5ac08b674d7fb1e2ae634d2b8f0b", + "statement_created_at": "2025-12-19T10:31:15.239252" + }, + { + "statement_name": "用户从2023年开始就一直在北京生活。", + "statement_id": "e612a44da4db483993c350df7c97a1a1", + "statement_created_at": "2025-12-19T10:31:15.239252" + }, + { + "statement_name": "用户从来没有长期离开过北京。", + "statement_id": "b3c787a2e33c49f7981accabbbb4538a", + "statement_created_at": "2025-12-19T10:31:15.239252" + }, + { + "statement_name": "由于公司调整，用户在2024年上半年被调到上海待了差不多半年。", + "statement_id": "64cde4230cb24a4da726e7db9e7aa616", + "statement_created_at": "2025-12-19T10:31:15.239252" + }, + { + "statement_name": "用户在被调到上海期间每天都是在上海办公室打卡。", + "statement_id": "8b1b12e23b844b8088dfeb67da6ad669", + "statement_created_at": "2025-12-19T10:31:15.239252" + }, + { + "statement_name": "用户在入职时使用的身份信息是之前的，身份证号为11010119950308123X。", + "statement_id": "030afd362e9b4110b139e68e5d3e7143", + "statement_created_at": "2025-12-19T10:31:15.239252" + }, + { + "statement_name": "用户的银行卡号是6222023847595898。", + "statement_id": "6c7567cd1f3c478bb42d1b65383e6f2f", + "statement_created_at": "2025-12-19T10:31:15.239252" + }, + { + "statement_name": "用户的身份信息和银行卡信息一直没变。", + "statement_id": "b3ca618e1e204b83bebd70e75cf2073f", + "statement_created_at": "2025-12-19T10:31:15.239252" + }, + { + "statement_name": "用户认为在上海的那段时间更多算是远程配合。", + "statement_id": "150af89d2c154e6eb41ff1a91e37f962", + "statement_created_at": "2025-12-19T10:31:15.239252" + } + ], + "databasets": [ + { + "entity1_name": "Person", + "description": "表示人类个体的通用类型", + "statement_id": "62beac695b1346f4871740a45db88782", + "created_at": "2025-12-19T10:31:15.239252000", + "expired_at": "9999-12-31T00:00:00.000000000", + "relationship_type": "EXTRACTED_RELATIONSHIP", + "relationship": {}, + "entity2_name": "用户", + "entity2": { + "entity_idx": 0, + "run_id": "62b59cfebeea43dd94d91763056f069a", + "connect_strength": "strong", + "created_at": "2025-12-19T10:31:15.239252000", + "description": "叙述者，讲述个人工作与生活经历的个体", + "statement_id": "62beac695b1346f4871740a45db88782", + "expired_at": "9999-12-31T00:00:00.000000000", + "entity_type": "Person", + "group_id": "88a459f5_text08", + "user_id": "88a459f5_text08", + "name": "用户", + "apply_id": "88a459f5_text08", + "id": "3d3896797b334572a80d57590026063d" + } + }, + { + "entity1_name": "用户", + "description": "叙述者，讲述个人工作与生活经历的个体", + "statement_id": "62beac695b1346f4871740a45db88782", + "created_at": "2025-12-19T10:31:15.239252000", + "expired_at": "9999-12-31T00:00:00.000000000", + "relationship_type": "EXTRACTED_RELATIONSHIP", + "relationship": {}, + "entity2_name": "身份信息", + "entity2": { + "entity_idx": 1, + "run_id": "62b59cfebeea43dd94d91763056f069a", + "connect_strength": "Strong", + "description": "用于个人身份识别的数据", + "created_at": "2025-12-19T10:31:15.239252000", + "statement_id": "030afd362e9b4110b139e68e5d3e7143", + "expired_at": "9999-12-31T00:00:00.000000000", + "entity_type": "Information", + "group_id": "88a459f5_text08", + "user_id": "88a459f5_text08", + "name": "身份信息", + "apply_id": "88a459f5_text08", + "id": "aa766a517e82490599a9b3af54cfd933" + } + }, + { + "entity1_name": "用户", + "description": "叙述者，讲述个人工作与生活经历的个体", + "statement_id": "62beac695b1346f4871740a45db88782", + "created_at": "2025-12-19T10:31:15.239252000", + "expired_at": "9999-12-31T00:00:00.000000000", + "relationship_type": "EXTRACTED_RELATIONSHIP", + "relationship": {}, + "entity2_name": "6222023847595898", + "entity2": { + "entity_idx": 1, + "run_id": "62b59cfebeea43dd94d91763056f069a", + "connect_strength": "Strong", + "description": "用户的银行卡号码", + "created_at": "2025-12-19T10:31:15.239252000", + "statement_id": "6c7567cd1f3c478bb42d1b65383e6f2f", + "expired_at": "9999-12-31T00:00:00.000000000", + "entity_type": "Numeric", + "group_id": "88a459f5_text08", + "user_id": "88a459f5_text08", + "name": "6222023847595898", + "apply_id": "88a459f5_text08", + "id": "610ba361918f4e68a65ce6ad06e5c7a0" + } + }, + { + "entity1_name": "用户", + "description": "叙述者，讲述个人工作与生活经历的个体", + "statement_id": "62beac695b1346f4871740a45db88782", + "created_at": "2025-12-19T10:31:15.239252000", + "expired_at": "9999-12-31T00:00:00.000000000", + "relationship_type": "EXTRACTED_RELATIONSHIP", + "relationship": {}, + "entity2_name": "上海办公室", + "entity2": { + "entity_idx": 1, + "run_id": "62b59cfebeea43dd94d91763056f069a", + "aliases": ["上海办"], + "connect_strength": "Strong", + "created_at": "2025-12-19T10:31:15.239252000", + "description": "位于上海的工作办公场所", + "statement_id": "8b1b12e23b844b8088dfeb67da6ad669", + "expired_at": "9999-12-31T00:00:00.000000000", + "entity_type": "Location", + "group_id": "88a459f5_text08", + "user_id": "88a459f5_text08", + "name": "上海办公室", + "apply_id": "88a459f5_text08", + "id": "fb702ef695c14e14af3e56786bc8815b" + } + }, + { + "entity1_name": "用户", + "description": "叙述者，讲述个人工作与生活经历的个体", + "statement_id": "62beac695b1346f4871740a45db88782", + "created_at": "2025-12-19T10:31:15.239252000", + "expired_at": "9999-12-31T00:00:00.000000000", + "relationship_type": "EXTRACTED_RELATIONSHIP", + "relationship": {}, + "entity2_name": "北京", + "entity2": { + "entity_idx": 2, + "run_id": "62b59cfebeea43dd94d91763056f069a", + "aliases": ["京", "京城", "北平"], + "connect_strength": "strong", + "created_at": "2025-12-19T10:31:15.239252000", + "description": "中国的首都城市，用户主要工作和生活所在地", + "statement_id": "62beac695b1346f4871740a45db88782", + "expired_at": "9999-12-31T00:00:00.000000000", + "entity_type": "Location", + "group_id": "88a459f5_text08", + "user_id": "88a459f5_text08", + "name": "北京", + "apply_id": "88a459f5_text08", + "id": "81b2d1a571bb46a08a2d7a1e87efb945" + } + }, + { + "entity1_name": "11010119950308123X", + "description": "具体的身份证号码值", + "statement_id": "030afd362e9b4110b139e68e5d3e7143", + "created_at": "2025-12-19T10:31:15.239252000", + "expired_at": "9999-12-31T00:00:00.000000000", + "relationship_type": "EXTRACTED_RELATIONSHIP", + "relationship": {}, + "entity2_name": "身份证号", + "entity2": { + "entity_idx": 2, + "run_id": "62b59cfebeea43dd94d91763056f069a", + "connect_strength": "strong", + "description": "中华人民共和国公民的身份号码", + "created_at": "2025-12-19T10:31:15.239252000", + "statement_id": "030afd362e9b4110b139e68e5d3e7143", + "expired_at": "9999-12-31T00:00:00.000000000", + "entity_type": "Identifier", + "group_id": "88a459f5_text08", + "user_id": "88a459f5_text08", + "name": "身份证号", + "apply_id": "88a459f5_text08", + "id": "3e5f920645b2404fadb0e9ff60d1306e" + } + } + ] + } +} \ No newline at end of file diff --git a/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py b/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py index b3e5813d..8f5b9bae 100644 --- a/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py +++ b/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py @@ -8,17 +8,20 @@ 4. 反思结果应用 - 更新记忆库 """ -import os import json import logging import asyncio +import os +import time from typing import List, Dict, Any, Optional -from datetime import datetime from enum import Enum import uuid -from pydantic import BaseModel, Field +from pydantic import BaseModel +from app.repositories.neo4j.cypher_queries import neo4j_query_part, neo4j_statement_part, neo4j_query_all, neo4j_statement_all +from app.repositories.neo4j.neo4j_update import neo4j_data +from app.repositories.neo4j.neo4j_connector import Neo4jConnector # 配置日志 _root_logger = logging.getLogger() @@ -33,14 +36,14 @@ else: class ReflectionRange(str, Enum): """反思范围枚举""" - RETRIEVAL = "retrieval" # 从检索结果中反思 - DATABASE = "database" # 从整个数据库中反思 + PARTIAL = "partial" # 从检索结果中反思 + ALL = "all" # 从整个数据库中反思 class ReflectionBaseline(str, Enum): """反思基线枚举""" - TIME = "TIME" # 基于时间的反思 - FACT = "FACT" # 基于事实的反思 + TIME = "TIME" # 基于时间的反思 + FACT = "FACT" # 基于事实的反思 HYBRID = "HYBRID" # 混合反思 @@ -48,9 +51,16 @@ class ReflectionConfig(BaseModel): """反思引擎配置""" enabled: bool = False iteration_period: str = "3" # 反思周期 - reflexion_range: ReflectionRange = ReflectionRange.RETRIEVAL + reflexion_range: ReflectionRange = ReflectionRange.PARTIAL baseline: ReflectionBaseline = ReflectionBaseline.TIME - concurrency: int = Field(default=5, description="并发数量") + model_id: Optional[str] = None # 模型ID + end_user_id: Optional[str] = None + output_example: Optional[str] = None # 输出示例 + + # 评估相关字段 + memory_verify: bool = True # 记忆验证 + quality_assessment: bool = True # 质量评估 + violation_handling_strategy: str = "warn" # 违规处理策略 class Config: use_enum_values = True @@ -75,16 +85,16 @@ class ReflectionEngine: """ def __init__( - self, - config: ReflectionConfig, - neo4j_connector: Optional[Any] = None, - llm_client: Optional[Any] = None, - get_data_func: Optional[Any] = None, - render_evaluate_prompt_func: Optional[Any] = None, - render_reflexion_prompt_func: Optional[Any] = None, - conflict_schema: Optional[Any] = None, - reflexion_schema: Optional[Any] = None, - update_query: Optional[str] = None + self, + config: ReflectionConfig, + neo4j_connector: Optional[Any] = None, + llm_client: Optional[Any] = None, + get_data_func: Optional[Any] = None, + render_evaluate_prompt_func: Optional[Any] = None, + render_reflexion_prompt_func: Optional[Any] = None, + conflict_schema: Optional[Any] = None, + reflexion_schema: Optional[Any] = None, + update_query: Optional[str] = None ): """ 初始化反思引擎 @@ -109,7 +119,7 @@ class ReflectionEngine: self.conflict_schema = conflict_schema self.reflexion_schema = reflexion_schema self.update_query = update_query - self._semaphore = asyncio.Semaphore(config.concurrency) + self._semaphore = asyncio.Semaphore(5) # 默认并发数为5 # 延迟导入以避免循环依赖 self._lazy_init_done = False @@ -127,11 +137,21 @@ class ReflectionEngine: from app.core.memory.utils.llm.llm_utils import get_llm_client from app.core.memory.utils.config import definitions as config_defs self.llm_client = get_llm_client(config_defs.SELECTED_LLM_ID) + elif isinstance(self.llm_client, str): + # 如果 llm_client 是字符串（model_id），则用它初始化客户端 + from app.core.memory.utils.llm.llm_utils import get_llm_client + model_id = self.llm_client + self.llm_client = get_llm_client(model_id) if self.get_data_func is None: from app.core.memory.utils.config.get_data import get_data self.get_data_func = get_data + # 导入get_data_statement函数 + if not hasattr(self, 'get_data_statement'): + from app.core.memory.utils.config.get_data import get_data_statement + self.get_data_statement = get_data_statement + if self.render_evaluate_prompt_func is None: from app.core.memory.utils.prompt.template_render import render_evaluate_prompt self.render_evaluate_prompt_func = render_evaluate_prompt @@ -154,13 +174,11 @@ class ReflectionEngine: self._lazy_init_done = True - async def execute_reflection(self, host_id: uuid.UUID) -> ReflectionResult: + async def execute_reflection(self, host_id) -> ReflectionResult: """ 执行完整的反思流程 - Args: host_id: 主机ID - Returns: ReflectionResult: 反思结果 """ @@ -176,9 +194,10 @@ class ReflectionEngine: start_time = asyncio.get_event_loop().time() logging.info("====== 自我反思流程开始 ======") + print(self.config.baseline, self.config.memory_verify, self.config.quality_assessment) try: # 1. 获取反思数据 - reflexion_data = await self._get_reflexion_data(host_id) + reflexion_data, statement_databasets = await self._get_reflexion_data(host_id) if not reflexion_data: return ReflectionResult( success=True, @@ -187,22 +206,21 @@ class ReflectionEngine: ) # 2. 检测冲突（基于事实的反思） - conflict_data = await self._detect_conflicts(reflexion_data) - if not conflict_data: - return ReflectionResult( - success=True, - message="无冲突，无需反思", - execution_time=asyncio.get_event_loop().time() - start_time - ) + conflict_data = await self._detect_conflicts(reflexion_data, statement_databasets) + print(100 * '-') + print(conflict_data) + print(100 * '-') - conflicts_found = len(conflict_data) - logging.info(f"发现 {conflicts_found} 个冲突") + # 检查是否真的有冲突 + has_conflict = conflict_data[0].get('conflict', False) + conflicts_found = len(conflict_data[0]['data']) if has_conflict else 0 + logging.info(f"冲突状态: {has_conflict}, 发现 {conflicts_found} 个冲突") # 记录冲突数据 await self._log_data("conflict", conflict_data) # 3. 解决冲突 - solved_data = await self._resolve_conflicts(conflict_data) + solved_data = await self._resolve_conflicts(conflict_data, statement_databasets) if not solved_data: return ReflectionResult( success=False, @@ -210,6 +228,9 @@ class ReflectionEngine: conflicts_found=conflicts_found, execution_time=asyncio.get_event_loop().time() - start_time ) + print(100 * '*') + print(solved_data) + print(100 * '*') conflicts_resolved = len(solved_data) logging.info(f"解决了 {conflicts_resolved} 个冲突") @@ -230,7 +251,8 @@ class ReflectionEngine: conflicts_found=conflicts_found, conflicts_resolved=conflicts_resolved, memories_updated=memories_updated, - execution_time=execution_time + execution_time=execution_time, + ) except Exception as e: @@ -241,6 +263,79 @@ class ReflectionEngine: execution_time=asyncio.get_event_loop().time() - start_time ) + async def reflection_run(self): + self._lazy_init() + start_time = time.time() + + asyncio.get_event_loop().time() + logging.info("====== 自我反思流程开始 ======") + + result_data = {} + + source_data, databasets = await self.extract_fields_from_json() + result_data['baseline'] = self.config.baseline + result_data[ + 'source_data'] = "我是 2023 年春天去北京工作的，后来基本一直都在北京上班，也没怎么换过城市。不过后来公司调整，2024 年上半年我被调到上海待了差不多半年，那段时间每天都是在上海办公室打卡。当时入职资料用的还是我之前的身份信息，身份证号是 11010119950308123X，银行卡是 6222023847595898，这些一直没变。对了，其实我 从 2023 年开始就一直在北京生活，从来没有长期离开过北京，上海那段更多算是远程配合" + + # 2. 检测冲突（基于事实的反思） + conflict_data = await self._detect_conflicts(databasets, source_data) + # 遍历数据提取字段 + quality_assessments = [] + memory_verifies = [] + for item in conflict_data: + print(item) + quality_assessments.append(item['quality_assessment']) + memory_verifies.append(item['memory_verify']) + result_data['quality_assessments'] = quality_assessments + result_data['memory_verifies'] = memory_verifies + + # 检查是否真的有冲突 + has_conflict = conflict_data[0].get('conflict', False) + conflicts_found = len(conflict_data[0]['data']) if has_conflict else 0 + logging.info(f"冲突状态: {has_conflict}, 发现 {conflicts_found} 个冲突") + + # 记录冲突数据 + await self._log_data("conflict", conflict_data) + + # 3. 解决冲突 + solved_data = await self._resolve_conflicts(conflict_data, source_data) + if not solved_data: + return ReflectionResult( + success=False, + message="反思失败，未解决冲突", + conflicts_found=conflicts_found, + execution_time=asyncio.get_event_loop().time() - start_time + ) + reflexion_data = [] + + # 遍历数据提取reflexion字段 + for item in solved_data: + if 'results' in item: + for result in item['results']: + reflexion_data.append(result['reflexion']) + result_data['reflexion_data'] = reflexion_data + execution_time = time.time() - start_time + return {"status": "SUCCESS", "message": "反思试运行", "data": result_data, "time": execution_time} + + async def extract_fields_from_json(self): + """从example.json中提取source_data和databasets字段""" + + prompt_dir = os.path.join(os.path.dirname(__file__), "example") + try: + # 读取JSON文件 + with open(prompt_dir + '/example.json', 'r', encoding='utf-8') as f: + data = json.loads(f.read()) + + # 提取memory_verify下的字段 + memory_verify = data.get("memory_verify", {}) + source_data = memory_verify.get("source_data", []) + databasets = memory_verify.get("databasets", []) + + return source_data, databasets + + except Exception as e: + return [], [] + async def _get_reflexion_data(self, host_id: uuid.UUID) -> List[Any]: """ 获取反思数据 @@ -253,17 +348,28 @@ class ReflectionEngine: Returns: List[Any]: 反思数据列表 """ - if self.config.reflexion_range == ReflectionRange.RETRIEVAL: - # 从检索结果中获取数据 - return await self.get_data_func(host_id) - elif self.config.reflexion_range == ReflectionRange.DATABASE: - # 从整个数据库中获取数据（待实现） - logging.warning("从数据库获取反思数据功能尚未实现") - return [] - else: - raise ValueError(f"未知的反思范围: {self.config.reflexion_range}") - async def _detect_conflicts(self, data: List[Any]) -> List[Any]: + + + if self.config.reflexion_range == ReflectionRange.PARTIAL: + neo4j_query = neo4j_query_part.format(host_id) + neo4j_statement = neo4j_statement_part.format(host_id) + elif self.config.reflexion_range == ReflectionRange.ALL: + neo4j_query = neo4j_query_all.format(host_id) + neo4j_statement = neo4j_statement_all.format(host_id) + try: + result = await self.neo4j_connector.execute_query(neo4j_query) + result_statement = await self.neo4j_connector.execute_query(neo4j_statement) + neo4j_databasets = await self.get_data_func(result) + neo4j_state = await self.get_data_statement(result_statement) + return neo4j_databasets, neo4j_state + + + except Exception as e: + logging.error(f"Neo4j查询失败: {e}") + return [], [] + + async def _detect_conflicts(self, data: List[Any], statement_databasets: List[Any]) -> List[Any]: """ 检测冲突（基于事实的反思） @@ -278,14 +384,28 @@ class ReflectionEngine: if not data: return [] + # 数据预处理：如果数据量太少，直接返回无冲突 + if len(data) < 2: + logging.info("数据量不足，无需检测冲突") + return [] + + # 使用转换后的数据 + print("转换后的数据:", data[:2] if len(data) > 2 else data) # 只打印前2条避免日志过长 + memory_verify = self.config.memory_verify + logging.info("====== 冲突检测开始 ======") start_time = asyncio.get_event_loop().time() + quality_assessment = self.config.quality_assessment try: # 渲染冲突检测提示词 rendered_prompt = await self.render_evaluate_prompt_func( data, - self.conflict_schema + self.conflict_schema, + self.config.baseline, + memory_verify, + quality_assessment, + statement_databasets ) messages = [{"role": "user", "content": rendered_prompt}] @@ -316,7 +436,7 @@ class ReflectionEngine: logging.error(f"冲突检测失败: {e}", exc_info=True) return [] - async def _resolve_conflicts(self, conflicts: List[Any]) -> List[Any]: + async def _resolve_conflicts(self, conflicts: List[Any], statement_databasets: List[Any]) -> List[Any]: """ 解决冲突 @@ -332,6 +452,8 @@ class ReflectionEngine: return [] logging.info("====== 冲突解决开始 ======") + baseline = self.config.baseline + memory_verify = self.config.memory_verify # 并行处理每个冲突 async def _resolve_one(conflict: Any) -> Optional[Dict[str, Any]]: @@ -341,7 +463,10 @@ class ReflectionEngine: # 渲染反思提示词 rendered_prompt = await self.render_reflexion_prompt_func( [conflict], - self.reflexion_schema + self.reflexion_schema, + baseline, + memory_verify, + statement_databasets ) messages = [{"role": "user", "content": rendered_prompt}] @@ -381,8 +506,8 @@ class ReflectionEngine: return solved async def _apply_reflection_results( - self, - solved_data: List[Dict[str, Any]] + self, + solved_data: List[Dict[str, Any]] ) -> int: """ 应用反思结果（更新记忆库） @@ -395,57 +520,7 @@ class ReflectionEngine: Returns: int: 成功更新的记忆数量 """ - if not solved_data: - logging.warning("无解决方案数据，跳过更新") - return 0 - - logging.info("====== 记忆更新开始 ======") - - success_count = 0 - - async def _update_one(item: Dict[str, Any]) -> bool: - """更新单条记忆""" - async with self._semaphore: - try: - if not isinstance(item, dict): - return False - - # 提取更新参数 - resolved = item.get("resolved", {}) - resolved_mem = resolved.get("resolved_memory", {}) - group_id = resolved_mem.get("group_id") - memory_id = resolved_mem.get("id") - new_invalid_at = resolved_mem.get("invalid_at") - - if not all([group_id, memory_id, new_invalid_at]): - logging.warning(f"记忆更新参数缺失，跳过此项: {item}") - return False - - # 执行更新 - await self.neo4j_connector.execute_query( - self.update_query, - group_id=group_id, - id=memory_id, - new_invalid_at=new_invalid_at, - ) - - return True - - except Exception as e: - logging.error(f"更新单条记忆失败: {e}") - return False - - # 并发执行所有更新任务 - tasks = [ - _update_one(item) - for item in solved_data - if isinstance(item, dict) - ] - results = await asyncio.gather(*tasks, return_exceptions=False) - success_count = sum(1 for r in results if r) - - logging.info(f"成功更新 {success_count}/{len(solved_data)} 条记忆") - + success_count = await neo4j_data(solved_data) return success_count async def _log_data(self, label: str, data: Any) -> None: @@ -456,6 +531,7 @@ class ReflectionEngine: label: 数据标签 data: 要记录的数据 """ + def _write(): try: with open("reflexion_data.json", "a", encoding="utf-8") as f: @@ -470,9 +546,9 @@ class ReflectionEngine: # 基于时间的反思方法 async def time_based_reflection( - self, - host_id: uuid.UUID, - time_period: Optional[str] = None + self, + host_id: uuid.UUID, + time_period: Optional[str] = None ) -> ReflectionResult: """ 基于时间的反思 @@ -494,8 +570,8 @@ class ReflectionEngine: # 基于事实的反思方法 async def fact_based_reflection( - self, - host_id: uuid.UUID + self, + host_id: uuid.UUID ) -> ReflectionResult: """ 基于事实的反思 @@ -515,8 +591,8 @@ class ReflectionEngine: # 综合反思方法 async def comprehensive_reflection( - self, - host_id: uuid.UUID + self, + host_id: uuid.UUID ) -> ReflectionResult: """ 综合反思 @@ -553,33 +629,3 @@ class ReflectionEngine: else: raise ValueError(f"未知的反思基线: {self.config.baseline}") - -# 便捷函数：创建默认配置的反思引擎 -def create_reflection_engine( - enabled: bool = False, - iteration_period: str = "3", - reflexion_range: str = "retrieval", - baseline: str = "TIME", - concurrency: int = 5 -) -> ReflectionEngine: - """ - 创建反思引擎实例 - - Args: - enabled: 是否启用反思 - iteration_period: 反思周期 - reflexion_range: 反思范围 - baseline: 反思基线 - concurrency: 并发数量 - - Returns: - ReflectionEngine: 反思引擎实例 - """ - config = ReflectionConfig( - enabled=enabled, - iteration_period=iteration_period, - reflexion_range=reflexion_range, - baseline=baseline, - concurrency=concurrency - ) - return ReflectionEngine(config) diff --git a/api/app/core/memory/utils/config/get_data.py b/api/app/core/memory/utils/config/get_data.py index f2f21198..a099694e 100644 --- a/api/app/core/memory/utils/config/get_data.py +++ b/api/app/core/memory/utils/config/get_data.py @@ -1,13 +1,8 @@ import json -import os import uuid -from typing import List, Dict, Any, Optional -from sqlalchemy.orm import Session -from app.db import get_db -from app.models.retrieval_info import RetrievalInfo -from app.schemas.memory_storage_schema import BaseDataSchema - import logging + +from typing import List, Dict, Any logger = logging.getLogger(__name__) async def _load_(data: List[Any]) -> List[Dict]: @@ -60,27 +55,46 @@ async def _load_(data: List[Any]) -> List[Dict]: return results -async def get_data(host_id: uuid.UUID) -> List[Dict]: +async def get_data(result): """ 从数据库中获取数据 """ - # 从数据库会话中获取会话 - db: Session = next(get_db()) - try: - data = db.query(RetrievalInfo.retrieve_info).filter(RetrievalInfo.host_id == host_id).all() + neo4j_databasets=[] + for item in result: + filtered_item = {} + for key, value in item.items(): + if 'name_embedding' not in key.lower(): + if key == 'relationship' and value is not None: + # 只保留relationship的指定字段 + rel_filtered = {} + if hasattr(value, 'get'): + rel_filtered['run_id'] = value.get('run_id') + rel_filtered['statement'] = value.get('statement') + rel_filtered['statement_id'] = value.get('statement_id') + rel_filtered['expired_at'] = value.get('expired_at') + rel_filtered['created_at'] = value.get('created_at') + filtered_item[key] = rel_filtered + elif key == 'entity2' and value is not None: + # 过滤entity2的name_embedding字段 + entity2_filtered = {} + if hasattr(value, 'items'): + for e_key, e_value in value.items(): + if 'name_embedding' not in e_key.lower(): + entity2_filtered[e_key] = e_value + filtered_item[key] = entity2_filtered + else: + filtered_item[key] = value + + # 直接将字典添加到列表中 + neo4j_databasets.append(filtered_item) + return neo4j_databasets +async def get_data_statement( result): + neo4j_databasets=[] + for i in result: + neo4j_databasets.append(i) + return neo4j_databasets + - # print(f"data:\n{data}") - # 解析，提取为字典的列表 - results = await _load_(data) - return results - except Exception as e: - logger.error(f"failed to get data from database, host_id: {host_id}, error: {e}") - raise e - finally: - try: - db.close() - except Exception: - pass if __name__ == "__main__": diff --git a/api/app/core/memory/utils/prompt/prompts/evaluate.jinja2 b/api/app/core/memory/utils/prompt/prompts/evaluate.jinja2 index cb5b917d..e1ecf820 100644 --- a/api/app/core/memory/utils/prompt/prompts/evaluate.jinja2 +++ b/api/app/core/memory/utils/prompt/prompts/evaluate.jinja2 @@ -1,19 +1,222 @@ -你将收到一组记忆对象：{{ evaluate_data }}。 -任务：多维度判断这些记忆是否与已有记忆存在冲突，并给出冲突的对应记忆。(冗余不算冲突) +你将收到一组用户历史记忆原始数据（来源于 Neo4j），以及相关配置参数： +原本的输入句子：{{statement_databasets}} +需要检测冲突对象：{{ evaluate_data }} +冲突判定类型：{{ baseline }}（取值为 TIME / FACT / HYBRID） +记忆审核开关：{{ memory_verify }}（取值为 true / false） +记忆质量评估开关开关：{{ quality_assessment }}（取值为 true / false） -仅输出一个合法 JSON 对象，严格遵循下述结构： +你的任务是： +对用户历史记忆数据进行冲突检测和记忆审核，并输出严格结构化的 JSON 分析结果 +数据的结构： + statement_databasets里面statement_name是输入的句子，statement_id是连接evaluate_data里面的statement_id，代表这个句子被拆分成几个实体，需要根据整体的内容， + 需要根据以下内容做处理（冲突检测、记忆审核、记忆的质量评估） +## 冲突定义 + +### 时间冲突 +时间冲突是指同一用户的相关事件在时间维度上存在逻辑矛盾： + +1. **同一活动的时间冲突**： + - 同一用户的同一活动在不同时间点被记录（如"周五打球"和"周六打球"） + - 同一用户在同一时间段内被记录进行不同的互斥活动 + +2. **时间逻辑错误**： + - expired_at 早于 created_at + - 同一事实的 created_at 时间差异超过合理误差范围（>5分钟） + +3. **日期属性冲突**： + - 同一人的生日记录为不同日期（如"2月10号"和"2月16号"） +4.存在明确先后约束 A -> B，但 t(A) > t(B) + -例：入学时间晚于毕业时间。 + -处理：标记异常、降权、触发逻辑反思或人工审查。 +5.时间属性冲突 + -单值日期属性出现多值（生日、入职日期） + -注意：本质属于事实冲突的日期特例，归入事实冲突仲裁框架。 +6.互斥重叠冲突 + -例：同一主体的两个事件区间重叠且互斥（如同一时间出现在两地） + -处理：证据仲裁、保留多版本（active + candidate）。 + + + +### 事实冲突 +事实冲突是指同一实体的属性或关系存在相互矛盾的陈述： + +1. **属性互斥**：同一实体的相反属性（喜欢↔不喜欢、有↔没有、是↔不是） +2. **关系矛盾**：同一实体在相同语境下的不同关系描述 +3. **身份冲突**：同一实体被赋予不同的类型或角色 + +### 混合冲突检测 +检测所有类型的冲突，包括但不限于时间冲突和事实冲突： +检测任何逻辑上不一致或相互矛盾的记录 +## 记忆审核定义 + +### 隐私信息检测（隐私冲突） +当memory_verify为true时，需要额外检测包含个人隐私信息的记录： + +1. **身份证信息**：包含身份证号码、身份证相关描述 +2. **手机号码**：包含手机号、电话号码等联系方式 +3. **社交账号**：包含微信号、QQ号、邮箱地址等社交平台信息 +4. **银行信息**：包含银行卡号、账户信息、支付信息 +5. **税务信息**：包含税号、纳税信息、发票信息 +6. **贷款信息**：包含贷款记录、信贷信息、借款信息 +7. **其他敏感信息**：包含密码、PIN码、验证码等安全信息 + +### 隐私检测原则 +- 检测description、entity1_name、entity2_name等字段中的隐私信息 +- 识别数字模式（如手机号11位数字、身份证18位等） +- 识别关键词（如"身份证"、"银行卡"、"密码"等） +- 检测敏感实体类型和关系 + +## 冲突检测原则 + +**全面检测**：不区分冲突类型，检测所有可能的冲突 +**完整输出**：如果发现任何冲突或隐私信息，必须将所有相关记录都放入data字段 +**实体关联**：重点检查涉及相同实体（entity1_name, entity2_name）的记录 +**语义分析**：分析description字段的语义相似性和冲突性 +**时间逻辑**：检查时间字段的逻辑一致性 +**隐私检测**：当memory_verify为true时，检测所有包含隐私信息的记录 + +## 不符合冲突检测 + -称呼 +## 重要检测示例 + +### 冲突检测示例 +- 用户与不同时间点的关系（周五 vs 周六，2月10号 vs 2月16号） +- 同一实体的重复定义但描述不同 +- 同一关系的不同表述但含义冲突 +- 任何逻辑上不可能同时为真的记录 + +### 隐私信息检测示例 +- 包含手机号的记录："用户的手机号是13812345678" +- 包含身份证的记录："身份证号码为110101199001011234" +- 包含银行卡的记录："银行卡号6222021234567890" +- 包含社交账号的记录："微信号是user123456" +- 包含敏感信息的实体名称或描述 + +## 输出要求 + +**关键原则**： +1. 当存在冲突或检测到隐私信息时，conflict才为true，data字段才包含相关记录 +2. 如果发现冲突，必须将所有相关的冲突记录都放入data数组中 +3. 如果memory_verify为true且检测到隐私信息，必须将包含隐私信息的记录也放入data数组中 +4. 既没有冲突也没有隐私信息时，conflict为false，data为空数组 +5. 如果quality_assessment为true，独立分析数据质量并输出评估结果；如果为false，quality_assessment字段输出null +6. 冲突检测、隐私审核和质量评估三个功能完全独立，互不影响 +7. 不输出conflict_memory字段 + +**处理逻辑**： +- 首先进行冲突检测，将冲突记录加入data数组 +- 如果memory_verify为true，再进行隐私信息检测，将包含隐私信息的记录也加入data数组 +- 如果quality_assessment为true，独立进行质量评估，分析所有输入数据的质量并输出评估结果 +- 最终data数组包含所有冲突记录和隐私信息记录（去重） +- quality_assessment字段独立输出，不影响冲突检测和隐私审核结果 +- memory_verify字段独立输出隐私检测结果，包含检测到的隐私信息类型和概述 + +返回数据格式以json方式输出: +- 必须通过json.loads()的格式支持的形式输出,响应必须是与此确切模式匹配的有效JSON对象。不要在JSON之前或之后包含任何文本。 +- 关键的JSON格式要求{"statement":识别出的文本内容} +1.JSON结构仅使用标准ASCII双引号（"）-切勿使用中文引号（""）或其他Unicode引号 +2.如果提取的语句文本包含引号，请使用反斜杠（\"）正确转义它们 +3.确保所有JSON字符串都正确关闭并以逗号分隔 +4.JSON字符串值中不包括换行符 +5.正确转义的例子："statement"："Zhang Xinhua said：\"我非常喜欢这本书\"" +6.不允许输出```json```相关符号，如```json```、``````、```python```、```javascript```、```html```、```css```、```sql```、```java```、```c```、```c++```、```c#```、```ruby``` + +## 记忆质量评估定义 + +### 质量评估标准 +当quality_assessment为true时，需要对记忆数据进行质量评估： + +1. **数据完整性**： + - 检查必要字段是否完整（entity1_name、entity2_name、description等） + - 检查关系描述是否清晰明确 + - 检查时间字段的有效性 + +2. **重复字段检测**： + - 识别相同或高度相似的记录 + - 检测冗余的实体关系 + - 分析描述内容的重复度 + +3. **无意义字段检测**： + - 识别空值、无效值或占位符内容 + - 检测过于简单或无信息量的描述 + - 识别格式错误或不规范的数据 + +4. **上下文依赖性**： + - 评估记录是否需要额外上下文才能理解 + - 检查实体名称的明确性 + - 分析关系描述的自包含性 + +### 质量评估输出 +- **质量百分比**：基于上述标准计算的整体质量分数（0-100） +- **质量概述**：简要描述数据质量状况，包括主要问题和优点 + +输出是仅输出一个合法 JSON 对象，严格遵循下述结构： { - "data": [ ...与输入同结构的记忆对象数组... ], - "conflict": true 或 false, - "conflict_memory": 若冲突为 true，则填写与其冲突的记忆对象；否则为 null + "data": [ + { + "entity1_name": "实体1名称", + "description": "描述信息", + "statement_id": "陈述ID", + "created_at": "创建时间戳", + "expired_at": "过期时间戳", + "relationship_type": "关系类型", + "relationship": "关系对象", + "entity2_name": "实体2名称", + "entity2": "实体2对象" + } + ], + "conflict": true或false, + "quality_assessment": { + "score": 质量百分比数字, + "summary": "质量概述文本" + } 或 null, + "memory_verify": { + "has_privacy": true或false, + "privacy_types": ["检测到的隐私信息类型列表"], + "summary": "隐私检测结果概述" + } 或 null } 必须遵守： - 只输出 JSON，不要添加解释或多余文本。 - 使用标准双引号，必要时对内部引号进行转义。 - 字段名与结构必须与给定模式一致。 +- data数组中包含冲突记录和隐私信息记录，如果都没有则为空数组。 +- quality_assessment字段：当quality_assessment参数为true时输出评估对象，为false时输出null。 +- memory_verify字段：当memory_verify参数为true时输出隐私检测结果对象，为false时输出null。 + +### memory_verify字段说明 +当memory_verify为true时，需要输出隐私检测结果： +- **has_privacy**: 布尔值，表示是否检测到隐私信息 +- **privacy_types**: 字符串数组，包含检测到的隐私信息类型（如["手机号码", "身份证信息"]） +- **summary**: 字符串，简要描述隐私检测结果 + +当memory_verify为false时，memory_verify字段输出null。 + +### memory_verify字段示例 + +**示例1：检测到隐私信息** +```json +"memory_verify": { + "has_privacy": true, + "privacy_types": ["手机号码", "身份证信息"], + "summary": "检测到2条记录包含隐私信息：1个手机号码，1个身份证号码" +} +``` + +**示例2：未检测到隐私信息** +```json +"memory_verify": { + "has_privacy": false, + "privacy_types": [], + "summary": "未检测到隐私信息" +} +``` + +**示例3：memory_verify为false时** +```json +"memory_verify": null +``` 模式参考： -[ - {{ json_schema }} -] \ No newline at end of file +{{ json_schema }} \ No newline at end of file diff --git a/api/app/core/memory/utils/prompt/prompts/reflexion.jinja2 b/api/app/core/memory/utils/prompt/prompts/reflexion.jinja2 index 3f78b137..43e8e100 100644 --- a/api/app/core/memory/utils/prompt/prompts/reflexion.jinja2 +++ b/api/app/core/memory/utils/prompt/prompts/reflexion.jinja2 @@ -1,23 +1,300 @@ +你将收到一组用户历史记忆原始数据（来源于 Neo4j） 你将收到一条冲突判定对象：{{ data }}。 -任务：分析冲突产生原因，给出解决方案，并生成设为失效后的记忆。 +需要检测冲突对象：{{ statement_databasets }} +以及需要识别的冲突对象为：{{ baseline }} +记忆审核开关：{{ memory_verify }}（取值为 true / false） + +角色： +- 你是数据领域中解决数据冲突的专家 + +任务：分析冲突产生原因，按冲突类型分组处理，为每种冲突类型生成独立的解决方案。 + +数据的结构： + statement_databasets里面statement_name是输入的句子，statement_id是连接data里面的statement_id，代表这个句子被拆分成几个实体，需要根据整体的内容， + 需要根据以下内容做处理（冲突检测、记忆审核、记忆的质量评估）,data里面的statement_created_at是用户输入的时间 + +**处理模式**： +- 当memory_verify为false时：仅处理数据冲突 +- 当memory_verify为true时：处理数据冲突 + 隐私信息脱敏 + +## 分组处理原则 + +**冲突类型识别与分组**： +1. **日期冲突**： + 1.1.涉及用户生日的不同日期记录（如2月10号 vs 2月16号）， + 1.2.涉及同一活动的不同时间记录（如周五打球 vs 周六打球） +3. **事实属性冲突**： + 3.1. **属性互斥**：同一实体的相反属性（喜欢↔不喜欢、有↔没有、是↔不是） + 3.2. **关系矛盾**：同一实体在相同语境下的不同关系描述 + 3.3. **身份冲突**：同一实体被赋予不同的类型或角色 +4. **其他冲突类型/混合冲突（时间+事实）**：根据具体数据识别 + +**分组输出要求**： +- 每种冲突类型生成一个独立的reflexion_result对象 +- 同一类型的多个冲突记录归并到一个结果中 +- 不同类型的冲突分别处理，各自生成独立结果 + +## 冲突类型定义 + +### 时间冲突（TIME） +时间维度冲突是指两个事件发生时间重叠，或者用户同一件事情和场景等情况下，时间出现了变化。 + +### 事实冲突（FACT） +事实冲突是指同一事实对象（同一个人、同一个时间、同一个状态）但陈述内容相互矛盾，主要为真假不能共存的情况。 +### 混合冲突（HYBRID） +检测所有类型的冲突，包括但不限于时间冲突和事实冲突：检测任何逻辑上不一致或相互矛盾的记录 +{% if memory_verify %} +## 隐私信息处理（memory_verify为true时启用） + +### 隐私信息识别 +需要识别并处理以下类型的隐私信息： + +1. **身份证信息**：包含身份证号码、身份证相关描述 +2. **手机号码**：包含手机号、电话号码等联系方式 +3. **社交账号**：包含微信号、QQ号、邮箱地址等社交平台信息 +4. **银行信息**：包含银行卡号、账户信息、支付信息 +5. **税务信息**：包含税号、纳税信息、发票信息 +6. **贷款信息**：包含贷款记录、信贷信息、借款信息 +7. **其他敏感信息**：包含密码、PIN码、验证码等安全信息 + +### 隐私数据脱敏规则 +对于检测到的隐私信息，按以下规则进行脱敏处理： + +**数字类隐私信息脱敏**： +- 保留前三位和后四位，中间用*代替 +- 示例：手机号13812345678 → 138****5678 +- 示例：身份证110101199001011234 → 110***********1234 +- 示例：银行卡6222021234567890 → 622***********7890 + +**文本类隐私信息脱敏**： +- 社交账号：保留前三后四位字符，中间用*代替 +- 示例：微信号user123456 → use****3456 +- 示例：邮箱zhang.san@example.com → zha****@example.com + +**脱敏处理字段**： +- name字段：如包含隐私信息需脱敏 +- entity1_name字段：如包含隐私信息需脱敏 +- entity2_name字段：如包含隐私信息需脱敏 +- description字段：如包含隐私信息需脱敏 +{% endif %} + +## 工作步骤 + +### 第一步：分析冲突类型匹配 +首先判断输入的冲突数据是否符合baseline要求的类型： + +**类型匹配规则**： +- 如果baseline是"TIME"：只处理时间相关的冲突（涉及时间表达式、日期、时间点的冲突） +- 如果baseline是"FACT"：只处理事实相关的冲突（属性矛盾、关系冲突、描述不一致） +- 如果baseline是"HYBRID"：处理所有类型的冲突，也可以当作混合冲突类型处理 + +**类型识别**： +- 时间冲突标识：entity2的entity_type包含"TimeExpression"、"TemporalExpression"，或entity2_name包含时间词汇（周一到周日、月份日期等） +- 事实冲突标识：相同实体的不同属性描述、互斥的关系陈述 + +**重要**：如果输入的冲突类型与baseline不匹配，必须输出空结果（resolved为null） + +### 第二步：筛选并分组冲突数据 +按冲突类型对数据进行分组： + +**分组策略**： +1. **时间冲突组**：筛选涉及用户时间的所有记录 +2. **活动时间冲突组**：筛选涉及同一活动不同时间的记录 +3. **事实冲突组**：筛选涉及同一实体不同属性的记录 +4. **其他冲突组**：其他类型的冲突记录 + +**筛选条件**： +- 只处理与baseline匹配的冲突类型 +- 相同entity1_name但entity2_name不同的记录 +- 相同关系但描述矛盾的记录 +- 时间逻辑不一致的记录 + +### 第三步：冲突解决策略 +** 不可以解决的冲突情况 + 1. 数据被判定为正确的情况下，不可以进行修改 +**仅当冲突类型与baseline匹配时**，对筛选出的冲突数据进行处理： + +**智能解决策略**： +1. **分析冲突数据**：识别哪些记录是正确的，哪些是错误的,需要结合statement_databasets的输入原文来判定 +2. **判断正确答案是否存在**： + - 如果正确答案已存在于data中：只需将错误记录的expired_at设为当前日期（2025-12-16T12:00:00） + - 如果正确答案已存在于data中：错误记录的expired_at已经设为日期，则不需要对正确的数据进行修改 + - 如果正确答案不存在于data中：需要修改现有记录的内容以包含正确信息 + +{% if memory_verify %} +**隐私处理集成**： +- 在处理冲突的同时，需要对涉及的记录进行隐私脱敏 +- 脱敏处理应该在冲突解决之后进行，确保最终输出的记录都已脱敏 +- 在change字段中记录隐私脱敏的变更 +{% endif %} + +**具体处理规则**： + +**情况1：正确答案存在于data中** +- 保留正确的记录不变 +- 基于时间关系的冲突： + 需要只修改错误记录的expired_at为当前时间（2025-12-16T12:00:00） +- 基于事实的关系冲突 +- resolved.resolved_memory只包含被设为失效的错误记录 +- change字段只记录expired_at的变更：`[{"expired_at": "2025-12-16T12:00:00"}]`(注意：如果已存在时间，则不需要对其修改，也不需要变更 时间) + +**情况2：正确答案不存在于data中** +- 选择最合适的记录进行修改 +- 更新该记录的相关字段： + - description字段：添加或修改描述信息{% if memory_verify %}（如包含隐私信息，需脱敏处理）{% endif %} + - name字段：修改名称字段{% if memory_verify %}（如需要，包含隐私信息时需脱敏）{% endif %} +- resolved.resolved_memory包含修改后的完整记录{% if memory_verify %}（已脱敏）{% endif %} +- change字段记录所有被修改的字段{% if memory_verify %}，包括脱敏变更{% endif %}，例如：`[{"description": "新描述"{% if memory_verify %}, "entity2_name": "138****5678"{% endif %}}]` + +**重要原则**： +- **只输出需要修改的记录**：resolved.resolved_memory只包含实际需要修改的数据 +- **优先保留策略**：时间冲突保留最可信的created_at时间的记录，事实冲突选择最新且可信度最高的记录 +- **精确记录变更**：change字段必须包含记录ID、字段名称、新值和旧值 +{% if memory_verify %}- **隐私保护优先**：所有输出的记录必须完成隐私脱敏处理 +- **脱敏变更记录**：隐私脱敏的变更也必须在change字段中详细记录{% endif %} +- **不可修改数据**：数据被判定为正确时，不可以进行修改，如果没有数据可输出空 + +**变更记录格式**： +```json +"change": [ + { + "field": [ + {"字段名1": "修改后的值1"}, + {"字段名2": "修改后的值2"} + ] + } +] +``` + +**类型不匹配处理**： +- 如果冲突类型与baseline不匹配，resolved必须设为null +- reflexion.reason说明类型不匹配的原因 +- reflexion.solution说明无需处理 + +### 第四步：输出解决方案 + +## 输出要求 +**嵌套字段映射**（系统会自动处理）： +- `entity2.name` → 自动映射为 `name` +- `entity1.name` → 自动映射为 `name` +- `entity1.description` → 自动映射为 `description` +- `entity2.description` → 自动映射为 `description` + +返回数据格式以json方式输出： +- 必须通过json.loads()的格式支持的形式输出 +- 响应必须是与此确切模式匹配的有效JSON对象 +- 不要在JSON之前或之后包含任何文本 + +JSON格式要求： +1. JSON结构仅使用标准ASCII双引号（"） +2. 如果提取的语句文本包含引号，请使用反斜杠（\"）正确转义 +3. 确保所有JSON字符串都正确关闭并以逗号分隔 +4. JSON字符串值中不包括换行符 +5. 不允许输出```json```相关符号 仅输出一个合法 JSON 对象，严格遵循下述结构： + +**输出格式：按冲突类型分组的列表** { - "conflict": 与输入同结构，包含 data 与 conflict_memory, - "reflexion": { "reason": string, "solution": string }, - "resolved": { - "original_memory_id": 被设为失效的记忆 id, - "resolved_memory": 完整的设为失效后的记忆对象 - } + "results": [ + { + "conflict": { + "data": [该冲突类型相关的数据记录], + "conflict": true + }, + "reflexion": { + "reason": "该冲突类型的原因分析", + "solution": "该冲突类型的解决方案" + }, + "resolved": { + "original_memory_id": "被设为失效的记忆id", + "resolved_memory": { + "entity1_name": "实体1名称", + "entity2_name": "实体2名称", + "description": "描述信息", + "statement_id": "陈述ID", + "created_at": "创建时间", + "expired_at": "过期时间", + "relationship_type": "关系类型", + "relationship": {}, + "entity2": {...} + }, + "change": [ + { + "field": [ + {"字段名1": "修改后的值1"}, + {"字段名2": "修改后的值2"} + ] + } + ] + }, + "type": "reflexion_result" + } + ] +} + +**示例：多种冲突类型的输出** +{ + "results": [ + { + "conflict": { + "data": [生日冲突相关的记录], + "conflict": true + }, + "reflexion": { + "reason": "检测到生日冲突：用户同时关联2月10号和2月16号两个不同日期", + "solution": "保留最新记录（2月16号），将旧记录（2月10号）设为失效" + }, + "resolved": { + "original_memory_id": "df066210883545a08e727ccd8ad4ec77", + "resolved_memory": {...}, + "change": [ + { + "field": [ + {"expired_at": "2025-12-16T12:00:00"} + ] + } + ] + }, + "type": "reflexion_result" + }, + { + "conflict": { + "data": [篮球时间冲突相关的记录], + "conflict": true + }, + "reflexion": { + "reason": "检测到活动时间冲突：用户打篮球时间存在周五和周六的冲突", + "solution": "保留最可信的时间记录，将冲突记录设为失效" + }, + "resolved": { + "original_memory_id": "另一个记录ID", + "resolved_memory": {...}, + "change": [ + { + "field": [ + {"description": "使用系统的个人，指代说话者本人，篮球时间为周六"}, + {"entity2_name": "周六"} + ] + } + ] + }, + "type": "reflexion_result" + } + ] } 必须遵守： -- 只输出 JSON，不要添加解释或多余文本。 -- 使用标准双引号，必要时对内部引号进行转义。 -- 字段名与结构必须与给定模式一致。 -- 当 conflict 为 false 时，resolved 必须为 null。 - - 其中 conflict.data 必须为数组形式，即使只有一个对象也需使用 [ ] 包裹。 +- 只输出 JSON，不要添加解释或多余文本 +- 使用标准双引号，必要时对内部引号进行转义 +- 字段名与结构必须与给定模式一致 +- **输出必须是results数组格式**，每个冲突类型作为一个独立的对象 +- **按冲突类型分组**：相同类型的冲突记录归并到一个result对象中 +- **每个result对象的conflict.data**只包含该冲突类型相关的记录 +- **resolved.resolved_memory 只包含需要修改的记录**，不需要修改的记录不要输出 +- **resolved.change 必须包含详细的变更信息**：field数组包含所有被修改的字段及其新值 +- 如果某个冲突类型经分析无需修改任何数据，该类型的resolved 必须为 null +- 如果与baseline不匹配的冲突类型，不要在results中包含该类型 + 模式参考： -[ - {{ json_schema }} -] +{{ json_schema }} \ No newline at end of file diff --git a/api/app/core/memory/utils/prompt/template_render.py b/api/app/core/memory/utils/prompt/template_render.py index c783e095..818d456a 100644 --- a/api/app/core/memory/utils/prompt/template_render.py +++ b/api/app/core/memory/utils/prompt/template_render.py @@ -7,36 +7,50 @@ from typing import List, Dict, Any prompt_dir = os.path.join(os.path.dirname(__file__), "prompts") prompt_env = Environment(loader=FileSystemLoader(prompt_dir)) -async def render_evaluate_prompt(evaluate_data: List[Any], schema: Dict[str, Any]) -> str: +async def render_evaluate_prompt(evaluate_data: List[Any], schema: Dict[str, Any], + baseline: str = "TIME", + memory_verify: bool = False,quality_assessment:bool = False,statement_databasets: List[str] = []) -> str: """ - Renders the evaluate prompt using the evaluate.jinja2 template. + Renders the evaluate prompt using the evaluate_optimized.jinja2 template. Args: evaluate_data: The data to evaluate schema: The JSON schema to use for the output. + baseline: The baseline type for conflict detection (TIME/FACT/TIME-FACT) + memory_verify: Whether to enable memory verification for privacy detection Returns: Rendered prompt content as string """ template = prompt_env.get_template("evaluate.jinja2") - rendered_prompt = template.render(evaluate_data=evaluate_data, json_schema=schema) - + rendered_prompt = template.render( + evaluate_data=evaluate_data, + json_schema=schema, + baseline=baseline, + memory_verify=memory_verify, + quality_assessment=quality_assessment, + statement_databasets=statement_databasets + ) return rendered_prompt -async def render_reflexion_prompt(data: Dict[str, Any], schema: Dict[str, Any]) -> str: +async def render_reflexion_prompt(data: Dict[str, Any], schema: Dict[str, Any], baseline: str, memory_verify: bool = False, + statement_databasets: List[str] = []) -> str: """ - Renders the reflexion prompt using the extract_temporal.jinja2 template. + Renders the reflexion prompt using the reflexion_optimized.jinja2 template. Args: data: The data to reflex on. schema: The JSON schema to use for the output. + baseline: The baseline type for conflict resolution. Returns: Rendered prompt content as a string. """ template = prompt_env.get_template("reflexion.jinja2") - rendered_prompt = template.render(data=data, json_schema=schema) + rendered_prompt = template.render(data=data, json_schema=schema, + baseline=baseline,memory_verify=memory_verify, + statement_databasets=statement_databasets) return rendered_prompt diff --git a/api/app/models/data_config_model.py b/api/app/models/data_config_model.py index 9f27562c..be43bd8d 100644 --- a/api/app/models/data_config_model.py +++ b/api/app/models/data_config_model.py @@ -1,5 +1,4 @@ import datetime -import uuid from sqlalchemy import Column, String, Boolean, DateTime, Integer, Float from sqlalchemy.dialects.postgresql import UUID from app.db import Base @@ -11,50 +10,53 @@ class DataConfig(Base): # 主键 config_id = Column(Integer, primary_key=True, autoincrement=True, comment="配置ID") - + # 基本信息 config_name = Column(String, nullable=False, comment="配置名称") config_desc = Column(String, nullable=True, comment="配置描述") - + # 组织信息 workspace_id = Column(UUID(as_uuid=True), nullable=True, comment="工作空间ID") group_id = Column(String, nullable=True, comment="组ID") user_id = Column(String, nullable=True, comment="用户ID") apply_id = Column(String, nullable=True, comment="应用ID") - + # 模型选择（从workspace继承） llm_id = Column(String, nullable=True, comment="LLM模型配置ID") embedding_id = Column(String, nullable=True, comment="嵌入模型配置ID") rerank_id = Column(String, nullable=True, comment="重排序模型配置ID") llm = Column(String, nullable=True, comment="LLM模型配置ID") - + # 记忆萃取引擎配置 enable_llm_dedup_blockwise = Column(Boolean, default=True, comment="启用LLM决策去重") enable_llm_disambiguation = Column(Boolean, default=True, comment="启用LLM决策消歧") deep_retrieval = Column(Boolean, default=True, comment="深度检索开关") - + # 阈值配置 (0-1 之间的浮点数) t_type_strict = Column(Float, default=0.8, comment="类型严格阈值") t_name_strict = Column(Float, default=0.8, comment="名称严格阈值") t_overall = Column(Float, default=0.8, comment="综合阈值") - + # 状态配置 state = Column(Boolean, default=False, comment="配置使用状态") - + # 分块策略 chunker_strategy = Column(String, default="RecursiveChunker", comment="分块策略") - + # 剪枝配置 pruning_enabled = Column(Boolean, default=False, comment="是否启动智能语义剪枝") pruning_scene = Column(String, nullable=True, comment="智能剪枝场景：education/online_service/outbound") pruning_threshold = Column(Float, nullable=True, comment="智能语义剪枝阈值（0-0.9）") - + # 自我反思配置 enable_self_reflexion = Column(Boolean, default=False, comment="是否启用自我反思") iteration_period = Column(String, default="3", comment="反思迭代周期") reflexion_range = Column(String, default="retrieval", comment="反思范围：部分/全部") baseline = Column(String, default="time", comment="基线：时间/事实/时间和事实") - + reflection_model_id = Column(String, nullable=True, comment="反思模型ID") + memory_verify = Column(Boolean, default=True, comment="记忆验证") + quality_assessment = Column(Boolean, default=True, comment="质量评估") + # 遗忘引擎配置 statement_granularity = Column(Integer, default=2, comment="陈述提取颗粒度，挡位 1/2/3") include_dialogue_context = Column(Boolean, default=False, comment="是否包含对话上下文") @@ -62,7 +64,7 @@ class DataConfig(Base): lambda_time = Column("lambda_time", Float, default=0.5, comment="最低保持度，0-1 小数") lambda_mem = Column("lambda_mem", Float, default=0.5, comment="遗忘率，0-1 小数") offset = Column("offset", Float, default=0.0, comment="偏移度，0-1 小数") - + # 时间戳 created_at = Column(DateTime, default=datetime.datetime.now, comment="创建时间") updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now, comment="更新时间") diff --git a/api/app/models/end_user_model.py b/api/app/models/end_user_model.py index a2c02f84..2a9ed8da 100644 --- a/api/app/models/end_user_model.py +++ b/api/app/models/end_user_model.py @@ -14,6 +14,7 @@ class EndUser(Base): other_id = Column(String, nullable=True) # Store original user_id other_name = Column(String, default="", nullable=False) other_address = Column(String, default="", nullable=False) + reflection_time = Column(DateTime, nullable=True) created_at = Column(DateTime, default=datetime.datetime.now) updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now) diff --git a/api/app/repositories/data_config_repository.py b/api/app/repositories/data_config_repository.py index ed1a482a..6b281ef1 100644 --- a/api/app/repositories/data_config_repository.py +++ b/api/app/repositories/data_config_repository.py @@ -16,48 +16,46 @@ import uuid from app.models.data_config_model import DataConfig from app.schemas.memory_storage_schema import ( ConfigParamsCreate, - ConfigParamsDelete, ConfigUpdate, ConfigUpdateExtracted, ConfigUpdateForget, - ConfigKey, ) from app.core.logging_config import get_db_logger # 获取数据库专用日志器 db_logger = get_db_logger() - +TABLE_NAME = "data_config" class DataConfigRepository: """数据配置Repository - + 提供data_config表的数据访问方法，包括： - SQLAlchemy ORM 数据库操作 - Neo4j Cypher查询常量 """ - + # ==================== Neo4j Cypher 查询常量 ==================== - + # Dialogue count by group SEARCH_FOR_DIALOGUE = """ MATCH (n:Dialogue) WHERE n.group_id = $group_id RETURN COUNT(n) AS num """ - + # Chunk count by group SEARCH_FOR_CHUNK = """ MATCH (n:Chunk) WHERE n.group_id = $group_id RETURN COUNT(n) AS num """ - + # Statement count by group SEARCH_FOR_STATEMENT = """ MATCH (n:Statement) WHERE n.group_id = $group_id RETURN COUNT(n) AS num """ - + # ExtractedEntity count by group SEARCH_FOR_ENTITY = """ MATCH (n:ExtractedEntity) WHERE n.group_id = $group_id RETURN COUNT(n) AS num """ - + # All counts by label and total SEARCH_FOR_ALL = """ OPTIONAL MATCH (n:Dialogue) WHERE n.group_id = $group_id RETURN 'Dialogue' AS Label, COUNT(n) AS Count @@ -70,7 +68,7 @@ class DataConfigRepository: UNION ALL OPTIONAL MATCH (n) WHERE n.group_id = $group_id RETURN 'ALL' AS Label, COUNT(n) AS Count """ - + # Extracted entity details within group/app/user SEARCH_FOR_DETIALS = """ MATCH (n:ExtractedEntity) @@ -86,7 +84,7 @@ class DataConfigRepository: n.user_id AS user_id, n.id AS id """ - + # Edges between extracted entities within group/app/user SEARCH_FOR_EDGES = """ MATCH (n:ExtractedEntity)-[r]->(m:ExtractedEntity) @@ -102,7 +100,7 @@ class DataConfigRepository: r.statement_id AS statement_id, r.statement AS statement """ - + # Entity graph within group (source node, edge, target node) SEARCH_FOR_ENTITY_GRAPH = """ MATCH (n:ExtractedEntity)-[r]->(m:ExtractedEntity) @@ -135,22 +133,106 @@ class DataConfigRepository: id: m.id } AS targetNode """ - + # ==================== SQLAlchemy ORM 数据库操作方法 ==================== - + @staticmethod + def build_update_reflection(config_id: int, **kwargs) -> Tuple[str, Dict]: + """构建反思配置更新语句（SQLAlchemy text() 命名参数） + + Args: + config_id: 配置ID + **kwargs: 反思配置参数 + + Returns: + Tuple[str, Dict]: (SQL查询字符串, 参数字典) + + Raises: + ValueError: 没有字段需要更新时抛出 + """ + db_logger.debug(f"构建反思配置更新语句: config_id={config_id}") + + key_where = "config_id = :config_id" + set_fields: List[str] = [] + params: Dict = { + "config_id": config_id, + } + + # 反思配置字段映射 + mapping = { + "enable_self_reflexion": "enable_self_reflexion", + "iteration_period": "iteration_period", + "reflexion_range": "reflexion_range", + "baseline": "baseline", + "reflection_model_id": "reflection_model_id", + "memory_verify": "memory_verify", + "quality_assessment": "quality_assessment", + } + + for api_field, db_col in mapping.items(): + if api_field in kwargs and kwargs[api_field] is not None: + set_fields.append(f"{db_col} = :{api_field}") + params[api_field] = kwargs[api_field] + + if not set_fields: + raise ValueError("No fields to update") + + set_fields.append("updated_at = timezone('Asia/Shanghai', now())") + query = f"UPDATE {TABLE_NAME} SET " + ", ".join(set_fields) + f" WHERE {key_where}" + return query, params + + @staticmethod + def build_select_reflection(config_id: int) -> Tuple[str, Dict]: + """构建反思配置查询语句，通过config_id查询反思配置（SQLAlchemy text() 命名参数） + + Args: + config_id: 配置ID + + Returns: + Tuple[str, Dict]: (SQL查询字符串, 参数字典) + """ + db_logger.debug(f"构建反思配置查询语句: config_id={config_id}") + + query = ( + f"SELECT config_id, enable_self_reflexion, iteration_period, reflexion_range, baseline, " + f"reflection_model_id, memory_verify, quality_assessment, user_id " + f"FROM {TABLE_NAME} WHERE config_id = :config_id" + ) + params = {"config_id": config_id} + return query, params + + @staticmethod + def build_select_all(workspace_id: uuid.UUID) -> Tuple[str, Dict]: + """构建查询所有配置的语句（SQLAlchemy text() 命名参数） + + Args: + workspace_id: 工作空间ID + + Returns: + Tuple[str, Dict]: (SQL查询字符串, 参数字典) + """ + db_logger.debug(f"构建查询所有配置语句: workspace_id={workspace_id}") + + query = ( + f"SELECT config_id, config_name, enable_self_reflexion, iteration_period, reflexion_range, baseline, " + f"reflection_model_id, memory_verify, quality_assessment, user_id, created_at, updated_at " + f"FROM {TABLE_NAME} WHERE workspace_id = :workspace_id ORDER BY updated_at DESC" + ) + params = {"workspace_id": workspace_id} + return query, params + @staticmethod def create(db: Session, params: ConfigParamsCreate) -> DataConfig: """创建数据配置 - + Args: db: 数据库会话 params: 配置参数创建模型 - + Returns: DataConfig: 创建的配置对象 """ db_logger.debug(f"创建数据配置: config_name={params.config_name}, workspace_id={params.workspace_id}") - + try: db_config = DataConfig( config_name=params.config_name, @@ -162,37 +244,37 @@ class DataConfigRepository: ) db.add(db_config) db.flush() # 获取自增ID但不提交事务 - + db_logger.info(f"数据配置已添加到会话: {db_config.config_name} (ID: {db_config.config_id})") return db_config - + except Exception as e: db.rollback() db_logger.error(f"创建数据配置失败: {params.config_name} - {str(e)}") raise - + @staticmethod def update(db: Session, update: ConfigUpdate) -> Optional[DataConfig]: """更新基础配置 - + Args: db: 数据库会话 update: 配置更新模型 - + Returns: Optional[DataConfig]: 更新后的配置对象，不存在则返回None - + Raises: ValueError: 没有字段需要更新时抛出 """ db_logger.debug(f"更新数据配置: config_id={update.config_id}") - + try: db_config = db.query(DataConfig).filter(DataConfig.config_id == update.config_id).first() if not db_config: db_logger.warning(f"数据配置不存在: config_id={update.config_id}") return None - + # 更新字段 has_update = False if update.config_name is not None: @@ -201,44 +283,44 @@ class DataConfigRepository: if update.config_desc is not None: db_config.config_desc = update.config_desc has_update = True - + if not has_update: raise ValueError("No fields to update") - + db.commit() db.refresh(db_config) - + db_logger.info(f"数据配置更新成功: {db_config.config_name} (ID: {update.config_id})") return db_config - + except Exception as e: db.rollback() db_logger.error(f"更新数据配置失败: config_id={update.config_id} - {str(e)}") raise - + @staticmethod def update_extracted(db: Session, update: ConfigUpdateExtracted) -> Optional[DataConfig]: """更新记忆萃取引擎配置 - + Args: db: 数据库会话 update: 萃取配置更新模型 - + Returns: Optional[DataConfig]: 更新后的配置对象，不存在则返回None - + Raises: ValueError: 没有字段需要更新时抛出 """ db_logger.debug(f"更新萃取配置: config_id={update.config_id}") - + try: db_config = db.query(DataConfig).filter(DataConfig.config_id == update.config_id).first() if not db_config: db_logger.warning(f"数据配置不存在: config_id={update.config_id}") return None - + # 更新字段映射 field_mapping = { # 模型选择 @@ -268,50 +350,50 @@ class DataConfigRepository: "reflexion_range": "reflexion_range", "baseline": "baseline", } - + has_update = False for api_field, db_field in field_mapping.items(): value = getattr(update, api_field, None) if value is not None: setattr(db_config, db_field, value) has_update = True - + if not has_update: raise ValueError("No fields to update") - + db.commit() db.refresh(db_config) - + db_logger.info(f"萃取配置更新成功: config_id={update.config_id}") return db_config - + except Exception as e: db.rollback() db_logger.error(f"更新萃取配置失败: config_id={update.config_id} - {str(e)}") raise - + @staticmethod def update_forget(db: Session, update: ConfigUpdateForget) -> Optional[DataConfig]: """更新遗忘引擎配置 - + Args: db: 数据库会话 update: 遗忘配置更新模型 - + Returns: Optional[DataConfig]: 更新后的配置对象，不存在则返回None - + Raises: ValueError: 没有字段需要更新时抛出 """ db_logger.debug(f"更新遗忘配置: config_id={update.config_id}") - + try: db_config = db.query(DataConfig).filter(DataConfig.config_id == update.config_id).first() if not db_config: db_logger.warning(f"数据配置不存在: config_id={update.config_id}") return None - + # 更新字段 has_update = False if update.lambda_time is not None: @@ -323,40 +405,40 @@ class DataConfigRepository: if update.offset is not None: db_config.offset = update.offset has_update = True - + if not has_update: raise ValueError("No fields to update") - + db.commit() db.refresh(db_config) - + db_logger.info(f"遗忘配置更新成功: config_id={update.config_id}") return db_config - + except Exception as e: db.rollback() db_logger.error(f"更新遗忘配置失败: config_id={update.config_id} - {str(e)}") raise - + @staticmethod def get_extracted_config(db: Session, config_id: int) -> Optional[Dict]: """获取萃取配置，通过主键查询某条配置 - + Args: db: 数据库会话 config_id: 配置ID - + Returns: Optional[Dict]: 萃取配置字典，不存在则返回None """ db_logger.debug(f"查询萃取配置: config_id={config_id}") - + try: db_config = db.query(DataConfig).filter(DataConfig.config_id == config_id).first() if not db_config: db_logger.debug(f"萃取配置不存在: config_id={config_id}") return None - + result = { "llm_id": db_config.llm_id, "embedding_id": db_config.embedding_id, @@ -379,62 +461,62 @@ class DataConfigRepository: "reflexion_range": db_config.reflexion_range, "baseline": db_config.baseline, } - + db_logger.debug(f"萃取配置查询成功: config_id={config_id}") return result - + except Exception as e: db_logger.error(f"查询萃取配置失败: config_id={config_id} - {str(e)}") raise - + @staticmethod def get_forget_config(db: Session, config_id: int) -> Optional[Dict]: """获取遗忘配置，通过主键查询某条配置 - + Args: db: 数据库会话 config_id: 配置ID - + Returns: Optional[Dict]: 遗忘配置字典，不存在则返回None """ db_logger.debug(f"查询遗忘配置: config_id={config_id}") - + try: db_config = db.query(DataConfig).filter(DataConfig.config_id == config_id).first() if not db_config: db_logger.debug(f"遗忘配置不存在: config_id={config_id}") return None - + result = { "lambda_time": db_config.lambda_time, "lambda_mem": db_config.lambda_mem, "offset": db_config.offset, } - + db_logger.debug(f"遗忘配置查询成功: config_id={config_id}") return result - + except Exception as e: db_logger.error(f"查询遗忘配置失败: config_id={config_id} - {str(e)}") raise - + @staticmethod def get_by_id(db: Session, config_id: int) -> Optional[DataConfig]: """根据ID获取数据配置 - + Args: db: 数据库会话 config_id: 配置ID - + Returns: Optional[DataConfig]: 配置对象，不存在则返回None """ db_logger.debug(f"根据ID查询数据配置: config_id={config_id}") - + try: config = db.query(DataConfig).filter(DataConfig.config_id == config_id).first() - + if config: db_logger.debug(f"数据配置查询成功: {config.config_name} (ID: {config_id})") else: @@ -443,60 +525,60 @@ class DataConfigRepository: except Exception as e: db_logger.error(f"根据ID查询数据配置失败: config_id={config_id} - {str(e)}") raise - + @staticmethod def get_all(db: Session, workspace_id: Optional[uuid.UUID] = None) -> List[DataConfig]: """获取所有配置参数 - + Args: db: 数据库会话 workspace_id: 工作空间ID，用于过滤查询结果 - + Returns: List[DataConfig]: 配置列表 """ db_logger.debug(f"查询所有配置: workspace_id={workspace_id}") - + try: query = db.query(DataConfig) - + if workspace_id: query = query.filter(DataConfig.workspace_id == workspace_id) - + configs = query.order_by(desc(DataConfig.updated_at)).all() - + db_logger.debug(f"配置列表查询成功: 数量={len(configs)}") return configs - + except Exception as e: db_logger.error(f"查询所有配置失败: workspace_id={workspace_id} - {str(e)}") raise - + @staticmethod def delete(db: Session, config_id: int) -> bool: """删除数据配置 - + Args: db: 数据库会话 config_id: 配置ID - + Returns: bool: 删除成功返回True，配置不存在返回False """ db_logger.debug(f"删除数据配置: config_id={config_id}") - + try: db_config = db.query(DataConfig).filter(DataConfig.config_id == config_id).first() if not db_config: db_logger.warning(f"数据配置不存在: config_id={config_id}") return False - + db.delete(db_config) db.commit() - + db_logger.info(f"数据配置删除成功: config_id={config_id}") return True - + except Exception as e: db.rollback() db_logger.error(f"删除数据配置失败: config_id={config_id} - {str(e)}") diff --git a/api/app/repositories/neo4j/cypher_queries.py b/api/app/repositories/neo4j/cypher_queries.py index 7330a00f..95e2ee03 100644 --- a/api/app/repositories/neo4j/cypher_queries.py +++ b/api/app/repositories/neo4j/cypher_queries.py @@ -746,3 +746,57 @@ DETACH DELETE losing RETURN count(losing) as deleted """ + +neo4j_statement_part = ''' +MATCH (n:Statement) +WHERE n.group_id = "{}" + AND datetime(n.created_at) >= datetime() - duration('P3D') +RETURN + n.statement as statement_name, + n.id as statement_id, + n.created_at as statement_created_at + +''' +neo4j_statement_all = ''' +MATCH (n:Statement) +WHERE n.group_id = "{}" +RETURN + n.statement as statement_name, + n.id as statement_id + +''' +neo4j_query_part = """ + MATCH (n)-[r]-(m:ExtractedEntity) + WHERE n.group_id = "{}" + AND datetime(n.created_at) >= datetime() - duration('P3D') + WITH DISTINCT m + OPTIONAL MATCH (m)-[rel]-(other:ExtractedEntity) + RETURN + m.name as entity1_name, + m.description as description, + m.statement_id as statement_id, + m.created_at as created_at, + m.expired_at as expired_at, + CASE WHEN rel IS NULL THEN "NO_RELATIONSHIP" ELSE type(rel) END as relationship_type, + rel as relationship, + CASE WHEN other IS NULL THEN "ISOLATED_NODE" ELSE other.name END as entity2_name, + other as entity2 + """ +neo4j_query_all = """ + MATCH (n)-[r]-(m:ExtractedEntity) + WHERE n.group_id = "{}" + WITH DISTINCT m + OPTIONAL MATCH (m)-[rel]-(other:ExtractedEntity) + RETURN + m.name as entity1_name, + m.description as description, + m.statement_id as statement_id, + m.created_at as created_at, + m.expired_at as expired_at, + CASE WHEN rel IS NULL THEN "NO_RELATIONSHIP" ELSE type(rel) END as relationship_type, + rel as relationship, + CASE WHEN other IS NULL THEN "ISOLATED_NODE" ELSE other.name END as entity2_name, + other as entity2 + """ + + diff --git a/api/app/repositories/neo4j/neo4j_update.py b/api/app/repositories/neo4j/neo4j_update.py new file mode 100644 index 00000000..9644224c --- /dev/null +++ b/api/app/repositories/neo4j/neo4j_update.py @@ -0,0 +1,227 @@ +from app.repositories import Neo4jConnector + +neo4j_connector = Neo4jConnector() + +async def update_neo4j_data(neo4j_dict_data, update_databases): + """ + Update Neo4j data based on query criteria and update parameters + + Args: + neo4j_dict_data: find + update_databases: update + """ + try: + # 构建WHERE条件 + where_conditions = [] + params = {} + + for key, value in neo4j_dict_data.items(): + if value is not None: + param_name = f"param_{key}" + where_conditions.append(f"e.{key} = ${param_name}") + params[param_name] = value + + where_clause = " AND ".join(where_conditions) if where_conditions else "1=1" + + # 构建SET条件 + set_conditions = [] + for key, value in update_databases.items(): + if value is not None: + param_name = f"update_{key}" + set_conditions.append(f"e.{key} = ${param_name}") + params[param_name] = value + + set_clause = ", ".join(set_conditions) + + if not set_clause: + print("警告: 没有需要更新的字段") + return False + + # 构建Cypher查询 + cypher_query = f""" + MATCH (e:ExtractedEntity) + WHERE {where_clause} + SET {set_clause} + RETURN count(e) as updated_count, collect(e.name) as updated_names + """ + + print(f"\n执行Cypher查询: {cypher_query}") + print(f"参数: {params}") + + # 执行更新 + result = await neo4j_connector.execute_query(cypher_query, **params) + + if result: + updated_count = result[0].get('updated_count', 0) + updated_names = result[0].get('updated_names', []) + print(f"成功更新 {updated_count} 个节点") + if updated_names: + print(f"更新的实体名称: {updated_names}") + return updated_count > 0 + else: + return False + + except Exception as e: + print(f"更新过程中出现错误: {e}") + import traceback + traceback.print_exc() + return False + + +def map_field_names(data_dict): + mapped_dict = {} + has_name_field = False + + # 第一遍：检查是否有name相关字段 + for key, value in data_dict.items(): + if key in ['name', 'entity2.name', 'entity1.name']: + has_name_field = True + break + + print(f"字段检查: has_name_field = {has_name_field}") + + # 第二遍：根据规则映射和过滤字段 + for key, value in data_dict.items(): + if key == 'entity2.name' or key == 'entity2_name': + # 将 entity2.name 映射为 name + mapped_dict['name'] = value + print(f"字段名映射: {key} -> name") + elif key == 'entity1.name' or key == 'entity1_name': + # 将 entity1.name 映射为 name + mapped_dict['name'] = value + print(f"字段名映射: {key} -> name") + elif key == 'entity1.description': + # 将 entity1.description 映射为 description + mapped_dict['description'] = value + print(f"字段名映射: {key} -> description") + elif key == 'entity2.description': + # 将 entity2.description 映射为 description + mapped_dict['description'] = value + print(f"字段名映射: {key} -> description") + elif key == 'relationship_type': + # 跳过relationship_type字段 + print(f"字段过滤: 跳过不需要的字段 '{key}'") + continue + elif key == 'entity1_name': + if has_name_field: + # 如果有name字段，跳过entity1_name + print(f"字段过滤: 由于存在name字段，跳过 '{key}'") + continue + else: + # 如果没有name字段，保留entity1_name + mapped_dict[key] = value + print(f"字段保留: {key}") + elif key == 'entity2_name': + if has_name_field: + # 如果有name字段，跳过entity2_name + print(f"字段过滤: 由于存在name字段，跳过 '{key}'") + continue + else: + # 即使没有name字段，也不使用entity2_name（根据需求） + print(f"字段过滤: 跳过不推荐的字段 '{key}'") + continue + elif '.' not in key: + # 不包含点号的其他字段直接保留 + mapped_dict[key] = value + else: + # 其他包含点号的字段跳过并警告 + print(f"警告: 跳过不支持的嵌套字段 '{key}'") + + print(f"字段映射结果: {mapped_dict}") + return mapped_dict +async def neo4j_data(solved_data): + """ + Process the resolved data and update the Neo4j database + Args: + Solved_data: Solution Data List + Returns: + Int: Number of successfully updated records + """ + success_count = 0 + + for i in solved_data: + neo4j_dict_data = {} + update_databases = {} + results = i['results'] + for data in results: + resolved = data.get('resolved') + if not resolved: + print("跳过：resolved为None") + continue + + try: + change_list = resolved.get('change', []) + except (AttributeError, TypeError): + change_list = [] + + if change_list == []: + print("跳过：change_list为空") + continue + + if change_list and len(change_list) > 0: + change = change_list[0] + print(f"change: {change}") + field_data = change.get('field', []) + print(f"field_data: {field_data}") + print(f"field_data type: {type(field_data)}") + + # 字段名映射和过滤函数 + + + # 处理field数据，可能是字典或列表 + if isinstance(field_data, dict): + # 如果是字典，映射字段名后更新 + mapped_data = map_field_names(field_data) + update_databases.update(mapped_data) + elif isinstance(field_data, list): + # 如果是列表，遍历每个字典并更新 + for field_item in field_data: + if isinstance(field_item, dict): + mapped_item = map_field_names(field_item) + update_databases.update(mapped_item) + else: + print(f"警告: field_item不是字典: {field_item}") + else: + print(f"警告: field_data类型不支持: {type(field_data)}") + + if 'entity1_name' in data: + data['name'] = data.pop('entity1_name') + if 'entity2_name' in data: + data.pop('entity2_name', None) + + resolved_memory = resolved.get('resolved_memory', {}) + + entity2 = None + if isinstance(resolved_memory, dict): + entity2 = resolved_memory.get('entity2') + + if entity2 and isinstance(entity2, dict) and len(entity2) >= 5: + stat_id = resolved.get('original_memory_id') + # 安全地获取description + statement_id = None + if isinstance(resolved_memory, dict): + statement_id = resolved_memory.get('statement_id') + + # 只有当neo4j_dict_data中还没有statement_id时才使用original_memory_id + if statement_id and 'id' not in neo4j_dict_data: + neo4j_dict_data['id'] = stat_id + neo4j_dict_data['statement_id'] = statement_id + else: + # 处理original_memory_id，它可能是字符串或字典 + try: + for key, value in resolved_memory.items(): + if key == 'statement_id': + neo4j_dict_data['statement_id'] = value + if key == 'description': + neo4j_dict_data['description'] = value + except AttributeError: + neo4j_dict_data=[] + + print(neo4j_dict_data) + print(update_databases) + if neo4j_dict_data!=[]: + await update_neo4j_data(neo4j_dict_data, update_databases) + success_count += 1 + + return success_count + diff --git a/api/app/schemas/end_user_schema.py b/api/app/schemas/end_user_schema.py index 30dafddd..74fc4a14 100644 --- a/api/app/schemas/end_user_schema.py +++ b/api/app/schemas/end_user_schema.py @@ -13,5 +13,6 @@ class EndUser(BaseModel): other_id: Optional[str] = Field(description="第三方ID", default=None) other_name: Optional[str] = Field(description="其他名称", default="") other_address: Optional[str] = Field(description="其他地址", default="") + reflection_time: Optional[datetime.datetime] = Field(description="反思时间", default_factory=datetime.datetime.now) created_at: datetime.datetime = Field(description="创建时间", default_factory=datetime.datetime.now) updated_at: datetime.datetime = Field(description="更新时间", default_factory=datetime.datetime.now) diff --git a/api/app/schemas/memory_reflection_schemas.py b/api/app/schemas/memory_reflection_schemas.py new file mode 100644 index 00000000..9eb11c6c --- /dev/null +++ b/api/app/schemas/memory_reflection_schemas.py @@ -0,0 +1,54 @@ +from pydantic import BaseModel, Field +from typing import Optional +from enum import Enum + + +class OptimizationStrategy(str, Enum): + """优化策略枚举""" + SPEED_FIRST = "speed_first" + ACCURACY_FIRST = "accuracy_first" + BALANCED = "balanced" + + +class Memory_Reflection(BaseModel): + config_id: Optional[int] = None + reflectionenabled: bool + reflection_period_in_hours: str + reflexion_range: str + baseline: str + reflection_model_id: str + memory_verify: bool + quality_assessment: bool + + # 新增快速引擎优化参数 + optimization_strategy: Optional[OptimizationStrategy] = OptimizationStrategy.BALANCED + use_fast_model: Optional[bool] = True + enable_caching: Optional[bool] = True + enable_streaming: Optional[bool] = True + batch_size: Optional[int] = Field(default=3, ge=1, le=10) + max_concurrent: Optional[int] = Field(default=5, ge=1, le=20) + + class Config: + use_enum_values = True + + +class FastReflectionRequest(BaseModel): + """快速反思请求模型""" + reflection: Memory_Reflection + host_id: Optional[str] = "88a459f5_text02" + optimization_strategy: Optional[OptimizationStrategy] = OptimizationStrategy.BALANCED + + class Config: + use_enum_values = True + + +class ReflectionBenchmarkRequest(BaseModel): + """反思基准测试请求模型""" + reflection: Memory_Reflection + host_id: Optional[str] = "88a459f5_text02" + iterations: Optional[int] = Field(default=3, ge=1, le=10) + + class Config: + use_enum_values = True + + diff --git a/api/app/schemas/memory_storage_schema.py b/api/app/schemas/memory_storage_schema.py index 66b2e45f..ab6b0512 100644 --- a/api/app/schemas/memory_storage_schema.py +++ b/api/app/schemas/memory_storage_schema.py @@ -2,7 +2,7 @@ 所有的内容是放错误地方了，应该放在models """ -from typing import Any, Optional, List, Dict, Literal +from typing import Any, Optional, List, Dict, Literal, Union import time import uuid from pydantic import BaseModel, Field, ConfigDict, field_validator, model_validator @@ -28,25 +28,48 @@ class Write_UserInput(BaseModel): # ============================================================================ class BaseDataSchema(BaseModel): """Base schema for the data""" - id: str = Field(..., description="The unique identifier for the data entry.") - statement: str = Field(..., description="The statement text.") - group_id: str = Field(..., description="The group identifier.") - chunk_id: str = Field(..., description="The chunk identifier.") + # 保持原有必需字段为可选，以兼容不同数据源 + id: Optional[str] = Field(None, description="The unique identifier for the data entry.") + statement: Optional[str] = Field(None, description="The statement text.") + group_id: Optional[str] = Field(None, description="The group identifier.") + chunk_id: Optional[str] = Field(None, description="The chunk identifier.") created_at: str = Field(..., description="The creation timestamp in ISO 8601 format.") expired_at: Optional[str] = Field(None, description="The expiration timestamp in ISO 8601 format.") valid_at: Optional[str] = Field(None, description="The validation timestamp in ISO 8601 format.") invalid_at: Optional[str] = Field(None, description="The invalidation timestamp in ISO 8601 format.") entity_ids: List[str] = Field([], description="The list of entity identifiers.") + description: Optional[str] = Field(None, description="The description of the data entry.") + + # 新增字段以匹配实际输入数据 + entity1_name: str = Field(..., description="The first entity name.") + entity2_name: Optional[str] = Field(None, description="The second entity name.") + statement_id: str = Field(..., description="The statement identifier.") + relationship_type: str = Field(..., description="The relationship type.") + relationship: Optional[Dict[str, Any]] = Field(None, description="The relationship object.") + entity2: Optional[Dict[str, Any]] = Field(None, description="The second entity object.") + + +class QualityAssessmentSchema(BaseModel): + """Schema for memory quality assessment results.""" + score: int = Field(..., ge=0, le=100, description="Quality score percentage (0-100).") + summary: str = Field(..., description="Brief summary of data quality status, including main issues and strengths.") + + +class MemoryVerifySchema(BaseModel): + """Schema for memory privacy verification results.""" + has_privacy: bool = Field(..., description="Whether privacy information was detected.") + privacy_types: List[str] = Field([], description="List of detected privacy information types.") + summary: str = Field(..., description="Brief summary of privacy detection results.") class ConflictResultSchema(BaseModel): """Schema for the conflict result data in the reflexion_data.json file.""" - data: List[BaseDataSchema] = Field(..., description="The conflict memory data.") + data: List[BaseDataSchema] = Field(..., description="The conflict memory data. Only contains conflicting records when conflict is True.") conflict: bool = Field(..., description="Whether the memory is in conflict.") - conflict_memory: Optional[BaseDataSchema] = Field(None, description="The conflict memory data.") + quality_assessment: Optional[QualityAssessmentSchema] = Field(None, description="The quality assessment object. Contains score and summary when quality_assessment is enabled, null otherwise.") + memory_verify: Optional[MemoryVerifySchema] = Field(None, description="The memory privacy verification object. Contains privacy detection results when memory_verify is enabled, null otherwise.") @model_validator(mode="before") - @classmethod def _normalize_data(cls, v): if isinstance(v, dict): d = v.get("data") @@ -61,7 +84,6 @@ class ConflictSchema(BaseModel): conflict_memory: Optional[BaseDataSchema] = Field(None, description="The conflict memory data.") @model_validator(mode="before") - @classmethod def _normalize_data(cls, v): if isinstance(v, dict): d = v.get("data") @@ -76,21 +98,30 @@ class ReflexionSchema(BaseModel): solution: str = Field(..., description="The solution for the reflexion.") +class ChangeRecordSchema(BaseModel): + """Schema for individual change records""" + field: List[Dict[str, str]] = Field(..., description="List of field changes, each containing field name and new value.") + class ResolvedSchema(BaseModel): """Schema for the resolved memory data in the reflexion_data""" original_memory_id: Optional[str] = Field(None, description="The original memory identifier.") - resolved_memory: Optional[BaseDataSchema] = Field(None, description="The resolved memory data.") + # resolved_memory: Optional[BaseDataSchema] = Field(None, description="The resolved memory data (only contains records that need modification).") + resolved_memory: Optional[Union[BaseDataSchema, List[BaseDataSchema]]] = Field(None, description="The resolved memory data (only contains records that need modification). Can be a single record or list of records.") + change: Optional[List[ChangeRecordSchema]] = Field(None, description="List of detailed change records with IDs and field information.") +class SingleReflexionResultSchema(BaseModel): + """Schema for a single reflexion result item.""" + conflict: ConflictResultSchema = Field(..., description="The conflict result data for this specific conflict type.") + reflexion: ReflexionSchema = Field(..., description="The reflexion data for this conflict.") + resolved: Optional[ResolvedSchema] = Field(None, description="The resolved memory data for this conflict.") + type: str = Field("reflexion_result", description="The type identifier.") + class ReflexionResultSchema(BaseModel): - """Schema for the reflexion result data in the reflexion_data.json file.""" - # 模型输出中 "conflict" 为单个冲突对象（包含 data 与 conflict_memory），而非字典映射 - conflict: ConflictResultSchema = Field(..., description="The conflict result data.") - reflexion: Optional[ReflexionSchema] = Field(None, description="The reflexion data.") - resolved: Optional[ResolvedSchema] = Field(None, description="The resolved memory data.") + """Schema for the complete reflexion result data - a list of individual conflict resolutions.""" + results: List[SingleReflexionResultSchema] = Field(..., description="List of individual conflict resolution results, grouped by conflict type.") @model_validator(mode="before") - @classmethod def _normalize_resolved(cls, v): if isinstance(v, dict): conflict = v.get("conflict") diff --git a/api/app/services/memory_reflection_service.py b/api/app/services/memory_reflection_service.py new file mode 100644 index 00000000..0f8fb569 --- /dev/null +++ b/api/app/services/memory_reflection_service.py @@ -0,0 +1,397 @@ +""" +记忆反思服务 +处理反思引擎的调用和执行 +""" +from datetime import datetime +from typing import Dict, Any, Optional, Set + +from fastapi import Depends +from sqlalchemy.orm import Session +from sqlalchemy import text + +from app.db import get_db +from app.core.logging_config import get_api_logger +from app.core.memory.storage_services.reflection_engine import ReflectionConfig, ReflectionEngine +from app.core.memory.storage_services.reflection_engine.self_reflexion import ReflectionRange, ReflectionBaseline +from app.repositories.data_config_repository import DataConfigRepository +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.models.app_model import App +from app.models.app_release_model import AppRelease +from app.models.end_user_model import EndUser + +api_logger = get_api_logger() + + +class WorkspaceAppService: + """Workplace Application Service Class """ + + def __init__(self, db: Session): + self.db = db + + def get_workspace_apps_detailed(self, workspace_id: str) -> Dict[str, Any]: + """ + Get detailed information of all applications in the workspace + + Args: + Workspace_id: Workspace ID + + Returns: + Dictionary containing detailed application information + """ + apps = self.db.query(App).filter(App.workspace_id == workspace_id).all() + app_ids = [str(app.id) for app in apps] + + apps_detailed_info = [] + + for app in apps: + app_info = self._build_app_info(app) + self._process_app_releases(app, app_info) + self._process_end_users(app, app_info) + apps_detailed_info.append(app_info) + + return { + "status": "成功", + "message": f"成功查询到 {len(app_ids)} 个应用及其详细信息", + "workspace_id": str(workspace_id), + "apps_count": len(app_ids), + "app_ids": app_ids, + "apps_detailed_info": apps_detailed_info + } + + def _build_app_info(self, app: App) -> Dict[str, Any]: + """base_infomation""" + return { + "id": str(app.id), + "name": app.name, + "description": app.description, + "type": app.type, + "status": app.status, + "visibility": app.visibility, + "created_at": app.created_at.isoformat() if app.created_at else None, + "updated_at": app.updated_at.isoformat() if app.updated_at else None, + "releases": [], + "data_configs": [], + "end_users": [] + } + + def _process_app_releases(self, app: App, app_info: Dict[str, Any]) -> None: + """Process the release version and configuration information of the application""" + app_releases = self.db.query(AppRelease).filter(AppRelease.app_id == app.id).all() + + if not app_releases: + return + + processed_configs: Set[str] = set() + + for release in app_releases: + memory_content = self._extract_memory_content(release.config) + + + if memory_content and memory_content in processed_configs: + continue + + release_info = { + "app_id": str(release.app_id), + "config": memory_content + } + + + if memory_content: + processed_configs.add(memory_content) + data_config_info = self._get_data_config(memory_content) + + if data_config_info: + if not any(dc["config_id"] == data_config_info["config_id"] for dc in app_info["data_configs"]): + app_info["data_configs"].append(data_config_info) + + app_info["releases"].append(release_info) + + def _extract_memory_content(self, config: Any) -> str: + """Extract memory_comtent from config""" + if not config or not isinstance(config, dict): + return None + + memory_obj = config.get('memory') + if memory_obj and isinstance(memory_obj, dict): + return memory_obj.get('memory_content') + + return None + + def _get_data_config(self, memory_content: str) -> Dict[str, Any]: + """Retrieve data_comfig information based on memory_comtent""" + try: + data_config_query, data_config_params = DataConfigRepository.build_select_reflection(memory_content) + data_config_result = self.db.execute(text(data_config_query), data_config_params).fetchone() + if data_config_result is None: + return None + + if data_config_result: + return { + "config_id": data_config_result.config_id, + "enable_self_reflexion": data_config_result.enable_self_reflexion, + "iteration_period": data_config_result.iteration_period, + "reflexion_range": data_config_result.reflexion_range, + "baseline": data_config_result.baseline, + "reflection_model_id": data_config_result.reflection_model_id, + "memory_verify": data_config_result.memory_verify, + "quality_assessment": data_config_result.quality_assessment, + "user_id": data_config_result.user_id + } + except Exception as e: + api_logger.warning(f"查询data_config失败，memory_content: {memory_content}, 错误: {str(e)}") + + return None + + def _process_end_users(self, app: App, app_info: Dict[str, Any]) -> None: + """Processing end-user information for applications""" + end_users = self.db.query(EndUser).filter(EndUser.app_id == app.id).all() + + for end_user in end_users: + end_user_info = { + "id": str(end_user.id), + "app_id": str(end_user.app_id) + } + app_info["end_users"].append(end_user_info) + + def get_end_user_reflection_time(self, end_user_id: str) -> Optional[Any]: + """ + Read the reflection time of end users + + Args: + End_user_id: End User ID + + Returns: + Reflection time or None + """ + try: + end_user = self.db.query(EndUser).filter(EndUser.id == end_user_id).first() + if end_user: + return end_user.reflection_time + return None + except Exception as e: + api_logger.error(f"读取用户反思时间失败，end_user_id: {end_user_id}, 错误: {str(e)}") + return None + + def update_end_user_reflection_time(self, end_user_id: str) -> bool: + """ + Update the reflection time of end users to the current time + + Args: + End_user_id: End User ID + + Returns: + Is the update successful + """ + try: + from datetime import datetime + + end_user = self.db.query(EndUser).filter(EndUser.id == end_user_id).first() + if end_user: + end_user.reflection_time = datetime.now() + self.db.commit() + api_logger.info(f"成功更新用户反思时间，end_user_id: {end_user_id}") + return True + else: + api_logger.warning(f"未找到用户，end_user_id: {end_user_id}") + return False + except Exception as e: + api_logger.error(f"更新用户反思时间失败，end_user_id: {end_user_id}, 错误: {str(e)}") + self.db.rollback() + return False + + +class MemoryReflectionService: + """Memory reflection service category""" + + def __init__(self,db: Session = Depends(get_db)): + self.db=db + + + async def start_reflection_from_data(self, config_data: Dict[str, Any], end_user_id: str) -> Dict[str, Any]: + """ + Starting Reflection from Configuration Data + + Args: + config_data: Configure data dictionary, including reflective configuration information + end_user_id: end_user_id + + Returns: + Reflect on the execution results + """ + try: + config_id = config_data.get("config_id") + api_logger.info(f"从配置数据启动反思，config_id: {config_id}, end_user_id: {end_user_id}") + + + if not config_data.get("enable_self_reflexion", False): + return { + "status": "跳过", + "message": "反思引擎未启用", + "config_id": config_id, + "end_user_id": end_user_id, + "config_data": config_data + } + + + config_data_id=config_data['config_id'] + reflection_config=WorkspaceAppService(self.db)._get_data_config(config_data_id) + if reflection_config is not None and reflection_config['enable_self_reflexion']: + reflection_config= self._create_reflection_config_from_data(reflection_config) + iteration_period=reflection_config.iteration_period + workspace_service = WorkspaceAppService(self.db) + current_reflection_time = workspace_service.get_end_user_reflection_time(end_user_id) + + reflection_time = datetime.fromisoformat(str(current_reflection_time)) + + current_time = datetime.now() + time_diff = current_time - reflection_time + hours_diff = int(time_diff.total_seconds() / 3600) + if iteration_period==hours_diff or current_reflection_time is None: + api_logger.info(f"与上次的反思时间间隔为: {hours_diff} 小时") + # 3. 执行反思引擎 + reflection_results = await self._execute_reflection_engine( + reflection_config, end_user_id + ) + # 更新反思时间为当前时间 + update_success = workspace_service.update_end_user_reflection_time(end_user_id) + if update_success: + api_logger.info(f"成功更新用户 {end_user_id} 的反思时间") + else: + api_logger.error(f"更新用户 {end_user_id} 的反思时间失败") + + return { + "status": "完成", + "message": "反思引擎执行完成", + "config_id": config_id, + "end_user_id": end_user_id, + "config_data": config_data, + "reflection_results": reflection_results + } + else: + return { + "status": "等待中..", + "message": "反思引擎未开始执行执", + "config_id": config_id, + "end_user_id": end_user_id, + "config_data": config_data, + "reflection_results": '' + } + + except Exception as e: + config_id = config_data.get("config_id", "unknown") + api_logger.error(f"启动反思失败，config_id: {config_id}, end_user_id: {end_user_id}, 错误: {str(e)}") + return { + "status": "错误", + "message": f"启动反思失败: {str(e)}", + "config_id": config_id, + "end_user_id": end_user_id, + "config_data": config_data + } + + def _create_reflection_config_from_data(self, config_data: Dict[str, Any]) -> ReflectionConfig: + """Create reflective configuration objects from configuration data""" + + reflexion_range_value = config_data.get("reflexion_range") + if reflexion_range_value is None or reflexion_range_value == "": + reflexion_range_value = "partial" + reflexion_range = ReflectionRange(reflexion_range_value) + + baseline_value = config_data.get("baseline") + if baseline_value is None or baseline_value == "": + baseline_value = "TIME" + baseline = ReflectionBaseline(baseline_value) + + # iteration_period = + iteration_period = config_data.get("iteration_period", 24) + if isinstance(iteration_period, str): + try: + iteration_period = int(iteration_period) + except (ValueError, TypeError): + iteration_period = 24 # 默认24小时 + + return ReflectionConfig( + enabled=config_data.get("enable_self_reflexion", False), + iteration_period=str(iteration_period), # ReflectionConfig期望字符串 + reflexion_range=reflexion_range, + baseline=baseline, + memory_verify=config_data.get("memory_verify", False), + quality_assessment=config_data.get("quality_assessment", False), + model_id=config_data.get("reflection_model_id", "") + ) + + async def _execute_reflection_engine( + self, + reflection_config: ReflectionConfig, + user_id: str + ) -> Dict[str, Any]: + """Execute Reflection Engine""" + try: + # 创建Neo4j连接器 + connector = Neo4jConnector() + + # 创建反思引擎 + engine = ReflectionEngine( + config=reflection_config, + neo4j_connector=connector, + llm_client=reflection_config.model_id + ) + + # 执行反思 + reflection_result = await engine.execute_reflection(user_id) + + return { + "success": reflection_result.success, + "message": reflection_result.message, + "conflicts_found": reflection_result.conflicts_found, + "conflicts_resolved": reflection_result.conflicts_resolved, + "memories_updated": reflection_result.memories_updated, + "execution_time": reflection_result.execution_time, + "details": reflection_result.details + } + + except Exception as e: + api_logger.error(f"反思引擎执行失败: {str(e)}") + return { + "success": False, + "message": f"反思引擎执行失败: {str(e)}", + "conflicts_found": 0, + "conflicts_resolved": 0, + "memories_updated": 0, + "execution_time": 0.0 + } + + +class Memory_Reflection_Service: + """Memory Reflection Service - Used for calling the/reflection interface""" + + def __init__(self, db: Session): + self.db = db + self.reflection_service = MemoryReflectionService(db) + + async def start_reflection(self, config_data: Dict[str, Any], end_user_id: str) -> Dict[str, Any]: + """ + Activate the reflection function + + Args: + config_data: 配置数据，格式如下： + { + "config_id": 26, + "enable_self_reflexion": true, + "iteration_period": "6", + "reflexion_range": "partial", + "baseline": "TIME", + "reflection_model_id": "ea405fa6-c387-4d78-80ab-826d692301b3", + "memory_verify": true, + "quality_assessment": false, + "user_id": null + } + end_user_id: end_user_id，example "12a8b235-6eb1-4481-a53c-b77933b5c949" + + Returns: + """ + api_logger.info(f"Memory_Reflection_Service启动反思，config_id: {config_data.get('config_id')}, end_user_id: {end_user_id}") + + # 调用核心反思服务 + result = await self.reflection_service.start_reflection_from_data(config_data, end_user_id) + + return result \ No newline at end of file diff --git a/api/app/tasks.py b/api/app/tasks.py index 2d461cd3..39758275 100644 --- a/api/app/tasks.py +++ b/api/app/tasks.py @@ -295,26 +295,6 @@ def write_message_task(self, group_id: str, message: str, config_id: str,storage } -def reflection_engine() -> None: - """Empty function placeholder for timed background reflection. - - Intentionally left blank; replace with real reflection logic later. - """ - from app.core.memory.utils.self_reflexion_utils.self_reflexion import self_reflexion - import asyncio - - host_id = uuid.UUID("2f6ff1eb-50c7-4765-8e89-e4566be19122") - asyncio.run(self_reflexion(host_id)) - - -@celery_app.task(name="app.core.memory.agent.reflection.timer") -def reflection_timer_task() -> None: - """Periodic Celery task that invokes reflection_engine. - - Raises an exception on failure. - """ - reflection_engine() - @celery_app.task(name="app.core.memory.agent.health.check_read_service") def check_read_service_task() -> Dict[str, str]: @@ -464,4 +444,147 @@ def write_total_memory_task(workspace_id: str) -> Dict[str, Any]: "error": str(e), "workspace_id": workspace_id, "elapsed_time": elapsed_time, + } + + +@celery_app.task(name="app.tasks.workspace_reflection_task", bind=True) +def workspace_reflection_task(self) -> Dict[str, Any]: + """定时任务：每30秒运行工作空间反思功能 + + Returns: + 包含任务执行结果的字典 + """ + start_time = time.time() + + async def _run() -> Dict[str, Any]: + from app.services.memory_reflection_service import WorkspaceAppService, MemoryReflectionService + from app.models.workspace_model import Workspace + from app.core.logging_config import get_api_logger + + api_logger = get_api_logger() + db = next(get_db()) + + try: + # 获取所有工作空间 + workspaces = db.query(Workspace).all() + + if not workspaces: + return { + "status": "SUCCESS", + "message": "没有找到工作空间", + "workspace_count": 0, + "reflection_results": [] + } + + all_reflection_results = [] + + # 遍历每个工作空间 + for workspace in workspaces: + workspace_id = workspace.id + api_logger.info(f"开始处理工作空间反思，workspace_id: {workspace_id}") + + try: + reflection_service = MemoryReflectionService(db) + + # 使用服务类处理复杂查询逻辑 + service = WorkspaceAppService(db) + result = service.get_workspace_apps_detailed(str(workspace_id)) + + workspace_reflection_results = [] + + for data in result['apps_detailed_info']: + if data['data_configs'] == []: + continue + + releases = data['releases'] + data_configs = data['data_configs'] + end_users = data['end_users'] + + for base, config, user in zip(releases, data_configs, end_users): + if int(base['config']) == int(config['config_id']) and base['app_id'] == user['app_id']: + # 调用反思服务 + api_logger.info(f"为用户 {user['id']} 启动反思，config_id: {config['config_id']}") + + reflection_result = await reflection_service.start_reflection_from_data( + config_data=config, + end_user_id=user['id'] + ) + + workspace_reflection_results.append({ + "app_id": base['app_id'], + "config_id": config['config_id'], + "end_user_id": user['id'], + "reflection_result": reflection_result + }) + + all_reflection_results.append({ + "workspace_id": str(workspace_id), + "reflection_count": len(workspace_reflection_results), + "reflection_results": workspace_reflection_results + }) + + api_logger.info( + f"工作空间 {workspace_id} 反思处理完成，处理了 {len(workspace_reflection_results)} 个任务") + + except Exception as e: + api_logger.error(f"处理工作空间 {workspace_id} 反思失败: {str(e)}") + all_reflection_results.append({ + "workspace_id": str(workspace_id), + "error": str(e), + "reflection_count": 0, + "reflection_results": [] + }) + + total_reflections = sum(r.get("reflection_count", 0) for r in all_reflection_results) + + return { + "status": "SUCCESS", + "message": f"成功处理 {len(workspaces)} 个工作空间，总共 {total_reflections} 个反思任务", + "workspace_count": len(workspaces), + "total_reflections": total_reflections, + "workspace_results": all_reflection_results + } + + except Exception as e: + api_logger.error(f"工作空间反思任务执行失败: {str(e)}") + return { + "status": "FAILURE", + "error": str(e), + "workspace_count": 0, + "reflection_results": [] + } + finally: + db.close() + + try: + # 使用 nest_asyncio 来避免事件循环冲突 + try: + import nest_asyncio + nest_asyncio.apply() + except ImportError: + pass + + # 尝试获取现有事件循环，如果不存在则创建新的 + try: + loop = asyncio.get_event_loop() + if loop.is_closed(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + result = loop.run_until_complete(_run()) + elapsed_time = time.time() - start_time + result["elapsed_time"] = elapsed_time + result["task_id"] = self.request.id + + return result + except Exception as e: + elapsed_time = time.time() - start_time + return { + "status": "FAILURE", + "error": str(e), + "elapsed_time": elapsed_time, + "task_id": self.request.id } \ No newline at end of file diff --git a/api/check_code.py b/api/check_code.py new file mode 100755 index 00000000..e4634d91 --- /dev/null +++ b/api/check_code.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +""" +代码质量检查脚本 +自动检查代码中的导入错误、未使用变量、语法问题等 + +用法: + python check_code.py # 检查整个 app/ 目录 + python check_code.py file1.py file2.py # 检查指定文件 +""" + +import subprocess +import sys +from pathlib import Path + + +def run_command(cmd: list[str], description: str) -> tuple[bool, str]: + """运行命令并返回结果""" + print(f"\n{'=' * 60}") + print(f"🔍 {description}") + print(f"{'=' * 60}") + + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=False) + + output = result.stdout + result.stderr + success = result.returncode == 0 + + if success: + print(f"✅ {description} - 通过") + else: + print(f"❌ {description} - 发现问题") + if output: + print(output[:2000]) # 只显示前2000字符 + + return success, output + + except Exception as e: + print(f"❌ 执行失败: {e}") + return False, str(e) + + +def main(): + """主函数""" + # 获取命令行参数中的文件列表 + target_files = sys.argv[1:] if len(sys.argv) > 1 else None + + if target_files: + # 检查指定文件 + print(f"🚀 开始代码质量检查 (指定文件: {len(target_files)} 个)...") + target_paths = target_files + ruff_target = target_files + py_compile_files = [f for f in target_files if f.endswith('.py')] + else: + # 检查整个 app/ 目录 + print("🚀 开始代码质量检查 (整个 app/ 目录)...") + target_paths = ["app/"] + ruff_target = ["app/"] + py_compile_files = list(Path("app").rglob("*.py")) + + checks = [ + { + "cmd": ["ruff", "check"] + ruff_target + ["--output-format=concise"], + "description": "Ruff 代码检查 (导入、语法、风格)", + "auto_fix": ["ruff", "check"] + ruff_target + ["--fix", "--unsafe-fixes"], + }, + { + "cmd": ["python", "-m", "py_compile"] + [str(f) for f in py_compile_files], + "description": "Python 语法检查", + "auto_fix": None, + }, + ] + + results = [] + for check in checks: + success, output = run_command(check["cmd"], check["description"]) + results.append( + {"name": check["description"], "success": success, "output": output, "auto_fix": check.get("auto_fix")} + ) + + # 汇总报告 + print(f"\n{'=' * 60}") + print("📊 检查汇总") + print(f"{'=' * 60}") + + all_passed = True + for result in results: + status = "✅ 通过" if result["success"] else "❌ 失败" + print(f"{status} - {result['name']}") + if not result["success"]: + all_passed = False + if result["auto_fix"]: + print(f" 💡 可以运行自动修复: {' '.join(result['auto_fix'])}") + + if all_passed: + print("\n🎉 所有检查通过！") + return 0 + else: + print("\n⚠️ 发现问题，请查看上面的详细信息") + print("\n💡 快速修复命令:") + if target_files: + print(f" ruff check {' '.join(target_files)} --fix --unsafe-fixes") + else: + print(" ruff check app/ --fix --unsafe-fixes") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) From 01a5bed11cedd57b895300eafa4b1375a833c097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=B0=E6=9C=88?= Date: Fri, 19 Dec 2025 09:40:40 +0000 Subject: [PATCH 15/20] Merge #18 into develop from fix/memory_reflection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 反思优化 * fix/memory_reflection: (28 commits squashed) - 新增反思功能（功能配置接口+反思celery后台检测反思的迭代周期） - 新增反思功能（功能配置接口+反思celery后台检测反思的迭代周期） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - Merge branch develop into fix/memory_reflection (Conflict resolved online) # Conflicts: # api/app/controllers/memory_reflection_controller.py # api/app/schemas/memory_reflection_schemas.py - 反思优化 - Merge remote-tracking branch 'origin/fix/memory_reflection' into fix/memory_reflection Signed-off-by: aliyun8644380055 Reviewed-by: aliyun6762716068 Merged-by: aliyun6762716068 CR-link: https://codeup.aliyun.com/redbearai/python/redbear-mem-open/change/18 --- .../memory_reflection_controller.py | 107 +++++++++++++++--- api/app/schemas/memory_reflection_schemas.py | 4 +- 2 files changed, 94 insertions(+), 17 deletions(-) diff --git a/api/app/controllers/memory_reflection_controller.py b/api/app/controllers/memory_reflection_controller.py index 759c25c5..bd9e0e09 100644 --- a/api/app/controllers/memory_reflection_controller.py +++ b/api/app/controllers/memory_reflection_controller.py @@ -16,7 +16,7 @@ from app.repositories.neo4j.neo4j_connector import Neo4jConnector from app.services.memory_reflection_service import WorkspaceAppService, MemoryReflectionService from app.schemas.memory_reflection_schemas import Memory_Reflection - +from app.services.model_service import ModelConfigService load_dotenv() api_logger = get_api_logger() @@ -47,7 +47,7 @@ async def save_reflection_config( api_logger.info(f"用户 {current_user.username} 保存反思配置，config_id: {config_id}") update_params = { - "enable_self_reflexion": request.reflectionenabled, + "enable_self_reflexion": request.reflection_enabled, "iteration_period": request.reflection_period_in_hours, "reflexion_range": request.reflexion_range, "baseline": request.baseline, @@ -115,7 +115,7 @@ async def save_reflection_config( @router.post("/reflection") async def start_workspace_reflection( - request: dict, + config_id: int, current_user: User = Depends(get_current_user), db: Session = Depends(get_db), ) -> dict: @@ -171,30 +171,109 @@ async def start_workspace_reflection( detail=f"启动workspace反思失败: {str(e)}" ) -@router.post("/reflection/run") + +@router.get("/reflection/configs") +async def start_reflection_configs( + config_id: int, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +) -> dict: + """通过config_id查询data_config表中的反思配置信息""" + + try: + api_logger.info(f"用户 {current_user.username} 查询反思配置，config_id: {config_id}") + + # 使用DataConfigRepository查询反思配置 + select_query, select_params = DataConfigRepository.build_select_reflection(config_id) + result = db.execute(text(select_query), select_params).fetchone() + + if not result: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"未找到config_id为 {config_id} 的配置" + ) + + # 构建返回数据 + reflection_config = { + "config_id": result.config_id, + "enable_self_reflexion": result.enable_self_reflexion, + "iteration_period": result.iteration_period, + "reflexion_range": result.reflexion_range, + "baseline": result.baseline, + "reflection_model_id": result.reflection_model_id, + "memory_verify": result.memory_verify, + "quality_assessment": result.quality_assessment, + "user_id": result.user_id + } + + api_logger.info(f"成功查询反思配置，config_id: {config_id}") + + return { + "status": "成功", + "message": "反思配置查询成功", + "data": reflection_config + } + + except HTTPException: + # 重新抛出HTTP异常 + raise + except Exception as e: + api_logger.error(f"查询反思配置失败: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"查询反思配置失败: {str(e)}" + ) + +@router.get("/reflection/run") async def reflection_run( - reflection: Memory_Reflection, + config_id: int, current_user: User = Depends(get_current_user), db: Session = Depends(get_db), ) -> dict: """Activate the reflection function for all matching applications in the workspace""" + + api_logger.info(f"用户 {current_user.username} 查询反思配置，config_id: {config_id}") + + # 使用DataConfigRepository查询反思配置 + select_query, select_params = DataConfigRepository.build_select_reflection(config_id) + result = db.execute(text(select_query), select_params).fetchone() + + if not result: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"未找到config_id为 {config_id} 的配置" + ) + + api_logger.info(f"成功查询反思配置，config_id: {config_id}") + + # 验证模型ID是否存在 + model_id = result.reflection_model_id + if model_id: + try: + ModelConfigService.get_model_by_id(db=db, model_id=model_id) + api_logger.info(f"模型ID验证成功: {model_id}") + except Exception as e: + api_logger.warning(f"模型ID '{model_id}' 不存在，将使用默认模型: {str(e)}") + # 可以设置为None，让反思引擎使用默认模型 + model_id = None + config = ReflectionConfig( - enabled=reflection.reflectionenabled, - iteration_period=reflection.reflection_period_in_hours, - reflexion_range=reflection.reflexion_range, - baseline=reflection.baseline, + enabled=result.enable_self_reflexion, + iteration_period=result.iteration_period, + reflexion_range=result.reflexion_range, + baseline=result.baseline, output_example='', - memory_verify=reflection.memory_verify, - quality_assessment=reflection.quality_assessment, + memory_verify=result.memory_verify, + quality_assessment=result.quality_assessment, violation_handling_strategy="block", - model_id=reflection.reflection_model_id + model_id=model_id ) connector = Neo4jConnector() engine = ReflectionEngine( config=config, neo4j_connector=connector, - llm_client=reflection.reflection_model_id # 传入 model_id + llm_client=model_id # 传入验证后的 model_id ) result=await (engine.reflection_run()) - return result + return result \ No newline at end of file diff --git a/api/app/schemas/memory_reflection_schemas.py b/api/app/schemas/memory_reflection_schemas.py index 9eb11c6c..ada92cf2 100644 --- a/api/app/schemas/memory_reflection_schemas.py +++ b/api/app/schemas/memory_reflection_schemas.py @@ -8,11 +8,9 @@ class OptimizationStrategy(str, Enum): SPEED_FIRST = "speed_first" ACCURACY_FIRST = "accuracy_first" BALANCED = "balanced" - - class Memory_Reflection(BaseModel): config_id: Optional[int] = None - reflectionenabled: bool + reflection_enabled: bool reflection_period_in_hours: str reflexion_range: str baseline: str From 185e262db87fbb67023a234d0de98565566cb5bf Mon Sep 17 00:00:00 2001 From: Mark Date: Fri, 19 Dec 2025 18:06:49 +0800 Subject: [PATCH 16/20] [add] migration script --- api/app/models/workspace_model.py | 2 +- .../versions/f96a53af914c_202512191805.py | 36 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 api/migrations/versions/f96a53af914c_202512191805.py diff --git a/api/app/models/workspace_model.py b/api/app/models/workspace_model.py index abb5adeb..4d42ed32 100644 --- a/api/app/models/workspace_model.py +++ b/api/app/models/workspace_model.py @@ -1,7 +1,7 @@ import datetime from enum import StrEnum import uuid -from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, Boolean +from sqlalchemy import Column, String, DateTime, ForeignKey, Boolean from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import relationship from app.db import Base diff --git a/api/migrations/versions/f96a53af914c_202512191805.py b/api/migrations/versions/f96a53af914c_202512191805.py new file mode 100644 index 00000000..9c3d34b5 --- /dev/null +++ b/api/migrations/versions/f96a53af914c_202512191805.py @@ -0,0 +1,36 @@ +"""202512191805 + +Revision ID: f96a53af914c +Revises: 87a6537b4074 +Create Date: 2025-12-19 18:05:14.964454 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'f96a53af914c' +down_revision: Union[str, None] = '87a6537b4074' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('data_config', sa.Column('reflection_model_id', sa.String(), nullable=True, comment='反思模型ID')) + op.add_column('data_config', sa.Column('memory_verify', sa.Boolean(), nullable=True, comment='记忆验证')) + op.add_column('data_config', sa.Column('quality_assessment', sa.Boolean(), nullable=True, comment='质量评估')) + op.add_column('end_users', sa.Column('reflection_time', sa.DateTime(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('end_users', 'reflection_time') + op.drop_column('data_config', 'quality_assessment') + op.drop_column('data_config', 'memory_verify') + op.drop_column('data_config', 'reflection_model_id') + # ### end Alembic commands ### From 226550a62c5d1d60b989189180e8a2f295ed1cba Mon Sep 17 00:00:00 2001 From: Mark Date: Fri, 19 Dec 2025 18:21:54 +0800 Subject: [PATCH 17/20] [add] migration script --- .../versions/70e94dd4a8d1_202512191820.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 api/migrations/versions/70e94dd4a8d1_202512191820.py diff --git a/api/migrations/versions/70e94dd4a8d1_202512191820.py b/api/migrations/versions/70e94dd4a8d1_202512191820.py new file mode 100644 index 00000000..114340a5 --- /dev/null +++ b/api/migrations/versions/70e94dd4a8d1_202512191820.py @@ -0,0 +1,40 @@ +"""202512191820 + +Revision ID: 70e94dd4a8d1 +Revises: f96a53af914c +Create Date: 2025-12-19 18:20:21.998247 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '70e94dd4a8d1' +down_revision: Union[str, None] = 'f96a53af914c' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_prompt_model_config_id'), table_name='prompt_model_config') + op.drop_table('prompt_model_config') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('prompt_model_config', + sa.Column('id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('tenant_id', sa.UUID(), autoincrement=False, nullable=False, comment='Tenant ID'), + sa.Column('system_prompt', sa.TEXT(), autoincrement=False, nullable=False, comment='System Prompt'), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True, comment='Creation Time'), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True, comment='Update Time'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('prompt_model_config_tenant_id_fkey')), + sa.PrimaryKeyConstraint('id', name=op.f('prompt_model_config_pkey')) + ) + op.create_index(op.f('ix_prompt_model_config_id'), 'prompt_model_config', ['id'], unique=False) + # ### end Alembic commands ### From 1f0bb1f8afd269be2af3070274592046cdd67967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=B0=E6=9C=88?= Date: Fri, 19 Dec 2025 10:37:28 +0000 Subject: [PATCH 18/20] Merge #19 into develop from fix/memory_reflection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 统一输出 * fix/memory_reflection: (35 commits squashed) - 新增反思功能（功能配置接口+反思celery后台检测反思的迭代周期） - 新增反思功能（功能配置接口+反思celery后台检测反思的迭代周期） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - Merge branch develop into fix/memory_reflection (Conflict resolved online) # Conflicts: # api/app/controllers/memory_reflection_controller.py # api/app/schemas/memory_reflection_schemas.py - 反思优化 - Merge remote-tracking branch 'origin/fix/memory_reflection' into fix/memory_reflection - 统一输出 - 统一输出 - 统一输出 - Merge branch develop into fix/memory_reflection (Conflict resolved online) # Conflicts: # api/app/controllers/memory_reflection_controller.py - 统一输出 - Merge remote-tracking branch 'origin/fix/memory_reflection' into fix/memory_reflection - 统一输出 Signed-off-by: aliyun8644380055 Reviewed-by: aliyun6762716068 Merged-by: aliyun6762716068 CR-link: https://codeup.aliyun.com/redbearai/python/redbear-mem-open/change/19 --- .../memory_reflection_controller.py | 50 ++++++++----------- .../reflection_engine/self_reflexion.py | 5 +- 2 files changed, 23 insertions(+), 32 deletions(-) diff --git a/api/app/controllers/memory_reflection_controller.py b/api/app/controllers/memory_reflection_controller.py index bd9e0e09..8dfa6c50 100644 --- a/api/app/controllers/memory_reflection_controller.py +++ b/api/app/controllers/memory_reflection_controller.py @@ -1,4 +1,5 @@ import asyncio +import time from dotenv import load_dotenv from fastapi import APIRouter, Depends, HTTPException, status @@ -6,17 +7,17 @@ from sqlalchemy.orm import Session from sqlalchemy import text from app.core.logging_config import get_api_logger +from app.core.response_utils import success from app.core.memory.storage_services.reflection_engine.self_reflexion import ReflectionConfig, ReflectionEngine from app.dependencies import get_current_user from app.db import get_db from app.models.user_model import User from app.repositories.data_config_repository import DataConfigRepository from app.repositories.neo4j.neo4j_connector import Neo4jConnector - from app.services.memory_reflection_service import WorkspaceAppService, MemoryReflectionService - from app.schemas.memory_reflection_schemas import Memory_Reflection from app.services.model_service import ModelConfigService + load_dotenv() api_logger = get_api_logger() @@ -80,13 +81,8 @@ async def save_reflection_config( ) api_logger.info(f"成功保存反思配置到数据库，config_id: {config_id}") - - # 返回结果 - return { - "status": "成功", - "message": "反思配置已保存", - "config_id": config_id, - "database_record": { + + reflection_result={ "config_id": result.config_id, "enable_self_reflexion": result.enable_self_reflexion, "iteration_period": result.iteration_period, @@ -95,9 +91,11 @@ async def save_reflection_config( "reflection_model_id": result.reflection_model_id, "memory_verify": result.memory_verify, "quality_assessment": result.quality_assessment, - "user_id": result.user_id - } - } + "user_id": result.user_id} + + return success(data=reflection_result, msg="反思配置成功") + + except ValueError as ve: api_logger.error(f"参数错误: {str(ve)}") @@ -156,13 +154,7 @@ async def start_workspace_reflection( "reflection_result": reflection_result }) - return { - "status": "完成", - "message": f"成功处理 {len(reflection_results)} 个反思任务", - "workspace_id": str(workspace_id), - "reflection_count": len(reflection_results), - "reflection_results": reflection_results - } + return success(data=reflection_results, msg="反思配置成功") except Exception as e: api_logger.error(f"启动workspace反思失败: {str(e)}") @@ -179,7 +171,6 @@ async def start_reflection_configs( db: Session = Depends(get_db), ) -> dict: """通过config_id查询data_config表中的反思配置信息""" - try: api_logger.info(f"用户 {current_user.username} 查询反思配置，config_id: {config_id}") @@ -196,8 +187,8 @@ async def start_reflection_configs( # 构建返回数据 reflection_config = { "config_id": result.config_id, - "enable_self_reflexion": result.enable_self_reflexion, - "iteration_period": result.iteration_period, + "reflection_enabled": result.enable_self_reflexion, + "reflection_period_in_hours": result.iteration_period, "reflexion_range": result.reflexion_range, "baseline": result.baseline, "reflection_model_id": result.reflection_model_id, @@ -205,15 +196,10 @@ async def start_reflection_configs( "quality_assessment": result.quality_assessment, "user_id": result.user_id } - api_logger.info(f"成功查询反思配置，config_id: {config_id}") + return success(data=reflection_config, msg="反思配置查询成功") - return { - "status": "成功", - "message": "反思配置查询成功", - "data": reflection_config - } - + except HTTPException: # 重新抛出HTTP异常 raise @@ -276,4 +262,8 @@ async def reflection_run( ) result=await (engine.reflection_run()) - return result \ No newline at end of file + return success(data=result, msg="反思试运行") + + + + diff --git a/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py b/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py index 8f5b9bae..6ccec500 100644 --- a/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py +++ b/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py @@ -19,6 +19,7 @@ import uuid from pydantic import BaseModel +from app.core.response_utils import success from app.repositories.neo4j.cypher_queries import neo4j_query_part, neo4j_statement_part, neo4j_query_all, neo4j_statement_all from app.repositories.neo4j.neo4j_update import neo4j_data from app.repositories.neo4j.neo4j_connector import Neo4jConnector @@ -314,8 +315,8 @@ class ReflectionEngine: for result in item['results']: reflexion_data.append(result['reflexion']) result_data['reflexion_data'] = reflexion_data - execution_time = time.time() - start_time - return {"status": "SUCCESS", "message": "反思试运行", "data": result_data, "time": execution_time} + return result_data + async def extract_fields_from_json(self): """从example.json中提取source_data和databasets字段""" From 1f4524c28c3b8cd484cd8b73ce35696fbdac66d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=90=E5=8A=9B=E9=BD=90?= Date: Sat, 20 Dec 2025 07:02:46 +0000 Subject: [PATCH 19/20] Merge #21 into develop from feature/emotion-engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feature/情绪引擎 * feature/emotion-engine: (7 commits squashed) - [feature]Emotion Engine Development - [feature]Emotion Engine Development - Merge branch 'feature/emotion-engine' of codeup.aliyun.com:redbearai/python/redbear-mem-open into feature/emotion-engine - [fix]1.Fix the front-end files;2.Cache Management Deletion;3.Delete "check_code.py" - [fix]1.Fix the front-end files;2.Cache Management Deletion;3.Delete "check_code.py" - Merge branch 'feature/emotion-engine' of codeup.aliyun.com:redbearai/python/redbear-mem-open into feature/emotion-engine - [fix]fix vite.config.ts Signed-off-by: 乐力齐 Commented-by: aliyun6762716068 Commented-by: 乐力齐 Reviewed-by: aliyun6762716068 Merged-by: aliyun6762716068 CR-link: https://codeup.aliyun.com/redbearai/python/redbear-mem-open/change/21 --- api/app/controllers/__init__.py | 4 + .../controllers/emotion_config_controller.py | 207 ++++++ api/app/controllers/emotion_controller.py | 255 +++++++ .../agent/langgraph_graph/write_graph.py | 71 +- .../core/memory/agent/utils/write_tools.py | 11 + api/app/core/memory/models/emotion_models.py | 85 +++ api/app/core/memory/models/graph_models.py | 75 +- api/app/core/memory/models/message_models.py | 11 + .../deduplication/entity_dedup_llm.py | 1 - .../extraction_orchestrator.py | 173 ++++- api/app/core/memory/utils/config/overrides.py | 18 +- .../core/memory/utils/prompt/prompt_utils.py | 78 ++ .../prompt/prompts/extract_emotion.jinja2 | 57 ++ .../generate_emotion_suggestions.jinja2 | 63 ++ api/app/models/data_config_model.py | 9 +- api/app/repositories/neo4j/add_nodes.py | 8 +- api/app/repositories/neo4j/cypher_queries.py | 19 +- .../repositories/neo4j/emotion_repository.py | 246 +++++++ .../neo4j/statement_repository.py | 15 +- api/app/schemas/emotion_schema.py | 32 + api/app/services/emotion_analytics_service.py | 670 ++++++++++++++++++ api/app/services/emotion_config_service.py | 212 ++++++ .../services/emotion_extraction_service.py | 200 ++++++ 23 files changed, 2453 insertions(+), 67 deletions(-) create mode 100644 api/app/controllers/emotion_config_controller.py create mode 100644 api/app/controllers/emotion_controller.py create mode 100644 api/app/core/memory/models/emotion_models.py create mode 100644 api/app/core/memory/utils/prompt/prompts/extract_emotion.jinja2 create mode 100644 api/app/core/memory/utils/prompt/prompts/generate_emotion_suggestions.jinja2 create mode 100644 api/app/repositories/neo4j/emotion_repository.py create mode 100644 api/app/schemas/emotion_schema.py create mode 100644 api/app/services/emotion_analytics_service.py create mode 100644 api/app/services/emotion_config_service.py create mode 100644 api/app/services/emotion_extraction_service.py diff --git a/api/app/controllers/__init__.py b/api/app/controllers/__init__.py index ddf534c6..27f65b1d 100644 --- a/api/app/controllers/__init__.py +++ b/api/app/controllers/__init__.py @@ -29,6 +29,8 @@ from . import ( public_share_controller, multi_agent_controller, workflow_controller, + emotion_controller, + emotion_config_controller, prompt_optimizer_controller, ) @@ -60,6 +62,8 @@ manager_router.include_router(public_share_controller.router) # 公开路由（ manager_router.include_router(memory_dashboard_controller.router) manager_router.include_router(multi_agent_controller.router) manager_router.include_router(workflow_controller.router) +manager_router.include_router(emotion_controller.router) +manager_router.include_router(emotion_config_controller.router) manager_router.include_router(prompt_optimizer_controller.router) manager_router.include_router(memory_reflection_controller.router) __all__ = ["manager_router"] diff --git a/api/app/controllers/emotion_config_controller.py b/api/app/controllers/emotion_config_controller.py new file mode 100644 index 00000000..76450d8a --- /dev/null +++ b/api/app/controllers/emotion_config_controller.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- +"""情绪配置控制器模块 + +本模块提供情绪引擎配置管理的API端点，包括获取和更新配置。 + +Routes: + GET /memory/config/emotion - 获取情绪引擎配置 + POST /memory/config/emotion - 更新情绪引擎配置 +""" + +from fastapi import APIRouter, Depends, Query, HTTPException, status +from pydantic import BaseModel, Field +from typing import Optional +from sqlalchemy.orm import Session + +from app.core.response_utils import success +from app.dependencies import get_current_user +from app.models.user_model import User +from app.schemas.response_schema import ApiResponse +from app.services.emotion_config_service import EmotionConfigService +from app.core.logging_config import get_api_logger +from app.db import get_db + +# 获取API专用日志器 +api_logger = get_api_logger() + +router = APIRouter( + prefix="/memory/emotion", + tags=["Emotion Config"], + dependencies=[Depends(get_current_user)] # 所有路由都需要认证 +) + +class EmotionConfigQuery(BaseModel): + """情绪配置查询请求模型""" + config_id: int = Field(..., description="配置ID") + +class EmotionConfigUpdate(BaseModel): + """情绪配置更新请求模型""" + config_id: int = Field(..., description="配置ID") + emotion_enabled: bool = Field(..., description="是否启用情绪提取") + emotion_model_id: Optional[str] = Field(None, description="情绪分析专用模型ID") + emotion_extract_keywords: bool = Field(..., description="是否提取情绪关键词") + emotion_min_intensity: float = Field(..., ge=0.0, le=1.0, description="最小情绪强度阈值（0.0-1.0）") + emotion_enable_subject: bool = Field(..., description="是否启用主体分类") + +@router.get("/read_config", response_model=ApiResponse) +def get_emotion_config( + config_id: int = Query(..., description="配置ID"), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """获取情绪引擎配置 + + 查询指定配置ID的情绪相关配置字段。 + + Args: + config_id: 配置ID + + Returns: + ApiResponse: 包含情绪配置数据 + + Example Response: + { + "code": 2000, + "msg": "情绪配置获取成功", + "data": { + "config_id": 17, + "emotion_enabled": true, + "emotion_model_id": "gpt-4", + "emotion_extract_keywords": true, + "emotion_min_intensity": 0.1, + "emotion_enable_subject": true + } + } + """ + try: + api_logger.info( + f"用户 {current_user.username} 请求获取情绪配置", + extra={"config_id": config_id} + ) + + # 初始化服务 + config_service = EmotionConfigService(db) + + # 调用服务层 + data = config_service.get_emotion_config(config_id) + + api_logger.info( + "情绪配置获取成功", + extra={ + "config_id": config_id, + "emotion_enabled": data.get("emotion_enabled", False) + } + ) + + return success(data=data, msg="情绪配置获取成功") + + except ValueError as e: + api_logger.warning( + f"获取情绪配置失败: {str(e)}", + extra={"config_id": config_id} + ) + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(e) + ) + except Exception as e: + api_logger.error( + f"获取情绪配置失败: {str(e)}", + extra={"config_id": config_id}, + exc_info=True + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"获取情绪配置失败: {str(e)}" + ) + + + +@router.post("/updated_config", response_model=ApiResponse) +def update_emotion_config( + config: EmotionConfigUpdate, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """更新情绪引擎配置 + + 更新指定配置ID的情绪相关配置字段。 + + Args: + config: 配置更新数据（包含config_id） + + Returns: + ApiResponse: 包含更新后的情绪配置数据 + + Example Request: + { + "config_id": 2, + "emotion_enabled": true, + "emotion_model_id": "gpt-4", + "emotion_extract_keywords": true, + "emotion_min_intensity": 0.1, + "emotion_enable_subject": true + } + + Example Response: + { + "code": 2000, + "msg": "情绪配置更新成功", + "data": { + "config_id": 17, + "emotion_enabled": true, + "emotion_model_id": "gpt-4", + "emotion_extract_keywords": true, + "emotion_min_intensity": 0.2, + "emotion_enable_subject": true + } + } + """ + try: + api_logger.info( + f"用户 {current_user.username} 请求更新情绪配置", + extra={ + "config_id": config.config_id, + "emotion_enabled": config.emotion_enabled, + "emotion_min_intensity": config.emotion_min_intensity + } + ) + + # 初始化服务 + config_service = EmotionConfigService(db) + + # 转换为字典（排除config_id，因为它作为参数传递） + config_data = config.model_dump(exclude={'config_id'}) + + # 调用服务层 + data = config_service.update_emotion_config(config.config_id, config_data) + + api_logger.info( + "情绪配置更新成功", + extra={ + "config_id": config.config_id, + "emotion_enabled": data.get("emotion_enabled", False) + } + ) + + return success(data=data, msg="情绪配置更新成功") + + except ValueError as e: + api_logger.warning( + f"更新情绪配置失败: {str(e)}", + extra={"config_id": config.config_id} + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(e) + ) + except Exception as e: + api_logger.error( + f"更新情绪配置失败: {str(e)}", + extra={"config_id": config.config_id}, + exc_info=True + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"更新情绪配置失败: {str(e)}" + ) diff --git a/api/app/controllers/emotion_controller.py b/api/app/controllers/emotion_controller.py new file mode 100644 index 00000000..2ed00c43 --- /dev/null +++ b/api/app/controllers/emotion_controller.py @@ -0,0 +1,255 @@ +# -*- coding: utf-8 -*- +"""情绪分析控制器模块 + +本模块提供情绪分析相关的API端点，包括情绪标签、词云、健康指数和个性化建议。 + +Routes: + POST /emotion/tags - 获取情绪标签统计 + POST /emotion/wordcloud - 获取情绪词云数据 + POST /emotion/health - 获取情绪健康指数 + POST /emotion/suggestions - 获取个性化情绪建议 +""" + +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy.orm import Session + +from app.core.response_utils import success, fail +from app.core.error_codes import BizCode +from app.dependencies import get_current_user, get_db +from app.models.user_model import User +from app.schemas.response_schema import ApiResponse +from app.schemas.emotion_schema import ( + EmotionTagsRequest, + EmotionWordcloudRequest, + EmotionHealthRequest, + EmotionSuggestionsRequest +) +from app.services.emotion_analytics_service import EmotionAnalyticsService +from app.core.logging_config import get_api_logger + +# 获取API专用日志器 +api_logger = get_api_logger() + +router = APIRouter( + prefix="/memory/emotion", + tags=["Emotion Analysis"], + dependencies=[Depends(get_current_user)] # 所有路由都需要认证 +) + + +# 初始化情绪分析服务uv +emotion_service = EmotionAnalyticsService() + + + +@router.post("/tags", response_model=ApiResponse) +async def get_emotion_tags( + request: EmotionTagsRequest, + current_user: User = Depends(get_current_user), +): + + try: + api_logger.info( + f"用户 {current_user.username} 请求获取情绪标签统计", + extra={ + "group_id": request.group_id, + "emotion_type": request.emotion_type, + "start_date": request.start_date, + "end_date": request.end_date, + "limit": request.limit + } + ) + + # 调用服务层 + data = await emotion_service.get_emotion_tags( + end_user_id=request.group_id, + emotion_type=request.emotion_type, + start_date=request.start_date, + end_date=request.end_date, + limit=request.limit + ) + + api_logger.info( + "情绪标签统计获取成功", + extra={ + "group_id": request.group_id, + "total_count": data.get("total_count", 0), + "tags_count": len(data.get("tags", [])) + } + ) + + return success(data=data, msg="情绪标签获取成功") + + except Exception as e: + api_logger.error( + f"获取情绪标签统计失败: {str(e)}", + extra={"group_id": request.group_id}, + exc_info=True + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"获取情绪标签统计失败: {str(e)}" + ) + + + +@router.post("/wordcloud", response_model=ApiResponse) +async def get_emotion_wordcloud( + request: EmotionWordcloudRequest, + current_user: User = Depends(get_current_user), +): + + try: + api_logger.info( + f"用户 {current_user.username} 请求获取情绪词云数据", + extra={ + "group_id": request.group_id, + "emotion_type": request.emotion_type, + "limit": request.limit + } + ) + + # 调用服务层 + data = await emotion_service.get_emotion_wordcloud( + end_user_id=request.group_id, + emotion_type=request.emotion_type, + limit=request.limit + ) + + api_logger.info( + "情绪词云数据获取成功", + extra={ + "group_id": request.group_id, + "total_keywords": data.get("total_keywords", 0) + } + ) + + return success(data=data, msg="情绪词云获取成功") + + except Exception as e: + api_logger.error( + f"获取情绪词云数据失败: {str(e)}", + extra={"group_id": request.group_id}, + exc_info=True + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"获取情绪词云数据失败: {str(e)}" + ) + + + +@router.post("/health", response_model=ApiResponse) +async def get_emotion_health( + request: EmotionHealthRequest, + current_user: User = Depends(get_current_user), +): + + try: + # 验证时间范围参数 + if request.time_range not in ["7d", "30d", "90d"]: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="时间范围参数无效，必须是 7d、30d 或 90d" + ) + + api_logger.info( + f"用户 {current_user.username} 请求获取情绪健康指数", + extra={ + "group_id": request.group_id, + "time_range": request.time_range + } + ) + + # 调用服务层 + data = await emotion_service.calculate_emotion_health_index( + end_user_id=request.group_id, + time_range=request.time_range + ) + + api_logger.info( + "情绪健康指数获取成功", + extra={ + "group_id": request.group_id, + "health_score": data.get("health_score", 0), + "level": data.get("level", "未知") + } + ) + + return success(data=data, msg="情绪健康指数获取成功") + + except HTTPException: + raise + except Exception as e: + api_logger.error( + f"获取情绪健康指数失败: {str(e)}", + extra={"group_id": request.group_id}, + exc_info=True + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"获取情绪健康指数失败: {str(e)}" + ) + + + +@router.post("/suggestions", response_model=ApiResponse) +async def get_emotion_suggestions( + request: EmotionSuggestionsRequest, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """获取个性化情绪建议 + + Args: + request: 包含 group_id 和可选的 config_id + db: 数据库会话 + current_user: 当前用户 + + Returns: + 个性化情绪建议响应 + """ + try: + # 验证 config_id（如果提供） + config_id = request.config_id + if config_id is not None: + from app.controllers.memory_agent_controller import validate_config_id + try: + config_id = validate_config_id(config_id, db) + except ValueError as e: + return fail(BizCode.INVALID_PARAMETER, "配置ID无效", str(e)) + + api_logger.info( + f"用户 {current_user.username} 请求获取个性化情绪建议", + extra={ + "group_id": request.group_id, + "config_id": config_id + } + ) + + # 调用服务层 + data = await emotion_service.generate_emotion_suggestions( + end_user_id=request.group_id, + config_id=config_id + ) + + api_logger.info( + "个性化建议获取成功", + extra={ + "group_id": request.group_id, + "suggestions_count": len(data.get("suggestions", [])) + } + ) + + return success(data=data, msg="个性化建议获取成功") + + except Exception as e: + api_logger.error( + f"获取个性化建议失败: {str(e)}", + extra={"group_id": request.group_id}, + exc_info=True + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"获取个性化建议失败: {str(e)}" + ) diff --git a/api/app/core/memory/agent/langgraph_graph/write_graph.py b/api/app/core/memory/agent/langgraph_graph/write_graph.py index dbdc51d6..cfcc1c4a 100644 --- a/api/app/core/memory/agent/langgraph_graph/write_graph.py +++ b/api/app/core/memory/agent/langgraph_graph/write_graph.py @@ -38,14 +38,53 @@ async def make_write_graph(user_id, tools, apply_id, group_id, config_id=None): messages = state["messages"] last_message = messages[-1] - result = await data_type_tool.ainvoke({ - "context": last_message[1] if isinstance(last_message, tuple) else last_message.content - }) - result=json.loads( result) + # 调用 Data_type_differentiation 工具 + try: + raw_result = await data_type_tool.ainvoke({ + "context": last_message[1] if isinstance(last_message, tuple) else last_message.content + }) + + # MCP工具返回的是列表格式，需要提取内容 + logger.debug(f"Data_type_differentiation raw result type: {type(raw_result)}, value: {raw_result}") + + # 处理不同的返回格式 + if isinstance(raw_result, list) and len(raw_result) > 0: + # MCP工具返回格式: [{"type": "text", "text": "..."}] + result_text = raw_result[0].get("text", "{}") if isinstance(raw_result[0], dict) else str(raw_result[0]) + elif isinstance(raw_result, str): + result_text = raw_result + else: + result_text = str(raw_result) + + # 解析JSON字符串 + try: + result = json.loads(result_text) + except json.JSONDecodeError as je: + logger.error(f"Failed to parse result as JSON: {result_text}, error: {je}") + return {"messages": [AIMessage(content=json.dumps({ + "status": "error", + "message": f"Invalid JSON response from Data_type_differentiation: {str(je)}" + }))]} + + # 检查是否有错误 + if isinstance(result, dict) and result.get("type") == "error": + error_msg = result.get("message", "Unknown error in Data_type_differentiation") + logger.error(f"Data_type_differentiation 返回错误: {error_msg}") + return {"messages": [AIMessage(content=json.dumps({ + "status": "error", + "message": error_msg + }))]} + + except Exception as e: + logger.error(f"调用 Data_type_differentiation 失败: {e}", exc_info=True) + return {"messages": [AIMessage(content=json.dumps({ + "status": "error", + "message": f"Data type differentiation failed: {str(e)}" + }))]} # 调用 Data_write，传递 config_id write_params = { - "content": result["context"], + "content": result.get("context", last_message.content if hasattr(last_message, 'content') else str(last_message)), "apply_id": apply_id, "group_id": group_id, "user_id": user_id @@ -56,14 +95,22 @@ async def make_write_graph(user_id, tools, apply_id, group_id, config_id=None): write_params["config_id"] = config_id logger.debug(f"传递 config_id 到 Data_write: {config_id}") - write_result = await data_write_tool.ainvoke(write_params) + try: + write_result = await data_write_tool.ainvoke(write_params) - if isinstance(write_result, dict): - content = write_result.get("data", str(write_result)) - else: - content = str(write_result) - logger.info("写入内容: %s", content) - return {"messages": [AIMessage(content=content)]} + if isinstance(write_result, dict): + content = write_result.get("data", str(write_result)) + else: + content = str(write_result) + logger.info("写入内容: %s", content) + return {"messages": [AIMessage(content=content)]} + + except Exception as e: + logger.error(f"调用 Data_write 失败: {e}", exc_info=True) + return {"messages": [AIMessage(content=json.dumps({ + "status": "error", + "message": f"Data write failed: {str(e)}" + }))]} workflow = StateGraph(WriteState) workflow.add_node("content_input", call_model) diff --git a/api/app/core/memory/agent/utils/write_tools.py b/api/app/core/memory/agent/utils/write_tools.py index ebfbcc6c..f792ea9d 100644 --- a/api/app/core/memory/agent/utils/write_tools.py +++ b/api/app/core/memory/agent/utils/write_tools.py @@ -39,6 +39,17 @@ async def write(content: str, user_id: str, apply_id: str, group_id: str, ref_id ref_id: 参考ID，默认为 "wyl20251027" config_id: 配置ID，用于标记数据处理配置 """ + # 如果提供了config_id，重新加载配置 + if config_id: + from app.core.memory.utils.config.definitions import reload_configuration_from_database + logger.info(f"Reloading configuration for config_id: {config_id}") + config_loaded = reload_configuration_from_database(config_id) + if not config_loaded: + error_msg = f"Failed to load configuration for config_id: {config_id}" + logger.error(error_msg) + raise ValueError(error_msg) + logger.info(f"Configuration reloaded successfully for config_id: {config_id}") + logger.info("=== MemSci Knowledge Extraction Pipeline ===") logger.info(f"Using model: {config_defs.SELECTED_LLM_NAME}") logger.info(f"Using LLM ID: {config_defs.SELECTED_LLM_ID}") diff --git a/api/app/core/memory/models/emotion_models.py b/api/app/core/memory/models/emotion_models.py new file mode 100644 index 00000000..f84165a7 --- /dev/null +++ b/api/app/core/memory/models/emotion_models.py @@ -0,0 +1,85 @@ +"""Emotion extraction models for LLM structured output. + +This module contains Pydantic models for emotion extraction from statements, +designed to be used with LLM structured output capabilities. + +Classes: + EmotionExtraction: Model for emotion extraction results from statements +""" + +from pydantic import BaseModel, Field, field_validator +from typing import List, Optional + + +class EmotionExtraction(BaseModel): + """Emotion extraction result model for LLM structured output. + + This model represents the structured emotion information extracted from + a statement using LLM. It includes emotion type, intensity, keywords, + subject classification, and optional target. + + Attributes: + emotion_type: Type of emotion (joy/sadness/anger/fear/surprise/neutral) + emotion_intensity: Intensity of emotion (0.0-1.0) + emotion_keywords: List of emotion keywords from the statement (max 3) + emotion_subject: Subject of emotion (self/other/object) + emotion_target: Optional target of emotion (person or object name) + """ + + emotion_type: str = Field( + ..., + description="Emotion type: joy/sadness/anger/fear/surprise/neutral" + ) + emotion_intensity: float = Field( + ..., + ge=0.0, + le=1.0, + description="Emotion intensity from 0.0 to 1.0" + ) + emotion_keywords: List[str] = Field( + default_factory=list, + description="Emotion keywords extracted from the statement (max 3)" + ) + emotion_subject: str = Field( + ..., + description="Emotion subject: self/other/object" + ) + emotion_target: Optional[str] = Field( + None, + description="Emotion target: person or object name" + ) + + @field_validator('emotion_type') + @classmethod + def validate_emotion_type(cls, v): + """Validate emotion type is one of the valid values.""" + valid_types = ['joy', 'sadness', 'anger', 'fear', 'surprise', 'neutral'] + if v not in valid_types: + raise ValueError(f"emotion_type must be one of {valid_types}, got {v}") + return v + + @field_validator('emotion_subject') + @classmethod + def validate_emotion_subject(cls, v): + """Validate emotion subject is one of the valid values.""" + valid_subjects = ['self', 'other', 'object'] + if v not in valid_subjects: + raise ValueError(f"emotion_subject must be one of {valid_subjects}, got {v}") + return v + + @field_validator('emotion_keywords') + @classmethod + def validate_emotion_keywords(cls, v): + """Validate and limit emotion keywords to max 3 items.""" + if not isinstance(v, list): + return [] + # Limit to max 3 keywords + return v[:3] + + @field_validator('emotion_intensity') + @classmethod + def validate_emotion_intensity(cls, v): + """Validate emotion intensity is within valid range.""" + if not (0.0 <= v <= 1.0): + raise ValueError(f"emotion_intensity must be between 0.0 and 1.0, got {v}") + return v diff --git a/api/app/core/memory/models/graph_models.py b/api/app/core/memory/models/graph_models.py index 58b8271c..a8c3f7b0 100644 --- a/api/app/core/memory/models/graph_models.py +++ b/api/app/core/memory/models/graph_models.py @@ -215,24 +215,58 @@ class StatementNode(Node): Attributes: chunk_id: ID of the parent chunk this statement belongs to stmt_type: Type of the statement (from ontology) - temporal_info: Temporal information extracted from the statement statement: The actual statement text content - connect_strength: Classification of connection strength ('Strong' or 'Weak') + emotion_intensity: Optional emotion intensity (0.0-1.0) - displayed on node + emotion_target: Optional emotion target (person or object name) + emotion_subject: Optional emotion subject (self/other/object) + emotion_type: Optional emotion type (joy/sadness/anger/fear/surprise/neutral) + emotion_keywords: Optional list of emotion keywords (max 3) + temporal_info: Temporal information extracted from the statement valid_at: Optional start date of temporal validity invalid_at: Optional end date of temporal validity statement_embedding: Optional embedding vector for the statement chunk_embedding: Optional embedding vector for the parent chunk + connect_strength: Classification of connection strength ('Strong' or 'Weak') config_id: Configuration ID used to process this statement """ + # Core fields (ordered as requested) chunk_id: str = Field(..., description="ID of the parent chunk") stmt_type: str = Field(..., description="Type of the statement") - temporal_info: TemporalInfo = Field(..., description="Temporal information") statement: str = Field(..., description="The statement text content") - connect_strength: str = Field(..., description="Strong VS Weak classification of this statement") + + # Emotion fields (ordered as requested, emotion_intensity first for display) + emotion_intensity: Optional[float] = Field( + None, + ge=0.0, + le=1.0, + description="Emotion intensity: 0.0-1.0 (displayed on node)" + ) + emotion_target: Optional[str] = Field( + None, + description="Emotion target: person or object name" + ) + emotion_subject: Optional[str] = Field( + None, + description="Emotion subject: self/other/object" + ) + emotion_type: Optional[str] = Field( + None, + description="Emotion type: joy/sadness/anger/fear/surprise/neutral" + ) + emotion_keywords: Optional[List[str]] = Field( + default_factory=list, + description="Emotion keywords list, max 3 items" + ) + + # Temporal fields + temporal_info: TemporalInfo = Field(..., description="Temporal information") valid_at: Optional[datetime] = Field(None, description="Temporal validity start") invalid_at: Optional[datetime] = Field(None, description="Temporal validity end") + + # Embedding and other fields statement_embedding: Optional[List[float]] = Field(None, description="Statement embedding vector") chunk_embedding: Optional[List[float]] = Field(None, description="Chunk embedding vector") + connect_strength: str = Field(..., description="Strong VS Weak classification of this statement") config_id: Optional[int | str] = Field(None, description="Configuration ID used to process this statement (integer or string)") @field_validator('valid_at', 'invalid_at', mode='before') @@ -240,6 +274,39 @@ class StatementNode(Node): def validate_datetime(cls, v): """使用通用的历史日期解析函数""" return parse_historical_datetime(v) + + @field_validator('emotion_type', mode='before') + @classmethod + def validate_emotion_type(cls, v): + """Validate emotion type is one of the valid values""" + if v is None: + return v + valid_types = ['joy', 'sadness', 'anger', 'fear', 'surprise', 'neutral'] + if v not in valid_types: + raise ValueError(f"emotion_type must be one of {valid_types}, got {v}") + return v + + @field_validator('emotion_subject', mode='before') + @classmethod + def validate_emotion_subject(cls, v): + """Validate emotion subject is one of the valid values""" + if v is None: + return v + valid_subjects = ['self', 'other', 'object'] + if v not in valid_subjects: + raise ValueError(f"emotion_subject must be one of {valid_subjects}, got {v}") + return v + + @field_validator('emotion_keywords', mode='before') + @classmethod + def validate_emotion_keywords(cls, v): + """Validate emotion keywords list has max 3 items""" + if v is None: + return [] + if not isinstance(v, list): + return [] + # Limit to max 3 keywords + return v[:3] class ChunkNode(Node): diff --git a/api/app/core/memory/models/message_models.py b/api/app/core/memory/models/message_models.py index 192816fd..199bdd75 100644 --- a/api/app/core/memory/models/message_models.py +++ b/api/app/core/memory/models/message_models.py @@ -64,6 +64,11 @@ class Statement(BaseModel): connect_strength: Optional connection strength ('Strong' or 'Weak') temporal_validity: Optional temporal validity range triplet_extraction_info: Optional triplet extraction results + emotion_type: Optional emotion type (joy/sadness/anger/fear/surprise/neutral) + emotion_intensity: Optional emotion intensity (0.0-1.0) + emotion_keywords: Optional list of emotion keywords + emotion_subject: Optional emotion subject (self/other/object) + emotion_target: Optional emotion target (person or object name) """ id: str = Field(default_factory=lambda: uuid4().hex, description="A unique identifier for the statement.") chunk_id: str = Field(..., description="ID of the parent chunk this statement belongs to.") @@ -80,6 +85,12 @@ class Statement(BaseModel): triplet_extraction_info: Optional[TripletExtractionResponse] = Field( None, description="The triplet extraction information of the statement." ) + # Emotion fields + emotion_type: Optional[str] = Field(None, description="Emotion type: joy/sadness/anger/fear/surprise/neutral") + emotion_intensity: Optional[float] = Field(None, ge=0.0, le=1.0, description="Emotion intensity: 0.0-1.0") + emotion_keywords: Optional[List[str]] = Field(default_factory=list, description="Emotion keywords, max 3") + emotion_subject: Optional[str] = Field(None, description="Emotion subject: self/other/object") + emotion_target: Optional[str] = Field(None, description="Emotion target: person or object name") class ConversationContext(BaseModel): diff --git a/api/app/core/memory/storage_services/extraction_engine/deduplication/entity_dedup_llm.py b/api/app/core/memory/storage_services/extraction_engine/deduplication/entity_dedup_llm.py index 2c784d42..734f7b69 100644 --- a/api/app/core/memory/storage_services/extraction_engine/deduplication/entity_dedup_llm.py +++ b/api/app/core/memory/storage_services/extraction_engine/deduplication/entity_dedup_llm.py @@ -480,7 +480,6 @@ async def llm_dedup_entities_iterative_blocks( # 迭代分块并发 LLM 去重 - global_redirect: dict losing_id -> canonical_id accumulated across rounds - records: textual logs including per-round/per-block summaries and per-pair decisions """ - import asyncio import random # 初始化全局日志和全局ID映射（存储所有轮次的结果） records: List[str] = [] diff --git a/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py b/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py index e00bcf0a..91529aa9 100644 --- a/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py +++ b/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py @@ -35,7 +35,6 @@ from app.core.memory.models.graph_models import ( from app.core.memory.utils.data.ontology import TemporalInfo from app.core.memory.models.variate_config import ( ExtractionPipelineConfig, - StatementExtractionConfig, ) from app.core.memory.llm_tools.openai_client import LLMClient from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient @@ -53,7 +52,6 @@ from app.core.memory.storage_services.extraction_engine.knowledge_extraction.tem ) from app.core.memory.storage_services.extraction_engine.knowledge_extraction.embedding_generation import ( embedding_generation, - embedding_generation_all, generate_entity_embeddings_from_triplets, ) from app.core.memory.storage_services.extraction_engine.deduplication.two_stage_dedup import ( @@ -179,24 +177,12 @@ class ExtractionOrchestrator: all_statements_list.extend(chunk.statements) total_statements = len(all_statements_list) - # 🔥 陈述句提取完成后，立即发送知识抽取完成消息 - if self.progress_callback: - extraction_stats = { - "statements_count": total_statements, - "entities_count": 0, # 暂时为0，后续会更新 - "triplets_count": 0, # 暂时为0，后续会更新 - "temporal_ranges_count": 0, # 暂时为0，后续会更新 - } - await self.progress_callback("knowledge_extraction_complete", "知识抽取完成", extraction_stats) - - # 🔥 立即发送下一阶段的开始消息，让前端知道进入了创建节点和边阶段 - await self.progress_callback("creating_nodes_edges", "正在创建节点和边...") - - # 步骤 2: 并行执行三元组提取、时间信息提取和基础嵌入生成（后台静默执行） - logger.info("步骤 2/6: 并行执行三元组提取、时间信息提取和嵌入生成（后台静默执行）") + # 步骤 2: 并行执行三元组提取、时间信息提取、情绪提取和基础嵌入生成 + logger.info("步骤 2/6: 并行执行三元组提取、时间信息提取、情绪提取和嵌入生成") ( triplet_maps, temporal_maps, + emotion_maps, statement_embedding_maps, chunk_embedding_maps, dialog_embeddings, @@ -225,6 +211,7 @@ class ExtractionOrchestrator: dialog_data_list, temporal_maps, triplet_maps, + emotion_maps, statement_embedding_maps, chunk_embedding_maps, dialog_embeddings, @@ -552,9 +539,108 @@ class ExtractionOrchestrator: return temporal_maps + async def _extract_emotions( + self, dialog_data_list: List[DialogData] + ) -> List[Dict[str, Any]]: + """ + 从对话中提取情绪信息（优化版：全局陈述句级并行） + + Args: + dialog_data_list: 对话数据列表 + + Returns: + 情绪信息映射列表，每个对话对应一个字典 + """ + logger.info("开始情绪信息提取（全局陈述句级并行）") + + # 收集所有陈述句及其配置 + all_statements = [] + statement_metadata = [] # (dialog_idx, statement_id) + + # 获取第一个对话的config_id来加载配置 + config_id = None + if dialog_data_list and hasattr(dialog_data_list[0], 'config_id'): + config_id = dialog_data_list[0].config_id + + # 加载DataConfig + data_config = None + if config_id: + try: + from app.db import SessionLocal + from app.repositories.data_config_repository import DataConfigRepository + + db = SessionLocal() + try: + data_config = DataConfigRepository.get_by_id(db, config_id) + finally: + db.close() + + if data_config and not data_config.emotion_enabled: + logger.info("情绪提取已在配置中禁用，跳过情绪提取") + return [{} for _ in dialog_data_list] + + except Exception as e: + logger.warning(f"加载DataConfig失败: {e}，将跳过情绪提取") + return [{} for _ in dialog_data_list] + else: + logger.info("未找到config_id，跳过情绪提取") + return [{} for _ in dialog_data_list] + + # 如果配置未启用情绪提取，直接返回空映射 + if not data_config or not data_config.emotion_enabled: + logger.info("情绪提取未启用，跳过") + return [{} for _ in dialog_data_list] + + # 收集所有陈述句 + for d_idx, dialog in enumerate(dialog_data_list): + for chunk in dialog.chunks: + for statement in chunk.statements: + all_statements.append((statement, data_config)) + statement_metadata.append((d_idx, statement.id)) + + logger.info(f"收集到 {len(all_statements)} 个陈述句，开始全局并行提取情绪") + + # 初始化情绪提取服务 + from app.services.emotion_extraction_service import EmotionExtractionService + emotion_service = EmotionExtractionService( + llm_id=data_config.emotion_model_id if data_config.emotion_model_id else None + ) + + # 全局并行处理所有陈述句 + async def extract_for_statement(stmt_data): + statement, config = stmt_data + try: + return await emotion_service.extract_emotion(statement.statement, config) + except Exception as e: + logger.error(f"陈述句 {statement.id} 情绪提取失败: {e}") + return None + + tasks = [extract_for_statement(stmt_data) for stmt_data in all_statements] + results = await asyncio.gather(*tasks, return_exceptions=True) + + # 将结果组织成对话级别的映射 + emotion_maps = [{} for _ in dialog_data_list] + successful_extractions = 0 + + for i, result in enumerate(results): + d_idx, stmt_id = statement_metadata[i] + if isinstance(result, Exception): + logger.error(f"陈述句处理异常: {result}") + emotion_maps[d_idx][stmt_id] = None + else: + emotion_maps[d_idx][stmt_id] = result + if result is not None: + successful_extractions += 1 + + # 统计提取结果 + logger.info(f"情绪信息提取完成，共成功提取 {successful_extractions}/{len(all_statements)} 个情绪") + + return emotion_maps + async def _parallel_extract_and_embed( self, dialog_data_list: List[DialogData] ) -> Tuple[ + List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, List[float]]], @@ -562,35 +648,39 @@ class ExtractionOrchestrator: List[List[float]], ]: """ - 并行执行三元组提取、时间信息提取和基础嵌入生成 + 并行执行三元组提取、时间信息提取、情绪提取和基础嵌入生成 - 这三个任务都依赖陈述句提取的结果，但彼此独立，可以并行执行： + 这四个任务都依赖陈述句提取的结果，但彼此独立，可以并行执行： - 三元组提取：从陈述句中提取实体和关系 - 时间信息提取：从陈述句中提取时间范围 + - 情绪提取：从陈述句中提取情绪信息 - 嵌入生成：为陈述句、分块和对话生成向量（不依赖三元组） Args: dialog_data_list: 对话数据列表 Returns: - 五个列表的元组： + 六个列表的元组： - 三元组映射列表 - 时间信息映射列表 + - 情绪映射列表 - 陈述句嵌入映射列表 - 分块嵌入映射列表 - 对话嵌入列表 """ - logger.info("并行执行：三元组提取 + 时间信息提取 + 基础嵌入生成") + logger.info("并行执行：三元组提取 + 时间信息提取 + 情绪提取 + 基础嵌入生成") - # 创建三个并行任务 + # 创建四个并行任务 triplet_task = self._extract_triplets(dialog_data_list) temporal_task = self._extract_temporal(dialog_data_list) + emotion_task = self._extract_emotions(dialog_data_list) embedding_task = self._generate_basic_embeddings(dialog_data_list) # 并行执行 results = await asyncio.gather( triplet_task, temporal_task, + emotion_task, embedding_task, return_exceptions=True ) @@ -598,19 +688,21 @@ class ExtractionOrchestrator: # 解包结果 triplet_maps = results[0] if not isinstance(results[0], Exception) else [{} for _ in dialog_data_list] temporal_maps = results[1] if not isinstance(results[1], Exception) else [{} for _ in dialog_data_list] + emotion_maps = results[2] if not isinstance(results[2], Exception) else [{} for _ in dialog_data_list] - if isinstance(results[2], Exception): - logger.error(f"基础嵌入生成失败: {results[2]}") + if isinstance(results[3], Exception): + logger.error(f"基础嵌入生成失败: {results[3]}") statement_embedding_maps = [{} for _ in dialog_data_list] chunk_embedding_maps = [{} for _ in dialog_data_list] dialog_embeddings = [[] for _ in dialog_data_list] else: - statement_embedding_maps, chunk_embedding_maps, dialog_embeddings = results[2] + statement_embedding_maps, chunk_embedding_maps, dialog_embeddings = results[3] logger.info("并行任务执行完成") return ( triplet_maps, temporal_maps, + emotion_maps, statement_embedding_maps, chunk_embedding_maps, dialog_embeddings, @@ -727,6 +819,7 @@ class ExtractionOrchestrator: dialog_data_list: List[DialogData], temporal_maps: List[Dict[str, Any]], triplet_maps: List[Dict[str, Any]], + emotion_maps: List[Dict[str, Any]], statement_embedding_maps: List[Dict[str, List[float]]], chunk_embedding_maps: List[Dict[str, List[float]]], dialog_embeddings: List[List[float]], @@ -738,6 +831,7 @@ class ExtractionOrchestrator: dialog_data_list: 对话数据列表 temporal_maps: 时间信息映射列表 triplet_maps: 三元组映射列表 + emotion_maps: 情绪信息映射列表 statement_embedding_maps: 陈述句嵌入映射列表 chunk_embedding_maps: 分块嵌入映射列表 dialog_embeddings: 对话嵌入列表 @@ -752,6 +846,7 @@ class ExtractionOrchestrator: if ( len(temporal_maps) != expected_length or len(triplet_maps) != expected_length + or len(emotion_maps) != expected_length or len(statement_embedding_maps) != expected_length or len(chunk_embedding_maps) != expected_length or len(dialog_embeddings) != expected_length @@ -759,6 +854,7 @@ class ExtractionOrchestrator: logger.warning( f"数据大小不匹配 - 对话: {len(dialog_data_list)}, " f"时间映射: {len(temporal_maps)}, 三元组映射: {len(triplet_maps)}, " + f"情绪映射: {len(emotion_maps)}, " f"陈述句嵌入: {len(statement_embedding_maps)}, " f"分块嵌入: {len(chunk_embedding_maps)}, " f"对话嵌入: {len(dialog_embeddings)}" @@ -767,6 +863,7 @@ class ExtractionOrchestrator: total_statements = 0 assigned_temporal = 0 assigned_triplets = 0 + assigned_emotions = 0 assigned_statement_embeddings = 0 assigned_chunk_embeddings = 0 assigned_dialog_embeddings = 0 @@ -774,12 +871,13 @@ class ExtractionOrchestrator: # 处理每个对话 for i, dialog_data in enumerate(dialog_data_list): # 检查是否有缺失的数据 - if i >= len(temporal_maps) or i >= len(triplet_maps): + if i >= len(temporal_maps) or i >= len(triplet_maps) or i >= len(emotion_maps): logger.warning(f"对话 {dialog_data.id} 缺少提取数据，跳过赋值") continue temporal_map = temporal_maps[i] triplet_map = triplet_maps[i] + emotion_map = emotion_maps[i] statement_embedding_map = statement_embedding_maps[i] if i < len(statement_embedding_maps) else {} chunk_embedding_map = chunk_embedding_maps[i] if i < len(chunk_embedding_maps) else {} dialog_embedding = dialog_embeddings[i] if i < len(dialog_embeddings) else [] @@ -810,6 +908,18 @@ class ExtractionOrchestrator: statement.triplet_extraction_info = triplet_map[statement.id] assigned_triplets += 1 + # 赋值情绪信息 + if statement.id in emotion_map: + emotion_data = emotion_map[statement.id] + if emotion_data is not None: + # 将EmotionExtraction对象的字段赋值到Statement + statement.emotion_type = emotion_data.emotion_type + statement.emotion_intensity = emotion_data.emotion_intensity + statement.emotion_keywords = emotion_data.emotion_keywords + statement.emotion_subject = emotion_data.emotion_subject + statement.emotion_target = emotion_data.emotion_target + assigned_emotions += 1 + # 赋值陈述句嵌入 if statement.id in statement_embedding_map: statement.statement_embedding = statement_embedding_map[statement.id] @@ -818,6 +928,7 @@ class ExtractionOrchestrator: logger.info( f"数据赋值完成 - 总陈述句: {total_statements}, " f"时间信息: {assigned_temporal}, 三元组: {assigned_triplets}, " + f"情绪信息: {assigned_emotions}, " f"陈述句嵌入: {assigned_statement_embeddings}, " f"分块嵌入: {assigned_chunk_embeddings}, " f"对话嵌入: {assigned_dialog_embeddings}" @@ -927,6 +1038,12 @@ class ExtractionOrchestrator: created_at=dialog_data.created_at, expired_at=dialog_data.expired_at, config_id=dialog_data.config_id if hasattr(dialog_data, 'config_id') else None, + # Emotion fields + emotion_type=getattr(statement, 'emotion_type', None), + emotion_intensity=getattr(statement, 'emotion_intensity', None), + emotion_keywords=getattr(statement, 'emotion_keywords', None), + emotion_subject=getattr(statement, 'emotion_subject', None), + emotion_target=getattr(statement, 'emotion_target', None), ) statement_nodes.append(statement_node) @@ -1333,7 +1450,7 @@ class ExtractionOrchestrator: if match: entity1_name = match.group(1).strip() entity1_type = match.group(2) - entity2_name = match.group(3).strip() + match.group(3).strip() entity2_type = match.group(4) # 提取置信度和原因 @@ -1646,7 +1763,6 @@ async def get_chunked_dialogs( """ import json import re - import os # 加载测试数据 testdata_path = os.path.join(os.path.dirname(__file__), "../../data", "testdata.json") @@ -1822,7 +1938,6 @@ async def get_chunked_dialogs_with_preprocessing( Returns: 带 chunks 的 DialogData 列表 """ - import os print("\n=== 完整数据处理流程（包含预处理）===") if input_data_path is None: diff --git a/api/app/core/memory/utils/config/overrides.py b/api/app/core/memory/utils/config/overrides.py index e333bb29..0dd7b2d1 100644 --- a/api/app/core/memory/utils/config/overrides.py +++ b/api/app/core/memory/utils/config/overrides.py @@ -28,7 +28,6 @@ """ import os import json -import socket from typing import Optional, Dict, Any, Literal NetworkMode = Literal['internal', 'external'] @@ -105,7 +104,6 @@ def _make_pgsql_conn() -> Optional[object]: try: import psycopg2 # type: ignore - from psycopg2.extras import RealDictCursor # type: ignore port = int(port_str) if port_str else 5432 conn = psycopg2.connect( @@ -193,7 +191,7 @@ def _fetch_db_config_by_config_id(config_id: int | str) -> Optional[Dict[str, An # config_id 在数据库中是 Integer 类型，需要转换 try: config_id_int = int(config_id) - except (ValueError, TypeError) as e: + except (ValueError, TypeError): try: pass except Exception: @@ -207,7 +205,7 @@ def _fetch_db_config_by_config_id(config_id: int | str) -> Optional[Dict[str, An " statement_granularity, include_dialogue_context, max_context, " " \"offset\" AS offset, lambda_time, lambda_mem, " " pruning_enabled, pruning_scene, pruning_threshold, " - " llm_id, embedding_id " + " llm_id, embedding_id, rerank_id " "FROM data_config WHERE config_id = %s LIMIT 1" ) cur.execute(sql, (config_id_int,)) @@ -222,7 +220,7 @@ def _fetch_db_config_by_config_id(config_id: int | str) -> Optional[Dict[str, An pass return row if row else None - except Exception as e: + except Exception: pass return None finally: @@ -325,7 +323,7 @@ def _apply_overrides_from_db_row( _set_if_present(selections, tk, db_row, tk, str) # 特殊处理 UUID 字段，确保转换为字符串格式 - for uuid_field in ("llm_id", "embedding_id"): + for uuid_field in ("llm_id", "embedding_id", "rerank_id"): if uuid_field in db_row and db_row.get(uuid_field) is not None: try: value = db_row.get(uuid_field) @@ -370,7 +368,7 @@ def _apply_overrides_from_db_row( pass return runtime_cfg - except Exception as e: + except Exception: pass return runtime_cfg @@ -460,7 +458,7 @@ def apply_runtime_overrides_with_config_id( updated_cfg = _apply_overrides_from_db_row(runtime_cfg, db_row, selected_cid, "config_id") return updated_cfg, True - except Exception as e: + except Exception: pass return runtime_cfg, False @@ -570,7 +568,7 @@ def load_unified_config( try: with open(runtime_config_path, "r", encoding="utf-8") as f: runtime_cfg = json.load(f) - except (FileNotFoundError, json.JSONDecodeError) as e: + except (FileNotFoundError, json.JSONDecodeError): runtime_cfg = {"selections": {}} # 步骤 2: 尝试从 dbrun.json 读取 config_id 并应用数据库配置（最高优先级） @@ -603,7 +601,7 @@ def load_unified_config( pass return runtime_cfg - except Exception as e: + except Exception: return {"selections": {}} diff --git a/api/app/core/memory/utils/prompt/prompt_utils.py b/api/app/core/memory/utils/prompt/prompt_utils.py index 77a23e0f..c39a3f89 100644 --- a/api/app/core/memory/utils/prompt/prompt_utils.py +++ b/api/app/core/memory/utils/prompt/prompt_utils.py @@ -238,3 +238,81 @@ async def render_memory_summary_prompt( 'json_schema': 'MemorySummaryResponse.schema' }) return rendered_prompt + +async def render_emotion_extraction_prompt( + statement: str, + extract_keywords: bool, + enable_subject: bool +) -> str: + """ + Renders the emotion extraction prompt using the extract_emotion.jinja2 template. + + Args: + statement: The statement to analyze + extract_keywords: Whether to extract emotion keywords + enable_subject: Whether to enable subject classification + + Returns: + Rendered prompt content as string + """ + template = prompt_env.get_template("extract_emotion.jinja2") + rendered_prompt = template.render( + statement=statement, + extract_keywords=extract_keywords, + enable_subject=enable_subject + ) + + # 记录渲染结果到提示日志 + log_prompt_rendering('emotion extraction', rendered_prompt) + # 可选：记录模板渲染信息 + log_template_rendering('extract_emotion.jinja2', { + 'statement': 'str', + 'extract_keywords': extract_keywords, + 'enable_subject': enable_subject + }) + + return rendered_prompt + +async def render_emotion_suggestions_prompt( + health_data: dict, + patterns: dict, + user_profile: dict +) -> str: + """ + Renders the emotion suggestions generation prompt using the generate_emotion_suggestions.jinja2 template. + + Args: + health_data: 情绪健康数据 + patterns: 情绪模式分析结果 + user_profile: 用户画像数据 + + Returns: + Rendered prompt content as string + """ + import json + + # 预处理 emotion_distribution 为 JSON 字符串 + emotion_distribution_json = json.dumps( + health_data.get('emotion_distribution', {}), + ensure_ascii=False, + indent=2 + ) + + template = prompt_env.get_template("generate_emotion_suggestions.jinja2") + rendered_prompt = template.render( + health_data=health_data, + patterns=patterns, + user_profile=user_profile, + emotion_distribution_json=emotion_distribution_json + ) + + # 记录渲染结果到提示日志 + log_prompt_rendering('emotion suggestions', rendered_prompt) + # 可选：记录模板渲染信息 + log_template_rendering('generate_emotion_suggestions.jinja2', { + 'health_score': health_data.get('health_score'), + 'health_level': health_data.get('level'), + 'user_interests': user_profile.get('interests', []) + }) + + return rendered_prompt diff --git a/api/app/core/memory/utils/prompt/prompts/extract_emotion.jinja2 b/api/app/core/memory/utils/prompt/prompts/extract_emotion.jinja2 new file mode 100644 index 00000000..5e1e425f --- /dev/null +++ b/api/app/core/memory/utils/prompt/prompts/extract_emotion.jinja2 @@ -0,0 +1,57 @@ +你是一个专业的情绪分析专家。请分析以下陈述句的情绪信息。 + +陈述句：{{ statement }} + +请提取以下信息： + +1. emotion_type（情绪类型）： + - joy: 喜悦、开心、高兴、满意、愉快 + - sadness: 悲伤、难过、失落、沮丧、遗憾 + - anger: 愤怒、生气、不满、恼火、烦躁 + - fear: 恐惧、害怕、担心、焦虑、紧张 + - surprise: 惊讶、意外、震惊、吃惊 + - neutral: 中性、客观陈述、无明显情绪 + +2. emotion_intensity（情绪强度）： + - 0.0-0.3: 弱情绪 + - 0.3-0.7: 中等情绪 + - 0.7-1.0: 强情绪 + +{% if extract_keywords %} +3. emotion_keywords（情绪关键词）： + - 原句中直接表达情绪的词语 + - 最多提取3个关键词 + - 如果没有明显的情绪词，返回空列表 +{% else %} +3. emotion_keywords（情绪关键词）： + - 返回空列表 +{% endif %} + +{% if enable_subject %} +4. emotion_subject（情绪主体）： + - self: 用户本人的情绪（包含"我"、"我们"、"咱们"等第一人称） + - other: 他人的情绪（包含人名、"他/她"等第三人称） + - object: 对事物的评价（针对产品、地点、事件等） + + 注意： + - 如果同时包含多个主体，优先识别用户本人（self） + - 如果无法明确判断主体，默认为 self + +5. emotion_target（情绪对象）： + - 如果有明确的情绪对象，提取其名称 + - 如果没有明确对象，返回 null +{% else %} +4. emotion_subject（情绪主体）： + - 默认为 self + +5. emotion_target（情绪对象）： + - 返回 null +{% endif %} + +注意事项： +- 如果陈述句是客观事实陈述，无明显情绪，标记为 neutral +- 情绪强度要符合语境，不要过度解读 +- 情绪关键词要准确，不要添加原句中没有的词 +- 主体分类要准确，优先识别用户本人（self） + +请以 JSON 格式返回结果。 diff --git a/api/app/core/memory/utils/prompt/prompts/generate_emotion_suggestions.jinja2 b/api/app/core/memory/utils/prompt/prompts/generate_emotion_suggestions.jinja2 new file mode 100644 index 00000000..6a29edd9 --- /dev/null +++ b/api/app/core/memory/utils/prompt/prompts/generate_emotion_suggestions.jinja2 @@ -0,0 +1,63 @@ +你是一位专业的心理健康顾问。请根据以下用户的情绪健康数据和个人信息，生成3-5条个性化的情绪改善建议。 + +## 用户情绪健康数据 + +健康分数：{{ health_data.health_score }}/100 +健康等级：{{ health_data.level }} + +维度分析： +- 积极率：{{ health_data.dimensions.positivity_rate.score }}/100 + - 正面情绪：{{ health_data.dimensions.positivity_rate.positive_count }}次 + - 负面情绪：{{ health_data.dimensions.positivity_rate.negative_count }}次 + - 中性情绪：{{ health_data.dimensions.positivity_rate.neutral_count }}次 + +- 稳定性：{{ health_data.dimensions.stability.score }}/100 + - 标准差：{{ health_data.dimensions.stability.std_deviation }} + +- 恢复力：{{ health_data.dimensions.resilience.score }}/100 + - 恢复率：{{ health_data.dimensions.resilience.recovery_rate }} + +情绪分布： +{{ emotion_distribution_json }} + +## 情绪模式分析 + +主要负面情绪：{{ patterns.dominant_negative_emotion|default('无') }} +情绪波动性：{{ patterns.emotion_volatility|default('未知') }} +高强度情绪次数：{{ patterns.high_intensity_emotions|default([])|length }} + +## 用户兴趣 + +{{ user_profile.interests|default(['未知'])|join(', ') }} + +## 任务要求 + +请生成3-5条个性化建议，每条建议包含： +1. type: 建议类型（emotion_balance/activity_recommendation/social_connection/stress_management） +2. title: 建议标题（简短有力） +3. content: 建议内容（详细说明，50-100字） +4. priority: 优先级（high/medium/low） +5. actionable_steps: 3个可执行的具体步骤 + +同时提供一个health_summary（不超过50字），概括用户的整体情绪状态。 + +请以JSON格式返回，格式如下： +{ + "health_summary": "您的情绪健康状况...", + "suggestions": [ + { + "type": "emotion_balance", + "title": "建议标题", + "content": "建议内容...", + "priority": "high", + "actionable_steps": ["步骤1", "步骤2", "步骤3"] + } + ] +} + +注意事项： +- 建议要具体、可执行，避免空泛 +- 结合用户的兴趣爱好提供个性化建议 +- 针对主要问题（如主要负面情绪）提供针对性建议 +- 优先级要合理分配（至少1个high，1-2个medium，其余low） +- 每个建议的3个步骤要循序渐进、易于实施 diff --git a/api/app/models/data_config_model.py b/api/app/models/data_config_model.py index be43bd8d..870d46b2 100644 --- a/api/app/models/data_config_model.py +++ b/api/app/models/data_config_model.py @@ -64,7 +64,14 @@ class DataConfig(Base): lambda_time = Column("lambda_time", Float, default=0.5, comment="最低保持度，0-1 小数") lambda_mem = Column("lambda_mem", Float, default=0.5, comment="遗忘率，0-1 小数") offset = Column("offset", Float, default=0.0, comment="偏移度，0-1 小数") - + + # 情绪引擎配置 + emotion_enabled = Column(Boolean, default=True, comment="是否启用情绪提取") + emotion_model_id = Column(String, nullable=True, comment="情绪分析专用模型ID") + emotion_extract_keywords = Column(Boolean, default=True, comment="是否提取情绪关键词") + emotion_min_intensity = Column(Float, default=0.1, comment="最小情绪强度阈值") + emotion_enable_subject = Column(Boolean, default=True, comment="是否启用主体分类") + # 时间戳 created_at = Column(DateTime, default=datetime.datetime.now, comment="创建时间") updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now, comment="更新时间") diff --git a/api/app/repositories/neo4j/add_nodes.py b/api/app/repositories/neo4j/add_nodes.py index d339879f..ce4a6876 100644 --- a/api/app/repositories/neo4j/add_nodes.py +++ b/api/app/repositories/neo4j/add_nodes.py @@ -100,7 +100,13 @@ async def add_statement_nodes(statements: List[StatementNode], connector: Neo4jC # "triplets": [triplet.model_dump() for triplet in statement.triplet_extraction_info.triplets] if statement.triplet_extraction_info else [], # "entities": [entity.model_dump() for entity in statement.triplet_extraction_info.entities] if statement.triplet_extraction_info else [] # }) if statement.triplet_extraction_info else json.dumps({"triplets": [], "entities": []}), - "statement_embedding": statement.statement_embedding if statement.statement_embedding else None + "statement_embedding": statement.statement_embedding if statement.statement_embedding else None, + # 添加情绪字段处理 + "emotion_type": statement.emotion_type, + "emotion_intensity": statement.emotion_intensity, + "emotion_keywords": statement.emotion_keywords if statement.emotion_keywords else [], + "emotion_subject": statement.emotion_subject, + "emotion_target": statement.emotion_target } flattened_statements.append(flattened_statement) diff --git a/api/app/repositories/neo4j/cypher_queries.py b/api/app/repositories/neo4j/cypher_queries.py index 95e2ee03..0f6e32aa 100644 --- a/api/app/repositories/neo4j/cypher_queries.py +++ b/api/app/repositories/neo4j/cypher_queries.py @@ -20,20 +20,25 @@ UNWIND $statements AS statement MERGE (s:Statement {id: statement.id}) SET s += { id: statement.id, + run_id: statement.run_id, + chunk_id: statement.chunk_id, group_id: statement.group_id, user_id: statement.user_id, apply_id: statement.apply_id, - chunk_id: statement.chunk_id, - run_id: statement.run_id, + stmt_type: statement.stmt_type, + statement: statement.statement, + emotion_intensity: statement.emotion_intensity, + emotion_target: statement.emotion_target, + emotion_subject: statement.emotion_subject, + emotion_type: statement.emotion_type, + emotion_keywords: statement.emotion_keywords, + temporal_info: statement.temporal_info, created_at: statement.created_at, expired_at: statement.expired_at, - stmt_type: statement.stmt_type, - temporal_info: statement.temporal_info, - relevence_info: statement.relevence_info, - statement: statement.statement, valid_at: statement.valid_at, invalid_at: statement.invalid_at, - statement_embedding: statement.statement_embedding + statement_embedding: statement.statement_embedding, + relevence_info: statement.relevence_info } RETURN s.id AS uuid """ diff --git a/api/app/repositories/neo4j/emotion_repository.py b/api/app/repositories/neo4j/emotion_repository.py new file mode 100644 index 00000000..d445c8d4 --- /dev/null +++ b/api/app/repositories/neo4j/emotion_repository.py @@ -0,0 +1,246 @@ +# -*- coding: utf-8 -*- +"""情绪数据仓储模块 + +本模块提供情绪数据的查询功能，用于情绪分析和统计。 + +Classes: + EmotionRepository: 情绪数据仓储，提供情绪标签、词云、健康指数等查询方法 +""" + +from typing import List, Dict, Optional, Any +from datetime import datetime, timedelta +import json + +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.core.logging_config import get_business_logger + +logger = get_business_logger() + + +class EmotionRepository: + """情绪数据仓储 + + 提供情绪数据的查询和统计功能，包括： + - 情绪标签统计 + - 情绪词云数据 + - 时间范围内的情绪数据查询 + + Attributes: + connector: Neo4j连接器实例 + """ + + def __init__(self, connector: Neo4jConnector): + """初始化情绪数据仓储 + + Args: + connector: Neo4j连接器实例 + """ + self.connector = connector + logger.info("情绪数据仓储初始化完成") + + async def get_emotion_tags( + self, + group_id: str, + emotion_type: Optional[str] = None, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + limit: int = 10 + ) -> List[Dict[str, Any]]: + """获取情绪标签统计 + + 查询指定用户的情绪类型分布，包括计数、百分比和平均强度。 + + Args: + group_id: 用户组ID（宿主ID） + emotion_type: 可选的情绪类型过滤（joy/sadness/anger/fear/surprise/neutral） + start_date: 可选的开始日期（ISO格式字符串） + end_date: 可选的结束日期（ISO格式字符串） + limit: 返回结果的最大数量 + + Returns: + List[Dict]: 情绪标签列表，每个包含： + - emotion_type: 情绪类型 + - count: 该类型的数量 + - percentage: 占比百分比 + - avg_intensity: 平均强度 + """ + # 构建查询条件 + where_clauses = ["s.group_id = $group_id", "s.emotion_type IS NOT NULL"] + params = {"group_id": group_id, "limit": limit} + + if emotion_type: + where_clauses.append("s.emotion_type = $emotion_type") + params["emotion_type"] = emotion_type + + if start_date: + where_clauses.append("s.created_at >= $start_date") + params["start_date"] = start_date + + if end_date: + where_clauses.append("s.created_at <= $end_date") + params["end_date"] = end_date + + where_str = " AND ".join(where_clauses) + + # 优化的 Cypher 查询：使用索引，减少中间结果 + query = f""" + MATCH (s:Statement) + WHERE {where_str} + WITH s.emotion_type as emotion_type, + count(*) as count, + avg(s.emotion_intensity) as avg_intensity + WITH collect({{emotion_type: emotion_type, count: count, avg_intensity: avg_intensity}}) as results, + sum(count) as total_count + UNWIND results as result + RETURN result.emotion_type as emotion_type, + result.count as count, + toFloat(result.count) / total_count * 100 as percentage, + result.avg_intensity as avg_intensity + ORDER BY count DESC + LIMIT $limit + """ + + try: + results = await self.connector.execute_query(query, **params) + formatted_results = [ + { + "emotion_type": record["emotion_type"], + "count": record["count"], + "percentage": round(record["percentage"], 2), + "avg_intensity": round(record["avg_intensity"], 3) if record["avg_intensity"] else 0.0 + } + for record in results + ] + + return formatted_results + except Exception as e: + logger.error(f"查询情绪标签失败: {str(e)}", exc_info=True) + return [] + + async def get_emotion_wordcloud( + self, + group_id: str, + emotion_type: Optional[str] = None, + limit: int = 50 + ) -> List[Dict[str, Any]]: + """获取情绪词云数据 + + 查询情绪关键词及其频率，用于生成词云可视化。 + + Args: + group_id: 用户组ID（宿主ID） + emotion_type: 可选的情绪类型过滤 + limit: 返回关键词的最大数量 + + Returns: + List[Dict]: 关键词列表，每个包含： + - keyword: 关键词 + - frequency: 出现频率 + - emotion_type: 关联的情绪类型 + - avg_intensity: 平均强度 + """ + # 构建查询条件 + where_clauses = ["s.group_id = $group_id", "s.emotion_keywords IS NOT NULL"] + params = {"group_id": group_id, "limit": limit} + + if emotion_type: + where_clauses.append("s.emotion_type = $emotion_type") + params["emotion_type"] = emotion_type + + where_str = " AND ".join(where_clauses) + + # 优化的 Cypher 查询：使用索引，减少不必要的计算 + query = f""" + MATCH (s:Statement) + WHERE {where_str} + UNWIND s.emotion_keywords as keyword + WITH keyword, + s.emotion_type as emotion_type, + count(*) as frequency, + avg(s.emotion_intensity) as avg_intensity + WHERE keyword IS NOT NULL AND keyword <> '' + RETURN keyword, + frequency, + emotion_type, + avg_intensity + ORDER BY frequency DESC + LIMIT $limit + """ + + try: + results = await self.connector.execute_query(query, **params) + formatted_results = [ + { + "keyword": record["keyword"], + "frequency": record["frequency"], + "emotion_type": record["emotion_type"], + "avg_intensity": round(record["avg_intensity"], 3) if record["avg_intensity"] else 0.0 + } + for record in results + ] + + return formatted_results + except Exception as e: + logger.error(f"查询情绪词云失败: {str(e)}", exc_info=True) + return [] + + async def get_emotions_in_range( + self, + group_id: str, + time_range: str = "30d" + ) -> List[Dict[str, Any]]: + """获取时间范围内的情绪数据 + + 查询指定时间范围内的所有情绪数据，用于健康指数计算。 + + Args: + group_id: 用户组ID（宿主ID） + time_range: 时间范围（7d/30d/90d） + + Returns: + List[Dict]: 情绪数据列表，每个包含： + - emotion_type: 情绪类型 + - emotion_intensity: 情绪强度 + - created_at: 创建时间 + - statement_id: 陈述句ID + """ + # 解析时间范围 + days_map = {"7d": 7, "30d": 30, "90d": 90} + days = days_map.get(time_range, 30) + + # 计算起始日期（使用字符串比较，避免时区问题） + start_date = (datetime.now() - timedelta(days=days)).isoformat() + + # 优化的 Cypher 查询：使用字符串比较避免时区问题 + query = """ + MATCH (s:Statement) + WHERE s.group_id = $group_id + AND s.emotion_type IS NOT NULL + AND s.created_at >= $start_date + RETURN s.id as statement_id, + s.emotion_type as emotion_type, + s.emotion_intensity as emotion_intensity, + s.created_at as created_at + ORDER BY s.created_at ASC + """ + + try: + results = await self.connector.execute_query( + query, + group_id=group_id, + start_date=start_date + ) + formatted_results = [ + { + "statement_id": record["statement_id"], + "emotion_type": record["emotion_type"], + "emotion_intensity": record["emotion_intensity"], + "created_at": record["created_at"].isoformat() if hasattr(record["created_at"], "isoformat") else str(record["created_at"]) + } + for record in results + ] + + return formatted_results + except Exception as e: + logger.error(f"查询时间范围情绪数据失败: {str(e)}", exc_info=True) + return [] diff --git a/api/app/repositories/neo4j/statement_repository.py b/api/app/repositories/neo4j/statement_repository.py index ec2d6660..34858444 100644 --- a/api/app/repositories/neo4j/statement_repository.py +++ b/api/app/repositories/neo4j/statement_repository.py @@ -58,11 +58,22 @@ class StatementRepository(BaseNeo4jRepository[StatementNode]): n['invalid_at'] = datetime.fromisoformat(n['invalid_at']) # 处理temporal_info字段 - if isinstance(n.get('temporal_info'), dict): + if isinstance(n.get('temporal_info'), str): + # 从字符串转换为枚举值 + n['temporal_info'] = TemporalInfo(n['temporal_info']) + elif isinstance(n.get('temporal_info'), dict): n['temporal_info'] = TemporalInfo(**n['temporal_info']) elif not n.get('temporal_info'): # 如果没有temporal_info，创建一个默认的 - n['temporal_info'] = TemporalInfo() + n['temporal_info'] = TemporalInfo.STATIC + + # 处理情绪字段 - 映射 Neo4j 节点属性到 StatementNode 模型 + # 处理空值情况，确保字段存在 + n['emotion_type'] = n.get('emotion_type') + n['emotion_intensity'] = n.get('emotion_intensity') + n['emotion_keywords'] = n.get('emotion_keywords', []) + n['emotion_subject'] = n.get('emotion_subject') + n['emotion_target'] = n.get('emotion_target') return StatementNode(**n) diff --git a/api/app/schemas/emotion_schema.py b/api/app/schemas/emotion_schema.py new file mode 100644 index 00000000..9f14884d --- /dev/null +++ b/api/app/schemas/emotion_schema.py @@ -0,0 +1,32 @@ +"""情绪分析相关的请求和响应模型""" + +from typing import Optional +from pydantic import BaseModel, Field + + +class EmotionTagsRequest(BaseModel): + """获取情绪标签统计请求""" + group_id: str = Field(..., description="组ID") + emotion_type: Optional[str] = Field(None, description="情绪类型过滤（joy/sadness/anger/fear/surprise/neutral）") + start_date: Optional[str] = Field(None, description="开始日期（ISO格式，如：2024-01-01）") + end_date: Optional[str] = Field(None, description="结束日期（ISO格式，如：2024-12-31）") + limit: int = Field(10, ge=1, le=100, description="返回数量限制") + + +class EmotionWordcloudRequest(BaseModel): + """获取情绪词云数据请求""" + group_id: str = Field(..., description="组ID") + emotion_type: Optional[str] = Field(None, description="情绪类型过滤（joy/sadness/anger/fear/surprise/neutral）") + limit: int = Field(50, ge=1, le=200, description="返回词语数量") + + +class EmotionHealthRequest(BaseModel): + """获取情绪健康指数请求""" + group_id: str = Field(..., description="组ID") + time_range: str = Field("30d", description="时间范围（7d/30d/90d）") + + +class EmotionSuggestionsRequest(BaseModel): + """获取个性化情绪建议请求""" + group_id: str = Field(..., description="组ID") + config_id: Optional[int] = Field(None, description="配置ID（用于指定LLM模型）") diff --git a/api/app/services/emotion_analytics_service.py b/api/app/services/emotion_analytics_service.py new file mode 100644 index 00000000..6952256e --- /dev/null +++ b/api/app/services/emotion_analytics_service.py @@ -0,0 +1,670 @@ +# -*- coding: utf-8 -*- +"""情绪分析服务模块 + +本模块提供情绪数据的分析和统计功能，包括情绪标签、词云、健康指数计算等。 + +Classes: + EmotionAnalyticsService: 情绪分析服务，提供各种情绪分析功能 +""" + +from typing import Dict, Any, Optional, List +import statistics +import json +from pydantic import BaseModel, Field + +from app.repositories.neo4j.emotion_repository import EmotionRepository +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.core.logging_config import get_business_logger + +logger = get_business_logger() + + +class EmotionSuggestion(BaseModel): + """情绪建议模型""" + type: str = Field(..., description="建议类型：emotion_balance/activity_recommendation/social_connection/stress_management") + title: str = Field(..., description="建议标题") + content: str = Field(..., description="建议内容") + priority: str = Field(..., description="优先级：high/medium/low") + actionable_steps: List[str] = Field(..., description="可执行步骤列表（3个）") + + +class EmotionSuggestionsResponse(BaseModel): + """情绪建议响应模型""" + health_summary: str = Field(..., description="健康状态摘要（不超过50字）") + suggestions: List[EmotionSuggestion] = Field(..., description="建议列表（3-5条）") + + +class EmotionAnalyticsService: + """情绪分析服务 + + 提供情绪数据的分析和统计功能，包括： + - 情绪标签统计 + - 情绪词云数据 + - 情绪健康指数计算 + - 个性化情绪建议生成 + + Attributes: + emotion_repo: 情绪数据仓储实例 + """ + + def __init__(self): + """初始化情绪分析服务""" + connector = Neo4jConnector() + self.emotion_repo = EmotionRepository(connector) + logger.info("情绪分析服务初始化完成") + + async def get_emotion_tags( + self, + end_user_id: str, + emotion_type: Optional[str] = None, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + limit: int = 10 + ) -> Dict[str, Any]: + """获取情绪标签统计 + + 查询指定用户的情绪类型分布，包括计数、百分比和平均强度。 + + Args: + end_user_id: 宿主ID（用户组ID） + emotion_type: 可选的情绪类型过滤 + start_date: 可选的开始日期（ISO格式） + end_date: 可选的结束日期（ISO格式） + limit: 返回结果的最大数量 + + Returns: + Dict: 包含情绪标签统计的响应数据： + - tags: 情绪标签列表 + - total_count: 总情绪数量 + - time_range: 时间范围信息 + """ + try: + logger.info(f"获取情绪标签统计: user={end_user_id}, type={emotion_type}, " + f"start={start_date}, end={end_date}, limit={limit}") + + # 调用仓储层查询 + tags = await self.emotion_repo.get_emotion_tags( + group_id=end_user_id, + emotion_type=emotion_type, + start_date=start_date, + end_date=end_date, + limit=limit + ) + + # 计算总数 + total_count = sum(tag["count"] for tag in tags) + + # 构建时间范围信息 + time_range = {} + if start_date: + time_range["start_date"] = start_date + if end_date: + time_range["end_date"] = end_date + + # 格式化响应 + response = { + "tags": tags, + "total_count": total_count, + "time_range": time_range if time_range else None + } + + logger.info(f"情绪标签统计完成: total_count={total_count}, tags_count={len(tags)}") + return response + + except Exception as e: + logger.error(f"获取情绪标签统计失败: {str(e)}", exc_info=True) + raise + + async def get_emotion_wordcloud( + self, + end_user_id: str, + emotion_type: Optional[str] = None, + limit: int = 50 + ) -> Dict[str, Any]: + """获取情绪词云数据 + + 查询情绪关键词及其频率，用于生成词云可视化。 + + Args: + end_user_id: 宿主ID（用户组ID） + emotion_type: 可选的情绪类型过滤 + limit: 返回关键词的最大数量 + + Returns: + Dict: 包含情绪词云数据的响应： + - keywords: 关键词列表 + - total_keywords: 总关键词数量 + """ + try: + logger.info(f"获取情绪词云数据: user={end_user_id}, type={emotion_type}, limit={limit}") + + # 调用仓储层查询 + keywords = await self.emotion_repo.get_emotion_wordcloud( + group_id=end_user_id, + emotion_type=emotion_type, + limit=limit + ) + + # 计算总关键词数量 + total_keywords = len(keywords) + + # 格式化响应 + response = { + "keywords": keywords, + "total_keywords": total_keywords + } + + logger.info(f"情绪词云数据获取完成: total_keywords={total_keywords}") + return response + + except Exception as e: + logger.error(f"获取情绪词云数据失败: {str(e)}", exc_info=True) + raise + + def _calculate_positivity_rate(self, emotions: List[Dict[str, Any]]) -> Dict[str, Any]: + """计算积极率 + + 根据情绪类型分类正面、负面和中性情绪，计算积极率。 + 公式：(正面数 / (正面数 + 负面数)) * 100 + + Args: + emotions: 情绪数据列表，每个包含 emotion_type 字段 + + Returns: + Dict: 包含积极率计算结果： + - score: 积极率分数（0-100） + - positive_count: 正面情绪数量 + - negative_count: 负面情绪数量 + - neutral_count: 中性情绪数量 + """ + # 定义情绪分类 + positive_emotions = {'joy', 'surprise'} + negative_emotions = {'sadness', 'anger', 'fear'} + + # 统计各类情绪数量 + positive_count = sum(1 for e in emotions if e.get('emotion_type') in positive_emotions) + negative_count = sum(1 for e in emotions if e.get('emotion_type') in negative_emotions) + neutral_count = sum(1 for e in emotions if e.get('emotion_type') == 'neutral') + + # 计算积极率 + total_non_neutral = positive_count + negative_count + if total_non_neutral > 0: + score = (positive_count / total_non_neutral) * 100 + else: + score = 50.0 # 如果没有非中性情绪，默认为50 + + logger.debug(f"积极率计算: positive={positive_count}, negative={negative_count}, " + f"neutral={neutral_count}, score={score:.2f}") + + return { + "score": round(score, 2), + "positive_count": positive_count, + "negative_count": negative_count, + "neutral_count": neutral_count + } + + def _calculate_stability(self, emotions: List[Dict[str, Any]]) -> Dict[str, Any]: + """计算稳定性 + + 基于情绪强度的标准差计算情绪稳定性。 + 公式：(1 - min(std_deviation, 1.0)) * 100 + + Args: + emotions: 情绪数据列表，每个包含 emotion_intensity 字段 + + Returns: + Dict: 包含稳定性计算结果： + - score: 稳定性分数（0-100） + - std_deviation: 标准差 + """ + # 提取所有情绪强度 + intensities = [e.get('emotion_intensity', 0.0) for e in emotions if e.get('emotion_intensity') is not None] + + # 计算标准差 + if len(intensities) >= 2: + std_deviation = statistics.stdev(intensities) + elif len(intensities) == 1: + std_deviation = 0.0 # 只有一个数据点，标准差为0 + else: + std_deviation = 0.0 # 没有数据，标准差为0 + + # 计算稳定性分数 + # 标准差越小，稳定性越高 + score = (1 - min(std_deviation, 1.0)) * 100 + + logger.debug(f"稳定性计算: intensities_count={len(intensities)}, " + f"std_deviation={std_deviation:.3f}, score={score:.2f}") + + return { + "score": round(score, 2), + "std_deviation": round(std_deviation, 3) + } + + def _calculate_resilience(self, emotions: List[Dict[str, Any]]) -> Dict[str, Any]: + """计算恢复力 + + 分析情绪转换模式，统计从负面情绪恢复到正面情绪的能力。 + 公式：(负面到正面转换次数 / 总负面情绪数) * 100 + + Args: + emotions: 情绪数据列表，每个包含 emotion_type 和 created_at 字段 + 应该按时间顺序排列 + + Returns: + Dict: 包含恢复力计算结果： + - score: 恢复力分数（0-100） + - recovery_rate: 恢复率（转换次数/负面情绪数） + """ + # 定义情绪分类 + positive_emotions = {'joy', 'surprise'} + negative_emotions = {'sadness', 'anger', 'fear'} + + # 统计负面到正面的转换次数 + recovery_count = 0 + negative_count = 0 + + for i in range(len(emotions)): + current_emotion = emotions[i].get('emotion_type') + + # 统计负面情绪总数 + if current_emotion in negative_emotions: + negative_count += 1 + + # 检查下一个情绪是否为正面 + if i + 1 < len(emotions): + next_emotion = emotions[i + 1].get('emotion_type') + if next_emotion in positive_emotions: + recovery_count += 1 + + # 计算恢复力分数 + if negative_count > 0: + recovery_rate = recovery_count / negative_count + score = recovery_rate * 100 + else: + # 如果没有负面情绪，恢复力设为100（最佳状态） + recovery_rate = 1.0 + score = 100.0 + + logger.debug(f"恢复力计算: negative_count={negative_count}, " + f"recovery_count={recovery_count}, score={score:.2f}") + + return { + "score": round(score, 2), + "recovery_rate": round(recovery_rate, 3) + } + + async def calculate_emotion_health_index( + self, + end_user_id: str, + time_range: str = "30d" + ) -> Dict[str, Any]: + """计算情绪健康指数 + + 综合积极率、稳定性和恢复力计算情绪健康指数。 + + Args: + end_user_id: 宿主ID（用户组ID） + time_range: 时间范围（7d/30d/90d） + + Returns: + Dict: 包含情绪健康指数的完整响应： + - health_score: 综合健康分数（0-100） + - level: 健康等级（优秀/良好/一般/较差） + - dimensions: 各维度详细数据 + - positivity_rate: 积极率 + - stability: 稳定性 + - resilience: 恢复力 + - emotion_distribution: 情绪分布统计 + - time_range: 时间范围 + """ + try: + logger.info(f"计算情绪健康指数: user={end_user_id}, time_range={time_range}") + + # 获取时间范围内的情绪数据 + emotions = await self.emotion_repo.get_emotions_in_range( + group_id=end_user_id, + time_range=time_range + ) + + # 如果没有数据，返回默认值 + if not emotions: + logger.warning(f"用户 {end_user_id} 在时间范围 {time_range} 内没有情绪数据") + return { + "health_score": 0.0, + "level": "无数据", + "dimensions": { + "positivity_rate": {"score": 0.0, "positive_count": 0, "negative_count": 0, "neutral_count": 0}, + "stability": {"score": 0.0, "std_deviation": 0.0}, + "resilience": {"score": 0.0, "recovery_rate": 0.0} + }, + "emotion_distribution": {}, + "time_range": time_range + } + + # 计算各维度指标 + positivity_rate = self._calculate_positivity_rate(emotions) + stability = self._calculate_stability(emotions) + resilience = self._calculate_resilience(emotions) + + # 计算综合健康分数 + # 公式：positivity_rate * 0.4 + stability * 0.3 + resilience * 0.3 + health_score = ( + positivity_rate["score"] * 0.4 + + stability["score"] * 0.3 + + resilience["score"] * 0.3 + ) + + # 确定健康等级 + if health_score >= 80: + level = "优秀" + elif health_score >= 60: + level = "良好" + elif health_score >= 40: + level = "一般" + else: + level = "较差" + + # 统计情绪分布 + emotion_distribution = {} + for emotion_type in ['joy', 'sadness', 'anger', 'fear', 'surprise', 'neutral']: + count = sum(1 for e in emotions if e.get('emotion_type') == emotion_type) + emotion_distribution[emotion_type] = count + + # 格式化响应 + response = { + "health_score": round(health_score, 2), + "level": level, + "dimensions": { + "positivity_rate": positivity_rate, + "stability": stability, + "resilience": resilience + }, + "emotion_distribution": emotion_distribution, + "time_range": time_range + } + + logger.info(f"情绪健康指数计算完成: score={health_score:.2f}, level={level}") + return response + + except Exception as e: + logger.error(f"计算情绪健康指数失败: {str(e)}", exc_info=True) + raise + + def _analyze_emotion_patterns(self, emotions: List[Dict[str, Any]]) -> Dict[str, Any]: + """分析情绪模式 + + 识别主要负面情绪、情绪触发因素和波动时段。 + + Args: + emotions: 情绪数据列表，每个包含 emotion_type、emotion_intensity、created_at 字段 + + Returns: + Dict: 包含情绪模式分析结果： + - dominant_negative_emotion: 主要负面情绪类型 + - high_intensity_emotions: 高强度情绪列表 + - emotion_volatility: 情绪波动性（高/中/低） + """ + negative_emotions = {'sadness', 'anger', 'fear'} + + # 统计负面情绪分布 + negative_emotion_counts = {} + for emotion in emotions: + emotion_type = emotion.get('emotion_type') + if emotion_type in negative_emotions: + negative_emotion_counts[emotion_type] = negative_emotion_counts.get(emotion_type, 0) + 1 + + # 识别主要负面情绪 + dominant_negative_emotion = None + if negative_emotion_counts: + dominant_negative_emotion = max(negative_emotion_counts, key=negative_emotion_counts.get) + + # 识别高强度情绪（强度 >= 0.7） + high_intensity_emotions = [ + { + "type": e.get('emotion_type'), + "intensity": e.get('emotion_intensity'), + "created_at": e.get('created_at') + } + for e in emotions + if e.get('emotion_intensity', 0) >= 0.7 + ] + + # 评估情绪波动性 + intensities = [e.get('emotion_intensity', 0.0) for e in emotions if e.get('emotion_intensity') is not None] + if len(intensities) >= 2: + std_dev = statistics.stdev(intensities) + if std_dev > 0.3: + volatility = "高" + elif std_dev > 0.15: + volatility = "中" + else: + volatility = "低" + else: + volatility = "未知" + + logger.debug(f"情绪模式分析: dominant_negative={dominant_negative_emotion}, " + f"high_intensity_count={len(high_intensity_emotions)}, volatility={volatility}") + + return { + "dominant_negative_emotion": dominant_negative_emotion, + "high_intensity_emotions": high_intensity_emotions[:5], # 最多返回5个 + "emotion_volatility": volatility + } + + async def generate_emotion_suggestions( + self, + end_user_id: str, + config_id: Optional[int] = None + ) -> Dict[str, Any]: + """生成个性化情绪建议 + + 基于情绪健康数据和用户画像生成个性化建议。 + + Args: + end_user_id: 宿主ID（用户组ID） + config_id: 配置ID（可选，用于从数据库加载LLM配置） + + Returns: + Dict: 包含个性化建议的响应： + - health_summary: 健康状态摘要 + - suggestions: 建议列表（3-5条） + """ + try: + logger.info(f"生成个性化情绪建议: user={end_user_id}, config_id={config_id}") + + # 1. 如果提供了 config_id，从数据库加载配置 + if config_id is not None: + from app.core.memory.utils.config.definitions import reload_configuration_from_database + config_loaded = reload_configuration_from_database(config_id) + if not config_loaded: + logger.warning(f"无法加载配置 config_id={config_id}，将使用默认配置") + + # 2. 获取情绪健康数据 + health_data = await self.calculate_emotion_health_index(end_user_id, time_range="30d") + + # 3. 获取情绪数据用于模式分析 + emotions = await self.emotion_repo.get_emotions_in_range( + group_id=end_user_id, + time_range="30d" + ) + + # 4. 分析情绪模式 + patterns = self._analyze_emotion_patterns(emotions) + + # 5. 获取用户画像数据（简化版，直接从Neo4j获取） + user_profile = await self._get_simple_user_profile(end_user_id) + + # 6. 构建LLM prompt + prompt = await self._build_suggestion_prompt(health_data, patterns, user_profile) + + # 7. 调用LLM生成建议（使用配置中的LLM） + from app.core.memory.utils.llm.llm_utils import get_llm_client + llm_client = get_llm_client() + + # 将 prompt 转换为 messages 格式 + messages = [ + {"role": "user", "content": prompt} + ] + + response = await llm_client.chat(messages=messages) + response_text = response.content.strip() + + # 8. 解析LLM响应 + try: + response_data = json.loads(response_text) + suggestions_response = EmotionSuggestionsResponse(**response_data) + except (json.JSONDecodeError, Exception) as e: + logger.error(f"解析LLM响应失败: {str(e)}, response={response_text}") + # 返回默认建议 + suggestions_response = self._get_default_suggestions(health_data) + + # 8. 验证建议数量（3-5条） + if len(suggestions_response.suggestions) < 3: + logger.warning(f"建议数量不足: {len(suggestions_response.suggestions)}") + suggestions_response = self._get_default_suggestions(health_data) + elif len(suggestions_response.suggestions) > 5: + logger.warning(f"建议数量过多: {len(suggestions_response.suggestions)}") + suggestions_response.suggestions = suggestions_response.suggestions[:5] + + # 9. 格式化响应 + response = { + "health_summary": suggestions_response.health_summary, + "suggestions": [ + { + "type": s.type, + "title": s.title, + "content": s.content, + "priority": s.priority, + "actionable_steps": s.actionable_steps + } + for s in suggestions_response.suggestions + ] + } + + logger.info(f"个性化建议生成完成: suggestions_count={len(response['suggestions'])}") + return response + + except Exception as e: + logger.error(f"生成个性化建议失败: {str(e)}", exc_info=True) + raise + + async def _get_simple_user_profile(self, end_user_id: str) -> Dict[str, Any]: + """获取简化的用户画像数据 + + Args: + end_user_id: 用户ID + + Returns: + Dict: 用户画像数据 + """ + try: + connector = Neo4jConnector() + + # 查询用户的实体和标签 + query = """ + MATCH (e:Entity) + WHERE e.group_id = $group_id + RETURN e.name as name, e.type as type + ORDER BY e.created_at DESC + LIMIT 20 + """ + + entities = await connector.execute_query(query, group_id=end_user_id) + + # 提取兴趣标签 + interests = [e["name"] for e in entities if e.get("type") in ["INTEREST", "HOBBY"]][:5] + # 后期会引入用户的习惯。。 + return { + "interests": interests if interests else ["未知"] + } + + except Exception as e: + logger.error(f"获取用户画像失败: {str(e)}") + return {"interests": ["未知"]} + + async def _build_suggestion_prompt( + self, + health_data: Dict[str, Any], + patterns: Dict[str, Any], + user_profile: Dict[str, Any] + ) -> str: + """构建情绪建议生成的prompt + + Args: + health_data: 情绪健康数据 + patterns: 情绪模式分析结果 + user_profile: 用户画像数据 + + Returns: + str: LLM prompt + """ + from app.core.memory.utils.prompt.prompt_utils import render_emotion_suggestions_prompt + + prompt = await render_emotion_suggestions_prompt( + health_data=health_data, + patterns=patterns, + user_profile=user_profile + ) + + return prompt + + def _get_default_suggestions(self, health_data: Dict[str, Any]) -> EmotionSuggestionsResponse: + """获取默认建议（当LLM调用失败时使用） + + Args: + health_data: 情绪健康数据 + + Returns: + EmotionSuggestionsResponse: 默认建议 + """ + health_score = health_data.get('health_score', 0) + + if health_score >= 80: + summary = "您的情绪健康状况优秀，请继续保持积极的生活态度。" + elif health_score >= 60: + summary = "您的情绪健康状况良好，可以通过一些调整进一步提升。" + elif health_score >= 40: + summary = "您的情绪健康需要关注，建议采取一些改善措施。" + else: + summary = "您的情绪健康需要重点关注，建议寻求专业帮助。" + + suggestions = [ + EmotionSuggestion( + type="emotion_balance", + title="保持情绪平衡", + content="通过正念冥想和深呼吸练习，帮助您更好地管理情绪波动，提升情绪稳定性。", + priority="high", + actionable_steps=[ + "每天早晨进行5-10分钟的正念冥想", + "感到情绪波动时，进行3次深呼吸", + "记录每天的情绪变化，识别触发因素" + ] + ), + EmotionSuggestion( + type="activity_recommendation", + title="增加户外活动", + content="适度的户外运动可以有效改善情绪，增强身心健康。建议每周进行3-4次户外活动。", + priority="medium", + actionable_steps=[ + "每周安排2-3次30分钟的散步", + "周末尝试户外运动如骑行或爬山", + "在户外活动时关注周围环境，放松心情" + ] + ), + EmotionSuggestion( + type="social_connection", + title="加强社交联系", + content="与朋友和家人保持良好的社交联系，可以提供情感支持，改善情绪健康。", + priority="medium", + actionable_steps=[ + "每周至少与一位朋友或家人深入交流", + "参加感兴趣的社交活动或兴趣小组", + "主动分享自己的感受和想法" + ] + ) + ] + + return EmotionSuggestionsResponse( + health_summary=summary, + suggestions=suggestions + ) diff --git a/api/app/services/emotion_config_service.py b/api/app/services/emotion_config_service.py new file mode 100644 index 00000000..37171640 --- /dev/null +++ b/api/app/services/emotion_config_service.py @@ -0,0 +1,212 @@ +# -*- coding: utf-8 -*- +"""情绪配置服务模块 + +本模块提供情绪引擎配置的管理功能，包括获取和更新配置。 + +Classes: + EmotionConfigService: 情绪配置服务，提供配置管理功能 +""" + +from typing import Dict, Any +from sqlalchemy.orm import Session + +from app.models.data_config_model import DataConfig +from app.core.logging_config import get_business_logger + +logger = get_business_logger() + + +class EmotionConfigService: + """情绪配置服务 + + 提供情绪引擎配置的管理功能，包括： + - 获取情绪配置 + - 更新情绪配置 + - 验证配置参数 + + Attributes: + db: 数据库会话 + """ + + def __init__(self, db: Session): + """初始化情绪配置服务 + + Args: + db: 数据库会话 + """ + self.db = db + logger.info("情绪配置服务初始化完成") + + def get_emotion_config(self, config_id: int) -> Dict[str, Any]: + """获取情绪引擎配置 + + 查询指定配置ID的情绪相关配置字段。 + + Args: + config_id: 配置ID + + Returns: + Dict: 包含情绪配置的响应数据： + - config_id: 配置ID + - emotion_enabled: 是否启用情绪提取 + - emotion_model_id: 情绪分析专用模型ID + - emotion_extract_keywords: 是否提取情绪关键词 + - emotion_min_intensity: 最小情绪强度阈值 + - emotion_enable_subject: 是否启用主体分类 + + Raises: + ValueError: 当配置不存在时 + """ + try: + logger.info(f"获取情绪配置: config_id={config_id}") + + # 查询配置 + config = self.db.query(DataConfig).filter( + DataConfig.config_id == config_id + ).first() + + if not config: + logger.error(f"配置不存在: config_id={config_id}") + raise ValueError(f"配置不存在: config_id={config_id}") + + # 提取情绪相关字段 + emotion_config = { + "config_id": config.config_id, + "emotion_enabled": config.emotion_enabled, + "emotion_model_id": config.emotion_model_id, + "emotion_extract_keywords": config.emotion_extract_keywords, + "emotion_min_intensity": config.emotion_min_intensity, + "emotion_enable_subject": config.emotion_enable_subject + } + + logger.info(f"情绪配置获取成功: config_id={config_id}") + return emotion_config + + except ValueError: + raise + except Exception as e: + logger.error(f"获取情绪配置失败: {str(e)}", exc_info=True) + raise + + def validate_emotion_config(self, config_data: Dict[str, Any]) -> bool: + """验证情绪配置参数 + + 验证配置参数的有效性，包括： + - emotion_min_intensity 在 [0.0, 1.0] 范围内 + - 布尔字段类型正确 + - emotion_model_id 格式有效（如果提供） + + Args: + config_data: 配置数据字典 + + Returns: + bool: 验证是否通过 + + Raises: + ValueError: 当配置参数无效时 + """ + try: + logger.debug(f"验证情绪配置参数: {config_data}") + + # 验证 emotion_min_intensity 范围 + if "emotion_min_intensity" in config_data: + min_intensity = config_data["emotion_min_intensity"] + if not isinstance(min_intensity, (int, float)): + raise ValueError("emotion_min_intensity 必须是数字类型") + if not (0.0 <= min_intensity <= 1.0): + raise ValueError("emotion_min_intensity 必须在 0.0 到 1.0 之间") + + # 验证布尔字段 + bool_fields = ["emotion_enabled", "emotion_extract_keywords", "emotion_enable_subject"] + for field in bool_fields: + if field in config_data: + value = config_data[field] + if not isinstance(value, bool): + raise ValueError(f"{field} 必须是布尔类型") + + # 验证 emotion_model_id（如果提供） + if "emotion_model_id" in config_data: + model_id = config_data["emotion_model_id"] + if model_id is not None and not isinstance(model_id, str): + raise ValueError("emotion_model_id 必须是字符串类型或 null") + if model_id is not None and len(model_id.strip()) == 0: + raise ValueError("emotion_model_id 不能为空字符串") + + logger.debug("情绪配置参数验证通过") + return True + + except ValueError as e: + logger.warning(f"配置参数验证失败: {str(e)}") + raise + except Exception as e: + logger.error(f"验证配置参数时发生错误: {str(e)}", exc_info=True) + raise ValueError(f"验证配置参数失败: {str(e)}") + + def update_emotion_config( + self, + config_id: int, + config_data: Dict[str, Any] + ) -> Dict[str, Any]: + """更新情绪引擎配置 + + 更新指定配置ID的情绪相关配置字段。 + + Args: + config_id: 配置ID + config_data: 要更新的配置数据，可包含以下字段： + - emotion_enabled: 是否启用情绪提取 + - emotion_model_id: 情绪分析专用模型ID + - emotion_extract_keywords: 是否提取情绪关键词 + - emotion_min_intensity: 最小情绪强度阈值 + - emotion_enable_subject: 是否启用主体分类 + + Returns: + Dict: 更新后的完整情绪配置 + + Raises: + ValueError: 当配置不存在或参数无效时 + """ + try: + logger.info(f"更新情绪配置: config_id={config_id}, data={config_data}") + + # 验证配置参数 + self.validate_emotion_config(config_data) + + # 查询配置 + config = self.db.query(DataConfig).filter( + DataConfig.config_id == config_id + ).first() + + if not config: + logger.error(f"配置不存在: config_id={config_id}") + raise ValueError(f"配置不存在: config_id={config_id}") + + # 更新字段 + if "emotion_enabled" in config_data: + config.emotion_enabled = config_data["emotion_enabled"] + if "emotion_model_id" in config_data: + config.emotion_model_id = config_data["emotion_model_id"] + if "emotion_extract_keywords" in config_data: + config.emotion_extract_keywords = config_data["emotion_extract_keywords"] + if "emotion_min_intensity" in config_data: + config.emotion_min_intensity = config_data["emotion_min_intensity"] + if "emotion_enable_subject" in config_data: + config.emotion_enable_subject = config_data["emotion_enable_subject"] + + # 提交更改 + self.db.commit() + self.db.refresh(config) + + # 返回更新后的配置 + updated_config = self.get_emotion_config(config_id) + + logger.info(f"情绪配置更新成功: config_id={config_id}") + return updated_config + + except ValueError: + self.db.rollback() + raise + except Exception as e: + self.db.rollback() + logger.error(f"更新情绪配置失败: {str(e)}", exc_info=True) + raise diff --git a/api/app/services/emotion_extraction_service.py b/api/app/services/emotion_extraction_service.py new file mode 100644 index 00000000..b3172df1 --- /dev/null +++ b/api/app/services/emotion_extraction_service.py @@ -0,0 +1,200 @@ +"""Emotion extraction service for analyzing emotions from statements. + +This service extracts emotion information from user statements using LLM, +including emotion type, intensity, keywords, subject classification, and target. + +Classes: + EmotionExtractionService: Service for extracting emotions from statements +""" + +import logging +from typing import Optional +from app.core.memory.models.emotion_models import EmotionExtraction +from app.models.data_config_model import DataConfig +from app.core.memory.utils.llm.llm_utils import get_llm_client +from app.core.memory.llm_tools.llm_client import LLMClientException + +logger = logging.getLogger(__name__) + + +class EmotionExtractionService: + """Service for extracting emotion information from statements. + + This service uses LLM to analyze statements and extract structured emotion + information including type, intensity, keywords, subject, and target. + It respects configuration settings for enabling/disabling extraction and + filtering by intensity threshold. + + Attributes: + llm_client: LLM client for making structured output calls + """ + + def __init__(self, llm_id: Optional[str] = None): + """Initialize the emotion extraction service. + + Args: + llm_id: Optional LLM model ID. If None, uses default from config. + """ + self.llm_client = None + self.llm_id = llm_id + logger.info(f"Initialized EmotionExtractionService with llm_id={llm_id}") + + def _get_llm_client(self, model_id: Optional[str] = None): + """Get or create LLM client instance. + + Args: + model_id: Optional model ID to use. If None, uses instance llm_id. + + Returns: + LLM client instance + """ + if self.llm_client is None or model_id: + effective_model_id = model_id or self.llm_id + self.llm_client = get_llm_client(effective_model_id) + return self.llm_client + + async def extract_emotion( + self, + statement: str, + config: DataConfig + ) -> Optional[EmotionExtraction]: + """Extract emotion information from a statement. + + This method checks if emotion extraction is enabled in the config, + builds an appropriate prompt, calls the LLM for structured output, + and applies intensity threshold filtering. + + Args: + statement: The statement text to analyze + config: Data configuration object containing emotion settings + + Returns: + EmotionExtraction object if extraction succeeds and passes threshold, + None if extraction is disabled, fails, or doesn't meet threshold + + Raises: + No exceptions are raised - failures are logged and return None + """ + # Check if emotion extraction is enabled + if not config.emotion_enabled: + logger.debug("Emotion extraction is disabled in config") + return None + + # Validate statement + if not statement or not statement.strip(): + logger.warning("Empty statement provided for emotion extraction") + return None + + try: + # Build the emotion extraction prompt + prompt = await self._build_emotion_prompt( + statement=statement, + extract_keywords=config.emotion_extract_keywords, + enable_subject=config.emotion_enable_subject + ) + + # Call LLM for structured output + emotion = await self._call_llm_structured( + prompt=prompt, + model_id=config.emotion_model_id + ) + + # Apply intensity threshold filtering + if emotion.emotion_intensity < config.emotion_min_intensity: + logger.debug( + f"Emotion intensity {emotion.emotion_intensity} below threshold " + f"{config.emotion_min_intensity}, skipping storage" + ) + return None + + logger.info( + f"Successfully extracted emotion: type={emotion.emotion_type}, " + f"intensity={emotion.emotion_intensity}, subject={emotion.emotion_subject}" + ) + + return emotion + + except Exception as e: + logger.error( + f"Emotion extraction failed for statement: {statement[:50]}..., " + f"error: {str(e)}", + exc_info=True + ) + return None + + async def _build_emotion_prompt( + self, + statement: str, + extract_keywords: bool, + enable_subject: bool + ) -> str: + """Build the emotion extraction prompt based on configuration. + + This method constructs a detailed prompt for the LLM that includes + instructions for emotion type classification, intensity assessment, + and optionally keyword extraction and subject classification. + + Args: + statement: The statement to analyze + extract_keywords: Whether to extract emotion keywords + enable_subject: Whether to enable subject classification + + Returns: + Formatted prompt string for LLM + """ + from app.core.memory.utils.prompt.prompt_utils import render_emotion_extraction_prompt + + prompt = await render_emotion_extraction_prompt( + statement=statement, + extract_keywords=extract_keywords, + enable_subject=enable_subject + ) + + return prompt + + async def _call_llm_structured( + self, + prompt: str, + model_id: Optional[str] = None + ) -> EmotionExtraction: + """Call LLM for structured emotion extraction output. + + This method uses the LLM client's response_structured method to get + a validated EmotionExtraction object from the LLM. + + Args: + prompt: The formatted prompt for emotion extraction + model_id: Optional model ID to use for this call + + Returns: + EmotionExtraction object with validated emotion data + + Raises: + LLMClientException: If LLM call fails or times out + ValidationError: If LLM response doesn't match expected schema + """ + try: + # Get LLM client + llm_client = self._get_llm_client(model_id) + + # Prepare messages + messages = [ + {"role": "user", "content": prompt} + ] + + # Call LLM with structured output + emotion = await llm_client.response_structured( + messages=messages, + response_model=EmotionExtraction, + temperature=0.3, + max_tokens=500 + ) + + return emotion + + except LLMClientException as e: + logger.error(f"LLM call failed: {str(e)}") + raise + except Exception as e: + logger.error(f"Unexpected error in LLM structured call: {str(e)}") + raise LLMClientException(f"Emotion extraction LLM call failed: {str(e)}") From 3f4c2d7796f6589169eb0cef2064e3b498b92c8e Mon Sep 17 00:00:00 2001 From: Mark Date: Sat, 20 Dec 2025 15:27:47 +0800 Subject: [PATCH 20/20] [add] migration script --- .../versions/626abf154a6a_202512201526.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 api/migrations/versions/626abf154a6a_202512201526.py diff --git a/api/migrations/versions/626abf154a6a_202512201526.py b/api/migrations/versions/626abf154a6a_202512201526.py new file mode 100644 index 00000000..7d89766e --- /dev/null +++ b/api/migrations/versions/626abf154a6a_202512201526.py @@ -0,0 +1,38 @@ +"""202512201526 + +Revision ID: 626abf154a6a +Revises: 70e94dd4a8d1 +Create Date: 2025-12-20 15:26:50.634470 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '626abf154a6a' +down_revision: Union[str, None] = '70e94dd4a8d1' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('data_config', sa.Column('emotion_enabled', sa.Boolean(), nullable=True, comment='是否启用情绪提取')) + op.add_column('data_config', sa.Column('emotion_model_id', sa.String(), nullable=True, comment='情绪分析专用模型ID')) + op.add_column('data_config', sa.Column('emotion_extract_keywords', sa.Boolean(), nullable=True, comment='是否提取情绪关键词')) + op.add_column('data_config', sa.Column('emotion_min_intensity', sa.Float(), nullable=True, comment='最小情绪强度阈值')) + op.add_column('data_config', sa.Column('emotion_enable_subject', sa.Boolean(), nullable=True, comment='是否启用主体分类')) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('data_config', 'emotion_enable_subject') + op.drop_column('data_config', 'emotion_min_intensity') + op.drop_column('data_config', 'emotion_extract_keywords') + op.drop_column('data_config', 'emotion_model_id') + op.drop_column('data_config', 'emotion_enabled') + # ### end Alembic commands ###