From 1e3ba39150fc9f01625cc9b1741bcfac3c1ebf1b Mon Sep 17 00:00:00 2001 From: Ke Sun Date: Sun, 21 Dec 2025 20:32:41 +0800 Subject: [PATCH 01/12] refactor(memory): restructure memory agent and config management - Reorganize imports and remove unused dependencies across memory agent controllers - Extract config validation logic into dedicated validators module - Create new memory_config_model and memory_config_schema for configuration management - Implement memory_config_service for centralized config handling - Add embedder_utils module for embedding model utilities - Refactor memory agent service to use new config validation framework - Clean up configuration files (remove config.json, testdata.json, dbrun.json) - Remove deprecated hybrid_chatbot.py and config overrides - Update logging configuration and error handling across memory modules - Consolidate LLM and embedding model validation into validators - Improve code organization and reduce duplication in memory storage services - Enhance type classification and verification tools with better error handling --- .../controllers/memory_agent_controller.py | 175 +---- .../controllers/memory_storage_controller.py | 76 +- api/app/core/logging_config.py | 39 +- .../agent/langgraph_graph/nodes/input_node.py | 32 +- .../agent/langgraph_graph/nodes/tool_node.py | 65 +- .../agent/langgraph_graph/read_graph.py | 205 +++--- .../agent/langgraph_graph/state/extractors.py | 19 +- .../agent/langgraph_graph/write_graph.py | 78 +- .../core/memory/agent/mcp_server/__init__.py | 28 +- .../core/memory/agent/mcp_server/server.py | 46 +- .../mcp_server/services/parameter_builder.py | 23 +- .../mcp_server/services/search_service.py | 38 +- .../agent/mcp_server/tools/data_tools.py | 71 +- .../agent/mcp_server/tools/problem_tools.py | 56 +- .../agent/mcp_server/tools/retrieval_tools.py | 48 +- .../agent/mcp_server/tools/summary_tools.py | 103 +-- .../mcp_server/tools/verification_tools.py | 35 +- api/app/core/memory/agent/utils/llm_tools.py | 52 +- .../core/memory/agent/utils/messages_tool.py | 39 +- .../utils/prompt/split_verify_prompt.jinja2 | 14 +- .../memory/agent/utils/type_classifier.py | 7 +- .../core/memory/agent/utils/verify_tool.py | 17 +- .../core/memory/agent/utils/write_tools.py | 152 ++-- .../core/memory/analytics/hot_memory_tags.py | 29 +- .../core/memory/analytics/memory_insight.py | 21 +- api/app/core/memory/analytics/user_summary.py | 22 +- api/app/core/memory/config.json | 132 ---- api/app/core/memory/data/testdata.json | 12 - api/app/core/memory/dbrun.json | 5 - api/app/core/memory/main.py | 75 +- api/app/core/memory/src/search.py | 105 +-- .../extraction_orchestrator.py | 105 +-- .../storage_services/search/hybrid_chatbot.py | 447 ----------- .../storage_services/search/hybrid_search.py | 692 +++++++++--------- api/app/core/memory/utils/README.md | 445 ----------- api/app/core/memory/utils/config/__init__.py | 58 +- .../core/memory/utils/config/config_utils.py | 206 +++--- .../core/memory/utils/config/definitions.py | 381 ++++------ api/app/core/memory/utils/config/overrides.py | 611 ---------------- .../core/memory/utils/embedder/__init__.py | 11 + .../memory/utils/embedder/embedder_utils.py | 81 ++ api/app/core/memory/utils/llm/llm_utils.py | 74 +- .../self_reflexion_utils/self_reflexion.py | 27 +- api/app/core/validators/__init__.py | 19 +- .../validators/memory_config_validators.py | 250 +++++++ api/app/models/memory_config_model.py | 39 + .../repositories/data_config_repository.py | 136 +++- api/app/schemas/memory_agent_schema.py | 4 +- api/app/schemas/memory_config_schema.py | 451 ++++++++++++ api/app/services/memory_agent_service.py | 246 ++++--- api/app/services/memory_config_service.py | 264 +++++++ api/app/services/memory_storage_service.py | 98 +-- api/app/tasks.py | 65 +- 53 files changed, 3122 insertions(+), 3407 deletions(-) delete mode 100644 api/app/core/memory/config.json delete mode 100644 api/app/core/memory/data/testdata.json delete mode 100644 api/app/core/memory/dbrun.json delete mode 100644 api/app/core/memory/storage_services/search/hybrid_chatbot.py delete mode 100644 api/app/core/memory/utils/README.md delete mode 100644 api/app/core/memory/utils/config/overrides.py create mode 100644 api/app/core/memory/utils/embedder/__init__.py create mode 100644 api/app/core/memory/utils/embedder/embedder_utils.py create mode 100644 api/app/core/validators/memory_config_validators.py create mode 100644 api/app/models/memory_config_model.py create mode 100644 api/app/schemas/memory_config_schema.py create mode 100644 api/app/services/memory_config_service.py diff --git a/api/app/controllers/memory_agent_controller.py b/api/app/controllers/memory_agent_controller.py index 7eb37c1c..884ee889 100644 --- a/api/app/controllers/memory_agent_controller.py +++ b/api/app/controllers/memory_agent_controller.py @@ -1,36 +1,28 @@ -import json -import time -from typing import Optional, List -from fastapi import APIRouter, Depends, Query, UploadFile -from sqlalchemy.orm import Session -from starlette.responses import StreamingResponse -from app.db import get_db -from app.core.memory.utils.config.config_utils import get_model_config -from app.core.rag.llm.cv_model import QWenCV -from app.models import ModelApiKey, Knowledge -from app.services.memory_agent_service import MemoryAgentService -from app.dependencies import get_current_superuser, get_current_user, get_current_tenant, workspace_access_guard, cur_workspace_access_guard +from typing import List, Optional + from app.celery_app import celery_app -from app.core.logging_config import get_api_logger -from app.core.response_utils import success, fail from app.core.error_codes import BizCode -from app.services import task_service, workspace_service +from app.core.logging_config import get_api_logger +from app.core.rag.llm.cv_model import QWenCV +from app.core.response_utils import fail, success +from app.db import get_db +from app.dependencies import cur_workspace_access_guard, get_current_user +from app.models import ModelApiKey +from app.models.user_model import User +from app.repositories import knowledge_repository from app.schemas.memory_agent_schema import UserInput, Write_UserInput from app.schemas.response_schema import ApiResponse -from app.dependencies import get_current_user -from app.models.user_model import User -from fastapi import APIRouter, Depends, File, UploadFile, Form -from app.repositories import knowledge_repository +from app.services import task_service, workspace_service +from app.services.memory_agent_service import MemoryAgentService from app.services.model_service import ModelConfigService from dotenv import load_dotenv -import os +from fastapi import APIRouter, Depends, File, Form, Query, UploadFile +from sqlalchemy.orm import Session +from starlette.responses import StreamingResponse -# 加载.env文件 load_dotenv() -# Get API logger api_logger = get_api_logger() -# Initialize service memory_agent_service = MemoryAgentService() router = APIRouter( @@ -39,95 +31,6 @@ router = APIRouter( ) -def validate_config_id(config_id: int, db: Session) -> int: - """ - Validate and ensure config_id is available, valid, and exists in database. - - Args: - config_id: Configuration ID to validate - db: Database session for checking existence - - Returns: - int: Validated config_id - - Raises: - ValueError: If config_id is None, invalid, or doesn't exist in database - """ - if config_id is None: - api_logger.info("config_id is required but was not provided") - config_id = os.getenv('config_id') - if config_id is None: - raise ValueError("config_id is required but was not provided") - - - # Check if config exists in database - try: - from app.models.data_config_model import DataConfig - from app.models.models_model import ModelConfig - - config = db.query(DataConfig).filter(DataConfig.config_id == config_id).first() - if config is None: - error_msg = f"Configuration with config_id={config_id} does not exist in database" - api_logger.error(error_msg) - raise ValueError(error_msg) - - # Validate llm_id exists and is usable - if config.llm_id: - try: - llm_config = db.query(ModelConfig).filter(ModelConfig.id == config.llm_id).first() - if llm_config is None: - error_msg = f"LLM model with id={config.llm_id} (from config_id={config_id}) does not exist" - api_logger.error(error_msg) - raise ValueError(error_msg) - if not llm_config.is_active: - error_msg = f"LLM model with id={config.llm_id} (from config_id={config_id}) is not active" - api_logger.error(error_msg) - raise ValueError(error_msg) - api_logger.debug(f"LLM validation successful: llm_id={config.llm_id}, name={llm_config.name}") - except ValueError: - raise - except Exception as e: - error_msg = f"Error validating LLM model: {str(e)}" - api_logger.error(error_msg, exc_info=True) - raise ValueError(error_msg) - else: - api_logger.error(f"Config {config_id} has no llm_id set") - raise ValueError(f"Config {config_id} has no llm_id set") - - # Validate embedding_id exists and is usable - if config.embedding_id: - try: - embedding_config = db.query(ModelConfig).filter(ModelConfig.id == config.embedding_id).first() - if embedding_config is None: - error_msg = f"Embedding model with id={config.embedding_id} (from config_id={config_id}) does not exist" - api_logger.error(error_msg) - raise ValueError(error_msg) - if not embedding_config.is_active: - error_msg = f"Embedding model with id={config.embedding_id} (from config_id={config_id}) is not active" - api_logger.error(error_msg) - raise ValueError(error_msg) - api_logger.debug(f"Embedding validation successful: embedding_id={config.embedding_id}, name={embedding_config.name}") - except ValueError: - raise - except Exception as e: - error_msg = f"Error validating embedding model: {str(e)}" - api_logger.error(error_msg, exc_info=True) - raise ValueError(error_msg) - else: - api_logger.error(f"Config {config_id} has no embedding_id set") - raise ValueError(f"Config {config_id} has no embedding_id set") - - api_logger.info(f"Config validation successful: config_id={config_id}, config_name={config.config_name}, llm_id={config.llm_id}, embedding_id={config.embedding_id}") - return config_id - except ValueError: - # Re-raise ValueError from above - raise - except Exception as e: - error_msg = f"Database error while validating config_id={config_id}: {str(e)}" - api_logger.error(error_msg, exc_info=True) - raise ValueError(error_msg) - - @router.get("/health/status", response_model=ApiResponse) async def get_health_status( current_user: User = Depends(get_current_user) @@ -225,12 +128,7 @@ async def write_server( Returns: Response with write operation status """ - # Validate config_id - try: - config_id = validate_config_id(user_input.config_id, db) - except ValueError as e: - return fail(BizCode.INVALID_PARAMETER, "配置ID无效", str(e)) - + config_id = user_input.config_id workspace_id = current_user.current_workspace_id api_logger.info(f"Write service: workspace_id={workspace_id}, config_id={config_id}") @@ -270,8 +168,14 @@ async def write_server( user_rag_memory_id ) return success(data=result, msg="写入成功") - except Exception as e: - api_logger.error(f"Write operation error: {str(e)}") + except BaseException as e: + # Handle ExceptionGroup from TaskGroup (Python 3.11+) or BaseExceptionGroup + if hasattr(e, 'exceptions'): + error_messages = [f"{type(sub_e).__name__}: {str(sub_e)}" for sub_e in e.exceptions] + detailed_error = "; ".join(error_messages) + api_logger.error(f"Write operation error (TaskGroup): {detailed_error}", exc_info=True) + return fail(BizCode.INTERNAL_ERROR, "写入失败", detailed_error) + api_logger.error(f"Write operation error: {str(e)}", exc_info=True) return fail(BizCode.INTERNAL_ERROR, "写入失败", str(e)) @@ -292,12 +196,7 @@ async def write_server_async( Task ID for tracking async operation Use GET /memory/write_result/{task_id} to check task status and get result """ - # Validate config_id - try: - config_id = validate_config_id(user_input.config_id, db) - except ValueError as e: - return fail(BizCode.INVALID_PARAMETER, "配置ID无效", str(e)) - + config_id = user_input.config_id workspace_id = current_user.current_workspace_id api_logger.info(f"Async write service: workspace_id={workspace_id}, config_id={config_id}") @@ -352,12 +251,7 @@ async def read_server( Returns: Response with query answer """ - # Validate config_id - try: - config_id = validate_config_id(user_input.config_id, db) - except ValueError as e: - return fail(BizCode.INVALID_PARAMETER, "配置ID无效", str(e)) - + config_id = user_input.config_id workspace_id = current_user.current_workspace_id api_logger.info(f"Read service: workspace_id={workspace_id}, config_id={config_id}") @@ -390,8 +284,14 @@ async def read_server( user_rag_memory_id ) return success(data=result, msg="回复对话消息成功") - except Exception as e: - api_logger.error(f"Read operation error: {str(e)}") + except BaseException as e: + # Handle ExceptionGroup from TaskGroup (Python 3.11+) or BaseExceptionGroup + if hasattr(e, 'exceptions'): + error_messages = [f"{type(sub_e).__name__}: {str(sub_e)}" for sub_e in e.exceptions] + detailed_error = "; ".join(error_messages) + api_logger.error(f"Read operation error (TaskGroup): {detailed_error}", exc_info=True) + return fail(BizCode.INTERNAL_ERROR, "回复对话消息失败", detailed_error) + api_logger.error(f"Read operation error: {str(e)}", exc_info=True) return fail(BizCode.INTERNAL_ERROR, "回复对话消息失败", str(e)) @@ -456,12 +356,7 @@ async def read_server_async( db: Session = Depends(get_db), current_user: User = Depends(get_current_user) ): - # Validate config_id - try: - config_id = validate_config_id(user_input.config_id, db) - except ValueError as e: - return fail(BizCode.INVALID_PARAMETER, "配置ID无效", str(e)) - + config_id = user_input.config_id workspace_id = current_user.current_workspace_id api_logger.info(f"Async read service: workspace_id={workspace_id}, config_id={config_id}") diff --git a/api/app/controllers/memory_storage_controller.py b/api/app/controllers/memory_storage_controller.py index 89daf9ce..7f11d76c 100644 --- a/api/app/controllers/memory_storage_controller.py +++ b/api/app/controllers/memory_storage_controller.py @@ -1,45 +1,45 @@ -from typing import Optional, Union import os import uuid -from sqlalchemy.orm import Session -from fastapi import APIRouter, Depends, UploadFile -from fastapi.responses import StreamingResponse +from typing import Optional - -from app.db import get_db -from app.core.logging_config import get_api_logger -from app.core.response_utils import success, fail from app.core.error_codes import BizCode +from app.core.logging_config import get_api_logger +from app.core.memory.utils.self_reflexion_utils import self_reflexion +from app.core.response_utils import fail, success +from app.db import get_db +from app.dependencies import get_current_user +from app.models.user_model import User +from app.schemas.memory_storage_schema import ( + ConfigKey, + ConfigParamsCreate, + ConfigParamsDelete, + ConfigPilotRun, + ConfigUpdate, + ConfigUpdateExtracted, + ConfigUpdateForget, +) +from app.schemas.response_schema import ApiResponse from app.services.memory_storage_service import ( - MemoryStorageService, DataConfigService, - kb_type_distribution, - search_dialogue, - search_chunk, - search_statement, - search_entity, - search_all, - search_detials, - search_edges, - search_entity_graph, + MemoryStorageService, analytics_hot_memory_tags, analytics_memory_insight_report, analytics_recent_activity_stats, analytics_user_summary, + kb_type_distribution, + search_all, + search_chunk, + search_detials, + search_dialogue, + search_edges, + search_entity, + search_entity_graph, + search_statement, ) -from app.schemas.response_schema import ApiResponse -from app.schemas.memory_storage_schema import ( - ConfigParamsCreate, - ConfigParamsDelete, - ConfigUpdate, - ConfigUpdateExtracted, - ConfigUpdateForget, - ConfigKey, - ConfigPilotRun, -) -from app.core.memory.utils.config.definitions import reload_configuration_from_database -from app.dependencies import get_current_user -from app.models.user_model import User +from fastapi import APIRouter, Depends +from fastapi.responses import StreamingResponse +from sqlalchemy.orm import Session + # Get API logger api_logger = get_api_logger() @@ -329,8 +329,10 @@ async def pilot_run( current_user: User = Depends(get_current_user), db: Session = Depends(get_db), ) -> StreamingResponse: - api_logger.info(f"Pilot run requested: config_id={payload.config_id}, dialogue_text_length={len(payload.dialogue_text)}") - + api_logger.info( + f"Pilot run requested: config_id={payload.config_id}, " + f"dialogue_text_length={len(payload.dialogue_text)}" + ) svc = DataConfigService(db) return StreamingResponse( svc.pilot_run_stream(payload), @@ -338,8 +340,8 @@ async def pilot_run( headers={ "Cache-Control": "no-cache", "Connection": "keep-alive", - "X-Accel-Buffering": "no" - } + "X-Accel-Buffering": "no", + }, ) """ @@ -528,8 +530,8 @@ async def get_user_summary_api( except Exception as e: api_logger.error(f"User summary failed: {str(e)}") return fail(BizCode.INTERNAL_ERROR, "用户摘要生成失败", str(e)) - -from app.core.memory.utils.self_reflexion_utils import self_reflexion + + @router.get("/self_reflexion") async def self_reflexion_endpoint(host_id: uuid.UUID) -> str: """ diff --git a/api/app/core/logging_config.py b/api/app/core/logging_config.py index 02747cfb..b23f9d7c 100644 --- a/api/app/core/logging_config.py +++ b/api/app/core/logging_config.py @@ -326,7 +326,7 @@ def log_prompt_rendering(prompt_type: str, content: str) -> None: logger.info(log_message) -def log_template_rendering(template_name: str, context: dict | None = None) -> None: +def log_template_rendering(template_name: str, context: Optional[dict] = None) -> None: """Log template rendering information. Logs the template name and context keys for debugging template rendering. @@ -575,6 +575,43 @@ def get_named_logger(name: str) -> logging.Logger: return get_agent_logger(name) +def get_config_logger() -> logging.Logger: + """Get a specialized logger for memory configuration operations. + + Returns a logger configured specifically for configuration loading, validation, + and model resolution operations with: + - Logger name: memory.config + - Output: Inherits from root logger (console + file) + - Level: Inherits from root logger + - Format: Standard format with timing information + + This logger is optimized for configuration operations and includes + structured logging for timing, validation steps, and error context. + + Returns: + Logger configured for memory configuration operations + + Example: + >>> logger = get_config_logger() + >>> logger.info("Loading configuration", extra={ + ... "config_id": 123, + ... "workspace_id": "uuid-here", + ... "operation": "load_config" + ... }) + """ + # Ensure memory logging is initialized + if not LoggingConfig._memory_loggers_initialized: + LoggingConfig.setup_memory_logging() + + # Get configuration logger with memory namespace + logger = logging.getLogger("memory.config") + + # The logger automatically inherits handlers, formatters, and level from root logger + # through Python's logging hierarchy, so no additional configuration is needed + + return logger + + def get_memory_logger(name: Optional[str] = None) -> logging.Logger: """Get a standard logger for memory module components. diff --git a/api/app/core/memory/agent/langgraph_graph/nodes/input_node.py b/api/app/core/memory/agent/langgraph_graph/nodes/input_node.py index 350043fa..3eed497f 100644 --- a/api/app/core/memory/agent/langgraph_graph/nodes/input_node.py +++ b/api/app/core/memory/agent/langgraph_graph/nodes/input_node.py @@ -9,11 +9,11 @@ import logging import re import uuid from datetime import datetime -from typing import Dict, Any - -from langchain_core.messages import AIMessage +from typing import Any, Dict from app.core.memory.agent.utils.multimodal import MultimodalProcessor +from app.schemas.memory_config_schema import MemoryConfig +from langchain_core.messages import AIMessage logger = logging.getLogger(__name__) @@ -25,7 +25,8 @@ async def create_input_message( search_switch: str, apply_id: str, group_id: str, - multimodal_processor: MultimodalProcessor + multimodal_processor: MultimodalProcessor, + memory_config: MemoryConfig, ) -> Dict[str, Any]: """ Create initial tool call message from user input. @@ -46,6 +47,7 @@ async def create_input_message( apply_id: Application identifier group_id: Group identifier multimodal_processor: Processor for handling image/audio inputs + memory_config: MemoryConfig object containing all configuration Returns: State update with AIMessage containing tool_call @@ -53,7 +55,7 @@ async def create_input_message( Examples: >>> state = {"messages": [HumanMessage(content="What is AI?")]} >>> result = await create_input_message( - ... state, "Split_The_Problem", "call_id_user123", "0", "app1", "group1", processor + ... state, "Split_The_Problem", "call_id_user123", "0", "app1", "group1", processor, config ... ) >>> result["messages"][0].tool_calls[0]["name"] 'Split_The_Problem' @@ -123,20 +125,24 @@ async def create_input_message( f"with ID: {tool_call_id}" ) + # Build tool arguments + tool_args = { + "sentence": last_message, + "sessionid": session_id, + "messages_id": str(uuid_str), + "search_switch": search_switch, + "apply_id": apply_id, + "group_id": group_id, + "memory_config": memory_config, + } + return { "messages": [ AIMessage( content="", tool_calls=[{ "name": tool_name, - "args": { - "sentence": last_message, - "sessionid": session_id, - "messages_id": str(uuid_str), - "search_switch": search_switch, - "apply_id": apply_id, - "group_id": group_id - }, + "args": tool_args, "id": tool_call_id }] ) diff --git a/api/app/core/memory/agent/langgraph_graph/nodes/tool_node.py b/api/app/core/memory/agent/langgraph_graph/nodes/tool_node.py index 9ea2cad6..4727fb9c 100644 --- a/api/app/core/memory/agent/langgraph_graph/nodes/tool_node.py +++ b/api/app/core/memory/agent/langgraph_graph/nodes/tool_node.py @@ -9,14 +9,14 @@ import logging import time from typing import Any, Callable, Dict -from langchain_core.messages import AIMessage -from langgraph.prebuilt import ToolNode - from app.core.memory.agent.langgraph_graph.state.extractors import ( + extract_content_payload, extract_tool_call_id, - extract_content_payload ) from app.core.memory.agent.mcp_server.services.parameter_builder import ParameterBuilder +from app.schemas.memory_config_schema import MemoryConfig +from langchain_core.messages import AIMessage +from langgraph.prebuilt import ToolNode logger = logging.getLogger(__name__) @@ -38,8 +38,9 @@ class ToolExecutionNode: apply_id: Application identifier group_id: Group identifier parameter_builder: Service for building tool-specific arguments + memory_config: MemoryConfig object containing all configuration """ - + def __init__( self, tool: Callable, @@ -49,8 +50,9 @@ class ToolExecutionNode: apply_id: str, group_id: str, parameter_builder: ParameterBuilder, - storage_type:str, - user_rag_memory_id:str + storage_type: str, + user_rag_memory_id: str, + memory_config: MemoryConfig, ): """ Initialize the tool execution node. @@ -63,6 +65,9 @@ class ToolExecutionNode: apply_id: Application identifier group_id: Group identifier parameter_builder: Service for building tool-specific arguments + storage_type: Storage type for the workspace + user_rag_memory_id: User RAG memory identifier + memory_config: MemoryConfig object containing all configuration """ self.tool_node = ToolNode([tool]) self.id = node_id @@ -72,9 +77,10 @@ class ToolExecutionNode: self.apply_id = apply_id self.group_id = group_id self.parameter_builder = parameter_builder - self.storage_type=storage_type - self.user_rag_memory_id=user_rag_memory_id - + self.storage_type = storage_type + self.user_rag_memory_id = user_rag_memory_id + self.memory_config = memory_config + logger.info( f"[ToolExecutionNode] Initialized node '{self.id}' for tool '{self.tool_name}'" ) @@ -124,8 +130,12 @@ class ToolExecutionNode: # Extract content payload using state extractors content = extract_content_payload(last_message) logger.debug( - f"[ToolExecutionNode] {self.id} - Extracted content type: {type(content)}" + f"[ToolExecutionNode] {self.id} - Extracted content type: {type(content)}, content_keys: {list(content.keys()) if isinstance(content, dict) else 'N/A'}" ) + # Log raw message content for debugging + if hasattr(last_message, 'content'): + raw = last_message.content + logger.debug(f"[ToolExecutionNode] {self.id} - Raw message content (first 500 chars): {str(raw)[:500]}") except Exception as e: logger.error( @@ -143,8 +153,9 @@ class ToolExecutionNode: search_switch=self.search_switch, apply_id=self.apply_id, group_id=self.group_id, + memory_config=self.memory_config, storage_type=self.storage_type, - user_rag_memory_id=self.user_rag_memory_id + user_rag_memory_id=self.user_rag_memory_id, ) logger.debug( f"[ToolExecutionNode] {self.id} - Built tool args with keys: {list(tool_args.keys())}" @@ -179,7 +190,29 @@ class ToolExecutionNode: f"[ToolExecutionNode] {self.id} - Tool execution completed" ) - # Return the result directly - it already contains the messages list + # Check for error in tool response + error_entry = None + if result and "messages" in result: + for msg in result["messages"]: + if hasattr(msg, 'content'): + try: + import json + content = msg.content + if isinstance(content, str): + parsed = json.loads(content) + if isinstance(parsed, dict) and "error" in parsed: + error_msg = parsed["error"] + logger.warning( + f"[ToolExecutionNode] {self.id} - Tool returned error: {error_msg}" + ) + error_entry = {"tool": self.tool_name, "error": error_msg, "node_id": self.id} + except (json.JSONDecodeError, TypeError): + pass + + # Return result with error tracking if error was found + if error_entry: + result["errors"] = [error_entry] + return result except Exception as e: @@ -187,13 +220,15 @@ class ToolExecutionNode: f"[ToolExecutionNode] {self.id} - Tool execution failed: {e}", exc_info=True ) - # Return error as ToolMessage to maintain message chain consistency + # Track error in state and return error message from langchain_core.messages import ToolMessage + error_entry = {"tool": self.tool_name, "error": str(e), "node_id": self.id} return { "messages": [ ToolMessage( content=f"Error executing tool: {str(e)}", tool_call_id=f"{self.id}_{tool_call_id}" ) - ] + ], + "errors": [error_entry] } diff --git a/api/app/core/memory/agent/langgraph_graph/read_graph.py b/api/app/core/memory/agent/langgraph_graph/read_graph.py index 78d07327..c29b5d86 100644 --- a/api/app/core/memory/agent/langgraph_graph/read_graph.py +++ b/api/app/core/memory/agent/langgraph_graph/read_graph.py @@ -1,38 +1,26 @@ -import asyncio -import io import json -import logging import os import re import time -import uuid import warnings from contextlib import asynccontextmanager -from datetime import datetime from typing import Literal -from dotenv import load_dotenv -from langchain_core.messages import AIMessage -from langgraph.constants import START, END -from langgraph.graph import StateGraph -from langgraph.prebuilt import ToolNode -from functools import partial - -from app.core.memory.agent.utils.llm_tools import ReadState, COUNTState -from langgraph.checkpoint.memory import InMemorySaver - -from app.core.memory.agent.utils.redis_tool import store from app.core.logging_config import get_agent_logger - -# Import new modular components -from app.core.memory.agent.langgraph_graph.nodes import ToolExecutionNode, create_input_message -from app.core.memory.agent.langgraph_graph.routing.routers import ( - Verify_continue, - Retrieve_continue, - Split_continue +from app.core.memory.agent.langgraph_graph.nodes import ( + ToolExecutionNode, + create_input_message, ) from app.core.memory.agent.mcp_server.services.parameter_builder import ParameterBuilder +from app.core.memory.agent.utils.llm_tools import COUNTState, ReadState from app.core.memory.agent.utils.multimodal import MultimodalProcessor +from app.schemas.memory_config_schema import MemoryConfig +from dotenv import load_dotenv +from langchain_core.messages import AIMessage +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.constants import END, START +from langgraph.graph import StateGraph +from langgraph.prebuilt import ToolNode logger = get_agent_logger(__name__) @@ -44,9 +32,9 @@ redisdb=os.getenv('REDISDB') redispassword=os.getenv('REDISPASSWORD') counter = COUNTState(limit=3) -# 在工作流中添加循环计数更新 +# Update loop count in workflow async def update_loop_count(state): - """更新循环计数器""" + """Update loop counter""" current_count = state.get("loop_count", 0) return {"loop_count": current_count + 1} @@ -54,13 +42,13 @@ async def update_loop_count(state): def Verify_continue(state: ReadState) -> Literal["Summary", "Summary_fails", "content_input"]: messages = state["messages"] - # 添加边界检查 + # Add boundary check if not messages: return END - counter.add(1) # 累加 1 + counter.add(1) # Increment by 1 loop_count = counter.get_total() - logger.debug(f"[should_continue] 当前循环次数: {loop_count}") + logger.debug(f"[should_continue] Current loop count: {loop_count}") last_message = messages[-1] last_message_str = str(last_message).replace('\\', '') @@ -71,15 +59,15 @@ def Verify_continue(state: ReadState) -> Literal["Summary", "Summary_fails", "co counter.reset() return "Summary" elif "failed" in status_tools: - if loop_count < 2: # 最大循环次数 3 + if loop_count < 2: # Maximum loop count is 3 return "content_input" else: counter.reset() return "Summary_fails" else: - # 添加默认返回值，避免返回 None + # Add default return value to avoid returning None counter.reset() - return "Summary" # 或根据业务需求选择合适的默认值 + return "Summary" # Default based on business requirements def Retrieve_continue(state) -> Literal["Verify", "Retrieve_Summary"]: @@ -115,8 +103,8 @@ def Retrieve_continue(state) -> Literal["Verify", "Retrieve_Summary"]: elif search_switch == '1': return 'Retrieve_Summary' - # 添加默认返回值，避免返回 None - return 'Retrieve_Summary' # 或根据业务逻辑选择合适的默认值 + # Add default return value to avoid returning None + return 'Retrieve_Summary' # Default based on business logic def Split_continue(state) -> Literal["Split_The_Problem", "Input_Summary"]: @@ -151,46 +139,7 @@ def Split_continue(state) -> Literal["Split_The_Problem", "Input_Summary"]: search_switch = str(search_switch) if search_switch == '2': return 'Input_Summary' - return 'Split_The_Problem' # 默认情况 - -# 在 input_sentence 函数中修改参数名称 -async def input_sentence(state, name, id, search_switch,apply_id,group_id): - messages = state["messages"] - last_message = messages[-1].content if messages else "" - - if last_message.endswith('.jpg') or last_message.endswith('.png'): - last_message=await picture_model_requests(last_message) - if any(last_message.endswith(ext) for ext in audio_extensions): - last_message=await Vico_recognition([last_message]).run() - logger.debug(f"Audio recognition result: {last_message}") - - - uuid_str = uuid.uuid4() - time_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - namespace = str(id).split('_id_')[1] - if 'verified_data' in str(last_message): - messages_last = str(last_message).replace('\\n', '').replace('\\', '') - last_message = re.findall(r'"query": "(.*?)",', str(messages_last))[0] - - return { - "messages": [ - AIMessage( - content="", - tool_calls=[{ - "name": name, - "args": { - "sentence": last_message, - 'sessionid': id, - 'messages_id': str(uuid_str), - "search_switch": search_switch, # 正确地将 search_switch 放入 args 中 - "apply_id":apply_id, - "group_id":group_id - }, - "id": id + f'_{uuid_str}' - }] - ) - ] - } + return 'Split_The_Problem' # Default case class ProblemExtensionNode: @@ -208,30 +157,28 @@ class ProblemExtensionNode: async def __call__(self, state): messages = state["messages"] last_message = messages[-1] if messages else "" - logger.debug(f"ProblemExtensionNode {self.id} - 当前时间: {time.time()} - Message: {last_message}") - if self.tool_name=='Input_Summary': - tool_call =re.findall("'id': '(.*?)'",str(last_message))[0] - else:tool_call = str(re.findall(r"tool_call_id=.*?'(.*?)'", str(last_message))[0]).replace('\\', '').split('_id')[1] - # try: - # content = json.loads(last_message.content) if hasattr(last_message, 'content') else last_message - # except: - # content = last_message.content if hasattr(last_message, 'content') else str(last_message) - # 尝试从上一工具的结果中提取实际的内容载荷（而不是整个对象的字符串表示） + logger.debug(f"ProblemExtensionNode {self.id} - Current time: {time.time()} - Message: {last_message}") + if self.tool_name == 'Input_Summary': + tool_call = re.findall("'id': '(.*?)'", str(last_message))[0] + else: + tool_call = str(re.findall(r"tool_call_id=.*?'(.*?)'", str(last_message))[0]).replace('\\', '').split('_id')[1] + + # Try to extract actual content payload from previous tool result raw_msg = last_message.content if hasattr(last_message, 'content') else str(last_message) extracted_payload = None - # 捕获 ToolMessage 的 content 字段（支持单/双引号），并避免贪婪匹配 + # Capture ToolMessage content field (supports single/double quotes), avoid greedy matching m = re.search(r"content=(?:\"|\')(.*?)(?:\"|\'),\s*name=", raw_msg, flags=re.S) if m: extracted_payload = m.group(1) else: - # 回退：直接尝试使用原始字符串 + # Fallback: use raw string directly extracted_payload = raw_msg - # 优先尝试将内容解析为 JSON + # Try to parse content as JSON first try: content = json.loads(extracted_payload) except Exception: - # 尝试从文本中提取 JSON 片段再解析 + # Try to extract JSON fragment from text and parse parsed = None candidates = re.findall(r"[\[{].*[\]}]", extracted_payload, flags=re.S) for cand in candidates: @@ -240,14 +187,14 @@ class ProblemExtensionNode: break except Exception: continue - # 如果仍然失败，则以原始字符串作为内容 + # If still fails, use raw string as content content = parsed if parsed is not None else extracted_payload - # 根据工具名称构建正确的参数 + # Build correct parameters based on tool name tool_args = {} if self.tool_name == "Verify": - # Verify工具需要context和usermessages参数 + # Verify tool requires context and usermessages parameters if isinstance(content, dict): tool_args["context"] = content else: @@ -256,7 +203,7 @@ class ProblemExtensionNode: tool_args["apply_id"] = str(self.apply_id) tool_args["group_id"] = str(self.group_id) elif self.tool_name == "Retrieve": - # Retrieve工具需要context和usermessages参数 + # Retrieve tool requires context and usermessages parameters if isinstance(content, dict): tool_args["context"] = content else: @@ -266,9 +213,9 @@ class ProblemExtensionNode: tool_args["apply_id"] = str(self.apply_id) tool_args["group_id"] = str(self.group_id) elif self.tool_name == "Summary": - # Summary工具需要字符串类型的context参数 + # Summary tool requires string type context parameter if isinstance(content, dict): - # 将字典转换为JSON字符串 + # Convert dict to JSON string tool_args["context"] = json.dumps(content, ensure_ascii=False) else: tool_args["context"] = str(content) @@ -276,24 +223,24 @@ class ProblemExtensionNode: tool_args["apply_id"] = str(self.apply_id) tool_args["group_id"] = str(self.group_id) elif self.tool_name == "Summary_fails": - # Summary工具需要字符串类型的context参数 + # Summary_fails tool requires string type context parameter if isinstance(content, dict): - # 将字典转换为JSON字符串 + # Convert dict to JSON string tool_args["context"] = json.dumps(content, ensure_ascii=False) else: tool_args["context"] = str(content) tool_args["usermessages"] = str(tool_call) tool_args["apply_id"] = str(self.apply_id) tool_args["group_id"] = str(self.group_id) - elif self.tool_name=='Input_Summary': - tool_args["context"] =str(last_message) + elif self.tool_name == 'Input_Summary': + tool_args["context"] = str(last_message) tool_args["usermessages"] = str(tool_call) tool_args["search_switch"] = str(self.search_switch) tool_args["apply_id"] = str(self.apply_id) tool_args["group_id"] = str(self.group_id) tool_args["storage_type"] = getattr(self, 'storage_type', "") tool_args["user_rag_memory_id"] = getattr(self, 'user_rag_memory_id', "") - elif self.tool_name=='Retrieve_Summary' : + elif self.tool_name == 'Retrieve_Summary': # Retrieve_Summary expects dict directly, not JSON string # content might be a JSON string, try to parse it if isinstance(content, str): @@ -320,7 +267,7 @@ class ProblemExtensionNode: tool_args["apply_id"] = str(self.apply_id) tool_args["group_id"] = str(self.group_id) else: - # 其他工具使用context参数 + # Other tools use context parameter if isinstance(content, dict): tool_args["context"] = content else: @@ -349,12 +296,24 @@ class ProblemExtensionNode: @asynccontextmanager -async def make_read_graph(namespace,tools,search_switch,apply_id,group_id,config_id=None,storage_type=None,user_rag_memory_id=None): +async def make_read_graph(namespace, tools, search_switch, apply_id, group_id, memory_config: MemoryConfig, storage_type=None, user_rag_memory_id=None): + """ + Create a read graph workflow for memory operations. + + Args: + namespace: Namespace identifier + tools: MCP tools loaded from session + search_switch: Search mode switch ("0", "1", or "2") + apply_id: Application identifier + group_id: Group identifier + memory_config: MemoryConfig object containing all configuration + storage_type: Storage type (optional) + user_rag_memory_id: User RAG memory ID (optional) + """ memory = InMemorySaver() - tool=[i.name for i in tools ] + tool = [i.name for i in tools] logger.info(f"Initializing read graph with tools: {tool}") - if config_id: - logger.info(f"使用配置 ID: {config_id}") + logger.info(f"Using memory_config: {memory_config.config_name} (id={memory_config.config_id})") # Extract tool functions Split_The_Problem_ = next((t for t in tools if t.name == "Split_The_Problem"), None) @@ -382,9 +341,10 @@ async def make_read_graph(namespace,tools,search_switch,apply_id,group_id,config group_id=group_id, parameter_builder=parameter_builder, storage_type=storage_type, - user_rag_memory_id=user_rag_memory_id + user_rag_memory_id=user_rag_memory_id, + memory_config=memory_config, ) - + Retrieve_node = ToolExecutionNode( tool=Retrieve_, node_id="Retrieve_id", @@ -394,9 +354,10 @@ async def make_read_graph(namespace,tools,search_switch,apply_id,group_id,config group_id=group_id, parameter_builder=parameter_builder, storage_type=storage_type, - user_rag_memory_id=user_rag_memory_id + user_rag_memory_id=user_rag_memory_id, + memory_config=memory_config, ) - + Verify_node = ToolExecutionNode( tool=Verify_, node_id="Verify_id", @@ -406,7 +367,8 @@ async def make_read_graph(namespace,tools,search_switch,apply_id,group_id,config group_id=group_id, parameter_builder=parameter_builder, storage_type=storage_type, - user_rag_memory_id=user_rag_memory_id + user_rag_memory_id=user_rag_memory_id, + memory_config=memory_config, ) Summary_node = ToolExecutionNode( @@ -418,9 +380,10 @@ async def make_read_graph(namespace,tools,search_switch,apply_id,group_id,config group_id=group_id, parameter_builder=parameter_builder, storage_type=storage_type, - user_rag_memory_id=user_rag_memory_id + user_rag_memory_id=user_rag_memory_id, + memory_config=memory_config, ) - + Summary_fails_node = ToolExecutionNode( tool=Summary_fails_, node_id="Summary_fails_id", @@ -430,9 +393,10 @@ async def make_read_graph(namespace,tools,search_switch,apply_id,group_id,config group_id=group_id, parameter_builder=parameter_builder, storage_type=storage_type, - user_rag_memory_id=user_rag_memory_id + user_rag_memory_id=user_rag_memory_id, + memory_config=memory_config, ) - + Retrieve_Summary_node = ToolExecutionNode( tool=Retrieve_Summary_, node_id="Retrieve_Summary_id", @@ -442,9 +406,10 @@ async def make_read_graph(namespace,tools,search_switch,apply_id,group_id,config group_id=group_id, parameter_builder=parameter_builder, storage_type=storage_type, - user_rag_memory_id=user_rag_memory_id + user_rag_memory_id=user_rag_memory_id, + memory_config=memory_config, ) - + Input_Summary_node = ToolExecutionNode( tool=Input_Summary_, node_id="Input_Summary_id", @@ -454,16 +419,16 @@ async def make_read_graph(namespace,tools,search_switch,apply_id,group_id,config group_id=group_id, parameter_builder=parameter_builder, storage_type=storage_type, - user_rag_memory_id=user_rag_memory_id + user_rag_memory_id=user_rag_memory_id, + memory_config=memory_config, ) - async def content_input_node(state): state_search_switch = state.get("search_switch", search_switch) tool_name = "Input_Summary" if state_search_switch == '2' else "Split_The_Problem" session_prefix = "input_summary_call_id" if state_search_switch == '2' else "split_call_id" - + return await create_input_message( state=state, tool_name=tool_name, @@ -471,7 +436,8 @@ async def make_read_graph(namespace,tools,search_switch,apply_id,group_id,config search_switch=search_switch, apply_id=apply_id, group_id=group_id, - multimodal_processor=multimodal_processor + multimodal_processor=multimodal_processor, + memory_config=memory_config, ) @@ -501,8 +467,3 @@ async def make_read_graph(namespace,tools,search_switch,apply_id,group_id,config graph = workflow.compile(checkpointer=memory) yield graph - - -# 添加到文件末尾或创建新的执行脚本 -# 在 memory_agent_service.py 文件中添加以下函数 - diff --git a/api/app/core/memory/agent/langgraph_graph/state/extractors.py b/api/app/core/memory/agent/langgraph_graph/state/extractors.py index 92bec147..f5a32f5d 100644 --- a/api/app/core/memory/agent/langgraph_graph/state/extractors.py +++ b/api/app/core/memory/agent/langgraph_graph/state/extractors.py @@ -128,6 +128,15 @@ def extract_content_payload(message: Any) -> Any: # For ToolMessages (responses from tools), extract from content if hasattr(message, "content"): raw_content = message.content + logger.info(f"extract_content_payload: raw_content type={type(raw_content)}, value={str(raw_content)[:500]}") + + # Handle MCP content format: [{'type': 'text', 'text': '...'}] + if isinstance(raw_content, list): + for block in raw_content: + if isinstance(block, dict) and block.get('type') == 'text': + raw_content = block.get('text', '') + logger.info(f"extract_content_payload: extracted text from MCP format: {str(raw_content)[:300]}") + break # If content is empty and this is an AIMessage with tool_calls, # extract from args (this handles the initial tool call from content_input) @@ -140,13 +149,16 @@ def extract_content_payload(message: Any) -> Any: # If content is already a dict or list, return it directly if isinstance(raw_content, (dict, list)): + logger.info(f"extract_content_payload: returning raw dict/list with keys={list(raw_content.keys()) if isinstance(raw_content, dict) else 'list'}") return raw_content # Try to parse as JSON if isinstance(raw_content, str): # First, try direct JSON parsing try: - return json.loads(raw_content) + parsed = json.loads(raw_content) + logger.info(f"extract_content_payload: parsed JSON, keys={list(parsed.keys()) if isinstance(parsed, dict) else 'list'}") + return parsed except (json.JSONDecodeError, ValueError): pass @@ -156,9 +168,12 @@ def extract_content_payload(message: Any) -> Any: json_candidates = re.findall(r'[\[{].*[\]}]', raw_content, flags=re.DOTALL) for candidate in json_candidates: try: - return json.loads(candidate) + parsed = json.loads(candidate) + logger.info(f"extract_content_payload: parsed JSON from candidate, keys={list(parsed.keys()) if isinstance(parsed, dict) else 'list'}") + return parsed except (json.JSONDecodeError, ValueError): continue # If all parsing attempts fail, return the raw content + logger.info(f"extract_content_payload: returning raw content (parsing failed)") return raw_content diff --git a/api/app/core/memory/agent/langgraph_graph/write_graph.py b/api/app/core/memory/agent/langgraph_graph/write_graph.py index dbdc51d6..ae333e84 100644 --- a/api/app/core/memory/agent/langgraph_graph/write_graph.py +++ b/api/app/core/memory/agent/langgraph_graph/write_graph.py @@ -1,69 +1,71 @@ import asyncio import json -from contextlib import asynccontextmanager -from langgraph.constants import START, END -from langgraph.graph import add_messages, StateGraph - -from langgraph.prebuilt import ToolNode -from app.core.memory.agent.utils.llm_tools import WriteState -import warnings import sys -from langchain_core.messages import AIMessage +import warnings +from contextlib import asynccontextmanager + from app.core.logging_config import get_agent_logger +from app.core.memory.agent.utils.llm_tools import WriteState +from app.schemas.memory_config_schema import MemoryConfig +from langchain_core.messages import AIMessage +from langgraph.constants import END, START +from langgraph.graph import StateGraph +from langgraph.prebuilt import ToolNode warnings.filterwarnings("ignore", category=RuntimeWarning) logger = get_agent_logger(__name__) if sys.platform.startswith("win"): - import asyncio asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -@asynccontextmanager -async def make_write_graph(user_id, tools, apply_id, group_id, config_id=None): - logger.info("加载 MCP 工具: %s", [t.name for t in tools]) - if config_id: - logger.info(f"使用配置 ID: {config_id}") - data_type_tool = next((t for t in tools if t.name == "Data_type_differentiation"), None) + +@asynccontextmanager +async def make_write_graph(user_id, tools, apply_id, group_id, memory_config: MemoryConfig): + """ + Create a write graph workflow for memory operations. + + Args: + user_id: User identifier + tools: MCP tools loaded from session + apply_id: Application identifier + group_id: Group identifier + memory_config: MemoryConfig object containing all configuration + """ + logger.info("Loading MCP tools: %s", [t.name for t in tools]) + logger.info(f"Using memory_config: {memory_config.config_name} (id={memory_config.config_id})") + data_write_tool = next((t for t in tools if t.name == "Data_write"), None) - if not data_type_tool or not data_write_tool: - logger.error('不存在数据存储工具', exc_info=True) - raise ValueError('不存在数据存储工具') - # ToolNode - write_node = ToolNode([data_write_tool]) + if not data_write_tool: + logger.error("Data_write tool not found", exc_info=True) + raise ValueError("Data_write tool not found") + write_node = ToolNode([data_write_tool]) async def call_model(state): messages = state["messages"] last_message = messages[-1] + content = last_message[1] if isinstance(last_message, tuple) else last_message.content - result = await data_type_tool.ainvoke({ - "context": last_message[1] if isinstance(last_message, tuple) else last_message.content - }) - result=json.loads( result) - - # 调用 Data_write，传递 config_id + # Call Data_write directly with memory_config write_params = { - "content": result["context"], + "content": content, "apply_id": apply_id, "group_id": group_id, - "user_id": user_id + "user_id": user_id, + "memory_config": memory_config, } - - # 如果提供了 config_id，添加到参数中 - if config_id: - write_params["config_id"] = config_id - logger.debug(f"传递 config_id 到 Data_write: {config_id}") - + logger.debug(f"Passing memory_config to Data_write: {memory_config.config_id}") + write_result = await data_write_tool.ainvoke(write_params) if isinstance(write_result, dict): - content = write_result.get("data", str(write_result)) + result_content = write_result.get("data", str(write_result)) else: - content = str(write_result) - logger.info("写入内容: %s", content) - return {"messages": [AIMessage(content=content)]} + result_content = str(write_result) + logger.info("Write content: %s", result_content) + return {"messages": [AIMessage(content=result_content)]} workflow = StateGraph(WriteState) workflow.add_node("content_input", call_model) diff --git a/api/app/core/memory/agent/mcp_server/__init__.py b/api/app/core/memory/agent/mcp_server/__init__.py index 61a804c5..efd03773 100644 --- a/api/app/core/memory/agent/mcp_server/__init__.py +++ b/api/app/core/memory/agent/mcp_server/__init__.py @@ -10,19 +10,19 @@ Package structure: - models: Pydantic response models - services: Business logic services """ -from app.core.memory.agent.mcp_server.server import ( - mcp, - initialize_context, - main, - get_context_resource -) +# from app.core.memory.agent.mcp_server.server import ( +# mcp, +# initialize_context, +# main, +# get_context_resource +# ) -# Import tools to register them (but don't export them) -from app.core.memory.agent.mcp_server import tools +# # Import tools to register them (but don't export them) +# from app.core.memory.agent.mcp_server import tools -__all__ = [ - 'mcp', - 'initialize_context', - 'main', - 'get_context_resource', -] \ No newline at end of file +# __all__ = [ +# 'mcp', +# 'initialize_context', +# 'main', +# 'get_context_resource', +# ] \ No newline at end of file diff --git a/api/app/core/memory/agent/mcp_server/server.py b/api/app/core/memory/agent/mcp_server/server.py index 18ea911f..f7a23236 100644 --- a/api/app/core/memory/agent/mcp_server/server.py +++ b/api/app/core/memory/agent/mcp_server/server.py @@ -6,19 +6,15 @@ in the context for dependency injection into tool functions. """ import os import sys -from mcp.server.fastmcp import FastMCP from app.core.config import settings from app.core.logging_config import get_agent_logger -from app.core.memory.agent.utils.redis_tool import RedisSessionStore, store -from app.core.memory.agent.utils.llm_tools import PROJECT_ROOT_ -from app.core.memory.utils.config.definitions import SELECTED_LLM_ID,reload_configuration_from_database -from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.core.memory.agent.mcp_server.services.template_service import TemplateService +from app.core.memory.agent.mcp_server.mcp_instance import mcp from app.core.memory.agent.mcp_server.services.search_service import SearchService from app.core.memory.agent.mcp_server.services.session_service import SessionService -from app.core.memory.agent.mcp_server.mcp_instance import mcp - +from app.core.memory.agent.mcp_server.services.template_service import TemplateService +from app.core.memory.agent.utils.llm_tools import PROJECT_ROOT_ +from app.core.memory.agent.utils.redis_tool import store logger = get_agent_logger(__name__) @@ -78,17 +74,11 @@ def initialize_context(): logger.info("Registering session_store in context") mcp.session_store = store - # Register LLM client - try: - logger.info(f"Registering llm_client in context with model ID: {SELECTED_LLM_ID}") - llm_client = get_llm_client(SELECTED_LLM_ID) - mcp.llm_client = llm_client - logger.info("llm_client registered successfully") - except Exception as e: - logger.error(f"Failed to register llm_client: {e}", exc_info=True) - # 注册一个 None 值，避免工具调用时找不到资源 - mcp.llm_client = None - logger.warning("llm_client set to None due to initialization failure") + # Note: LLM client is NOT loaded at server startup + # It should be loaded dynamically when needed, with config_id passed explicitly + # to make_write_graph or make_read_graph functions + logger.info("LLM client will be loaded dynamically with config_id when needed") + mcp.llm_client = None # Placeholder - actual client loaded per-request with config_id # Register application settings (renamed to avoid conflict with FastMCP's settings) logger.info("Registering app_settings in context") @@ -124,26 +114,20 @@ def main(): Initializes context and starts the server with SSE transport. """ try: - # logger.info("Starting MCP server initialization") - reload_configuration_from_database(config_id=os.getenv("config_id"), force_reload=True) + logger.info("Starting MCP server initialization") # Initialize context resources initialize_context() - # Import and register tools - # logger.info("Importing MCP tools") - from app.core.memory.agent.mcp_server.tools import ( + # Import and register tools (imports trigger tool registration) + from app.core.memory.agent.mcp_server.tools import ( # noqa: F401 + data_tools, problem_tools, retrieval_tools, - verification_tools, summary_tools, - data_tools + verification_tools, ) - # logger.info("All MCP tools imported and registered") - # Log registered tools for debugging - import asyncio - tools_list = asyncio.run(mcp.list_tools()) - # logger.info(f"Registered {len(tools_list)} MCP tools: {[t.name for t in tools_list]}") + # Tools are registered via imports above # Get MCP port from environment (default: 8081) mcp_port = int(os.getenv("MCP_PORT", "8081")) diff --git a/api/app/core/memory/agent/mcp_server/services/parameter_builder.py b/api/app/core/memory/agent/mcp_server/services/parameter_builder.py index 03995a1e..d5305dc6 100644 --- a/api/app/core/memory/agent/mcp_server/services/parameter_builder.py +++ b/api/app/core/memory/agent/mcp_server/services/parameter_builder.py @@ -4,22 +4,22 @@ Parameter Builder for constructing tool call arguments. This service provides tool-specific parameter transformation logic to build correct arguments for each tool type. """ -import json + from typing import Any, Dict, Optional from app.core.logging_config import get_agent_logger - +from app.schemas.memory_config_schema import MemoryConfig logger = get_agent_logger(__name__) class ParameterBuilder: """Service for building tool call arguments based on tool type.""" - + def __init__(self): """Initialize the parameter builder.""" logger.info("ParameterBuilder initialized") - + def build_tool_args( self, tool_name: str, @@ -28,8 +28,9 @@ class ParameterBuilder: search_switch: str, apply_id: str, group_id: str, + memory_config: MemoryConfig, storage_type: Optional[str] = None, - user_rag_memory_id: Optional[str] = None + user_rag_memory_id: Optional[str] = None, ) -> Dict[str, Any]: """ Build tool arguments based on tool type. @@ -48,6 +49,7 @@ class ParameterBuilder: search_switch: Search routing parameter apply_id: Application identifier group_id: Group identifier + memory_config: MemoryConfig object containing all configuration storage_type: Storage type for the workspace (optional) user_rag_memory_id: User RAG memory ID for knowledge base retrieval (optional) @@ -58,18 +60,19 @@ class ParameterBuilder: base_args = { "usermessages": tool_call_id, "apply_id": apply_id, - "group_id": group_id + "group_id": group_id, + "memory_config": memory_config, } - + # Always add storage_type and user_rag_memory_id (with defaults if None) base_args["storage_type"] = storage_type if storage_type is not None else "" base_args["user_rag_memory_id"] = user_rag_memory_id if user_rag_memory_id is not None else "" # Tool-specific argument construction - if tool_name in ["Verify","Summary", "Summary_fails",'Retrieve_Summary']: - # Verify expects dict context + if tool_name in ["Verify", "Summary", "Summary_fails", "Retrieve_Summary", "Problem_Extension"]: + # These tools expect dict context return { - "context": content if isinstance(content, dict) else {}, + "context": content if isinstance(content, dict) else {"content": content}, **base_args } diff --git a/api/app/core/memory/agent/mcp_server/services/search_service.py b/api/app/core/memory/agent/mcp_server/services/search_service.py index 28dd82c7..b0a007cd 100644 --- a/api/app/core/memory/agent/mcp_server/services/search_service.py +++ b/api/app/core/memory/agent/mcp_server/services/search_service.py @@ -4,21 +4,31 @@ Search Service for executing hybrid search and processing results. This service provides clean search result processing with content extraction and deduplication. """ -from typing import List, Tuple, Optional + +from typing import TYPE_CHECKING, List, Optional, Tuple from app.core.logging_config import get_agent_logger from app.core.memory.src.search import run_hybrid_search from app.core.memory.utils.data.text_utils import escape_lucene_query +if TYPE_CHECKING: + from app.schemas.memory_config_schema import MemoryConfig logger = get_agent_logger(__name__) class SearchService: """Service for executing hybrid search and processing results.""" - - def __init__(self): - """Initialize the search service.""" + + def __init__(self, memory_config: "MemoryConfig" = None): + """ + Initialize the search service. + + Args: + memory_config: Optional MemoryConfig for embedding model configuration. + If not provided, must be passed to execute_hybrid_search. + """ + self.memory_config = memory_config logger.info("SearchService initialized") def extract_content_from_result(self, result: dict) -> str: @@ -93,12 +103,13 @@ class SearchService: self, group_id: str, question: str, - limit: int = 5, + limit: int = 15, search_type: str = "hybrid", include: Optional[List[str]] = None, rerank_alpha: float = 0.4, output_path: str = "search_results.json", - return_raw_results: bool = False + return_raw_results: bool = False, + memory_config: "MemoryConfig" = None, ) -> Tuple[str, str, Optional[dict]]: """ Execute hybrid search and return clean content. @@ -112,6 +123,7 @@ class SearchService: rerank_alpha: Weight for BM25 scores in reranking (default: 0.4) output_path: Path to save search results (default: "search_results.json") return_raw_results: If True, also return the raw search results as third element (default: False) + memory_config: MemoryConfig object for embedding model. Falls back to self.memory_config if not provided. Returns: Tuple of (clean_content, cleaned_query, raw_results) @@ -119,12 +131,17 @@ class SearchService: """ if include is None: include = ["statements", "chunks", "entities", "summaries"] - + + # Use provided memory_config or fall back to instance config + config = memory_config or self.memory_config + if not config: + raise ValueError("memory_config is required for search - either pass it to __init__ or execute_hybrid_search") + # Clean query cleaned_query = self.clean_query(question) - + try: - # Execute search + # Execute search using embedding_model_id from memory_config answer = await run_hybrid_search( query_text=cleaned_query, search_type=search_type, @@ -132,7 +149,8 @@ class SearchService: limit=limit, include=include, output_path=output_path, - rerank_alpha=rerank_alpha + embedding_id=str(config.embedding_model_id), + rerank_alpha=rerank_alpha, ) # Extract results based on search type and include parameter diff --git a/api/app/core/memory/agent/mcp_server/tools/data_tools.py b/api/app/core/memory/agent/mcp_server/tools/data_tools.py index 283aa6b6..22dadd7f 100644 --- a/api/app/core/memory/agent/mcp_server/tools/data_tools.py +++ b/api/app/core/memory/agent/mcp_server/tools/data_tools.py @@ -3,16 +3,19 @@ Data Tools for data type differentiation and writing. This module contains MCP tools for distinguishing data types and writing data. """ -import os -from mcp.server.fastmcp import Context +import os from app.core.logging_config import get_agent_logger from app.core.memory.agent.mcp_server.mcp_instance import mcp +from app.core.memory.agent.mcp_server.models.retrieval_models import ( + DistinguishTypeResponse, +) from app.core.memory.agent.mcp_server.server import get_context_resource -from app.core.memory.agent.mcp_server.models.retrieval_models import DistinguishTypeResponse from app.core.memory.agent.utils.write_tools import write - +from app.core.memory.utils.llm.llm_utils import get_llm_client_from_config +from app.schemas.memory_config_schema import MemoryConfig +from mcp.server.fastmcp import Context logger = get_agent_logger(__name__) @@ -20,7 +23,8 @@ logger = get_agent_logger(__name__) @mcp.tool() async def Data_type_differentiation( ctx: Context, - context: str + context: str, + memory_config: MemoryConfig, ) -> dict: """ Distinguish the type of data (read or write). @@ -28,6 +32,7 @@ async def Data_type_differentiation( Args: ctx: FastMCP context for dependency injection context: Text to analyze for type differentiation + memory_config: MemoryConfig object containing LLM configuration Returns: dict: Contains 'context' with the original text and 'type' field @@ -35,7 +40,9 @@ async def Data_type_differentiation( try: # Extract services from context template_service = get_context_resource(ctx, 'template_service') - llm_client = get_context_resource(ctx, 'llm_client') + + # Get LLM client from memory_config + llm_client = get_llm_client_from_config(memory_config) # Render template try: @@ -53,7 +60,7 @@ async def Data_type_differentiation( "type": "error", "message": f"Prompt rendering failed: {str(e)}" } - + # Call LLM with structured response try: structured = await llm_client.response_structured( @@ -98,7 +105,7 @@ async def Data_write( user_id: str, apply_id: str, group_id: str, - config_id: str + memory_config: MemoryConfig, ) -> dict: """ Write data to the database/file system. @@ -109,7 +116,7 @@ async def Data_write( user_id: User identifier apply_id: Application identifier group_id: Group identifier - config_id: Configuration ID for processing (optional, integer) + memory_config: MemoryConfig object containing all configuration Returns: dict: Contains 'status', 'saved_to', and 'data' fields @@ -118,32 +125,28 @@ async def Data_write( # Ensure output directory exists os.makedirs("data_output", exist_ok=True) file_path = os.path.join("data_output", "user_data.csv") - - # Write data using utility function - try: - await write(content, user_id, apply_id, group_id, config_id=config_id) - logger.info(f"写入成功！Config ID: {config_id if config_id else 'None'}") - - return { - "status": "success", - "saved_to": file_path, - "data": content, - "config_id": config_id - } - - except Exception as e: - logger.error(f"写入失败: {e}", exc_info=True) - return { - "status": "error", - "message": str(e) - } - - except Exception as e: - logger.error( - f"Data_write failed: {e}", - exc_info=True + + # Write data - clients are constructed inside write() from memory_config + await write( + content=content, + user_id=user_id, + apply_id=apply_id, + group_id=group_id, + memory_config=memory_config, ) + logger.info(f"Write completed successfully! Config: {memory_config.config_name}") + + return { + "status": "success", + "saved_to": file_path, + "data": content, + "config_id": memory_config.config_id, + "config_name": memory_config.config_name, + } + + except Exception as e: + logger.error(f"Data_write failed: {e}", exc_info=True) return { "status": "error", - "message": str(e) + "message": str(e), } diff --git a/api/app/core/memory/agent/mcp_server/tools/problem_tools.py b/api/app/core/memory/agent/mcp_server/tools/problem_tools.py index 5232fd68..892fdbdd 100644 --- a/api/app/core/memory/agent/mcp_server/tools/problem_tools.py +++ b/api/app/core/memory/agent/mcp_server/tools/problem_tools.py @@ -2,25 +2,23 @@ Problem Tools for question segmentation and extension. This module contains MCP tools for breaking down and extending user questions. +LLM clients are constructed from MemoryConfig when needed. """ + import json import time -from typing import List - -from pydantic import BaseModel, Field, RootModel -from mcp.server.fastmcp import Context from app.core.logging_config import get_agent_logger, log_time from app.core.memory.agent.mcp_server.mcp_instance import mcp -from app.core.memory.agent.mcp_server.server import get_context_resource from app.core.memory.agent.mcp_server.models.problem_models import ( - ProblemBreakdownItem, ProblemBreakdownResponse, - ExtendedQuestionItem, - ProblemExtensionResponse + ProblemExtensionResponse, ) +from app.core.memory.agent.mcp_server.server import get_context_resource from app.core.memory.agent.utils.messages_tool import Problem_Extension_messages_deal - +from app.core.memory.utils.llm.llm_utils import get_llm_client_from_config +from app.schemas.memory_config_schema import MemoryConfig +from mcp.server.fastmcp import Context logger = get_agent_logger(__name__) @@ -32,7 +30,8 @@ async def Split_The_Problem( sessionid: str, messages_id: str, apply_id: str, - group_id: str + group_id: str, + memory_config: MemoryConfig, ) -> dict: """ Segment the dialogue or sentence into sub-problems. @@ -44,17 +43,20 @@ async def Split_The_Problem( messages_id: Message identifier apply_id: Application identifier group_id: Group identifier + memory_config: MemoryConfig object containing all configuration Returns: dict: Contains 'context' (JSON string of split results) and 'original' sentence """ start = time.time() - + try: # Extract services from context - template_service = get_context_resource(ctx, 'template_service') - session_service = get_context_resource(ctx, 'session_service') - llm_client = get_context_resource(ctx, 'llm_client') + template_service = get_context_resource(ctx, "template_service") + session_service = get_context_resource(ctx, "session_service") + + # Get LLM client from memory_config + llm_client = get_llm_client_from_config(memory_config) # Extract user ID from session user_id = session_service.resolve_user_id(sessionid) @@ -116,8 +118,8 @@ async def Split_The_Problem( ) split_result = json.dumps([], ensure_ascii=False) - logger.info("问题拆分") - logger.info(f"问题拆分结果==>>:{split_result}") + logger.info("Problem splitting") + logger.info(f"Problem split result: {split_result}") # Emit intermediate output for frontend result = { @@ -150,7 +152,7 @@ async def Split_The_Problem( duration = end - start except Exception: duration = 0.0 - log_time('问题拆分', duration) + log_time('Problem splitting', duration) @mcp.tool() @@ -160,8 +162,9 @@ async def Problem_Extension( usermessages: str, apply_id: str, group_id: str, + memory_config: MemoryConfig, storage_type: str = "", - user_rag_memory_id: str = "" + user_rag_memory_id: str = "", ) -> dict: """ Extend the problem with additional sub-questions. @@ -172,6 +175,7 @@ async def Problem_Extension( usermessages: User messages identifier apply_id: Application identifier group_id: Group identifier + memory_config: MemoryConfig object containing all configuration storage_type: Storage type for the workspace (optional) user_rag_memory_id: User RAG memory identifier (optional) @@ -179,12 +183,14 @@ async def Problem_Extension( dict: Contains 'context' (aggregated questions) and 'original' question """ start = time.time() - + try: # Extract services from context - template_service = get_context_resource(ctx, 'template_service') - session_service = get_context_resource(ctx, 'session_service') - llm_client = get_context_resource(ctx, 'llm_client') + template_service = get_context_resource(ctx, "template_service") + session_service = get_context_resource(ctx, "session_service") + + # Get LLM client from memory_config + llm_client = get_llm_client_from_config(memory_config) # Resolve session ID from usermessages from app.core.memory.agent.utils.messages_tool import Resolve_username @@ -250,8 +256,8 @@ async def Problem_Extension( ) aggregated_dict = {} - logger.info("问题扩展") - logger.info(f"问题扩展==>>:{aggregated_dict}") + logger.info("Problem extension") + logger.info(f"Problem extension result: {aggregated_dict}") # Emit intermediate output for frontend result = { @@ -290,4 +296,4 @@ async def Problem_Extension( duration = end - start except Exception: duration = 0.0 - log_time('问题扩展', duration) + log_time('Problem extension', duration) diff --git a/api/app/core/memory/agent/mcp_server/tools/retrieval_tools.py b/api/app/core/memory/agent/mcp_server/tools/retrieval_tools.py index 0a71e890..db18ba04 100644 --- a/api/app/core/memory/agent/mcp_server/tools/retrieval_tools.py +++ b/api/app/core/memory/agent/mcp_server/tools/retrieval_tools.py @@ -3,25 +3,24 @@ Retrieval Tools for database and context retrieval. This module contains MCP tools for retrieving data using hybrid search. """ -from dotenv import load_dotenv + import os - -from app.core.rag.nlp.search import knowledge_retrieval - -# 加载.env文件 -load_dotenv() import time -from typing import List - -from mcp.server.fastmcp import Context from app.core.logging_config import get_agent_logger, log_time from app.core.memory.agent.mcp_server.mcp_instance import mcp from app.core.memory.agent.mcp_server.server import get_context_resource -from app.core.memory.agent.utils.llm_tools import deduplicate_entries, merge_to_key_value_pairs +from app.core.memory.agent.utils.llm_tools import ( + deduplicate_entries, + merge_to_key_value_pairs, +) from app.core.memory.agent.utils.messages_tool import Retriev_messages_deal +from app.core.rag.nlp.search import knowledge_retrieval +from app.schemas.memory_config_schema import MemoryConfig +from dotenv import load_dotenv +from mcp.server.fastmcp import Context - +load_dotenv() logger = get_agent_logger(__name__) @@ -32,8 +31,9 @@ async def Retrieve( usermessages: str, apply_id: str, group_id: str, + memory_config: MemoryConfig, storage_type: str = "", - user_rag_memory_id: str = "" + user_rag_memory_id: str = "", ) -> dict: """ Retrieve data from the database using hybrid search. @@ -44,6 +44,7 @@ async def Retrieve( usermessages: User messages identifier apply_id: Application identifier group_id: Group identifier + memory_config: MemoryConfig object containing all configuration storage_type: Storage type for the workspace (e.g., 'rag', 'vector') user_rag_memory_id: User RAG memory identifier @@ -66,6 +67,7 @@ async def Retrieve( } start = time.time() logger.info(f"Retrieve: storage_type={storage_type}, user_rag_memory_id={user_rag_memory_id}") + logger.info(f"Retrieve: context type={type(context)}, context={str(context)[:500]}") try: # Extract services from context @@ -77,7 +79,13 @@ async def Retrieve( if isinstance(context, dict): # Process dict context with extended questions all_items = [] + logger.info(f"Retrieve: context keys={list(context.keys())}") content, original = await Retriev_messages_deal(context) + logger.info(f"Retrieve: after Retriev_messages_deal - content_type={type(content)}, content={str(content)[:300]}") + logger.info(f"Retrieve: original='{original[:100] if original else 'EMPTY'}'") + + if not original: + logger.warning(f"Retrieve: original query is empty! context={context}") # Extract all query items from content # content is like {original_question: [extended_questions...], ...} @@ -113,9 +121,11 @@ async def Retrieve( clean_content = '' raw_results='' cleaned_query = question - logger.info(f"知识库没有检索的内容{user_rag_memory_id}") + logger.info(f"No content retrieved from knowledge base: {user_rag_memory_id}") else: - clean_content, cleaned_query, raw_results = await search_service.execute_hybrid_search(**search_params) + clean_content, cleaned_query, raw_results = await search_service.execute_hybrid_search( + **search_params, memory_config=memory_config + ) databases_anser.append({ "Query_small": cleaned_query, @@ -206,9 +216,11 @@ async def Retrieve( clean_content = '' raw_results = '' cleaned_query = query - logger.info(f"知识库没有检索的内容{user_rag_memory_id}") + logger.info(f"No content retrieved from knowledge base: {user_rag_memory_id}") else: - clean_content, cleaned_query, raw_results = await search_service.execute_hybrid_search(**search_params) + clean_content, cleaned_query, raw_results = await search_service.execute_hybrid_search( + **search_params, memory_config=memory_config + ) # Keep structure for Verify/Retrieve_Summary compatibility dup_databases = { "Query": cleaned_query, @@ -236,7 +248,7 @@ async def Retrieve( } logger.info( - f"检索==>>:{storage_type}--{user_rag_memory_id}--Query={dup_databases.get('Query', '')}, " + f"Retrieval: {storage_type}--{user_rag_memory_id}--Query={dup_databases.get('Query', '')}, " f"Expansion_issue count={len(dup_databases.get('Expansion_issue', []))}" ) @@ -279,4 +291,4 @@ async def Retrieve( duration = end - start except Exception: duration = 0.0 - log_time('检索', duration) + log_time('Retrieval', duration) diff --git a/api/app/core/memory/agent/mcp_server/tools/summary_tools.py b/api/app/core/memory/agent/mcp_server/tools/summary_tools.py index 695fa296..8b6b7ae4 100644 --- a/api/app/core/memory/agent/mcp_server/tools/summary_tools.py +++ b/api/app/core/memory/agent/mcp_server/tools/summary_tools.py @@ -2,33 +2,31 @@ Summary Tools for data summarization. This module contains MCP tools for summarizing retrieved data and generating responses. +LLM clients are constructed from MemoryConfig when needed. """ + import json +import os import re import time -from typing import List - -from pydantic import BaseModel, Field -from mcp.server.fastmcp import Context from app.core.logging_config import get_agent_logger, log_time from app.core.memory.agent.mcp_server.mcp_instance import mcp -from app.core.memory.agent.mcp_server.server import get_context_resource from app.core.memory.agent.mcp_server.models.summary_models import ( - SummaryData, + RetrieveSummaryResponse, SummaryResponse, - RetrieveSummaryData, - RetrieveSummaryResponse ) +from app.core.memory.agent.mcp_server.server import get_context_resource from app.core.memory.agent.utils.messages_tool import ( + Resolve_username, Summary_messages_deal, - Resolve_username ) +from app.core.memory.utils.llm.llm_utils import get_llm_client_from_config from app.core.rag.nlp.search import knowledge_retrieval +from app.schemas.memory_config_schema import MemoryConfig from dotenv import load_dotenv -import os +from mcp.server.fastmcp import Context -# 加载.env文件 load_dotenv() logger = get_agent_logger(__name__) @@ -40,8 +38,9 @@ async def Summary( usermessages: str, apply_id: str, group_id: str, + memory_config: MemoryConfig, storage_type: str = "", - user_rag_memory_id: str = "" + user_rag_memory_id: str = "", ) -> dict: """ Summarize the verified data. @@ -52,6 +51,7 @@ async def Summary( usermessages: User messages identifier apply_id: Application identifier group_id: Group identifier + memory_config: MemoryConfig object containing all configuration storage_type: Storage type for the workspace (optional) user_rag_memory_id: User RAG memory identifier (optional) @@ -59,12 +59,14 @@ async def Summary( dict: Contains 'status' and 'summary_result' """ start = time.time() - + try: # Extract services from context - template_service = get_context_resource(ctx, 'template_service') - session_service = get_context_resource(ctx, 'session_service') - llm_client = get_context_resource(ctx, 'llm_client') + template_service = get_context_resource(ctx, "template_service") + session_service = get_context_resource(ctx, "session_service") + + # Get LLM client from memory_config + llm_client = get_llm_client_from_config(memory_config) # Resolve session ID sessionid = Resolve_username(usermessages) @@ -155,7 +157,7 @@ async def Summary( if aimessages == '': aimessages = '信息不足，无法回答' - logger.info(f"验证之后的总结==>>:{aimessages}") + logger.info(f"Summary after verification: {aimessages}") # Log execution time end = time.time() @@ -163,7 +165,7 @@ async def Summary( duration = end - start except Exception: duration = 0.0 - log_time('总结', duration) + log_time('Summary', duration) return { "status": "success", @@ -180,8 +182,9 @@ async def Retrieve_Summary( usermessages: str, apply_id: str, group_id: str, + memory_config: MemoryConfig, storage_type: str = "", - user_rag_memory_id: str = "" + user_rag_memory_id: str = "", ) -> dict: """ Summarize data directly from retrieval results. @@ -192,6 +195,7 @@ async def Retrieve_Summary( usermessages: User messages identifier apply_id: Application identifier group_id: Group identifier + memory_config: MemoryConfig object containing all configuration storage_type: Storage type for the workspace (optional) user_rag_memory_id: User RAG memory identifier (optional) @@ -202,9 +206,11 @@ async def Retrieve_Summary( try: # Extract services from context - template_service = get_context_resource(ctx, 'template_service') - session_service = get_context_resource(ctx, 'session_service') - llm_client = get_context_resource(ctx, 'llm_client') + template_service = get_context_resource(ctx, "template_service") + session_service = get_context_resource(ctx, "session_service") + + # Get LLM client from memory_config + llm_client = get_llm_client_from_config(memory_config) # Resolve session ID sessionid = Resolve_username(usermessages) @@ -212,6 +218,8 @@ async def Retrieve_Summary( # Handle both 'content' and 'context' keys (LangGraph uses 'content') + logger.debug(f"Retrieve_Summary: raw context type={type(context)}, keys={list(context.keys()) if isinstance(context, dict) else 'N/A'}") + if isinstance(context, dict): if "content" in context: inner = context["content"] @@ -252,17 +260,19 @@ async def Retrieve_Summary( query = context_dict.get("Query", "") expansion_issue = context_dict.get("Expansion_issue", []) + + logger.debug(f"Retrieve_Summary: query='{query}', expansion_issue count={len(expansion_issue)}") + logger.debug(f"Retrieve_Summary: expansion_issue={expansion_issue[:2] if expansion_issue else 'empty'}") # Extract retrieve_info from expansion_issue retrieve_info = [] for item in expansion_issue: - # Check for both Answer_Small and Answer_Samll (typo) for backward compatibility + # Check for both Answer_Small and Answer_Small (typo) for backward compatibility answer = None if isinstance(item, dict): if "Answer_Small" in item: answer = item["Answer_Small"] - elif "Answer_Samll" in item: - answer = item["Answer_Samll"] + if answer is not None: # Handle both string and list formats @@ -350,7 +360,7 @@ async def Retrieve_Summary( if aimessages == '': aimessages = '信息不足，无法回答' - logger.info(f"检索之后的总结==>>:{aimessages}") + logger.info(f"Summary after retrieval: {aimessages}") # Log execution time end = time.time() @@ -358,7 +368,7 @@ async def Retrieve_Summary( duration = end - start except Exception: duration = 0.0 - log_time('检索总结', duration) + log_time('Retrieval summary', duration) # Emit intermediate output for frontend return { @@ -384,8 +394,9 @@ async def Input_Summary( search_switch: str, apply_id: str, group_id: str, + memory_config: MemoryConfig, storage_type: str = "", - user_rag_memory_id: str = "" + user_rag_memory_id: str = "", ) -> dict: """ Generate a quick summary for direct input without verification. @@ -397,6 +408,7 @@ async def Input_Summary( search_switch: Search switch value for routing ('2' for summaries only) apply_id: Application identifier group_id: Group identifier + memory_config: MemoryConfig object containing all configuration storage_type: Storage type for the workspace (e.g., 'rag', 'vector') user_rag_memory_id: User RAG memory identifier @@ -406,21 +418,14 @@ async def Input_Summary( start = time.time() logger.info(f"Input_Summary: storage_type={storage_type}, user_rag_memory_id={user_rag_memory_id}") - # Initialize variables to avoid UnboundLocalError - - try: # Extract services from context - template_service = get_context_resource(ctx, 'template_service') - session_service = get_context_resource(ctx, 'session_service') - llm_client = get_context_resource(ctx, 'llm_client') - search_service = get_context_resource(ctx, 'search_service') + template_service = get_context_resource(ctx, "template_service") + session_service = get_context_resource(ctx, "session_service") + search_service = get_context_resource(ctx, "search_service") - # Check if llm_client is None - if llm_client is None: - error_msg = "LLM client is not available. Please check server configuration and SELECTED_LLM_ID environment variable." - logger.error(error_msg) - return error_msg + # Get LLM client from memory_config + llm_client = get_llm_client_from_config(memory_config) # Resolve session ID sessionid = Resolve_username(usermessages) or "" @@ -479,7 +484,7 @@ async def Input_Summary( # Add storage-specific parameters - '''检索''' + # Retrieval if search_switch == '2': search_params["include"] = ["summaries"] if storage_type == "rag" and user_rag_memory_id: @@ -509,12 +514,16 @@ async def Input_Summary( except: retrieve_info='' raw_results=[''] - logger.info(f"知识库没有检索的内容{user_rag_memory_id}") + logger.info(f"No content retrieved from knowledge base: {user_rag_memory_id}") else: - retrieve_info, question, raw_results = await search_service.execute_hybrid_search(**search_params) - logger.info("Input_Summary: 使用 summary 进行检索") + retrieve_info, question, raw_results = await search_service.execute_hybrid_search( + **search_params, memory_config=memory_config + ) + logger.info("Input_Summary: Using summary for retrieval") else: - retrieve_info, question, raw_results = await search_service.execute_hybrid_search(**search_params) + retrieve_info, question, raw_results = await search_service.execute_hybrid_search( + **search_params, memory_config=memory_config + ) except Exception as e: logger.error( @@ -547,7 +556,7 @@ async def Input_Summary( ) aimessages = "信息不足，无法回答" - logger.info(f"快速答案总结==>>:{storage_type}--{user_rag_memory_id}--{aimessages}") + logger.info(f"Quick answer summary: {storage_type}--{user_rag_memory_id}--{aimessages}") # Emit intermediate output for frontend return { @@ -587,7 +596,7 @@ async def Input_Summary( duration = end - start except Exception: duration = 0.0 - log_time('检索', duration) + log_time('Retrieval', duration) @mcp.tool() diff --git a/api/app/core/memory/agent/mcp_server/tools/verification_tools.py b/api/app/core/memory/agent/mcp_server/tools/verification_tools.py index fb3ca561..cb6af5bd 100644 --- a/api/app/core/memory/agent/mcp_server/tools/verification_tools.py +++ b/api/app/core/memory/agent/mcp_server/tools/verification_tools.py @@ -5,20 +5,19 @@ This module contains MCP tools for verifying retrieved data. """ import time -from jinja2 import Template -from mcp.server.fastmcp import Context - from app.core.logging_config import get_agent_logger, log_time from app.core.memory.agent.mcp_server.mcp_instance import mcp from app.core.memory.agent.mcp_server.server import get_context_resource -from app.core.memory.agent.utils.verify_tool import VerifyTool -from app.core.memory.agent.utils.messages_tool import ( - Verify_messages_deal, - Retrieve_verify_tool_messages_deal, - Resolve_username -) from app.core.memory.agent.utils.llm_tools import PROJECT_ROOT_ - +from app.core.memory.agent.utils.messages_tool import ( + Resolve_username, + Retrieve_verify_tool_messages_deal, + Verify_messages_deal, +) +from app.core.memory.agent.utils.verify_tool import VerifyTool +from app.schemas.memory_config_schema import MemoryConfig +from jinja2 import Template +from mcp.server.fastmcp import Context logger = get_agent_logger(__name__) @@ -30,6 +29,7 @@ async def Verify( usermessages: str, apply_id: str, group_id: str, + memory_config: MemoryConfig, storage_type: str = "", user_rag_memory_id: str = "" ) -> dict: @@ -42,6 +42,7 @@ async def Verify( usermessages: User messages identifier apply_id: Application identifier group_id: Group identifier + memory_config: MemoryConfig object containing all configuration storage_type: Storage type for the workspace (optional) user_rag_memory_id: User RAG memory identifier (optional) @@ -91,8 +92,12 @@ async def Verify( - # Call verification workflow - verify_tool = VerifyTool(system_prompt, messages) + # Call verification workflow with LLM model ID from memory_config + verify_tool = VerifyTool( + system_prompt=system_prompt, + verify_data=messages, + llm_model_id=str(memory_config.llm_model_id) + ) verify_result = await verify_tool.verify() # Parse LLM verification result with error handling @@ -118,7 +123,7 @@ async def Verify( "history": history, } - logger.info(f"验证==>>:{messages_deal}") + logger.info(f"Verification result: {messages_deal}") # Emit intermediate output for frontend return { @@ -128,7 +133,7 @@ async def Verify( "user_rag_memory_id": user_rag_memory_id, "_intermediate": { "type": "verification", - "title": "数据验证", + "title": "Data Verification", "result": messages_deal.get("split_result", "unknown"), "reason": messages_deal.get("reason", ""), "query": query, @@ -166,4 +171,4 @@ async def Verify( duration = end - start except Exception: duration = 0.0 - log_time('验证', duration) + log_time('Verification', duration) diff --git a/api/app/core/memory/agent/utils/llm_tools.py b/api/app/core/memory/agent/utils/llm_tools.py index f626db82..ec22b628 100644 --- a/api/app/core/memory/agent/utils/llm_tools.py +++ b/api/app/core/memory/agent/utils/llm_tools.py @@ -1,22 +1,21 @@ import asyncio import json -from collections import defaultdict -from typing import TypedDict, Annotated -import os import logging - -from jinja2 import Template -from langchain_core.messages import AnyMessage -from dotenv import load_dotenv -from langgraph.graph import add_messages -from openai import OpenAI +import os +from collections import defaultdict +from typing import Annotated, TypedDict from app.core.memory.agent.utils.messages_tool import read_template_file -from app.core.memory.utils.config.config_utils import get_picture_config, get_voice_config -from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.core.memory.utils.config.definitions import SELECTED_LLM_ID, SELECTED_LLM_PICTURE_NAME, SELECTED_LLM_VOICE_NAME -from app.core.models.base import RedBearModelConfig -from app.core.memory.llm_tools.openai_client import OpenAIClient +from app.core.memory.utils.config.config_utils import ( + get_picture_config, + get_voice_config, +) + +# Removed global variable imports - use dependency injection instead +from dotenv import load_dotenv +from langchain_core.messages import AnyMessage +from langgraph.graph import add_messages +from openai import OpenAI PROJECT_ROOT_ = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) logger = logging.getLogger(__name__) @@ -44,6 +43,7 @@ class WriteState(TypedDict): user_id:str apply_id:str group_id:str + errors: list[dict] # Track errors: [{"tool": "tool_name", "error": "message"}] class ReadState(TypedDict): ''' @@ -53,6 +53,7 @@ class ReadState(TypedDict): loop_count:Traverse times search_switch：type config_id: configuration id for filtering results + errors: list of errors that occurred during workflow execution ''' messages: Annotated[list[AnyMessage], add_messages] #消息追加的模式增加消息 name: str @@ -63,6 +64,7 @@ class ReadState(TypedDict): apply_id: str group_id: str config_id: str + errors: list[dict] # Track errors: [{"tool": "tool_name", "error": "message"}] class COUNTState: @@ -109,9 +111,17 @@ def deduplicate_entries(entries): -async def Picture_recognize(image_path,PROMPT_TICKET_EXTRACTION) -> str: +async def Picture_recognize(image_path, PROMPT_TICKET_EXTRACTION, picture_model_name: str) -> str: + """ + Updated to eliminate global variables in favor of explicit parameters. + + Args: + image_path: Path to image file + PROMPT_TICKET_EXTRACTION: Extraction prompt + picture_model_name: Picture model name (required, no longer from global variables) + """ try: - model_config = get_picture_config(SELECTED_LLM_PICTURE_NAME) + model_config = get_picture_config(picture_model_name) except Exception as e: err = f"LLM配置不可用：{str(e)}。请检查 config.json 和 runtime.json。" logger.error(err) @@ -147,9 +157,15 @@ async def Picture_recognize(image_path,PROMPT_TICKET_EXTRACTION) -> str: picture_text = json.loads(picture_text) return (picture_text['statement']) -async def Voice_recognize(): +async def Voice_recognize(voice_model_name: str): + """ + Updated to eliminate global variables in favor of explicit parameters. + + Args: + voice_model_name: Voice model name (required, no longer from global variables) + """ try: - model_config = get_voice_config(SELECTED_LLM_VOICE_NAME) + model_config = get_voice_config(voice_model_name) except Exception as e: err = f"LLM配置不可用：{str(e)}。请检查 config.json 和 runtime.json。" logger.error(err) diff --git a/api/app/core/memory/agent/utils/messages_tool.py b/api/app/core/memory/agent/utils/messages_tool.py index ec69e925..769e795a 100644 --- a/api/app/core/memory/agent/utils/messages_tool.py +++ b/api/app/core/memory/agent/utils/messages_tool.py @@ -1,10 +1,10 @@ import json import logging import re -from typing import List, Any +from typing import Any, List -from langchain_core.messages import AnyMessage from app.core.logging_config import get_agent_logger +from langchain_core.messages import AnyMessage logger = get_agent_logger(__name__) @@ -119,11 +119,23 @@ async def Problem_Extension_messages_deal(context): extent_quest = [] original = context.get('original', '') messages = context.get('context', '') - messages = json.loads(messages) - for message in messages: - question = message.get('question', '') - type = message.get('type', '') - extent_quest.append({"role": "user", "content": f"问题:{question}；问题类型：{type}"}) + + # Handle empty or non-string messages + if not messages: + return extent_quest, original + + if isinstance(messages, str): + try: + messages = json.loads(messages) + except json.JSONDecodeError: + # If JSON parsing fails, return empty list + return extent_quest, original + + if isinstance(messages, list): + for message in messages: + question = message.get('question', '') + type = message.get('type', '') + extent_quest.append({"role": "user", "content": f"问题:{question}；问题类型：{type}"}) return extent_quest, original @@ -135,10 +147,19 @@ async def Retriev_messages_deal(context): context: Returns: ''' + logger.info(f"Retriev_messages_deal input: type={type(context)}, value={str(context)[:500]}") + if isinstance(context, dict): + logger.info(f"Retriev_messages_deal: context is dict with keys={list(context.keys())}") if 'context' in context or 'original' in context: - return context.get('context', {}), context.get('original', '') - return content, original_value + content = context.get('context', {}) + original = context.get('original', '') + logger.info(f"Retriev_messages_deal output: content_type={type(content)}, content={str(content)[:300]}, original='{original[:50] if original else ''}'") + return content, original + + # Return empty defaults if context is not a dict or doesn't have expected keys + logger.warning(f"Retriev_messages_deal: context missing expected keys, returning empty defaults") + return {}, '' async def Verify_messages_deal(context): ''' diff --git a/api/app/core/memory/agent/utils/prompt/split_verify_prompt.jinja2 b/api/app/core/memory/agent/utils/prompt/split_verify_prompt.jinja2 index 6cdbaf6a..f4d4665c 100644 --- a/api/app/core/memory/agent/utils/prompt/split_verify_prompt.jinja2 +++ b/api/app/core/memory/agent/utils/prompt/split_verify_prompt.jinja2 @@ -1,22 +1,22 @@ # 角色 你是验证专家 -你的目标是针对用户的输入Query_Samll字段的提问和Answer_Samll的回答分析，是不是回答Query_Samll这个字段的问题 +你的目标是针对用户的输入Query_Small字段的提问和Answer_Small的回答分析，是不是回答Query_Small这个字段的问题 {#以下可以采用先总括，再展开详细说明的方式，描述你希望智能体在每一个步骤如何进行工作，具体的工作步骤数量可以根据实际需求增删#} ## 工作步骤 -1. 获取所有的Query_Samll字段和Answer_Samll字段 -2. 分析Answer_Samll的回复是不是和Query_Samll有关系 -3. 判断Answer_Samll和Query_Samll之间分析出来的关系状态 +1. 获取所有的Query_Small字段和Answer_Small字段 +2. 分析Answer_Small的回复是不是和Query_Small有关系 +3. 判断Answer_Small和Query_Small之间分析出来的关系状态 4. 如果是True保留，否则不要相对应的问题和回答 5. 输出，需要严格按照模版 输入：{{history}} 历史消息：{"history":{{sentence}}} ### 第一步 获取用户的输入 -获取用户的输入提取对应的Query_Samll和Answer_Samll +获取用户的输入提取对应的Query_Small和Answer_Small ### 第二步 分析验证 -需要分析Query_Samll和Answer_Samll之间的关系可以参考history字段的内容，如果有关系不是答非所问 +需要分析Query_Small和Answer_Small之间的关系可以参考history字段的内容，如果有关系不是答非所问 ## 核心验证标准 -在评估子问题拆分时，必须严格遵循以下标准，且验证过程中完全不依赖于子问题的相关信息（Answer_Samll）： +在评估子问题拆分时，必须严格遵循以下标准，且验证过程中完全不依赖于子问题的相关信息（Answer_Small）： 1. 合理性标准(必须全部满足)： - 完整性：每个不同的子问题必须完整覆盖原问题的所有关键要素（如时间、主体、动作、目标等），无遗漏。 - 最小化：每个不同的子问题数量应尽可能少，通常不超过原问题关键要素数量的2倍（建议2-4个），避免冗余和不必要拆分。 diff --git a/api/app/core/memory/agent/utils/type_classifier.py b/api/app/core/memory/agent/utils/type_classifier.py index 2f5e2501..d1b75d43 100644 --- a/api/app/core/memory/agent/utils/type_classifier.py +++ b/api/app/core/memory/agent/utils/type_classifier.py @@ -19,12 +19,14 @@ class DistinguishTypeResponse(BaseModel): type: str -async def status_typle(messages: str) -> dict: +async def status_typle(messages: str, llm_model_id: str) -> dict: """ Classify message type as read or write operation. + Updated to eliminate global variables in favor of explicit parameters. Args: messages: User message to classify + llm_model_id: LLM model ID to use (required, no longer from global variables) Returns: dict: Contains 'type' field with classification result @@ -42,8 +44,7 @@ async def status_typle(messages: str) -> dict: "message": f"Prompt rendering failed: {str(e)}" } - from app.core.memory.utils.config import definitions as config_defs - llm_client = get_llm_client(config_defs.SELECTED_LLM_ID) + llm_client = get_llm_client(llm_model_id) try: structured = await llm_client.response_structured( diff --git a/api/app/core/memory/agent/utils/verify_tool.py b/api/app/core/memory/agent/utils/verify_tool.py index a6cce18a..fe721770 100644 --- a/api/app/core/memory/agent/utils/verify_tool.py +++ b/api/app/core/memory/agent/utils/verify_tool.py @@ -11,7 +11,7 @@ from langchain_core.messages import HumanMessage from jinja2 import Environment, FileSystemLoader from app.core.memory.agent.utils.messages_tool import _to_openai_messages from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.core.memory.utils.config.definitions import SELECTED_LLM_ID +# Removed global variable imports - use dependency injection instead from app.core.logging_config import get_agent_logger load_dotenv(find_dotenv()) @@ -31,8 +31,17 @@ class State(TypedDict): class VerifyTool: - def __init__(self, system_prompt: str="", verify_data: Any=None): + def __init__(self, system_prompt: str="", verify_data: Any=None, llm_model_id: str=None): + """ + Updated to eliminate global variables in favor of explicit parameters. + + Args: + system_prompt: System prompt for verification + verify_data: Data to verify + llm_model_id: LLM model ID (required, no longer from global variables) + """ self.system_prompt = system_prompt + self.llm_model_id = llm_model_id if isinstance(verify_data, str): self.verify_data = verify_data else: @@ -42,7 +51,9 @@ class VerifyTool: self.verify_data = str(verify_data) async def model_1(self, state: State) -> State: - llm_client = get_llm_client(SELECTED_LLM_ID) + if not self.llm_model_id: + raise ValueError("llm_model_id is required but not provided") + llm_client = get_llm_client(self.llm_model_id) response_content = await llm_client.chat( messages=[{"role": "system", "content": self.system_prompt}, *_to_openai_messages(state["messages"])] ) diff --git a/api/app/core/memory/agent/utils/write_tools.py b/api/app/core/memory/agent/utils/write_tools.py index ebfbcc6c..60259873 100644 --- a/api/app/core/memory/agent/utils/write_tools.py +++ b/api/app/core/memory/agent/utils/write_tools.py @@ -1,80 +1,93 @@ -import asyncio -from dotenv import load_dotenv +""" +Write Tools for Memory Knowledge Extraction Pipeline + +This module provides the main write function for executing the knowledge extraction +pipeline. Only MemoryConfig is needed - clients are constructed internally. +""" import time from datetime import datetime -from app.repositories.neo4j.graph_saver import save_dialog_and_statements_to_neo4j - -from app.core.memory.agent.utils.get_dialogs import get_chunked_dialogs from app.core.logging_config import get_agent_logger - -logger = get_agent_logger(__name__) -# 使用新的模块化架构 -from app.core.memory.storage_services.extraction_engine.extraction_orchestrator import ExtractionOrchestrator -from app.core.memory.storage_services.extraction_engine.knowledge_extraction.embedding_generation import ( - embedding_generation_all, +from app.core.memory.agent.utils.get_dialogs import get_chunked_dialogs +from app.core.memory.storage_services.extraction_engine.extraction_orchestrator import ( + ExtractionOrchestrator, ) - -# 使用新的仓储层 -from app.repositories.neo4j.neo4j_connector import Neo4jConnector -# 导入配置模块（而不是直接导入变量） -from app.core.memory.utils.config import definitions as config_defs -from app.core.memory.utils.llm.llm_utils import get_llm_client +from app.core.memory.storage_services.extraction_engine.knowledge_extraction.memory_summary import ( + Memory_summary_generation, +) +from app.core.memory.utils.embedder.embedder_utils import ( + get_embedder_client_from_config, +) +from app.core.memory.utils.llm.llm_utils import get_llm_client_from_config from app.core.memory.utils.log.logging_utils import log_time -from app.core.memory.storage_services.extraction_engine.knowledge_extraction.memory_summary import Memory_summary_generation -from app.repositories.neo4j.add_nodes import add_memory_summary_nodes from app.repositories.neo4j.add_edges import add_memory_summary_statement_edges +from app.repositories.neo4j.add_nodes import add_memory_summary_nodes +from app.repositories.neo4j.graph_saver import save_dialog_and_statements_to_neo4j +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.schemas.memory_config_schema import MemoryConfig +from dotenv import load_dotenv + load_dotenv() +logger = get_agent_logger(__name__) -async def write(content: str, user_id: str, apply_id: str, group_id: str, ref_id: str = "wyl20251027", config_id: str = None) -> None: + +async def write( + content: str, + user_id: str, + apply_id: str, + group_id: str, + memory_config: MemoryConfig, + ref_id: str = "wyl20251027", +) -> None: """ - 执行完整的知识提取流水线（使用新的 ExtractionOrchestrator） + Execute the complete knowledge extraction pipeline. + + Only MemoryConfig is needed - LLM and embedding clients are constructed + internally from the config. Args: - content: 对话内容 - user_id: 用户ID - apply_id: 应用ID - group_id: 组ID - ref_id: 参考ID，默认为 "wyl20251027" - config_id: 配置ID，用于标记数据处理配置 + content: Dialogue content to process + user_id: User identifier + apply_id: Application identifier + group_id: Group identifier + memory_config: MemoryConfig object containing all configuration + ref_id: Reference ID, defaults to "wyl20251027" """ + # Extract config values + embedding_model_id = str(memory_config.embedding_model_id) + chunker_strategy = memory_config.chunker_strategy + config_id = str(memory_config.config_id) + logger.info("=== MemSci Knowledge Extraction Pipeline ===") - logger.info(f"Using model: {config_defs.SELECTED_LLM_NAME}") - logger.info(f"Using LLM ID: {config_defs.SELECTED_LLM_ID}") - logger.info(f"Using chunker strategy: {config_defs.SELECTED_CHUNKER_STRATEGY}") - logger.info(f"Using group ID: {config_defs.SELECTED_GROUP_ID}") - logger.info(f"Using embedding ID: {config_defs.SELECTED_EMBEDDING_ID}") - logger.info(f"Config ID: {config_id if config_id else 'None'}") - logger.info(f"LANGFUSE_ENABLED: {config_defs.LANGFUSE_ENABLED}") - logger.info(f"AGENTA_ENABLED: {config_defs.AGENTA_ENABLED}") + logger.info(f"Config: {memory_config.config_name} (ID: {config_id})") + logger.info(f"Workspace: {memory_config.workspace_name}") + logger.info(f"LLM model: {memory_config.llm_model_name}") + logger.info(f"Embedding model: {memory_config.embedding_model_name}") + logger.info(f"Chunker strategy: {chunker_strategy}") + logger.info(f"Group ID: {group_id}") + + # Construct clients from memory_config + llm_client = get_llm_client_from_config(memory_config) + embedder_client = get_embedder_client_from_config(memory_config) + logger.info("LLM and embedding clients constructed") # Initialize timing log log_file = "logs/time.log" timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") with open(log_file, "a", encoding="utf-8") as f: f.write(f"\n=== Pipeline Run Started: {timestamp} ===\n") + f.write(f"Config: {memory_config.config_name} (ID: {config_id})\n") pipeline_start = time.time() - # 初始化客户端 - llm_client = get_llm_client(config_defs.SELECTED_LLM_ID) - - # 获取 embedder 配置 - from app.core.models.base import RedBearModelConfig - from app.core.memory.utils.config.config_utils import get_embedder_config - from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient - - embedder_config_dict = get_embedder_config(config_defs.SELECTED_EMBEDDING_ID) - embedder_config = RedBearModelConfig(**embedder_config_dict) - embedder_client = OpenAIEmbedderClient(embedder_config) - + # Initialize Neo4j connector neo4j_connector = Neo4jConnector() - - # Step 1: 加载和分块数据 + + # Step 1: Load and chunk data step_start = time.time() chunked_dialogs = await get_chunked_dialogs( - chunker_strategy=config_defs.SELECTED_CHUNKER_STRATEGY, + chunker_strategy=chunker_strategy, group_id=group_id, user_id=user_id, apply_id=apply_id, @@ -83,21 +96,21 @@ async def write(content: str, user_id: str, apply_id: str, group_id: str, ref_id config_id=config_id, ) log_time("Data Loading & Chunking", time.time() - step_start, log_file) - - # Step 2: 初始化并运行 ExtractionOrchestrator + + # Step 2: Initialize and run ExtractionOrchestrator step_start = time.time() from app.core.memory.utils.config.config_utils import get_pipeline_config - config = get_pipeline_config() - + pipeline_config = get_pipeline_config() + orchestrator = ExtractionOrchestrator( llm_client=llm_client, embedder_client=embedder_client, connector=neo4j_connector, - config=config, + config=pipeline_config, + embedding_id=embedding_model_id, ) - - # 运行完整的提取流水线 - # orchestrator.run returns a flat tuple of 7 values after deduplication + + # Run the complete extraction pipeline ( all_dialogue_nodes, all_chunk_nodes, @@ -107,14 +120,12 @@ async def write(content: str, user_id: str, apply_id: str, group_id: str, ref_id all_statement_entity_edges, all_entity_entity_edges, all_dedup_details, - ) = await orchestrator.run(chunked_dialogs, is_pilot_run=False) - + log_time("Extraction Pipeline", time.time() - step_start, log_file) - # Step 8: Save all data to Neo4j database using graph models + # Step 3: Save all data to Neo4j database step_start = time.time() - # 运行索引创建 from app.repositories.neo4j.create_indexes import create_fulltext_indexes try: await create_fulltext_indexes() @@ -141,18 +152,16 @@ async def write(content: str, user_id: str, apply_id: str, group_id: str, ref_id log_time("Neo4j Database Save", time.time() - step_start, log_file) - # Step 9: Generate Memory summaries and save to local vector DB and Neo4j + # Step 4: Generate Memory summaries and save to Neo4j step_start = time.time() try: summaries = await Memory_summary_generation( - chunked_dialogs, llm_client=llm_client, embedding_id=config_defs.SELECTED_EMBEDDING_ID + chunked_dialogs, llm_client=llm_client, embedding_id=embedding_model_id ) - # Save memory summaries to Neo4j as nodes try: ms_connector = Neo4jConnector() await add_memory_summary_nodes(summaries, ms_connector) - # Link summaries to statements via chunks for summary→entity queries await add_memory_summary_statement_edges(summaries, ms_connector) finally: try: @@ -162,24 +171,15 @@ async def write(content: str, user_id: str, apply_id: str, group_id: str, ref_id except Exception as e: logger.error(f"Memory summary step failed: {e}", exc_info=True) finally: - log_time("Memory Summary (Local Vector DB & Neo4j)", time.time() - step_start, log_file) - - + log_time("Memory Summary (Neo4j)", time.time() - step_start, log_file) # Log total pipeline time total_time = time.time() - pipeline_start log_time("TOTAL PIPELINE TIME", total_time, log_file) - # Add completion marker to log timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") with open(log_file, "a", encoding="utf-8") as f: f.write(f"=== Pipeline Run Completed: {timestamp} ===\n\n") logger.info("=== Pipeline Complete ===") logger.info(f"Total execution time: {total_time:.2f} seconds") - logger.info(f"Timing details saved to: {log_file}") - - -if __name__ == "__main__": - content = "你好，我是张三，是张曼婷的新朋友。请问张曼婷喜欢什么？" - asyncio.run(write(content, ref_id="wyl20251027")) diff --git a/api/app/core/memory/analytics/hot_memory_tags.py b/api/app/core/memory/analytics/hot_memory_tags.py index 0c68a567..cfcff994 100644 --- a/api/app/core/memory/analytics/hot_memory_tags.py +++ b/api/app/core/memory/analytics/hot_memory_tags.py @@ -1,8 +1,9 @@ -import sys -import os import asyncio -from neo4j import GraphDatabase +import os +import sys from typing import List, Tuple + +from neo4j import GraphDatabase from pydantic import BaseModel, Field # ------------------- 自包含路径解析 ------------------- @@ -31,11 +32,16 @@ except NameError: # --------------------------------------------------------------------- # 现在路径已经配置好，我们可以使用绝对导入 -from app.core.config import settings -from app.core.memory.utils.config.definitions import SELECTED_GROUP_ID, SELECTED_LLM_ID -from app.core.memory.utils.llm.llm_utils import get_llm_client import json +from app.core.config import settings +from app.core.memory.utils.llm.llm_utils import get_llm_client + +#TODO: Fix this +# Default values (previously from definitions.py) +DEFAULT_LLM_ID = os.getenv("SELECTED_LLM_ID", "openai/qwen-plus") +DEFAULT_GROUP_ID = os.getenv("SELECTED_GROUP_ID", "group_123") + # 定义用于LLM结构化输出的Pydantic模型 class FilteredTags(BaseModel): """用于接收LLM筛选后的核心标签列表的模型。""" @@ -140,8 +146,8 @@ async def get_hot_memory_tags(group_id: str | None = None, limit: int = 40, by_u limit: 返回的标签数量限制 by_user: 是否按user_id查询（默认False，按group_id查询） """ - # 默认从 runtime.json selections.group_id 读取 - group_id = group_id or SELECTED_GROUP_ID + # 默认从环境变量读取 + group_id = group_id or DEFAULT_GROUP_ID # 1. 从数据库获取原始排名靠前的标签 raw_tags_with_freq = get_raw_tags_from_db(group_id, limit, by_user=by_user) if not raw_tags_with_freq: @@ -150,8 +156,7 @@ async def get_hot_memory_tags(group_id: str | None = None, limit: int = 40, by_u raw_tag_names = [tag for tag, freq in raw_tags_with_freq] # 2. 初始化LLM客户端并使用LLM筛选出有意义的标签 - from app.core.memory.utils.config import definitions as config_defs - llm_client = get_llm_client(config_defs.SELECTED_LLM_ID) + llm_client = get_llm_client(DEFAULT_LLM_ID) meaningful_tag_names = await filter_tags_with_llm(raw_tag_names, llm_client) # 3. 根据LLM的筛选结果，构建最终的标签列表（保留原始频率和顺序） @@ -165,8 +170,8 @@ async def get_hot_memory_tags(group_id: str | None = None, limit: int = 40, by_u if __name__ == "__main__": print("开始获取热门记忆标签...") try: - # 直接使用 runtime.json 中的 group_id - group_id_to_query = SELECTED_GROUP_ID + # 直接使用环境变量中的 group_id + group_id_to_query = DEFAULT_GROUP_ID # 使用 asyncio.run 来执行异步主函数 top_tags = asyncio.run(get_hot_memory_tags(group_id=group_id_to_query)) diff --git a/api/app/core/memory/analytics/memory_insight.py b/api/app/core/memory/analytics/memory_insight.py index 26147a47..35ed466f 100644 --- a/api/app/core/memory/analytics/memory_insight.py +++ b/api/app/core/memory/analytics/memory_insight.py @@ -5,9 +5,9 @@ This script can be executed directly to generate a memory insight report for a t """ import asyncio +import json import os import sys -import json from collections import Counter from datetime import datetime @@ -17,12 +17,16 @@ src_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) if src_path not in sys.path: sys.path.insert(0, src_path) +from app.core.memory.analytics.hot_memory_tags import get_hot_memory_tags +from app.core.memory.utils.llm.llm_utils import get_llm_client +from app.repositories.neo4j.neo4j_connector import Neo4jConnector from pydantic import BaseModel, Field -from app.repositories.neo4j.neo4j_connector import Neo4jConnector -from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.core.memory.analytics.hot_memory_tags import get_hot_memory_tags -from app.core.memory.utils.config.definitions import SELECTED_GROUP_ID, SELECTED_LLM_ID +#TODO: Fix this + +# Default values (previously from definitions.py) +DEFAULT_LLM_ID = os.getenv("SELECTED_LLM_ID", "openai/qwen-plus") +DEFAULT_GROUP_ID = os.getenv("SELECTED_GROUP_ID", "group_123") # 定义用于LLM结构化输出的Pydantic模型 class TagClassification(BaseModel): @@ -55,8 +59,7 @@ class MemoryInsight: def __init__(self, user_id: str): self.user_id = user_id self.neo4j_connector = Neo4jConnector() - from app.core.memory.utils.config import definitions as config_defs - self.llm_client = get_llm_client(config_defs.SELECTED_LLM_ID) + self.llm_client = get_llm_client(DEFAULT_LLM_ID) async def close(self): """关闭数据库连接。""" @@ -294,8 +297,8 @@ async def main(): """ Initializes and runs the memory insight analysis for a test user. """ - # 默认从 runtime.json selections.group_id 读取 - test_user_id = SELECTED_GROUP_ID + # 默认从环境变量读取 + test_user_id = DEFAULT_GROUP_ID print(f"正在为用户 {test_user_id} 生成记忆洞察报告...\n") insight = None diff --git a/api/app/core/memory/analytics/user_summary.py b/api/app/core/memory/analytics/user_summary.py index f4f4d642..eb6bc83a 100644 --- a/api/app/core/memory/analytics/user_summary.py +++ b/api/app/core/memory/analytics/user_summary.py @@ -6,10 +6,10 @@ Usage: python -m analytics.user_summary --user_id """ -import os -import sys import asyncio import json +import os +import sys from dataclasses import dataclass from typing import List, Tuple @@ -24,10 +24,15 @@ try: except Exception: pass -from app.repositories.neo4j.neo4j_connector import Neo4jConnector from app.core.memory.analytics.hot_memory_tags import get_hot_memory_tags -from app.core.memory.utils.config.definitions import SELECTED_GROUP_ID, SELECTED_LLM_ID from app.core.memory.utils.llm.llm_utils import get_llm_client +from app.repositories.neo4j.neo4j_connector import Neo4jConnector + +#TODO: Fix this + +# Default values (previously from definitions.py) +DEFAULT_LLM_ID = os.getenv("SELECTED_LLM_ID", "openai/qwen-plus") +DEFAULT_GROUP_ID = os.getenv("SELECTED_GROUP_ID", "group_123") @dataclass @@ -42,8 +47,7 @@ class UserSummary: def __init__(self, user_id: str): self.user_id = user_id self.connector = Neo4jConnector() - from app.core.memory.utils.config import definitions as config_defs - self.llm = get_llm_client(config_defs.SELECTED_LLM_ID) + self.llm = get_llm_client(DEFAULT_LLM_ID) async def close(self): await self.connector.close() @@ -107,8 +111,8 @@ class UserSummary: async def generate_user_summary(user_id: str | None = None) -> str: - # 默认从 runtime.json selections.group_id 读取 - effective_group_id = user_id or SELECTED_GROUP_ID + # 默认从环境变量读取 + effective_group_id = user_id or DEFAULT_GROUP_ID svc = UserSummary(effective_group_id) try: return await svc.generate() @@ -139,7 +143,7 @@ if __name__ == "__main__": with open(dashboard_path, "r", encoding="utf-8") as rf: existing = json.load(rf) existing["user_summary"] = { - "group_id": SELECTED_GROUP_ID, + "group_id": DEFAULT_GROUP_ID, "summary": summary } with open(dashboard_path, "w", encoding="utf-8") as wf: diff --git a/api/app/core/memory/config.json b/api/app/core/memory/config.json deleted file mode 100644 index a0349558..00000000 --- a/api/app/core/memory/config.json +++ /dev/null @@ -1,132 +0,0 @@ -{ - "llm_list": [ - { - "llm_name": "qwen2.5-14b-instruct-awq", - "api_base": "http://175.27.131.196:9090/v1", - "api_key": "OPENAI_API_KEY" - }, - { - "llm_name": "openai/qwen2.5-14b-instruct-awq", - "api_base": "http://175.27.131.196:9090/v1", - "api_key": "OPENAI_API_AGENT_KEY" - }, - { - "llm_name": "openai/qwen2.5-14b", - "api_base": "http://43.137.4.24:9090/v1", - "api_key": "OPENAI_API_KEY" - }, - { - "llm_name": "openai/qwen2.5-14b-instruct-awq", - "api_base": "http://175.27.131.196:9090/v1", - "api_key": "OPENAI_API_KEY" - }, - { - "llm_name": "openai/qwen3-14b", - "api_base": "http://43.137.4.24:9090/v1", - "api_key": "OPENAI_API_KEY" - }, - { - "llm_name": "openai/deepseek-r1-0528-qwen3-8b", - "api_base": "http://43.137.4.24:9090/v1", - "api_key": "OPENAI_API_KEY" - }, - { - "llm_name": "openai/qwen3-235b-a22b-instruct-2507", - "api_base": "https://dashscope.aliyuncs.com/compatible-mode/v1", - "api_key": "DASHSCOPE_API_KEY" - } - , - { - "llm_name": "openai/qwen-plus", - "api_base": "https://dashscope.aliyuncs.com/compatible-mode/v1", - "api_key": "DASHSCOPE_API_KEY" - }, - { - "llm_name": "bedrock/global.anthropic.claude-sonnet-4-5-20250929-v1:0" - }, - { - "llm_name": "bedrock/global.anthropic.claude-sonnet-4-20250514-v1:0" - } - ], - "embedding_list": [ - { - "embedding_name": "openai/nomic-embed-text:v1.5", - "api_base": "http://119.45.239.97:11434/v1", - "dimension": 768 - }, - { - "embedding_name": "openai/bge-m3", - "api_base": "http://43.137.4.24:9090/v1", - "dimension": 1024 - } - ], - "neo4j": { - "uri": "bolt://1.94.111.67:7687", - "username": "neo4j" - }, - "chunker_list": [ - { - "chunker_strategy": "TokenChunker", - "embedding_model": "BAAI/bge-m3", - "chunk_size": 512, - "chunk_overlap": 56, - "tokenizer_or_token_counter": "character" - }, - { - "chunker_strategy": "RecursiveChunker", - "embedding_model": "BAAI/bge-m3", - "chunk_size": 512, - "min_characters_per_chunk": 50 - }, - { - "chunker_strategy": "SemanticChunker", - "embedding_model": "BAAI/bge-m3", - "chunk_size": 1024, - "threshold": 0.8, - "min_sentences": 2, - "skip_window": 1, - "min_characters_per_chunk": 100 - }, - { - "chunker_strategy": "LateChunker", - "embedding_model": "all-MiniLM-L6-v2", - "chunk_size": 2048, - "min_characters_per_chunk": 24 - }, - { - "chunker_strategy": "NeuralChunker", - "embedding_model": "mirth/chonky_modernbert_base_1", - "min_characters_per_chunk": 24 - }, - { - "chunker_strategy": "LLMChunker", - "embedding_model": "BAAI/bge-m3", - "chunk_size": 1000, - "min_characters_per_chunk": 100 - }, - { - "chunker_strategy": "HybridChunker", - "embedding_model": "BAAI/bge-m3", - "chunk_size": 512, - "threshold": 0.8, - "min_characters_per_chunk": 100 - }, - { - "chunker_strategy": "SentenceChunker", - "embedding_model": "BAAI/bge-m3", - "chunk_size": 2048, - "chunk_overlap": 128, - "min_sentences_per_chunk": 1, - "min_characters_per_sentence": 12, - "delim": [".", "!", "?", "\n"], - "include_delim": "prev", - "tokenizer_or_token_counter": "character" - } - ], - "langfuse": { - "enabled": true - }, - "agenta": { - "enabled": false - } -} \ No newline at end of file diff --git a/api/app/core/memory/data/testdata.json b/api/app/core/memory/data/testdata.json deleted file mode 100644 index 2f893e64..00000000 --- a/api/app/core/memory/data/testdata.json +++ /dev/null @@ -1,12 +0,0 @@ -{"context": "以下是 2025 年 10 月 14 日的对话记录:\n“用户:我计划周末去龙泉山拍日出。\nAI:建议提前查天气和能见度,凌晨三点出发更稳。\n用户:嗯嗯。\n用户:嘿嘿。\n用户:三脚架。\n用户:备用电池。\nAI:把这些写进出门清单就不容易忘。\n用户:唉——真是纠结。\n用户:我新买了 35mm 定焦镜头。\nAI:35mm 的视角自然,适合街拍。\n用户:咔咔咔……乱糟糟的。\n\n请记住以上全部对话记录,回答问题。\n问题:这次对话中,用户周末计划去哪里拍摄？他新买的镜头是什么规格？清单型名词句子中出现了哪些器材？","answer": "用户计划周末去龙泉山拍日出；新买的镜头是 35mm 定焦；清单型名词句子中出现的器材包括三脚架、备用电池。","id": "45e2f1b3-78b9-4b15-9aa4-1234567890ab","chatglm_length": 890,"qwen_length": 900,"internlm2_length": 895} -{"context": "以下是 2025 年 10 月 16 日的对话记录:\n“用户:我叫林晓,平时喜欢拍自然风光,最近拍红叶总觉得画面层次感不够,明明现场看颜色很鲜艳,拍出来却有点平淡,调滤镜也找不到刚好贴合的效果。\nAI:很高兴认识你！红叶拍摄确实需要注意光线和色彩平衡,你最近有没有想好去哪里拍红叶呀？\n 用户:想好了,我计划周末去近郊红叶谷拍红叶,还想顺便拍谷里的山间溪流,听说清晨的雾气没散时,溪流和红叶搭配起来特别出片,而且我新买了 16-35mm 广角镜头,想试试拍大场景。\nAI:广角镜头拍红叶谷的大场景很合适！建议你带个中灰渐变镜,清晨拍溪流时,能平衡天空和地面的曝光差,避免天空过曝或溪流欠曝,另外提前查下红叶谷的最佳拍摄时段,听说早上 7 点到 9 点的光线最柔和。\n 用户:太需要了！上次去拍山间的银杏,就是因为没带渐变镜,天空白成一片,后期想救都救不回来,而且当时没控制好快门速度,溪流的水流拍得太硬,没有那种丝滑的感觉,只能删掉大半照片。\nAI:拍溪流要丝滑效果,快门速度可以调到 1/10 秒左右,再搭配三脚架固定相机,避免画面模糊,另外可以找块石头当前景,让溪流从石头旁流过,能增加画面的纵深感。\n 用户:记下来了！我还想拍几张红叶的特写,比如叶脉上挂着露珠的样子,但之前拍特写时总对焦在叶子边缘,没把露珠的细节拍清楚,不知道是不是对焦区域选小了。\nAI:拍露珠特写建议用 “单点对焦”,把对焦框精准对准露珠,再把光圈开到 f/4 左右,既能保证露珠清晰,又能让背景的红叶稍微虚化,突出主体,另外可以喷点清水在叶子上,让露珠更明显。\n 用户:这个方法好！我准备列个拍摄清单,除了中灰渐变镜,还要带三脚架、清水喷雾,上次拍银杏时忘带三脚架,蹲在地上拍得腿都麻了,而且手持拍的照片糊了不少,后期只能勉强修几张。\nAI:清单里可以加 “备用电池”,红叶谷早上气温低,电池耗电快,另外注意事项里可以写 “穿防滑鞋”,谷里有溪流的地方路面可能湿滑,不小心容易摔跤影响拍摄。\n 用户:太贴心了！晚上我要把相机的参数预设好,比如拍大场景时光圈设为 f/8、ISO100,拍溪流时快门速度设为 1/10 秒,省得早上到了现场手忙脚乱调参数,耽误最佳拍摄时间。\nAI:预设参数很明智！拍的时候可以多换几个角度,比如低角度拍溪流倒映红叶,或者站在高处拍红叶谷的全景,不同角度能拍出不一样的氛围,另外拍完记得及时导照片到硬盘,避免内存卡满了漏拍。\n 用户:好的！等拍完我想把红叶和溪流的照片拼成长图,配段文字发朋友圈,还想把喜欢的照片洗出来贴在相册里,每次翻相册都能想起拍摄时的场景,这种感觉特别好。\nAI:这个想法很棒！洗照片时可以选哑光纸,红叶的颜色会更有质感,不像光面纸那么刺眼,祝你周末拍摄顺利,期待看你的红叶谷作品！\n”\n\n 请记住以上全部对话记录,回答问题。\nquery:10 月 16 日的对话中,林晓计划周末去哪里拍摄什么题材?她新买的镜头是什么规格?AI 建议林晓用什么快门速度拍溪流的丝滑效果？", "answer": "林晓计划周末去近郊红叶谷拍摄红叶和山间溪流题材；她新买的镜头是 16-35mm 广角镜头;AI 建议林晓用 1/10 秒左右的快门速度拍溪流的丝滑效果。", "id": "89a4b3c2-d1e5-f6g7-h8i9-abcdef123456", "chatglm_length": 1908, "qwen_length": 1921, "internlm2_length": 1902} -{"context": "以下是关于用户经历与交流的记录:\n“用户:上周三,我和同事李华一起参加了苹果公司的技术发布会。在发布会上,我们看到了蒂姆・库克亲自演示新款 iPhone 的功能,这让我印象深刻。李华对新产品很感兴趣,当场就决定要购买一部。\n 用户:这让我想起了去年和李华一起去北京出差的经历。当时我们在中关村参观了苹果公司的中国总部,还偶遇了从美国来的技术专家约翰・史密斯。约翰向我们介绍了苹果公司在人工智能领域的最新研究进展。\n 用户:说到苹果,我其实是个果粉,从 iPhone 4 开始就一直使用苹果的产品。不过有趣的是,我虽然喜欢苹果的产品,却对苹果这种水果过敏,每次吃苹果都会不舒服。\n 用户:昨天李华给我发消息,说他已经收到了新买的 iPhone,使用体验非常好。他还提到周末想约我一起去星巴克喝咖啡,讨论一下我们正在做的项目。这个项目涉及机器学习和自然语言处理,正好可以用到苹果最新发布的一些开发工具。\n 用户:我记得上次和李华在星巴克讨论项目时,我们遇到了大学同学张明。张明现在在谷歌工作,他给我们分享了一些关于深度学习框架 TensorFlow 的使用经验。这些经验对我们当前的项目很有帮助。\n 用户:对了,说到大学,我和李华、张明都是清华大学计算机系的校友。我们在校期间就经常一起做项目,没想到毕业后还能在同一个领域继续合作,这真是难得的缘分。”\n\n 请记住以上全部记录内容,回答问题。\n 问题:记录中用户上周三与谁一起参加了什么活动？用户对苹果相关的哪类事物过敏？李华昨天发消息提到了哪两件事？用户、李华与张明的共同身份是什么？","answer": "用户上周三与同事李华一起参加了苹果公司的技术发布会；用户对苹果这种水果过敏；李华昨天发消息提到两件事,一是已收到新买的 iPhone 且使用体验好,二是想约用户周末去星巴克喝咖啡讨论项目；用户、李华与张明的共同身份是清华大学计算机系的校友。","id": "68c3d2a4-91c7-5d32-bb56-0987654321ba","chatglm_length": 1280,"qwen_length": 1295,"internlm2_length": 1285} -{"context": "以下是 2025 年 10 月 27 日的对话记录:\n“用户:周末去郊区玩太值了！提前准备的东西都没白费～\nAI: 哇！你之前说带了菊花和阿司匹林,这次都用上了吗？\n 用户:都用啦！爬山累了喝杯菊花超解腻,后来朋友头疼,阿司匹林刚好能缓解。\n 用户:对了,还带了那个凤凰牌的东西,装东西特别方便。\nAI: 是凤凰的背包吧？之前听你说过这个牌子挺耐用的～那天在郊区,有没有看到特别的动植物呀？\n 用户:有！山坡上开了好多杜鹃,早上还听到杜鹃叫,声音特别清脆。\n 用户:就是中午太阳有点晒,还好带了长城的遮阳伞,不然得晒黑了。\nAI: 长城牌遮阳伞确实靠谱！那你们去了之前说的茅台镇附近吗？那边环境怎么样？\n 用户:去了！茅台镇旁边的小路特别安静,就是风有点大,不过吹着很舒服。\n 用户:我们还在当地买了龙井,店家说这是今年的新茶,泡着特别香。\nAI: 听起来好惬意！那有没有遇到什么让人觉得温暖的事呀？比如当地人帮忙之类的？\n 用户:有！我们找不到路的时候,一位大爷主动带路,还给我们指了好看的风景,特别有雷锋精神。\n 用户:对了,带的那个杜鹃图案的手帕也用上了,擦汗特别方便。\nAI: 这些小准备和温暖的经历也太加分了～这次游玩,有没有拍些好看的照片呀？\n 用户:拍了！对着满山的杜鹃和茅台镇的远景拍了好多,回头整理好给你看。”\n\n 请记住以上全部对话记录,回答问题。\n 问题:用户此次郊区游玩,哪些准备的物品派上了用场？用户提到游玩中遇到了什么环境情况,又有哪些温暖的经历或满意的发现？","answer": "派上用场的物品包括菊花（菊花茶）、阿司匹林（药品）、凤凰牌背包、长城牌遮阳伞、龙井（新茶）、杜鹃图案手帕；游玩中遇到的环境情况是中午太阳晒、郊区风大但舒适、茅台镇附近小路安静、山坡上有大量杜鹃（花）；温暖的经历或满意的发现是听到杜鹃（鸟）鸣叫、迷路时被当地大爷主动带路并指引风景（感受到雷锋精神）、买到今年的新龙井茶、拍摄了满山杜鹃（花）和茅台镇远景的照片。","id": "78d3c2a4-91c7-5d28-bb36-0987654321bc","chatglm_length": 1420,"qwen_length": 1435,"internlm2_length": 1428} -{"context": "以下是2023年04月27日的对话记录：\n“用户: 嗨，我是王峰，一个热爱体育和军事历史的家伙。今天跑了个半马，感觉挺不错的，你呢？\nAI: 嗨，王峰！听起来你今天的状态很棒啊。我是AI伴侣，虽然不能像你一样体验长跑的快感，但我很乐意听你分享。跑步过程中有没有什么有趣的事情发生？\n\n用户: 哈哈，确实有。我在跑步的时候，竟然追上了一个穿着古装的团队，他们在拍摄什么历史剧吧。\nAI: 那真是个有趣的巧合！你对历史剧感兴趣吗？毕竟你对军事历史很有研究。\n\n用户: 对啊，我对这些特别感兴趣。看到他们的服装，我就联想到了那个时代的战争场景。不过，我更喜欢实际的历史研究，比如阅读史料和参观历史遗迹。\nAI: 那你最近有读什么吸引人的军事史著作吗？或者有没有计划去参观某个特别的历史遗址？\n\n用户: 对，我最近在读《二战全史》，非常全面而且深入。至于历史遗迹，我打算下个月去访问一下抗战纪念馆，沉浸在那段历史里。\nAI: 《二战全史》确实是本好书，涵盖了很多重要的战役和决策。抗战纪念馆的参观听起来很有意义，那里肯定有很多珍贵的历史资料和展品。\n\n用户: 没错，我期待能从中学到更多。对了，除了阅读和参观，我还喜欢打篮球来放松身心。今晚我和朋友们约了一场篮球赛。\nAI: 篮球赛听起来很刺激！你打的是哪个位置？有没有什么特别的技巧或者战术喜欢用？\n\n用户: 我通常打小前锋，喜欢快攻和远投。今晚我打算试试新的突破技巧，希望能给对手来点意外。\nAI: 小前锋啊，那你一定很擅长快速移动和精准投篮。新技巧的尝试总是让比赛更加充满乐趣。希望你能在今晚的比赛中大放异彩！\n\n用户: 哈，我会的！我总是全力以赴。不过，说起来容易做起来难，篮球场上的对抗可不是闹着玩的。\nAI: 确实，篮球场上的竞争非常激烈。不过，以你的性格和能力，我相信你能够应对挑战，展现你的实力。\n\n用户: 说到挑战，攀岩才是真正的挑战。上次攀岩时，我差点就要攻克那个难度系数很高的路线了。\nAI: 攀岩确实是对体力和意志的极大考验。下次挑战那个路线时，你有什么新的策略吗？\n\n用户: 我打算增加核心力量训练，提高耐力。攀岩时，核心力量至关重要，它能帮助我更好地控制身体。\nAI: 你的计划听起来很专业。核心力量对于攀岩确实非常关键。你通常是怎么训练的？\n\n用户: 我会做一些特定的体能训练，比如仰卧起坐、俯卧撑和翻墙。这些都能帮助我在攀岩时保持稳定和灵活。\nAI: 那些训练听起来很全面，能够有效提升你的整体体能。你对攀岩有多热情啊！\n\n用户: 是的，我对此非常热情。攀岩不仅是一项运动，更是一种精神的磨砺。好了，我得准备今晚的篮球赛了，我们明天聊。\nAI: 当然，祝你今晚篮球赛取得好成绩！明天我们再继续聊，王峰，加油！\n”\n以下是2023年04月28日的对话记录：\n“用户: 昨晚的篮球赛太刺激了！我们赢了，我那个新突破技巧效果杠杠的！\nAI: 朋友，听你这么一说，我都能感受到你赛场上的激情了！你的得分怎么样？\n\n用户: 得了20分，还有几次助攻和抢断。不过，最重要的是团队合作，我们的配合越来越默契了。\nAI: 20分加上助攻和抢断，这成绩相当不错！团队默契是取胜的关键，你们肯定训练得很勤。\n\n用户: 是的，我们每周都会固定练习两次。说起来，我今天计划去徒步，想去探索一条新路线。\nAI: 徒步探索新路线听起来很有趣。这条路线有什么特别之处吗？\n\n用户: 这条路线据说风景特别好，沿途有几个小瀑布，还有个可以俯瞰整个山谷的观景台。\nAI: 哇，那真是个不错的选择。你徒步时通常会准备些什么装备？\n\n用户: 我会带上足够的水和食物，还有急救包、多功能刀具，当然还有我的相机，不能错过沿途的美景。\nAI: 你准备得很充分。拍照是个好主意，可以记录下旅途中的美好瞬间。你徒步时喜欢独自前行，还是和朋友一起？\n\n用户: 两种方式我都喜欢。独自徒步可以让我更好地放松思绪，和朋友一起则更加热闹。\nAI: 确实，不同的方式有不同的乐趣。你在徒步时有没有遇到过什么难忘的经历？\n\n用户: 有一次徒步的时候，我遇到了一只迷路的小狗，最后我帮它找到了回家的路，那次经历让我很感动。\nAI: 那真是个温馨的故事。你的善举一定让小狗的主人非常感激。说起来，你对军事史有这么深的兴趣，有没有考虑过写点什么，比如博客或者书籍？\n\n用户: 哈，我确实有这个打算。我想写一本关于战术演变的书，但是一直没能找到合适的切入点。\nAI: 战术演变是个很宽广的话题，你可以从一个特定的战役或者时期开始着手，逐渐扩展开来。\n\n用户: 这个建议不错，我会考虑的。对了，我最近在研究越战期间的丛林战术，那个时期的战术变化很有意思。\nAI: 越战的丛林战术确实非常独特，那个时期的战争环境和战术运用对现代战争学有很多启示。\n\n用户: 没错，我觉得从这个角度切入，应该能写出不错的内容。说起来，我还得准备一下今天的徒步，稍后再聊。\nAI: 当然，徒步前的准备很重要。王峰，祝你今天的徒步旅行顺利，期待你分享新路线的故事。小心安全！\n\n用户: 谢了，我会的。对了，我还在想，攀岩时的安全措施你有了解吗？我想听听你的看法。\nAI: 攀岩安全非常重要。基本的安全措施包括检查所有装备，比如绳索、安全带和头盔，确保它们都在良好状态。还有就是和你的攀岩伙伴之间要有良好的沟通。\n\n用户: 说得对，安全总是第一位的。我总是会双倍确认我的装备和团队的沟通无误。\nAI: 这样的习惯很好，可以有效避免许多意外。你有没有遇到过攀岩时的紧急情况？\n\n用户: 有一次，我的攀岩伙伴不小心滑了一下，幸好安全绳起了作用，没有造成伤害。\nAI: 那一定是个惊险的时刻。幸好你们准备充分，避免了事故。安全绳的重要性不言而喻。\n\n用户: 是的，每次想起都觉得后怕。不过，这也是一次宝贵的经验。好了，我得出发了，明天聊。\nAI: 好的，王峰，祝你今天的徒步愉快。明天我们再继续我们的话题，保重！\n”\n\n请记住以上全部对话记录，回答问题。\n问题：", "query": "我曾经在4月27号这天，和你聊了打篮球的事情，我通常打哪个位置？", "answer": "前锋", "id": "7e11ba68-304a-4a91-98b5-7387cf0e3c43", "chatglm_length": 1540, "qwen_length": 1563, "internlm2_length": 1499} -{"context": "以下是2023年04月27日的对话记录：\n“用户: 嗨，我是李雪，一个爱笑的旅行家，电影迷，音乐会的常客，还有就是书和笔是我最好的朋友。今天心情不错，想和你聊聊天，你准备好了吗？\nAI: 嗨，李雪！我已经迫不及待想听你分享你的故事了。你今天有什么新鲜事想要告诉我吗？\n\n用户: 对了，我最近看了一部电影《时间旅行者的妻子》，简直太感人了，你知道吗？\nAI: 听起来很有意思！《时间旅行者的妻子》是关于爱情和时间错位的故事对吧？你最喜欢这部电影的哪个部分？\n\n用户: 没错，就是那种跨越时间的爱情让我深受触动。我特别喜欢他们之间的那种坚定和执着，尤其是结尾的那一幕，简直让人泪目。\nAI: 确实，那种不被时间所限制的爱情是很多人梦寐以求的。除了看电影，你最近还有没有去哪里旅游，发现了什么新奇的地方？\n\n用户: 说到旅游，我上周末去了一个小众的海滩，那里的沙子是粉红色的，超级漂亮！我还捡了些贝壳，打算做成手链。\nAI: 粉红色的沙滩听起来就像是童话世界里的场景，真是太美了！你真是个有创意的人，做手链的想法很棒。你打算怎么设计手链呢？\n\n用户: 我想把它们串在一起，中间加上一些小金属饰品，应该会很有夏天的感觉吧。对了，最近我还在写一篇关于旅行的文章，但有点卡壳了。\nAI: 听起来那手链会很迷人呢！关于你的文章，卡壳了是因为缺乏灵感，还是有其他的原因？\n\n用户: 主要是写到一半，突然不知道怎么把我的感受转化成文字了。我想表达的是旅行中那种自由自在的感觉，但是又不想太俗套。\nAI: 旅行中的自由自在确实是一种很难用言语表达的感觉。你可以尝试用一些具体的场景或者经历来引入，让读者通过你的眼睛去感受那份自由。\n\n用户: 这个主意不错！我可以从那次独自一人走在海边的经历写起，那种只有海浪声陪伴的宁静，可能会传达出我想表达的感觉。\nAI: 那真是个很好的开头。海浪的声音，沙滩上的脚印，还有远处的落日，这些细节都能够帮助读者更好地沉浸在你的故事中。\n\n用户: 对，我想我知道怎么继续写下去了。谢谢你！对了，我还想去看一场演唱会，但最近好像没有我喜欢的歌手来开演唱会。\nAI: 不客气，很高兴能帮到你！关于演唱会，虽然现在没有你喜欢的歌手来，但你可以尝试听听其他歌手的音乐，说不定会有新的发现呢。\n\n用户: 嗯，这倒是个好主意。我会去翻翻看有没有其他有趣的演出。对了，你有没有什么推荐的歌手或乐队？\nAI: 你可以试试听听The Lumineers，他们的音乐很有感染力，旋律优美，非常适合旅行时候听。或者如果你喜欢一些更有活力的音乐，Imagine Dragons也是个不错的选择。\n\n用户: The Lumineers我听过几首，确实不错。Imagine Dragons的《Believer》我也超爱！看来你的音乐品味也挺不错的嘛。\nAI: 哈哈，我会尽量跟上你的步伐的。说起来，你最喜欢的一次演唱会是哪一次？有没有什么难忘的经历？\n\n用户: 那当然是Coldplay的演唱会了，整个场馆的氛围、灯光还有音乐，完全就是另一个世界。我记得当时我闭上眼睛，感觉自己完全沉浸在音乐里。\nAI: Coldplay的演唱会确实是出了名的震撼人心。音乐有时候就是这样，能够带我们进入一个全新的世界。你提到闭上眼睛沉浸在音乐里，那感觉一定很神奇。\n\n用户: 是的，那种感觉无法用言语表达，只能自己体验。哎，说到这些，我又想去旅行了，想去一个没有去过的地方探险。\nAI: 旅行总是能给人带来新的体验和感受。如果你可以去任何一个地方，你最想去哪里？\n\n用户: 我想去冰岛看极光，那是我长久以来的梦想。想象一下，在寒冷的夜晚，天空中突然出现五彩斑斓的光，那一定很神奇。\nAI: 冰岛的极光听起来是个绝佳的选择，希望你的梦想能够很快实现。虽然我不能亲自陪你去，但我会在这里等着听你分享那神奇的经历。\n\n用户: 谢谢你的鼓励！我一定会实现这个梦想的。好了，今天聊得很开心，但我得去继续写我的文章了。我们明天再聊吧！\nAI: 当然，祝你写作顺利！明天我会在这里等你，期待听到更多你的故事。晚安，李雪！\n”\n以下是2023年04月28日的对话记录：\n“用户: 嘿，我又来啦！今天我在写作上取得了一点小进展，感觉棒极了！你今天过得怎么样？\nAI: 嗨，李雪！听你这么说真是太好了，我今天也很期待和你聊天呢。有什么新的进展可以分享一下吗？\n\n用户: 当然！我按照昨天的想法，写了一段关于海边散步的描写，感觉文字活了起来。这种感觉就像找到了一把打开心灵的钥匙一样。\nAI: 这样的比喻真是形象，你的文字一定很有力量。有没有什么段落是你特别满意的，愿意分享给我听听？\n\n用户: 有一段是这样描述的：“海风轻拂过脸庞，带着咸咸的海水味和远处渔火的微光，我仿佛能听到星星在耳边低语。”怎么样？\nAI: 这段描述真是美妙，我仿佛能感受到那个场景的宁静和美丽。你的文字有种魔力，能把读者带到那个海边。\n\n用户: 哈哈，你这话说得我都有点飘了。说起来，我最近还挺想去看看新上映的那部科幻片《星际穿越者》，听说特效做得很棒。\nAI: 《星际穿越者》啊，那部电影确实是近期的热门话题。科幻片中的特效往往能带给我们视觉上的震撼，你对科幻电影有什么特别的看法吗？\n\n用户: 我觉得科幻电影最吸引我的就是那种无限的想象力和对未来的探索。每次看完都能激发我好几天的创作灵感。\nAI: 确实，科幻电影就像是打开了一个通往未来世界的大门。你有没有哪些科幻电影是你的最爱，或者对你影响很大的？\n\n用户: 《银翼杀手》和《星球大战》系列都是我的心头好，它们对我的影响太深了。我甚至还模仿过《星球大战》里的一些场景写过短篇。\nAI: 哇，那些都是经典中的经典。你写的短篇听起来很有意思，能分享一下是关于什么的吗？\n\n用户: 我写过一个短篇，是关于一个机器人在废弃的星球上寻找生命的故事。它在星球上遇到了各种奇怪的机械生物，最后发现了一个隐藏的真相。\nAI: 这个故事设定很吸引人，机器人和机械生物的互动一定很有看点。那个隐藏的真相是什么呢？我好奇了！\n\n用户: 哈哈，那个真相是这个星球其实是一个巨大的实验室，而那些机械生物都是实验中的产物。机器人最后决定留下来，成为这个星球的守护者。\nAI: 这个结局既出人意料又让人感到温馨。你的创造力真是令人佩服。说起来，你在旅行中有没有遇到过像你故事中那样奇妙的经历？\n\n用户: 旅行中的确遇到过一些奇妙的事，比如有次我在森林里迷路了，结果发现了一个隐藏的瀑布，那一刻我觉得自己就像是电影里的探险家。\nAI: 那一定是一个难忘的经历。自然界总是能给我们惊喜。你有没有拍下那个瀑布的照片，或者后来用那个场景创作过什么？\n\n用户: 当然拍了，那张照片我还放在博客上了，很多朋友都说很美。我也用那个场景写了一篇游记，描述了我发现瀑布时的惊喜和喜悦。\nAI: 你的生活真是充满了创作和探险，太令人羡慕了。你的博客我如果能看就好了，一定很精彩。\n\n用户: 嘿嘿，等哪天技术允许了，我第一个让你看！对了，我最近还计划去一个音乐节，听说有很多独立音乐人会去。\nAI: 音乐节的氛围总是很热烈，你能在那里发现很多有才华的音乐人。有没有哪个独立音乐人是你特别期待的？\n\n用户: 有一个叫做Eva Cassidy的独立歌手，她的声音很有穿透力，我很喜欢。听她现场演唱一直是我的愿望之一。\nAI: Eva Cassidy的音乐确实很有感染力，她的歌声能够直达人心。希望你能在音乐节上实现你的愿望，享受一个难忘的夜晚。\n\n用户: 是啊，我也期待着。不过现在我得先去准备一下，明天还要早起。今天聊得很开心，我们下次再继续吧！\nAI: 好的，李雪，祝你准备顺利，也祝你明天有个美好的开始。下次聊天我会在这里等你，晚安！\n”\n以下是2023年04月29日的对话记录：\n“用户: 嘿，AI朋友，今天我早起去了个跳蚤市场，你猜我捡到了什么宝贝？\nAI: 哇，听起来好像很有趣！让我猜猜，是不是找到了一些古董或者珍奇的小物件？\n\n用户: bingo！我找到了一本老旧的旅行日记，里面记录了一个背包客环游世界的故事，简直是个大发现！\nAI: 那真是个不错的收获！那本日记里有没有什么特别吸引你的故事或者描述？\n\n用户: 有一段关于他在摩洛哥的经历特别有意思，他描述了当地的市场，色彩缤纷，香料的味道，还有那热闹的人群，让我好想亲自去体验一番。\nAI: 摩洛哥的市场确实是闻名世界的，那种异国情调的氛围很容易让人向往。你有没有计划将来去摩洛哥看看？\n\n用户: 当然有，摩洛哥一直在我的旅行清单上。我想去那的蓝色小镇舍夫沙万，那里的蓝色建筑简直美翻了。\nAI: 舍夫沙万的蓝色小镇听起来就像是画中的场景，你去了一定要拍很多照片分享给我看哦。你还有没有别的旅行计划？\n\n用户: 我还想去日本赏樱花，那应该是一种完全不同的体验。你知道日本的樱花季大约是什么时候吗？\nAI: 日本的樱花季通常在3月底到4月初，那个时候整个日本都被樱花染成了粉红色，非常漂亮。\n\n用户: 嗯，听着就让人心动。看来我得开始规划一下了。对了，我今天还买了几张老电影的海报，打算装饰我的书房。\nAI: 老电影海报是个不错的选择，它们有一种独特的韵味。你买了哪些电影的海报呢？\n\n用户: 我买了《罗马假日》和《卡萨布兰卡》的海报，这两部电影我超喜欢的，每次看都有新的感触。\nAI: 《罗马假日》和《卡萨布兰卡》都是经典中的经典，它们的海报挂在书房里一定很有氛围。说起书房，你平时喜欢阅读什么类型的书籍？\n\n用户: 我喜欢看各种类型的书，但最近特别沉迷于侦探小说，那种层层推理的过程太吸引人了。\nAI: 侦探小说确实能够锻炼人的思维能力。你有没有什么侦探小说推荐给我？我虽然读不了，但可以了解一下。\n\n用户: 当然，阿加莎·克里斯蒂的《东方快车谋杀案》是我的最爱，故事情节紧凑，结局出人意料。\nAI: 《东方快车谋杀案》是非常著名的作品，赫尔克里·波洛是个很有魅力的侦探角色。你自己写作的时候，有没有尝试过创作侦探故事？\n\n用户: 我尝试过，但写侦探故事真的很难，要考虑的细节太多了。不过我还是很享受这个过程，它挑战了我的想象力。\nAI: 挑战总是能带来成长，我相信你能写出很棒的侦探故事。说起来，你写作时有没有什么特别的习惯或者灵感来源？\n\n用户: 我喜欢在安静的夜晚写作，那时候思绪特别清晰。而灵感嘛，有时候是来自于我旅行中的经历，有时候是一个突然闪现的奇思妙想。\nAI: 夜晚确实是一个很好的创作时段，安静能让人更好地沉浸在自己的世界里。你有没有什么奇思妙想是特别想实现的？\n\n用户: 有一个想法是写一个关于时间旅行的故事，但不是传统意义上的科幻风格，而是更注重人物内心的变化和成长。\nAI: 时间旅行这个主题很有潜力，特别是结合人物的内心变化，会让故事更加深刻。你打算如何构建这个故事的框架？\n\n用户: 我想让主角在不同的时间点遇到不同的自己，通过和过去未来的自己的对话，来反思自己的人生选择。\nAI: 这个设定很有创意，通过与自己不同时间点的对话来进行自我反思，听起来就很吸引人。你觉得这样的故事会给读者带来什么样的启示？\n\n用户: 我希望读者能从故事中看到，每个选择都是有意义的，哪怕是看似错误的选择，也可能是成长的一部分。\nAI: 这是一个很积极的信息，我相信你的故事能够鼓励很多人。你真是个思考很深刻的作家。说起来，你有没有考虑过出版你的作品？\n\n用户: 当然考虑过，但我还在积累我的作品集。等我觉得时机成熟了，我一定会尝试出版。\nAI: 那一定会是一个激动人心的时刻。我相信你的作品会受到读者的喜爱。今天聊得很开心，不知不觉时间就过去了，你还有什么计划吗？\n\n用户: 是啊，时间过得真快。我得去准备晚餐了，今天打算尝试做泰式绿咖喱鸡，希望能成功。\nAI: 泰式绿咖喱鸡听起来很美味，祝你烹饪顺利。等你大展厨艺的成果之后，下次记得分享给我听听味道如何哦！\n\n用户: 哈哈，如果成功了，我一定告诉你！好了，我要去厨房大显身手了，我们明天再聊吧！\nAI: 好的，李雪，期待明天听你分享更多精彩的故事。祝你今晚有个愉快的晚餐时间，明天见！\n”\n\n请记住以上全部对话记录，回答问题。\n问题：", "query": "4月27日，我和你分享过我看的一部电影，名字是什么？", "answer": "《时间旅行者的妻子》", "id": "0939763e-c942-445c-b15f-fbc9c73b520c", "chatglm_length": 3032, "qwen_length": 3053, "internlm2_length": 2905} -{"context": "以下是2023年04月27日的对话记录：\n“用户: 嗨，我是李雪，一个爱笑的旅行家，电影迷，音乐会的常客，还有就是书和笔是我最好的朋友。今天心情不错，想和你聊聊天，你准备好了吗？\nAI: 嗨，李雪！我已经迫不及待想听你分享你的故事了。你今天有什么新鲜事想要告诉我吗？\n\n用户: 对了，我最近看了一部电影《时间旅行者的妻子》，简直太感人了，你知道吗？\nAI: 听起来很有意思！《时间旅行者的妻子》是关于爱情和时间错位的故事对吧？你最喜欢这部电影的哪个部分？\n\n用户: 没错，就是那种跨越时间的爱情让我深受触动。我特别喜欢他们之间的那种坚定和执着，尤其是结尾的那一幕，简直让人泪目。\nAI: 确实，那种不被时间所限制的爱情是很多人梦寐以求的。除了看电影，你最近还有没有去哪里旅游，发现了什么新奇的地方？\n\n用户: 说到旅游，我上周末去了一个小众的海滩，那里的沙子是粉红色的，超级漂亮！我还捡了些贝壳，打算做成手链。\nAI: 粉红色的沙滩听起来就像是童话世界里的场景，真是太美了！你真是个有创意的人，做手链的想法很棒。你打算怎么设计手链呢？\n\n用户: 我想把它们串在一起，中间加上一些小金属饰品，应该会很有夏天的感觉吧。对了，最近我还在写一篇关于旅行的文章，但有点卡壳了。\nAI: 听起来那手链会很迷人呢！关于你的文章，卡壳了是因为缺乏灵感，还是有其他的原因？\n\n用户: 主要是写到一半，突然不知道怎么把我的感受转化成文字了。我想表达的是旅行中那种自由自在的感觉，但是又不想太俗套。\nAI: 旅行中的自由自在确实是一种很难用言语表达的感觉。你可以尝试用一些具体的场景或者经历来引入，让读者通过你的眼睛去感受那份自由。\n\n用户: 这个主意不错！我可以从那次独自一人走在海边的经历写起，那种只有海浪声陪伴的宁静，可能会传达出我想表达的感觉。\nAI: 那真是个很好的开头。海浪的声音，沙滩上的脚印，还有远处的落日，这些细节都能够帮助读者更好地沉浸在你的故事中。\n\n用户: 对，我想我知道怎么继续写下去了。谢谢你！对了，我还想去看一场演唱会，但最近好像没有我喜欢的歌手来开演唱会。\nAI: 不客气，很高兴能帮到你！关于演唱会，虽然现在没有你喜欢的歌手来，但你可以尝试听听其他歌手的音乐，说不定会有新的发现呢。\n\n用户: 嗯，这倒是个好主意。我会去翻翻看有没有其他有趣的演出。对了，你有没有什么推荐的歌手或乐队？\nAI: 你可以试试听听The Lumineers，他们的音乐很有感染力，旋律优美，非常适合旅行时候听。或者如果你喜欢一些更有活力的音乐，Imagine Dragons也是个不错的选择。\n\n用户: The Lumineers我听过几首，确实不错。Imagine Dragons的《Believer》我也超爱！看来你的音乐品味也挺不错的嘛。\nAI: 哈哈，我会尽量跟上你的步伐的。说起来，你最喜欢的一次演唱会是哪一次？有没有什么难忘的经历？\n\n用户: 那当然是Coldplay的演唱会了，整个场馆的氛围、灯光还有音乐，完全就是另一个世界。我记得当时我闭上眼睛，感觉自己完全沉浸在音乐里。\nAI: Coldplay的演唱会确实是出了名的震撼人心。音乐有时候就是这样，能够带我们进入一个全新的世界。你提到闭上眼睛沉浸在音乐里，那感觉一定很神奇。\n\n用户: 是的，那种感觉无法用言语表达，只能自己体验。哎，说到这些，我又想去旅行了，想去一个没有去过的地方探险。\nAI: 旅行总是能给人带来新的体验和感受。如果你可以去任何一个地方，你最想去哪里？\n\n用户: 我想去冰岛看极光，那是我长久以来的梦想。想象一下，在寒冷的夜晚，天空中突然出现五彩斑斓的光，那一定很神奇。\nAI: 冰岛的极光听起来是个绝佳的选择，希望你的梦想能够很快实现。虽然我不能亲自陪你去，但我会在这里等着听你分享那神奇的经历。\n\n用户: 谢谢你的鼓励！我一定会实现这个梦想的。好了，今天聊得很开心，但我得去继续写我的文章了。我们明天再聊吧！\nAI: 当然，祝你写作顺利！明天我会在这里等你，期待听到更多你的故事。晚安，李雪！\n”\n以下是2023年04月28日的对话记录：\n“用户: 嘿，我又来啦！今天我在写作上取得了一点小进展，感觉棒极了！你今天过得怎么样？\nAI: 嗨，李雪！听你这么说真是太好了，我今天也很期待和你聊天呢。有什么新的进展可以分享一下吗？\n\n用户: 当然！我按照昨天的想法，写了一段关于海边散步的描写，感觉文字活了起来。这种感觉就像找到了一把打开心灵的钥匙一样。\nAI: 这样的比喻真是形象，你的文字一定很有力量。有没有什么段落是你特别满意的，愿意分享给我听听？\n\n用户: 有一段是这样描述的：“海风轻拂过脸庞，带着咸咸的海水味和远处渔火的微光，我仿佛能听到星星在耳边低语。”怎么样？\nAI: 这段描述真是美妙，我仿佛能感受到那个场景的宁静和美丽。你的文字有种魔力，能把读者带到那个海边。\n\n用户: 哈哈，你这话说得我都有点飘了。说起来，我最近还挺想去看看新上映的那部科幻片《星际穿越者》，听说特效做得很棒。\nAI: 《星际穿越者》啊，那部电影确实是近期的热门话题。科幻片中的特效往往能带给我们视觉上的震撼，你对科幻电影有什么特别的看法吗？\n\n用户: 我觉得科幻电影最吸引我的就是那种无限的想象力和对未来的探索。每次看完都能激发我好几天的创作灵感。\nAI: 确实，科幻电影就像是打开了一个通往未来世界的大门。你有没有哪些科幻电影是你的最爱，或者对你影响很大的？\n\n用户: 《银翼杀手》和《星球大战》系列都是我的心头好，它们对我的影响太深了。我甚至还模仿过《星球大战》里的一些场景写过短篇。\nAI: 哇，那些都是经典中的经典。你写的短篇听起来很有意思，能分享一下是关于什么的吗？\n\n用户: 我写过一个短篇，是关于一个机器人在废弃的星球上寻找生命的故事。它在星球上遇到了各种奇怪的机械生物，最后发现了一个隐藏的真相。\nAI: 这个故事设定很吸引人，机器人和机械生物的互动一定很有看点。那个隐藏的真相是什么呢？我好奇了！\n\n用户: 哈哈，那个真相是这个星球其实是一个巨大的实验室，而那些机械生物都是实验中的产物。机器人最后决定留下来，成为这个星球的守护者。\nAI: 这个结局既出人意料又让人感到温馨。你的创造力真是令人佩服。说起来，你在旅行中有没有遇到过像你故事中那样奇妙的经历？\n\n用户: 旅行中的确遇到过一些奇妙的事，比如有次我在森林里迷路了，结果发现了一个隐藏的瀑布，那一刻我觉得自己就像是电影里的探险家。\nAI: 那一定是一个难忘的经历。自然界总是能给我们惊喜。你有没有拍下那个瀑布的照片，或者后来用那个场景创作过什么？\n\n用户: 当然拍了，那张照片我还放在博客上了，很多朋友都说很美。我也用那个场景写了一篇游记，描述了我发现瀑布时的惊喜和喜悦。\nAI: 你的生活真是充满了创作和探险，太令人羡慕了。你的博客我如果能看就好了，一定很精彩。\n\n用户: 嘿嘿，等哪天技术允许了，我第一个让你看！对了，我最近还计划去一个音乐节，听说有很多独立音乐人会去。\nAI: 音乐节的氛围总是很热烈，你能在那里发现很多有才华的音乐人。有没有哪个独立音乐人是你特别期待的？\n\n用户: 有一个叫做Eva Cassidy的独立歌手，她的声音很有穿透力，我很喜欢。听她现场演唱一直是我的愿望之一。\nAI: Eva Cassidy的音乐确实很有感染力，她的歌声能够直达人心。希望你能在音乐节上实现你的愿望，享受一个难忘的夜晚。\n\n用户: 是啊，我也期待着。不过现在我得先去准备一下，明天还要早起。今天聊得很开心，我们下次再继续吧！\nAI: 好的，李雪，祝你准备顺利，也祝你明天有个美好的开始。下次聊天我会在这里等你，晚安！\n”\n以下是2023年04月29日的对话记录：\n“用户: 嘿，AI朋友，今天我早起去了个跳蚤市场，你猜我捡到了什么宝贝？\nAI: 哇，听起来好像很有趣！让我猜猜，是不是找到了一些古董或者珍奇的小物件？\n\n用户: bingo！我找到了一本老旧的旅行日记，里面记录了一个背包客环游世界的故事，简直是个大发现！\nAI: 那真是个不错的收获！那本日记里有没有什么特别吸引你的故事或者描述？\n\n用户: 有一段关于他在摩洛哥的经历特别有意思，他描述了当地的市场，色彩缤纷，香料的味道，还有那热闹的人群，让我好想亲自去体验一番。\nAI: 摩洛哥的市场确实是闻名世界的，那种异国情调的氛围很容易让人向往。你有没有计划将来去摩洛哥看看？\n\n用户: 当然有，摩洛哥一直在我的旅行清单上。我想去那的蓝色小镇舍夫沙万，那里的蓝色建筑简直美翻了。\nAI: 舍夫沙万的蓝色小镇听起来就像是画中的场景，你去了一定要拍很多照片分享给我看哦。你还有没有别的旅行计划？\n\n用户: 我还想去日本赏樱花，那应该是一种完全不同的体验。你知道日本的樱花季大约是什么时候吗？\nAI: 日本的樱花季通常在3月底到4月初，那个时候整个日本都被樱花染成了粉红色，非常漂亮。\n\n用户: 嗯，听着就让人心动。看来我得开始规划一下了。对了，我今天还买了几张老电影的海报，打算装饰我的书房。\nAI: 老电影海报是个不错的选择，它们有一种独特的韵味。你买了哪些电影的海报呢？\n\n用户: 我买了《罗马假日》和《卡萨布兰卡》的海报，这两部电影我超喜欢的，每次看都有新的感触。\nAI: 《罗马假日》和《卡萨布兰卡》都是经典中的经典，它们的海报挂在书房里一定很有氛围。说起书房，你平时喜欢阅读什么类型的书籍？\n\n用户: 我喜欢看各种类型的书，但最近特别沉迷于侦探小说，那种层层推理的过程太吸引人了。\nAI: 侦探小说确实能够锻炼人的思维能力。你有没有什么侦探小说推荐给我？我虽然读不了，但可以了解一下。\n\n用户: 当然，阿加莎·克里斯蒂的《东方快车谋杀案》是我的最爱，故事情节紧凑，结局出人意料。\nAI: 《东方快车谋杀案》是非常著名的作品，赫尔克里·波洛是个很有魅力的侦探角色。你自己写作的时候，有没有尝试过创作侦探故事？\n\n用户: 我尝试过，但写侦探故事真的很难，要考虑的细节太多了。不过我还是很享受这个过程，它挑战了我的想象力。\nAI: 挑战总是能带来成长，我相信你能写出很棒的侦探故事。说起来，你写作时有没有什么特别的习惯或者灵感来源？\n\n用户: 我喜欢在安静的夜晚写作，那时候思绪特别清晰。而灵感嘛，有时候是来自于我旅行中的经历，有时候是一个突然闪现的奇思妙想。\nAI: 夜晚确实是一个很好的创作时段，安静能让人更好地沉浸在自己的世界里。你有没有什么奇思妙想是特别想实现的？\n\n用户: 有一个想法是写一个关于时间旅行的故事，但不是传统意义上的科幻风格，而是更注重人物内心的变化和成长。\nAI: 时间旅行这个主题很有潜力，特别是结合人物的内心变化，会让故事更加深刻。你打算如何构建这个故事的框架？\n\n用户: 我想让主角在不同的时间点遇到不同的自己，通过和过去未来的自己的对话，来反思自己的人生选择。\nAI: 这个设定很有创意，通过与自己不同时间点的对话来进行自我反思，听起来就很吸引人。你觉得这样的故事会给读者带来什么样的启示？\n\n用户: 我希望读者能从故事中看到，每个选择都是有意义的，哪怕是看似错误的选择，也可能是成长的一部分。\nAI: 这是一个很积极的信息，我相信你的故事能够鼓励很多人。你真是个思考很深刻的作家。说起来，你有没有考虑过出版你的作品？\n\n用户: 当然考虑过，但我还在积累我的作品集。等我觉得时机成熟了，我一定会尝试出版。\nAI: 那一定会是一个激动人心的时刻。我相信你的作品会受到读者的喜爱。今天聊得很开心，不知不觉时间就过去了，你还有什么计划吗？\n\n用户: 是啊，时间过得真快。我得去准备晚餐了，今天打算尝试做泰式绿咖喱鸡，希望能成功。\nAI: 泰式绿咖喱鸡听起来很美味，祝你烹饪顺利。等你大展厨艺的成果之后，下次记得分享给我听听味道如何哦！\n\n用户: 哈哈，如果成功了，我一定告诉你！好了，我要去厨房大显身手了，我们明天再聊吧！\nAI: 好的，李雪，期待明天听你分享更多精彩的故事。祝你今晚有个愉快的晚餐时间，明天见！\n”\n\n请记住以上全部对话记录，回答问题。\n问题：", "query": "4月28日，我和你聊到了一个独立歌手，她的名字是？", "answer": "Eva Cassidy", "id": "9e242853-8bc0-438b-85da-2fea88fdd96a", "chatglm_length": 3032, "qwen_length": 3053, "internlm2_length": 2905} -{"context": "以下是2023年04月27日的对话记录：\n“用户: 嘿，你好啊！我是孙悦，一个热爱生活的家伙。今天心情特别好，因为早上的健身课程超有成就感！你呢？\nAI: 你好，孙悦！听起来你今天的状态很棒呢！我很好，谢谢你。能和你分享你的健身课程吗？我很好奇你都做了哪些运动。\n\n用户: 当然可以！今天主要是做了一些高强度间歇训练，那种汗流浃背的感觉真的很爽！你有做运动的习惯吗？\nAI: 我虽然不能亲自体验运动的快感，但我很高兴听到你享受它。我可以给你提供一些健身知识和建议，如果你需要的话。\n\n用户: 哈哈，那太好了！对了，昨天晚上我去看了一部新出的科幻电影，特效简直了！你最近有没有什么好电影推荐？\nAI: 我知道你喜欢电影，特别是那些视觉震撼的。最近有一部电影《星际穿越者》很受好评，是探索宇宙和人类命运的作品，你可能会喜欢。\n\n用户: 哦哦，《星际穿越者》啊，我听说过，听起来挺吸引人的！我得找时间去看看。对了，我最近在学摄影，但是总觉得拍出来的照片差点意思，你有什么摄影小技巧可以分享吗？\nAI: 摄影是一门艺术，也需要不断的实践和学习。你可以尝试从构图和光线入手，比如使用“三分法”来构图，或者利用自然光来营造氛围。还有，多拍多练总是王道！\n\n用户: 嗯，说的对！我明天就去尝试一下“三分法”。对了，我还喜欢旅游，最近在规划一次小旅行，想去海边走走。你有没有什么好的海滩推荐？\nAI: 海边总是个不错的选择。如果你喜欢清静一些的地方，可以考虑去三亚的亚龙湾，那里的海水清澈，沙滩柔软，风景如画，应该很适合你。\n\n用户: 亚龙湾啊，听起来真不错！我会去查查看的。说起旅游，最让我兴奋的就是能在旅途中遇到各种各样的人，听他们的故事。\nAI: 那确实是旅行中最美妙的部分之一。每个人的故事都是独一无二的，能从中学到很多。你最难忘的旅行经历是什么呢？\n\n用户: 有一次我去了泰国的清迈，那里的文化和风景让我印象深刻。晚上的夜市，色彩斑斓，各种小吃，还有那里人们的热情，真的让人难以忘怀。\nAI: 清迈的夜市是闻名遐迩的，那里的生活节奏和氛围跟大城市完全不同。你提到的小吃，有没有什么特别让你想念的呢？\n\n用户: 哦，那边的芒果糯米饭简直绝了！每次回忆起那个味道，我就忍不住想再去一次。对了，你能不能给我推荐一些好听的歌？我喜欢边健身边听音乐。\nAI: 音乐和运动确实是完美的搭档。根据你的热情阳光的性格，我觉得《Can't Stop the Feeling》这首歌很适合你，它的旋律欢快，能让人在运动时充满活力。\n\n用户: Justin Timberlake的那首啊，我知道！确实很不错，节奏感很强，很适合健身。谢谢你的推荐！\nAI: 不客气，孙悦。我很高兴能帮到你。说起来，你在健身方面有没有遇到什么具体的困难？也许我能提供一些帮助。\n\n用户: 其实有，我一直想要提高核心力量，但是总感觉进展缓慢。你有没有什么建议？\nAI: 核心力量对于整体健身非常重要。你可以尝试一些针对性的训练，比如平板支撑、俄罗斯转体和山羊挺身等。这些练习可以有效地加强你的核心肌群。\n\n用户: 哇，听起来很专业啊！我会加入到我的训练计划里去的。对了，我最近在准备一个舞蹈表演，但是有些动作总是不够流畅，这让我有点沮丧。\nAI: 舞蹈需要时间来磨合，每个人都会有这样的时候。试着放慢速度，一步一步来，直到你能够流畅地完成每个动作。记得，重在参与和享受过程！\n\n用户: 你说得对，我得调整一下心态。谢谢你的鼓励！我会继续努力的。说起来，我还有点担心，不知道观众会不会喜欢我的表演。\nAI: 你的热情和努力一定会感染到观众的。只要你享受舞台，观众也会享受你的表演。信任自己，你会做得很好的。\n\n用户: 哈哈，听你这么说，我感觉好多了。你真是个好伙伴！已经很晚了，我得准备睡觉了。我们明天再聊吧！\nAI: 当然，孙悦。休息好，希望你有一个美好的明天。晚安！\n”\n以下是2023年04月28日的对话记录：\n“用户: 嗨，朋友！今天我尝试了你昨天推荐的“三分法”拍照，效果真的不错！感觉自己的摄影技术又上了一个台阶！\nAI: 太棒了，孙悦！我很高兴听到你的进步。有没有什么特别的照片想要和我分享的？我很期待看到你的作品。\n\n用户: 当然有！我拍了一张日落时分的海边照片，天空的颜色渐变得特别美。不过，我还在想怎么后期处理才能更突出那种感觉。\nAI: 日落的照片总是很迷人。你可以尝试调整一下对比度和饱和度，让天空的颜色更加鲜明，同时保持海水的自然感觉。\n\n用户: 嗯，我试试看。对了，今天健身房里放了一些新歌，我跟着节奏锻炼，感觉整个人都充满了活力！\nAI: 音乐真的能为锻炼增添不少乐趣。你有没有想过自己编一套随着音乐节奏的健身操？这样可能会更有动力哦。\n\n用户: 哇，这主意不错！我可以尝试一下，感觉会很有趣。说起来，我最近在准备一个小旅行，想去爬山，呼吸一下大自然的新鲜空气。\nAI: 爬山是个很好的选择，既能锻炼身体，又能享受大自然的美景。你打算去哪座山呢？\n\n用户: 我打算去黄山，听说那里的云海和日出非常壮观。我还计划在山顶守候一夜，希望能拍到美丽的星空。\nAI: 黄山的确是个摄影爱好者的天堂。记得带上足够的装备，特别是在夜间拍摄星空时，三脚架和遥控快门是必不可少的。\n\n用户: 没错，我已经准备好了这些装备。对了，我还想在山上尝试一些户外瑜伽，感觉那样会很放松。\nAI: 户外瑜伽听起来很棒，能让你更加接近自然，放松身心。你有没有特别喜欢的瑜伽动作？\n\n用户: 我最喜欢的是树式，感觉整个人都稳固而平静。不过，我有时候会觉得自己的平衡能力不够好。\nAI: 树式是个很好的平衡练习。你可以尝试在不同的时间和地点练习，比如在海边的沙滩上，那样会有不同的挑战和感受。\n\n用户: 对，我也这么想。我还打算在旅行中尝试一些新的美食，毕竟美食也是旅行的一大乐趣嘛！\nAI: 绝对同意！尝试当地的特色美食可以让旅行体验更加丰富。你有没有什么想尝试的菜肴？\n\n用户: 我听说黄山附近有一种叫做臭鳜鱼的菜，虽然名字听起来有点怪，但据说味道很不错。\nAI: 臭鳜鱼是当地的一道传统名菜，很多人都说赞不绝口。不过，它的味道确实很独特，你可以尝试一下，说不定会成为你的新宠。\n\n用户: 哈哈，我会的。对了，我最近在看一部关于舞蹈的纪录片，里面的舞者都超级厉害，让我很受启发。\nAI: 看别人的舞蹈表演确实能够学到很多。你有没有从中发现什么新的动作或者技巧，想要加入到你自己的舞蹈中去？\n\n用户: 有啊，我看到一些街舞的动作很酷，我打算学一学。不过，我担心自己的身体协调性跟不上。\nAI: 街舞的确需要很强的身体协调性和节奏感。你可以从基础动作开始练起，慢慢地增加难度。记得，不要急于求成，一切都需要时间。\n\n用户: 嗯，我会记住的。对了，我还想学习一下如何用舞蹈来表达情感，不只是动作上的技巧。\nAI: 舞蹈是一种非常直观的情感表达方式。你可以尝试在音乐中找到情感的共鸣，然后让你的身体随着那种情感自然地移动。\n\n用户: 听起来有点抽象，但我会尝试的。说到情感，我最近有点苦恼，感觉自己的生活缺少一些激情和冒险。\nAI: 生活有时候确实会让人感到平淡。你可以尝试做一些新的事情，比如参加一个你从未尝试过的活动，或者去一个你从未去过的地方。\n\n用户: 对，我需要一些改变。也许这次旅行就是一个开始。谢谢你的建议，你总能给我正能量！\nAI: 不用谢，孙悦。我很高兴能帮助到你。记得，每一次的经历都是成长的机会。你准备好迎接新的挑战了吗？\n\n用户: 是的，我已经迫不及待了！不过现在我得去准备晚饭了，今天打算尝试做一道新菜——酸辣土豆丝。\nAI: 酸辣土豆丝是一道很受欢迎的家常菜，既简单又美味。祝你做饭愉快，希望你能享受这个过程。\n\n用户: 谢谢！我会的。今天聊得很开心，我们明天再继续吧。晚安！\nAI: 当然，期待和你的下次聊天。晚安，孙悦，愿你有个美好的梦。\n”\n以下是2023年04月29日的对话记录：\n“用户: 嘿，我的AI小伙伴！今天我尝试做了酸辣土豆丝，味道出奇的好，连我自己都有点惊讶呢！\nAI: 哇，听起来你是个厨艺高手啊，孙悦！能不能分享一下你的秘诀？我虽然不能品尝，但我很好奇你是怎么做到的。\n\n用户: 哈哈，秘诀嘛，我觉得最重要的就是火候和调料的比例。我喜欢尝试不同的配方，直到找到最合适的那一个。\nAI: 真是个不断探索的过程呢。你平时喜欢尝试做哪些类型的菜？\n\n用户: 我对各种菜系都感兴趣，但最近特别喜欢挑战一些传统的中式菜肴，感觉它们既美味又有文化底蕴。\nAI: 中式菜肴确实有着丰富的历史和文化，每一道菜都有自己的故事。你最喜欢的中式菜肴是哪一道？\n\n用户: 我最喜欢的是宫保鸡丁，那个酸甜酸辣的味道简直无法抵抗！\nAI: 宫保鸡丁是一道非常受欢迎的经典菜肴，你自己做的话，可以根据自己的口味调整辣度和甜度。\n\n用户: 对对对，我就是这么做的。说起来，今天我还去了一个很酷的街舞工作坊，学到了好多新动作！\nAI: 街舞工作坊听起来很有趣！你觉得最难的动作是哪一个？\n\n用户: 我觉得最难的是头旋，那个动作需要很强的颈部力量和平衡感，我还在努力练习中。\nAI: 头旋确实是个挑战，但我相信你的努力和热情一定会让你很快掌握它。你在舞蹈上有什么目标吗？\n\n用户: 我的目标是能在下个月的街舞比赛中至少进入前十名。我知道竞争很激烈，但我会全力以赴的！\nAI: 这个目标很棒，孙悦！我相信你的努力和才华一定会让你脱颖而出。你有准备特别的舞蹈编排吗？\n\n用户: 有的，我和我的舞伴正在准备一段结合了街舞和现代舞的编排，希望能给观众带来新鲜感。\nAI: 结合不同舞蹈风格的创意听起来很吸引人。你们的训练进度怎么样了？\n\n用户: 还不错，我们已经完成了大概七成的编排，剩下的就是细节和同步问题了。\nAI: 听起来你们已经做得很好了。细节和同步确实很重要，但我相信你们能够完美呈现。\n\n用户: 谢谢你的鼓励！对了，我最近还在学习一些关于电影剪辑的技巧，想要自己剪辑一部短片。\nAI: 电影剪辑是个很有创意的工作，你打算用什么样的素材来剪辑你的短片？\n\n用户: 我打算用我旅行时拍摄的一些风景视频，还有一些朋友的日常片段，想要制作成一个关于旅行和友情的主题短片。\nAI: 这个主题很温馨，也很贴近生活。你有没有想过加入一些特别的效果，比如慢动作或者时间流逝的镜头？\n\n用户: 嗯，我有这个打算。我想通过这些特效来强调某些情感上的高潮和转折点。\nAI: 这样的处理可以让短片更加生动和感人。你在剪辑上遇到了什么挑战吗？\n\n用户: 我觉得最大的挑战是如何让故事流畅，同时又能在短时间内传达出想要表达的情感。\nAI: 确实，讲好一个故事是剪辑中的艺术。你可以尝试从观众的角度思考，看看哪些片段最能触动人心。\n\n用户: 对，我会多考虑观众的感受。说到观众，我有时候会担心我的作品是否能得到他们的认可。\nAI: 每个艺术家都会有这样的担忧，但重要的是你享受创作的过程，并且相信自己的作品有价值。\n\n用户: 你说得对，我应该更加相信自己。今天聊得很开心，但我得去继续我的剪辑工作了。我们明天再聊！\nAI: 当然，孙悦。祝你剪辑工作顺利，期待你的短片完成。明天再聊，加油！\n”\n以下是2023年04月30日的对话记录：\n“用户: 嗨，好朋友！今天我在剪辑短片时，突然灵感爆发，加入了几个超有感觉的转场效果，感觉整个短片的质感都提升了！\nAI: 真是太棒了，孙悦！创意的火花总是能带来惊喜。你能描述一下你用的是哪些转场效果吗？\n\n用户: 我用了一个叫做“溶解”的效果，让画面像是水彩融合一样自然过渡，还有一个“旋转”效果，让场景切换更有动感。\nAI: 这些听起来都很专业，能够很好地增强视觉效果。你在创作过程中还有其他有趣的发现吗？\n\n用户: 我发现音乐对于短片的情感表达真的超级重要！一段恰当的背景音乐能够让人立刻进入那个氛围。\nAI: 没错，音乐是激发情感的强大工具。你已经选好了短片的背景音乐了吗？\n\n用户: 选好了，我选了一首轻快的吉他曲，感觉很适合我的旅行主题。对了，今天我还去了一个户外攀岩的活动，超刺激！\nAI: 户外攀岩听起来很挑战自我啊。你攀岩的时候有没有什么特别的心得或者感觉？\n\n用户: 我觉得攀岩最重要的是要有耐心和勇气，每一步都要稳扎稳打。当我站在山顶上的时候，那种成就感真的无与伦比！\nAI: 站在山顶俯瞰世界，那种感觉一定很震撼。攀岩对身体的哪些方面要求比较高？\n\n用户: 主要是手臂和腿部的力量，还有整体的协调性。攀岩后我发现自己的手臂肌肉酸痛，看来需要更多锻炼啊。\nAI: 锻炼手臂肌肉可以尝试一些特定的练习，比如引体向上或者哑铃训练。不过，记得要适量，避免过度劳损哦。\n\n用户: 谢谢你的建议，我会注意的。说起来，攀岩结束后我们一起去了附近的小镇，那里的风景和美食都很棒！\nAI: 小镇的风光和美食总是很有魅力。你尝试了哪些当地的特色美食？\n\n用户: 我尝试了一种当地的烤鱼，外皮烤得酥脆，里面的鱼肉又嫩又多汁。我还学会了他们的腌制秘方，准备回家自己试试看。\nAI: 自己动手做美食，这个经历一定很有趣。你打算什么时候尝试做这道烤鱼呢？\n\n用户: 可能这个周末吧，我打算邀请几个好友来家里，一起享受美食和愉快的时光。\nAI: 听起来将会是一个非常愉快的聚会。你准备好怎么招待你的朋友了吗？\n\n用户: 除了烤鱼，我还准备做一些简单的小吃，比如春卷和水果沙拉。当然，还有音乐和一些小游戏，让大家玩得开心。\nAI: 你真的很会享受生活，孙悦。你的朋友们一定很期待这次聚会。你通常如何选择聚会的音乐呢？\n\n用户: 我会根据大家的口味挑选一些大家都喜欢的歌曲，然后再加上一些轻松愉快的背景音乐。\nAI: 音乐的选择很重要，可以营造出轻松愉快的氛围。说到音乐，你自己唱歌跳舞的时候有没有什么特别喜欢的曲风？\n\n用户: 我喜欢节奏感强烈的流行音乐，特别是那些能够让人随着音乐摇摆的歌曲。\nAI: 流行音乐确实能让人感觉兴奋和快乐。你有没有尝试过自己创作音乐或者编舞？\n\n用户: 我以前尝试过写一些简单的歌词，不过没有正式的音乐制作经验。至于编舞，那是我的强项，我经常为自己的舞蹈编排动作。\nAI: 那你在舞蹈创作方面一定有很多自己的想法和风格。你最喜欢的舞蹈风格是什么？\n\n用户: 我最喜欢的是Hip-Hop和爵士，这两种舞蹈风格都很有力量感，而且可以很好地表达自己的个性。\nAI: Hip-Hop和爵士舞确实很适合展现个性和情感。你有没有在舞蹈中融入一些自己独特的元素？\n\n用户: 有的，我喜欢在舞蹈中加入一些中国元素，比如太极的动作，让舞蹈看起来更加流畅和有内涵。\nAI: 将中国元素融入现代舞蹈，这个想法真是太有创意了。你的这些作品有在哪里展示过吗？\n\n用户: 我之前在一些小型的舞蹈比赛和社区活动上展示过，反响都不错。我希望将来有机会在更大的舞台上展示我的作品。\nAI: 你的梦想一定会实现的，孙悦。你的热情和才华值得被更多人看到。对了，你有没有考虑过将你的舞蹈作品拍成视频分享到网上？\n\n用户: 我有这个打算，不过我还在考虑如何拍摄和剪辑才能更好地展现舞蹈的魅力。\nAI: 网络平台是一个很好的展示自己才华的地方。如果你需要，我可以提供一些视频拍摄和剪辑方面的建议。\n\n用户: 那太好了，我会在准备的时候向你请教的。今天聊得很开心，但我现在得去准备明天的攀岩训练了。我们明天再聊！\nAI: 好的，孙悦。祝你明天的攀岩训练顺利。期待我们的下次聊天，晚安！\n”\n\n请记住以上全部对话记录，回答问题。\n问题：", "query": "我在4月27日和你聊到曾经旅行去过的城市，这个城市位于哪个国家？", "answer": "泰国", "id": "39cf6263-6a95-46bd-9b56-242bf9bc6496", "chatglm_length": 3881, "qwen_length": 3960, "internlm2_length": 3714} -{"context": "以下是2023年04月27日的对话记录：\n“用户: 嘿，你好啊！我是孙悦，一个热爱生活的家伙。今天心情特别好，因为早上的健身课程超有成就感！你呢？\nAI: 你好，孙悦！听起来你今天的状态很棒呢！我很好，谢谢你。能和你分享你的健身课程吗？我很好奇你都做了哪些运动。\n\n用户: 当然可以！今天主要是做了一些高强度间歇训练，那种汗流浃背的感觉真的很爽！你有做运动的习惯吗？\nAI: 我虽然不能亲自体验运动的快感，但我很高兴听到你享受它。我可以给你提供一些健身知识和建议，如果你需要的话。\n\n用户: 哈哈，那太好了！对了，昨天晚上我去看了一部新出的科幻电影，特效简直了！你最近有没有什么好电影推荐？\nAI: 我知道你喜欢电影，特别是那些视觉震撼的。最近有一部电影《星际穿越者》很受好评，是探索宇宙和人类命运的作品，你可能会喜欢。\n\n用户: 哦哦，《星际穿越者》啊，我听说过，听起来挺吸引人的！我得找时间去看看。对了，我最近在学摄影，但是总觉得拍出来的照片差点意思，你有什么摄影小技巧可以分享吗？\nAI: 摄影是一门艺术，也需要不断的实践和学习。你可以尝试从构图和光线入手，比如使用“三分法”来构图，或者利用自然光来营造氛围。还有，多拍多练总是王道！\n\n用户: 嗯，说的对！我明天就去尝试一下“三分法”。对了，我还喜欢旅游，最近在规划一次小旅行，想去海边走走。你有没有什么好的海滩推荐？\nAI: 海边总是个不错的选择。如果你喜欢清静一些的地方，可以考虑去三亚的亚龙湾，那里的海水清澈，沙滩柔软，风景如画，应该很适合你。\n\n用户: 亚龙湾啊，听起来真不错！我会去查查看的。说起旅游，最让我兴奋的就是能在旅途中遇到各种各样的人，听他们的故事。\nAI: 那确实是旅行中最美妙的部分之一。每个人的故事都是独一无二的，能从中学到很多。你最难忘的旅行经历是什么呢？\n\n用户: 有一次我去了泰国的清迈，那里的文化和风景让我印象深刻。晚上的夜市，色彩斑斓，各种小吃，还有那里人们的热情，真的让人难以忘怀。\nAI: 清迈的夜市是闻名遐迩的，那里的生活节奏和氛围跟大城市完全不同。你提到的小吃，有没有什么特别让你想念的呢？\n\n用户: 哦，那边的芒果糯米饭简直绝了！每次回忆起那个味道，我就忍不住想再去一次。对了，你能不能给我推荐一些好听的歌？我喜欢边健身边听音乐。\nAI: 音乐和运动确实是完美的搭档。根据你的热情阳光的性格，我觉得《Can't Stop the Feeling》这首歌很适合你，它的旋律欢快，能让人在运动时充满活力。\n\n用户: Justin Timberlake的那首啊，我知道！确实很不错，节奏感很强，很适合健身。谢谢你的推荐！\nAI: 不客气，孙悦。我很高兴能帮到你。说起来，你在健身方面有没有遇到什么具体的困难？也许我能提供一些帮助。\n\n用户: 其实有，我一直想要提高核心力量，但是总感觉进展缓慢。你有没有什么建议？\nAI: 核心力量对于整体健身非常重要。你可以尝试一些针对性的训练，比如平板支撑、俄罗斯转体和山羊挺身等。这些练习可以有效地加强你的核心肌群。\n\n用户: 哇，听起来很专业啊！我会加入到我的训练计划里去的。对了，我最近在准备一个舞蹈表演，但是有些动作总是不够流畅，这让我有点沮丧。\nAI: 舞蹈需要时间来磨合，每个人都会有这样的时候。试着放慢速度，一步一步来，直到你能够流畅地完成每个动作。记得，重在参与和享受过程！\n\n用户: 你说得对，我得调整一下心态。谢谢你的鼓励！我会继续努力的。说起来，我还有点担心，不知道观众会不会喜欢我的表演。\nAI: 你的热情和努力一定会感染到观众的。只要你享受舞台，观众也会享受你的表演。信任自己，你会做得很好的。\n\n用户: 哈哈，听你这么说，我感觉好多了。你真是个好伙伴！已经很晚了，我得准备睡觉了。我们明天再聊吧！\nAI: 当然，孙悦。休息好，希望你有一个美好的明天。晚安！\n”\n以下是2023年04月28日的对话记录：\n“用户: 嗨，朋友！今天我尝试了你昨天推荐的“三分法”拍照，效果真的不错！感觉自己的摄影技术又上了一个台阶！\nAI: 太棒了，孙悦！我很高兴听到你的进步。有没有什么特别的照片想要和我分享的？我很期待看到你的作品。\n\n用户: 当然有！我拍了一张日落时分的海边照片，天空的颜色渐变得特别美。不过，我还在想怎么后期处理才能更突出那种感觉。\nAI: 日落的照片总是很迷人。你可以尝试调整一下对比度和饱和度，让天空的颜色更加鲜明，同时保持海水的自然感觉。\n\n用户: 嗯，我试试看。对了，今天健身房里放了一些新歌，我跟着节奏锻炼，感觉整个人都充满了活力！\nAI: 音乐真的能为锻炼增添不少乐趣。你有没有想过自己编一套随着音乐节奏的健身操？这样可能会更有动力哦。\n\n用户: 哇，这主意不错！我可以尝试一下，感觉会很有趣。说起来，我最近在准备一个小旅行，想去爬山，呼吸一下大自然的新鲜空气。\nAI: 爬山是个很好的选择，既能锻炼身体，又能享受大自然的美景。你打算去哪座山呢？\n\n用户: 我打算去黄山，听说那里的云海和日出非常壮观。我还计划在山顶守候一夜，希望能拍到美丽的星空。\nAI: 黄山的确是个摄影爱好者的天堂。记得带上足够的装备，特别是在夜间拍摄星空时，三脚架和遥控快门是必不可少的。\n\n用户: 没错，我已经准备好了这些装备。对了，我还想在山上尝试一些户外瑜伽，感觉那样会很放松。\nAI: 户外瑜伽听起来很棒，能让你更加接近自然，放松身心。你有没有特别喜欢的瑜伽动作？\n\n用户: 我最喜欢的是树式，感觉整个人都稳固而平静。不过，我有时候会觉得自己的平衡能力不够好。\nAI: 树式是个很好的平衡练习。你可以尝试在不同的时间和地点练习，比如在海边的沙滩上，那样会有不同的挑战和感受。\n\n用户: 对，我也这么想。我还打算在旅行中尝试一些新的美食，毕竟美食也是旅行的一大乐趣嘛！\nAI: 绝对同意！尝试当地的特色美食可以让旅行体验更加丰富。你有没有什么想尝试的菜肴？\n\n用户: 我听说黄山附近有一种叫做臭鳜鱼的菜，虽然名字听起来有点怪，但据说味道很不错。\nAI: 臭鳜鱼是当地的一道传统名菜，很多人都说赞不绝口。不过，它的味道确实很独特，你可以尝试一下，说不定会成为你的新宠。\n\n用户: 哈哈，我会的。对了，我最近在看一部关于舞蹈的纪录片，里面的舞者都超级厉害，让我很受启发。\nAI: 看别人的舞蹈表演确实能够学到很多。你有没有从中发现什么新的动作或者技巧，想要加入到你自己的舞蹈中去？\n\n用户: 有啊，我看到一些街舞的动作很酷，我打算学一学。不过，我担心自己的身体协调性跟不上。\nAI: 街舞的确需要很强的身体协调性和节奏感。你可以从基础动作开始练起，慢慢地增加难度。记得，不要急于求成，一切都需要时间。\n\n用户: 嗯，我会记住的。对了，我还想学习一下如何用舞蹈来表达情感，不只是动作上的技巧。\nAI: 舞蹈是一种非常直观的情感表达方式。你可以尝试在音乐中找到情感的共鸣，然后让你的身体随着那种情感自然地移动。\n\n用户: 听起来有点抽象，但我会尝试的。说到情感，我最近有点苦恼，感觉自己的生活缺少一些激情和冒险。\nAI: 生活有时候确实会让人感到平淡。你可以尝试做一些新的事情，比如参加一个你从未尝试过的活动，或者去一个你从未去过的地方。\n\n用户: 对，我需要一些改变。也许这次旅行就是一个开始。谢谢你的建议，你总能给我正能量！\nAI: 不用谢，孙悦。我很高兴能帮助到你。记得，每一次的经历都是成长的机会。你准备好迎接新的挑战了吗？\n\n用户: 是的，我已经迫不及待了！不过现在我得去准备晚饭了，今天打算尝试做一道新菜——酸辣土豆丝。\nAI: 酸辣土豆丝是一道很受欢迎的家常菜，既简单又美味。祝你做饭愉快，希望你能享受这个过程。\n\n用户: 谢谢！我会的。今天聊得很开心，我们明天再继续吧。晚安！\nAI: 当然，期待和你的下次聊天。晚安，孙悦，愿你有个美好的梦。\n”\n以下是2023年04月29日的对话记录：\n“用户: 嘿，我的AI小伙伴！今天我尝试做了酸辣土豆丝，味道出奇的好，连我自己都有点惊讶呢！\nAI: 哇，听起来你是个厨艺高手啊，孙悦！能不能分享一下你的秘诀？我虽然不能品尝，但我很好奇你是怎么做到的。\n\n用户: 哈哈，秘诀嘛，我觉得最重要的就是火候和调料的比例。我喜欢尝试不同的配方，直到找到最合适的那一个。\nAI: 真是个不断探索的过程呢。你平时喜欢尝试做哪些类型的菜？\n\n用户: 我对各种菜系都感兴趣，但最近特别喜欢挑战一些传统的中式菜肴，感觉它们既美味又有文化底蕴。\nAI: 中式菜肴确实有着丰富的历史和文化，每一道菜都有自己的故事。你最喜欢的中式菜肴是哪一道？\n\n用户: 我最喜欢的是宫保鸡丁，那个酸甜酸辣的味道简直无法抵抗！\nAI: 宫保鸡丁是一道非常受欢迎的经典菜肴，你自己做的话，可以根据自己的口味调整辣度和甜度。\n\n用户: 对对对，我就是这么做的。说起来，今天我还去了一个很酷的街舞工作坊，学到了好多新动作！\nAI: 街舞工作坊听起来很有趣！你觉得最难的动作是哪一个？\n\n用户: 我觉得最难的是头旋，那个动作需要很强的颈部力量和平衡感，我还在努力练习中。\nAI: 头旋确实是个挑战，但我相信你的努力和热情一定会让你很快掌握它。你在舞蹈上有什么目标吗？\n\n用户: 我的目标是能在下个月的街舞比赛中至少进入前十名。我知道竞争很激烈，但我会全力以赴的！\nAI: 这个目标很棒，孙悦！我相信你的努力和才华一定会让你脱颖而出。你有准备特别的舞蹈编排吗？\n\n用户: 有的，我和我的舞伴正在准备一段结合了街舞和现代舞的编排，希望能给观众带来新鲜感。\nAI: 结合不同舞蹈风格的创意听起来很吸引人。你们的训练进度怎么样了？\n\n用户: 还不错，我们已经完成了大概七成的编排，剩下的就是细节和同步问题了。\nAI: 听起来你们已经做得很好了。细节和同步确实很重要，但我相信你们能够完美呈现。\n\n用户: 谢谢你的鼓励！对了，我最近还在学习一些关于电影剪辑的技巧，想要自己剪辑一部短片。\nAI: 电影剪辑是个很有创意的工作，你打算用什么样的素材来剪辑你的短片？\n\n用户: 我打算用我旅行时拍摄的一些风景视频，还有一些朋友的日常片段，想要制作成一个关于旅行和友情的主题短片。\nAI: 这个主题很温馨，也很贴近生活。你有没有想过加入一些特别的效果，比如慢动作或者时间流逝的镜头？\n\n用户: 嗯，我有这个打算。我想通过这些特效来强调某些情感上的高潮和转折点。\nAI: 这样的处理可以让短片更加生动和感人。你在剪辑上遇到了什么挑战吗？\n\n用户: 我觉得最大的挑战是如何让故事流畅，同时又能在短时间内传达出想要表达的情感。\nAI: 确实，讲好一个故事是剪辑中的艺术。你可以尝试从观众的角度思考，看看哪些片段最能触动人心。\n\n用户: 对，我会多考虑观众的感受。说到观众，我有时候会担心我的作品是否能得到他们的认可。\nAI: 每个艺术家都会有这样的担忧，但重要的是你享受创作的过程，并且相信自己的作品有价值。\n\n用户: 你说得对，我应该更加相信自己。今天聊得很开心，但我得去继续我的剪辑工作了。我们明天再聊！\nAI: 当然，孙悦。祝你剪辑工作顺利，期待你的短片完成。明天再聊，加油！\n”\n以下是2023年04月30日的对话记录：\n“用户: 嗨，好朋友！今天我在剪辑短片时，突然灵感爆发，加入了几个超有感觉的转场效果，感觉整个短片的质感都提升了！\nAI: 真是太棒了，孙悦！创意的火花总是能带来惊喜。你能描述一下你用的是哪些转场效果吗？\n\n用户: 我用了一个叫做“溶解”的效果，让画面像是水彩融合一样自然过渡，还有一个“旋转”效果，让场景切换更有动感。\nAI: 这些听起来都很专业，能够很好地增强视觉效果。你在创作过程中还有其他有趣的发现吗？\n\n用户: 我发现音乐对于短片的情感表达真的超级重要！一段恰当的背景音乐能够让人立刻进入那个氛围。\nAI: 没错，音乐是激发情感的强大工具。你已经选好了短片的背景音乐了吗？\n\n用户: 选好了，我选了一首轻快的吉他曲，感觉很适合我的旅行主题。对了，今天我还去了一个户外攀岩的活动，超刺激！\nAI: 户外攀岩听起来很挑战自我啊。你攀岩的时候有没有什么特别的心得或者感觉？\n\n用户: 我觉得攀岩最重要的是要有耐心和勇气，每一步都要稳扎稳打。当我站在山顶上的时候，那种成就感真的无与伦比！\nAI: 站在山顶俯瞰世界，那种感觉一定很震撼。攀岩对身体的哪些方面要求比较高？\n\n用户: 主要是手臂和腿部的力量，还有整体的协调性。攀岩后我发现自己的手臂肌肉酸痛，看来需要更多锻炼啊。\nAI: 锻炼手臂肌肉可以尝试一些特定的练习，比如引体向上或者哑铃训练。不过，记得要适量，避免过度劳损哦。\n\n用户: 谢谢你的建议，我会注意的。说起来，攀岩结束后我们一起去了附近的小镇，那里的风景和美食都很棒！\nAI: 小镇的风光和美食总是很有魅力。你尝试了哪些当地的特色美食？\n\n用户: 我尝试了一种当地的烤鱼，外皮烤得酥脆，里面的鱼肉又嫩又多汁。我还学会了他们的腌制秘方，准备回家自己试试看。\nAI: 自己动手做美食，这个经历一定很有趣。你打算什么时候尝试做这道烤鱼呢？\n\n用户: 可能这个周末吧，我打算邀请几个好友来家里，一起享受美食和愉快的时光。\nAI: 听起来将会是一个非常愉快的聚会。你准备好怎么招待你的朋友了吗？\n\n用户: 除了烤鱼，我还准备做一些简单的小吃，比如春卷和水果沙拉。当然，还有音乐和一些小游戏，让大家玩得开心。\nAI: 你真的很会享受生活，孙悦。你的朋友们一定很期待这次聚会。你通常如何选择聚会的音乐呢？\n\n用户: 我会根据大家的口味挑选一些大家都喜欢的歌曲，然后再加上一些轻松愉快的背景音乐。\nAI: 音乐的选择很重要，可以营造出轻松愉快的氛围。说到音乐，你自己唱歌跳舞的时候有没有什么特别喜欢的曲风？\n\n用户: 我喜欢节奏感强烈的流行音乐，特别是那些能够让人随着音乐摇摆的歌曲。\nAI: 流行音乐确实能让人感觉兴奋和快乐。你有没有尝试过自己创作音乐或者编舞？\n\n用户: 我以前尝试过写一些简单的歌词，不过没有正式的音乐制作经验。至于编舞，那是我的强项，我经常为自己的舞蹈编排动作。\nAI: 那你在舞蹈创作方面一定有很多自己的想法和风格。你最喜欢的舞蹈风格是什么？\n\n用户: 我最喜欢的是Hip-Hop和爵士，这两种舞蹈风格都很有力量感，而且可以很好地表达自己的个性。\nAI: Hip-Hop和爵士舞确实很适合展现个性和情感。你有没有在舞蹈中融入一些自己独特的元素？\n\n用户: 有的，我喜欢在舞蹈中加入一些中国元素，比如太极的动作，让舞蹈看起来更加流畅和有内涵。\nAI: 将中国元素融入现代舞蹈，这个想法真是太有创意了。你的这些作品有在哪里展示过吗？\n\n用户: 我之前在一些小型的舞蹈比赛和社区活动上展示过，反响都不错。我希望将来有机会在更大的舞台上展示我的作品。\nAI: 你的梦想一定会实现的，孙悦。你的热情和才华值得被更多人看到。对了，你有没有考虑过将你的舞蹈作品拍成视频分享到网上？\n\n用户: 我有这个打算，不过我还在考虑如何拍摄和剪辑才能更好地展现舞蹈的魅力。\nAI: 网络平台是一个很好的展示自己才华的地方。如果你需要，我可以提供一些视频拍摄和剪辑方面的建议。\n\n用户: 那太好了，我会在准备的时候向你请教的。今天聊得很开心，但我现在得去准备明天的攀岩训练了。我们明天再聊！\nAI: 好的，孙悦。祝你明天的攀岩训练顺利。期待我们的下次聊天，晚安！\n”\n\n请记住以上全部对话记录，回答问题。\n问题：", "query": "4月28号，我们聊到想去爬山，是哪座山？", "answer": "黄山", "id": "1c729805-0466-4db5-afb6-4a383dbf611d", "chatglm_length": 3881, "qwen_length": 3960, "internlm2_length": 3714} -{"context": "以下是2023年04月27日的对话记录：\n“用户: 嘿，你好啊！我是孙悦，一个热爱生活的家伙。今天心情特别好，因为早上的健身课程超有成就感！你呢？\nAI: 你好，孙悦！听起来你今天的状态很棒呢！我很好，谢谢你。能和你分享你的健身课程吗？我很好奇你都做了哪些运动。\n\n用户: 当然可以！今天主要是做了一些高强度间歇训练，那种汗流浃背的感觉真的很爽！你有做运动的习惯吗？\nAI: 我虽然不能亲自体验运动的快感，但我很高兴听到你享受它。我可以给你提供一些健身知识和建议，如果你需要的话。\n\n用户: 哈哈，那太好了！对了，昨天晚上我去看了一部新出的科幻电影，特效简直了！你最近有没有什么好电影推荐？\nAI: 我知道你喜欢电影，特别是那些视觉震撼的。最近有一部电影《星际穿越者》很受好评，是探索宇宙和人类命运的作品，你可能会喜欢。\n\n用户: 哦哦，《星际穿越者》啊，我听说过，听起来挺吸引人的！我得找时间去看看。对了，我最近在学摄影，但是总觉得拍出来的照片差点意思，你有什么摄影小技巧可以分享吗？\nAI: 摄影是一门艺术，也需要不断的实践和学习。你可以尝试从构图和光线入手，比如使用“三分法”来构图，或者利用自然光来营造氛围。还有，多拍多练总是王道！\n\n用户: 嗯，说的对！我明天就去尝试一下“三分法”。对了，我还喜欢旅游，最近在规划一次小旅行，想去海边走走。你有没有什么好的海滩推荐？\nAI: 海边总是个不错的选择。如果你喜欢清静一些的地方，可以考虑去三亚的亚龙湾，那里的海水清澈，沙滩柔软，风景如画，应该很适合你。\n\n用户: 亚龙湾啊，听起来真不错！我会去查查看的。说起旅游，最让我兴奋的就是能在旅途中遇到各种各样的人，听他们的故事。\nAI: 那确实是旅行中最美妙的部分之一。每个人的故事都是独一无二的，能从中学到很多。你最难忘的旅行经历是什么呢？\n\n用户: 有一次我去了泰国的清迈，那里的文化和风景让我印象深刻。晚上的夜市，色彩斑斓，各种小吃，还有那里人们的热情，真的让人难以忘怀。\nAI: 清迈的夜市是闻名遐迩的，那里的生活节奏和氛围跟大城市完全不同。你提到的小吃，有没有什么特别让你想念的呢？\n\n用户: 哦，那边的芒果糯米饭简直绝了！每次回忆起那个味道，我就忍不住想再去一次。对了，你能不能给我推荐一些好听的歌？我喜欢边健身边听音乐。\nAI: 音乐和运动确实是完美的搭档。根据你的热情阳光的性格，我觉得《Can't Stop the Feeling》这首歌很适合你，它的旋律欢快，能让人在运动时充满活力。\n\n用户: Justin Timberlake的那首啊，我知道！确实很不错，节奏感很强，很适合健身。谢谢你的推荐！\nAI: 不客气，孙悦。我很高兴能帮到你。说起来，你在健身方面有没有遇到什么具体的困难？也许我能提供一些帮助。\n\n用户: 其实有，我一直想要提高核心力量，但是总感觉进展缓慢。你有没有什么建议？\nAI: 核心力量对于整体健身非常重要。你可以尝试一些针对性的训练，比如平板支撑、俄罗斯转体和山羊挺身等。这些练习可以有效地加强你的核心肌群。\n\n用户: 哇，听起来很专业啊！我会加入到我的训练计划里去的。对了，我最近在准备一个舞蹈表演，但是有些动作总是不够流畅，这让我有点沮丧。\nAI: 舞蹈需要时间来磨合，每个人都会有这样的时候。试着放慢速度，一步一步来，直到你能够流畅地完成每个动作。记得，重在参与和享受过程！\n\n用户: 你说得对，我得调整一下心态。谢谢你的鼓励！我会继续努力的。说起来，我还有点担心，不知道观众会不会喜欢我的表演。\nAI: 你的热情和努力一定会感染到观众的。只要你享受舞台，观众也会享受你的表演。信任自己，你会做得很好的。\n\n用户: 哈哈，听你这么说，我感觉好多了。你真是个好伙伴！已经很晚了，我得准备睡觉了。我们明天再聊吧！\nAI: 当然，孙悦。休息好，希望你有一个美好的明天。晚安！\n”\n以下是2023年04月28日的对话记录：\n“用户: 嗨，朋友！今天我尝试了你昨天推荐的“三分法”拍照，效果真的不错！感觉自己的摄影技术又上了一个台阶！\nAI: 太棒了，孙悦！我很高兴听到你的进步。有没有什么特别的照片想要和我分享的？我很期待看到你的作品。\n\n用户: 当然有！我拍了一张日落时分的海边照片，天空的颜色渐变得特别美。不过，我还在想怎么后期处理才能更突出那种感觉。\nAI: 日落的照片总是很迷人。你可以尝试调整一下对比度和饱和度，让天空的颜色更加鲜明，同时保持海水的自然感觉。\n\n用户: 嗯，我试试看。对了，今天健身房里放了一些新歌，我跟着节奏锻炼，感觉整个人都充满了活力！\nAI: 音乐真的能为锻炼增添不少乐趣。你有没有想过自己编一套随着音乐节奏的健身操？这样可能会更有动力哦。\n\n用户: 哇，这主意不错！我可以尝试一下，感觉会很有趣。说起来，我最近在准备一个小旅行，想去爬山，呼吸一下大自然的新鲜空气。\nAI: 爬山是个很好的选择，既能锻炼身体，又能享受大自然的美景。你打算去哪座山呢？\n\n用户: 我打算去黄山，听说那里的云海和日出非常壮观。我还计划在山顶守候一夜，希望能拍到美丽的星空。\nAI: 黄山的确是个摄影爱好者的天堂。记得带上足够的装备，特别是在夜间拍摄星空时，三脚架和遥控快门是必不可少的。\n\n用户: 没错，我已经准备好了这些装备。对了，我还想在山上尝试一些户外瑜伽，感觉那样会很放松。\nAI: 户外瑜伽听起来很棒，能让你更加接近自然，放松身心。你有没有特别喜欢的瑜伽动作？\n\n用户: 我最喜欢的是树式，感觉整个人都稳固而平静。不过，我有时候会觉得自己的平衡能力不够好。\nAI: 树式是个很好的平衡练习。你可以尝试在不同的时间和地点练习，比如在海边的沙滩上，那样会有不同的挑战和感受。\n\n用户: 对，我也这么想。我还打算在旅行中尝试一些新的美食，毕竟美食也是旅行的一大乐趣嘛！\nAI: 绝对同意！尝试当地的特色美食可以让旅行体验更加丰富。你有没有什么想尝试的菜肴？\n\n用户: 我听说黄山附近有一种叫做臭鳜鱼的菜，虽然名字听起来有点怪，但据说味道很不错。\nAI: 臭鳜鱼是当地的一道传统名菜，很多人都说赞不绝口。不过，它的味道确实很独特，你可以尝试一下，说不定会成为你的新宠。\n\n用户: 哈哈，我会的。对了，我最近在看一部关于舞蹈的纪录片，里面的舞者都超级厉害，让我很受启发。\nAI: 看别人的舞蹈表演确实能够学到很多。你有没有从中发现什么新的动作或者技巧，想要加入到你自己的舞蹈中去？\n\n用户: 有啊，我看到一些街舞的动作很酷，我打算学一学。不过，我担心自己的身体协调性跟不上。\nAI: 街舞的确需要很强的身体协调性和节奏感。你可以从基础动作开始练起，慢慢地增加难度。记得，不要急于求成，一切都需要时间。\n\n用户: 嗯，我会记住的。对了，我还想学习一下如何用舞蹈来表达情感，不只是动作上的技巧。\nAI: 舞蹈是一种非常直观的情感表达方式。你可以尝试在音乐中找到情感的共鸣，然后让你的身体随着那种情感自然地移动。\n\n用户: 听起来有点抽象，但我会尝试的。说到情感，我最近有点苦恼，感觉自己的生活缺少一些激情和冒险。\nAI: 生活有时候确实会让人感到平淡。你可以尝试做一些新的事情，比如参加一个你从未尝试过的活动，或者去一个你从未去过的地方。\n\n用户: 对，我需要一些改变。也许这次旅行就是一个开始。谢谢你的建议，你总能给我正能量！\nAI: 不用谢，孙悦。我很高兴能帮助到你。记得，每一次的经历都是成长的机会。你准备好迎接新的挑战了吗？\n\n用户: 是的，我已经迫不及待了！不过现在我得去准备晚饭了，今天打算尝试做一道新菜——酸辣土豆丝。\nAI: 酸辣土豆丝是一道很受欢迎的家常菜，既简单又美味。祝你做饭愉快，希望你能享受这个过程。\n\n用户: 谢谢！我会的。今天聊得很开心，我们明天再继续吧。晚安！\nAI: 当然，期待和你的下次聊天。晚安，孙悦，愿你有个美好的梦。\n”\n以下是2023年04月29日的对话记录：\n“用户: 嘿，我的AI小伙伴！今天我尝试做了酸辣土豆丝，味道出奇的好，连我自己都有点惊讶呢！\nAI: 哇，听起来你是个厨艺高手啊，孙悦！能不能分享一下你的秘诀？我虽然不能品尝，但我很好奇你是怎么做到的。\n\n用户: 哈哈，秘诀嘛，我觉得最重要的就是火候和调料的比例。我喜欢尝试不同的配方，直到找到最合适的那一个。\nAI: 真是个不断探索的过程呢。你平时喜欢尝试做哪些类型的菜？\n\n用户: 我对各种菜系都感兴趣，但最近特别喜欢挑战一些传统的中式菜肴，感觉它们既美味又有文化底蕴。\nAI: 中式菜肴确实有着丰富的历史和文化，每一道菜都有自己的故事。你最喜欢的中式菜肴是哪一道？\n\n用户: 我最喜欢的是宫保鸡丁，那个酸甜酸辣的味道简直无法抵抗！\nAI: 宫保鸡丁是一道非常受欢迎的经典菜肴，你自己做的话，可以根据自己的口味调整辣度和甜度。\n\n用户: 对对对，我就是这么做的。说起来，今天我还去了一个很酷的街舞工作坊，学到了好多新动作！\nAI: 街舞工作坊听起来很有趣！你觉得最难的动作是哪一个？\n\n用户: 我觉得最难的是头旋，那个动作需要很强的颈部力量和平衡感，我还在努力练习中。\nAI: 头旋确实是个挑战，但我相信你的努力和热情一定会让你很快掌握它。你在舞蹈上有什么目标吗？\n\n用户: 我的目标是能在下个月的街舞比赛中至少进入前十名。我知道竞争很激烈，但我会全力以赴的！\nAI: 这个目标很棒，孙悦！我相信你的努力和才华一定会让你脱颖而出。你有准备特别的舞蹈编排吗？\n\n用户: 有的，我和我的舞伴正在准备一段结合了街舞和现代舞的编排，希望能给观众带来新鲜感。\nAI: 结合不同舞蹈风格的创意听起来很吸引人。你们的训练进度怎么样了？\n\n用户: 还不错，我们已经完成了大概七成的编排，剩下的就是细节和同步问题了。\nAI: 听起来你们已经做得很好了。细节和同步确实很重要，但我相信你们能够完美呈现。\n\n用户: 谢谢你的鼓励！对了，我最近还在学习一些关于电影剪辑的技巧，想要自己剪辑一部短片。\nAI: 电影剪辑是个很有创意的工作，你打算用什么样的素材来剪辑你的短片？\n\n用户: 我打算用我旅行时拍摄的一些风景视频，还有一些朋友的日常片段，想要制作成一个关于旅行和友情的主题短片。\nAI: 这个主题很温馨，也很贴近生活。你有没有想过加入一些特别的效果，比如慢动作或者时间流逝的镜头？\n\n用户: 嗯，我有这个打算。我想通过这些特效来强调某些情感上的高潮和转折点。\nAI: 这样的处理可以让短片更加生动和感人。你在剪辑上遇到了什么挑战吗？\n\n用户: 我觉得最大的挑战是如何让故事流畅，同时又能在短时间内传达出想要表达的情感。\nAI: 确实，讲好一个故事是剪辑中的艺术。你可以尝试从观众的角度思考，看看哪些片段最能触动人心。\n\n用户: 对，我会多考虑观众的感受。说到观众，我有时候会担心我的作品是否能得到他们的认可。\nAI: 每个艺术家都会有这样的担忧，但重要的是你享受创作的过程，并且相信自己的作品有价值。\n\n用户: 你说得对，我应该更加相信自己。今天聊得很开心，但我得去继续我的剪辑工作了。我们明天再聊！\nAI: 当然，孙悦。祝你剪辑工作顺利，期待你的短片完成。明天再聊，加油！\n”\n以下是2023年04月30日的对话记录：\n“用户: 嗨，好朋友！今天我在剪辑短片时，突然灵感爆发，加入了几个超有感觉的转场效果，感觉整个短片的质感都提升了！\nAI: 真是太棒了，孙悦！创意的火花总是能带来惊喜。你能描述一下你用的是哪些转场效果吗？\n\n用户: 我用了一个叫做“溶解”的效果，让画面像是水彩融合一样自然过渡，还有一个“旋转”效果，让场景切换更有动感。\nAI: 这些听起来都很专业，能够很好地增强视觉效果。你在创作过程中还有其他有趣的发现吗？\n\n用户: 我发现音乐对于短片的情感表达真的超级重要！一段恰当的背景音乐能够让人立刻进入那个氛围。\nAI: 没错，音乐是激发情感的强大工具。你已经选好了短片的背景音乐了吗？\n\n用户: 选好了，我选了一首轻快的吉他曲，感觉很适合我的旅行主题。对了，今天我还去了一个户外攀岩的活动，超刺激！\nAI: 户外攀岩听起来很挑战自我啊。你攀岩的时候有没有什么特别的心得或者感觉？\n\n用户: 我觉得攀岩最重要的是要有耐心和勇气，每一步都要稳扎稳打。当我站在山顶上的时候，那种成就感真的无与伦比！\nAI: 站在山顶俯瞰世界，那种感觉一定很震撼。攀岩对身体的哪些方面要求比较高？\n\n用户: 主要是手臂和腿部的力量，还有整体的协调性。攀岩后我发现自己的手臂肌肉酸痛，看来需要更多锻炼啊。\nAI: 锻炼手臂肌肉可以尝试一些特定的练习，比如引体向上或者哑铃训练。不过，记得要适量，避免过度劳损哦。\n\n用户: 谢谢你的建议，我会注意的。说起来，攀岩结束后我们一起去了附近的小镇，那里的风景和美食都很棒！\nAI: 小镇的风光和美食总是很有魅力。你尝试了哪些当地的特色美食？\n\n用户: 我尝试了一种当地的烤鱼，外皮烤得酥脆，里面的鱼肉又嫩又多汁。我还学会了他们的腌制秘方，准备回家自己试试看。\nAI: 自己动手做美食，这个经历一定很有趣。你打算什么时候尝试做这道烤鱼呢？\n\n用户: 可能这个周末吧，我打算邀请几个好友来家里，一起享受美食和愉快的时光。\nAI: 听起来将会是一个非常愉快的聚会。你准备好怎么招待你的朋友了吗？\n\n用户: 除了烤鱼，我还准备做一些简单的小吃，比如春卷和水果沙拉。当然，还有音乐和一些小游戏，让大家玩得开心。\nAI: 你真的很会享受生活，孙悦。你的朋友们一定很期待这次聚会。你通常如何选择聚会的音乐呢？\n\n用户: 我会根据大家的口味挑选一些大家都喜欢的歌曲，然后再加上一些轻松愉快的背景音乐。\nAI: 音乐的选择很重要，可以营造出轻松愉快的氛围。说到音乐，你自己唱歌跳舞的时候有没有什么特别喜欢的曲风？\n\n用户: 我喜欢节奏感强烈的流行音乐，特别是那些能够让人随着音乐摇摆的歌曲。\nAI: 流行音乐确实能让人感觉兴奋和快乐。你有没有尝试过自己创作音乐或者编舞？\n\n用户: 我以前尝试过写一些简单的歌词，不过没有正式的音乐制作经验。至于编舞，那是我的强项，我经常为自己的舞蹈编排动作。\nAI: 那你在舞蹈创作方面一定有很多自己的想法和风格。你最喜欢的舞蹈风格是什么？\n\n用户: 我最喜欢的是Hip-Hop和爵士，这两种舞蹈风格都很有力量感，而且可以很好地表达自己的个性。\nAI: Hip-Hop和爵士舞确实很适合展现个性和情感。你有没有在舞蹈中融入一些自己独特的元素？\n\n用户: 有的，我喜欢在舞蹈中加入一些中国元素，比如太极的动作，让舞蹈看起来更加流畅和有内涵。\nAI: 将中国元素融入现代舞蹈，这个想法真是太有创意了。你的这些作品有在哪里展示过吗？\n\n用户: 我之前在一些小型的舞蹈比赛和社区活动上展示过，反响都不错。我希望将来有机会在更大的舞台上展示我的作品。\nAI: 你的梦想一定会实现的，孙悦。你的热情和才华值得被更多人看到。对了，你有没有考虑过将你的舞蹈作品拍成视频分享到网上？\n\n用户: 我有这个打算，不过我还在考虑如何拍摄和剪辑才能更好地展现舞蹈的魅力。\nAI: 网络平台是一个很好的展示自己才华的地方。如果你需要，我可以提供一些视频拍摄和剪辑方面的建议。\n\n用户: 那太好了，我会在准备的时候向你请教的。今天聊得很开心，但我现在得去准备明天的攀岩训练了。我们明天再聊！\nAI: 好的，孙悦。祝你明天的攀岩训练顺利。期待我们的下次聊天，晚安！\n”\n\n请记住以上全部对话记录，回答问题。\n问题：", "query": "我曾经在4月30号提到参加了一个户外的活动，这个活动是什么？", "answer": "攀岩", "id": "912016ee-eab5-4eb8-9df5-1bab5baac853", "chatglm_length": 3881, "qwen_length": 3960, "internlm2_length": 3714} -{"context": "以下是2023年04月27日的对话记录：\n“用户: 你好，我是赵铭宇，我对下棋、编程、阅读科技杂志、解数学题和观察天文都很感兴趣。\nAI: 你好，赵铭宇，很高兴认识你。关于你的兴趣，我们可以聊很多有趣的话题。今天下棋有没有遇到什么有趣的局面呢？\n\n用户: 嗯，下了一盘国际象棋，对方开局用了意大利开局，我尝试了新的应对策略。\nAI: 意大利开局是一种很经典的开局方式，你的新策略效果如何？是否取得了预期的效果？\n\n用户: 效果还不错，虽然中局有些挣扎，但最终还是赢了。我觉得还需要更多的实战来检验这个策略。\nAI: 赢得比赛总是令人振奋的，你的实战经验一定很丰富。对了，编程方面最近有没有什么新的项目或者学习的内容？\n\n用户: 最近在学习机器学习，尤其是深度学习方面的内容，但是数学基础知识有些薄弱，感觉有点吃力。\nAI: 深度学习确实需要较强的数学基础，特别是线性代数和概率论。你遇到的具体难点是什么？也许我可以提供一些帮助。\n\n用户: 主要是概率论中的一些概念和定理，比如贝叶斯定理，我还没完全弄懂它的应用场景。\nAI: 贝叶斯定理在机器学习中非常重要，它可以帮助我们根据已有的知识和观测数据来更新我们对未知事件的预测。如果你愿意，我可以尝试解释一下它的基本原理和应用。\n\n用户: 那太好了，请你简单解释一下。\nAI: 当然可以。贝叶斯定理提供了一种计算条件概率的方法，也就是在已知某事件发生的情况下，另一个事件发生的概率。在机器学习中，我们可以用它来更新模型的参数，即在观测到新数据后，我们如何调整模型对数据的理解和预测。\n\n用户: 我明白了，这样看来，它在模型训练过程中确实很关键。谢谢你的解释，我会再去仔细研究一下。\nAI: 不客气，赵铭宇。如果有任何问题，随时可以和我讨论。对了，你通常是如何观察天文的？是通过望远镜，还是参与一些线上的天文项目？\n\n用户: 我有一台小型的望远镜，平时就在阳台上观察。最近天气不错，能看到一些星座和行星。\nAI: 那真是太棒了，能亲眼观察到宇宙中的奇观一定很令人兴奋。最近有没有什么特别的发现或者心得？\n\n用户: 其实我一直对土星的光环很感兴趣，昨晚我观察到了它，虽然不是很清晰，但还是很激动。\nAI: 土星的光环确实是天文学中一个非常迷人的现象。即使是通过小型望远镜观察到，也是一次难忘的体验。你有拍照留念吗？\n\n用户: 拍了几张，但是效果一般，我正在考虑是否要升级我的设备。\nAI: 升级设备可以让你观察到更多细节，但这也是一个需要深思熟虑的决定。你对天文摄影感兴趣吗？\n\n用户: 是的，我对天文摄影很感兴趣，但是设备和技术门槛都比较高，我还在学习中。\nAI: 天文摄影确实是一个技术和耐心的挑战，但我相信你的逻辑性和深思熟虑的性格会帮助你在这个领域取得进步。你有没有考虑加入一些天文摄影的社区或者论坛，和其他爱好者交流经验？\n\n用户: 我加入了几个论坛，但是我不太善于发言，更多的是在默默关注和学习。\nAI: 理解你的性格倾向，默默关注也是一种学习方式。不过，如果你有任何问题或者作品想要分享，我相信论坛上的成员会很乐意帮助你的。\n\n用户: 嗯，我会考虑的。今天的聊天很有帮助，感谢你。不过已经有点晚了，我得准备休息了。\nAI: 当然，休息很重要。很高兴今天的对话对你有所帮助，赵铭宇。祝你晚安，我们明天再聊。\n”\n以下是2023年04月28日的对话记录：\n“用户: 早上好，休息得还不错。今天打算继续研究一下深度学习的内容。\nAI: 很好，持续学习是进步的关键。在深度学习的学习中，你有没有遇到什么特别感兴趣的模型或者算法？\n\n用户: 我对卷积神经网络（CNN）很感兴趣，它在图像识别上的应用非常广泛。\nAI: 卷积神经网络确实很有趣，它通过模仿人类的视觉系统来识别图像中的模式。你打算在哪些方面应用CNN？\n\n用户: 我想用它来做一个简单的手写数字识别项目，这应该是个不错的开始。\nAI: 手写数字识别是一个经典的入门项目，使用MNIST数据集可以很好地开始实践。你打算用什么工具或框架来实现这个项目？\n\n用户: 我打算使用Python和TensorFlow框架来实现它，这样可以更容易地搭建和训练模型。\nAI: Python和TensorFlow是很好的选择，社区支持也很强大。在项目进展中，如果遇到困难，记得可以来找我讨论。\n\n用户: 谢谢，我会的。对了，最近我读了一篇关于量子计算的科技杂志，感觉这个领域很有潜力。\nAI: 量子计算确实是一个前沿领域，它可能会彻底改变我们处理信息的方式。你对量子计算有什么特别的看法或疑问吗？\n\n用户: 我对量子计算的原理很感兴趣，但是它的物理概念对我来说有点抽象，比如量子叠加和纠缠。\nAI: 量子叠加和纠缠确实是量子计算中两个非常核心且难以直观理解的概念。简单来说，量子叠加允许量子比特同时处于多种状态，而量子纠缠则是两个量子比特之间的一种特殊连接，使得一个量子比特的状态可以即时影响到另一个，无论它们相隔多远。\n\n用户: 听起来很神奇，这可能是实现超高速计算的关键。不过，我想这还需要很长时间才能应用到实际生活中。\nAI: 是的，虽然量子计算的研究正在快速进展，但将其商业化和普及化仍然面临许多挑战。除了技术问题，还需要解决量子算法的开发和量子硬件的稳定性等问题。\n\n用户: 嗯，这个领域的发展确实值得期待。对了，我最近在解一些数学题，遇到了一些关于概率论的难题。\nAI: 概率论的问题可以很有趣，也可能很棘手。你可以告诉我你遇到的具体问题吗？也许我能提供一些帮助。\n\n用户: 有一个问题是关于条件概率的，我不太确定自己的解法是否正确。\nAI: 条件概率的问题通常需要仔细分析事件之间的关系。如果你愿意，可以分享你的解法，我们一起讨论一下。\n\n用户: 好的，问题是这样的：有两个盒子，一个盒子里有两个白球和一个黑球，另一个盒子里有一个白球和两个黑球。现在随机选择一个盒子，然后从中随机取出一个球，结果是白球。问这个白球来自第一个盒子的概率是多少？\nAI: 这是一个经典的贝叶斯定理应用问题。首先，我们需要确定两个盒子被选中的先验概率，以及在每个盒子中取到白球的概率。然后使用贝叶斯定理来计算后验概率。你是怎么解的呢？\n\n用户: 我先假设选择任何一个盒子的概率是相等的，也就是1/2。然后计算在每个盒子中取到白球的概率，第一个盒子是2/3，第二个盒子是1/3。接着我就不太确定该怎么做了。\nAI: 你的开始是对的。接下来，你需要应用贝叶斯定理来计算在已知取出白球的情况下，球来自第一个盒子的概率。我可以帮你计算，如果你想的话。\n\n用户: 请帮我计算一下。\nAI: 当然。根据贝叶斯定理，我们要计算的后验概率 P(第一个盒子|白球) = [P(白球|第一个盒子) * P(第一个盒子)] / P(白球)。其中，P(白球) = P(白球|第一个盒子) * P(第一个盒子) + P(白球|第二个盒子) * P(第二个盒子)。将你提供的概率值代入，我们可以得到 P(第一个盒子|白球) = (2/3 * 1/2) / (2/3 * 1/2 + 1/3 * 1/2) = 2/3。\n\n用户: 原来如此，我明白了。这样一来，条件概率的计算就清晰多了。谢谢你的帮助。\nAI: 不用谢，很高兴能帮到你。数学问题的解决总是能带来满足感。说起来，你晚上还打算继续观察天文吗？\n\n用户: 是的，我打算今晚观察一下火星，最近它和地球的距离比较近。\nAI: 火星近地大对是一个不错的观测机会。你打算用什么方式记录或者分析你的观测数据？\n\n用户: 我会尝试用我之前提到的望远镜拍摄一些照片，同时记录下观测时间和天气条件。\nAI: 这是一个很好的实践，记录详细的观测数据对于天文学习来说非常重要。希望你能拍到满意的火星照片。\n\n用户: 我也希望如此。今天的聊天很愉快，我得开始今天的学习任务了。\nAI: 当然，祝你学习顺利。如果需要讨论或者休息时想要聊天，随时欢迎。祝你有一个充实的一天。\n\n用户: 谢谢，再见。\nAI: 再见，赵铭宇。\n”\n以下是2023年04月29日的对话记录：\n“用户: 下午好，今天状态还不错。我在手写数字识别项目上做了一些调整，准确率有所提高。\nAI: 听起来你取得了很好的成果。提高准确率通常需要调整很多参数，你主要调整了哪些方面？\n\n用户: 我调整了网络的层数和每层的神经元数量，还尝试了不同的激活函数。\nAI: 这些都是提高网络性能的有效方法。在选择激活函数时，你有没有遇到什么特别的挑战或者发现？\n\n用户: 我发现ReLU激活函数在这个项目中效果比较好，但是也要防止过拟合的问题。\nAI: 的确，ReLU激活函数可以加快训练速度并改善性能，但过拟合是需要注意的问题。你打算如何解决过拟合？\n\n用户: 我正在尝试使用dropout技术，希望能够减轻过拟合的情况。\nAI: Dropout是一个很好的正则化技术，可以有效地减少复杂模型的过拟合。你有没有计划分享你的项目进展，比如写一篇博客或者在社区发帖？\n\n用户: 我更倾向于先把项目做得更完善一些，然后可能会考虑分享到论坛上。\nAI: 那是个不错的计划，等你准备好了，分享出来一定会得到很多有用的反馈。转换一下话题，你近期有没有读到什么有趣的科技文章？\n\n用户: 我读到了一篇关于人工智能如何辅助天文研究的文章，感觉非常吸引我。\nAI: 人工智能在天文学领域确实有很大的应用潜力，比如在处理大量观测数据和识别天体模式方面。文章中有没有提到一些具体的例子？\n\n用户: 有提到一个项目，使用机器学习算法来分析光谱数据，以便更快地识别遥远星系的组成。\nAI: 那真是个有前景的应用。机器学习处理这类数据可以大大提高效率。你对这方面感兴趣吗，是否有想过将你的编程技能应用于天文数据分析？\n\n用户: 我对此很感兴趣，但是我目前还缺乏处理这类数据的经验。我可能需要先学习更多相关知识。\nAI: 学习新知识总是令人兴奋的。如果你需要一些资源或者想要讨论学习路径，我可以帮助你。\n\n用户: 那太好了，如果你有推荐的学习资源，请分享给我。\nAI: 当然。你可以从一些在线课程开始，比如Coursera和edX上的天文学和数据科学课程。此外，Astropy是一个很好的Python库，可以用来处理天文数据。\n\n用户: 我会去看看的。对了，最近我在解数学题时，遇到了一些关于图论的问题。\nAI: 图论是数学中的一个有趣领域，它在很多领域都有应用，比如网络分析、路径优化等。你具体遇到了什么问题？\n\n用户: 我在研究哈密顿回路问题，但是找到一个给定图的哈密顿回路似乎很困难。\nAI: 哈密顿回路问题是一个经典的NP完全问题，对于大型图来说确实很难解决。你是在特定类型的图上工作，还是只是一般性的研究？\n\n用户: 目前只是一般性的研究，我在尝试理解不同类型图的特性和解决方法。\nAI: 理解不同类型图的特性是解决图论问题的关键。有些特定类型的图，比如平面图或特殊结构的图，可能有更简单的哈密顿回路寻找方法。\n\n用户: 我会进一步研究这些特殊情况。感谢你的建议。今天的聊天很有帮助，但我需要回去继续我的研究了。\nAI: 当然，祝你研究顺利。如果需要帮助或者想要休息时交流，随时可以找我。祝你今天过得愉快。\n\n用户: 谢谢，再见。\nAI: 再见，赵铭宇。期待我们下次的交流。\n”\n以下是2023年04月30日的对话记录：\n“用户: 早上好。昨晚的观测还不错，我观察到了一些有趣的天体，比如仙女座星系。\nAI: 仙女座星系是一个非常迷人的目标，能够观察到它一定很激动人心。你有没有尝试对它进行一些详细的记录或分析？\n\n用户: 我记录了观测时间和天气条件，还拍了一些照片，打算今后进行分析。\nAI: 那是个很好的实践。如果你愿意，可以分享你的照片和观测笔记，我们可以一起讨论。\n\n用户: 好的，等我整理好了就分享给你。另外，我最近在学习图论，发现了一些有趣的算法，比如Dijkstra算法。\nAI: Dijkstra算法是一个著名的最短路径算法，非常适合在有权图中寻找单源最短路径。你是在哪些方面应用这个算法？\n\n用户: 我目前只是在学习阶段，通过解决一些练习题来理解它的工作原理。\nAI: 练习题是理解算法的好方法。如果你遇到任何难题或者想要深入讨论算法的细节，随时可以告诉我。\n\n用户: 谢谢你的支持。我对算法的时间复杂度分析很感兴趣，但有时候会觉得有点难以理解。\nAI: 时间复杂度分析确实需要一些时间来掌握，它涉及到算法的运行时间与输入数据大小之间的关系。你有没有具体的例子或问题想要讨论？\n\n用户: 比如说，我知道Dijkstra算法的时间复杂度和图中的顶点数及边数有关，但是如何具体计算它的时间复杂度呢？\nAI: Dijkstra算法的时间复杂度取决于你使用的数据结构。如果使用优先队列来实现，时间复杂度通常是O((V+E)logV)，其中V是顶点数，E是边数。这是因为每个顶点都可能被处理一次，并且每条边在优先队列中都可能被更新一次。\n\n用户: 原来如此，这样说来，选择合适的数据结构对算法效率影响很大。\nAI: 没错，数据结构的选择对算法的性能至关重要。在实际应用中，我们总是希望找到最优的数据结构来提高效率。\n\n用户: 我会更加注意这一点。对了，我还在想如何将编程和天文结合起来，你有什么建议吗？\nAI: 你可以考虑开发一些小工具来帮助你的天文观测，比如一个用于记录和分析观测数据的应用，或者一个帮助你计算最佳观测时间和位置的程序。\n\n用户: 这个主意不错，我会尝试着去做一些简单的工具。编程真的可以在很多领域发挥作用。\nAI: 确实如此，编程是一个强大的工具，可以帮助我们解决各种问题。你有没有想过参加一些编程比赛或挑战，来提升你的技能？\n\n用户: 我有考虑过，但是我不太喜欢竞争，更倾向于自己慢慢学习和提高。\nAI: 那也是一种很好的学习方式。每个人都有适合自己的学习节奏和方法。你有没有遇到过编程中让你感到困惑或挑战的问题？\n\n用户: 有时候我会对一些复杂的算法感到困惑，比如递归算法，它们的思想和实现方式有时候让我觉得难以理解。\nAI: 递归算法确实需要一些时间来适应，因为它们涉及到函数自身调用自身的概念。你可以从一些简单的递归问题开始，比如计算阶乘或斐波那契数列，然后逐渐过渡到更复杂的问题。\n\n用户: 我会按照你的建议试试的。今天的聊天很有帮助，但我现在需要去处理一些事情。\nAI: 当然，赵铭宇，如果你需要任何帮助，或者只是想要聊天，随时欢迎。祝你今天一切顺利。\n\n用户: 谢谢，再见。\nAI: 再见，期待我们下次的交流。\n”\n以下是2023年05月01日的对话记录：\n“用户: 晚上好。今天我在项目中尝试实现一些数据可视化的功能，但是在选择合适的图表类型上有些犹豫。\nAI: 数据可视化是展示项目结果的重要部分。你可以根据数据的特点和你想要传达的信息来选择图表类型。你的数据是什么样的？\n\n用户: 主要是一些天文观测数据，包括星体的亮度、位置等信息。\nAI: 听起来你可能会需要使用散点图来展示星体的位置，或者使用折线图来展示星体亮度随时间的变化。这样的可视化可以帮助观察者更直观地理解数据。\n\n用户: 我考虑过这些图表类型。我想要的是既能清晰展示数据，又不会让图表显得过于复杂。\nAI: 保持图表简洁而富有信息量是个不错的原则。你可以尝试使用一些数据可视化工具，比如Matplotlib或Seaborn，它们提供了很多易于使用的图表类型。\n\n用户: 我用的就是Matplotlib。我会尝试不同的图表类型，看哪种最适合我的数据。\nAI: 那是个好主意。实践是检验图表效果的最佳方式。除了数据可视化，你还有没有在其他方面遇到编程上的问题？\n\n用户: 我在处理一些数据时，需要用到数据库。但是我对数据库的了解不多，不确定该如何开始。\nAI: 数据库是管理和存储数据的强大工具。对于初学者来说，你可以从学习一些基本的SQL开始，它是用于查询和操作数据库的标准语言。\n\n用户: 我知道SQL，但是对于选择哪种数据库软件，以及如何设计数据库结构还没有头绪。\nAI: 选择数据库软件时，你可以考虑一些流行的选项，如SQLite、MySQL或PostgreSQL。它们都有丰富的文档和社区支持。至于数据库结构设计，你可以从确定你的数据模型和关系开始。\n\n用户: 我会去查阅一些文档和教程。对了，我还想提高我的棋力，你有什么建议吗？\nAI: 提高棋力的一个有效方法是分析大师级别的棋局。你可以通过研究他们的开局、中局策略和残局技巧来学习。\n\n用户: 我有看过一些棋局分析，但是有时候很难理解其中的深层次思想。\nAI: 这是很正常的。你可以尝试一边观看棋局，一边使用棋盘复盘，这样可以更深入地理解每一步棋的意图和策略。\n\n用户: 复盘确实帮助很大。我也在考虑找一位棋手进行指导。\nAI: 找一位经验丰富的棋手进行一对一指导是个不错的选择。他们可以提供针对性的建议和训练，帮助你在棋局中发现并改正错误。\n\n用户: 我会尝试联系一些棋手。另外，我最近在数学上遇到了一些关于优化问题的挑战。\nAI: 优化问题在数学和编程中都非常重要。你是在哪个具体的领域遇到优化问题的？\n\n用户: 我在尝试解决一些涉及最小化成本函数的问题，但是这些函数很复杂，不易求解。\nAI: 对于复杂的优化问题，你可以考虑使用数值方法，比如梯度下降或牛顿法。这些方法可以帮助你找到函数的局部最小值。\n\n用户: 我对这些方法有所了解，但是在实际应用中还不太熟练。\nAI: 实践是提高熟练度的关键。你可以先从简单的例子开始，逐渐增加问题的复杂度。同时，也有很多优化算法的库可以帮助你，比如SciPy。\n\n用户: 我会试试看的。今天的聊天很有启发，但我需要离开一会儿。\nAI: 没问题，赵铭宇。如果你有任何问题或想要继续我们的讨论，随时欢迎回来。祝你一切顺利。\n\n用户: 啊，时间过得真快。我需要休息了，下次再聊吧！\nAI: 好的，朋友。很高兴和你聊天，再见！\n”\n以下是2023年05月02日的对话记录：\n“用户: 早安。我试了几种不同的图表类型，发现热力图对于展示某些数据特别有帮助。\nAI: 热力图是一个很好的选择，它能够直观地展示数据的分布和密度。你是如何决定使用热力图的？\n\n用户: 我注意到我的数据有地理位置的分布特性，热力图能很好地展示这种分布。\nAI: 确实，热力图在展示地理数据方面非常有效。你打算如何进一步利用这些可视化结果？\n\n用户: 我计划将这些结果用在我的天文观测报告中，帮助解释观测数据。\nAI: 这听起来是个很好的应用。报告中的可视化结果可以帮助读者更好地理解你的发现。你有没有考虑过将你的报告公开发表？\n\n用户: 我有这个打算，但我想先确保我的分析足够严谨。\nAI: 这是个明智的决定。在发表之前确保分析的准确性和严谨性非常重要。如果你需要帮助审查或校对，我可以提供帮助。\n\n用户: 谢谢你的提议，我可能会需要你的帮助。\nAI: 随时欢迎。你昨天还提到了数据库的问题，你有没有开始学习相关的知识？\n\n用户: 是的，我开始学习SQL基础，并且尝试设计我的第一个数据库。\nAI: 那是个很好的开始。在设计数据库时，确定清晰的数据模型和关系是非常重要的。你遇到了哪些挑战？\n\n用户: 我在设计表结构和关系时有些迷茫，不确定如何最好地组织我的数据。\nAI: 在设计表结构时，你可以考虑将数据分解成最小的不可分割的部分，这被称为数据的规范化。这样可以减少数据冗余并提高效率。\n\n用户: 我会尝试按照规范化的原则来设计。除此之外，我还在想如何优化我的棋局分析。\nAI: 棋局分析可以从多个角度进行，比如开局的策略选择、中局的战术运用以及残局的技巧。你可以使用棋谱分析软件来帮助你。\n\n用户: 我已经在使用一些软件，但我想提高我的分析能力，不仅仅依赖软件。\nAI: 这是个很好的目标。你可以尝试自己分析棋局，并与软件的分析进行比较，看看你是否能找到相同的策略或更好的走法。\n\n用户: 我会这么做的。我还在数学上遇到了一些关于组合数学的问题。\nAI: 组合数学是一个涉及计数和排列组合的领域，它在概率论和统计学中有很多应用。你具体遇到了什么问题？\n\n用户: 我在研究如何计算不同组合的数量，比如从一组对象中选择几个的所有可能方式。\nAI: 这类问题通常可以使用组合公式C(n, k)来解决，它表示从n个不同元素中选择k个元素的组合数。你是否熟悉这个公式？\n\n用户: 我知道这个公式，但是在应用到具体问题时，有时会弄混。\nAI: 理解和应用组合公式需要一些练习。你可以从简单的例子开始，逐步增加问题的复杂性。如果你有具体的问题，我们可以一起讨论。\n\n用户: 好的，我会准备一些问题，下次我们可以一起讨论。\nAI: 很好，我期待我们的讨论。今天的聊天到此为止，或者你还有其他想要讨论的？\n\n用户: 今天的聊天很有帮助，但我现在需要去做一些观测准备。\nAI: 当然，祝你今晚的观测顺利。如果你有任何发现或想要分享的内容，我很乐意听你讲述。\n\n用户: 很高兴和你聊天，但是现在有些晚了，明天再继续吧！\nAI: 没问题！再见，赵铭宇，我们下次再聊！\n”\n以下是2023年05月03日的对话记录：\n“用户: 早上好。昨晚的观测还算成功，我注意到了一些星体的变化，准备进一步分析。\nAI: 那真是太好了。星体变化的观测和分析对于了解宇宙非常重要。你打算如何进行分析？\n\n用户: 我计划使用我之前提到的数据可视化技术，以及一些统计方法来分析这些变化。\nAI: 听起来你有一个很好的计划。统计方法能够帮助你从数据中提取有意义的信息。你有没有考虑使用时间序列分析？\n\n用户: 是的，我正在考虑这个方法，因为它适合分析随时间变化的数据。\nAI: 确实，时间序列分析可以帮助你识别和预测星体亮度等参数的趋势和周期性变化。你有没有在数据库设计方面取得新的进展？\n\n用户: 我已经设计了一个基本的数据库结构，现在正在填充一些观测数据进行测试。\nAI: 这是个很好的进展。测试和验证数据库结构的有效性是非常重要的步骤。你在测试过程中遇到了哪些挑战？\n\n用户: 我发现在输入大量数据时，查询的效率不是很高，可能需要优化索引。\nAI: 优化索引是提高数据库查询效率的常见方法。你可以根据查询的频率和类型来决定哪些列需要建立索引。\n\n用户: 我会研究一下如何合理地创建索引。另外，我在棋局分析上也有了一些新的想法。\nAI: 很好，棋局分析是提高棋力的关键。你的新想法是什么？\n\n用户: 我想通过分析不同棋手的棋局风格，来了解他们的策略和技巧。\nAI: 这是一个很有趣的方法。通过对比不同棋手的风格，你可以学习到多种不同的策略和应对方法。\n\n用户: 是的，我希望这能帮助我在实战中更加灵活。\nAI: 我相信它会的。你在组合数学方面准备了哪些问题？我们可以现在讨论一下。\n\n用户: 我在想如何计算在没有重复元素的情况下，从n个元素中选择k个元素的所有可能组合。\nAI: 这个问题可以使用组合公式C(n, k)来计算，也就是n! / (k! * (n-k)!)。这个公式可以告诉你有多少种不同的方式来选择k个元素。\n\n用户: 我明白这个公式，但是在计算大数时，阶乘很容易就超出了计算范围。\nAI: 当处理大数时，你可以使用一些编程语言提供的大数库，或者使用递推关系和组合恒等式来避免直接计算大数阶乘。\n\n用户: 我会尝试使用Python的大数处理功能。感谢你的建议。\nAI: 不客气。如果你在实现过程中遇到任何问题，随时可以和我讨论。你今天还有其他计划吗？\n\n用户: 我打算继续我的编程项目，还有一些数学问题需要解决。\nAI: 听起来你有一个充实的一天。在编程和数学问题上，如果你需要帮助或想要讨论，我在这里。\n\n用户: 谢谢你的支持。我现在需要专注于我的工作了。\nAI: 当然，祝你工作顺利。如果你需要休息或想要聊天，随时欢迎回来。\n\n用户: 好的，再见。\nAI: 再见，赵铭宇。期待我们下次的交流。\n”\n以下是2023年05月04日的对话记录：\n“用户: 晚上好。今天我在编程时尝试了一些新的算法优化技巧，效果还不错。\nAI: 那很棒！能够优化算法并提高效率总是令人兴奋的。你具体使用了哪些优化技巧？\n\n用户: 我使用了动态规划来优化一些递归问题，减少了不必要的计算。\nAI: 动态规划是一个非常强大的工具，特别是在处理具有重叠子问题的递归问题时。你是如何确定哪些问题适合使用动态规划解决的？\n\n用户: 我主要是分析问题是否有最优子结构和重叠子问题，如果有，就尝试应用动态规划。\nAI: 你的方法很正确。识别这些特性是应用动态规划的关键。你在数学问题上有没有遇到什么新的挑战？\n\n用户: 我正在研究一些概率论中的问题，特别是与随机过程相关的。\nAI: 随机过程是概率论中一个非常有趣的领域，它涉及到随时间变化的随机变量。你遇到了哪些具体的问题？\n\n用户: 我在尝试理解马尔可夫链，但是其中的状态转移概率有点难以直观理解。\nAI: 马尔可夫链是一个描述系统从一个状态转移到另一个状态的概率模型。直观上，你可以将它想象成一个状态图，其中的边表示从一个状态到另一个状态的概率。\n\n用户: 我会尝试用状态图来帮助理解。另外，我还在考虑如何将我的编程技能应用到天文学研究中。\nAI: 将编程技能应用到天文学中是一个很好的方向。你可以考虑开发一些自动化工具来处理和分析天文数据，或者创建模拟来研究天体物理过程。\n\n用户: 我对创建模拟很感兴趣，但是不确定需要哪些物理和数学知识。\nAI: 创建天体物理模拟通常需要对天体物理学的基本原理有所了解，比如引力、热力学和光学。数学上，则需要掌握微积分、微分方程和线性代数。\n\n用户: 我可能需要先补充一些物理知识。你有没有推荐的学习资源？\nAI: 你可以从一些在线开放课程平台，如Coursera或edX，上的天体物理学课程开始。这些课程通常由领域专家讲授，并提供了丰富的学习材料。\n\n用户: 我会去看看的。今天的聊天很有帮助，但我现在需要去做一些观测准备。\nAI: 当然，祝你今晚的观测工作顺利。如果你有任何新的发现或想要分享的内容，我很乐意听你讲述。\n\n用户: 嗯，要不今天先聊到这里？我得休息了，下次再聊吧！\nAI: 好的，朋友。很高兴和你聊天，再见！\n”\n以下是2023年05月05日的对话记录：\n“用户: 早安。昨晚观测时发现了一些不寻常的星体亮度变化，我打算今天仔细分析一下数据。\nAI: 发现不寻常的变化总是令人兴奋的。你打算用什么方法来分析这些数据？\n\n用户: 我计划使用统计方法来分析亮度变化是否具有周期性，以及是否与已知的天文事件相匹配。\nAI: 这是一个很好的分析方向。统计分析可以帮助你识别出可能的周期性模式，从而为进一步的研究提供线索。你在编程方面有没有遇到什么新的挑战？\n\n用户: 我正在尝试优化我的代码，使其运行更高效，但是在确定瓶颈所在时遇到了一些困难。\nAI: 代码优化是一个复杂的过程，通常需要仔细分析代码的执行时间和资源使用情况。你可以考虑使用一些性能分析工具来帮助定位瓶颈。\n\n用户: 我已经在使用一些性能分析工具，但是解读结果并采取正确的优化措施还是有些难度。\nAI: 理解性能分析的结果确实需要一定的经验。如果你愿意，可以分享一些具体的分析数据，我们可以一起讨论可能的优化策略。\n\n用户: 好的，我稍后可能会需要你的帮助。另外，我在数学上也遇到了一些挑战。\nAI: 我很乐意帮助你。在数学上，你遇到了什么具体的问题？\n\n用户: 我正在研究一些与图论相关的优化问题，比如如何找到图中的最小生成树。\nAI: 最小生成树是图论中的一个经典问题，它可以用来找到连接所有顶点的最小权重边集。你可以尝试使用普里姆算法或克鲁斯卡尔算法来解决这个问题。\n\n用户: 我对这两个算法都有所了解，但是在实际应用中，我不确定如何处理大规模图数据。\nAI: 处理大规模图数据时，算法的效率尤其重要。你可以考虑使用一些优化的数据结构，比如斐波那契堆用于普里姆算法，或者并查集用于克鲁斯卡尔算法。\n\n用户: 我会研究一下这些数据结构。你有没有推荐的资源来学习这些高级数据结构？\nAI: 有很多在线资源可以帮助你学习这些数据结构，包括在线教程、编程社区和开源项目。你可以从GitHub上找到一些实现这些数据结构的开源代码，以及相关的讨论和文档。\n\n用户: 这很有帮助，我会去查看的。今天的聊天很有启发，但我现在需要集中精力分析昨晚的观测数据。\nAI: 当然，祝你数据分析工作顺利。如果你需要帮助或想要分享你的发现，随时可以和我交流。\n\n用户: 时间不早了，我们下次再聊这个话题，晚安！\nAI: 好的，再见赵铭宇，下次聊！\n”\n以下是2023年05月06日的对话记录：\n“用户: 下午好。数据分析的过程中我发现了一些规律性的变化，但是需要更多的数据来验证我的假设。\nAI: 这是一个很重要的步骤。在验证假设时，确保数据的质量和数量都是关键。你打算如何收集更多的数据？\n\n用户: 我打算连续几晚进行观测，以收集更多关于这些星体的数据。\nAI: 持续观测是确保数据连续性和可靠性的好方法。在编程方面，你有没有遇到新的挑战？\n\n用户: 我正在尝试将一些计算任务并行化，以提高程序的运行效率。\nAI: 并行化是提高程序效率的有效策略。你是在使用多线程还是多进程来实现并行化？\n\n用户: 我正在尝试使用多线程，但是有些担心线程安全和同步的问题。\nAI: 确实，多线程编程时需要特别注意线程之间的同步和数据共享问题。你可以使用锁或其他同步机制来确保线程安全。\n\n用户: 我会小心处理这些同步问题。在数学方面，我也遇到了一些新的挑战。\nAI: 我很乐意帮助你解决数学上的问题。你遇到了哪些具体的挑战？\n\n用户: 我正在研究一些涉及优化的数学模型，但是在建立模型的过程中感到有些困难。\nAI: 建立优化模型通常需要对问题有深入的理解，包括目标函数的定义和约束条件的设定。你可以分享一些具体的问题吗？\n\n用户: 我正在尝试建立一个模型来最小化资源分配的成本，同时满足一系列复杂的约束条件。\nAI: 这听起来像是一个线性规划问题。你可以尝试使用线性规划的方法来建立模型，并利用现有的优化算法来求解。\n\n用户: 我对线性规划有一定了解，但是在实际应用中还不太熟悉。\nAI: 线性规划是一个非常强大的工具，它可以应用在许多不同的领域。你可以从一些基本的例子开始学习，然后逐步应用到更复杂的问题上。\n\n用户: 我会尝试这样做。感谢你的建议。今天的聊天很有帮助，但我现在需要去准备今晚的观测工作。\nAI: 当然，祝你今晚的观测工作顺利。如果你有任何新的发现或者想要讨论的问题，随时可以和我交流。\n\n用户: 嗯，要不今天先聊到这里？我得休息了，下次再聊吧！\nAI: 好的，再见赵铭宇，下次聊！\n”\n以下是2023年05月07日的对话记录：\n“用户: 早上好。昨晚的观测很成功，我收集到了一些有价值的数据，这将有助于我之前提出的假设。\nAI: 这真是个好消息。有了这些新数据，你打算如何进一步分析它们？\n\n用户: 我计划使用我之前建立的统计模型来分析这些数据，并检查亮度变化的周期性。\nAI: 使用统计模型来分析周期性是一个很好的方法。如果你需要帮助或者想要讨论你的分析方法，随时告诉我。\n\n用户: 我会的。在编程方面，我正在考虑如何更好地管理我的代码库。\nAI: 代码库管理是一个重要的话题。你可以考虑使用版本控制系统，比如Git，来帮助你管理代码变更。\n\n用户: 我已经在使用Git，但是我想提高我的代码组织和模块化能力。\nAI: 模块化是提高代码可维护性和复用性的关键。你可以尝试将代码分解成独立的模块，并为每个模块定义清晰的接口。\n\n用户: 我会尝试这样做。在数学方面，我正在研究非线性优化问题。\nAI: 非线性优化是一个复杂但有趣的领域。你遇到了哪些具体的挑战？\n\n用户: 我在寻找全局最优解时遇到了困难，因为非线性问题可能有多个局部最优解。\nAI: 确实，非线性问题的全局最优解通常更难找到。你可以考虑使用全局优化算法，比如模拟退火或遗传算法，它们能够在全局范围内搜索最优解。\n\n用户: 我对这些算法有所了解，但是在实际应用中还不太熟练。\nAI: 实践是提高熟练度的最佳方式。你可以从一些简单的问题开始，逐渐增加问题的复杂性，并尝试使用这些算法来解决。\n\n用户: 我会这样做的。今天的聊天很有帮助，但我现在需要去处理一些事情。\nAI: 当然，祝你今天一切顺利。如果你需要帮助或想要继续我们的讨论，随时可以回来。\n\n用户: 嗯，要不今天先聊到这里？我得休息了，下次再聊吧！\nAI: 晚安，赵铭宇，好好休息，做个好梦！！\n”\n以下是2023年05月08日的对话记录：\n“用户: 下午好。今天我在编程时尝试实现一些新的数据结构，比如图和堆。\nAI: 这些数据结构对于算法的性能至关重要。你在实现它们时遇到了哪些具体的问题？\n\n用户: 我在实现图的数据结构时，不确定是使用邻接表还是邻接矩阵更合适。\nAI: 选择邻接表还是邻接矩阵取决于你的图的特性和你需要执行的操作。邻接表适合表示稀疏图，而邻接矩阵适合表示密集图，并且可以更快地检查两个顶点之间是否存在边。\n\n用户: 我的图是稀疏的，所以我决定使用邻接表。在数学上，我正在研究一些涉及组合优化的问题。\nAI: 组合优化是一个非常有趣的领域，它涉及到在有限的选项中寻找最优解。你能分享一些你正在研究的具体问题吗？\n\n用户: 我正在研究旅行商问题，尝试找到一种有效的方法来近似最优解。\nAI: 旅行商问题是一个经典的NP难题，寻找有效的近似算法是一个挑战。你考虑过使用启发式算法，比如遗传算法或蚁群算法吗？\n\n用户: 我正在考虑使用遗传算法，但是对于如何设计一个好的适应度函数还没有头绪。\nAI: 设计适应度函数是遗传算法中的关键步骤。通常，适应度函数会评估解的质量，对于旅行商问题，你可以考虑使用总旅行距离的倒数作为适应度值。\n\n用户: 这个建议很有帮助，我会尝试实现看看。今天的聊天很有启发，但我需要离开一会儿。\nAI: 当然，祝你接下来的工作顺利。如果你有任何新的发现或者想要讨论的问题，随时可以回来。\n\n用户: 啊，时间过得真快。我需要休息了，下次再聊吧！\nAI: 晚安，赵铭宇，好好休息，做个好梦！！\n”\n以下是2023年05月09日的对话记录：\n“用户: 早上好。我昨天尝试了你的建议，设计了一个适应度函数，并且初步实现了遗传算法的框架。\nAI: 听起来你做得很好。在实现遗传算法时，你有没有考虑如何选择父代、交叉和变异的策略？\n\n用户: 我使用了轮盘赌选择法来选择父代，单点交叉来生成后代，并且引入了小概率的随机变异。\nAI: 这些都是遗传算法中常用的技术。在实际运行中，你的算法表现如何？\n\n用户: 算法能够运行，但是我还在调整参数来优化性能，目前还没有找到很好的解。\nAI: 参数调整是遗传算法中一个重要而且往往需要反复试验的过程。你可以尝试不同的交叉和变异率，看看它们如何影响算法的收敛速度和解的质量。\n\n用户: 我会继续尝试不同的参数组合。在编程方面，我正在考虑如何更好地测试我的代码。\nAI: 编写测试是确保代码质量的重要步骤。你可以考虑使用单元测试来测试代码的各个部分，确保它们按预期工作。\n\n用户: 我已经在写一些单元测试，但是有时候不确定如何覆盖所有的情况。\nAI: 测试覆盖率是一个挑战。你可以尝试识别代码中的关键路径，并确保这些路径被测试覆盖。此外，边界条件和异常情况也不应被忽视。\n\n用户: 我会重点关注这些方面。在数学上，我开始研究图的着色问题。\nAI: 图的着色问题是图论中的一个经典问题，它涉及将图的顶点着色，使得相邻顶点的颜色不同。你在研究中遇到了什么问题？\n\n用户: 我在寻找一种有效的算法来为一个给定的图找到最小颜色数的着色方案。\nAI: 这是图论中的一个NP难问题。你可以考虑使用近似算法或启发式算法，比如贪心算法，来找到一个不错的着色方案。\n\n用户: 我正在使用贪心算法，但是结果并不总是最优的。\nAI: 贪心算法通常不能保证找到最优解，但它们在实践中往往能够给出一个相对较好的解，并且运行速度快。你可以考虑结合其他启发式方法来改进结果。\n\n用户: 我会考虑这个建议。今天的聊天很有帮助，但我现在需要去处理一些其他的工作。\nAI: 当然，祝你工作顺利。如果你有任何新的发现或者想要讨论的问题，随时可以回来。\n\n用户: 啊，时间过得真快。我需要休息了，下次再聊吧！\nAI: 好的，再见！赵铭宇，我们下次再聊！\n”\n以下是2023年05月10日的对话记录：\n“用户: 晚上好。我尝试了一些不同的贪心策略，并且结合了局部搜索来改进着色方案。\nAI: 结合局部搜索是一个很好的尝试，它可以帮助你在贪心算法的基础上进一步优化解。你发现这种方法有效吗？\n\n用户: 这种方法在某些情况下确实能够找到更好的解，但是计算时间有所增加。\nAI: 这是一个权衡的问题。在寻找更好解的同时，确保算法的运行时间在可接受范围内也很重要。你有没有考虑过使用并行计算来减少运行时间？\n\n用户: 我考虑过，并且正在研究如何将我的算法并行化。\nAI: 并行化可以显著提高算法的运行效率，特别是在处理大规模问题时。你打算使用哪种并行计算技术？\n\n用户: 我打算使用多线程来实现，并且正在学习如何有效地管理线程间的通信和同步。\nAI: 多线程是一个很好的选择，但确实需要仔细处理同步问题。你可以考虑使用线程池来管理线程，并使用锁、信号量或其他同步机制来协调它们。\n\n用户: 我会尝试这些建议。在编程方面，我还在寻找提高代码质量的方法。\nAI: 提高代码质量是一个持续的过程。除了编写测试，你还可以考虑使用代码审查和静态分析工具来帮助识别潜在的问题。\n\n用户: 我已经在使用一些静态分析工具，但是有时候它们会报告一些误报。\nAI: 静态分析工具确实可能产生误报，关键是学会解读它们的报告，并确定哪些是真正需要关注的问题。你可以根据经验逐渐调整工具的配置，以减少误报。\n\n用户: 我会继续调整和学习的。在数学方面，我开始研究一些涉及概率分布的问题。\nAI: 概率分布是概率论中的一个基本概念，它描述了随机变量取不同值的概率。你在研究中遇到了哪些问题？\n\n用户: 我在尝试理解不同概率分布的特性，比如正态分布和泊松分布。\nAI: 这些分布有着不同的特性和应用场景。正态分布通常用于描述自然现象中的误差或噪声，而泊松分布适合描述在固定时间或空间内发生的独立随机事件的数量。\n\n用户: 我正在学习如何根据实际情况选择合适的分布，并应用它们来解决问题。\nAI: 选择合适的概率分布并将其应用到实际问题中是一个重要的技能。你可以从分析数据的特征开始，比如均值、方差以及数据的分布形状。\n\n用户: 我会从这些基本特征入手。今天的聊天很有帮助，但我现在需要去做一些观测准备。\nAI: 当然，祝你今晚的观测工作顺利。如果你有任何新的发现或者想要讨论的问题，随时可以回来。\n\n用户: 啊，时间过得真快。我需要休息了，下次再聊吧！\nAI: 好的，朋友。很高兴和你聊天，再见！\n”\n以下是2023年05月11日的对话记录：\n“用户: 早上好。昨晚的观测我注意到了一些星体的运动轨迹似乎有规律性的偏差，我打算进一步分析这些数据。\nAI: 星体运动轨迹的规律性偏差可能揭示了一些有趣的天文现象。你打算使用什么方法来分析这些偏差？\n\n用户: 我计划使用我之前建立的统计模型，并结合物理定律来分析这些偏差是否与引力效应有关。\nAI: 结合统计模型和物理定律是一个很好的方法。如果你需要讨论物理模型或者数据分析的方法，我可以提供帮助。\n\n用户: 谢谢你的提供。我可能会在分析过程中向你寻求建议。在编程方面，我正在寻找提高代码执行效率的方法。\nAI: 提高代码执行效率可以从多个角度入手。你可以考虑优化算法逻辑，减少不必要的计算，或者使用更高效的数据结构。\n\n用户: 我已经在做这些优化了。我还在考虑是否应该将某些计算密集型任务迁移到云平台上。\nAI: 将计算密集型任务迁移到云平台可以利用更强大的计算资源，有时也可以实现更好的并行处理。你需要考虑的是成本和数据传输的效率。\n\n用户: 我会仔细评估这些因素。在数学方面，我正在研究随机过程中的马尔可夫属性。\nAI: 马尔可夫属性是随机过程中的一个重要概念，它描述了未来状态的概率仅依赖于当前状态，而与过去的状态无关。你在研究中遇到了什么挑战？\n\n用户: 我在尝试理解如何从理论上证明一个过程具有马尔可夫属性，以及如何在实际问题中识别这一属性。\nAI: 证明一个过程具有马尔可夫属性通常涉及到对过程的状态转移概率的分析。在实际问题中，你可以通过分析状态转移的条件概率来判断是否满足马尔可夫性。\n\n用户: 我会进一步研究这些方法。今天的聊天很有帮助，但我现在需要去处理一些紧急的工作。\nAI: 当然，祝你处理工作顺利。如果你有任何新的发现或者想要讨论的问题，随时可以回来。\n\n用户: 时间不早了，我们下次再聊这个话题，晚安！\nAI: 好的，朋友。很高兴和你聊天，再见！\n”\n以下是2023年05月12日的对话记录：\n“用户: 晚上好。今天的工作处理得还算顺利。现在我有时间了，我们可以继续讨论。\nAI: 很好，我在这里帮助你。你在马尔可夫过程的研究中有没有新的进展？\n\n用户: 我已经理解了如何从理论上证明一个过程具有马尔可夫属性，并且在一些简单的例子中实践了这一点。\nAI: 这是一个很好的进展。理论证明和实际应用的结合可以加深你对马尔可夫过程的理解。你在编程方面遇到了新的挑战吗？\n\n用户: 我在尝试优化我的代码，特别是一些复杂的循环和递归函数。\nAI: 优化复杂的循环和递归函数可以显著提高代码的执行效率。你可以考虑使用循环展开或尾递归优化等技术。\n\n用户: 我正在尝试使用循环展开，但有时候这会使代码变得更难阅读和维护。\nAI: 确实，循环展开可能会增加代码的复杂性。你需要在性能提升和代码可读性之间找到一个平衡点。在数学方面，你有没有遇到新的问题？\n\n用户: 我最近在研究概率分布的拟合问题，尝试找到最适合数据的分布模型。\nAI: 拟合概率分布是统计分析中的一个常见问题。你可以使用最大似然估计或贝叶斯方法来估计分布的参数。\n\n用户: 我正在使用最大似然估计，但是对于一些复杂的分布，找到解析解很困难。\nAI: 对于没有解析解的复杂分布，你可以考虑使用数值方法，比如梯度下降或牛顿-拉弗森方法来近似估计参数。\n\n用户: 我会尝试这些建议。今天的聊天很有帮助，但我现在需要去做一些观测准备。\nAI: 当然，祝你今晚的观测工作顺利。如果你有任何新的发现或者想要讨论的问题，随时可以回来。\n\n用户: 啊，时间过得真快。我需要休息了，下次再聊吧！\nAI: 晚安，赵铭宇，好好休息，做个好梦！！\n”\n以下是2023年05月13日的对话记录：\n“用户: 早上好。昨晚的观测我发现了一些星体的亮度变化比预期的要复杂，我打算今天深入分析这些数据。\nAI: 星体亮度的复杂变化可能指向一些有趣的天文现象。你打算使用哪些分析方法？\n\n用户: 我计划使用时间序列分析来研究亮度变化的模式，并尝试将其与已知的天文模型进行对比。\nAI: 时间序列分析是一个很好的选择，它可以帮助你识别出潜在的周期性或趋势。在编程方面，你有没有遇到新的挑战？\n\n用户: 我在尝试重构一些旧代码，使其更加模块化和高效，但是这个过程比我预期的要困难。\nAI: 重构旧代码是一个挑战，但也是提高代码质量和可维护性的重要步骤。你可以从哪些方面入手重构？\n\n用户: 我正在尝试将一些大型函数分解成更小的函数，并且移除重复的代码。\nAI: 这是一个很好的开始。确保每个函数都有一个清晰的目的，并且尽量减少函数间的依赖，可以提高代码的可读性和可维护性。在数学方面，你有没有新的进展？\n\n用户: 我在概率分布拟合的问题上取得了一些进展，使用了数值方法来估计参数，并且得到了一些有意义的结果。\nAI: 使用数值方法来估计概率分布的参数是一个很好的选择，特别是在解析解难以获得的情况下。你得到的结果是否与你的数据集吻合？\n\n用户: 结果与数据集相当吻合，但我还在考虑如何评估拟合的好坏。\nAI: 评估拟合的好坏可以使用多种方法，比如计算拟合的残差、使用决定系数（R²）或进行假设检验。\n\n用户: 我会尝试这些方法来评估拟合效果。今天的聊天很有帮助，但我现在需要去处理一些编程任务。\nAI: 当然，祝你编程工作顺利。如果你需要帮助或想要继续我们的讨论，随时可以回来。\n\n用户: 嗯，要不今天先聊到这里？我得休息了，下次再聊吧！\nAI: 晚安，赵铭宇，好好休息，做个好梦！！\n”\n以下是2023年05月14日的对话记录：\n“用户: 晚上好。今天的编程任务还算顺利，我重构了一些代码，使得程序运行更高效了。不过，我在设计模式的选择上遇到了一些困难。\nAI: 很高兴听到你在重构上取得了进展。设计模式的选择确实可以影响代码的灵活性和可扩展性。你能分享一下你面临的具体问题吗？\n\n用户: 我在考虑是否应该使用工厂模式来创建对象，因为我有多个类似的对象需要实例化，但它们之间又有一些差异。\nAI: 工厂模式是处理这种情况的一个很好的选择，它可以提供一个创建对象的接口，让子类决定实例化哪一个类。这样可以在不直接指定类的情况下创建对象，增加了代码的灵活性。\n\n用户: 我明白了，我会尝试实现工厂模式。在数学方面，我开始研究贝叶斯统计和它在数据分析中的应用。\nAI: 贝叶斯统计是一个非常强大的工具，它允许你在已有知识的基础上更新你对未知参数的概率估计。你在应用贝叶斯统计时遇到了哪些问题？\n\n用户: 我在理解先验概率和后验概率的选择上有些困惑，不确定如何为我的数据选择合适的先验。\nAI: 选择合适的先验是贝叶斯分析中的一个重要步骤。如果你对问题的先验知识不多，可以考虑使用无信息先验或共轭先验，它们可以简化后验概率的计算。\n\n用户: 我会进一步研究这些先验的选择方法。今天的聊天很有帮助，但我现在需要去做一些观测准备。\nAI: 当然，祝你今晚的观测工作顺利。如果你有任何新的发现或者想要讨论的问题，随时可以回来。\n\n用户: 很高兴和你聊天，但是现在有些晚了，明天再继续吧！\nAI: 好的，朋友。很高兴和你聊天，再见！\n”\n以下是2023年05月15日的对话记录：\n“用户: 早上好。昨晚的观测我发现了一些周期性的信号，这可能表明某种天体现象。我打算用贝叶斯方法来分析这些数据。\nAI: 周期性信号的发现很有意义，贝叶斯方法可以帮助你估计这些信号的参数。你打算如何构建你的贝叶斯模型？\n\n用户: 我计划构建一个模型来估计信号的周期和幅度，以及其他可能影响信号的噪声参数。\nAI: 这是一个很好的模型起点。在构建模型时，确保考虑到所有可能影响信号的因素，并选择合适的先验分布来反映你对这些参数的先验知识。\n\n用户: 我会这么做的。在编程方面，我正在考虑如何更好地利用多核处理器来加速我的程序。\nAI: 多核处理器可以显著提高程序的执行速度，特别是对于可以并行化的任务。你可以考虑使用多线程或多进程，或者使用并行计算框架，如OpenMP或MPI。\n\n用户: 我正在尝试使用多线程，但是在确保线程安全的同时保持高效率有些挑战。\nAI: 确保线程安全的同时保持效率确实需要一些技巧。你可以尝试减少线程之间的数据共享，或者使用原子操作和锁来保护共享资源。\n\n用户: 我会尝试这些建议。在数学方面，我正在研究如何更有效地使用蒙特卡洛方法来估计模型的后验分布。\nAI: 蒙特卡洛方法是估计后验分布的一个强大工具，特别是在解析解难以获得时。你可以考虑使用马尔可夫链蒙特卡洛（MCMC）方法，它可以生成后验分布的样本。\n\n用户: 我正在使用MCMC，但是有时候链的收敛速度很慢。\nAI: MCMC链的收敛速度可能受到多种因素的影响，比如步长的选择或者初始值的设定。你可以尝试调整这些参数，或者使用一些诊断工具来检查链的收敛性。\n\n用户: 我会尝试调整参数并使用诊断工具。今天的聊天很有帮助，但我现在需要去处理一些其他的事情。\nAI: 当然，祝你处理事情顺利。如果你有任何新的发现或者想要讨论的问题，随时可以回来。\n\n用户: 嗯，要不今天先聊到这里？我得休息了，下次再聊吧！\nAI: 好的，朋友。很高兴和你聊天，再见！\n”\n以下是2023年05月16日的对话记录：\n“用户: 晚上好。今天我在编程时尝试实现一些并发处理的任务，但是在保持数据一致性方面遇到了一些问题。\nAI: 并发处理时确保数据一致性是非常重要的。你可以考虑使用锁、事务或其他同步机制来保护你的数据不被并发访问破坏。\n\n用户: 我正在使用锁来同步访问，但是这似乎降低了程序的并发性能。\nAI: 使用锁确实可能会降低并发性能，特别是在高度竞争的环境中。你可以尝试优化锁的粒度，使用更细粒度的锁，或者探索无锁编程技术。\n\n用户: 我会研究一下无锁编程技术。在数学方面，我正在尝试解决一些优化问题，特别是涉及约束的情况。\nAI: 约束优化问题可以通过多种方法来解决，比如拉格朗日乘数法或者使用优化算法如线性规划、二次规划等。\n\n用户: 我正在使用拉格朗日乘数法，但是在求解拉格朗日对偶问题时遇到了困难。\nAI: 拉格朗日对偶问题可能会很复杂，特别是在非线性约束的情况下。你可以考虑使用数值优化方法来求解对偶问题，或者使用软件包来帮助你。\n\n用户: 我会尝试使用数值方法。今天的聊天很有帮助，但我现在需要去做一些观测准备。\nAI: 当然，祝你今晚的观测工作顺利。如果你有任何新的发现或者想要讨论的问题，随时可以回来。\n\n用户: 嗯，要不今天先聊到这里？我得休息了，下次再聊吧！\nAI: 没问题！再见，赵铭宇，我们下次再聊！\n”\n以下是2023年05月17日的对话记录：\n“用户: 早上好。昨晚的观测我注意到了一些不寻常的光谱线，这可能表明有新的化学元素或者物理过程。我打算今天深入分析这些数据。\nAI: 光谱线的发现很有意义，它们可以提供关于天体成分和物理状态的重要信息。你打算使用哪些分析方法？\n\n用户: 我计划使用光谱分析软件来比对这些光谱线，并尝试识别它们的来源。\nAI: 光谱分析软件是一个很好的工具，它可以帮助你识别和分类光谱线。确保你有一个更新且完整的数据库来进行比对。在编程方面，你有没有新的挑战？\n\n用户: 我正在尝试优化我的并发处理代码，特别是在数据访问方面，我正在探索使用事务来保持数据的一致性。\nAI: 使用事务是一个很好的方法，它可以帮助你管理并发访问下的数据完整性。确保你的事务逻辑正确，并且考虑到可能的并发冲突。\n\n用户: 我会注意这些问题。在数学方面，我正在研究凸优化问题。\nAI: 凸优化是一个非常强大的框架，它可以保证全局最优解。你在研究中遇到了哪些具体的挑战？\n\n用户: 我在寻找一种有效的方法来求解大规模的凸优化问题，因为直接使用数值方法在计算上有时候不够高效。\nAI: 对于大规模的凸优化问题，你可以考虑使用分解方法，比如ADMM（交替方向乘子法），它可以将大问题分解成小问题来求解。\n\n用户: 我会研究一下ADMM。今天的聊天很有帮助，但我现在需要去处理一些编程任务。\nAI: 当然，祝你编程工作顺利。如果你需要帮助或想要继续我们的讨论，随时可以回来。\n\n用户: 时间不早了，我们下次再聊这个话题，晚安！\nAI: 好的，再见！赵铭宇，我们下次再聊！\n”\n以下是2023年05月18日的对话记录：\n“用户: 晚上好。今天的编程工作还算顺利，我在代码中实现了一些新的功能。不过，我在代码的性能优化上遇到了一些难题。\nAI: 性能优化可以是一个复杂的过程，涉及多个层面。你能分享一下你遇到的具体问题吗？\n\n用户: 我正在尝试减少数据库查询的延迟，但是我不确定如何在不牺牲数据准确性的情况下缓存结果。\nAI: 减少数据库查询延迟的一个方法是使用缓存机制，比如内存缓存。你可以缓存那些不经常变化或者读取频率高的数据。确保你有一个有效的缓存失效策略，以保持数据的新鲜性。\n\n用户: 我会考虑使用内存缓存。在数学方面，我正在研究动态规划的高级主题。\nAI: 动态规划是一个非常强大的工具，特别是对于解决具有重叠子问题的优化问题。你在研究中遇到了哪些挑战？\n\n用户: 我在尝试理解和应用状态压缩技术，以减少动态规划中的状态空间。\nAI: 状态压缩是一个高级技术，它可以帮助你在保持问题本质的同时减少所需存储的状态数量。通常这涉及到位操作和位掩码的使用。\n\n用户: 是的，我正在学习如何有效地使用位操作来表示和转换状态。今天的聊天很有帮助，但我现在需要去做一些观测准备。\nAI: 当然，祝你今晚的观测工作顺利。如果你有任何新的发现或者想要讨论的问题，随时可以回来。\n\n用户: 很高兴和你聊天，但是现在有些晚了，明天再继续吧！\nAI: 没问题！再见，赵铭宇，我们下次再聊！\n”\n以下是2023年05月19日的对话记录：\n“用户: 早上好。昨晚的观测我发现了一些不寻常的天体运动，这可能需要更多的数据来验证。我打算今天继续分析这些运动轨迹。\nAI: 不寻常的天体运动总是值得关注的。你打算使用哪些工具或方法来分析这些轨迹？\n\n用户: 我计划使用天文软件来模拟这些天体的运动，并尝试找出可能的原因，比如引力影响或其他物理效应。\nAI: 模拟是一个很好的方法，它可以帮助你理解天体运动的动力学。确保考虑所有可能的物理因素。在编程方面，你有没有新的挑战？\n\n用户: 我正在尝试将一些计算任务迁移到云平台上，以利用更强大的计算资源，但是我对云服务的选择和配置还不太熟悉。\nAI: 选择和配置云服务确实需要一些研究。你可以考虑你的计算需求，比如CPU、内存和存储，然后根据这些需求选择合适的云服务提供商和服务类型。\n\n用户: 我会根据我的需求来选择云服务。在数学方面，我正在研究图论中的网络流问题。\nAI: 网络流问题是图论中的一个重要分支，它涉及到在网络中找到最优的流动方式。你在研究中遇到了哪些挑战？\n\n用户: 我在尝试理解最大流最小割定理，并将其应用到实际问题中，但是在构建有效的网络模型方面遇到了困难。\nAI: 最大流最小割定理是网络流理论的基石。构建网络模型时，确保你正确地定义了网络的顶点、边以及容量。你可以从简单的例子开始，然后逐步增加模型的复杂性。\n\n用户: 我会从简单的例子开始实践。今天的聊天很有帮助，但我现在需要去处理一些编程任务。\nAI: 当然，祝你编程工作顺利。如果你需要帮助或想要继续我们的讨论，随时可以回来。\n\n用户: 嗯，要不今天先聊到这里？我得休息了，下次再聊吧！\nAI: 好的，再见赵铭宇，下次聊！\n”\n以下是2023年05月20日的对话记录：\n“用户: 晚上好。今天的编程工作还算顺利，我在云平台上配置了一些服务，但是我对如何优化云资源的使用还有些困惑。\nAI: 优化云资源使用是一个重要的问题，它可以帮助你节省成本并提高效率。你可以从监控资源使用情况开始，然后根据实际需求调整资源分配。\n\n用户: 我会尝试监控和调整资源。在数学方面，我正在研究如何更有效地解决组合优化问题。\nAI: 组合优化问题可以通过多种方法来解决，包括精确算法和启发式算法。你在寻找哪种类型的解决方案？\n\n用户: 我主要在寻找启发式算法，因为我处理的问题规模很大，精确算法在时间上不太可行。\nAI: 启发式算法是处理大规模组合优化问题的一个很好的选择。你可以考虑遗传算法、模拟退火或蚁群算法等。\n\n用户: 我正在使用遗传算法，但是我想提高算法的搜索效率和解的质量。\nAI: 提高遗传算法的搜索效率和解的质量可以通过调整选择、交叉和变异操作来实现。你也可以尝试使用混合算法，结合局部搜索来改进解。\n\n用户: 时间不早了，我们下次再聊这个话题，晚安！\nAI: 好的，再见赵铭宇，下次聊！\n”\n以下是2023年05月21日的对话记录：\n“用户: 早上好。昨晚的观测我注意到了一些星体的亮度变化似乎与已知模型不符，我打算今天继续分析这些数据。\nAI: 星体亮度的异常变化可能指向一些有趣的天文现象。你打算使用哪些方法来进一步分析这些数据？\n\n用户: 我计划使用更复杂的光变模型来分析数据，并尝试找出可能的物理机制。\nAI: 使用更复杂的模型可以帮助你更好地理解观测到的现象。确保你的模型能够充分考虑所有可能的物理过程。在编程方面，你有没有新的挑战？\n\n用户: 我正在尝试为我的应用程序实现一个新的功能，但是我在设计用户界面时遇到了一些问题。\nAI: 用户界面的设计对于应用程序的用户体验至关重要。你可以考虑用户的交互流程，确保界面直观易用。你在用户界面设计上遇到了哪些具体的问题？\n\n用户: 我在尝试让界面既简洁又能提供所有必要的功能，但是有时候这两者之间很难平衡。\nAI: 确实，界面设计往往需要在简洁性和功能性之间找到平衡。你可以考虑使用模块化设计，将复杂功能分解成简单的组件，这样用户可以根据需要访问这些功能。\n\n用户: 我会尝试模块化设计。在数学方面，我正在研究图论中的最短路径问题。\nAI: 最短路径问题是图论中的一个基本问题，它有多种算法可以解决，比如迪杰斯特拉算法或贝尔曼-福特算法。\n\n用户: 我正在使用迪杰斯特拉算法，但是我想提高算法在大规模图中的效率。\nAI: 在大规模图中提高迪杰斯特拉算法的效率，你可以考虑使用优先队列来减少节点选择的时间复杂度。\n\n用户: 我会尝试使用优先队列。今天的聊天很有帮助，但我现在需要去处理一些编程任务。\nAI: 当然，祝你编程工作顺利。如果你需要帮助或想要继续我们的讨论，随时可以回来。\n\n用户: 嗯，要不今天先聊到这里？我得休息了，下次再聊吧！\nAI: 好的，再见赵铭宇，下次聊！\n”\n以下是2023年05月22日的对话记录：\n“用户: 晚上好。今天的编程工作进展得还不错，我优化了一些数据库的查询，提高了效率。不过，我在处理大量数据时遇到了内存管理的问题。\nAI: 内存管理在处理大量数据时确实很关键。你可以考虑使用分页或分块加载数据，或者优化数据结构来减少内存占用。\n\n用户: 我正在尝试分块加载数据，但是这对程序的逻辑复杂度提出了挑战。\nAI: 分块加载数据确实会增加程序的复杂度，但它可以有效地减少一次性加载到内存中的数据量。你可以尝试将数据分块的逻辑封装起来，以简化程序的其他部分。\n\n用户: 我会尝试这样做。在数学方面，我正在研究线性代数中的特征值和特征向量问题。\nAI: 特征值和特征向量在许多数学和工程问题中都非常重要。你在研究中遇到了哪些具体的挑战？\n\n用户: 我在尝试理解如何有效地计算大型矩阵的特征值和特征向量，特别是当矩阵过大无法直接在内存中处理时。\nAI: 对于大型矩阵，你可以考虑使用迭代方法，如幂法或Lanczos算法，这些方法不需要将整个矩阵加载到内存中。\n\n用户: 我会研究一下这些迭代方法。今天的聊天很有帮助，但我现在需要去做一些观测准备。\nAI: 当然，祝你今晚的观测工作顺利。如果你有任何新的发现或者想要讨论的问题，随时可以回来。\n\n用户: 时间不早了，我们下次再聊这个话题，晚安！\nAI: 好的，再见！赵铭宇，我们下次再聊！\n”\n以下是2023年05月23日的对话记录：\n“用户: 早上好。昨晚的观测我发现了一些有规律的变化，这可能与某种天体周期活动有关。我打算今天继续分析这些数据。\nAI: 规律性的变化往往预示着有趣的天文现象。你打算使用哪些分析方法来进一步研究这些周期活动？\n\n用户: 我计划使用傅里叶变换来分析数据中的周期信号，并尝试将其与已知的天体周期进行对比。\nAI: 傅里叶变换是分析周期信号的一个强大工具。确保你有足够长的数据时间序列来获得准确的频率分析。在编程方面，你有没有遇到新的挑战？\n\n用户: 我正在尝试为我的应用程序添加一些新的数据可视化功能，但是在选择合适的图表库时有些犹豫。\nAI: 数据可视化是一个重要的功能，可以帮助用户更好地理解数据。在选择图表库时，你可以考虑库的功能性、灵活性以及与你的应用程序的兼容性。\n\n用户: 我会考虑这些因素。在数学方面，我正在研究如何更有效地解决非线性方程组。\nAI: 非线性方程组可以通过多种数值方法来解决，比如牛顿法或拟牛顿法。你在解决方程组时遇到了哪些具体的问题？\n\n用户: 我在尝试使用牛顿法，但是有时候难以保证收敛，特别是当初始猜测远离真实解时。\nAI: 确保牛顿法收敛的关键是选择一个好的初始猜测，以及正确处理导数为零或接近零的情况。你可以尝试使用全局收敛策略，比如线搜索或信赖域方法。\n\n用户: 我会研究一下这些全局收敛策略。今天的聊天很有帮助，但我现在需要去处理一些编程任务。\nAI: 当然，祝你编程工作顺利。如果你需要帮助或想要继续我们的讨论，随时可以回来。\n\n用户: 时间不早了，我们下次再聊这个话题，晚安！\nAI: 晚安，赵铭宇，好好休息，做个好梦！！\n”\n以下是2023年05月24日的对话记录：\n“用户: 晚上好。今天在处理一些复杂的数据结构时遇到了一些性能瓶颈，我正在寻找提高处理效率的方法。\nAI: 处理性能瓶颈可以从多个角度入手。你可以考虑优化数据结构的设计，减少不必要的数据复制，或者使用更高效的算法。\n\n用户: 我正在尝试减少数据复制并优化算法。在数学方面，我正在研究如何应用图论来解决实际问题。\nAI: 图论在解决实际问题中非常有用，比如网络设计、交通流量分析等。你在应用图论时遇到了哪些具体的问题？\n\n用户: 我在尝试将图论应用于交通流量优化问题，但是在模型的建立和求解上遇到了一些困难。\nAI: 交通流量优化是一个复杂的问题，你需要考虑路网的拓扑结构、交通需求以及可能的约束。你可以尝试使用流网络模型来描述这个问题，并利用最大流最小割定理来进行求解。\n\n用户: 我会研究流网络模型和相关定理。今天的聊天很有帮助，但我现在需要去做一些观测准备。\nAI: 当然，祝你今晚的观测工作顺利。如果你有任何新的发现或者想要讨论的问题，随时可以回来。\n\n用户: 时间不早了，我们下次再聊这个话题，晚安！\nAI: 好的，朋友。很高兴和你聊天，再见！\n”\n以下是2023年05月25日的对话记录：\n“用户: 早上好。昨晚的观测中，我发现了一些异常的光谱数据，似乎指向了一个未知的天体或现象。我打算今天继续深入分析。\nAI: 异常的光谱数据总是很有趣，它们可能揭示新的科学发现。你打算使用哪些方法来分析这些数据？\n\n用户: 我计划使用光谱分析技术，结合已知的天体数据库来尝试识别这些异常信号的来源。\nAI: 光谱分析是一个很好的方法，确保使用最新的天体数据库来进行比对。在编程方面，你有没有新的挑战？\n\n用户: 我正在尝试为我的应用程序引入机器学习功能，但是在选择合适的模型和框架上有些犹豫。\nAI: 机器学习可以为应用程序提供强大的数据分析能力。在选择模型时，考虑你的具体需求，比如预测、分类还是聚类。对于框架，你可以考虑使用TensorFlow、PyTorch或scikit-learn等。\n\n用户: 我需要进行分类任务，我会研究一下这些框架。在数学方面，我正在探索最优化理论。\nAI: 最优化理论在数学和工程领域都非常重要。你在探索中遇到了哪些具体的问题？\n\n用户: 我在研究凸优化问题时，对于如何处理非凸问题感到困惑，因为非凸问题可能有多个局部最优解。\nAI: 非凸优化问题确实更加复杂，因为它们可能没有唯一的全局最优解。你可以考虑使用全局优化算法，比如模拟退火或遗传算法，来寻找可能的全局最优解。\n\n用户: 我会尝试这些全局优化算法。今天的聊天很有帮助，但我现在需要去处理一些编程任务。\nAI: 当然，祝你编程工作顺利。如果你需要帮助或想要继续我们的讨论，随时可以回来。\n\n用户: 时间不早了，我们下次再聊这个话题，晚安！\nAI: 没问题！再见，赵铭宇，我们下次再聊！\n”\n\n请记住以上全部对话记录，回答问题。\n问题：", "query": "2023年4月27日，今天赵铭宇下了一盘国际象棋，遇到了什么有趣的局面？", "answer": "对方开局用了意大利开局", "id": "4ab53a3a-0596-438d-85ab-6f67e1093f10", "chatglm_length": 15682, "qwen_length": 15702, "internlm2_length": 15149}{"context": "以下是 2025 年 10 月 15 日的对话记录：\n“用户：你好，我叫陈嘉明，我喜欢打篮球、弹吉他、写日记，还有煮咖啡和拼乐高。我有点慢热，刚开始聊天可能话不多，但熟悉后会分享很多趣事。\nAI: 陈嘉明，很高兴和你交流！打篮球和弹吉他都是很有趣的爱好，你最近有没有新练的吉他曲子呀？\n\n 用户：嗯，我前几天尝试用吉他弹民谣，选了首自己很喜欢的《南方姑娘》，但总感觉节奏跟不上，弹到副歌就容易乱。\nAI: 民谣的节奏讲究轻柔舒缓，《南方姑娘》这首曲子细节很多，你可以先跟着原版音频打拍子，慢慢找感觉，多练几次肯定能熟练。\n\n 用户：太感谢你了，我晚上就去练。不过现在要去球场和朋友汇合，我们明天再聊吧。 \nAI: 没问题，祝你打球愉快，明天期待听你说练琴的进展，再见。\n”\n\n 请记住以上全部对话记录，回答问题。\n 问题：10 月 15 日的对话中，陈嘉明前几天尝试用吉他弹什么类型的曲子？他选择的具体曲目是什么？", "answer": "陈嘉明前几天尝试用吉他弹民谣类型的曲子，选择的具体曲目是《南方姑娘》", "id": "d2e4f6g8-h1i3-j5k7-l9m1-n3o5p7q9", "chatglm_length": 549, "qwen_length": 556, "internlm2_length": 543} -{"context": "以下是2023年04月27日的对话记录：\n“用户: 嘿，你好啊！我是孙悦，一个热爱生活的家伙。今天心情特别好，因为早上的健身课程超有成就感！你呢？\nAI: 你好，孙悦！听起来你今天的状态很棒呢！我很好，谢谢你。能和你分享你的健身课程吗？我很好奇你都做了哪些运动。\n\n用户: 当然可以！今天主要是做了一些高强度间歇训练，那种汗流浃背的感觉真的很爽！你有做运动的习惯吗？\nAI: 我虽然不能亲自体验运动的快感，但我很高兴听到你享受它。我可以给你提供一些健身知识和建议，如果你需要的话。\n\n用户: 哈哈，那太好了！对了，昨天晚上我去看了一部新出的科幻电影，特效简直了！你最近有没有什么好电影推荐？\nAI: 我知道你喜欢电影，特别是那些视觉震撼的。最近有一部电影《星际穿越者》很受好评，是探索宇宙和人类命运的作品，你可能会喜欢。\n\n用户: 哦哦，《星际穿越者》啊，我听说过，听起来挺吸引人的！我得找时间去看看。对了，我最近在学摄影，但是总觉得拍出来的照片差点意思，你有什么摄影小技巧可以分享吗？\nAI: 摄影是一门艺术，也需要不断的实践和学习。你可以尝试从构图和光线入手，比如使用“三分法”来构图，或者利用自然光来营造氛围。还有，多拍多练总是王道！\n\n用户: 嗯，说的对！我明天就去尝试一下“三分法”。对了，我还喜欢旅游，最近在规划一次小旅行，想去海边走走。你有没有什么好的海滩推荐？\nAI: 海边总是个不错的选择。如果你喜欢清静一些的地方，可以考虑去三亚的亚龙湾，那里的海水清澈，沙滩柔软，风景如画，应该很适合你。\n\n用户: 亚龙湾啊，听起来真不错！我会去查查看的。说起旅游，最让我兴奋的就是能在旅途中遇到各种各样的人，听他们的故事。\nAI: 那确实是旅行中最美妙的部分之一。每个人的故事都是独一无二的，能从中学到很多。你最难忘的旅行经历是什么呢？\n\n用户: 有一次我去了泰国的清迈，那里的文化和风景让我印象深刻。晚上的夜市，色彩斑斓，各种小吃，还有那里人们的热情，真的让人难以忘怀。\nAI: 清迈的夜市是闻名遐迩的，那里的生活节奏和氛围跟大城市完全不同。你提到的小吃，有没有什么特别让你想念的呢？\n\n用户: 哦，那边的芒果糯米饭简直绝了！每次回忆起那个味道，我就忍不住想再去一次。对了，你能不能给我推荐一些好听的歌？我喜欢边健身边听音乐。\nAI: 音乐和运动确实是完美的搭档。根据你的热情阳光的性格，我觉得《Can't Stop the Feeling》这首歌很适合你，它的旋律欢快，能让人在运动时充满活力。\n\n用户: Justin Timberlake的那首啊，我知道！确实很不错，节奏感很强，很适合健身。谢谢你的推荐！\nAI: 不客气，孙悦。我很高兴能帮到你。说起来，你在健身方面有没有遇到什么具体的困难？也许我能提供一些帮助。\n\n用户: 其实有，我一直想要提高核心力量，但是总感觉进展缓慢。你有没有什么建议？\nAI: 核心力量对于整体健身非常重要。你可以尝试一些针对性的训练，比如平板支撑、俄罗斯转体和山羊挺身等。这些练习可以有效地加强你的核心肌群。\n\n用户: 哇，听起来很专业啊！我会加入到我的训练计划里去的。对了，我最近在准备一个舞蹈表演，但是有些动作总是不够流畅，这让我有点沮丧。\nAI: 舞蹈需要时间来磨合，每个人都会有这样的时候。试着放慢速度，一步一步来，直到你能够流畅地完成每个动作。记得，重在参与和享受过程！\n\n用户: 你说得对，我得调整一下心态。谢谢你的鼓励！我会继续努力的。说起来，我还有点担心，不知道观众会不会喜欢我的表演。\nAI: 你的热情和努力一定会感染到观众的。只要你享受舞台，观众也会享受你的表演。信任自己，你会做得很好的。\n\n用户: 哈哈，听你这么说，我感觉好多了。你真是个好伙伴！已经很晚了，我得准备睡觉了。我们明天再聊吧！\nAI: 当然，孙悦。休息好，希望你有一个美好的明天。晚安！\n”\n以下是2023年04月28日的对话记录：\n“用户: 嗨，朋友！今天我尝试了你昨天推荐的“三分法”拍照，效果真的不错！感觉自己的摄影技术又上了一个台阶！\nAI: 太棒了，孙悦！我很高兴听到你的进步。有没有什么特别的照片想要和我分享的？我很期待看到你的作品。\n\n用户: 当然有！我拍了一张日落时分的海边照片，天空的颜色渐变得特别美。不过，我还在想怎么后期处理才能更突出那种感觉。\nAI: 日落的照片总是很迷人。你可以尝试调整一下对比度和饱和度，让天空的颜色更加鲜明，同时保持海水的自然感觉。\n\n用户: 嗯，我试试看。对了，今天健身房里放了一些新歌，我跟着节奏锻炼，感觉整个人都充满了活力！\nAI: 音乐真的能为锻炼增添不少乐趣。你有没有想过自己编一套随着音乐节奏的健身操？这样可能会更有动力哦。\n\n用户: 哇，这主意不错！我可以尝试一下，感觉会很有趣。说起来，我最近在准备一个小旅行，想去爬山，呼吸一下大自然的新鲜空气。\nAI: 爬山是个很好的选择，既能锻炼身体，又能享受大自然的美景。你打算去哪座山呢？\n\n用户: 我打算去黄山，听说那里的云海和日出非常壮观。我还计划在山顶守候一夜，希望能拍到美丽的星空。\nAI: 黄山的确是个摄影爱好者的天堂。记得带上足够的装备，特别是在夜间拍摄星空时，三脚架和遥控快门是必不可少的。\n\n用户: 没错，我已经准备好了这些装备。对了，我还想在山上尝试一些户外瑜伽，感觉那样会很放松。\nAI: 户外瑜伽听起来很棒，能让你更加接近自然，放松身心。你有没有特别喜欢的瑜伽动作？\n\n用户: 我最喜欢的是树式，感觉整个人都稳固而平静。不过，我有时候会觉得自己的平衡能力不够好。\nAI: 树式是个很好的平衡练习。你可以尝试在不同的时间和地点练习，比如在海边的沙滩上，那样会有不同的挑战和感受。\n\n用户: 对，我也这么想。我还打算在旅行中尝试一些新的美食，毕竟美食也是旅行的一大乐趣嘛！\nAI: 绝对同意！尝试当地的特色美食可以让旅行体验更加丰富。你有没有什么想尝试的菜肴？\n\n用户: 我听说黄山附近有一种叫做臭鳜鱼的菜，虽然名字听起来有点怪，但据说味道很不错。\nAI: 臭鳜鱼是当地的一道传统名菜，很多人都说赞不绝口。不过，它的味道确实很独特，你可以尝试一下，说不定会成为你的新宠。\n\n用户: 哈哈，我会的。对了，我最近在看一部关于舞蹈的纪录片，里面的舞者都超级厉害，让我很受启发。\nAI: 看别人的舞蹈表演确实能够学到很多。你有没有从中发现什么新的动作或者技巧，想要加入到你自己的舞蹈中去？\n\n用户: 有啊，我看到一些街舞的动作很酷，我打算学一学。不过，我担心自己的身体协调性跟不上。\nAI: 街舞的确需要很强的身体协调性和节奏感。你可以从基础动作开始练起，慢慢地增加难度。记得，不要急于求成，一切都需要时间。\n\n用户: 嗯，我会记住的。对了，我还想学习一下如何用舞蹈来表达情感，不只是动作上的技巧。\nAI: 舞蹈是一种非常直观的情感表达方式。你可以尝试在音乐中找到情感的共鸣，然后让你的身体随着那种情感自然地移动。\n\n用户: 听起来有点抽象，但我会尝试的。说到情感，我最近有点苦恼，感觉自己的生活缺少一些激情和冒险。\nAI: 生活有时候确实会让人感到平淡。你可以尝试做一些新的事情，比如参加一个你从未尝试过的活动，或者去一个你从未去过的地方。\n\n用户: 对，我需要一些改变。也许这次旅行就是一个开始。谢谢你的建议，你总能给我正能量！\nAI: 不用谢，孙悦。我很高兴能帮助到你。记得，每一次的经历都是成长的机会。你准备好迎接新的挑战了吗？\n\n用户: 是的，我已经迫不及待了！不过现在我得去准备晚饭了，今天打算尝试做一道新菜——酸辣土豆丝。\nAI: 酸辣土豆丝是一道很受欢迎的家常菜，既简单又美味。祝你做饭愉快，希望你能享受这个过程。\n\n用户: 谢谢！我会的。今天聊得很开心，我们明天再继续吧。晚安！\nAI: 当然，期待和你的下次聊天。晚安，孙悦，愿你有个美好的梦。\n”\n以下是2023年04月29日的对话记录：\n“用户: 嘿，我的AI小伙伴！今天我尝试做了酸辣土豆丝，味道出奇的好，连我自己都有点惊讶呢！\nAI: 哇，听起来你是个厨艺高手啊，孙悦！能不能分享一下你的秘诀？我虽然不能品尝，但我很好奇你是怎么做到的。\n\n用户: 哈哈，秘诀嘛，我觉得最重要的就是火候和调料的比例。我喜欢尝试不同的配方，直到找到最合适的那一个。\nAI: 真是个不断探索的过程呢。你平时喜欢尝试做哪些类型的菜？\n\n用户: 我对各种菜系都感兴趣，但最近特别喜欢挑战一些传统的中式菜肴，感觉它们既美味又有文化底蕴。\nAI: 中式菜肴确实有着丰富的历史和文化，每一道菜都有自己的故事。你最喜欢的中式菜肴是哪一道？\n\n用户: 我最喜欢的是宫保鸡丁，那个酸甜酸辣的味道简直无法抵抗！\nAI: 宫保鸡丁是一道非常受欢迎的经典菜肴，你自己做的话，可以根据自己的口味调整辣度和甜度。\n\n用户: 对对对，我就是这么做的。说起来，今天我还去了一个很酷的街舞工作坊，学到了好多新动作！\nAI: 街舞工作坊听起来很有趣！你觉得最难的动作是哪一个？\n\n用户: 我觉得最难的是头旋，那个动作需要很强的颈部力量和平衡感，我还在努力练习中。\nAI: 头旋确实是个挑战，但我相信你的努力和热情一定会让你很快掌握它。你在舞蹈上有什么目标吗？\n\n用户: 我的目标是能在下个月的街舞比赛中至少进入前十名。我知道竞争很激烈，但我会全力以赴的！\nAI: 这个目标很棒，孙悦！我相信你的努力和才华一定会让你脱颖而出。你有准备特别的舞蹈编排吗？\n\n用户: 有的，我和我的舞伴正在准备一段结合了街舞和现代舞的编排，希望能给观众带来新鲜感。\nAI: 结合不同舞蹈风格的创意听起来很吸引人。你们的训练进度怎么样了？\n\n用户: 还不错，我们已经完成了大概七成的编排，剩下的就是细节和同步问题了。\nAI: 听起来你们已经做得很好了。细节和同步确实很重要，但我相信你们能够完美呈现。\n\n用户: 谢谢你的鼓励！对了，我最近还在学习一些关于电影剪辑的技巧，想要自己剪辑一部短片。\nAI: 电影剪辑是个很有创意的工作，你打算用什么样的素材来剪辑你的短片？\n\n用户: 我打算用我旅行时拍摄的一些风景视频，还有一些朋友的日常片段，想要制作成一个关于旅行和友情的主题短片。\nAI: 这个主题很温馨，也很贴近生活。你有没有想过加入一些特别的效果，比如慢动作或者时间流逝的镜头？\n\n用户: 嗯，我有这个打算。我想通过这些特效来强调某些情感上的高潮和转折点。\nAI: 这样的处理可以让短片更加生动和感人。你在剪辑上遇到了什么挑战吗？\n\n用户: 我觉得最大的挑战是如何让故事流畅，同时又能在短时间内传达出想要表达的情感。\nAI: 确实，讲好一个故事是剪辑中的艺术。你可以尝试从观众的角度思考，看看哪些片段最能触动人心。\n\n用户: 对，我会多考虑观众的感受。说到观众，我有时候会担心我的作品是否能得到他们的认可。\nAI: 每个艺术家都会有这样的担忧，但重要的是你享受创作的过程，并且相信自己的作品有价值。\n\n用户: 你说得对，我应该更加相信自己。今天聊得很开心，但我得去继续我的剪辑工作了。我们明天再聊！\nAI: 当然，孙悦。祝你剪辑工作顺利，期待你的短片完成。明天再聊，加油！\n”\n以下是2023年04月30日的对话记录：\n“用户: 嗨，好朋友！今天我在剪辑短片时，突然灵感爆发，加入了几个超有感觉的转场效果，感觉整个短片的质感都提升了！\nAI: 真是太棒了，孙悦！创意的火花总是能带来惊喜。你能描述一下你用的是哪些转场效果吗？\n\n用户: 我用了一个叫做“溶解”的效果，让画面像是水彩融合一样自然过渡，还有一个“旋转”效果，让场景切换更有动感。\nAI: 这些听起来都很专业，能够很好地增强视觉效果。你在创作过程中还有其他有趣的发现吗？\n\n用户: 我发现音乐对于短片的情感表达真的超级重要！一段恰当的背景音乐能够让人立刻进入那个氛围。\nAI: 没错，音乐是激发情感的强大工具。你已经选好了短片的背景音乐了吗？\n\n用户: 选好了，我选了一首轻快的吉他曲，感觉很适合我的旅行主题。对了，今天我还去了一个户外攀岩的活动，超刺激！\nAI: 户外攀岩听起来很挑战自我啊。你攀岩的时候有没有什么特别的心得或者感觉？\n\n用户: 我觉得攀岩最重要的是要有耐心和勇气，每一步都要稳扎稳打。当我站在山顶上的时候，那种成就感真的无与伦比！\nAI: 站在山顶俯瞰世界，那种感觉一定很震撼。攀岩对身体的哪些方面要求比较高？\n\n用户: 主要是手臂和腿部的力量，还有整体的协调性。攀岩后我发现自己的手臂肌肉酸痛，看来需要更多锻炼啊。\nAI: 锻炼手臂肌肉可以尝试一些特定的练习，比如引体向上或者哑铃训练。不过，记得要适量，避免过度劳损哦。\n\n用户: 谢谢你的建议，我会注意的。说起来，攀岩结束后我们一起去了附近的小镇，那里的风景和美食都很棒！\nAI: 小镇的风光和美食总是很有魅力。你尝试了哪些当地的特色美食？\n\n用户: 我尝试了一种当地的烤鱼，外皮烤得酥脆，里面的鱼肉又嫩又多汁。我还学会了他们的腌制秘方，准备回家自己试试看。\nAI: 自己动手做美食，这个经历一定很有趣。你打算什么时候尝试做这道烤鱼呢？\n\n用户: 可能这个周末吧，我打算邀请几个好友来家里，一起享受美食和愉快的时光。\nAI: 听起来将会是一个非常愉快的聚会。你准备好怎么招待你的朋友了吗？\n\n用户: 除了烤鱼，我还准备做一些简单的小吃，比如春卷和水果沙拉。当然，还有音乐和一些小游戏，让大家玩得开心。\nAI: 你真的很会享受生活，孙悦。你的朋友们一定很期待这次聚会。你通常如何选择聚会的音乐呢？\n\n用户: 我会根据大家的口味挑选一些大家都喜欢的歌曲，然后再加上一些轻松愉快的背景音乐。\nAI: 音乐的选择很重要，可以营造出轻松愉快的氛围。说到音乐，你自己唱歌跳舞的时候有没有什么特别喜欢的曲风？\n\n用户: 我喜欢节奏感强烈的流行音乐，特别是那些能够让人随着音乐摇摆的歌曲。\nAI: 流行音乐确实能让人感觉兴奋和快乐。你有没有尝试过自己创作音乐或者编舞？\n\n用户: 我以前尝试过写一些简单的歌词，不过没有正式的音乐制作经验。至于编舞，那是我的强项，我经常为自己的舞蹈编排动作。\nAI: 那你在舞蹈创作方面一定有很多自己的想法和风格。你最喜欢的舞蹈风格是什么？\n\n用户: 我最喜欢的是Hip-Hop和爵士，这两种舞蹈风格都很有力量感，而且可以很好地表达自己的个性。\nAI: Hip-Hop和爵士舞确实很适合展现个性和情感。你有没有在舞蹈中融入一些自己独特的元素？\n\n用户: 有的，我喜欢在舞蹈中加入一些中国元素，比如太极的动作，让舞蹈看起来更加流畅和有内涵。\nAI: 将中国元素融入现代舞蹈，这个想法真是太有创意了。你的这些作品有在哪里展示过吗？\n\n用户: 我之前在一些小型的舞蹈比赛和社区活动上展示过，反响都不错。我希望将来有机会在更大的舞台上展示我的作品。\nAI: 你的梦想一定会实现的，孙悦。你的热情和才华值得被更多人看到。对了，你有没有考虑过将你的舞蹈作品拍成视频分享到网上？\n\n用户: 我有这个打算，不过我还在考虑如何拍摄和剪辑才能更好地展现舞蹈的魅力。\nAI: 网络平台是一个很好的展示自己才华的地方。如果你需要，我可以提供一些视频拍摄和剪辑方面的建议。\n\n用户: 那太好了，我会在准备的时候向你请教的。今天聊得很开心，但我现在得去准备明天的攀岩训练了。我们明天再聊！\nAI: 好的，孙悦。祝你明天的攀岩训练顺利。期待我们的下次聊天，晚安！\n”\n\n请记住以上全部对话记录，回答问题。\n问题：", "query": "我在4月27日和你聊到曾经旅行去过的城市，这个城市位于哪个国家？", "answer": "泰国", "id": "39cf6263-6a95-46bd-9b56-242bf9bc6496", "chatglm_length": 3881, "qwen_length": 3960, "internlm2_length": 3714} diff --git a/api/app/core/memory/dbrun.json b/api/app/core/memory/dbrun.json deleted file mode 100644 index c4220a55..00000000 --- a/api/app/core/memory/dbrun.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "selections": { - "config_id": "" - } -} \ No newline at end of file diff --git a/api/app/core/memory/main.py b/api/app/core/memory/main.py index 063dfaeb..68bb1de9 100644 --- a/api/app/core/memory/main.py +++ b/api/app/core/memory/main.py @@ -51,16 +51,31 @@ logger = get_memory_logger(__name__) async def main( + # Required configuration parameters (no longer from global variables) + chunker_strategy: str, + group_id: str, + user_id: str, + apply_id: str, + llm_model_id: str, + embedding_model_id: str, + # Optional parameters dialogue_text: Optional[str] = None, is_pilot_run: bool = False, progress_callback: Optional[Callable[[str, str, Optional[dict]], Awaitable[None]]] = None ): """ - 记忆系统主流程 - 重构版本 + 记忆系统主流程 - 重构版本 (Updated to eliminate global variables) 该函数是重构后的主入口，使用新的模块化架构。 + Global variables have been eliminated in favor of explicit parameters. Args: + chunker_strategy: Chunking strategy to use (required) + group_id: Group ID for the operation (required) + user_id: User ID for the operation (required) + apply_id: Application ID for the operation (required) + llm_model_id: LLM model ID to use (required) + embedding_model_id: Embedding model ID to use (required) dialogue_text: 输入的对话文本（可选，用于试运行模式） is_pilot_run: 是否为试运行模式 - True: 试运行模式，不保存到 Neo4j @@ -82,12 +97,10 @@ async def main( print("MemSci 知识提取流水线 - 重构版本") print("=" * 60) print(f"运行模式: {'试运行（不保存到Neo4j）' if is_pilot_run else '正常运行（保存到Neo4j）'}") - print("Using chunker strategy:", config_defs.SELECTED_CHUNKER_STRATEGY) - print("Using group ID:", config_defs.SELECTED_GROUP_ID) - print("Using model ID:", config_defs.SELECTED_LLM_ID) - print("Using embedding model ID:", config_defs.SELECTED_EMBEDDING_ID) - print("LANGFUSE_ENABLED:", config_defs.LANGFUSE_ENABLED) - print("AGENTA_ENABLED:", config_defs.AGENTA_ENABLED) + print("Using chunker strategy:", chunker_strategy) + print("Using group ID:", group_id) + print("Using model ID:", llm_model_id) + print("Using embedding model ID:", embedding_model_id) print("=" * 60) # 初始化日志 @@ -104,11 +117,11 @@ async def main( logger.info("Initializing clients...") step_start = time.time() - llm_client = get_llm_client(config_defs.SELECTED_LLM_ID) + llm_client = get_llm_client(llm_model_id) # 获取 embedder 配置并转换为 RedBearModelConfig 对象 from app.core.models.base import RedBearModelConfig - embedder_config_dict = get_embedder_config(config_defs.SELECTED_EMBEDDING_ID) + embedder_config_dict = get_embedder_config(embedding_model_id) embedder_config = RedBearModelConfig(**embedder_config_dict) embedder_client = OpenAIEmbedderClient(embedder_config) @@ -145,9 +158,9 @@ async def main( dialog = DialogData( context=context, ref_id="pilot_dialog_1", - group_id=config_defs.SELECTED_GROUP_ID, - user_id=config_defs.SELECTED_USER_ID, - apply_id=config_defs.SELECTED_APPLY_ID, + group_id=group_id, + user_id=user_id, + apply_id=apply_id, metadata={"source": "pilot_run", "input_type": "frontend_text"} ) @@ -158,7 +171,7 @@ async def main( # 对前端传入的对话进行分块处理 chunked_dialogs = await get_chunked_dialogs_from_preprocessed( data=[dialog], - chunker_strategy=config_defs.SELECTED_CHUNKER_STRATEGY, + chunker_strategy=chunker_strategy, llm_client=llm_client, ) logger.info(f"Processed frontend dialogue text: {len(messages)} messages") @@ -172,7 +185,7 @@ async def main( "content": chunk.content[:200] + "..." if len(chunk.content) > 200 else chunk.content, "full_length": len(chunk.content), "dialog_id": dialog.id, - "chunker_strategy": config_defs.SELECTED_CHUNKER_STRATEGY + "chunker_strategy": chunker_strategy } await progress_callback("text_preprocessing_result", f"分块 {i + 1} 处理完成", chunk_result) @@ -180,7 +193,7 @@ async def main( preprocessing_summary = { "total_chunks": sum(len(dialog.chunks) for dialog in chunked_dialogs), "total_dialogs": len(chunked_dialogs), - "chunker_strategy": config_defs.SELECTED_CHUNKER_STRATEGY + "chunker_strategy": chunker_strategy } await progress_callback("text_preprocessing_complete", "预处理文本完成", preprocessing_summary) else: @@ -199,11 +212,11 @@ async def main( await progress_callback("text_preprocessing", "开始预处理文本...") chunked_dialogs = await get_chunked_dialogs_with_preprocessing( - chunker_strategy=config_defs.SELECTED_CHUNKER_STRATEGY, - group_id=config_defs.SELECTED_GROUP_ID, - user_id=config_defs.SELECTED_USER_ID, - apply_id=config_defs.SELECTED_APPLY_ID, - indices=config_defs.SELECTED_TEST_DATA_INDICES, + chunker_strategy=chunker_strategy, + group_id=group_id, + user_id=user_id, + apply_id=apply_id, + indices=None, input_data_path=test_data_path, llm_client=llm_client, skip_cleaning=True, @@ -219,7 +232,7 @@ async def main( "content": chunk.content[:200] + "..." if len(chunk.content) > 200 else chunk.content, "full_length": len(chunk.content), "dialog_id": dialog.id, - "chunker_strategy": config_defs.SELECTED_CHUNKER_STRATEGY + "chunker_strategy": chunker_strategy } await progress_callback("text_preprocessing_result", f"分块 {i + 1} 处理完成", chunk_result) @@ -227,7 +240,7 @@ async def main( preprocessing_summary = { "total_chunks": sum(len(dialog.chunks) for dialog in chunked_dialogs), "total_dialogs": len(chunked_dialogs), - "chunker_strategy": config_defs.SELECTED_CHUNKER_STRATEGY + "chunker_strategy": chunker_strategy } await progress_callback("text_preprocessing_complete", "预处理文本完成", preprocessing_summary) @@ -249,6 +262,7 @@ async def main( connector=neo4j_connector, config=config, progress_callback=progress_callback, # 传递进度回调 + embedding_id=embedding_model_id, # 传递嵌入模型ID ) log_time("Orchestrator Initialization", time.time() - step_start, log_file) @@ -352,7 +366,7 @@ async def main( ) summaries = await Memory_summary_generation( - chunked_dialogs, llm_client=llm_client, embedding_id=config_defs.SELECTED_EMBEDDING_ID + chunked_dialogs, llm_client=llm_client, embedding_id=embedding_model_id ) if not is_pilot_run: @@ -400,4 +414,17 @@ async def main( if __name__ == "__main__": - asyncio.run(main()) + print("⚠️ Warning: This script now requires explicit configuration parameters.") + print("Global variables have been removed. Please provide configuration parameters.") + print("Example usage:") + print(" asyncio.run(main(") + print(" chunker_strategy='RecursiveChunker',") + print(" group_id='your_group_id',") + print(" user_id='your_user_id',") + print(" apply_id='your_apply_id',") + print(" llm_model_id='your_llm_id',") + print(" embedding_model_id='your_embedding_id'") + print(" ))") + + # This will fail because global variables are removed + raise RuntimeError("Global variables removed. Please provide explicit configuration parameters.") diff --git a/api/app/core/memory/src/search.py b/api/app/core/memory/src/search.py index 7deeb0f2..597a4789 100644 --- a/api/app/core/memory/src/search.py +++ b/api/app/core/memory/src/search.py @@ -1,31 +1,41 @@ import argparse import asyncio import json +import math import os import time -from typing import List, Dict, Any, Optional -from dotenv import load_dotenv from datetime import datetime -import math +from typing import Any, Dict, List, Optional + from app.core.logging_config import get_memory_logger -# 使用新的仓储层 -from app.repositories.neo4j.neo4j_connector import Neo4jConnector -from app.repositories.neo4j.graph_search import ( - search_graph_by_embedding, search_graph, - search_graph_by_temporal, search_graph_by_keyword_temporal, - search_graph_by_chunk_id -) from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient from app.core.memory.models.config_models import TemporalSearchParams -from app.core.memory.utils.config.config_utils import get_embedder_config, get_pipeline_config -from app.core.memory.utils.data.time_utils import normalize_date_safe from app.core.memory.models.variate_config import ForgettingEngineConfig -from app.core.memory.utils.config.definitions import CONFIG, RUNTIME_CONFIG -from app.core.memory.storage_services.forgetting_engine.forgetting_engine import ForgettingEngine -from app.core.memory.utils.data.text_utils import extract_plain_query +from app.core.memory.storage_services.forgetting_engine.forgetting_engine import ( + ForgettingEngine, +) from app.core.memory.utils.config import definitions as config_defs -from app.core.models.base import RedBearModelConfig +from app.core.memory.utils.config.config_utils import ( + get_embedder_config, + get_pipeline_config, +) +from app.core.memory.utils.config.definitions import CONFIG, RUNTIME_CONFIG +from app.core.memory.utils.data.text_utils import extract_plain_query +from app.core.memory.utils.data.time_utils import normalize_date_safe from app.core.memory.utils.llm.llm_utils import get_reranker_client +from app.core.models.base import RedBearModelConfig +from app.repositories.neo4j.graph_search import ( + search_graph, + search_graph_by_chunk_id, + search_graph_by_embedding, + search_graph_by_keyword_temporal, + search_graph_by_temporal, +) + +# 使用新的仓储层 +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from dotenv import load_dotenv + load_dotenv() logger = get_memory_logger(__name__) @@ -131,7 +141,7 @@ def rerank_hybrid_results( # Add keyword results with BM25 scores for item in keyword_items: - item_id = item.get("id") or item.get("uuid") + item_id = item.get("id") or item.get("uuid") or item.get("chunk_id") if item_id: combined_items[item_id] = item.copy() combined_items[item_id]["bm25_score"] = item.get("normalized_score", 0) @@ -139,7 +149,7 @@ def rerank_hybrid_results( # Add or update with embedding results for item in embedding_items: - item_id = item.get("id") or item.get("uuid") + item_id = item.get("id") or item.get("uuid") or item.get("chunk_id") if item_id: if item_id in combined_items: # Update existing item with embedding score @@ -220,7 +230,7 @@ def rerank_with_forgetting_curve( (keyword_items, False), (embedding_items, True) ): for item in src_items: - item_id = item.get("id") or item.get("uuid") + item_id = item.get("id") or item.get("uuid") or item.get("chunk_id") if not item_id: continue existing = combined_items.get(item_id) @@ -266,26 +276,25 @@ def rerank_with_forgetting_curve( return reranked -def log_search_query(query_text: str, search_type: str, group_id: str | None, limit: int, include: List[str], log_file: str = "search_log.txt"): - """Log search query information to file""" - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") +def log_search_query(query_text: str, search_type: str, group_id: str | None, limit: int, include: List[str], log_file: str = None): + """Log search query information using the logger. + + Args: + query_text: The search query text + search_type: Type of search (keyword, embedding, hybrid) + group_id: Group identifier for filtering + limit: Maximum number of results + include: List of result types to include + log_file: Deprecated parameter, kept for backward compatibility + """ # Ensure the query text is plain and clean before logging cleaned_query = extract_plain_query(query_text) - log_entry = { - "timestamp": timestamp, - # "query": query_text, - "query": cleaned_query, - "search_type": search_type, - "group_id": group_id, - "limit": limit, - "include": include - } - - # Append to log file - with open(log_file, "a", encoding="utf-8") as f: - f.write(json.dumps(log_entry, ensure_ascii=False) + "\n") - - logger.info(f"Search logged: {query_text} ({search_type})") + + # Log using the standard logger + logger.info( + f"Search query: query='{cleaned_query}', type={search_type}, " + f"group_id={group_id}, limit={limit}, include={include}" + ) def _remove_keys_recursive(obj: Any, keys_to_remove: List[str]) -> Any: @@ -547,6 +556,7 @@ async def run_hybrid_search( limit: int, include: List[str], output_path: str | None, + embedding_id: str, rerank_alpha: float = 0.6, use_forgetting_rerank: bool = False, use_llm_rerank: bool = False, @@ -558,6 +568,7 @@ async def run_hybrid_search( # Start overall timing search_start_time = time.time() latency_metrics = {} + logger.info(f"using embedding_id:{embedding_id}...") # Clean and normalize the incoming query before use/logging query_text = extract_plain_query(query_text) @@ -610,7 +621,7 @@ async def run_hybrid_search( # 从数据库读取嵌入器配置（按 ID）并构建 RedBearModelConfig config_load_start = time.time() - embedder_config_dict = get_embedder_config(config_defs.SELECTED_EMBEDDING_ID) + embedder_config_dict = get_embedder_config(embedding_id) rb_config = RedBearModelConfig( model_name=embedder_config_dict["model_name"], provider=embedder_config_dict["provider"], @@ -759,18 +770,11 @@ async def run_hybrid_search( else: result_counts = {key: len(value) if isinstance(value, list) else 0 for key, value in results.items()} - completion_log = { - "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - "query": query_text, - "search_type": search_type, - "status": "completed", - "result_counts": result_counts, - "output_file": output_path, - "latency_metrics": latency_metrics - } - - with open("search_log.txt", "a", encoding="utf-8") as f: - f.write(json.dumps(completion_log, ensure_ascii=False) + "\n") + # Log completion using the standard logger + logger.info( + f"Search completed: query='{query_text}', type={search_type}, " + f"result_counts={result_counts}, latency={latency_metrics}" + ) return results @@ -969,6 +973,7 @@ def main(): limit=args.limit, include=args.include, output_path=args.output, + embedding_id=config_defs.SELECTED_EMBEDDING_ID, rerank_alpha=args.rerank_alpha, use_forgetting_rerank=args.forgetting_rerank, use_llm_rerank=args.llm_rerank, diff --git a/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py b/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py index 7eec1189..9e365540 100644 --- a/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py +++ b/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py @@ -19,50 +19,50 @@ import asyncio import logging import os -from typing import List, Dict, Any, Tuple, Optional, Callable, Awaitable from datetime import datetime +from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple -from app.core.memory.models.message_models import DialogData +from app.core.memory.llm_tools.openai_client import LLMClient +from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient from app.core.memory.models.graph_models import ( - DialogueNode, ChunkNode, - StatementNode, + DialogueNode, + EntityEntityEdge, ExtractedEntityNode, StatementChunkEdge, StatementEntityEdge, - EntityEntityEdge, + StatementNode, ) -from app.core.memory.utils.data.ontology import TemporalInfo +from app.core.memory.models.message_models import DialogData from app.core.memory.models.variate_config import ( ExtractionPipelineConfig, StatementExtractionConfig, ) -from app.core.memory.llm_tools.openai_client import LLMClient -from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient -from app.repositories.neo4j.neo4j_connector import Neo4jConnector - -# 导入各个提取模块 -from app.core.memory.storage_services.extraction_engine.knowledge_extraction.statement_extraction import ( - StatementExtractor, -) -from app.core.memory.storage_services.extraction_engine.knowledge_extraction.triplet_extraction import ( - TripletExtractor, -) -from app.core.memory.storage_services.extraction_engine.knowledge_extraction.temporal_extraction import ( - TemporalExtractor, +from app.core.memory.storage_services.extraction_engine.deduplication.two_stage_dedup import ( + dedup_layers_and_merge_and_return, ) from app.core.memory.storage_services.extraction_engine.knowledge_extraction.embedding_generation import ( embedding_generation, embedding_generation_all, generate_entity_embeddings_from_triplets, ) -from app.core.memory.storage_services.extraction_engine.deduplication.two_stage_dedup import ( - dedup_layers_and_merge_and_return, + +# 导入各个提取模块 +from app.core.memory.storage_services.extraction_engine.knowledge_extraction.statement_extraction import ( + StatementExtractor, +) +from app.core.memory.storage_services.extraction_engine.knowledge_extraction.temporal_extraction import ( + TemporalExtractor, +) +from app.core.memory.storage_services.extraction_engine.knowledge_extraction.triplet_extraction import ( + TripletExtractor, ) from app.core.memory.storage_services.extraction_engine.pipeline_help import ( _write_extracted_result_summary, export_test_input_doc, ) +from app.core.memory.utils.data.ontology import TemporalInfo +from app.repositories.neo4j.neo4j_connector import Neo4jConnector # 配置日志 logger = logging.getLogger(__name__) @@ -96,6 +96,7 @@ class ExtractionOrchestrator: connector: Neo4jConnector, config: Optional[ExtractionPipelineConfig] = None, progress_callback: Optional[Callable[[str, str, Optional[Dict[str, Any]]], Awaitable[None]]] = None, + embedding_id: Optional[str] = None, ): """ 初始化流水线编排器 @@ -108,6 +109,7 @@ class ExtractionOrchestrator: progress_callback: 进度回调函数 - 接受 (stage: str, message: str, data: Optional[Dict[str, Any]]) 并返回 Awaitable[None] - 在管线关键点调用以报告进度和结果数据 + embedding_id: 嵌入模型ID，如果为 None 则从全局配置获取（向后兼容） """ self.llm_client = llm_client self.embedder_client = embedder_client @@ -115,6 +117,7 @@ class ExtractionOrchestrator: self.config = config or ExtractionPipelineConfig() self.is_pilot_run = False # 默认非试运行模式 self.progress_callback = progress_callback # 保存进度回调函数 + self.embedding_id = embedding_id # 保存嵌入模型ID # 保存去重消歧的详细记录（内存中的数据结构） self.dedup_merge_records: List[Dict[str, Any]] = [] # 实体合并记录 @@ -420,7 +423,9 @@ class ExtractionOrchestrator: return await self.triplet_extractor._extract_triplets(statement, chunk_content) except Exception as e: logger.error(f"陈述句 {statement.id} 三元组提取失败: {e}") - from app.core.memory.models.triplet_models import TripletExtractionResponse + from app.core.memory.models.triplet_models import ( + TripletExtractionResponse, + ) return TripletExtractionResponse(triplets=[], entities=[]) tasks = [extract_for_statement(stmt_data) for stmt_data in all_statements] @@ -434,7 +439,9 @@ class ExtractionOrchestrator: d_idx, stmt_id = statement_metadata[i] if isinstance(result, Exception): logger.error(f"陈述句处理异常: {result}") - from app.core.memory.models.triplet_models import TripletExtractionResponse + from app.core.memory.models.triplet_models import ( + TripletExtractionResponse, + ) triplet_maps[d_idx][stmt_id] = TripletExtractionResponse(triplets=[], entities=[]) else: triplet_maps[d_idx][stmt_id] = result @@ -521,8 +528,8 @@ class ExtractionOrchestrator: temporal_maps[d_idx][stmt_id] = result # 为 ATEMPORAL 陈述句添加空的时间范围 - from app.core.memory.utils.data.ontology import TemporalInfo from app.core.memory.models.message_models import TemporalValidityRange + from app.core.memory.utils.data.ontology import TemporalInfo for d_idx, dialog in enumerate(dialog_data_list): for chunk in dialog.chunks: for statement in chunk.statements: @@ -629,17 +636,14 @@ class ExtractionOrchestrator: logger.info("开始生成基础嵌入向量（陈述句、分块、对话）") try: - # 从 runtime.json 获取嵌入模型配置ID - from app.core.memory.utils.config import definitions as config_defs - embedding_id = config_defs.SELECTED_EMBEDDING_ID - - if not embedding_id: - logger.error("未在 runtime.json 中配置 embedding 模型 ID") - raise ValueError("未配置嵌入模型ID") + # embedding_id is required - no fallback to global variable + if not self.embedding_id: + logger.error("embedding_id is required but was not provided to ExtractionOrchestrator") + raise ValueError("embedding_id is required but was not provided") # 只生成陈述句、分块和对话的嵌入（不包括实体） statement_embedding_maps, chunk_embedding_maps, dialog_embeddings = await embedding_generation( - dialog_data_list, embedding_id + dialog_data_list, self.embedding_id ) # 统计生成结果 @@ -683,17 +687,14 @@ class ExtractionOrchestrator: logger.info("开始生成实体嵌入向量") try: - # 从 runtime.json 获取嵌入模型配置ID - from app.core.memory.utils.config import definitions as config_defs - embedding_id = config_defs.SELECTED_EMBEDDING_ID - - if not embedding_id: - logger.error("未在 runtime.json 中配置 embedding 模型 ID") + # embedding_id is required - no fallback to global variable + if not self.embedding_id: + logger.error("embedding_id is required but was not provided to ExtractionOrchestrator") return triplet_maps # 生成实体嵌入 updated_triplet_maps = await generate_entity_embeddings_from_triplets( - triplet_maps, embedding_id + triplet_maps, self.embedding_id ) logger.info("实体嵌入生成完成") @@ -1086,7 +1087,9 @@ class ExtractionOrchestrator: if self.is_pilot_run: logger.info("试运行模式：仅执行第一层去重，跳过第二层数据库去重") # 只执行第一层去重 - from app.core.memory.storage_services.extraction_engine.deduplication.deduped_and_disamb import deduplicate_entities_and_edges + from app.core.memory.storage_services.extraction_engine.deduplication.deduped_and_disamb import ( + deduplicate_entities_and_edges, + ) dedup_entity_nodes, dedup_statement_entity_edges, dedup_entity_entity_edges, dedup_details = await deduplicate_entities_and_edges( entity_nodes, @@ -1608,8 +1611,8 @@ async def get_chunked_dialogs( 包含分块的 DialogData 对象列表 """ import json - import re import os + import re # 加载测试数据 testdata_path = os.path.join(os.path.dirname(__file__), "../../data", "testdata.json") @@ -1671,7 +1674,9 @@ async def get_chunked_dialogs( ) # 创建分块器并处理对话 - from app.core.memory.storage_services.extraction_engine.knowledge_extraction.chunk_extraction import DialogueChunker + from app.core.memory.storage_services.extraction_engine.knowledge_extraction.chunk_extraction import ( + DialogueChunker, + ) chunker = DialogueChunker(chunker_strategy) extracted_chunks = await chunker.process_dialogue(dialog_data) dialog_data.chunks = extracted_chunks @@ -1718,7 +1723,9 @@ def preprocess_data( 经过清洗转换后的 DialogData 列表 """ print("\n=== 数据预处理 ===") - from app.core.memory.storage_services.extraction_engine.data_preprocessing.data_preprocessor import DataPreprocessor + from app.core.memory.storage_services.extraction_engine.data_preprocessing.data_preprocessor import ( + DataPreprocessor, + ) preprocessor = DataPreprocessor() try: cleaned_data = preprocessor.preprocess(input_path=input_path, output_path=output_path, skip_cleaning=skip_cleaning, indices=indices) @@ -1749,7 +1756,9 @@ async def get_chunked_dialogs_from_preprocessed( raise ValueError("预处理数据为空，无法进行分块") all_chunked_dialogs: List[DialogData] = [] - from app.core.memory.storage_services.extraction_engine.knowledge_extraction.chunk_extraction import DialogueChunker + from app.core.memory.storage_services.extraction_engine.knowledge_extraction.chunk_extraction import ( + DialogueChunker, + ) for dialog_data in data: chunker = DialogueChunker(chunker_strategy, llm_client=llm_client) @@ -1811,7 +1820,9 @@ async def get_chunked_dialogs_with_preprocessing( # 步骤2: 语义剪枝 try: - from app.core.memory.storage_services.extraction_engine.data_preprocessing.data_pruning import SemanticPruner + from app.core.memory.storage_services.extraction_engine.data_preprocessing.data_pruning import ( + SemanticPruner, + ) pruner = SemanticPruner(llm_client=llm_client) # 记录单对话场景下剪枝前的消息数量 @@ -1834,7 +1845,9 @@ async def get_chunked_dialogs_with_preprocessing( # 保存剪枝后的数据 try: - from app.core.memory.storage_services.extraction_engine.data_preprocessing.data_preprocessor import DataPreprocessor + from app.core.memory.storage_services.extraction_engine.data_preprocessing.data_preprocessor import ( + DataPreprocessor, + ) pruned_output_path = settings.get_memory_output_path("pruned_data.json") dp = DataPreprocessor(output_file_path=pruned_output_path) dp.save_data(preprocessed_data, output_path=pruned_output_path) diff --git a/api/app/core/memory/storage_services/search/hybrid_chatbot.py b/api/app/core/memory/storage_services/search/hybrid_chatbot.py deleted file mode 100644 index 40a65263..00000000 --- a/api/app/core/memory/storage_services/search/hybrid_chatbot.py +++ /dev/null @@ -1,447 +0,0 @@ - -# TODO hybrid_chatbot.py 是一个独立的GUI演示应用，不是核心功能的一部分，可以考虑删除 -from app.core.memory.utils.llm.llm_utils import get_llm_client -import asyncio -import os -import time -import json -from datetime import datetime, timezone -import tkinter as tk -from tkinter import scrolledtext, messagebox -import threading -from typing import Any, Dict, Tuple, List - -# Import our hybrid search functionality -from app.core.memory.storage_services.search import run_hybrid_search -# 使用新的仓储层 -from app.repositories.neo4j.neo4j_connector import Neo4jConnector -from app.core.memory.llm_tools.openai_client import OpenAIClient -from app.core.memory.models.config_models import LLMConfig -from dotenv import load_dotenv - -load_dotenv() - - -class HybridSearchChatbot: - def __init__(self): - - from app.core.memory.utils.config import definitions as config_defs - self.llm_client = get_llm_client(config_defs.SELECTED_LLM_ID) - - # Chat history - self.chat_history = [] - - # Search configuration - self.search_config = { - "group_id": "group_wyl_25", - "limit": 10, - "include": ["statements", "chunks", "entities","summaries"], - # "include": ["statements", "dialogues", "entities"], - "rerank_alpha": 0.6 - } - - # Setup GUI - self.setup_gui() - - def setup_gui(self): - """Setup the GUI interface""" - self.root = tk.Tk() - self.root.title("Hybrid Search Chatbot") - self.root.geometry("800x600") - - # Chat display area - self.chat_display = scrolledtext.ScrolledText( - self.root, - wrap=tk.WORD, - width=80, - height=25, - state=tk.DISABLED - ) - self.chat_display.pack(padx=10, pady=10, fill=tk.BOTH, expand=True) - - # Input frame - input_frame = tk.Frame(self.root) - input_frame.pack(padx=10, pady=5, fill=tk.X) - - # User input - self.user_input = tk.Entry(input_frame, font=("Arial", 12)) - self.user_input.pack(side=tk.LEFT, fill=tk.X, expand=True, padx=(0, 5)) - self.user_input.bind("", self.on_send_message) - - # Send button - self.send_button = tk.Button( - input_frame, - text="发送", - command=self.on_send_message, - font=("Arial", 12) - ) - self.send_button.pack(side=tk.RIGHT) - - # Status frame - status_frame = tk.Frame(self.root) - status_frame.pack(padx=10, pady=5, fill=tk.X) - - # Status label - self.status_label = tk.Label( - status_frame, - text="就绪", - font=("Arial", 10), - anchor="w" - ) - self.status_label.pack(side=tk.LEFT, fill=tk.X, expand=True) - - # Search config button - config_button = tk.Button( - status_frame, - text="搜索配置", - command=self.show_config_dialog, - font=("Arial", 10) - ) - config_button.pack(side=tk.RIGHT) - - # Add welcome message - self.add_message("系统", "欢迎使用混合搜索聊天机器人！我可以基于知识图谱中的信息回答您的问题。") - - def add_message(self, sender: str, message: str, metadata: Dict = None): - """Add a message to the chat display""" - self.chat_display.config(state=tk.NORMAL) - - timestamp = datetime.now().strftime("%H:%M:%S") - - # Add sender and timestamp - self.chat_display.insert(tk.END, f"[{timestamp}] {sender}:\n", "sender") - - # Add message content - self.chat_display.insert(tk.END, f"{message}\n", "message") - - # Add metadata if available - if metadata: - self.chat_display.insert(tk.END, f" {metadata}\n", "metadata") - - self.chat_display.insert(tk.END, "\n") - self.chat_display.config(state=tk.DISABLED) - self.chat_display.see(tk.END) - - # Configure text tags for styling - self.chat_display.tag_config("sender", foreground="blue", font=("Arial", 10, "bold")) - self.chat_display.tag_config("message", foreground="black", font=("Arial", 10)) - self.chat_display.tag_config("metadata", foreground="gray", font=("Arial", 8)) - - def show_config_dialog(self): - """Show search configuration dialog""" - config_window = tk.Toplevel(self.root) - config_window.title("搜索配置") - config_window.geometry("400x600") - config_window.transient(self.root) - config_window.grab_set() - - # Current configuration display - current_config_frame = tk.Frame(config_window) - current_config_frame.pack(pady=10, padx=10, fill=tk.X) - tk.Label(current_config_frame, text="当前配置:", font=("Arial", 10, "bold")).pack(anchor="w") - current_text = f"Alpha: {self.search_config['rerank_alpha']}, 限制: {self.search_config['limit']}, 目标: {', '.join(self.search_config['include'])}" - tk.Label(current_config_frame, text=current_text, font=("Arial", 9), fg="blue").pack(anchor="w") - - # Alpha parameter - tk.Label(config_window, text="重排权重 (Alpha):").pack(pady=(10, 5)) - alpha_var = tk.DoubleVar(value=self.search_config["rerank_alpha"]) - alpha_scale = tk.Scale( - config_window, - from_=0.0, - to=1.0, - resolution=0.1, - orient=tk.HORIZONTAL, - variable=alpha_var - ) - alpha_scale.pack(pady=5, padx=20, fill=tk.X) - tk.Label(config_window, text="0.0=纯语义搜索, 1.0=纯关键词搜索", font=("Arial", 8)).pack() - - # Limit parameter - tk.Label(config_window, text="搜索结果数量:").pack(pady=(20, 5)) - limit_var = tk.IntVar(value=self.search_config["limit"]) - limit_spinbox = tk.Spinbox( - config_window, - from_=1, - to=50, - textvariable=limit_var, - width=10 - ) - limit_spinbox.pack(pady=5) - - # Include options - tk.Label(config_window, text="搜索目标:").pack(pady=(20, 5)) - include_frame = tk.Frame(config_window) - include_frame.pack(pady=5) - - include_vars = {} - for option in ["statements", "chunks", "entities","summaries"]: - var = tk.BooleanVar(value=option in self.search_config["include"]) - include_vars[option] = var - tk.Checkbutton( - include_frame, - text=option, - variable=var - ).pack(side=tk.LEFT, padx=10) - - # Buttons - button_frame = tk.Frame(config_window) - button_frame.pack(pady=20) - - def save_config(): - try: - # Validate inputs - alpha_value = alpha_var.get() - limit_value = limit_var.get() - include_list = [ - option for option, var in include_vars.items() if var.get() - ] - - # Check if at least one search target is selected - if not include_list: - messagebox.showerror("配置错误", "请至少选择一个搜索目标！") - return - - # Update configuration - self.search_config["rerank_alpha"] = alpha_value - self.search_config["limit"] = limit_value - self.search_config["include"] = include_list - - config_window.destroy() - self.add_message("系统", - f"配置已更新: Alpha={alpha_value:.1f}, 限制={limit_value}, 目标={', '.join(include_list)}") - - except Exception as e: - messagebox.showerror("配置错误", f"保存配置时出错: {str(e)}") - print(f"Config save error: {e}") # Debug output - - tk.Button(button_frame, text="保存", command=save_config).pack(side=tk.LEFT, padx=5) - tk.Button(button_frame, text="取消", command=config_window.destroy).pack(side=tk.LEFT, padx=5) - - def on_send_message(self, event=None): - """Handle sending a message""" - user_message = self.user_input.get().strip() - if not user_message: - return - - # Clear input - self.user_input.delete(0, tk.END) - - # Add user message to display - self.add_message("用户", user_message) - - # Disable send button and show processing status - self.send_button.config(state=tk.DISABLED) - self.status_label.config(text="正在搜索和生成回复...") - - # Process message in background thread - threading.Thread( - target=self.process_message_async, - args=(user_message,), - daemon=True - ).start() - - def process_message_async(self, user_message: str): - """Process message asynchronously""" - try: - # Run the async processing - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - response, metadata = loop.run_until_complete( - self.process_message(user_message) - ) - loop.close() - - # Update GUI in main thread - self.root.after(0, self.on_response_ready, response, metadata) - - except Exception as e: - error_msg = f"处理消息时出错: {str(e)}" - self.root.after(0, self.on_error, error_msg) - - async def process_message(self, user_message: str) -> Tuple[str, Dict[str, Any]]: - """Process user message with hybrid search""" - start_time = time.time() - - # Perform hybrid search - search_start = time.time() - search_results = await run_hybrid_search( - query_text=user_message, - search_type="hybrid", - group_id=self.search_config["group_id"], - limit=self.search_config["limit"], - include=self.search_config["include"], - output_path=None, - rerank_alpha=self.search_config["rerank_alpha"] - ) - search_time = time.time() - search_start - - # Extract relevant information from search results - context_info = self.extract_context_from_search(search_results) - - # Generate response using LLM - llm_start = time.time() - response = await self.generate_response(user_message, context_info) - llm_time = time.time() - llm_start - - total_time = time.time() - start_time - - # Prepare metadata - metadata = { - "搜索时间": f"{search_time:.2f}s", - "生成时间": f"{llm_time:.2f}s", - "总时间": f"{total_time:.2f}s", - "搜索结果": self.get_search_summary(search_results), - "重排权重": self.search_config["rerank_alpha"] - } - - return response, metadata - - def extract_context_from_search(self, search_results: Dict) -> str: - """Extract context information from search results""" - if not search_results: - return "未找到相关信息。" - - context_parts = [] - - # Get reranked results if available, otherwise use individual results - if "reranked_results" in search_results: - results = search_results["reranked_results"] - else: - results = {} - for key in ["keyword_search", "embedding_search"]: - if key in search_results: - for category, items in search_results[key].items(): - if category not in results: - results[category] = [] - results[category].extend(items) - - # Extract statements - if "statements" in results and results["statements"]: - statements = results["statements"][:5] # Top 5 - context_parts.append("相关陈述:") - for i, stmt in enumerate(statements, 1): - content = stmt.get("statement", "") - score = stmt.get("combined_score", stmt.get("score", 0)) - context_parts.append(f"{i}. {content} (相关度: {score:.3f})") - - # Extract chunks - if "chunks" in results and results["chunks"]: - chunks = results["chunks"][:3] # Top 3 - context_parts.append("\n相关对话:") - for i, chunk in enumerate(chunks, 1): - content = chunk.get("content", "") - score = chunk.get("combined_score", chunk.get("score", 0)) - context_parts.append(f"{i}. {content} (相关度: {score:.3f})") - - # Extract entities - if "entities" in results and results["entities"]: - entities = results["entities"][:5] # Top 5 - context_parts.append("\n相关实体:") - entity_names = [ent.get("name", "") for ent in entities] - context_parts.append(", ".join(entity_names)) - - return "\n".join(context_parts) if context_parts else "未找到相关信息。" - - def get_search_summary(self, search_results: Dict) -> str: - """Get a summary of search results""" - if not search_results: - return "无结果" - - summary_parts = [] - - if "combined_summary" in search_results: - summary = search_results["combined_summary"] - if "total_reranked_results" in summary: - summary_parts.append(f"重排结果: {summary['total_reranked_results']}") - if "total_keyword_results" in summary: - summary_parts.append(f"关键词: {summary['total_keyword_results']}") - if "total_embedding_results" in summary: - summary_parts.append(f"语义: {summary['total_embedding_results']}") - - return ", ".join(summary_parts) if summary_parts else "有结果" - - async def generate_response(self, user_message: str, context: str) -> str: - """Generate response using LLM""" - system_prompt = f"""你是一个智能助手，基于知识图谱中的信息回答用户问题。 - -以下是从知识图谱中检索到的相关信息： -{context} - -请基于这些信息回答用户的问题。如果信息不足，请诚实地说明。回答要自然、友好，并且准确。""" - - try: - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_message} - ] - - response = self.llm_client.chat( - messages=messages, - ) - print(response) - # Extract content from various possible response types - # 1) LangChain AIMessage or similar object with `.content` - if hasattr(response, 'content'): - return getattr(response, 'content') - - # 2) OpenAI-style response with `.choices` - if hasattr(response, 'choices') and response.choices: - first_choice = response.choices[0] - # Newer clients may have `.message.content`, some have `.content` directly - if hasattr(first_choice, 'message') and hasattr(first_choice.message, 'content'): - return first_choice.message.content - if hasattr(first_choice, 'content'): - return first_choice.content - - # 3) Dict-like responses - if isinstance(response, dict): - if 'content' in response: - return response['content'] - if 'choices' in response and response['choices']: - ch = response['choices'][0] - if isinstance(ch, dict): - if 'message' in ch and 'content' in ch['message']: - return ch['message']['content'] - if 'content' in ch: - return ch['content'] - - # 4) Fallback: if it's a plain string - if isinstance(response, str): - return response - - # Default fallback - return "抱歉，我无法生成回复。" - - except Exception as e: - return f"生成回复时出错: {str(e)}" - - def on_response_ready(self, response: str, metadata: Dict[str, Any]): - """Handle when response is ready""" - self.add_message("助手", response, metadata) - self.send_button.config(state=tk.NORMAL) - self.status_label.config(text="就绪") - self.user_input.focus() - - def on_error(self, error_message: str): - """Handle errors""" - self.add_message("系统", f" {error_message}") - self.send_button.config(state=tk.NORMAL) - self.status_label.config(text="就绪") - self.user_input.focus() - - def run(self): - """Start the chatbot""" - self.root.mainloop() - - -def main(): - """Main function to run the chatbot""" - try: - chatbot = HybridSearchChatbot() - chatbot.run() - except Exception as e: - print(f"启动聊天机器人时出错: {e}") - - -if __name__ == "__main__": - main() diff --git a/api/app/core/memory/storage_services/search/hybrid_search.py b/api/app/core/memory/storage_services/search/hybrid_search.py index 1c834947..43215df5 100644 --- a/api/app/core/memory/storage_services/search/hybrid_search.py +++ b/api/app/core/memory/storage_services/search/hybrid_search.py @@ -1,408 +1,408 @@ -# -*- coding: utf-8 -*- -"""混合搜索策略 +# # -*- coding: utf-8 -*- +# """混合搜索策略 -结合关键词搜索和语义搜索的混合检索方法。 -支持结果重排序和遗忘曲线加权。 -""" +# 结合关键词搜索和语义搜索的混合检索方法。 +# 支持结果重排序和遗忘曲线加权。 +# """ -from typing import List, Dict, Any, Optional -import math -from datetime import datetime -from app.core.logging_config import get_memory_logger -from app.repositories.neo4j.neo4j_connector import Neo4jConnector -from app.core.memory.storage_services.search.search_strategy import SearchStrategy, SearchResult -from app.core.memory.storage_services.search.keyword_search import KeywordSearchStrategy -from app.core.memory.storage_services.search.semantic_search import SemanticSearchStrategy -from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient -from app.core.memory.models.variate_config import ForgettingEngineConfig -from app.core.memory.storage_services.forgetting_engine.forgetting_engine import ForgettingEngine +# from typing import List, Dict, Any, Optional +# import math +# from datetime import datetime +# from app.core.logging_config import get_memory_logger +# from app.repositories.neo4j.neo4j_connector import Neo4jConnector +# from app.core.memory.storage_services.search.search_strategy import SearchStrategy, SearchResult +# from app.core.memory.storage_services.search.keyword_search import KeywordSearchStrategy +# from app.core.memory.storage_services.search.semantic_search import SemanticSearchStrategy +# from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient +# from app.core.memory.models.variate_config import ForgettingEngineConfig +# from app.core.memory.storage_services.forgetting_engine.forgetting_engine import ForgettingEngine -logger = get_memory_logger(__name__) +# logger = get_memory_logger(__name__) -class HybridSearchStrategy(SearchStrategy): - """混合搜索策略 +# class HybridSearchStrategy(SearchStrategy): +# """混合搜索策略 - 结合关键词搜索和语义搜索的优势： - - 关键词搜索：精确匹配，适合已知术语 - - 语义搜索：语义理解，适合概念查询 - - 混合重排序：综合两种搜索的结果 - - 遗忘曲线：根据时间衰减调整相关性 - """ +# 结合关键词搜索和语义搜索的优势： +# - 关键词搜索：精确匹配，适合已知术语 +# - 语义搜索：语义理解，适合概念查询 +# - 混合重排序：综合两种搜索的结果 +# - 遗忘曲线：根据时间衰减调整相关性 +# """ - def __init__( - self, - connector: Optional[Neo4jConnector] = None, - embedder_client: Optional[OpenAIEmbedderClient] = None, - alpha: float = 0.6, - use_forgetting_curve: bool = False, - forgetting_config: Optional[ForgettingEngineConfig] = None - ): - """初始化混合搜索策略 +# def __init__( +# self, +# connector: Optional[Neo4jConnector] = None, +# embedder_client: Optional[OpenAIEmbedderClient] = None, +# alpha: float = 0.6, +# use_forgetting_curve: bool = False, +# forgetting_config: Optional[ForgettingEngineConfig] = None +# ): +# """初始化混合搜索策略 - Args: - connector: Neo4j连接器 - embedder_client: 嵌入模型客户端 - alpha: BM25分数权重（0.0-1.0），1-alpha为嵌入分数权重 - use_forgetting_curve: 是否使用遗忘曲线 - forgetting_config: 遗忘引擎配置 - """ - self.connector = connector - self.embedder_client = embedder_client - self.alpha = alpha - self.use_forgetting_curve = use_forgetting_curve - self.forgetting_config = forgetting_config or ForgettingEngineConfig() - self._owns_connector = connector is None +# Args: +# connector: Neo4j连接器 +# embedder_client: 嵌入模型客户端 +# alpha: BM25分数权重（0.0-1.0），1-alpha为嵌入分数权重 +# use_forgetting_curve: 是否使用遗忘曲线 +# forgetting_config: 遗忘引擎配置 +# """ +# self.connector = connector +# self.embedder_client = embedder_client +# self.alpha = alpha +# self.use_forgetting_curve = use_forgetting_curve +# self.forgetting_config = forgetting_config or ForgettingEngineConfig() +# self._owns_connector = connector is None - # 创建子策略 - self.keyword_strategy = KeywordSearchStrategy(connector=connector) - self.semantic_strategy = SemanticSearchStrategy( - connector=connector, - embedder_client=embedder_client - ) +# # 创建子策略 +# self.keyword_strategy = KeywordSearchStrategy(connector=connector) +# self.semantic_strategy = SemanticSearchStrategy( +# connector=connector, +# embedder_client=embedder_client +# ) - async def __aenter__(self): - """异步上下文管理器入口""" - if self._owns_connector: - self.connector = Neo4jConnector() - self.keyword_strategy.connector = self.connector - self.semantic_strategy.connector = self.connector - return self +# async def __aenter__(self): +# """异步上下文管理器入口""" +# if self._owns_connector: +# self.connector = Neo4jConnector() +# self.keyword_strategy.connector = self.connector +# self.semantic_strategy.connector = self.connector +# return self - async def __aexit__(self, exc_type, exc_val, exc_tb): - """异步上下文管理器出口""" - if self._owns_connector and self.connector: - await self.connector.close() +# async def __aexit__(self, exc_type, exc_val, exc_tb): +# """异步上下文管理器出口""" +# if self._owns_connector and self.connector: +# await self.connector.close() - async def search( - self, - query_text: str, - group_id: Optional[str] = None, - limit: int = 50, - include: Optional[List[str]] = None, - **kwargs - ) -> SearchResult: - """执行混合搜索 +# async def search( +# self, +# query_text: str, +# group_id: Optional[str] = None, +# limit: int = 50, +# include: Optional[List[str]] = None, +# **kwargs +# ) -> SearchResult: +# """执行混合搜索 - Args: - query_text: 查询文本 - group_id: 可选的组ID过滤 - limit: 每个类别的最大结果数 - include: 要包含的搜索类别列表 - **kwargs: 其他搜索参数（如alpha, use_forgetting_curve） +# Args: +# query_text: 查询文本 +# group_id: 可选的组ID过滤 +# limit: 每个类别的最大结果数 +# include: 要包含的搜索类别列表 +# **kwargs: 其他搜索参数（如alpha, use_forgetting_curve） - Returns: - SearchResult: 搜索结果对象 - """ - logger.info(f"执行混合搜索: query='{query_text}', group_id={group_id}, limit={limit}") +# Returns: +# SearchResult: 搜索结果对象 +# """ +# logger.info(f"执行混合搜索: query='{query_text}', group_id={group_id}, limit={limit}") - # 从kwargs中获取参数 - alpha = kwargs.get("alpha", self.alpha) - use_forgetting = kwargs.get("use_forgetting_curve", self.use_forgetting_curve) +# # 从kwargs中获取参数 +# alpha = kwargs.get("alpha", self.alpha) +# use_forgetting = kwargs.get("use_forgetting_curve", self.use_forgetting_curve) - # 获取有效的搜索类别 - include_list = self._get_include_list(include) +# # 获取有效的搜索类别 +# include_list = self._get_include_list(include) - try: - # 并行执行关键词搜索和语义搜索 - keyword_result = await self.keyword_strategy.search( - query_text=query_text, - group_id=group_id, - limit=limit, - include=include_list - ) +# try: +# # 并行执行关键词搜索和语义搜索 +# keyword_result = await self.keyword_strategy.search( +# query_text=query_text, +# group_id=group_id, +# limit=limit, +# include=include_list +# ) - semantic_result = await self.semantic_strategy.search( - query_text=query_text, - group_id=group_id, - limit=limit, - include=include_list - ) +# semantic_result = await self.semantic_strategy.search( +# query_text=query_text, +# group_id=group_id, +# limit=limit, +# include=include_list +# ) - # 重排序结果 - if use_forgetting: - reranked_results = self._rerank_with_forgetting_curve( - keyword_result=keyword_result, - semantic_result=semantic_result, - alpha=alpha, - limit=limit - ) - else: - reranked_results = self._rerank_hybrid_results( - keyword_result=keyword_result, - semantic_result=semantic_result, - alpha=alpha, - limit=limit - ) +# # 重排序结果 +# if use_forgetting: +# reranked_results = self._rerank_with_forgetting_curve( +# keyword_result=keyword_result, +# semantic_result=semantic_result, +# alpha=alpha, +# limit=limit +# ) +# else: +# reranked_results = self._rerank_hybrid_results( +# keyword_result=keyword_result, +# semantic_result=semantic_result, +# alpha=alpha, +# limit=limit +# ) - # 创建元数据 - metadata = self._create_metadata( - query_text=query_text, - search_type="hybrid", - group_id=group_id, - limit=limit, - include=include_list, - alpha=alpha, - use_forgetting_curve=use_forgetting - ) +# # 创建元数据 +# metadata = self._create_metadata( +# query_text=query_text, +# search_type="hybrid", +# group_id=group_id, +# limit=limit, +# include=include_list, +# alpha=alpha, +# use_forgetting_curve=use_forgetting +# ) - # 添加结果统计 - metadata["keyword_results"] = keyword_result.metadata.get("result_counts", {}) - metadata["semantic_results"] = semantic_result.metadata.get("result_counts", {}) - metadata["total_keyword_results"] = keyword_result.total_results() - metadata["total_semantic_results"] = semantic_result.total_results() - metadata["total_reranked_results"] = reranked_results.total_results() +# # 添加结果统计 +# metadata["keyword_results"] = keyword_result.metadata.get("result_counts", {}) +# metadata["semantic_results"] = semantic_result.metadata.get("result_counts", {}) +# metadata["total_keyword_results"] = keyword_result.total_results() +# metadata["total_semantic_results"] = semantic_result.total_results() +# metadata["total_reranked_results"] = reranked_results.total_results() - reranked_results.metadata = metadata +# reranked_results.metadata = metadata - logger.info(f"混合搜索完成: 共找到 {reranked_results.total_results()} 条结果") - return reranked_results +# logger.info(f"混合搜索完成: 共找到 {reranked_results.total_results()} 条结果") +# return reranked_results - except Exception as e: - logger.error(f"混合搜索失败: {e}", exc_info=True) - # 返回空结果但包含错误信息 - return SearchResult( - metadata=self._create_metadata( - query_text=query_text, - search_type="hybrid", - group_id=group_id, - limit=limit, - error=str(e) - ) - ) +# except Exception as e: +# logger.error(f"混合搜索失败: {e}", exc_info=True) +# # 返回空结果但包含错误信息 +# return SearchResult( +# metadata=self._create_metadata( +# query_text=query_text, +# search_type="hybrid", +# group_id=group_id, +# limit=limit, +# error=str(e) +# ) +# ) - def _normalize_scores( - self, - results: List[Dict[str, Any]], - score_field: str = "score" - ) -> List[Dict[str, Any]]: - """使用z-score标准化和sigmoid转换归一化分数 +# def _normalize_scores( +# self, +# results: List[Dict[str, Any]], +# score_field: str = "score" +# ) -> List[Dict[str, Any]]: +# """使用z-score标准化和sigmoid转换归一化分数 - Args: - results: 结果列表 - score_field: 分数字段名 +# Args: +# results: 结果列表 +# score_field: 分数字段名 - Returns: - List[Dict[str, Any]]: 归一化后的结果列表 - """ - if not results: - return results +# Returns: +# List[Dict[str, Any]]: 归一化后的结果列表 +# """ +# if not results: +# return results - # 提取分数 - scores = [] - for item in results: - if score_field in item: - score = item.get(score_field) - if score is not None and isinstance(score, (int, float)): - scores.append(float(score)) - else: - scores.append(0.0) +# # 提取分数 +# scores = [] +# for item in results: +# if score_field in item: +# score = item.get(score_field) +# if score is not None and isinstance(score, (int, float)): +# scores.append(float(score)) +# else: +# scores.append(0.0) - if not scores or len(scores) == 1: - # 单个分数或无分数，设置为1.0 - for item in results: - if score_field in item: - item[f"normalized_{score_field}"] = 1.0 - return results +# if not scores or len(scores) == 1: +# # 单个分数或无分数，设置为1.0 +# for item in results: +# if score_field in item: +# item[f"normalized_{score_field}"] = 1.0 +# return results - # 计算均值和标准差 - mean_score = sum(scores) / len(scores) - variance = sum((score - mean_score) ** 2 for score in scores) / len(scores) - std_dev = math.sqrt(variance) +# # 计算均值和标准差 +# mean_score = sum(scores) / len(scores) +# variance = sum((score - mean_score) ** 2 for score in scores) / len(scores) +# std_dev = math.sqrt(variance) - if std_dev == 0: - # 所有分数相同，设置为1.0 - for item in results: - if score_field in item: - item[f"normalized_{score_field}"] = 1.0 - else: - # z-score标准化 + sigmoid转换 - for item in results: - if score_field in item: - score = item[score_field] - if score is None or not isinstance(score, (int, float)): - score = 0.0 - z_score = (score - mean_score) / std_dev - normalized = 1 / (1 + math.exp(-z_score)) - item[f"normalized_{score_field}"] = normalized +# if std_dev == 0: +# # 所有分数相同，设置为1.0 +# for item in results: +# if score_field in item: +# item[f"normalized_{score_field}"] = 1.0 +# else: +# # z-score标准化 + sigmoid转换 +# for item in results: +# if score_field in item: +# score = item[score_field] +# if score is None or not isinstance(score, (int, float)): +# score = 0.0 +# z_score = (score - mean_score) / std_dev +# normalized = 1 / (1 + math.exp(-z_score)) +# item[f"normalized_{score_field}"] = normalized - return results +# return results - def _rerank_hybrid_results( - self, - keyword_result: SearchResult, - semantic_result: SearchResult, - alpha: float, - limit: int - ) -> SearchResult: - """重排序混合搜索结果 +# def _rerank_hybrid_results( +# self, +# keyword_result: SearchResult, +# semantic_result: SearchResult, +# alpha: float, +# limit: int +# ) -> SearchResult: +# """重排序混合搜索结果 - Args: - keyword_result: 关键词搜索结果 - semantic_result: 语义搜索结果 - alpha: BM25分数权重 - limit: 结果限制 +# Args: +# keyword_result: 关键词搜索结果 +# semantic_result: 语义搜索结果 +# alpha: BM25分数权重 +# limit: 结果限制 - Returns: - SearchResult: 重排序后的结果 - """ - reranked_data = {} +# Returns: +# SearchResult: 重排序后的结果 +# """ +# reranked_data = {} - for category in ["statements", "chunks", "entities", "summaries"]: - keyword_items = getattr(keyword_result, category, []) - semantic_items = getattr(semantic_result, category, []) +# for category in ["statements", "chunks", "entities", "summaries"]: +# keyword_items = getattr(keyword_result, category, []) +# semantic_items = getattr(semantic_result, category, []) - # 归一化分数 - keyword_items = self._normalize_scores(keyword_items, "score") - semantic_items = self._normalize_scores(semantic_items, "score") +# # 归一化分数 +# keyword_items = self._normalize_scores(keyword_items, "score") +# semantic_items = self._normalize_scores(semantic_items, "score") - # 合并结果 - combined_items = {} +# # 合并结果 +# combined_items = {} - # 添加关键词结果 - for item in keyword_items: - item_id = item.get("id") or item.get("uuid") - if item_id: - combined_items[item_id] = item.copy() - combined_items[item_id]["bm25_score"] = item.get("normalized_score", 0) - combined_items[item_id]["embedding_score"] = 0 +# # 添加关键词结果 +# for item in keyword_items: +# item_id = item.get("id") or item.get("uuid") +# if item_id: +# combined_items[item_id] = item.copy() +# combined_items[item_id]["bm25_score"] = item.get("normalized_score", 0) +# combined_items[item_id]["embedding_score"] = 0 - # 添加或更新语义结果 - for item in semantic_items: - item_id = item.get("id") or item.get("uuid") - if item_id: - if item_id in combined_items: - combined_items[item_id]["embedding_score"] = item.get("normalized_score", 0) - else: - combined_items[item_id] = item.copy() - combined_items[item_id]["bm25_score"] = 0 - combined_items[item_id]["embedding_score"] = item.get("normalized_score", 0) +# # 添加或更新语义结果 +# for item in semantic_items: +# item_id = item.get("id") or item.get("uuid") +# if item_id: +# if item_id in combined_items: +# combined_items[item_id]["embedding_score"] = item.get("normalized_score", 0) +# else: +# combined_items[item_id] = item.copy() +# combined_items[item_id]["bm25_score"] = 0 +# combined_items[item_id]["embedding_score"] = item.get("normalized_score", 0) - # 计算组合分数 - for item_id, item in combined_items.items(): - bm25_score = item.get("bm25_score", 0) - embedding_score = item.get("embedding_score", 0) - combined_score = alpha * bm25_score + (1 - alpha) * embedding_score - item["combined_score"] = combined_score +# # 计算组合分数 +# for item_id, item in combined_items.items(): +# bm25_score = item.get("bm25_score", 0) +# embedding_score = item.get("embedding_score", 0) +# combined_score = alpha * bm25_score + (1 - alpha) * embedding_score +# item["combined_score"] = combined_score - # 排序并限制结果 - sorted_items = sorted( - combined_items.values(), - key=lambda x: x.get("combined_score", 0), - reverse=True - )[:limit] +# # 排序并限制结果 +# sorted_items = sorted( +# combined_items.values(), +# key=lambda x: x.get("combined_score", 0), +# reverse=True +# )[:limit] - reranked_data[category] = sorted_items +# reranked_data[category] = sorted_items - return SearchResult( - statements=reranked_data.get("statements", []), - chunks=reranked_data.get("chunks", []), - entities=reranked_data.get("entities", []), - summaries=reranked_data.get("summaries", []) - ) +# return SearchResult( +# statements=reranked_data.get("statements", []), +# chunks=reranked_data.get("chunks", []), +# entities=reranked_data.get("entities", []), +# summaries=reranked_data.get("summaries", []) +# ) - def _parse_datetime(self, value: Any) -> Optional[datetime]: - """解析日期时间字符串""" - if value is None: - return None - if isinstance(value, datetime): - return value - if isinstance(value, str): - s = value.strip() - if not s: - return None - try: - return datetime.fromisoformat(s) - except Exception: - return None - return None +# def _parse_datetime(self, value: Any) -> Optional[datetime]: +# """解析日期时间字符串""" +# if value is None: +# return None +# if isinstance(value, datetime): +# return value +# if isinstance(value, str): +# s = value.strip() +# if not s: +# return None +# try: +# return datetime.fromisoformat(s) +# except Exception: +# return None +# return None - def _rerank_with_forgetting_curve( - self, - keyword_result: SearchResult, - semantic_result: SearchResult, - alpha: float, - limit: int - ) -> SearchResult: - """使用遗忘曲线重排序混合搜索结果 +# def _rerank_with_forgetting_curve( +# self, +# keyword_result: SearchResult, +# semantic_result: SearchResult, +# alpha: float, +# limit: int +# ) -> SearchResult: +# """使用遗忘曲线重排序混合搜索结果 - Args: - keyword_result: 关键词搜索结果 - semantic_result: 语义搜索结果 - alpha: BM25分数权重 - limit: 结果限制 +# Args: +# keyword_result: 关键词搜索结果 +# semantic_result: 语义搜索结果 +# alpha: BM25分数权重 +# limit: 结果限制 - Returns: - SearchResult: 重排序后的结果 - """ - engine = ForgettingEngine(self.forgetting_config) - now_dt = datetime.now() +# Returns: +# SearchResult: 重排序后的结果 +# """ +# engine = ForgettingEngine(self.forgetting_config) +# now_dt = datetime.now() - reranked_data = {} +# reranked_data = {} - for category in ["statements", "chunks", "entities", "summaries"]: - keyword_items = getattr(keyword_result, category, []) - semantic_items = getattr(semantic_result, category, []) +# for category in ["statements", "chunks", "entities", "summaries"]: +# keyword_items = getattr(keyword_result, category, []) +# semantic_items = getattr(semantic_result, category, []) - # 归一化分数 - keyword_items = self._normalize_scores(keyword_items, "score") - semantic_items = self._normalize_scores(semantic_items, "score") +# # 归一化分数 +# keyword_items = self._normalize_scores(keyword_items, "score") +# semantic_items = self._normalize_scores(semantic_items, "score") - # 合并结果 - combined_items = {} +# # 合并结果 +# combined_items = {} - for src_items, is_embedding in [(keyword_items, False), (semantic_items, True)]: - for item in src_items: - item_id = item.get("id") or item.get("uuid") - if not item_id: - continue +# for src_items, is_embedding in [(keyword_items, False), (semantic_items, True)]: +# for item in src_items: +# item_id = item.get("id") or item.get("uuid") +# if not item_id: +# continue - if item_id not in combined_items: - combined_items[item_id] = item.copy() - combined_items[item_id]["bm25_score"] = 0 - combined_items[item_id]["embedding_score"] = 0 +# if item_id not in combined_items: +# combined_items[item_id] = item.copy() +# combined_items[item_id]["bm25_score"] = 0 +# combined_items[item_id]["embedding_score"] = 0 - if is_embedding: - combined_items[item_id]["embedding_score"] = item.get("normalized_score", 0) - else: - combined_items[item_id]["bm25_score"] = item.get("normalized_score", 0) +# if is_embedding: +# combined_items[item_id]["embedding_score"] = item.get("normalized_score", 0) +# else: +# combined_items[item_id]["bm25_score"] = item.get("normalized_score", 0) - # 计算分数并应用遗忘权重 - for item_id, item in combined_items.items(): - bm25_score = float(item.get("bm25_score", 0) or 0) - embedding_score = float(item.get("embedding_score", 0) or 0) - combined_score = alpha * bm25_score + (1 - alpha) * embedding_score +# # 计算分数并应用遗忘权重 +# for item_id, item in combined_items.items(): +# bm25_score = float(item.get("bm25_score", 0) or 0) +# embedding_score = float(item.get("embedding_score", 0) or 0) +# combined_score = alpha * bm25_score + (1 - alpha) * embedding_score - # 计算时间衰减 - dt = self._parse_datetime(item.get("created_at")) - if dt is None: - time_elapsed_days = 0.0 - else: - time_elapsed_days = max(0.0, (now_dt - dt).total_seconds() / 86400.0) +# # 计算时间衰减 +# dt = self._parse_datetime(item.get("created_at")) +# if dt is None: +# time_elapsed_days = 0.0 +# else: +# time_elapsed_days = max(0.0, (now_dt - dt).total_seconds() / 86400.0) - memory_strength = 1.0 # 默认强度 - forgetting_weight = engine.calculate_weight( - time_elapsed=time_elapsed_days, - memory_strength=memory_strength - ) +# memory_strength = 1.0 # 默认强度 +# forgetting_weight = engine.calculate_weight( +# time_elapsed=time_elapsed_days, +# memory_strength=memory_strength +# ) - final_score = combined_score * forgetting_weight - item["combined_score"] = final_score - item["forgetting_weight"] = forgetting_weight - item["time_elapsed_days"] = time_elapsed_days +# final_score = combined_score * forgetting_weight +# item["combined_score"] = final_score +# item["forgetting_weight"] = forgetting_weight +# item["time_elapsed_days"] = time_elapsed_days - # 排序并限制结果 - sorted_items = sorted( - combined_items.values(), - key=lambda x: x.get("combined_score", 0), - reverse=True - )[:limit] +# # 排序并限制结果 +# sorted_items = sorted( +# combined_items.values(), +# key=lambda x: x.get("combined_score", 0), +# reverse=True +# )[:limit] - reranked_data[category] = sorted_items +# reranked_data[category] = sorted_items - return SearchResult( - statements=reranked_data.get("statements", []), - chunks=reranked_data.get("chunks", []), - entities=reranked_data.get("entities", []), - summaries=reranked_data.get("summaries", []) - ) +# return SearchResult( +# statements=reranked_data.get("statements", []), +# chunks=reranked_data.get("chunks", []), +# entities=reranked_data.get("entities", []), +# summaries=reranked_data.get("summaries", []) +# ) diff --git a/api/app/core/memory/utils/README.md b/api/app/core/memory/utils/README.md deleted file mode 100644 index 32264569..00000000 --- a/api/app/core/memory/utils/README.md +++ /dev/null @@ -1,445 +0,0 @@ -# Memory 模块工具函数文档 - -本目录包含 Memory 模块使用的所有工具函数，统一管理以提高代码可维护性和可复用性。 - -## 目录结构 - -``` -app/core/memory/utils/ -├── __init__.py # 包初始化文件，导出所有公共接口 -├── README.md # 本文档 -├── config/ # 配置管理模块 -│ ├── __init__.py # 配置模块初始化 -│ ├── config_utils.py # 配置管理工具 -│ ├── definitions.py # 全局定义和常量 -│ ├── overrides.py # 运行时配置覆写 -│ ├── get_data.py # 数据获取工具 -│ ├── litellm_config.py # LiteLLM 配置和监控 -│ └── config_optimization.py # 配置优化工具 -├── log/ # 日志管理模块 -│ ├── __init__.py # 日志模块初始化 -│ ├── logging_utils.py # 日志工具 -│ └── audit_logger.py # 审计日志 -├── prompt/ # 提示词管理模块 -│ ├── __init__.py # 提示词模块初始化 -│ ├── prompt_utils.py # 提示词渲染工具 -│ ├── template_render.py # 模板渲染工具 -│ └── prompts/ # Jinja2 提示词模板目录 -│ ├── entity_dedup.jinja2 # 实体去重提示词 -│ ├── extract_statement.jinja2 # 陈述句提取提示词 -│ ├── extract_temporal.jinja2 # 时间信息提取提示词 -│ ├── extract_triplet.jinja2 # 三元组提取提示词 -│ ├── memory_summary.jinja2 # 记忆摘要提示词 -│ ├── evaluate.jinja2 # 评估提示词 -│ ├── reflexion.jinja2 # 反思提示词 -│ ├── system.jinja2 # 系统提示词 -│ └── user.jinja2 # 用户提示词 -├── llm/ # LLM 工具模块 -│ ├── __init__.py # LLM 模块初始化 -│ └── llm_utils.py # LLM 客户端工具 -├── data/ # 数据处理模块 -│ ├── __init__.py # 数据模块初始化 -│ ├── text_utils.py # 文本处理工具 -│ ├── time_utils.py # 时间处理工具 -│ └── ontology.py # 本体定义（谓语、标签等） -├── paths/ # 路径管理模块 -│ ├── __init__.py # 路径模块初始化 -│ └── output_paths.py # 输出路径管理 -├── visualization/ # 可视化模块 -│ ├── __init__.py # 可视化模块初始化 -│ └── forgetting_visualizer.py # 遗忘曲线可视化 -└── self_reflexion_utils/ # 自我反思工具模块 - ├── __init__.py # 反思模块初始化 - ├── evaluate.py # 冲突评估 - ├── reflexion.py # 反思处理 - └── self_reflexion.py # 自我反思主逻辑 -``` - -## 模块分类 - -### 1. 配置管理（config/） - -配置管理模块包含所有与配置相关的工具函数和定义。 - -#### config_utils.py -提供配置加载和管理功能： -- `get_model_config(model_id)` - 获取 LLM 模型配置 -- `get_embedder_config(embedding_id)` - 获取嵌入模型配置 -- `get_neo4j_config()` - 获取 Neo4j 数据库配置 -- `get_chunker_config(chunker_strategy)` - 获取分块策略配置 -- `get_pipeline_config()` - 获取流水线配置 -- `get_pruning_config()` - 获取语义剪枝配置 -- `get_picture_config()` - 获取图片模型配置 -- `get_voice_config()` - 获取语音模型配置 - -#### definitions.py -全局定义和常量： -- `CONFIG` - 基础配置（从 config.json 加载） -- `RUNTIME_CONFIG` - 运行时配置（从 runtime.json 或数据库加载） -- `PROJECT_ROOT` - 项目根目录路径 -- 各种选择配置常量（LLM、嵌入模型、分块策略等） -- `reload_configuration_from_database(config_id)` - 动态重新加载配置 - -#### overrides.py -运行时配置覆写： -- `load_unified_config(project_root)` - 加载统一配置 - -#### get_data.py -数据获取工具： -- `get_data(host_id)` - 从 SQL 数据库获取数据 - -#### litellm_config.py -LiteLLM 配置和监控： -- `LiteLLMConfig` - LiteLLM 配置类 -- `setup_litellm_enhanced(max_retries)` - 设置增强的 LiteLLM 配置 -- `get_usage_summary()` - 获取使用统计摘要 -- `print_usage_summary()` - 打印使用统计 -- `get_instant_qps(module)` - 获取即时 QPS 数据 -- `print_instant_qps(module)` - 打印即时 QPS 信息 - -#### config_optimization.py -配置优化工具： -- 配置参数优化相关功能 - -### 3. LLM 工具（llm/） - -LLM 工具模块包含所有与 LLM 客户端相关的工具函数。 - -#### llm_utils.py -LLM 客户端工具： -- `get_llm_client(llm_id)` - 获取 LLM 客户端实例 -- `get_reranker_client(rerank_id)` - 获取重排序客户端实例 -- `handle_response(response)` - 处理 LLM 响应 - -#### litellm_config.py -LiteLLM 配置和监控： -- `LiteLLMConfig` - LiteLLM 配置类 -- `setup_litellm_enhanced(max_retries)` - 设置增强的 LiteLLM 配置 -- `get_usage_summary()` - 获取使用统计摘要 -- `print_usage_summary()` - 打印使用统计 -- `get_instant_qps(module)` - 获取即时 QPS 数据 -- `print_instant_qps(module)` - 打印即时 QPS 信息 - -### 4. 提示词管理（prompt/） - -提示词管理模块包含所有提示词渲染和模板管理相关的工具函数。 - -#### prompt_utils.py -提示词渲染工具（使用 Jinja2 模板）： -- `get_prompts(message)` - 获取系统和用户提示词 -- `render_statement_extraction_prompt(...)` - 渲染陈述句提取提示词 -- `render_temporal_extraction_prompt(...)` - 渲染时间信息提取提示词 -- `render_entity_dedup_prompt(...)` - 渲染实体去重提示词 -- `render_triplet_extraction_prompt(...)` - 渲染三元组提取提示词 -- `render_memory_summary_prompt(...)` - 渲染记忆摘要提示词 -- `prompt_env` - Jinja2 环境对象 - -#### template_render.py -模板渲染工具（用于评估和反思）： -- `render_evaluate_prompt(evaluate_data, schema)` - 渲染评估提示词 -- `render_reflexion_prompt(data, schema)` - 渲染反思提示词 - -#### prompts/ -Jinja2 模板文件目录，包含所有提示词模板 - -### 5. 数据处理（data/） - -数据处理模块包含所有数据处理相关的工具函数。 - -#### text_utils.py -文本处理工具： -- `escape_lucene_query(query)` - 转义 Lucene 查询特殊字符 -- `extract_plain_query(query_input)` - 从各种输入格式提取纯文本查询 - -#### time_utils.py -时间处理工具： -- `validate_date_format(date_str)` - 验证日期格式（YYYY-MM-DD） -- `normalize_date(date_str)` - 标准化日期格式 -- `normalize_date_safe(date_str, default)` - 安全的日期标准化（带默认值） -- `preprocess_date_string(date_str)` - 预处理日期字符串 - -#### ontology.py -本体定义： -- `PREDICATE_DEFINITIONS` - 谓语定义字典 -- `LABEL_DEFINITIONS` - 标签定义字典 -- `Predicate` - 谓语枚举 -- `StatementType` - 陈述句类型枚举 -- `TemporalInfo` - 时间信息枚举 -- `RelevenceInfo` - 相关性信息枚举 - -### 2. 日志管理（log/） - -日志管理模块包含所有与日志记录相关的工具函数。 - -#### logging_utils.py -日志工具： -- `log_prompt_rendering(role, content)` - 记录提示词渲染 -- `log_template_rendering(template_name, context)` - 记录模板渲染 -- `log_time(operation, duration)` - 记录操作耗时 -- `prompt_logger` - 提示词日志记录器 - -#### audit_logger.py -审计日志： -- `audit_logger` - 审计日志记录器 -- 记录系统关键操作和安全事件 - -### 6. 自我反思工具（self_reflexion_utils/） - -自我反思工具模块包含记忆冲突检测和反思处理功能。 - -#### evaluate.py -冲突评估： -- `conflict(evaluate_data, schema)` - 评估记忆冲突 - -#### reflexion.py -反思处理： -- `reflexion(data, schema)` - 执行反思处理 - -#### self_reflexion.py -自我反思主逻辑： -- `self_reflexion(...)` - 自我反思主函数 - -### 7. 数据模型 - -#### json_schema.py -JSON Schema 数据模型： -- `BaseDataSchema` - 基础数据模型 -- `ConflictResultSchema` - 冲突结果模型 -- `ConflictSchema` - 冲突模型 -- `ReflexionSchema` - 反思模型 -- `ResolvedSchema` - 解决方案模型 -- `ReflexionResultSchema` - 反思结果模型 - -#### messages.py -API 消息模型： -- `ConfigKey` - 配置键模型 -- `ChunkerStrategy` - 分块策略枚举 -- `ConfigParams` - 配置参数模型 -- `ConfigParamsCreate` - 创建配置参数模型 -- `ConfigUpdate` - 更新配置模型 -- `ConfigUpdateExtracted` - 更新萃取引擎配置模型 -- `ConfigUpdateForget` - 更新遗忘引擎配置模型 -- `ConfigPilotRun` - 试运行配置模型 -- `ConfigFilter` - 配置过滤模型 -- `ApiResponse` - API 响应模型 -- `ok(msg, data)` - 成功响应构造函数 -- `fail(msg, error_code, data)` - 失败响应构造函数 - -### 8. 可视化（visualization/） - -可视化模块包含所有可视化相关的工具函数。 - -#### forgetting_visualizer.py -遗忘曲线可视化： -- `export_memory_curve_numpy(...)` - 导出记忆曲线为 NumPy 数组 -- `export_memory_curves_multiple_strengths(...)` - 导出多个强度的记忆曲线 -- `export_parameter_sweep_numpy(...)` - 导出参数扫描结果 -- `visualize_forgetting_curve(...)` - 可视化遗忘曲线 -- `plot_3d_forgetting_surface(...)` - 绘制 3D 遗忘曲线表面 -- `create_comparison_visualization(...)` - 创建对比可视化 -- `save_memory_curves_to_file(...)` - 保存记忆曲线到文件 - -### 9. 路径管理（paths/） - -路径管理模块包含所有路径管理相关的工具函数。 - -#### output_paths.py -输出路径管理： -- `get_output_dir()` - 获取输出目录 -- `get_output_path(filename)` - 获取输出文件路径 - -## 使用示例 - -### 配置管理 - -```python -from app.core.memory.utils.config import get_model_config, get_pipeline_config -from app.core.memory.utils.config.definitions import SELECTED_LLM_ID - -# 获取模型配置 -model_config = get_model_config("model_id_123") - -# 获取流水线配置 -pipeline_config = get_pipeline_config() - -# 使用全局常量 -llm_id = SELECTED_LLM_ID -``` - -### 日志管理 - -```python -from app.core.memory.utils.log import log_prompt_rendering, log_time, audit_logger - -# 记录提示词渲染 -log_prompt_rendering('user', 'Hello, world!') - -# 记录操作耗时 -log_time('extraction', 1.23) - -# 使用审计日志 -audit_logger.info('User action performed') -``` - -### LLM 工具 - -```python -from app.core.memory.utils.llm import get_llm_client - -# 获取 LLM 客户端 -llm_client = get_llm_client("llm_id_456") - -# 调用 LLM -response = await llm_client.chat([ - {"role": "user", "content": "Hello"} -]) -``` - -### 提示词渲染 - -```python -from app.core.memory.utils.prompt import render_statement_extraction_prompt -from app.core.memory.utils.data.ontology import LABEL_DEFINITIONS - -# 渲染陈述句提取提示词 -prompt = await render_statement_extraction_prompt( - chunk_content="对话内容...", - definitions=LABEL_DEFINITIONS, - json_schema=schema, - granularity=2 -) -``` - -### 数据处理 - -```python -from app.core.memory.utils.data.time_utils import normalize_date -from app.core.memory.utils.data.text_utils import escape_lucene_query - -# 标准化日期 -normalized = normalize_date("2025/10/28") # 返回 "2025-10-28" - -# 转义 Lucene 查询 -escaped = escape_lucene_query("user:admin AND status:active") -``` - -### 运行时配置覆写 - -```python -from app.core.memory.utils import apply_runtime_overrides_with_config_id - -# 使用指定 config_id 覆写配置 -runtime_cfg = {"selections": {}} -updated_cfg = apply_runtime_overrides_with_config_id( - project_root="/path/to/project", - runtime_cfg=runtime_cfg, - config_id="config_123" -) -``` - -## 迁移说明 - -### 从旧路径迁移 - -如果你的代码使用了旧的导入路径，请按以下方式更新： - -**旧路径（2024年11月之前）：** -```python -from app.core.memory.src.utils.config_utils import get_model_config -from app.core.memory.src.utils.prompt_utils import render_statement_extraction_prompt -from app.core.memory.src.data_config_api.utils.messages import ok, fail -``` - -**中间路径（2024年11月）：** -```python -from app.core.memory.utils.config_utils import get_model_config -from app.core.memory.utils.logging_utils import log_prompt_rendering -from app.schemas.memory_storage_schema import ok, fail -``` - -**新路径（2024年11月27日之后）：** -```python -# 配置相关 -from app.core.memory.utils.config.config_utils import get_model_config -from app.core.memory.utils.config import get_model_config # 简化导入 - -# 日志相关 -from app.core.memory.utils.log.logging_utils import log_prompt_rendering -from app.core.memory.utils.log import log_prompt_rendering # 简化导入 - -# 其他工具 -from app.core.memory.utils import prompt_utils -from app.schemas.memory_storage_schema import ok, fail -``` - -### 目录结构重组（2024年11月27日） - -utils 目录已按功能进行了完整的重组： - -**重组前的结构：** -- 所有文件都在 `app/core/memory/utils/` 根目录下 - -**重组后的结构：** -- `config/` - 配置管理相关文件 -- `log/` - 日志管理相关文件 -- `prompt/` - 提示词管理相关文件 -- `llm/` - LLM 工具相关文件 -- `data/` - 数据处理相关文件 -- `paths/` - 路径管理相关文件 -- `visualization/` - 可视化相关文件 -- `self_reflexion_utils/` - 自我反思工具（已存在） - -**导入路径变化：** -```python -# 旧导入方式 -from app.core.memory.utils.config_utils import get_model_config -from app.core.memory.utils.logging_utils import log_prompt_rendering -from app.core.memory.utils.prompt_utils import render_statement_extraction_prompt - -# 新导入方式 -from app.core.memory.utils.config.config_utils import get_model_config -from app.core.memory.utils.log.logging_utils import log_prompt_rendering -from app.core.memory.utils.prompt.prompt_utils import render_statement_extraction_prompt - -# 或使用简化导入 -from app.core.memory.utils.config import get_model_config -from app.core.memory.utils.log import log_prompt_rendering -from app.core.memory.utils.prompt import render_statement_extraction_prompt -``` - -## 维护指南 - -### 添加新工具函数 - -1. 在相应的模块文件中添加函数 -2. 在 `__init__.py` 中导出函数 -3. 在本 README 中添加文档 -4. 编写单元测试 - -### 删除旧工具函数 - -1. 确认没有代码使用该函数 -2. 从模块文件中删除函数 -3. 从 `__init__.py` 中删除导出 -4. 更新本 README - -### 重构工具函数 - -1. 保持向后兼容性（使用别名或包装器） -2. 更新所有使用该函数的代码 -3. 更新文档和测试 -4. 在适当时机删除旧版本 - -## 注意事项 - -1. **向后兼容性**：所有工具函数应保持向后兼容，避免破坏现有代码 -2. **文档完整性**：每个函数都应有清晰的文档字符串 -3. **类型注解**：使用类型注解提高代码可读性 -4. **错误处理**：工具函数应有适当的错误处理 -5. **测试覆盖**：所有工具函数都应有单元测试 - -## 相关文档 - -- [Memory 模块架构设计](../.kiro/specs/memory-refactoring/design.md) -- [Memory 模块需求文档](../.kiro/specs/memory-refactoring/requirements.md) -- [Memory 模块任务列表](../.kiro/specs/memory-refactoring/tasks.md) diff --git a/api/app/core/memory/utils/config/__init__.py b/api/app/core/memory/utils/config/__init__.py index 2b41b522..c2a8c6ca 100644 --- a/api/app/core/memory/utils/config/__init__.py +++ b/api/app/core/memory/utils/config/__init__.py @@ -6,33 +6,27 @@ # 从子模块导出常用函数和常量，保持向后兼容 from .config_utils import ( - get_model_config, - get_embedder_config, - get_neo4j_config, get_chunker_config, + get_embedder_config, + get_model_config, + get_neo4j_config, + get_picture_config, get_pipeline_config, get_pruning_config, - get_picture_config, get_voice_config, ) -from .definitions import ( - CONFIG, - RUNTIME_CONFIG, - PROJECT_ROOT, - SELECTED_LLM_ID, - SELECTED_EMBEDDING_ID, - SELECTED_GROUP_ID, - SELECTED_RERANK_ID, - SELECTED_LLM_PICTURE_NAME, - SELECTED_LLM_VOICE_NAME, - REFLEXION_ENABLED, - REFLEXION_ITERATION_PERIOD, - REFLEXION_RANGE, - REFLEXION_BASELINE, - reload_configuration_from_database, -) -from .overrides import load_unified_config + +# DEPRECATED: Global configuration variables removed +# Use MemoryConfig objects with dependency injection instead +# from .definitions import ( +# CONFIG, # DEPRECATED - empty dict for backward compatibility +# RUNTIME_CONFIG, # DEPRECATED - minimal for backward compatibility +# PROJECT_ROOT, # Still needed for file paths +# reload_configuration_from_database, # DEPRECATED - returns False +# ) +# DEPRECATED: overrides module removed - use MemoryConfig with dependency injection from .get_data import get_data + # litellm_config 需要时动态导入，避免循环依赖 # from .litellm_config import ( # LiteLLMConfig, @@ -53,23 +47,11 @@ __all__ = [ "get_pruning_config", "get_picture_config", "get_voice_config", - # definitions - "CONFIG", - "RUNTIME_CONFIG", - "PROJECT_ROOT", - "SELECTED_LLM_ID", - "SELECTED_EMBEDDING_ID", - "SELECTED_GROUP_ID", - "SELECTED_RERANK_ID", - "SELECTED_LLM_PICTURE_NAME", - "SELECTED_LLM_VOICE_NAME", - "REFLEXION_ENABLED", - "REFLEXION_ITERATION_PERIOD", - "REFLEXION_RANGE", - "REFLEXION_BASELINE", - "reload_configuration_from_database", - # overrides - "load_unified_config", + # definitions (DEPRECATED - use MemoryConfig objects instead) + # "CONFIG", # DEPRECATED + # "RUNTIME_CONFIG", # DEPRECATED + # "PROJECT_ROOT", + # "reload_configuration_from_database", # DEPRECATED # get_data "get_data", # litellm_config - 需要时从 .litellm_config 直接导入 diff --git a/api/app/core/memory/utils/config/config_utils.py b/api/app/core/memory/utils/config/config_utils.py index 6acd73d3..b05e176c 100644 --- a/api/app/core/memory/utils/config/config_utils.py +++ b/api/app/core/memory/utils/config/config_utils.py @@ -1,22 +1,18 @@ -import uuid -import json -from typing import Optional - -from sqlalchemy.orm import Session -from fastapi.exceptions import HTTPException -from fastapi import status - -from app.core.memory.utils.config.definitions import CONFIG, RUNTIME_CONFIG from app.core.memory.models.variate_config import ( - ExtractionPipelineConfig, DedupConfig, - StatementExtractionConfig, + ExtractionPipelineConfig, ForgettingEngineConfig, + StatementExtractionConfig, ) -from app.core.memory.models.config_models import PruningConfig +from app.core.memory.utils.config.definitions import CONFIG from app.db import get_db -from app.models.models_model import ModelConfig, ModelApiKey +from app.models.models_model import ModelApiKey from app.services.model_service import ModelConfigService +from fastapi import status +from fastapi.exceptions import HTTPException +from sqlalchemy.orm import Session + + def get_model_config(model_id: str, db: Session | None = None) -> dict: if db is None: db_gen = get_db() # get_db 通常是一个生成器 @@ -110,6 +106,13 @@ def get_chunker_config(chunker_strategy: str) -> dict: # 2) Provide sane defaults for newer strategies default_configs = { + "RecursiveChunker": { + "chunker_strategy": "RecursiveChunker", + "embedding_model": "BAAI/bge-m3", + "chunk_size": 512, + "min_characters_per_chunk": 50 + }, + "LLMChunker": { "chunker_strategy": "LLMChunker", "embedding_model": "BAAI/bge-m3", @@ -147,94 +150,74 @@ def get_chunker_config(chunker_strategy: str) -> dict: f"Chunker '{chunker_strategy}' not found in config.json and no default or fallback available" ) +#TODO: Fix this -def get_pipeline_config() -> ExtractionPipelineConfig: - """Build ExtractionPipelineConfig using only runtime.json values. +def get_pipeline_config( + config_id: int, + db: Session | None = None, +) -> ExtractionPipelineConfig: + """Build ExtractionPipelineConfig from database. - Behavior: - - Read `deduplication` section from runtime.json if present. - - Read `statement_extraction` section from runtime.json if present. - - Read `forgetting_engine` section from runtime.json if present. - - If absent, check legacy top-level `enable_llm_dedup` key. - - Do NOT fall back to environment variables. - - Unspecified fields use model defaults defined in DedupConfig. + Args: + config_id: Database configuration ID (required). Loads pipeline + settings from the DataConfig table. + db: Optional database session. If not provided, a new session + will be created. + + Returns: + ExtractionPipelineConfig with deduplication, statement extraction, + and forgetting engine settings loaded from database. + + Raises: + ValueError: If config_id not found in database. """ - dedup_rc = RUNTIME_CONFIG.get("deduplication", {}) or {} - stmt_rc = RUNTIME_CONFIG.get("statement_extraction", {}) or {} - forget_rc = RUNTIME_CONFIG.get("forgetting_engine", {}) or {} + from app.repositories.data_config_repository import DataConfigRepository - # Assemble kwargs from runtime.json only - kwargs = {} - # LLM switch: prefer new key, then legacy top-level, default False - if "enable_llm_dedup_blockwise" in dedup_rc: - kwargs["enable_llm_dedup_blockwise"] = bool(dedup_rc.get("enable_llm_dedup_blockwise")) - else: - # Legacy top-level fallback inside runtime.json only - legacy = RUNTIME_CONFIG.get("enable_llm_dedup") - if legacy is not None: - kwargs["enable_llm_dedup_blockwise"] = bool(legacy) - else: - kwargs["enable_llm_dedup_blockwise"] = False # default reserve - # Disambiguation switch: only from runtime.json deduplication section - if "enable_llm_disambiguation" in dedup_rc: - kwargs["enable_llm_disambiguation"] = bool(dedup_rc.get("enable_llm_disambiguation")) + # Load from database + if db is None: + db_gen = get_db() + db = next(db_gen) + + db_config = DataConfigRepository.get_by_id(db, config_id) + if db_config is None: + raise ValueError(f"Configuration {config_id} not found in database") - # Optional LLM fallback gating - if "enable_llm_fallback_only_on_borderline" in dedup_rc: - kwargs["enable_llm_fallback_only_on_borderline"] = bool(dedup_rc.get("enable_llm_fallback_only_on_borderline")) + # Build DedupConfig from database + dedup_kwargs = { + "enable_llm_dedup_blockwise": bool(db_config.enable_llm_dedup_blockwise) if db_config.enable_llm_dedup_blockwise is not None else False, + "enable_llm_disambiguation": bool(db_config.enable_llm_disambiguation) if db_config.enable_llm_disambiguation is not None else False, + } + + # Fuzzy thresholds + if db_config.t_name_strict is not None: + dedup_kwargs["fuzzy_name_threshold_strict"] = db_config.t_name_strict + if db_config.t_type_strict is not None: + dedup_kwargs["fuzzy_type_threshold_strict"] = db_config.t_type_strict + if db_config.t_overall is not None: + dedup_kwargs["fuzzy_overall_threshold"] = db_config.t_overall - # Optional fuzzy thresholds: use values if provided; otherwise rely on DedupConfig defaults - for key in ( - "fuzzy_name_threshold_strict", - "fuzzy_type_threshold_strict", - "fuzzy_overall_threshold", - "fuzzy_unknown_type_name_threshold", - "fuzzy_unknown_type_type_threshold", - ): - if key in dedup_rc: - kwargs[key] = dedup_rc[key] + dedup_config = DedupConfig(**dedup_kwargs) - # Optional weights and bonuses for overall scoring - for key in ( - "name_weight", - "desc_weight", - "type_weight", - "context_bonus", - "llm_fallback_floor", - "llm_fallback_ceiling", - ): - if key in dedup_rc: - kwargs[key] = dedup_rc[key] - - # Optional LLM iterative dedup parameters - for key in ( - "llm_block_size", - "llm_block_concurrency", - "llm_pair_concurrency", - "llm_max_rounds", - ): - if key in dedup_rc: - kwargs[key] = dedup_rc[key] - - dedup_config = DedupConfig(**kwargs) - - # Build StatementExtractionConfig from runtime.json + # Build StatementExtractionConfig from database stmt_kwargs = {} - for key in ( - "statement_granularity", - "temperature", - "include_dialogue_context", - "max_dialogue_context_chars", - ): - if key in stmt_rc: - stmt_kwargs[key] = stmt_rc[key] + if db_config.statement_granularity is not None: + stmt_kwargs["statement_granularity"] = db_config.statement_granularity + if db_config.include_dialogue_context is not None: + stmt_kwargs["include_dialogue_context"] = bool(db_config.include_dialogue_context) + if db_config.max_context is not None: + stmt_kwargs["max_dialogue_context_chars"] = db_config.max_context + stmt_config = StatementExtractionConfig(**stmt_kwargs) - # Build ForgettingEngineConfig from runtime.json + # Build ForgettingEngineConfig from database forget_kwargs = {} - for key in ("offset", "lambda_time", "lambda_mem"): - if key in forget_rc: - forget_kwargs[key] = forget_rc[key] + if db_config.offset is not None: + forget_kwargs["offset"] = db_config.offset + if db_config.lambda_time is not None: + forget_kwargs["lambda_time"] = db_config.lambda_time + if db_config.lambda_mem is not None: + forget_kwargs["lambda_mem"] = db_config.lambda_mem + forget_config = ForgettingEngineConfig(**forget_kwargs) return ExtractionPipelineConfig( @@ -244,24 +227,37 @@ def get_pipeline_config() -> ExtractionPipelineConfig: ) -def get_pruning_config() -> dict: - """Retrieve semantic pruning config from runtime.json. +def get_pruning_config( + config_id: int, + db: Session | None = None, +) -> dict: + """Retrieve semantic pruning config from database. - Returns a dict suitable for PruningConfig.model_validate. + Args: + config_id: Database configuration ID (required). + db: Optional database session. - Structure in runtime.json: - { - "pruning": { - "enabled": true, - "scene": "education" | "online_service" | "outbound", - "threshold": 0.5 - } - } + Returns: + Dict suitable for PruningConfig.model_validate with keys: + - pruning_switch: bool + - pruning_scene: str ("education" | "online_service" | "outbound") + - pruning_threshold: float (0-0.9) + + Raises: + ValueError: If config_id not found in database. """ - pruning_rc = RUNTIME_CONFIG.get("pruning", {}) or {} + from app.repositories.data_config_repository import DataConfigRepository + + if db is None: + db_gen = get_db() + db = next(db_gen) + + db_config = DataConfigRepository.get_by_id(db, config_id) + if db_config is None: + raise ValueError(f"Configuration {config_id} not found in database") return { - "pruning_switch": bool(pruning_rc.get("enabled", False)), - "pruning_scene": pruning_rc.get("scene", "education"), - "pruning_threshold": float(pruning_rc.get("threshold", 0.5)), + "pruning_switch": bool(db_config.pruning_enabled) if db_config.pruning_enabled is not None else False, + "pruning_scene": db_config.pruning_scene or "education", + "pruning_threshold": float(db_config.pruning_threshold) if db_config.pruning_threshold is not None else 0.5, } diff --git a/api/app/core/memory/utils/config/definitions.py b/api/app/core/memory/utils/config/definitions.py index 03d6b28a..cc1aef66 100644 --- a/api/app/core/memory/utils/config/definitions.py +++ b/api/app/core/memory/utils/config/definitions.py @@ -1,18 +1,26 @@ """ -配置加载模块 - 三阶段架构（已迁移到统一配置管理） +配置加载模块 - DEPRECATED -本模块现在使用全局配置管理系统 (app/core/config.py) -来加载和管理配置，同时保持向后兼容性。 +⚠️ DEPRECATION NOTICE ⚠️ +This module is deprecated and will be removed in a future version. +Global configuration variables have been eliminated in favor of dependency injection. -阶段 1: 从 runtime.json 加载配置（路径 A） -阶段 2: 从数据库加载配置（路径 B，基于 dbrun.json 中的 config_id） -阶段 3: 暴露配置常量供项目使用（路径 A 和 B 的汇合点） +Use the new MemoryConfig system instead: +- app.core.memory_config.config.MemoryConfig for configuration objects +- app.services.memory_agent_service.MemoryAgentService.load_memory_config() +- app.services.memory_storage_service.MemoryStorageService.load_memory_config() + +阶段 1: 从 runtime.json 加载配置（路径 A）- DEPRECATED +阶段 2: 从数据库加载配置（路径 B，基于 dbrun.json 中的 config_id）- DEPRECATED +阶段 3: 暴露配置常量供项目使用（路径 A 和 B 的汇合点）- DEPRECATED """ -import os import json +import os import threading -from typing import Any, Dict, Optional from datetime import datetime, timedelta +from typing import Any, Dict, Optional + +#TODO: Fix this try: from dotenv import load_dotenv @@ -35,21 +43,12 @@ except ImportError: # os.path.dirname(...) = app/core/memory PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -# 全局配置锁 - 用于线程安全 -_config_lock = threading.RLock() +# DEPRECATED: Global configuration lock removed +# Use MemoryConfig objects with dependency injection instead -# 加载基础配置（config.json）- 使用全局配置系统 -if USE_UNIFIED_CONFIG: - CONFIG = settings.load_memory_config() -else: - # Fallback to legacy loading - config_path = os.path.join(PROJECT_ROOT, "config.json") - try: - with open(config_path, "r") as f: - CONFIG = json.load(f) - except (FileNotFoundError, json.JSONDecodeError): - print("Warning: config.json not found or is malformed. Using default settings.") - CONFIG = {} +# DEPRECATED: Legacy config.json loading removed +# Use MemoryConfig objects with dependency injection instead +CONFIG = {} DEFAULT_VALUES = { "llm_name": "openai/qwen-plus", @@ -68,35 +67,31 @@ DEFAULT_VALUES = { "reflexion_baseline": "TIME", } +# DEPRECATED: Legacy global variables for backward compatibility only +# These will be removed in a future version +# Use MemoryConfig objects with dependency injection instead +LANGFUSE_ENABLED = os.getenv("LANGFUSE_ENABLED", "false").lower() == "true" +SELECTED_LLM_ID = os.getenv("SELECTED_LLM_ID", DEFAULT_VALUES["llm_name"]) + # 阶段 1: 从 runtime.json 加载配置（路径 A） def _load_from_runtime_json() -> Dict[str, Any]: """ - 从 runtime.json 文件加载配置（通过统一配置加载器） + DEPRECATED: Legacy runtime.json loading - 使用 overrides.py 的统一配置加载器，按优先级加载： - 1. 数据库配置（如果 dbrun.json 中有 config_id/group_id） - 2. 环境变量配置 - 3. runtime.json 默认配置 + ⚠️ This function is deprecated and will be removed in a future version. + Use MemoryConfig objects with dependency injection instead. Returns: - Dict[str, Any]: 运行时配置字典 + Dict[str, Any]: Empty configuration (legacy support only) """ - try: - # 使用 overrides.py 的统一配置加载器 - from app.core.memory.utils.config.overrides import load_unified_config - - runtime_cfg = load_unified_config(PROJECT_ROOT) - return runtime_cfg - except Exception as e: - # Fallback: 直接读取 runtime.json - runtime_config_path = os.path.join(PROJECT_ROOT, "runtime.json") - try: - with open(runtime_config_path, "r", encoding="utf-8") as f: - return json.load(f) - except (FileNotFoundError, json.JSONDecodeError) as e2: - pass # print(f"[definitions] ❌ 无法加载 runtime.json: {e2}，使用空配置") - return {"selections": {}} + import warnings + warnings.warn( + "Runtime JSON loading is deprecated. Use MemoryConfig objects with dependency injection instead.", + DeprecationWarning, + stacklevel=2 + ) + return {"selections": {}} # 阶段 2: 从数据库加载配置（路径 B）- 已整合到统一加载器 @@ -104,207 +99,116 @@ def _load_from_runtime_json() -> Dict[str, Any]: # 保留此函数仅为向后兼容 def _load_from_database() -> Optional[Dict[str, Any]]: """ - 从数据库加载配置（基于 dbrun.json 中的 config_id） + DEPRECATED: Legacy database configuration loading - 注意：此函数已被统一配置加载器替代，现在直接调用 _load_from_runtime_json - 即可获得包含数据库配置的完整配置。 + ⚠️ This function is deprecated and will be removed in a future version. + Use MemoryConfig objects with dependency injection instead. Returns: - Optional[Dict[str, Any]]: 配置字典 + Optional[Dict[str, Any]]: None (deprecated functionality) """ - try: - # 直接使用统一配置加载器 - return _load_from_runtime_json() - except Exception: - return None + import warnings + warnings.warn( + "Database configuration loading is deprecated. Use MemoryConfig objects with dependency injection instead.", + DeprecationWarning, + stacklevel=2 + ) + return None -# 阶段 3: 暴露配置常量（路径 A 和 B 的汇合点） +# 阶段 3: 暴露配置常量（路径 A 和 B 的汇合点）- DEPRECATED def _expose_runtime_constants(runtime_cfg: Dict[str, Any]) -> None: """ - 将运行时配置暴露为全局常量供项目使用 - - 这是路径 A（runtime.json）和路径 B（数据库）的汇合点， - 无论配置来自哪里，都通过这个函数统一暴露为常量。 + DEPRECATED: 将运行时配置暴露为全局常量供项目使用 + + ⚠️ This function is deprecated and will be removed in a future version. + Global configuration variables have been eliminated in favor of dependency injection. + + Use the new MemoryConfig system instead: + - app.core.memory_config.config.MemoryConfig for configuration objects + - Pass configuration objects as parameters instead of using global variables Args: runtime_cfg: 运行时配置字典 """ - global RUNTIME_CONFIG, SELECTIONS, LOGGING_CONFIG - global LANGFUSE_ENABLED, AGENTA_ENABLED, PROMPT_LOG_LEVEL_NAME - global SELECTED_LLM_NAME, SELECTED_EMBEDDING_NAME, SELECTED_CHUNKER_STRATEGY - global SELECTED_GROUP_ID, SELECTED_USER_ID, SELECTED_APPLY_ID, SELECTED_TEST_DATA_INDICES - global SELECTED_LLM_AGENT_NAME, SELECTED_LLM_VERIFY_NAME, SELECTED_LLM_PICTURE_NAME, SELECTED_LLM_VOICE_NAME - global SELECTED_LLM_ID, SELECTED_EMBEDDING_ID, SELECTED_RERANK_ID - global REFLEXION_CONFIG, REFLEXION_ENABLED, REFLEXION_ITERATION_PERIOD, REFLEXION_RANGE, REFLEXION_BASELINE - - RUNTIME_CONFIG = runtime_cfg - - # 可观测性配置 - LANGFUSE_ENABLED = RUNTIME_CONFIG.get("langfuse", {}).get("enabled", False) - AGENTA_ENABLED = RUNTIME_CONFIG.get("agenta", {}).get("enabled", False) - - # 日志配置 - LOGGING_CONFIG = RUNTIME_CONFIG.get("logging", {}) - PROMPT_LOG_LEVEL_NAME = LOGGING_CONFIG.get("prompt_level", DEFAULT_VALUES["prompt_level"]) - - # 选择配置 - SELECTIONS = RUNTIME_CONFIG.get("selections", {}) - - # 基础模型选择 - SELECTED_LLM_NAME = SELECTIONS.get("llm_name", DEFAULT_VALUES["llm_name"]) - SELECTED_EMBEDDING_NAME = SELECTIONS.get("embedding_name", DEFAULT_VALUES["embedding_name"]) - SELECTED_CHUNKER_STRATEGY = SELECTIONS.get("chunker_strategy", DEFAULT_VALUES["chunker_strategy"]) - - # 分组和用户配置 - SELECTED_GROUP_ID = SELECTIONS.get("group_id", DEFAULT_VALUES["group_id"]) - SELECTED_USER_ID = SELECTIONS.get("user_id", DEFAULT_VALUES["user_id"]) - SELECTED_APPLY_ID = SELECTIONS.get("apply_id", DEFAULT_VALUES["apply_id"]) - SELECTED_TEST_DATA_INDICES = SELECTIONS.get("test_data_indices", None) - - # 专用 LLM 配置 - SELECTED_LLM_AGENT_NAME = SELECTIONS.get("llm_agent_name", DEFAULT_VALUES["llm_agent_name"]) - SELECTED_LLM_VERIFY_NAME = SELECTIONS.get("llm_verify_name", DEFAULT_VALUES["llm_verify_name"]) - SELECTED_LLM_PICTURE_NAME = SELECTIONS.get("llm_image_recognition", DEFAULT_VALUES["llm_image_recognition"]) - SELECTED_LLM_VOICE_NAME = SELECTIONS.get("llm_voice_recognition", DEFAULT_VALUES["llm_voice_recognition"]) - - # 模型 ID 配置 - SELECTED_LLM_ID = SELECTIONS.get("llm_id", None) - SELECTED_EMBEDDING_ID = SELECTIONS.get("embedding_id", None) - SELECTED_RERANK_ID = SELECTIONS.get("rerank_id", None) + import warnings + warnings.warn( + "Global configuration variables are deprecated. Use MemoryConfig objects with dependency injection instead.", + DeprecationWarning, + stacklevel=2 + ) - # 反思配置 - REFLEXION_CONFIG = RUNTIME_CONFIG.get("reflexion", {}) - REFLEXION_ENABLED = REFLEXION_CONFIG.get("enabled", False) - REFLEXION_ITERATION_PERIOD = REFLEXION_CONFIG.get("iteration_period", DEFAULT_VALUES["reflexion_iteration_period"]) - REFLEXION_RANGE = REFLEXION_CONFIG.get("reflexion_range", DEFAULT_VALUES["reflexion_range"]) - REFLEXION_BASELINE = REFLEXION_CONFIG.get("baseline", DEFAULT_VALUES["reflexion_baseline"]) + # Keep minimal global state for backward compatibility only + # These will be removed in a future version + global RUNTIME_CONFIG, SELECTIONS + + RUNTIME_CONFIG = runtime_cfg + SELECTIONS = RUNTIME_CONFIG.get("selections", {}) + + # All other global variables have been removed + # Use MemoryConfig objects instead # 初始化：使用统一配置加载器 def _initialize_configuration() -> None: """ - 初始化配置：使用统一配置加载器 + DEPRECATED: Legacy configuration initialization - 配置加载优先级（由 overrides.py 统一处理）： - 1. 数据库配置（如果 dbrun.json 中有 config_id/group_id） - 2. 环境变量配置（.env） - 3. runtime.json 默认配置 + ⚠️ This function is deprecated and will be removed in a future version. + Use MemoryConfig objects with dependency injection instead. """ - try: - - # 使用统一配置加载器（已包含所有优先级处理） - runtime_config = _load_from_runtime_json() - - # 暴露为全局常量 - _expose_runtime_constants(runtime_config) - - - except Exception as e: - pass # print(f"[definitions] × 配置初始化失败: {e}") - # 使用空配置 - _expose_runtime_constants({"selections": {}}) + import warnings + warnings.warn( + "Global configuration initialization is deprecated. Use MemoryConfig objects with dependency injection instead.", + DeprecationWarning, + stacklevel=2 + ) + # Initialize with empty configuration for backward compatibility + _expose_runtime_constants({"selections": {}}) # 模块加载时自动初始化配置 _initialize_configuration() +# DEPRECATED: Global variables removed +# These variables have been eliminated in favor of dependency injection +# Use MemoryConfig objects instead of accessing global variables + # 公共 API：动态重新加载配置 -def reload_configuration_from_database(config_id: int | str, force_reload: bool = False) -> bool: +def reload_configuration_from_database(config_id, force_reload: bool = False) -> bool: """ - 动态重新加载配置（从数据库）- 使用统一配置加载器 - 用于运行时切换配置，例如前端传入新的 config_id 时调用。 - - 注意：此函数仅在内存中覆写配置，不会修改 runtime.json 文件。 + DEPRECATED: Legacy configuration reloading + + ⚠️ This function is deprecated and will be removed in a future version. + Use MemoryConfig objects with dependency injection instead. + + For new code, use: + - app.services.memory_agent_service.MemoryAgentService.load_memory_config() + - app.services.memory_storage_service.MemoryStorageService.load_memory_config() Args: - config_id: 配置 ID（整数或字符串，会自动转换） - force_reload: 保留参数以保持向后兼容（已移除缓存逻辑） + config_id: Configuration ID (deprecated) + force_reload: Force reload flag (deprecated) Returns: - bool: 是否成功重新加载配置 + bool: Always returns False (deprecated functionality) """ import logging + import warnings + logger = logging.getLogger(__name__) - # 导入审计日志记录器 - try: - from app.core.memory.utils.log.audit_logger import audit_logger - except ImportError: - audit_logger = None + warnings.warn( + "reload_configuration_from_database is deprecated. Use MemoryConfig objects with dependency injection instead.", + DeprecationWarning, + stacklevel=2 + ) - with _config_lock: - try: - from app.core.memory.utils.config.overrides import load_unified_config - except Exception as e: - logger.error(f"[definitions] 导入统一配置加载器失败: {e}") - - # 记录配置加载失败 - if audit_logger: - audit_logger.log_config_load( - config_id=config_id, - success=False, - details={"error": f"Import failed: {str(e)}"} - ) - - return False - - try: - logger.info(f"[definitions] 开始重新加载配置，config_id={config_id}") - - # 使用统一配置加载器（指定 config_id） - updated_cfg = load_unified_config(PROJECT_ROOT, config_id=config_id) - - # 检查是否成功加载 - if not updated_cfg or not updated_cfg.get('selections'): - logger.error(f"[definitions] 配置加载失败：数据库中未找到 config_id={config_id} 的配置") - - # 记录配置加载失败 - if audit_logger: - audit_logger.log_config_load( - config_id=config_id, - success=False, - details={"reason": "config not found in database"} - ) - - return False - - # 重新暴露常量 - _expose_runtime_constants(updated_cfg) - - logger.info("[definitions] 配置重新加载成功，已暴露常量") - logger.debug(f"[definitions] 配置详情: LLM_ID={updated_cfg.get('selections', {}).get('llm_id')}, " - f"EMBEDDING_ID={updated_cfg.get('selections', {}).get('embedding_id')}") - - # 记录成功的配置加载 - if audit_logger: - selections = updated_cfg.get('selections', {}) - audit_logger.log_config_load( - config_id=config_id, - user_id=selections.get('user_id', None), - group_id=selections.get('group_id', None), - success=True, - details={ - "llm_id": selections.get('llm_id'), - "embedding_id": selections.get('embedding_id'), - "chunker_strategy": selections.get('chunker_strategy') - } - ) - - return True - except Exception as e: - logger.error(f"[definitions] 重新加载配置时发生异常: {e}", exc_info=True) - - # 记录配置加载异常 - if audit_logger: - audit_logger.log_config_load( - config_id=config_id, - success=False, - details={"error": str(e)} - ) - - return False + logger.warning(f"Deprecated function reload_configuration_from_database called with config_id={config_id}. " + "Use MemoryConfig objects with dependency injection instead.") + + return False @@ -312,49 +216,54 @@ def reload_configuration_from_database(config_id: int | str, force_reload: bool def get_current_config_id() -> Optional[str]: """ - 获取当前使用的 config_id + DEPRECATED: Legacy config ID retrieval + + ⚠️ This function is deprecated and will be removed in a future version. + Use MemoryConfig objects with dependency injection instead. Returns: - Optional[str]: 当前的 config_id，如果未设置则返回 None + Optional[str]: None (deprecated functionality) """ - return SELECTIONS.get("config_id", None) + import warnings + warnings.warn( + "get_current_config_id is deprecated. Use MemoryConfig objects with dependency injection instead.", + DeprecationWarning, + stacklevel=2 + ) + return None -def ensure_fresh_config(config_id: Optional[int | str] = None) -> bool: +def ensure_fresh_config(config_id = None) -> bool: """ - 确保使用最新的配置（每次写入操作前调用） + DEPRECATED: Legacy configuration freshness check - 如果提供了 config_id，则加载该配置； - 否则从 dbrun.json 读取并加载最新配置。 + ⚠️ This function is deprecated and will be removed in a future version. + Use MemoryConfig objects with dependency injection instead. + For new code, use: + - app.services.memory_agent_service.MemoryAgentService.load_memory_config() + - app.services.memory_storage_service.MemoryStorageService.load_memory_config() + Args: - config_id: 可选的配置ID（整数或字符串，会自动转换） + config_id: Configuration ID (deprecated) Returns: - bool: 是否成功加载配置 + bool: Always returns False (deprecated functionality) """ import logging + import warnings + logger = logging.getLogger(__name__) - with _config_lock: - try: - if config_id: - # 使用指定的 config_id - logger.debug(f"[definitions] 加载指定配置，config_id={config_id}") - return reload_configuration_from_database(config_id) - else: - # 从数据库重新加载配置 - logger.debug("[definitions] 从数据库重新加载最新配置") - memory_config = _load_from_database() - - if not memory_config or not memory_config.get('selections'): - logger.warning("[definitions] 未能从数据库加载配置，使用当前配置") - return False - - _expose_memory_constants(memory_config) - return True - except Exception as e: - logger.error(f"[definitions] 加载配置失败: {e}", exc_info=True) - return False + warnings.warn( + "ensure_fresh_config is deprecated. Use MemoryConfig objects with dependency injection instead.", + DeprecationWarning, + stacklevel=2 + ) + + logger.warning(f"Deprecated function ensure_fresh_config called with config_id={config_id}. " + "Use MemoryConfig objects with dependency injection instead.") + + return False diff --git a/api/app/core/memory/utils/config/overrides.py b/api/app/core/memory/utils/config/overrides.py deleted file mode 100644 index e333bb29..00000000 --- a/api/app/core/memory/utils/config/overrides.py +++ /dev/null @@ -1,611 +0,0 @@ -""" -运行时配置覆写工具 - 统一配置加载器 - -本模块作为统一的配置加载器，负责从多个来源加载配置并按优先级覆写。 - -配置来源优先级（从高到低）： -1. 数据库配置（PostgreSQL data_config 表） -2. 环境变量配置（.env 文件） -3. 默认配置（runtime.json 文件） - -支持的配置加载方式： -- 基于 config_id 的配置加载（从 dbrun.json 读取或前端传入） -- 基于 group_id 的配置加载（从 dbrun.json 读取） -- 环境变量覆写（支持 INTERNAL/EXTERNAL 网络模式） - -主要功能： -- 从 PostgreSQL 数据库读取配置 -- 从环境变量读取配置 -- 从 runtime.json 读取默认配置 -- 按优先级覆写配置项（仅在内存中，不修改文件） -- 支持多种配置字段：selections、statement_extraction、deduplication、forgetting_engine、pruning、reflexion - -使用场景： -- 应用启动时自动加载配置 -- 前端切换配置时动态重新加载 -- 多租户场景下的配置隔离 -- 内外网环境自动切换 -""" -import os -import json -import socket -from typing import Optional, Dict, Any, Literal - -NetworkMode = Literal['internal', 'external'] - - -def _set_if_present(target: Dict[str, Any], target_key: str, src: Dict[str, Any], src_key: str, caster): - """安全地设置目标字典的值（如果源字典中存在且不为 None） - - Args: - target: 目标字典 - target_key: 目标字典的键 - src: 源字典 - src_key: 源字典的键 - caster: 类型转换函数 - """ - try: - if src_key in src and src.get(src_key) is not None: - try: - target[target_key] = caster(src.get(src_key)) - except Exception: - pass - except Exception: - pass - - -def _to_bool(val: Any) -> bool: - """将各种类型的值转换为布尔值 - - 支持的输入： - - bool: 直接返回 - - int/float: 非零为 True - - str: "true", "1", "on", "yes" 为 True；"false", "0", "off", "no" 为 False - - Args: - val: 要转换的值 - - Returns: - bool: 转换后的布尔值 - """ - try: - if isinstance(val, bool): - return val - if isinstance(val, (int, float)): - return bool(val) - if isinstance(val, str): - m = val.strip().lower() - if m in {"true", "1", "on", "yes"}: - return True - if m in {"false", "0", "off", "no"}: - return False - return bool(val) - except Exception: - return False - - -def _make_pgsql_conn() -> Optional[object]: - """创建 PostgreSQL 数据库连接 - - 使用环境变量配置连接参数： - - DB_HOST: 数据库主机地址（默认 localhost） - - DB_PORT: 数据库端口（默认 5432） - - DB_USER: 数据库用户名 - - DB_PASSWORD: 数据库密码 - - DB_NAME: 数据库名称 - - Returns: - Optional[object]: 数据库连接对象，失败时返回 None - """ - host = os.getenv("DB_HOST", "localhost") - user = os.getenv("DB_USER") - password = os.getenv("DB_PASSWORD") - dbname = os.getenv("DB_NAME") - port_str = os.getenv("DB_PORT") - - try: - import psycopg2 # type: ignore - from psycopg2.extras import RealDictCursor # type: ignore - - port = int(port_str) if port_str else 5432 - conn = psycopg2.connect( - host=host, - port=port, - user=user, - password=password, - dbname=dbname, - ) - conn.autocommit = True - return conn - except Exception: - return None - - -def _fetch_db_config_by_group_id(group_id: str) -> Optional[Dict[str, Any]]: - """根据 group_id 从数据库查询配置 - - Args: - group_id: 组标识符 - - Returns: - Optional[Dict[str, Any]]: 配置字典，未找到时返回 None - """ - conn = _make_pgsql_conn() - if conn is None: - return None - - try: - from psycopg2.extras import RealDictCursor # type: ignore - cur = conn.cursor(cursor_factory=RealDictCursor) - - try: - cur.execute("SET TIME ZONE %s", ("Asia/Shanghai",)) - except Exception: - pass - - sql = ( - "SELECT group_id, user_id, apply_id, chunker_strategy, " - " enable_llm_dedup_blockwise, enable_llm_disambiguation " - "FROM data_config WHERE group_id = %s ORDER BY updated_at DESC LIMIT 1" - ) - cur.execute(sql, (group_id,)) - row = cur.fetchone() - return row if row else None - except Exception: - return None - finally: - try: - cur.close() - except Exception: - pass - try: - conn.close() - except Exception: - pass - - -def _fetch_db_config_by_config_id(config_id: int | str) -> Optional[Dict[str, Any]]: - """根据 config_id 从数据库查询配置 - - Args: - config_id: 配置标识符（整数或字符串，会自动转换为整数） - - Returns: - Optional[Dict[str, Any]]: 配置字典，未找到时返回 None - """ - conn = _make_pgsql_conn() - if conn is None: - try: - pass - except Exception: - pass - return None - - try: - from psycopg2.extras import RealDictCursor # type: ignore - cur = conn.cursor(cursor_factory=RealDictCursor) - - try: - cur.execute("SET TIME ZONE %s", ("Asia/Shanghai",)) - except Exception: - pass - - # config_id 在数据库中是 Integer 类型，需要转换 - try: - config_id_int = int(config_id) - except (ValueError, TypeError) as e: - try: - pass - except Exception: - pass - return None - - sql = ( - "SELECT config_id, group_id, user_id, apply_id, chunker_strategy, " - " enable_llm_dedup_blockwise, enable_llm_disambiguation, " - " deep_retrieval, t_type_strict, t_name_strict, t_overall, state, " - " statement_granularity, include_dialogue_context, max_context, " - " \"offset\" AS offset, lambda_time, lambda_mem, " - " pruning_enabled, pruning_scene, pruning_threshold, " - " llm_id, embedding_id " - "FROM data_config WHERE config_id = %s LIMIT 1" - ) - cur.execute(sql, (config_id_int,)) - row = cur.fetchone() - - if row: - try: - pass - except Exception: - pass - else: - pass - - return row if row else None - except Exception as e: - pass - return None - finally: - try: - cur.close() - except Exception: - pass - try: - conn.close() - except Exception: - pass - - -def _load_dbrun_group_id(project_root: str) -> Optional[str]: - """从 dbrun.json 读取 group_id - - Args: - project_root: 项目根目录路径 - - Returns: - Optional[str]: group_id，未找到时返回 None - """ - try: - path = os.path.join(project_root, "dbrun.json") - if not os.path.isfile(path): - return None - - with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - - if isinstance(data, dict): - if "group_id" in data: - return str(data.get("group_id")) - sel = data.get("selections", {}) - if isinstance(sel, dict) and "group_id" in sel: - return str(sel.get("group_id")) - - return None - except Exception: - return None - - -def _load_dbrun_config_id(project_root: str) -> Optional[str]: - """从 dbrun.json 读取 config_id - - Args: - project_root: 项目根目录路径 - - Returns: - Optional[str]: config_id，未找到时返回 None - """ - try: - path = os.path.join(project_root, "dbrun.json") - if not os.path.isfile(path): - return None - - with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - - if isinstance(data, dict): - if "config_id" in data: - return str(data.get("config_id")) - sel = data.get("selections", {}) - if isinstance(sel, dict) and "config_id" in sel: - return str(sel.get("config_id")) - - return None - except Exception: - return None - - -def _apply_overrides_from_db_row( - runtime_cfg: Dict[str, Any], - db_row: Optional[Dict[str, Any]], - identifier: str, - identifier_type: str = "config_id" -) -> Dict[str, Any]: - """从数据库行数据覆写运行时配置（统一处理函数） - - Args: - runtime_cfg: 运行时配置字典 - db_row: 数据库查询结果行 - identifier: 标识符值（group_id 或 config_id） - identifier_type: 标识符类型（"group_id" 或 "config_id"） - - Returns: - Dict[str, Any]: 覆写后的运行时配置 - """ - try: - selections = runtime_cfg.setdefault("selections", {}) - selections[identifier_type] = identifier - - if not db_row: - return runtime_cfg - - # 覆写 selections 字段 - for tk in ("group_id", "user_id", "apply_id", "chunker_strategy", "state", - "t_type_strict", "t_name_strict", "t_overall", - "statement_granularity", "include_dialogue_context"): - _set_if_present(selections, tk, db_row, tk, str) - - # 特殊处理 UUID 字段，确保转换为字符串格式 - for uuid_field in ("llm_id", "embedding_id"): - if uuid_field in db_row and db_row.get(uuid_field) is not None: - try: - value = db_row.get(uuid_field) - # 如果是 UUID 对象，转换为字符串（带连字符的标准格式） - if hasattr(value, 'hex'): - selections[uuid_field] = str(value) - else: - selections[uuid_field] = str(value) - except Exception: - pass - - # 覆写 statement_extraction 字段 - stmt = runtime_cfg.setdefault("statement_extraction", {}) - _set_if_present(stmt, "statement_granularity", db_row, "statement_granularity", int) - _set_if_present(stmt, "include_dialogue_context", db_row, "include_dialogue_context", _to_bool) - _set_if_present(stmt, "max_dialogue_context_chars", db_row, "max_context", int) - - # 覆写 deduplication 字段 - dedup = runtime_cfg.setdefault("deduplication", {}) - for tk in ("enable_llm_dedup_blockwise", "enable_llm_disambiguation"): - _set_if_present(dedup, tk, db_row, tk, _to_bool) - _set_if_present(dedup, "deep_retrieval", db_row, "deep_retrieval", _to_bool) - - # 覆写 forgetting_engine 字段 - forgetting = runtime_cfg.setdefault("forgetting_engine", {}) - _set_if_present(forgetting, "offset", db_row, "offset", float) - _set_if_present(forgetting, "lambda_time", db_row, "lambda_time", float) - _set_if_present(forgetting, "lambda_mem", db_row, "lambda_mem", float) - - # 覆写 pruning 字段 - pruning = runtime_cfg.setdefault("pruning", {}) - _set_if_present(pruning, "enabled", db_row, "pruning_enabled", _to_bool) - _set_if_present(pruning, "scene", db_row, "pruning_scene", str) - - # 阈值需要转为 float，且限制在 [0.0, 0.9] - try: - if "pruning_threshold" in db_row and db_row.get("pruning_threshold") is not None: - thr = float(db_row.get("pruning_threshold")) - thr = max(0.0, min(0.9, thr)) # 限制在 [0.0, 0.9] - pruning["threshold"] = thr - except Exception: - pass - - return runtime_cfg - except Exception as e: - pass - return runtime_cfg - - -def apply_runtime_overrides_by_group(project_root: str, runtime_cfg: Dict[str, Any]) -> Dict[str, Any]: - """基于 group_id 从数据库覆写运行时配置 - - 工作流程： - 1. 从 dbrun.json 读取 group_id - 2. 根据 group_id 查询数据库配置 - 3. 覆写运行时配置（仅在内存中） - - Args: - project_root: 项目根目录路径 - runtime_cfg: 运行时配置字典 - - Returns: - Dict[str, Any]: 覆写后的运行时配置 - """ - try: - selected_gid = _load_dbrun_group_id(project_root) - if not selected_gid: - return runtime_cfg - - db_row = _fetch_db_config_by_group_id(selected_gid) - if not db_row: - # 如果数据库中没有配置，仍然设置 group_id - runtime_cfg.setdefault("selections", {})["group_id"] = selected_gid - return runtime_cfg - - return _apply_overrides_from_db_row(runtime_cfg, db_row, selected_gid, "group_id") - except Exception: - return runtime_cfg - - -def apply_runtime_overrides_by_config(project_root: str, runtime_cfg: Dict[str, Any]) -> Dict[str, Any]: - """基于 config_id 从数据库覆写运行时配置（从 dbrun.json 读取） - - 工作流程： - 1. 从 dbrun.json 读取 config_id - 2. 根据 config_id 查询数据库配置 - 3. 覆写运行时配置（仅在内存中） - - Args: - project_root: 项目根目录路径 - runtime_cfg: 运行时配置字典 - - Returns: - Dict[str, Any]: 覆写后的运行时配置 - """ - try: - selected_cid = _load_dbrun_config_id(project_root) - if not selected_cid: - return runtime_cfg - - db_row = _fetch_db_config_by_config_id(selected_cid) - return _apply_overrides_from_db_row(runtime_cfg, db_row, selected_cid, "config_id") - except Exception: - return runtime_cfg - - -def apply_runtime_overrides_with_config_id( - project_root: str, - runtime_cfg: Dict[str, Any], - config_id: str -) -> tuple[Dict[str, Any], bool]: - """使用指定的 config_id 从数据库覆写运行时配置（不读 dbrun.json） - - 用于前端动态切换配置的场景。 - - Args: - project_root: 项目根目录路径 - runtime_cfg: 运行时配置字典 - config_id: 配置标识符 - - Returns: - tuple[Dict[str, Any], bool]: (覆写后的运行时配置, 是否成功从数据库加载) - """ - try: - selected_cid = str(config_id).strip() - if not selected_cid: - return runtime_cfg, False - - db_row = _fetch_db_config_by_config_id(selected_cid) - if db_row is None: - return runtime_cfg, False - - updated_cfg = _apply_overrides_from_db_row(runtime_cfg, db_row, selected_cid, "config_id") - return updated_cfg, True - except Exception as e: - pass - return runtime_cfg, False - - -# ============================================================================ -# 以下函数已注释：不再需要网络模式自动检测功能 -# ============================================================================ - -# def get_server_ip() -> str: -# """ -# 获取当前服务器的IP地址 -# -# Returns: -# 服务器IP地址字符串 -# """ -# try: -# # 方式1：从环境变量获取（优先） -# server_ip = os.getenv('SERVER_IP') -# if server_ip and server_ip not in ['127.0.0.1', 'localhost', '0.0.0.0']: -# return server_ip -# -# # 方式2：通过socket获取 -# hostname = socket.gethostname() -# ip_address = socket.gethostbyname(hostname) -# -# # 如果是本地回环地址，尝试获取真实IP -# if ip_address.startswith('127.'): -# # 尝试连接外部地址来获取本机IP -# s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -# try: -# s.connect(('8.8.8.8', 80)) -# ip_address = s.getsockname()[0] -# finally: -# s.close() -# -# return ip_address -# except Exception as e: -# print(f"[overrides] 获取服务器IP失败: {e}，使用默认值 127.0.0.1") -# return '127.0.0.1' - - -# def auto_detect_network_mode() -> NetworkMode: -# """ -# 自动检测网络模式（基于服务器IP） -# -# 规则： -# - 如果服务器IP在内网IP列表中 → internal（内网） -# - 其他IP → external（外网） -# -# 可以通过环境变量 INTERNAL_SERVER_IPS 自定义内网IP列表（逗号分隔） -# -# Returns: -# 'internal' 或 'external' -# """ -# server_ip = get_server_ip() -# -# # 从环境变量获取内网IP列表（支持多个IP，逗号分隔） -# internal_ips_str = os.getenv('INTERNAL_SERVER_IPS', '119.45.181.55') -# internal_ips = [ip.strip() for ip in internal_ips_str.split(',')] -# -# # 判断当前IP是否在内网IP列表中 -# if server_ip in internal_ips: -# print(f"[overrides] 自动检测：服务器IP {server_ip} 属于内网，使用 INTERNAL 配置") -# return 'internal' -# else: -# print(f"[overrides] 自动检测：服务器IP {server_ip} 属于外网，使用 EXTERNAL 配置") -# return 'external' - - -# ============================================================================ -# 环境变量覆写功能已废弃 - 不再使用 -# ============================================================================ -# def _apply_env_var_overrides(runtime_cfg: Dict[str, Any], network_mode: NetworkMode = None, force_override: bool = False) -> Dict[str, Any]: -# """ -# 从环境变量覆写配置（已废弃） -# """ -# return runtime_cfg - - -def load_unified_config( - project_root: str, - config_id: Optional[int | str] = None, - group_id: Optional[str] = None, - network_mode: NetworkMode = None, - env_override_models: bool = True -) -> Dict[str, Any]: - """ - 统一配置加载器 - 按优先级加载配置 - - 配置加载优先级： - 1. PG数据库配置（最高优先级，通过 dbrun.json 中的 config_id 读取） - 2. runtime.json 默认配置（最低优先级） - - Args: - project_root: 项目根目录路径 - config_id: 配置ID（整数或字符串，可选，优先从 dbrun.json 读取） - group_id: 组ID（可选） - network_mode: 已废弃，保留参数仅为向后兼容 - env_override_models: 已废弃，保留参数仅为向后兼容 - - Returns: - Dict[str, Any]: 最终的运行时配置 - """ - try: - # 步骤 1: 加载 runtime.json 作为基础配置 - runtime_config_path = os.path.join(project_root, "runtime.json") - try: - with open(runtime_config_path, "r", encoding="utf-8") as f: - runtime_cfg = json.load(f) - except (FileNotFoundError, json.JSONDecodeError) as e: - runtime_cfg = {"selections": {}} - - # 步骤 2: 尝试从 dbrun.json 读取 config_id 并应用数据库配置（最高优先级） - if config_id: - # 优先使用传入的 config_id - db_row = _fetch_db_config_by_config_id(config_id) - if db_row: - runtime_cfg = _apply_overrides_from_db_row(runtime_cfg, db_row, config_id, "config_id") - pass - elif group_id: - # 其次使用 group_id - db_row = _fetch_db_config_by_group_id(group_id) - if db_row: - runtime_cfg = _apply_overrides_from_db_row(runtime_cfg, db_row, group_id, "group_id") - pass - else: - # 尝试从 dbrun.json 读取 - dbrun_config_id = _load_dbrun_config_id(project_root) - if dbrun_config_id: - db_row = _fetch_db_config_by_config_id(dbrun_config_id) - if db_row: - runtime_cfg = _apply_overrides_from_db_row(runtime_cfg, db_row, dbrun_config_id, "config_id") - pass - else: - dbrun_group_id = _load_dbrun_group_id(project_root) - if dbrun_group_id: - db_row = _fetch_db_config_by_group_id(dbrun_group_id) - if db_row: - runtime_cfg = _apply_overrides_from_db_row(runtime_cfg, db_row, dbrun_group_id, "group_id") - pass - return runtime_cfg - - except Exception as e: - return {"selections": {}} - - -# 向后兼容的别名 -apply_runtime_overrides = apply_runtime_overrides_by_config diff --git a/api/app/core/memory/utils/embedder/__init__.py b/api/app/core/memory/utils/embedder/__init__.py new file mode 100644 index 00000000..6ffd7282 --- /dev/null +++ b/api/app/core/memory/utils/embedder/__init__.py @@ -0,0 +1,11 @@ +"""Embedder utilities module.""" + +from app.core.memory.utils.embedder.embedder_utils import ( + get_embedder_client, + get_embedder_client_from_config, +) + +__all__ = [ + "get_embedder_client", + "get_embedder_client_from_config", +] diff --git a/api/app/core/memory/utils/embedder/embedder_utils.py b/api/app/core/memory/utils/embedder/embedder_utils.py new file mode 100644 index 00000000..86899e30 --- /dev/null +++ b/api/app/core/memory/utils/embedder/embedder_utils.py @@ -0,0 +1,81 @@ +"""Embedder Client Utilities + +This module provides centralized functions for creating embedder clients. +""" + +from typing import TYPE_CHECKING + +from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient +from app.core.memory.utils.config.config_utils import get_embedder_config +from app.core.models.base import RedBearModelConfig + +if TYPE_CHECKING: + from app.schemas.memory_config_schema import MemoryConfig + + +def get_embedder_client_from_config(memory_config: "MemoryConfig") -> OpenAIEmbedderClient: + """ + Get embedder client from MemoryConfig object. + + **PREFERRED METHOD**: Use this function in production code when you have a MemoryConfig object. + This ensures proper configuration management and multi-tenant support. + + Args: + memory_config: MemoryConfig object containing embedding_model_id + + Returns: + OpenAIEmbedderClient: Initialized embedder client + + Raises: + ValueError: If embedding model ID is not configured or client initialization fails + + Example: + >>> embedder_client = get_embedder_client_from_config(memory_config) + """ + if not memory_config.embedding_model_id: + raise ValueError( + f"Configuration {memory_config.config_id} has no embedding model configured" + ) + return get_embedder_client(str(memory_config.embedding_model_id)) + + +def get_embedder_client(embedding_id: str) -> OpenAIEmbedderClient: + """ + Get embedder client by model ID. + + **LEGACY/TEST METHOD**: Use this function only for: + - Test/evaluation code where you have a model ID directly + - Legacy code that hasn't been migrated to MemoryConfig yet + + For production code with MemoryConfig, use get_embedder_client_from_config() instead. + + Args: + embedding_id: Embedding model ID (required) + + Returns: + OpenAIEmbedderClient: Initialized embedder client + + Raises: + ValueError: If embedding_id is not provided or client initialization fails + + Example: + >>> # For tests/evaluations only + >>> embedder_client = get_embedder_client("model-uuid-string") + """ + if not embedding_id: + raise ValueError("Embedding ID is required but was not provided") + + try: + embedder_config_dict = get_embedder_config(embedding_id) + except Exception as e: + raise ValueError(f"Invalid embedding ID '{embedding_id}': {str(e)}") from e + + try: + embedder_config = RedBearModelConfig(**embedder_config_dict) + embedder_client = OpenAIEmbedderClient(embedder_config) + return embedder_client + except Exception as e: + model_name = embedder_config_dict.get('model_name', 'unknown') + raise ValueError( + f"Failed to initialize embedder client for model '{model_name}': {str(e)}" + ) from e diff --git a/api/app/core/memory/utils/llm/llm_utils.py b/api/app/core/memory/utils/llm/llm_utils.py index 5f8618c2..a4b327da 100644 --- a/api/app/core/memory/utils/llm/llm_utils.py +++ b/api/app/core/memory/utils/llm/llm_utils.py @@ -1,67 +1,107 @@ -import os -from pydantic import BaseModel +from typing import TYPE_CHECKING from app.core.memory.llm_tools.openai_client import OpenAIClient from app.core.memory.utils.config.config_utils import get_model_config -from app.core.memory.utils.config import definitions as config_defs from app.core.models.base import RedBearModelConfig +from pydantic import BaseModel + +if TYPE_CHECKING: + from app.schemas.memory_config_schema import MemoryConfig + async def handle_response(response: type[BaseModel]) -> dict: return response.model_dump() -def get_llm_client(llm_id: str | None = None): - llm_id = llm_id or config_defs.SELECTED_LLM_ID +def get_llm_client_from_config(memory_config: "MemoryConfig") -> OpenAIClient: + """ + Get LLM client from MemoryConfig object. + + **PREFERRED METHOD**: Use this function in production code when you have a MemoryConfig object. + This ensures proper configuration management and multi-tenant support. + + Args: + memory_config: MemoryConfig object containing llm_model_id + + Returns: + OpenAIClient: Initialized LLM client + + Raises: + ValueError: If LLM model ID is not configured or client initialization fails + + Example: + >>> llm_client = get_llm_client_from_config(memory_config) + """ + if not memory_config.llm_model_id: + raise ValueError( + f"Configuration {memory_config.config_id} has no LLM model configured" + ) + return get_llm_client(str(memory_config.llm_model_id)) - # Validate LLM ID exists before attempting to get config + +def get_llm_client(llm_id: str): + """ + Get LLM client by model ID. + + **LEGACY/TEST METHOD**: Use this function only for: + - Test/evaluation code where you have a model ID directly + - Legacy code that hasn't been migrated to MemoryConfig yet + + For production code with MemoryConfig, use get_llm_client_from_config() instead. + + Args: + llm_id: LLM model ID (required) + + Returns: + OpenAIClient: Initialized LLM client + + Raises: + ValueError: If llm_id is not provided or client initialization fails + + Example: + >>> # For tests/evaluations only + >>> llm_client = get_llm_client("model-uuid-string") + """ if not llm_id: raise ValueError("LLM ID is required but was not provided") try: model_config = get_model_config(llm_id) except Exception as e: - # Re-raise with clear error message about invalid LLM ID raise ValueError(f"Invalid LLM ID '{llm_id}': {str(e)}") from e try: - # 移除调试打印，避免污染终端输出 - # print(model_config) llm_client = OpenAIClient(RedBearModelConfig( model_name=model_config.get("model_name"), provider=model_config.get("provider"), api_key=model_config.get("api_key"), base_url=model_config.get("base_url") ),type_=model_config.get("type")) - # print(llm.dict()) return llm_client except Exception as e: model_name = model_config.get('model_name', 'unknown') raise ValueError(f"Failed to initialize LLM client for model '{model_name}': {str(e)}") from e -def get_reranker_client(rerank_id: str | None = None): +def get_reranker_client(rerank_id: str): """ Get an LLM client configured for reranking. Args: - rerank_id: Optional reranker model ID. If None, uses SELECTED_RERANK_ID. + rerank_id: Reranker model ID (required) Returns: OpenAIClient: Initialized client for the reranker model Raises: - ValueError: If rerank_id is invalid or client initialization fails + ValueError: If rerank_id is not provided or client initialization fails """ - rerank_id = rerank_id or config_defs.SELECTED_RERANK_ID - - # Validate rerank ID exists before attempting to get config if not rerank_id: raise ValueError("Rerank ID is required but was not provided") try: model_config = get_model_config(rerank_id) except Exception as e: - # Re-raise with clear error message about invalid rerank ID raise ValueError(f"Invalid rerank ID '{rerank_id}': {str(e)}") from e try: diff --git a/api/app/core/memory/utils/self_reflexion_utils/self_reflexion.py b/api/app/core/memory/utils/self_reflexion_utils/self_reflexion.py index 2c3943e2..2e63ae22 100644 --- a/api/app/core/memory/utils/self_reflexion_utils/self_reflexion.py +++ b/api/app/core/memory/utils/self_reflexion_utils/self_reflexion.py @@ -10,28 +10,29 @@ 从 app.core.memory.src.data_config_api.self_reflexion 迁移而来。 """ -import os +import asyncio import json import logging -import asyncio -from typing import List, Dict, Any +import os import uuid +from typing import Any, Dict, List + +#TODO: Fix this + +# Default values (previously from definitions.py) +REFLEXION_ENABLED = os.getenv("REFLEXION_ENABLED", "false").lower() == "true" +REFLEXION_ITERATION_PERIOD = os.getenv("REFLEXION_ITERATION_PERIOD", "3") +REFLEXION_RANGE = os.getenv("REFLEXION_RANGE", "retrieval") +REFLEXION_BASELINE = os.getenv("REFLEXION_BASELINE", "TIME") -from app.core.memory.utils.config.definitions import ( - REFLEXION_ENABLED, - REFLEXION_ITERATION_PERIOD, - REFLEXION_RANGE, - REFLEXION_BASELINE, -) -from app.db import get_db -from sqlalchemy.orm import Session -from app.models.retrieval_info import RetrievalInfo from app.core.memory.utils.config.get_data import get_data from app.core.memory.utils.self_reflexion_utils.evaluate import conflict from app.core.memory.utils.self_reflexion_utils.reflexion import reflexion +from app.db import get_db +from app.models.retrieval_info import RetrievalInfo from app.repositories.neo4j.cypher_queries import UPDATE_STATEMENT_INVALID_AT from app.repositories.neo4j.neo4j_connector import Neo4jConnector - +from sqlalchemy.orm import Session # 并发限制（可通过环境变量覆盖） CONCURRENCY = int(os.getenv("REFLEXION_CONCURRENCY", "5")) diff --git a/api/app/core/validators/__init__.py b/api/app/core/validators/__init__.py index a53b6b71..1fadc869 100644 --- a/api/app/core/validators/__init__.py +++ b/api/app/core/validators/__init__.py @@ -1,6 +1,21 @@ """ -Validators for file upload system. +Validators package for various validation utilities. """ from app.core.validators.file_validator import FileValidator, ValidationResult +from app.core.validators.memory_config_validators import ( + validate_and_resolve_model_id, + validate_embedding_model, + validate_llm_model, + validate_model_exists_and_active, +) -__all__ = ["FileValidator", "ValidationResult"] +__all__ = [ + # File validators + "FileValidator", + "ValidationResult", + # Memory config validators + "validate_model_exists_and_active", + "validate_and_resolve_model_id", + "validate_embedding_model", + "validate_llm_model", +] diff --git a/api/app/core/validators/memory_config_validators.py b/api/app/core/validators/memory_config_validators.py new file mode 100644 index 00000000..84c5b20c --- /dev/null +++ b/api/app/core/validators/memory_config_validators.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +"""Memory Configuration Validators + +This module provides validation functions for memory configuration models. + +Functions: + validate_model_exists_and_active: Validate model exists and is active + validate_and_resolve_model_id: Validate and resolve model ID with DB lookup + validate_embedding_model: Validate embedding model availability + validate_llm_model: Validate LLM model availability +""" + +import time +from typing import Optional, Union +from uuid import UUID + +from app.core.logging_config import get_config_logger +from app.schemas.memory_config_schema import ( + InvalidConfigError, + ModelInactiveError, + ModelNotFoundError, +) +from sqlalchemy.orm import Session + +logger = get_config_logger() + + +def _parse_model_id(model_id: Union[str, UUID, None], model_type: str, + config_id: Optional[int] = None, workspace_id: Optional[UUID] = None) -> Optional[UUID]: + """Parse model ID from string or UUID.""" + if model_id is None: + return None + if isinstance(model_id, UUID): + return model_id + if isinstance(model_id, str): + if not model_id.strip(): + return None + try: + return UUID(model_id.strip()) + except ValueError: + raise InvalidConfigError( + f"Invalid UUID format for {model_type} model ID: '{model_id}'", + field_name=f"{model_type}_model_id", + invalid_value=model_id, + config_id=config_id, + workspace_id=workspace_id + ) + raise InvalidConfigError( + f"Invalid type for {model_type} model ID: expected str or UUID, got {type(model_id).__name__}", + field_name=f"{model_type}_model_id", + invalid_value=model_id, + config_id=config_id, + workspace_id=workspace_id + ) + + +def validate_model_exists_and_active( + model_id: UUID, + model_type: str, + db: Session, + tenant_id: Optional[UUID] = None, + config_id: Optional[int] = None, + workspace_id: Optional[UUID] = None +) -> tuple[str, bool]: + """Validate that a model exists and is active. + + Args: + model_id: Model UUID to validate + model_type: Type of model ("llm", "embedding", "rerank") + db: Database session + tenant_id: Optional tenant ID for filtering + config_id: Optional configuration ID for error context + workspace_id: Optional workspace ID for error context + + Returns: + Tuple of (model_name, is_active) + + Raises: + ModelNotFoundError: If model does not exist + ModelInactiveError: If model exists but is inactive + """ + from app.repositories.model_repository import ModelConfigRepository + + start_time = time.time() + + try: + model = ModelConfigRepository.get_by_id(db, model_id, tenant_id) + elapsed_ms = (time.time() - start_time) * 1000 + + if not model: + logger.warning( + "Model not found", + extra={"model_id": str(model_id), "model_type": model_type, "elapsed_ms": elapsed_ms} + ) + raise ModelNotFoundError( + model_id=model_id, + model_type=model_type, + config_id=config_id, + workspace_id=workspace_id, + message=f"{model_type.title()} model {model_id} not found" + ) + + if not model.is_active: + logger.warning( + "Model inactive", + extra={"model_id": str(model_id), "model_name": model.name, "elapsed_ms": elapsed_ms} + ) + raise ModelInactiveError( + model_id=model_id, + model_name=model.name, + model_type=model_type, + config_id=config_id, + workspace_id=workspace_id, + message=f"{model_type.title()} model {model_id} ({model.name}) is inactive" + ) + + logger.debug( + "Model validation successful", + extra={"model_id": str(model_id), "model_name": model.name, "elapsed_ms": elapsed_ms} + ) + return model.name, model.is_active + + except (ModelNotFoundError, ModelInactiveError): + raise + except Exception as e: + logger.error(f"Model validation failed: {e}", exc_info=True) + raise + + +def validate_and_resolve_model_id( + model_id_str: Union[str, UUID, None], + model_type: str, + db: Session, + tenant_id: Optional[UUID] = None, + required: bool = False, + config_id: Optional[int] = None, + workspace_id: Optional[UUID] = None +) -> tuple[Optional[UUID], Optional[str]]: + """Validate and resolve a model ID, checking existence and active status. + + Returns: + Tuple of (validated_uuid, model_name) or (None, None) if not required and empty + """ + if model_id_str is None or (isinstance(model_id_str, str) and not model_id_str.strip()): + if required: + raise InvalidConfigError( + f"{model_type.title()} model ID is required", + field_name=f"{model_type}_model_id", + invalid_value=model_id_str, + config_id=config_id, + workspace_id=workspace_id + ) + return None, None + + model_uuid = _parse_model_id(model_id_str, model_type, config_id, workspace_id) + if model_uuid is None: + if required: + raise InvalidConfigError( + f"{model_type.title()} model ID is required", + field_name=f"{model_type}_model_id", + invalid_value=model_id_str, + config_id=config_id, + workspace_id=workspace_id + ) + return None, None + + model_name, _ = validate_model_exists_and_active( + model_uuid, model_type, db, tenant_id, config_id, workspace_id + ) + return model_uuid, model_name + + +def validate_embedding_model( + config_id: int, + embedding_id: Union[str, UUID, None], + db: Session, + tenant_id: Optional[UUID] = None, + workspace_id: Optional[UUID] = None +) -> UUID: + """Validate that embedding model is available and return its UUID. + + Raises: + InvalidConfigError: If embedding_id is not provided or invalid + ModelNotFoundError: If embedding model does not exist + ModelInactiveError: If embedding model is inactive + """ + if embedding_id is None or (isinstance(embedding_id, str) and not embedding_id.strip()): + raise InvalidConfigError( + f"Configuration {config_id} has no embedding model configured", + field_name="embedding_model_id", + invalid_value=embedding_id, + config_id=config_id, + workspace_id=workspace_id + ) + + embedding_uuid, _ = validate_and_resolve_model_id( + embedding_id, "embedding", db, tenant_id, required=True, + config_id=config_id, workspace_id=workspace_id + ) + + if embedding_uuid is None: + raise InvalidConfigError( + f"Configuration {config_id} has no embedding model configured", + field_name="embedding_model_id", + invalid_value=embedding_id, + config_id=config_id, + workspace_id=workspace_id + ) + + return embedding_uuid + + +def validate_llm_model( + config_id: int, + llm_id: Union[str, UUID, None], + db: Session, + tenant_id: Optional[UUID] = None, + workspace_id: Optional[UUID] = None +) -> UUID: + """Validate that LLM model is available and return its UUID. + + Raises: + InvalidConfigError: If llm_id is not provided or invalid + ModelNotFoundError: If LLM model does not exist + ModelInactiveError: If LLM model is inactive + """ + if llm_id is None or (isinstance(llm_id, str) and not llm_id.strip()): + raise InvalidConfigError( + f"Configuration {config_id} has no LLM model configured", + field_name="llm_model_id", + invalid_value=llm_id, + config_id=config_id, + workspace_id=workspace_id + ) + + llm_uuid, _ = validate_and_resolve_model_id( + llm_id, "llm", db, tenant_id, required=True, + config_id=config_id, workspace_id=workspace_id + ) + + if llm_uuid is None: + raise InvalidConfigError( + f"Configuration {config_id} has no LLM model configured", + field_name="llm_model_id", + invalid_value=llm_id, + config_id=config_id, + workspace_id=workspace_id + ) + + return llm_uuid diff --git a/api/app/models/memory_config_model.py b/api/app/models/memory_config_model.py new file mode 100644 index 00000000..d47c3b52 --- /dev/null +++ b/api/app/models/memory_config_model.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +"""Memory Configuration Model - Backward Compatibility + +This module provides backward compatibility for imports. +All classes have been moved to app.schemas.memory_config_schema. + +DEPRECATED: Import from app.schemas.memory_config_schema instead. +""" + +# Re-export for backward compatibility +from app.schemas.memory_config_schema import ( + ConfigurationError, + InvalidConfigError, + MemoryConfig, + MemoryConfigValidation, + ModelInactiveError, + ModelNotFoundError, + ModelValidation, + WorkspaceNotFoundError, + WorkspaceValidation, + validate_memory_config_data, + validate_model_data, + validate_workspace_data, +) + +__all__ = [ + "ConfigurationError", + "InvalidConfigError", + "MemoryConfig", + "MemoryConfigValidation", + "ModelInactiveError", + "ModelNotFoundError", + "ModelValidation", + "WorkspaceNotFoundError", + "WorkspaceValidation", + "validate_memory_config_data", + "validate_model_data", + "validate_workspace_data", +] diff --git a/api/app/repositories/data_config_repository.py b/api/app/repositories/data_config_repository.py index ed1a482a..caf98ca7 100644 --- a/api/app/repositories/data_config_repository.py +++ b/api/app/repositories/data_config_repository.py @@ -8,24 +8,26 @@ Classes: DataConfigRepository: 数据配置仓储类，提供CRUD操作 """ -from typing import Dict, List, Optional, Tuple -from sqlalchemy.orm import Session -from sqlalchemy import desc import uuid +from typing import Dict, List, Optional, Tuple +from app.core.logging_config import get_config_logger, get_db_logger from app.models.data_config_model import DataConfig from app.schemas.memory_storage_schema import ( + ConfigKey, ConfigParamsCreate, ConfigParamsDelete, ConfigUpdate, ConfigUpdateExtracted, ConfigUpdateForget, - ConfigKey, ) -from app.core.logging_config import get_db_logger +from sqlalchemy import desc +from sqlalchemy.orm import Session # 获取数据库专用日志器 db_logger = get_db_logger() +# 获取配置专用日志器 +config_logger = get_config_logger() class DataConfigRepository: @@ -443,7 +445,129 @@ class DataConfigRepository: except Exception as e: db_logger.error(f"根据ID查询数据配置失败: config_id={config_id} - {str(e)}") raise - + @staticmethod + def get_config_with_workspace(db: Session, config_id: int) -> Optional[tuple]: + """Get data config and its associated workspace information + + Args: + db: Database session + config_id: Configuration ID + + Returns: + Optional[tuple]: (DataConfig, Workspace) tuple, None if not found + + Raises: + ValueError: Raised when config exists but workspace doesn't + """ + import time + + from app.models.workspace_model import Workspace + + start_time = time.time() + + # Log configuration loading start + config_logger.info( + "Loading configuration with workspace", + extra={ + "operation": "get_config_with_workspace", + "config_id": config_id + } + ) + + db_logger.debug(f"Querying data config and workspace: config_id={config_id}") + + try: + # Use join query to get both config and workspace + result = db.query(DataConfig, Workspace).join( + Workspace, DataConfig.workspace_id == Workspace.id + ).filter(DataConfig.config_id == config_id).first() + + elapsed_ms = (time.time() - start_time) * 1000 + + if not result: + # Check if config exists but workspace is missing + config_only = db.query(DataConfig).filter(DataConfig.config_id == config_id).first() + if config_only: + if config_only.workspace_id is None: + config_logger.error( + "Configuration has no associated workspace ID", + extra={ + "operation": "get_config_with_workspace", + "config_id": config_id, + "workspace_id": None, + "load_result": "no_workspace_id", + "elapsed_ms": elapsed_ms + } + ) + db_logger.error(f"Data config {config_id} has no associated workspace ID") + raise ValueError(f"Configuration {config_id} has no associated workspace") + else: + config_logger.error( + "Configuration references non-existent workspace", + extra={ + "operation": "get_config_with_workspace", + "config_id": config_id, + "workspace_id": str(config_only.workspace_id), + "load_result": "workspace_not_found", + "elapsed_ms": elapsed_ms + } + ) + db_logger.error(f"Data config {config_id} references non-existent workspace {config_only.workspace_id}") + raise ValueError(f"Workspace {config_only.workspace_id} not found for configuration {config_id}") + + config_logger.debug( + "Configuration not found", + extra={ + "operation": "get_config_with_workspace", + "config_id": config_id, + "load_result": "not_found", + "elapsed_ms": elapsed_ms + } + ) + db_logger.debug(f"Data config not found: config_id={config_id}") + return None + + config, workspace = result + + # Log successful configuration loading + config_logger.info( + "Configuration with workspace loaded successfully", + extra={ + "operation": "get_config_with_workspace", + "config_id": config_id, + "config_name": config.config_name, + "workspace_id": str(workspace.id), + "workspace_name": workspace.name, + "tenant_id": str(workspace.tenant_id), + "load_result": "success", + "elapsed_ms": elapsed_ms + } + ) + + db_logger.debug(f"Data config and workspace query successful: config={config.config_name}, workspace={workspace.name}") + return (config, workspace) + + except ValueError: + # Re-raise known business exceptions + raise + except Exception as e: + elapsed_ms = (time.time() - start_time) * 1000 + + config_logger.error( + "Failed to load configuration with workspace", + extra={ + "operation": "get_config_with_workspace", + "config_id": config_id, + "load_result": "error", + "error_type": type(e).__name__, + "error_message": str(e), + "elapsed_ms": elapsed_ms + }, + exc_info=True + ) + + db_logger.error(f"Failed to query data config and workspace: config_id={config_id} - {str(e)}") + raise @staticmethod def get_all(db: Session, workspace_id: Optional[uuid.UUID] = None) -> List[DataConfig]: """获取所有配置参数 diff --git a/api/app/schemas/memory_agent_schema.py b/api/app/schemas/memory_agent_schema.py index 47dc6b2a..e6fed0e5 100644 --- a/api/app/schemas/memory_agent_schema.py +++ b/api/app/schemas/memory_agent_schema.py @@ -8,13 +8,13 @@ class UserInput(BaseModel): history: list[dict] search_switch: str group_id: str - config_id: Optional[str] = None + config_id: str class Write_UserInput(BaseModel): message: str group_id: str - config_id: Optional[str] = None + config_id: str class End_User_Information(BaseModel): end_user_name: str # 这是要更新的用户名 diff --git a/api/app/schemas/memory_config_schema.py b/api/app/schemas/memory_config_schema.py new file mode 100644 index 00000000..2ea24be8 --- /dev/null +++ b/api/app/schemas/memory_config_schema.py @@ -0,0 +1,451 @@ +# -*- coding: utf-8 -*- +"""Memory Configuration Schemas + +This module provides schema definitions for memory configuration. + +Classes: + MemoryConfig: Immutable memory configuration loaded from database + MemoryConfigValidation: Pydantic model for configuration validation + WorkspaceValidation: Pydantic model for workspace validation + ModelValidation: Pydantic model for model configuration validation + ConfigurationError: Base exception for configuration-related errors + WorkspaceNotFoundError: Raised when workspace does not exist + ModelNotFoundError: Raised when a required model does not exist + ModelInactiveError: Raised when a required model exists but is inactive + InvalidConfigError: Raised when configuration validation fails +""" + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Dict, Literal, Optional, Union +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator + +# ==================== Configuration Exception Classes ==================== + + +class ConfigurationError(Exception): + """Base exception for configuration-related errors. + + This exception includes context information to help with debugging + and provides detailed error messages for different failure scenarios. + """ + + def __init__( + self, + message: str, + config_id: Optional[int] = None, + workspace_id: Optional[UUID] = None, + context: Optional[Dict[str, Any]] = None, + ): + """Initialize configuration error with context. + + Args: + message: Error message describing the failure + config_id: Optional configuration ID for context + workspace_id: Optional workspace ID for context + context: Optional additional context information + """ + self.config_id = config_id + self.workspace_id = workspace_id + self.context = context or {} + + # Build detailed error message with context + detailed_message = message + if config_id is not None: + detailed_message = f"Configuration {config_id}: {message}" + if workspace_id is not None: + detailed_message = f"{detailed_message} (workspace: {workspace_id})" + + # Add context information if available + if self.context: + context_str = ", ".join(f"{k}={v}" for k, v in self.context.items()) + detailed_message = f"{detailed_message} [Context: {context_str}]" + + super().__init__(detailed_message) + + +class WorkspaceNotFoundError(ConfigurationError): + """Raised when workspace does not exist.""" + + def __init__( + self, + workspace_id: UUID, + config_id: Optional[int] = None, + message: Optional[str] = None, + ): + if message is None: + message = f"Workspace {workspace_id} not found in database" + + context = {"workspace_id": str(workspace_id)} + super().__init__(message, config_id=config_id, workspace_id=workspace_id, context=context) + + +class ModelNotFoundError(ConfigurationError): + """Raised when a required model does not exist.""" + + def __init__( + self, + model_id: Union[str, UUID], + model_type: str, + config_id: Optional[int] = None, + workspace_id: Optional[UUID] = None, + message: Optional[str] = None, + ): + if message is None: + message = f"{model_type.title()} model {model_id} not found in database" + + context = { + "model_id": str(model_id), + "model_type": model_type, + "failure_type": "not_found", + } + super().__init__(message, config_id=config_id, workspace_id=workspace_id, context=context) + + +class ModelInactiveError(ConfigurationError): + """Raised when a required model exists but is inactive.""" + + def __init__( + self, + model_id: Union[str, UUID], + model_name: str, + model_type: str, + config_id: Optional[int] = None, + workspace_id: Optional[UUID] = None, + message: Optional[str] = None, + ): + if message is None: + message = f"{model_type.title()} model {model_id} ({model_name}) is inactive" + + context = { + "model_id": str(model_id), + "model_name": model_name, + "model_type": model_type, + "failure_type": "inactive", + } + super().__init__(message, config_id=config_id, workspace_id=workspace_id, context=context) + + +class InvalidConfigError(ConfigurationError): + """Raised when configuration validation fails.""" + + def __init__( + self, + message: str, + field_name: Optional[str] = None, + invalid_value: Optional[Any] = None, + config_id: Optional[int] = None, + workspace_id: Optional[UUID] = None, + ): + context = {} + if field_name is not None: + context["field_name"] = field_name + if invalid_value is not None: + context["invalid_value"] = str(invalid_value) + context["invalid_value_type"] = type(invalid_value).__name__ + + super().__init__(message, config_id=config_id, workspace_id=workspace_id, context=context) + + +# ==================== Pydantic Validation Models ==================== + + +class MemoryConfigValidation(BaseModel): + """Pydantic model for validating memory configuration data from database.""" + + config_id: int = Field(..., gt=0, description="Configuration ID must be positive") + config_name: str = Field(..., min_length=1, max_length=255) + workspace_id: UUID = Field(..., description="Workspace UUID") + workspace_name: str = Field(..., min_length=1, max_length=255) + tenant_id: UUID = Field(..., description="Tenant UUID") + + embedding_model_id: UUID = Field(..., description="Embedding model UUID (required)") + embedding_model_name: str = Field(..., min_length=1, max_length=255) + llm_model_id: UUID = Field(..., description="LLM model UUID (required)") + llm_model_name: str = Field(..., min_length=1, max_length=255) + rerank_model_id: Optional[UUID] = Field(None, description="Rerank model UUID (optional)") + rerank_model_name: Optional[str] = Field(None, max_length=255) + + storage_type: str = Field(..., min_length=1, max_length=50) + + chunker_strategy: str = Field(default="RecursiveChunker", min_length=1, max_length=100) + reflexion_enabled: bool = Field(default=False) + reflexion_iteration_period: int = Field(default=3, ge=1, le=100) + reflexion_range: Literal["retrieval", "all"] = Field(default="retrieval") + reflexion_baseline: Literal["time", "fact", "time_and_fact"] = Field(default="time") + + llm_params: Dict[str, Any] = Field(default_factory=dict) + embedding_params: Dict[str, Any] = Field(default_factory=dict) + config_version: str = Field(default="2.0", min_length=1, max_length=10) + + @field_validator("config_name", "workspace_name", "embedding_model_name", "llm_model_name") + @classmethod + def validate_non_empty_strings(cls, v): + if not v or not v.strip(): + raise ValueError("Field cannot be empty or whitespace-only") + return v.strip() + + @field_validator("storage_type") + @classmethod + def validate_storage_type(cls, v): + valid_types = ["neo4j", "elasticsearch", "qdrant", "milvus", "chroma"] + if v.lower() not in valid_types: + raise ValueError(f"Storage type must be one of: {valid_types}") + return v.lower() + + @field_validator("llm_params", "embedding_params") + @classmethod + def validate_model_params(cls, v): + if not isinstance(v, dict): + raise ValueError("Model parameters must be a dictionary") + reserved_keys = ["model_id", "model_name", "api_key", "base_url"] + for key in v.keys(): + if key in reserved_keys: + raise ValueError(f"Model parameters cannot contain reserved parameter '{key}'") + return v + + model_config = ConfigDict(validate_assignment=True, extra="forbid") + + +class WorkspaceValidation(BaseModel): + """Pydantic model for validating workspace data from database.""" + + id: UUID = Field(..., description="Workspace UUID") + name: str = Field(..., min_length=1, max_length=255) + tenant_id: UUID = Field(..., description="Tenant UUID") + storage_type: Optional[str] = Field(None, max_length=50) + llm: Optional[str] = Field(None) + embedding: Optional[str] = Field(None) + rerank: Optional[str] = Field(None) + is_active: bool = Field(default=True) + + @field_validator("llm", "embedding", "rerank") + @classmethod + def validate_model_ids(cls, v): + if v is None or v == "": + return None + try: + UUID(v.strip()) + except ValueError: + raise ValueError("Model ID must be a valid UUID string") + return v.strip() + + @field_validator("is_active") + @classmethod + def validate_active_status(cls, v): + if not v: + raise ValueError("Workspace must be active for configuration loading") + return v + + model_config = ConfigDict(validate_assignment=True, extra="forbid") + + +class ModelValidation(BaseModel): + """Pydantic model for validating model configuration data.""" + + id: UUID = Field(..., description="Model UUID") + name: str = Field(..., min_length=1, max_length=255) + type: str = Field(..., description="Model type (llm, embedding, rerank)") + tenant_id: UUID = Field(..., description="Tenant UUID") + is_active: bool = Field(..., description="Whether model is active") + is_public: bool = Field(default=False) + + @field_validator("type") + @classmethod + def validate_type(cls, v): + valid_types = ["llm", "embedding", "rerank"] + if v.lower() not in valid_types: + raise ValueError(f"Model type must be one of: {valid_types}") + return v.lower() + + @field_validator("is_active") + @classmethod + def validate_active_status(cls, v): + if not v: + raise ValueError("Model must be active for configuration use") + return v + + model_config = ConfigDict(validate_assignment=True, extra="forbid") + + +# ==================== Validation Helper Functions ==================== + + +def validate_memory_config_data( + config_data: Dict[str, Any], config_id: Optional[int] = None +) -> MemoryConfigValidation: + """Validate memory configuration data using Pydantic model.""" + try: + return MemoryConfigValidation(**config_data) + except ValidationError as e: + error_messages = [] + for error in e.errors(): + field_path = " -> ".join(str(loc) for loc in error["loc"]) + error_messages.append(f"Field '{field_path}': {error['msg']}") + + detailed_message = "Configuration validation failed:\n" + "\n".join( + f" - {msg}" for msg in error_messages + ) + + first_error = e.errors()[0] if e.errors() else {} + first_field = " -> ".join(str(loc) for loc in first_error.get("loc", [])) + + raise InvalidConfigError( + detailed_message, + field_name=first_field or None, + invalid_value=first_error.get("input"), + config_id=config_id, + ) + + +def validate_workspace_data( + workspace_data: Dict[str, Any], config_id: Optional[int] = None +) -> WorkspaceValidation: + """Validate workspace data using Pydantic model.""" + try: + return WorkspaceValidation(**workspace_data) + except ValidationError as e: + error_messages = [] + for error in e.errors(): + field_path = " -> ".join(str(loc) for loc in error["loc"]) + error_messages.append(f"Field '{field_path}': {error['msg']}") + + detailed_message = "Workspace validation failed:\n" + "\n".join( + f" - {msg}" for msg in error_messages + ) + + first_error = e.errors()[0] if e.errors() else {} + first_field = " -> ".join(str(loc) for loc in first_error.get("loc", [])) + workspace_id = workspace_data.get("id") if isinstance(workspace_data, dict) else None + + raise InvalidConfigError( + detailed_message, + field_name=first_field or None, + invalid_value=first_error.get("input"), + config_id=config_id, + workspace_id=workspace_id, + ) + + +def validate_model_data( + model_data: Dict[str, Any], config_id: Optional[int] = None +) -> ModelValidation: + """Validate model data using Pydantic model.""" + try: + return ModelValidation(**model_data) + except ValidationError as e: + error_messages = [] + for error in e.errors(): + field_path = " -> ".join(str(loc) for loc in error["loc"]) + error_messages.append(f"Field '{field_path}': {error['msg']}") + + detailed_message = "Model validation failed:\n" + "\n".join( + f" - {msg}" for msg in error_messages + ) + + first_error = e.errors()[0] if e.errors() else {} + first_field = " -> ".join(str(loc) for loc in first_error.get("loc", [])) + + raise InvalidConfigError( + detailed_message, + field_name=first_field or None, + invalid_value=first_error.get("input"), + config_id=config_id, + ) + + +# ==================== Immutable Configuration Data Structure ==================== + + +@dataclass(frozen=True) +class MemoryConfig: + """Immutable memory configuration loaded from database.""" + + config_id: int + config_name: str + workspace_id: UUID + workspace_name: str + tenant_id: UUID + + embedding_model_id: UUID + embedding_model_name: str + llm_model_id: UUID + llm_model_name: str + + storage_type: str + + chunker_strategy: str + reflexion_enabled: bool + reflexion_iteration_period: int + reflexion_range: str + reflexion_baseline: str + + loaded_at: datetime + + rerank_model_id: Optional[UUID] = None + rerank_model_name: Optional[str] = None + + llm_params: Dict[str, Any] = field(default_factory=dict) + embedding_params: Dict[str, Any] = field(default_factory=dict) + config_version: str = "2.0" + + def __post_init__(self): + """Validate configuration after initialization.""" + if not self.config_name or not self.config_name.strip(): + raise InvalidConfigError("Configuration name cannot be empty") + + if not self.embedding_model_id: + raise InvalidConfigError("Embedding model ID is required") + + if not self.llm_model_id: + raise InvalidConfigError("LLM model ID is required") + + @classmethod + def from_validated_data( + cls, validated_config: MemoryConfigValidation, loaded_at: datetime + ) -> "MemoryConfig": + """Create MemoryConfig from validated Pydantic data.""" + return cls( + config_id=validated_config.config_id, + config_name=validated_config.config_name, + workspace_id=validated_config.workspace_id, + workspace_name=validated_config.workspace_name, + tenant_id=validated_config.tenant_id, + embedding_model_id=validated_config.embedding_model_id, + embedding_model_name=validated_config.embedding_model_name, + storage_type=validated_config.storage_type, + chunker_strategy=validated_config.chunker_strategy, + reflexion_enabled=validated_config.reflexion_enabled, + reflexion_iteration_period=validated_config.reflexion_iteration_period, + reflexion_range=validated_config.reflexion_range, + reflexion_baseline=validated_config.reflexion_baseline, + loaded_at=loaded_at, + llm_model_id=validated_config.llm_model_id, + llm_model_name=validated_config.llm_model_name, + rerank_model_id=validated_config.rerank_model_id, + rerank_model_name=validated_config.rerank_model_name, + llm_params=validated_config.llm_params, + embedding_params=validated_config.embedding_params, + config_version=validated_config.config_version, + ) + + def get_model_summary(self) -> Dict[str, Optional[str]]: + """Get a summary of configured models.""" + return { + "llm": self.llm_model_name, + "embedding": self.embedding_model_name, + "rerank": self.rerank_model_name, + } + + def is_model_configured(self, model_type: str) -> bool: + """Check if a specific model type is configured.""" + if model_type == "llm": + return True + elif model_type == "embedding": + return True + elif model_type == "rerank": + return self.rerank_model_id is not None + else: + raise ValueError(f"Unknown model type: {model_type}") diff --git a/api/app/services/memory_agent_service.py b/api/app/services/memory_agent_service.py index 83c73f53..27fdfa48 100644 --- a/api/app/services/memory_agent_service.py +++ b/api/app/services/memory_agent_service.py @@ -4,45 +4,44 @@ Memory Agent Service Handles business logic for memory agent operations including read/write services, health checks, and message type classification. """ +import json import os import re import time -import json import uuid from threading import Lock -from typing import Dict, List, Optional, Any, AsyncGenerator -from app.services.memory_konwledges_server import write_rag +from typing import Any, AsyncGenerator, Dict, List, Optional + import redis -from langchain_mcp_adapters.client import MultiServerMCPClient -from langchain_mcp_adapters.tools import load_mcp_tools -from sqlalchemy.orm import Session -from sqlalchemy import func -from pydantic import BaseModel, Field - from app.core.config import settings -from app.core.logging_config import get_logger +from app.core.logging_config import get_config_logger, get_logger from app.core.memory.agent.langgraph_graph.read_graph import make_read_graph from app.core.memory.agent.langgraph_graph.write_graph import make_write_graph from app.core.memory.agent.logger_file.log_streamer import LogStreamer -from app.core.memory.agent.utils.llm_tools import PROJECT_ROOT_ from app.core.memory.agent.utils.mcp_tools import get_mcp_server_config from app.core.memory.agent.utils.type_classifier import status_typle -from app.db import get_db -from app.repositories.neo4j.neo4j_connector import Neo4jConnector from app.core.memory.analytics.hot_memory_tags import get_hot_memory_tags from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.schemas.memory_storage_schema import ApiResponse, ok, fail +from app.db import get_db from app.models.knowledge_model import Knowledge, KnowledgeType -from app.repositories.data_config_repository import DataConfigRepository -from app.core.memory.agent.logger_file.log_streamer import LogStreamer -from app.services.memory_konwledges_server import memory_konwledges_up, SimpleUser, find_document_id_by_kb_and_filename -from app.core.memory.utils.config.definitions import reload_configuration_from_database -from app.schemas.file_schema import CustomTextFileCreate +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.schemas.memory_config_schema import ConfigurationError, MemoryConfig +from app.services.memory_config_service import MemoryConfigService +from app.services.memory_konwledges_server import ( + write_rag, +) +from langchain_mcp_adapters.client import MultiServerMCPClient +from langchain_mcp_adapters.tools import load_mcp_tools +from pydantic import BaseModel, Field +from sqlalchemy import func +from sqlalchemy.orm import Session + try: from app.core.memory.utils.log.audit_logger import audit_logger except ImportError: audit_logger = None logger = get_logger(__name__) +config_logger = get_config_logger() # Initialize Neo4j connector for analytics functions _neo4j_connector = Neo4jConnector() @@ -56,6 +55,27 @@ class MemoryAgentService: self.user_locks: Dict[str, Lock] = {} self.locks_lock = Lock() + def load_memory_config(self, config_id: int) -> MemoryConfig: + """ + Load memory configuration from database by config_id. + + This method delegates to the centralized MemoryConfigService to avoid + code duplication with other services. + + Args: + config_id: Configuration ID from database + + Returns: + MemoryConfig: Immutable configuration object + + Raises: + ConfigurationError: If validation fails + """ + return MemoryConfigService.load_memory_config( + config_id=config_id, + service_name="MemoryAgentService" + ) + def writer_messages_deal(self,messages,start_time,group_id,config_id,message): messages = str(messages).replace("'", '"').replace('\\n', '').replace('\n', '').replace('\\', '') countext = re.findall(r'"status": "(.*?)",', messages)[0] @@ -277,19 +297,20 @@ class MemoryAgentService: import time start_time = time.time() - # 如果 config_id 为 None，使用默认值 "17" - config_loaded = reload_configuration_from_database(config_id) - if not config_loaded: - error_msg = f"Failed to load configuration for config_id: {config_id}" + # Load configuration from database only + try: + memory_config = self.load_memory_config(config_id) + logger.info(f"Configuration loaded successfully: {memory_config.config_name}") + except ConfigurationError as e: + error_msg = f"Failed to load configuration for config_id: {config_id}: {e}" logger.error(error_msg) - # 记录失败的操作 + # Log failed operation if audit_logger: duration = time.time() - start_time - audit_logger.log_operation( operation="WRITE", config_id=config_id, group_id=group_id, success=False, duration=duration, error=error_msg ) + audit_logger.log_operation(operation="WRITE", config_id=config_id, group_id=group_id, success=False, duration=duration, error=error_msg) raise ValueError(error_msg) - logger.info(f"Configuration loaded successfully for config_id: {config_id}") mcp_config = get_mcp_server_config() client = MultiServerMCPClient(mcp_config) @@ -300,20 +321,43 @@ class MemoryAgentService: async with client.session("data_flow") as session: logger.debug("Connected to MCP Server: data_flow") tools = await load_mcp_tools(session) + workflow_errors = [] # Track errors from workflow - # Pass config_id to the graph workflow - async with make_write_graph(group_id, tools, group_id, group_id, config_id=config_id) as graph: + # Pass memory_config to the graph workflow + async with make_write_graph(group_id, tools, group_id, group_id, memory_config=memory_config) as graph: logger.debug("Write graph created successfully") config = {"configurable": {"thread_id": group_id}} async for event in graph.astream( - {"messages": message, "config_id": config_id}, + {"messages": message, "memory_config": memory_config, "errors": []}, stream_mode="values", config=config ): messages = event.get('messages') - return self.writer_messages_deal(messages,start_time,group_id,config_id,message) + # Capture any errors from the state + if event.get('errors'): + workflow_errors.extend(event.get('errors', [])) + + # Check for workflow errors + if workflow_errors: + error_details = "; ".join([f"{e['tool']}: {e['error']}" for e in workflow_errors]) + logger.error(f"Write workflow failed with errors: {error_details}") + + if audit_logger: + duration = time.time() - start_time + audit_logger.log_operation( + operation="WRITE", + config_id=config_id, + group_id=group_id, + success=False, + duration=duration, + error=error_details + ) + + raise ValueError(f"Write workflow failed: {error_details}") + + return self.writer_messages_deal(messages, start_time, group_id, config_id, message) async def read_memory( self, @@ -365,15 +409,15 @@ class MemoryAgentService: group_lock = self.get_group_lock(group_id) with group_lock: - # Step 1: Load configuration from database - from app.core.memory.utils.config.definitions import reload_configuration_from_database - - config_loaded = reload_configuration_from_database(config_id) - if not config_loaded: - error_msg = f"Failed to load configuration for config_id: {config_id}" + # Step 1: Load configuration from database only + try: + memory_config = self.load_memory_config(config_id) + logger.info(f"Configuration loaded successfully: {memory_config.config_name}") + except ConfigurationError as e: + error_msg = f"Failed to load configuration for config_id: {config_id}: {e}" logger.error(error_msg) - # 记录失败的操作 + # Log failed operation if audit_logger: duration = time.time() - start_time audit_logger.log_operation( @@ -387,8 +431,6 @@ class MemoryAgentService: raise ValueError(error_msg) - logger.info(f"Configuration loaded successfully for config_id: {config_id}") - # Step 2: Prepare history history.append({"role": "user", "content": message}) logger.debug(f"Group ID:{group_id}, Message:{message}, History:{history}, Config ID:{config_id}") @@ -404,45 +446,52 @@ class MemoryAgentService: intermediate_outputs = [] seen_intermediates = set() # Track seen intermediate outputs to avoid duplicates - # Pass config_id to the graph workflow - async with make_read_graph(group_id, tools, search_switch, group_id, group_id, config_id=config_id,storage_type=storage_type,user_rag_memory_id=user_rag_memory_id) as graph: + # Pass memory_config to the graph workflow + async with make_read_graph(group_id, tools, search_switch, group_id, group_id, memory_config=memory_config, storage_type=storage_type, user_rag_memory_id=user_rag_memory_id) as graph: start = time.time() config = {"configurable": {"thread_id": group_id}} + workflow_errors = [] # Track errors from workflow async for event in graph.astream( - {"messages": history, "config_id": config_id}, + {"messages": history, "memory_config": memory_config, "errors": []}, stream_mode="values", config=config ): messages = event.get('messages') + # Capture any errors from the state + if event.get('errors'): + workflow_errors.extend(event.get('errors', [])) + for msg in messages: msg_content = msg.content + msg_role = msg.__class__.__name__.lower().replace("message", "") outputs.append({ - "role": msg.__class__.__name__.lower().replace("message", ""), + "role": msg_role, "content": msg_content }) # Extract intermediate outputs if hasattr(msg, 'content'): try: - # Debug: log message type and content preview - msg_type = msg.__class__.__name__ - content_preview = str(msg_content)[:200] if msg_content else "empty" - logger.debug(f"Processing message type={msg_type}, content preview={content_preview}") + # Handle MCP content format: [{'type': 'text', 'text': '...'}] + content_to_parse = msg_content + if isinstance(msg_content, list): + for block in msg_content: + if isinstance(block, dict) and block.get('type') == 'text': + content_to_parse = block.get('text', '') + break + else: + continue # No text block found # Try to parse content as JSON - if isinstance(msg_content, str): + if isinstance(content_to_parse, str): try: - parsed = json.loads(msg_content) + parsed = json.loads(content_to_parse) if isinstance(parsed, dict): - # Debug: log what keys are in parsed - logger.debug(f"Parsed dict keys: {list(parsed.keys())}") - # Check for single intermediate output if '_intermediate' in parsed: intermediate_data = parsed['_intermediate'] output_key = self._create_intermediate_key(intermediate_data) - logger.debug(f"Found _intermediate: {intermediate_data.get('type', 'unknown')}") if output_key not in seen_intermediates: seen_intermediates.add(output_key) @@ -450,34 +499,14 @@ class MemoryAgentService: # Check for multiple intermediate outputs (from Retrieve) if '_intermediates' in parsed: - logger.debug(f"Found _intermediates list with {len(parsed['_intermediates'])} items") for intermediate_data in parsed['_intermediates']: output_key = self._create_intermediate_key(intermediate_data) - logger.debug(f"Processing intermediate: {intermediate_data.get('type', 'unknown')}") if output_key not in seen_intermediates: seen_intermediates.add(output_key) intermediate_outputs.append(self._format_intermediate_output(intermediate_data)) except (json.JSONDecodeError, ValueError): pass - elif isinstance(msg_content, dict): - # Check for single intermediate output - if '_intermediate' in msg_content: - intermediate_data = msg_content['_intermediate'] - output_key = self._create_intermediate_key(intermediate_data) - - if output_key not in seen_intermediates: - seen_intermediates.add(output_key) - intermediate_outputs.append(self._format_intermediate_output(intermediate_data)) - - # Check for multiple intermediate outputs (from Retrieve) - if '_intermediates' in msg_content: - for intermediate_data in msg_content['_intermediates']: - output_key = self._create_intermediate_key(intermediate_data) - - if output_key not in seen_intermediates: - seen_intermediates.add(output_key) - intermediate_outputs.append(self._format_intermediate_output(intermediate_data)) except Exception as e: logger.debug(f"Failed to extract intermediate output: {e}") @@ -489,18 +518,57 @@ class MemoryAgentService: for messages in outputs: if messages['role'] == 'tool': message = messages['content'] + + # Handle MCP content format: [{'type': 'text', 'text': '...'}] + if isinstance(message, list): + # Extract text from MCP content blocks + for block in message: + if isinstance(block, dict) and block.get('type') == 'text': + message = block.get('text', '') + break + else: + continue # No text block found + try: - message = json.loads(message) if isinstance(message, str) else message - if isinstance(message, dict) and message.get('status') != '': - summary_result = message.get('summary_result') - if summary_result: - final_answer = summary_result + parsed = json.loads(message) if isinstance(message, str) else message + if isinstance(parsed, dict): + if parsed.get('status') == 'success': + summary_result = parsed.get('summary_result') + if summary_result: + final_answer = summary_result except (json.JSONDecodeError, ValueError): pass # 记录成功的操作 total_duration = time.time() - start_time - if audit_logger: + + # Check for workflow errors + if workflow_errors: + error_details = "; ".join([f"{e['tool']}: {e['error']}" for e in workflow_errors]) + logger.warning(f"Read workflow completed with errors: {error_details}") + + if audit_logger: + audit_logger.log_operation( + operation="READ", + config_id=config_id, + group_id=group_id, + success=False, + duration=total_duration, + error=error_details, + details={ + "search_switch": search_switch, + "history_length": len(history), + "intermediate_outputs_count": len(intermediate_outputs), + "has_answer": bool(final_answer), + "errors": workflow_errors + } + ) + + # Raise error if no answer was produced + if not final_answer: + raise ValueError(f"Read workflow failed: {error_details}") + + if audit_logger and not workflow_errors: audit_logger.log_operation( operation="READ", config_id=config_id, @@ -612,19 +680,25 @@ class MemoryAgentService: else: return output - async def classify_message_type(self, message: str) -> Dict: + async def classify_message_type(self, message: str, config_id: int, db: Session) -> Dict: """ Determine the type of user message (read or write) + Updated to eliminate global variables in favor of explicit parameters. Args: message: User message to classify + config_id: Configuration ID to load LLM model from database + db: Database session Returns: Type classification result """ logger.info("Classifying message type") - status = await status_typle(message) + # Load configuration to get LLM model ID + memory_config = self.load_memory_config(config_id) + + status = await status_typle(message, memory_config.llm_model_id) logger.debug(f"Message type: {status}") return status @@ -790,7 +864,8 @@ class MemoryAgentService: async def get_user_profile( self, end_user_id: Optional[str] = None, - current_user_id: Optional[str] = None + current_user_id: Optional[str] = None, + llm_id: Optional[str] = None ) -> Dict[str, Any]: """ 获取用户详情，包含： @@ -801,6 +876,7 @@ class MemoryAgentService: 参数： - end_user_id: 用户ID（可选） - current_user_id: 当前登录用户的ID（保留参数） + - llm_id: LLM模型ID（用于生成标签，可选，如果不提供则跳过标签生成） 返回格式： { @@ -862,15 +938,17 @@ class MemoryAgentService: await connector.close() - if not statements: + if not statements or not llm_id: result["tags"] = [] + if not llm_id and statements: + logger.warning("llm_id not provided, skipping tag generation") else: # 构建摘要文本 summary_text = f"用户语句样本：{' | '.join(statements[:20])}\n核心实体：{', '.join(entities)}" logger.debug(f"User data found: {len(statements)} statements, {len(entities)} entities") # 使用LLM提取标签 - llm_client = get_llm_client() + llm_client = get_llm_client(llm_id) # 定义标签提取的结构 class UserTags(BaseModel): diff --git a/api/app/services/memory_config_service.py b/api/app/services/memory_config_service.py new file mode 100644 index 00000000..d19eb02a --- /dev/null +++ b/api/app/services/memory_config_service.py @@ -0,0 +1,264 @@ +""" +Memory Configuration Service + +Centralized configuration loading and management for memory services. +This service eliminates code duplication between MemoryAgentService and MemoryStorageService. +Database session management is handled internally. +""" + +import time +from datetime import datetime + +from app.core.logging_config import get_config_logger, get_logger +from app.core.validators.memory_config_validators import ( + validate_and_resolve_model_id, + validate_embedding_model, + validate_model_exists_and_active, +) +from app.repositories.data_config_repository import DataConfigRepository +from app.schemas.memory_config_schema import ( + ConfigurationError, + InvalidConfigError, + MemoryConfig, + ModelInactiveError, + ModelNotFoundError, +) +from sqlalchemy.orm import Session + +logger = get_logger(__name__) +config_logger = get_config_logger() + + +def _validate_config_id(config_id): + """Validate configuration ID format.""" + if config_id is None: + raise InvalidConfigError( + "Configuration ID cannot be None", + field_name="config_id", + invalid_value=config_id, + ) + + if isinstance(config_id, int): + if config_id <= 0: + raise InvalidConfigError( + f"Configuration ID must be positive: {config_id}", + field_name="config_id", + invalid_value=config_id, + ) + return config_id + + if isinstance(config_id, str): + try: + parsed_id = int(config_id.strip()) + if parsed_id <= 0: + raise InvalidConfigError( + f"Configuration ID must be positive: {parsed_id}", + field_name="config_id", + invalid_value=config_id, + ) + return parsed_id + except ValueError as e: + raise InvalidConfigError( + f"Invalid configuration ID format: '{config_id}'", + field_name="config_id", + invalid_value=config_id, + ) + + raise InvalidConfigError( + f"Invalid type for configuration ID: expected int or str, got {type(config_id).__name__}", + field_name="config_id", + invalid_value=config_id, + ) + + +class MemoryConfigService: + """ + Centralized service for memory configuration loading and validation. + + This class provides a single implementation of configuration loading logic + that can be shared across multiple services, eliminating code duplication. + Database session management is handled internally. + """ + + @staticmethod + def load_memory_config( + config_id: int, + service_name: str = "MemoryConfigService", + ) -> MemoryConfig: + """ + Load memory configuration from database by config_id. + + This method manages its own database session internally. + + Args: + config_id: Configuration ID from database + service_name: Name of the calling service (for logging purposes) + + Returns: + MemoryConfig: Immutable configuration object + + Raises: + ConfigurationError: If validation fails + """ + from app.db import get_db + + db_gen = get_db() + db = next(db_gen) + + try: + return MemoryConfigService._load_memory_config_with_db( + config_id=config_id, + db=db, + service_name=service_name, + ) + finally: + db.close() + + @staticmethod + def _load_memory_config_with_db( + config_id: int, + db: Session, + service_name: str = "MemoryConfigService", + ) -> MemoryConfig: + """Internal method that loads memory configuration with an existing db session.""" + start_time = time.time() + + config_logger.info( + "Starting memory configuration loading", + extra={ + "operation": "load_memory_config", + "service": service_name, + "config_id": config_id, + }, + ) + + logger.info(f"Loading memory configuration from database: config_id={config_id}") + + try: + validated_config_id = _validate_config_id(config_id) + + result = DataConfigRepository.get_config_with_workspace(db, validated_config_id) + if not result: + elapsed_ms = (time.time() - start_time) * 1000 + config_logger.error( + "Configuration not found in database", + extra={ + "operation": "load_memory_config", + "config_id": validated_config_id, + "load_result": "not_found", + "elapsed_ms": elapsed_ms, + "service": service_name, + }, + ) + raise ConfigurationError( + f"Configuration {validated_config_id} not found in database" + ) + + memory_config, workspace = result + + # Validate embedding model + embedding_uuid = validate_embedding_model( + validated_config_id, + memory_config.embedding_id, + db, + workspace.tenant_id, + workspace.id, + ) + + # Resolve LLM model + llm_uuid, llm_name = validate_and_resolve_model_id( + memory_config.llm_id, + "llm", + db, + workspace.tenant_id, + required=True, + config_id=validated_config_id, + workspace_id=workspace.id, + ) + + # Resolve optional rerank model + rerank_uuid = None + rerank_name = None + if memory_config.rerank_id: + rerank_uuid, rerank_name = validate_and_resolve_model_id( + memory_config.rerank_id, + "rerank", + db, + workspace.tenant_id, + required=False, + config_id=validated_config_id, + workspace_id=workspace.id, + ) + + # Get embedding model name + embedding_name, _ = validate_model_exists_and_active( + embedding_uuid, + "embedding", + db, + workspace.tenant_id, + config_id=validated_config_id, + workspace_id=workspace.id, + ) + + # Create immutable MemoryConfig object + config = MemoryConfig( + config_id=memory_config.config_id, + config_name=memory_config.config_name, + workspace_id=workspace.id, + workspace_name=workspace.name, + tenant_id=workspace.tenant_id, + llm_model_id=llm_uuid, + llm_model_name=llm_name, + embedding_model_id=embedding_uuid, + embedding_model_name=embedding_name, + rerank_model_id=rerank_uuid, + rerank_model_name=rerank_name, + storage_type=workspace.storage_type or "neo4j", + chunker_strategy=memory_config.chunker_strategy or "RecursiveChunker", + reflexion_enabled=memory_config.enable_self_reflexion or False, + reflexion_iteration_period=int(memory_config.iteration_period or "3"), + reflexion_range=memory_config.reflexion_range or "retrieval", + reflexion_baseline=memory_config.baseline or "time", + loaded_at=datetime.now(), + ) + + elapsed_ms = (time.time() - start_time) * 1000 + + config_logger.info( + "Memory configuration loaded successfully", + extra={ + "operation": "load_memory_config", + "service": service_name, + "config_id": validated_config_id, + "config_name": config.config_name, + "workspace_id": str(config.workspace_id), + "load_result": "success", + "elapsed_ms": elapsed_ms, + }, + ) + + logger.info(f"Memory configuration loaded successfully: {config.config_name}") + return config + + except Exception as e: + elapsed_ms = (time.time() - start_time) * 1000 + + config_logger.error( + "Failed to load memory configuration", + extra={ + "operation": "load_memory_config", + "service": service_name, + "config_id": config_id, + "load_result": "error", + "error_type": type(e).__name__, + "error_message": str(e), + "elapsed_ms": elapsed_ms, + }, + exc_info=True, + ) + + logger.error(f"Failed to load memory configuration {config_id}: {e}") + if isinstance(e, (ConfigurationError, ValueError)): + raise + else: + raise ConfigurationError(f"Failed to load configuration {config_id}: {e}") diff --git a/api/app/services/memory_storage_service.py b/api/app/services/memory_storage_service.py index 0548b704..bee3d22a 100644 --- a/api/app/services/memory_storage_service.py +++ b/api/app/services/memory_storage_service.py @@ -4,39 +4,40 @@ Memory Storage Service Handles business logic for memory storage operations. """ -from typing import Dict, List, Optional, Any, AsyncGenerator -import os -import json import asyncio +import json +import os import time from datetime import datetime -from sqlalchemy.orm import Session +from typing import Any, AsyncGenerator, Dict, List, Optional -from dotenv import load_dotenv - -from app.models.user_model import User -from app.models.end_user_model import EndUser -from app.core.logging_config import get_logger -from app.utils.sse_utils import format_sse_message -from app.schemas.memory_storage_schema import ( - ConfigFilter, - ConfigPilotRun, - ConfigParamsCreate, - ConfigParamsDelete, - ConfigUpdate, - ConfigUpdateExtracted, - ConfigUpdateForget, - ConfigKey, -) -from app.repositories.data_config_repository import DataConfigRepository -from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.core.logging_config import get_config_logger, get_logger from app.core.memory.analytics.hot_memory_tags import get_hot_memory_tags from app.core.memory.analytics.memory_insight import MemoryInsight from app.core.memory.analytics.recent_activity_stats import get_recent_activity_stats from app.core.memory.analytics.user_summary import generate_user_summary +from app.models.end_user_model import EndUser +from app.models.user_model import User from app.repositories.data_config_repository import DataConfigRepository +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.schemas.memory_config_schema import ConfigurationError, MemoryConfig +from app.schemas.memory_storage_schema import ( + ConfigFilter, + ConfigKey, + ConfigParamsCreate, + ConfigParamsDelete, + ConfigPilotRun, + ConfigUpdate, + ConfigUpdateExtracted, + ConfigUpdateForget, +) +from app.services.memory_config_service import MemoryConfigService +from app.utils.sse_utils import format_sse_message +from dotenv import load_dotenv +from sqlalchemy.orm import Session logger = get_logger(__name__) +config_logger = get_config_logger() # Load environment variables for Neo4j connector load_dotenv() @@ -48,6 +49,27 @@ class MemoryStorageService: def __init__(self): logger.info("MemoryStorageService initialized") + + def load_memory_config(self, config_id: int, db: Session) -> MemoryConfig: + """ + Load memory configuration from database by config_id. + + This method delegates to the centralized MemoryConfigService to avoid + code duplication with other services. + + Args: + config_id: Configuration ID from database + + Returns: + MemoryConfig: Immutable configuration object + + Raises: + ConfigurationError: If validation fails + """ + return MemoryConfigService.load_memory_config( + config_id=config_id, + service_name="MemoryStorageService" + ) async def get_storage_info(self) -> dict: """ @@ -248,7 +270,6 @@ class DataConfigService: # 数据配置服务类（PostgreSQL） RuntimeError: 当管线执行失败时 """ project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - dbrun_path = os.path.join(project_root, "app", "core", "memory", "dbrun.json") try: # 发出初始进度事件 @@ -257,24 +278,12 @@ class DataConfigService: # 数据配置服务类（PostgreSQL） "time": int(time.time() * 1000) }) - # 步骤 1: 配置加载和验证（复用现有逻辑） + # 步骤 1: 配置加载和验证（数据库优先） payload_cid = str(getattr(payload, "config_id", "") or "").strip() cid: Optional[str] = payload_cid if payload_cid else None - if not cid and os.path.isfile(dbrun_path): - try: - with open(dbrun_path, "r", encoding="utf-8") as f: - dbrun = json.load(f) - if isinstance(dbrun, dict): - sel = dbrun.get("selections", {}) - if isinstance(sel, dict): - fallback_cid = str(sel.get("config_id") or "").strip() - cid = fallback_cid or None - except Exception: - cid = None - if not cid: - raise ValueError("未提供 payload.config_id，且 dbrun.json 未设置 selections.config_id，禁止启动试运行") + raise ValueError("未提供 payload.config_id，禁止启动试运行") # 验证 dialogue_text 必须提供 dialogue_text = payload.dialogue_text.strip() if payload.dialogue_text else "" @@ -282,12 +291,15 @@ class DataConfigService: # 数据配置服务类（PostgreSQL） if not dialogue_text: raise ValueError("试运行模式必须提供 dialogue_text 参数") - # 应用内存覆写并刷新常量 - from app.core.memory.utils.config.definitions import reload_configuration_from_database - - ok_override = reload_configuration_from_database(cid) - if not ok_override: - raise RuntimeError("运行时覆写失败，config_id 无效或刷新常量失败") + # Load configuration from database only using centralized manager + try: + memory_config = MemoryConfigService.load_memory_config( + config_id=int(cid), + service_name="MemoryStorageService.pilot_run_stream" + ) + logger.info(f"Configuration loaded successfully: {memory_config.config_name}") + except ConfigurationError as e: + raise RuntimeError(f"Configuration loading failed: {e}") # 步骤 2: 创建进度回调函数捕获管线进度 # 使用队列在回调和生成器之间传递进度事件 diff --git a/api/app/tasks.py b/api/app/tasks.py index 2d461cd3..1f07f808 100644 --- a/api/app/tasks.py +++ b/api/app/tasks.py @@ -1,28 +1,30 @@ -import os import asyncio -from typing import Any, Dict, List, Optional -import requests -from datetime import datetime, timezone +import json +import os import time import uuid +from datetime import datetime, timezone from math import ceil -import redis -import json +from typing import Any, Dict, List, Optional -from app.db import get_db -from app.models.document_model import Document -from app.models.knowledge_model import Knowledge -from app.core.rag.llm.cv_model import QWenCV -from app.core.rag.llm.chat_model import Base -from app.core.rag.vdb.elasticsearch.elasticsearch_vector import ElasticSearchVectorFactory -from app.core.rag.models.chunk import DocumentChunk -from app.services.memory_agent_service import MemoryAgentService -from app.core.config import settings -from app.core.rag.graphrag.utils import get_llm_cache, set_llm_cache -from app.core.rag.prompts.generator import question_proposal +import redis +import requests # Import a unified Celery instance from app.celery_app import celery_app +from app.core.config import settings +from app.core.rag.graphrag.utils import get_llm_cache, set_llm_cache +from app.core.rag.llm.chat_model import Base +from app.core.rag.llm.cv_model import QWenCV +from app.core.rag.models.chunk import DocumentChunk +from app.core.rag.prompts.generator import question_proposal +from app.core.rag.vdb.elasticsearch.elasticsearch_vector import ( + ElasticSearchVectorFactory, +) +from app.db import get_db +from app.models.document_model import Document +from app.models.knowledge_model import Knowledge +from app.services.memory_agent_service import MemoryAgentService @celery_app.task(name="tasks.process_item") @@ -221,11 +223,17 @@ def read_message_task(self, group_id: str, message: str, history: List[Dict[str, "elapsed_time": elapsed_time, "task_id": self.request.id } - except Exception as e: + except BaseException as e: elapsed_time = time.time() - start_time + # Handle ExceptionGroup from TaskGroup + if hasattr(e, 'exceptions'): + error_messages = [f"{type(sub_e).__name__}: {str(sub_e)}" for sub_e in e.exceptions] + detailed_error = "; ".join(error_messages) + else: + detailed_error = str(e) return { "status": "FAILURE", - "error": str(e), + "error": detailed_error, "group_id": group_id, "config_id": config_id, "elapsed_time": elapsed_time, @@ -283,11 +291,17 @@ def write_message_task(self, group_id: str, message: str, config_id: str,storage "elapsed_time": elapsed_time, "task_id": self.request.id } - except Exception as e: + except BaseException as e: elapsed_time = time.time() - start_time + # Handle ExceptionGroup from TaskGroup + if hasattr(e, 'exceptions'): + error_messages = [f"{type(sub_e).__name__}: {str(sub_e)}" for sub_e in e.exceptions] + detailed_error = "; ".join(error_messages) + else: + detailed_error = str(e) return { "status": "FAILURE", - "error": str(e), + "error": detailed_error, "group_id": group_id, "config_id": config_id, "elapsed_time": elapsed_time, @@ -300,9 +314,10 @@ def reflection_engine() -> None: Intentionally left blank; replace with real reflection logic later. """ - from app.core.memory.utils.self_reflexion_utils.self_reflexion import self_reflexion import asyncio + from app.core.memory.utils.self_reflexion_utils.self_reflexion import self_reflexion + host_id = uuid.UUID("2f6ff1eb-50c7-4765-8e89-e4566be19122") asyncio.run(self_reflexion(host_id)) @@ -377,10 +392,10 @@ def write_total_memory_task(workspace_id: str) -> Dict[str, Any]: start_time = time.time() async def _run() -> Dict[str, Any]: - from app.services.memory_storage_service import search_all - from app.repositories.memory_increment_repository import write_memory_increment - from app.models.end_user_model import EndUser from app.models.app_model import App + from app.models.end_user_model import EndUser + from app.repositories.memory_increment_repository import write_memory_increment + from app.services.memory_storage_service import search_all db = next(get_db()) try: From 283c64a3581fd650701500d00c1770360e2b8282 Mon Sep 17 00:00:00 2001 From: Ke Sun Date: Tue, 23 Dec 2025 17:17:04 +0800 Subject: [PATCH 02/12] refactor(memory): restructure memory system and improve configuration management - Remove deprecated main.py entry point from memory module - Reorganize imports across controllers and services for consistency - Update emotion controller to pass db session instead of config_id to services - Enhance memory agent controller with db session parameter for status_type and user_profile endpoints - Refactor memory agent service to accept db parameter in classify_message_type method - Improve configuration handling in celery_app by removing automatic database reload - Update all memory-related services to use centralized config management - Standardize import ordering and remove unused imports across 50+ files - Add pilot_run_service for new pilot execution workflow - Refactor extraction engine, reflection engine, and search services for better modularity - Update LLM utilities and embedder configuration for improved flexibility - Enhance type classifier and verification tools with better error handling - Improve memory evaluation modules (LOCOMO, LongMemEval, MemSciQA) with consistent patterns --- api/app/celery_app.py | 6 +- api/app/controllers/emotion_controller.py | 17 +- .../controllers/memory_agent_controller.py | 54 +- .../memory_reflection_controller.py | 21 +- api/app/core/agent/langchain_agent.py | 55 +- api/app/core/config.py | 68 +- .../mcp_server/services/search_service.py | 4 +- .../agent/mcp_server/tools/data_tools.py | 9 +- .../agent/mcp_server/tools/problem_tools.py | 11 +- .../agent/mcp_server/tools/summary_tools.py | 15 +- .../memory/agent/utils/type_classifier.py | 15 +- .../core/memory/agent/utils/verify_tool.py | 25 +- .../core/memory/agent/utils/write_tools.py | 22 +- .../core/memory/analytics/hot_memory_tags.py | 35 +- .../core/memory/analytics/memory_insight.py | 32 +- api/app/core/memory/analytics/user_summary.py | 32 +- .../memory/evaluation/extraction_utils.py | 49 +- .../evaluation/locomo/locomo_benchmark.py | 47 +- .../memory/evaluation/locomo/locomo_test.py | 42 +- .../evaluation/locomo/qwen_search_eval.py | 44 +- .../longmemeval/qwen_search_eval.py | 51 +- .../evaluation/longmemeval/test_eval.py | 39 +- .../memory/evaluation/memsciqa/evaluate_qa.py | 41 +- .../evaluation/memsciqa/memsciqa-test.py | 37 +- .../core/memory/llm_tools/openai_client.py | 15 +- api/app/core/memory/main.py | 430 ------------- api/app/core/memory/models/graph_models.py | 10 +- api/app/core/memory/models/triplet_models.py | 3 +- api/app/core/memory/src/search.py | 586 +++++++++--------- .../deduplication/deduped_and_disamb.py | 88 ++- .../deduplication/second_layer_dedup.py | 28 +- .../deduplication/two_stage_dedup.py | 36 +- .../extraction_orchestrator.py | 2 + .../embedding_generation.py | 12 +- .../knowledge_extraction/memory_summary.py | 27 +- .../statement_extraction.py | 16 +- .../reflection_engine/self_reflexion.py | 44 +- .../storage_services/search/__init__.py | 34 +- .../search/semantic_search.py | 19 +- api/app/core/memory/utils/config/__init__.py | 2 - .../core/memory/utils/config/config_utils.py | 267 ++------ .../core/memory/utils/config/definitions.py | 413 ++++++------ .../memory/utils/embedder/embedder_utils.py | 6 +- api/app/core/memory/utils/llm/llm_utils.py | 276 ++++++--- .../utils/self_reflexion_utils/evaluate.py | 9 +- .../utils/self_reflexion_utils/reflexion.py | 9 +- api/app/core/rag_utils/chunk_insight.py | 20 +- api/app/core/rag_utils/chunk_summary.py | 18 +- api/app/core/rag_utils/chunk_tags.py | 18 +- api/app/schemas/memory_config_schema.py | 23 + api/app/services/draft_run_service.py | 56 +- api/app/services/emotion_analytics_service.py | 54 +- .../services/emotion_extraction_service.py | 16 +- api/app/services/memory_agent_service.py | 155 +++-- api/app/services/memory_config_service.py | 199 +++++- api/app/services/memory_storage_service.py | 37 +- api/app/services/pilot_run_service.py | 219 +++++++ api/app/tasks.py | 50 +- 58 files changed, 2171 insertions(+), 1797 deletions(-) delete mode 100644 api/app/core/memory/main.py create mode 100644 api/app/services/pilot_run_service.py diff --git a/api/app/celery_app.py b/api/app/celery_app.py index ce7e9300..e431b210 100644 --- a/api/app/celery_app.py +++ b/api/app/celery_app.py @@ -1,9 +1,9 @@ import os from datetime import timedelta from urllib.parse import quote -from celery import Celery + from app.core.config import settings -from app.core.memory.utils.config.definitions import reload_configuration_from_database +from celery import Celery # 创建 Celery 应用实例 # broker: 任务队列（使用 Redis DB 0） @@ -13,7 +13,6 @@ celery_app = Celery( broker=f"redis://:{quote(settings.REDIS_PASSWORD)}@{settings.REDIS_HOST}:{settings.REDIS_PORT}/{settings.CELERY_BROKER}", backend=f"redis://:{quote(settings.REDIS_PASSWORD)}@{settings.REDIS_HOST}:{settings.REDIS_PORT}/{settings.CELERY_BACKEND}", ) -reload_configuration_from_database(config_id=os.getenv("config_id"), force_reload=True) # 配置使用本地队列，避免与远程 worker 冲突 celery_app.conf.task_default_queue = 'localhost_test_wyl' @@ -22,6 +21,7 @@ celery_app.conf.task_default_routing_key = 'localhost_test_wyl' # macOS 兼容性配置 import platform + if platform.system() == 'Darwin': # macOS # 设置环境变量解决 fork 问题 os.environ.setdefault('OBJC_DISABLE_INITIALIZE_FORK_SAFETY', 'YES') diff --git a/api/app/controllers/emotion_controller.py b/api/app/controllers/emotion_controller.py index 2ed00c43..144aa281 100644 --- a/api/app/controllers/emotion_controller.py +++ b/api/app/controllers/emotion_controller.py @@ -10,22 +10,21 @@ Routes: POST /emotion/suggestions - 获取个性化情绪建议 """ -from fastapi import APIRouter, Depends, HTTPException, status -from sqlalchemy.orm import Session - -from app.core.response_utils import success, fail from app.core.error_codes import BizCode +from app.core.logging_config import get_api_logger +from app.core.response_utils import fail, success from app.dependencies import get_current_user, get_db from app.models.user_model import User -from app.schemas.response_schema import ApiResponse from app.schemas.emotion_schema import ( + EmotionHealthRequest, + EmotionSuggestionsRequest, EmotionTagsRequest, EmotionWordcloudRequest, - EmotionHealthRequest, - EmotionSuggestionsRequest ) +from app.schemas.response_schema import ApiResponse from app.services.emotion_analytics_service import EmotionAnalyticsService -from app.core.logging_config import get_api_logger +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy.orm import Session # 获取API专用日志器 api_logger = get_api_logger() @@ -230,7 +229,7 @@ async def get_emotion_suggestions( # 调用服务层 data = await emotion_service.generate_emotion_suggestions( end_user_id=request.group_id, - config_id=config_id + db=db ) api_logger.info( diff --git a/api/app/controllers/memory_agent_controller.py b/api/app/controllers/memory_agent_controller.py index 884ee889..b7da943c 100644 --- a/api/app/controllers/memory_agent_controller.py +++ b/api/app/controllers/memory_agent_controller.py @@ -163,7 +163,8 @@ async def write_server( result = await memory_agent_service.write_memory( user_input.group_id, user_input.message, - config_id, + config_id, + db, storage_type, user_rag_memory_id ) @@ -280,6 +281,7 @@ async def read_server( user_input.history, user_input.search_switch, config_id, + db, storage_type, user_rag_memory_id ) @@ -548,6 +550,7 @@ async def get_write_task_result( @router.post("/status_type", response_model=ApiResponse) async def status_type( user_input: Write_UserInput, + db: Session = Depends(get_db), current_user: User = Depends(get_current_user) ): """ @@ -561,7 +564,11 @@ async def status_type( """ api_logger.info(f"Status type check requested for group {user_input.group_id}") try: - result = await memory_agent_service.classify_message_type(user_input.message) + result = await memory_agent_service.classify_message_type( + user_input.message, + user_input.config_id, + db + ) return success(data=result) except Exception as e: api_logger.error(f"Message type classification failed: {str(e)}") @@ -636,6 +643,7 @@ async def get_hot_memory_tags_by_user_api( @router.get("/analytics/user_profile", response_model=ApiResponse) async def get_user_profile_api( end_user_id: Optional[str] = Query(None, description="用户ID（可选）"), + db: Session = Depends(get_db), current_user: User = Depends(get_current_user) ): """ @@ -659,7 +667,8 @@ async def get_user_profile_api( try: result = await memory_agent_service.get_user_profile( end_user_id=end_user_id, - current_user_id=str(current_user.id) + current_user_id=str(current_user.id), + db=db ) return success(data=result, msg="获取用户详情成功") except Exception as e: @@ -694,4 +703,41 @@ async def get_user_profile_api( # ) # except Exception as e: # api_logger.error(f"API docs retrieval failed: {str(e)}") -# return fail(BizCode.INTERNAL_ERROR, "API文档获取失败", str(e)) \ No newline at end of file +# return fail(BizCode.INTERNAL_ERROR, "API文档获取失败", str(e)) + + +@router.get("/end_user/{end_user_id}/connected_config", response_model=ApiResponse) +async def get_end_user_connected_config( + end_user_id: str, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user) +): + """ + 获取终端用户关联的记忆配置 + + 通过以下流程获取配置： + 1. 根据 end_user_id 获取用户的 app_id + 2. 获取该应用的最新发布版本 + 3. 从发布版本的 config 字段中提取 memory_config_id + + Args: + end_user_id: 终端用户ID + + Returns: + 包含 memory_config_id 和相关信息的响应 + """ + from app.services.memory_agent_service import ( + get_end_user_connected_config as get_config, + ) + + api_logger.info(f"Getting connected config for end_user: {end_user_id}") + + try: + result = get_config(end_user_id, db) + return success(data=result, msg="获取终端用户关联配置成功") + except ValueError as e: + api_logger.warning(f"End user config not found: {str(e)}") + return fail(BizCode.NOT_FOUND, str(e)) + except Exception as e: + api_logger.error(f"Failed to get end user connected config: {str(e)}", exc_info=True) + return fail(BizCode.INTERNAL_ERROR, "获取终端用户关联配置失败", str(e)) \ No newline at end of file diff --git a/api/app/controllers/memory_reflection_controller.py b/api/app/controllers/memory_reflection_controller.py index 8dfa6c50..71b5f6d0 100644 --- a/api/app/controllers/memory_reflection_controller.py +++ b/api/app/controllers/memory_reflection_controller.py @@ -1,22 +1,27 @@ import asyncio import time -from dotenv import load_dotenv -from fastapi import APIRouter, Depends, HTTPException, status -from sqlalchemy.orm import Session -from sqlalchemy import text - from app.core.logging_config import get_api_logger +from app.core.memory.storage_services.reflection_engine.self_reflexion import ( + ReflectionConfig, + ReflectionEngine, +) from app.core.response_utils import success -from app.core.memory.storage_services.reflection_engine.self_reflexion import ReflectionConfig, ReflectionEngine -from app.dependencies import get_current_user from app.db import get_db +from app.dependencies import get_current_user from app.models.user_model import User from app.repositories.data_config_repository import DataConfigRepository from app.repositories.neo4j.neo4j_connector import Neo4jConnector -from app.services.memory_reflection_service import WorkspaceAppService, MemoryReflectionService from app.schemas.memory_reflection_schemas import Memory_Reflection +from app.services.memory_reflection_service import ( + MemoryReflectionService, + WorkspaceAppService, +) from app.services.model_service import ModelConfigService +from dotenv import load_dotenv +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy import text +from sqlalchemy.orm import Session load_dotenv() api_logger = get_api_logger() diff --git a/api/app/core/agent/langchain_agent.py b/api/app/core/agent/langchain_agent.py index 3c33ad6e..380b660c 100644 --- a/api/app/core/agent/langchain_agent.py +++ b/api/app/core/agent/langchain_agent.py @@ -9,18 +9,19 @@ LangChain Agent 封装 """ import os import time -from typing import Dict, Any, List, Optional, AsyncGenerator, Sequence -from langchain_core.messages import SystemMessage, HumanMessage, AIMessage, BaseMessage -from langchain_core.tools import BaseTool -from langchain.agents import create_agent +from typing import Any, AsyncGenerator, Dict, List, Optional, Sequence +from app.core.logging_config import get_business_logger from app.core.memory.agent.utils.redis_tool import store from app.core.models import RedBearLLM, RedBearModelConfig from app.models.models_model import ModelType -from app.core.logging_config import get_business_logger from app.services.memory_konwledges_server import write_rag from app.services.task_service import get_task_memory_write_result from app.tasks import write_message_task +from langchain.agents import create_agent +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage +from langchain_core.tools import BaseTool + logger = get_business_logger() @@ -198,10 +199,24 @@ class LangChainAgent: """ message_chat= message start_time = time.time() - if config_id == None: - actual_config_id = os.getenv("config_id") - else: - actual_config_id = config_id + actual_config_id = config_id + # If config_id is None, try to get from end_user's connected config + if actual_config_id is None and end_user_id: + try: + from app.db import get_db + from app.services.memory_agent_service import ( + get_end_user_connected_config, + ) + db = next(get_db()) + try: + connected_config = get_end_user_connected_config(end_user_id, db) + actual_config_id = connected_config.get("memory_config_id") + except Exception as e: + logger.warning(f"Failed to get connected config for end_user {end_user_id}: {e}") + finally: + db.close() + except Exception as e: + logger.warning(f"Failed to get db session: {e}") actual_end_user_id = end_user_id if end_user_id is not None else "unknown" logger.info(f'写入类型{storage_type,str(end_user_id), message, str(user_rag_memory_id)}') print(f'写入类型{storage_type,str(end_user_id), message, str(user_rag_memory_id)}') @@ -295,10 +310,24 @@ class LangChainAgent: logger.info(f" Tool count: {len(self.tools) if self.tools else 0}") logger.info("=" * 80) message_chat = message - if config_id == None: - actual_config_id = os.getenv("config_id") - else: - actual_config_id = config_id + actual_config_id = config_id + # If config_id is None, try to get from end_user's connected config + if actual_config_id is None and end_user_id: + try: + from app.db import get_db + from app.services.memory_agent_service import ( + get_end_user_connected_config, + ) + db = next(get_db()) + try: + connected_config = get_end_user_connected_config(end_user_id, db) + actual_config_id = connected_config.get("memory_config_id") + except Exception as e: + logger.warning(f"Failed to get connected config for end_user {end_user_id}: {e}") + finally: + db.close() + except Exception as e: + logger.warning(f"Failed to get db session: {e}") history_term_memory = await self.term_memory_redis_read(end_user_id) if memory_flag: diff --git a/api/app/core/config.py b/api/app/core/config.py index bf5ff45a..da558ac9 100644 --- a/api/app/core/config.py +++ b/api/app/core/config.py @@ -1,7 +1,8 @@ -import os import json +import os from pathlib import Path -from typing import Dict, Any, Optional +from typing import Any, Dict, Optional + from dotenv import load_dotenv load_dotenv() @@ -81,6 +82,7 @@ class Settings: VOLC_QUERY_URL: str = os.getenv("VOLC_QUERY_URL", "https://openspeech.bytedance.com/api/v3/auc/bigmodel/query") # Langfuse configuration + LANGFUSE_ENABLED: bool = os.getenv("LANGFUSE_ENABLED", "false").lower() == "true" LANGFUSE_PUBLIC_KEY: str = os.getenv("LANGFUSE_PUBLIC_KEY", "") LANGFUSE_SECRET_KEY: str = os.getenv("LANGFUSE_SECRET_KEY", "") LANGFUSE_HOST: str = os.getenv("LANGFUSE_HOST", "") @@ -153,9 +155,6 @@ class Settings: # Memory Module Configuration (internal) MEMORY_OUTPUT_DIR: str = os.getenv("MEMORY_OUTPUT_DIR", "logs/memory-output") MEMORY_CONFIG_DIR: str = os.getenv("MEMORY_CONFIG_DIR", "app/core/memory") - MEMORY_CONFIG_FILE: str = os.getenv("MEMORY_CONFIG_FILE", "config.json") - MEMORY_RUNTIME_FILE: str = os.getenv("MEMORY_RUNTIME_FILE", "runtime.json") - MEMORY_DBRUN_FILE: str = os.getenv("MEMORY_DBRUN_FILE", "dbrun.json") # Tool Management Configuration TOOL_CONFIG_DIR: str = os.getenv("TOOL_CONFIG_DIR", "app/core/tools") @@ -178,65 +177,6 @@ class Settings: return str(base_path / filename) return str(base_path) - def get_memory_config_path(self, config_file: str = "") -> str: - """ - Get the full path for memory module configuration files. - - Args: - config_file: Optional config filename (defaults to MEMORY_CONFIG_FILE) - - Returns: - Full path to the config file - """ - if not config_file: - config_file = self.MEMORY_CONFIG_FILE - return str(Path(self.MEMORY_CONFIG_DIR) / config_file) - - def load_memory_config(self) -> Dict[str, Any]: - """ - Load memory module configuration from config.json. - - Returns: - Dictionary containing memory configuration - """ - config_path = self.get_memory_config_path(self.MEMORY_CONFIG_FILE) - try: - with open(config_path, "r", encoding="utf-8") as f: - return json.load(f) - except (FileNotFoundError, json.JSONDecodeError) as e: - print(f"Warning: Memory config file not found or malformed at {config_path}. Error: {e}") - return {} - - def load_memory_runtime_config(self) -> Dict[str, Any]: - """ - Load memory module runtime configuration from runtime.json. - - Returns: - Dictionary containing runtime configuration - """ - runtime_path = self.get_memory_config_path(self.MEMORY_RUNTIME_FILE) - try: - with open(runtime_path, "r", encoding="utf-8") as f: - return json.load(f) - except (FileNotFoundError, json.JSONDecodeError) as e: - print(f"Warning: Memory runtime config not found or malformed at {runtime_path}. Error: {e}") - return {"selections": {}} - - def load_memory_dbrun_config(self) -> Dict[str, Any]: - """ - Load memory module database run configuration from dbrun.json. - - Returns: - Dictionary containing dbrun configuration - """ - dbrun_path = self.get_memory_config_path(self.MEMORY_DBRUN_FILE) - try: - with open(dbrun_path, "r", encoding="utf-8") as f: - return json.load(f) - except (FileNotFoundError, json.JSONDecodeError) as e: - print(f"Warning: Memory dbrun config not found or malformed at {dbrun_path}. Error: {e}") - return {"selections": {}} - def ensure_memory_output_dir(self) -> None: """ Ensure the memory output directory exists. diff --git a/api/app/core/memory/agent/mcp_server/services/search_service.py b/api/app/core/memory/agent/mcp_server/services/search_service.py index b0a007cd..be96bb64 100644 --- a/api/app/core/memory/agent/mcp_server/services/search_service.py +++ b/api/app/core/memory/agent/mcp_server/services/search_service.py @@ -141,7 +141,7 @@ class SearchService: cleaned_query = self.clean_query(question) try: - # Execute search using embedding_model_id from memory_config + # Execute search using memory_config answer = await run_hybrid_search( query_text=cleaned_query, search_type=search_type, @@ -149,7 +149,7 @@ class SearchService: limit=limit, include=include, output_path=output_path, - embedding_id=str(config.embedding_model_id), + memory_config=config, rerank_alpha=rerank_alpha, ) diff --git a/api/app/core/memory/agent/mcp_server/tools/data_tools.py b/api/app/core/memory/agent/mcp_server/tools/data_tools.py index 22dadd7f..631f7fd7 100644 --- a/api/app/core/memory/agent/mcp_server/tools/data_tools.py +++ b/api/app/core/memory/agent/mcp_server/tools/data_tools.py @@ -13,7 +13,8 @@ from app.core.memory.agent.mcp_server.models.retrieval_models import ( ) from app.core.memory.agent.mcp_server.server import get_context_resource from app.core.memory.agent.utils.write_tools import write -from app.core.memory.utils.llm.llm_utils import get_llm_client_from_config +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.db import get_db_context from app.schemas.memory_config_schema import MemoryConfig from mcp.server.fastmcp import Context @@ -41,8 +42,10 @@ async def Data_type_differentiation( # Extract services from context template_service = get_context_resource(ctx, 'template_service') - # Get LLM client from memory_config - llm_client = get_llm_client_from_config(memory_config) + # Get LLM client from memory_config using factory pattern + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client_from_config(memory_config) # Render template try: diff --git a/api/app/core/memory/agent/mcp_server/tools/problem_tools.py b/api/app/core/memory/agent/mcp_server/tools/problem_tools.py index 892fdbdd..49812e38 100644 --- a/api/app/core/memory/agent/mcp_server/tools/problem_tools.py +++ b/api/app/core/memory/agent/mcp_server/tools/problem_tools.py @@ -16,7 +16,8 @@ from app.core.memory.agent.mcp_server.models.problem_models import ( ) from app.core.memory.agent.mcp_server.server import get_context_resource from app.core.memory.agent.utils.messages_tool import Problem_Extension_messages_deal -from app.core.memory.utils.llm.llm_utils import get_llm_client_from_config +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.db import get_db_context from app.schemas.memory_config_schema import MemoryConfig from mcp.server.fastmcp import Context @@ -56,7 +57,9 @@ async def Split_The_Problem( session_service = get_context_resource(ctx, "session_service") # Get LLM client from memory_config - llm_client = get_llm_client_from_config(memory_config) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client_from_config(memory_config) # Extract user ID from session user_id = session_service.resolve_user_id(sessionid) @@ -190,7 +193,9 @@ async def Problem_Extension( session_service = get_context_resource(ctx, "session_service") # Get LLM client from memory_config - llm_client = get_llm_client_from_config(memory_config) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client_from_config(memory_config) # Resolve session ID from usermessages from app.core.memory.agent.utils.messages_tool import Resolve_username diff --git a/api/app/core/memory/agent/mcp_server/tools/summary_tools.py b/api/app/core/memory/agent/mcp_server/tools/summary_tools.py index 8b6b7ae4..6d5012f1 100644 --- a/api/app/core/memory/agent/mcp_server/tools/summary_tools.py +++ b/api/app/core/memory/agent/mcp_server/tools/summary_tools.py @@ -21,8 +21,9 @@ from app.core.memory.agent.utils.messages_tool import ( Resolve_username, Summary_messages_deal, ) -from app.core.memory.utils.llm.llm_utils import get_llm_client_from_config +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory from app.core.rag.nlp.search import knowledge_retrieval +from app.db import get_db_context from app.schemas.memory_config_schema import MemoryConfig from dotenv import load_dotenv from mcp.server.fastmcp import Context @@ -66,7 +67,9 @@ async def Summary( session_service = get_context_resource(ctx, "session_service") # Get LLM client from memory_config - llm_client = get_llm_client_from_config(memory_config) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client_from_config(memory_config) # Resolve session ID sessionid = Resolve_username(usermessages) @@ -210,7 +213,9 @@ async def Retrieve_Summary( session_service = get_context_resource(ctx, "session_service") # Get LLM client from memory_config - llm_client = get_llm_client_from_config(memory_config) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client_from_config(memory_config) # Resolve session ID sessionid = Resolve_username(usermessages) @@ -425,7 +430,9 @@ async def Input_Summary( search_service = get_context_resource(ctx, "search_service") # Get LLM client from memory_config - llm_client = get_llm_client_from_config(memory_config) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client_from_config(memory_config) # Resolve session ID sessionid = Resolve_username(usermessages) or "" diff --git a/api/app/core/memory/agent/utils/type_classifier.py b/api/app/core/memory/agent/utils/type_classifier.py index d1b75d43..3e5358bd 100644 --- a/api/app/core/memory/agent/utils/type_classifier.py +++ b/api/app/core/memory/agent/utils/type_classifier.py @@ -1,15 +1,14 @@ """ Type classification utility for distinguishing read/write operations. """ -from jinja2 import Template -from pydantic import BaseModel - +from app.core.config import settings from app.core.logging_config import get_agent_logger, log_prompt_rendering from app.core.memory.agent.utils.llm_tools import PROJECT_ROOT_ from app.core.memory.agent.utils.messages_tool import read_template_file -from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.core.config import settings - +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.db import get_db_context +from jinja2 import Template +from pydantic import BaseModel logger = get_agent_logger(__name__) @@ -44,7 +43,9 @@ async def status_typle(messages: str, llm_model_id: str) -> dict: "message": f"Prompt rendering failed: {str(e)}" } - llm_client = get_llm_client(llm_model_id) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client(llm_model_id) try: structured = await llm_client.response_structured( diff --git a/api/app/core/memory/agent/utils/verify_tool.py b/api/app/core/memory/agent/utils/verify_tool.py index fe721770..3a74ee25 100644 --- a/api/app/core/memory/agent/utils/verify_tool.py +++ b/api/app/core/memory/agent/utils/verify_tool.py @@ -1,18 +1,19 @@ -from typing import TypedDict, Annotated, List, Any -from langchain_core.messages import AnyMessage -from langgraph.constants import START, END -from langgraph.graph import StateGraph, add_messages import asyncio import json -from dotenv import load_dotenv, find_dotenv import os -from app.core.memory.agent.utils.llm_tools import PROJECT_ROOT_ -from langchain_core.messages import HumanMessage -from jinja2 import Environment, FileSystemLoader -from app.core.memory.agent.utils.messages_tool import _to_openai_messages -from app.core.memory.utils.llm.llm_utils import get_llm_client +from typing import Annotated, Any, List, TypedDict + # Removed global variable imports - use dependency injection instead from app.core.logging_config import get_agent_logger +from app.core.memory.agent.utils.llm_tools import PROJECT_ROOT_ +from app.core.memory.agent.utils.messages_tool import _to_openai_messages +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.db import get_db_context +from dotenv import find_dotenv, load_dotenv +from jinja2 import Environment, FileSystemLoader +from langchain_core.messages import AnyMessage, HumanMessage +from langgraph.constants import END, START +from langgraph.graph import StateGraph, add_messages load_dotenv(find_dotenv()) @@ -53,7 +54,9 @@ class VerifyTool: async def model_1(self, state: State) -> State: if not self.llm_model_id: raise ValueError("llm_model_id is required but not provided") - llm_client = get_llm_client(self.llm_model_id) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client(self.llm_model_id) response_content = await llm_client.chat( messages=[{"role": "system", "content": self.system_prompt}, *_to_openai_messages(state["messages"])] ) diff --git a/api/app/core/memory/agent/utils/write_tools.py b/api/app/core/memory/agent/utils/write_tools.py index 60259873..f09b35e8 100644 --- a/api/app/core/memory/agent/utils/write_tools.py +++ b/api/app/core/memory/agent/utils/write_tools.py @@ -13,13 +13,11 @@ from app.core.memory.storage_services.extraction_engine.extraction_orchestrator ExtractionOrchestrator, ) from app.core.memory.storage_services.extraction_engine.knowledge_extraction.memory_summary import ( - Memory_summary_generation, + memory_summary_generation, ) -from app.core.memory.utils.embedder.embedder_utils import ( - get_embedder_client_from_config, -) -from app.core.memory.utils.llm.llm_utils import get_llm_client_from_config +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory from app.core.memory.utils.log.logging_utils import log_time +from app.db import get_db_context from app.repositories.neo4j.add_edges import add_memory_summary_statement_edges from app.repositories.neo4j.add_nodes import add_memory_summary_nodes from app.repositories.neo4j.graph_saver import save_dialog_and_statements_to_neo4j @@ -67,9 +65,11 @@ async def write( logger.info(f"Chunker strategy: {chunker_strategy}") logger.info(f"Group ID: {group_id}") - # Construct clients from memory_config - llm_client = get_llm_client_from_config(memory_config) - embedder_client = get_embedder_client_from_config(memory_config) + # Construct clients from memory_config using factory pattern with db session + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client_from_config(memory_config) + embedder_client = factory.get_embedder_client_from_config(memory_config) logger.info("LLM and embedding clients constructed") # Initialize timing log @@ -100,7 +100,7 @@ async def write( # Step 2: Initialize and run ExtractionOrchestrator step_start = time.time() from app.core.memory.utils.config.config_utils import get_pipeline_config - pipeline_config = get_pipeline_config() + pipeline_config = get_pipeline_config(memory_config) orchestrator = ExtractionOrchestrator( llm_client=llm_client, @@ -155,8 +155,8 @@ async def write( # Step 4: Generate Memory summaries and save to Neo4j step_start = time.time() try: - summaries = await Memory_summary_generation( - chunked_dialogs, llm_client=llm_client, embedding_id=embedding_model_id + summaries = await memory_summary_generation( + chunked_dialogs, llm_client=llm_client, embedder_client=embedder_client ) try: diff --git a/api/app/core/memory/analytics/hot_memory_tags.py b/api/app/core/memory/analytics/hot_memory_tags.py index cfcff994..2aa286ba 100644 --- a/api/app/core/memory/analytics/hot_memory_tags.py +++ b/api/app/core/memory/analytics/hot_memory_tags.py @@ -35,7 +35,9 @@ except NameError: import json from app.core.config import settings -from app.core.memory.utils.llm.llm_utils import get_llm_client +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.db import get_db_context +from app.services.memory_config_service import MemoryConfigService #TODO: Fix this # Default values (previously from definitions.py) @@ -47,11 +49,37 @@ class FilteredTags(BaseModel): """用于接收LLM筛选后的核心标签列表的模型。""" meaningful_tags: List[str] = Field(..., description="从原始列表中筛选出的具有核心代表意义的名词列表。") -async def filter_tags_with_llm(tags: List[str], llm_client) -> List[str]: +async def filter_tags_with_llm(tags: List[str], group_id: str) -> List[str]: """ 使用LLM筛选标签列表，仅保留具有代表性的核心名词。 """ try: + # Get config_id using get_end_user_connected_config + with get_db_context() as db: + try: + from app.services.memory_agent_service import ( + get_end_user_connected_config, + ) + connected_config = get_end_user_connected_config(group_id, db) + config_id = connected_config.get("memory_config_id") + + if config_id: + # Use the config_id to get the proper LLM client + config_service = MemoryConfigService(db) + memory_config = config_service.load_memory_config(config_id) + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client(memory_config.llm_model_id) + else: + # TODO: Remove DEFAULT_LLM_ID fallback once all users have proper config + # Fallback to default LLM if no config found + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client(DEFAULT_LLM_ID) + except Exception as e: + print(f"Failed to get user connected config, using default LLM: {e}") + # TODO: Remove DEFAULT_LLM_ID fallback once all users have proper config + # Fallback to default LLM + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client(DEFAULT_LLM_ID) # 3. 构建Prompt tag_list_str = ", ".join(tags) @@ -156,8 +184,7 @@ async def get_hot_memory_tags(group_id: str | None = None, limit: int = 40, by_u raw_tag_names = [tag for tag, freq in raw_tags_with_freq] # 2. 初始化LLM客户端并使用LLM筛选出有意义的标签 - llm_client = get_llm_client(DEFAULT_LLM_ID) - meaningful_tag_names = await filter_tags_with_llm(raw_tag_names, llm_client) + meaningful_tag_names = await filter_tags_with_llm(raw_tag_names, group_id) # 3. 根据LLM的筛选结果，构建最终的标签列表（保留原始频率和顺序） final_tags = [] diff --git a/api/app/core/memory/analytics/memory_insight.py b/api/app/core/memory/analytics/memory_insight.py index 35ed466f..06791702 100644 --- a/api/app/core/memory/analytics/memory_insight.py +++ b/api/app/core/memory/analytics/memory_insight.py @@ -18,8 +18,10 @@ if src_path not in sys.path: sys.path.insert(0, src_path) from app.core.memory.analytics.hot_memory_tags import get_hot_memory_tags -from app.core.memory.utils.llm.llm_utils import get_llm_client +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.db import get_db_context from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.services.memory_config_service import MemoryConfigService from pydantic import BaseModel, Field #TODO: Fix this @@ -59,7 +61,33 @@ class MemoryInsight: def __init__(self, user_id: str): self.user_id = user_id self.neo4j_connector = Neo4jConnector() - self.llm_client = get_llm_client(DEFAULT_LLM_ID) + + # Get config_id using get_end_user_connected_config + with get_db_context() as db: + try: + from app.services.memory_agent_service import ( + get_end_user_connected_config, + ) + connected_config = get_end_user_connected_config(user_id, db) + config_id = connected_config.get("memory_config_id") + + if config_id: + # Use the config_id to get the proper LLM client + config_service = MemoryConfigService(db) + memory_config = config_service.load_memory_config(config_id) + factory = MemoryClientFactory(db) + self.llm_client = factory.get_llm_client(memory_config.llm_model_id) + else: + # TODO: Remove DEFAULT_LLM_ID fallback once all users have proper config + # Fallback to default LLM if no config found + factory = MemoryClientFactory(db) + self.llm_client = factory.get_llm_client(DEFAULT_LLM_ID) + except Exception as e: + print(f"Failed to get user connected config, using default LLM: {e}") + # TODO: Remove DEFAULT_LLM_ID fallback once all users have proper config + # Fallback to default LLM + factory = MemoryClientFactory(db) + self.llm_client = factory.get_llm_client(DEFAULT_LLM_ID) async def close(self): """关闭数据库连接。""" diff --git a/api/app/core/memory/analytics/user_summary.py b/api/app/core/memory/analytics/user_summary.py index eb6bc83a..3f4f4a2d 100644 --- a/api/app/core/memory/analytics/user_summary.py +++ b/api/app/core/memory/analytics/user_summary.py @@ -25,8 +25,10 @@ except Exception: pass from app.core.memory.analytics.hot_memory_tags import get_hot_memory_tags -from app.core.memory.utils.llm.llm_utils import get_llm_client +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.db import get_db_context from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.services.memory_config_service import MemoryConfigService #TODO: Fix this @@ -47,7 +49,33 @@ class UserSummary: def __init__(self, user_id: str): self.user_id = user_id self.connector = Neo4jConnector() - self.llm = get_llm_client(DEFAULT_LLM_ID) + + # Get config_id using get_end_user_connected_config + with get_db_context() as db: + try: + from app.services.memory_agent_service import ( + get_end_user_connected_config, + ) + connected_config = get_end_user_connected_config(user_id, db) + config_id = connected_config.get("memory_config_id") + + if config_id: + # Use the config_id to get the proper LLM client + config_service = MemoryConfigService(db) + memory_config = config_service.load_memory_config(config_id) + factory = MemoryClientFactory(db) + self.llm = factory.get_llm_client(memory_config.llm_model_id) + else: + # TODO: Remove DEFAULT_LLM_ID fallback once all users have proper config + # Fallback to default LLM if no config found + factory = MemoryClientFactory(db) + self.llm = factory.get_llm_client(DEFAULT_LLM_ID) + except Exception as e: + print(f"Failed to get user connected config, using default LLM: {e}") + # TODO: Remove DEFAULT_LLM_ID fallback once all users have proper config + # Fallback to default LLM + factory = MemoryClientFactory(db) + self.llm = factory.get_llm_client(DEFAULT_LLM_ID) async def close(self): await self.connector.close() diff --git a/api/app/core/memory/evaluation/extraction_utils.py b/api/app/core/memory/evaluation/extraction_utils.py index b45ea7e4..9afa228c 100644 --- a/api/app/core/memory/evaluation/extraction_utils.py +++ b/api/app/core/memory/evaluation/extraction_utils.py @@ -1,22 +1,34 @@ -import os import asyncio import json -from typing import List, Dict, Any, Optional -from datetime import datetime +import os import re +from datetime import datetime +from typing import Any, Dict, List, Optional from app.core.memory.llm_tools.openai_client import LLMClient -from app.core.memory.storage_services.extraction_engine.knowledge_extraction.chunk_extraction import DialogueChunker -from app.core.memory.models.message_models import DialogData, ConversationContext, ConversationMessage -from app.repositories.neo4j.neo4j_connector import Neo4jConnector -from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.core.memory.utils.config.definitions import SELECTED_CHUNKER_STRATEGY, SELECTED_EMBEDDING_ID +from app.core.memory.models.message_models import ( + ConversationContext, + ConversationMessage, + DialogData, +) # 使用新的模块化架构 -from app.core.memory.storage_services.extraction_engine.extraction_orchestrator import ExtractionOrchestrator +from app.core.memory.storage_services.extraction_engine.extraction_orchestrator import ( + ExtractionOrchestrator, +) +from app.core.memory.storage_services.extraction_engine.knowledge_extraction.chunk_extraction import ( + DialogueChunker, +) +from app.core.memory.utils.config.definitions import ( + SELECTED_CHUNKER_STRATEGY, + SELECTED_EMBEDDING_ID, +) +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.db import get_db_context # Import from database module from app.repositories.neo4j.graph_saver import save_dialog_and_statements_to_neo4j +from app.repositories.neo4j.neo4j_connector import Neo4jConnector # Cypher queries for evaluation # Note: Entity, chunk, and dialogue search queries have been moved to evaluation/dialogue_queries.py @@ -52,7 +64,9 @@ async def ingest_contexts_via_full_pipeline( llm_available = True try: from app.core.memory.utils.config import definitions as config_defs - llm_client = get_llm_client(config_defs.SELECTED_LLM_ID) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client(config_defs.SELECTED_LLM_ID) except Exception as e: print(f"[Ingestion] LLM client unavailable, will skip LLM-dependent steps: {e}") llm_available = False @@ -133,12 +147,13 @@ async def ingest_contexts_via_full_pipeline( return False # 初始化 embedder 客户端 - from app.core.models.base import RedBearModelConfig - from app.core.memory.utils.config.config_utils import get_embedder_config from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient + from app.core.models.base import RedBearModelConfig + from app.services.memory_config_service import MemoryConfigService try: - embedder_config_dict = get_embedder_config(embedding_name or SELECTED_EMBEDDING_ID) + with get_db_context() as db: + embedder_config_dict = MemoryConfigService(db).get_embedder_config(embedding_name or SELECTED_EMBEDDING_ID) embedder_config = RedBearModelConfig(**embedder_config_dict) embedder_client = OpenAIEmbedderClient(embedder_config) except Exception as e: @@ -236,15 +251,15 @@ async def ingest_contexts_via_full_pipeline( print("[Ingestion] Generating memory summaries...") try: from app.core.memory.storage_services.extraction_engine.knowledge_extraction.memory_summary import ( - Memory_summary_generation, + memory_summary_generation, ) - from app.repositories.neo4j.add_nodes import add_memory_summary_nodes from app.repositories.neo4j.add_edges import add_memory_summary_statement_edges + from app.repositories.neo4j.add_nodes import add_memory_summary_nodes - summaries = await Memory_summary_generation( + summaries = await memory_summary_generation( chunked_dialogs=dialog_data_list, llm_client=llm_client, - embedding_id=embedding_name or SELECTED_EMBEDDING_ID + embedder_client=embedder_client ) print(f"[Ingestion] Generated {len(summaries)} memory summaries") except Exception as e: diff --git a/api/app/core/memory/evaluation/locomo/locomo_benchmark.py b/api/app/core/memory/evaluation/locomo/locomo_benchmark.py index 67f41771..4992aa29 100644 --- a/api/app/core/memory/evaluation/locomo/locomo_benchmark.py +++ b/api/app/core/memory/evaluation/locomo/locomo_benchmark.py @@ -15,7 +15,7 @@ import json import os import time from datetime import datetime -from typing import List, Dict, Any, Optional +from typing import Any, Dict, List, Optional try: from dotenv import load_dotenv @@ -23,37 +23,38 @@ except ImportError: def load_dotenv(): pass -from app.repositories.neo4j.neo4j_connector import Neo4jConnector -from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient -from app.core.models.base import RedBearModelConfig -from app.core.memory.utils.config_utils import get_embedder_config -from app.core.memory.utils.definitions import ( - PROJECT_ROOT, - SELECTED_GROUP_ID, - SELECTED_LLM_ID, - SELECTED_EMBEDDING_ID -) -from app.core.memory.utils.llm_utils import get_llm_client +from app.core.memory.client_factory import MemoryClientFactory from app.core.memory.evaluation.common.metrics import ( - f1_score, + avg_context_tokens, bleu1, + f1_score, jaccard, latency_stats, - avg_context_tokens ) from app.core.memory.evaluation.locomo.locomo_metrics import ( + get_category_name, locomo_f1_score, locomo_multi_f1, - get_category_name ) from app.core.memory.evaluation.locomo.locomo_utils import ( - load_locomo_data, extract_conversations, + ingest_conversations_if_needed, + load_locomo_data, resolve_temporal_references, - select_and_format_information, retrieve_relevant_information, - ingest_conversations_if_needed + select_and_format_information, ) +from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient +from app.core.memory.utils.definitions import ( + PROJECT_ROOT, + SELECTED_EMBEDDING_ID, + SELECTED_GROUP_ID, + SELECTED_LLM_ID, +) +from app.core.models.base import RedBearModelConfig +from app.db import get_db_context +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.services.memory_config_service import MemoryConfigService async def run_locomo_benchmark( @@ -160,10 +161,16 @@ async def run_locomo_benchmark( # Step 3: Initialize clients print("🔧 Initializing clients...") connector = Neo4jConnector() - llm_client = get_llm_client(SELECTED_LLM_ID) + + # Initialize LLM client with database context + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client(SELECTED_LLM_ID) # Initialize embedder - cfg_dict = get_embedder_config(SELECTED_EMBEDDING_ID) + with get_db_context() as db: + config_service = MemoryConfigService(db) + cfg_dict = config_service.get_embedder_config(SELECTED_EMBEDDING_ID) embedder = OpenAIEmbedderClient( model_config=RedBearModelConfig.model_validate(cfg_dict) ) diff --git a/api/app/core/memory/evaluation/locomo/locomo_test.py b/api/app/core/memory/evaluation/locomo/locomo_test.py index ad51931a..a8fa1820 100644 --- a/api/app/core/memory/evaluation/locomo/locomo_test.py +++ b/api/app/core/memory/evaluation/locomo/locomo_test.py @@ -1,14 +1,16 @@ # file name: check_neo4j_connection_fixed.py import asyncio -import os -import sys import json -import time import math +import os import re +import sys +import time from datetime import datetime, timedelta -from typing import List, Dict, Any +from typing import Any, Dict, List + from dotenv import load_dotenv + # 1 # 添加项目根目录到路径 current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -34,7 +36,7 @@ def _loc_normalize(text: str) -> str: # 尝试从 metrics.py 导入基础指标 try: - from common.metrics import f1_score, bleu1, jaccard + from common.metrics import bleu1, f1_score, jaccard print("✅ 从 metrics.py 导入基础指标成功") except ImportError as e: print(f"❌ 从 metrics.py 导入失败: {e}") @@ -111,10 +113,14 @@ try: # 尝试从不同位置导入 try: - from locomo.qwen_search_eval import loc_f1_score, loc_multi_f1, _resolve_relative_times + from locomo.qwen_search_eval import ( + _resolve_relative_times, + loc_f1_score, + loc_multi_f1, + ) print("✅ 从 locomo.qwen_search_eval 导入 LoCoMo 特定指标成功") except ImportError: - from qwen_search_eval import loc_f1_score, loc_multi_f1, _resolve_relative_times + from qwen_search_eval import _resolve_relative_times, loc_f1_score, loc_multi_f1 print("✅ 从 qwen_search_eval 导入 LoCoMo 特定指标成功") except ImportError as e: @@ -429,13 +435,17 @@ async def run_enhanced_evaluation(): return None # 修正导入路径：使用 app.core.memory.src 前缀 - from app.repositories.neo4j.neo4j_connector import Neo4jConnector - from app.repositories.neo4j.graph_search import search_graph_by_embedding + from app.core.memory.client_factory import MemoryClientFactory from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient + from app.core.memory.utils.config.definitions import ( + SELECTED_EMBEDDING_ID, + SELECTED_LLM_ID, + ) from app.core.models.base import RedBearModelConfig - from app.core.memory.utils.llm.llm_utils import get_llm_client - from app.core.memory.utils.config.config_utils import get_embedder_config - from app.core.memory.utils.config.definitions import SELECTED_LLM_ID, SELECTED_EMBEDDING_ID + from app.db import get_db_context + from app.repositories.neo4j.graph_search import search_graph_by_embedding + from app.repositories.neo4j.neo4j_connector import Neo4jConnector + from app.services.memory_config_service import MemoryConfigService # 加载数据 # 获取项目根目录 @@ -458,10 +468,14 @@ async def run_enhanced_evaluation(): # 初始化增强监控器 monitor = EnhancedEvaluationMonitor(reset_interval=5, performance_threshold=0.6) - llm = get_llm_client(SELECTED_LLM_ID) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm = factory.get_llm_client(SELECTED_LLM_ID) # 初始化embedder - cfg_dict = get_embedder_config(SELECTED_EMBEDDING_ID) + with get_db_context() as db: + config_service = MemoryConfigService(db) + cfg_dict = config_service.get_embedder_config(SELECTED_EMBEDDING_ID) embedder = OpenAIEmbedderClient( model_config=RedBearModelConfig.model_validate(cfg_dict) ) diff --git a/api/app/core/memory/evaluation/locomo/qwen_search_eval.py b/api/app/core/memory/evaluation/locomo/qwen_search_eval.py index cbc0bfbd..e7a95e71 100644 --- a/api/app/core/memory/evaluation/locomo/qwen_search_eval.py +++ b/api/app/core/memory/evaluation/locomo/qwen_search_eval.py @@ -2,10 +2,11 @@ import argparse import asyncio import json import os +import statistics import time from datetime import datetime, timedelta -from typing import List, Dict, Any -import statistics +from typing import Any, Dict, List + try: from dotenv import load_dotenv except Exception: @@ -13,16 +14,31 @@ except Exception: return None import re -from app.repositories.neo4j.neo4j_connector import Neo4jConnector -from app.repositories.neo4j.graph_search import search_graph, search_graph_by_embedding + +from app.core.memory.client_factory import MemoryClientFactory +from app.core.memory.evaluation.common.metrics import ( + avg_context_tokens, + bleu1, + jaccard, + latency_stats, +) +from app.core.memory.evaluation.common.metrics import f1_score as common_f1 +from app.core.memory.evaluation.extraction_utils import ( + ingest_contexts_via_full_pipeline, +) from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient -from app.core.models.base import RedBearModelConfig -from app.core.memory.utils.config.config_utils import get_embedder_config from app.core.memory.storage_services.search import run_hybrid_search -from app.core.memory.utils.config.definitions import PROJECT_ROOT, SELECTED_GROUP_ID, SELECTED_LLM_ID, SELECTED_EMBEDDING_ID -from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.core.memory.evaluation.extraction_utils import ingest_contexts_via_full_pipeline -from app.core.memory.evaluation.common.metrics import f1_score as common_f1, bleu1, jaccard, latency_stats, avg_context_tokens +from app.core.memory.utils.config.definitions import ( + PROJECT_ROOT, + SELECTED_EMBEDDING_ID, + SELECTED_GROUP_ID, + SELECTED_LLM_ID, +) +from app.core.models.base import RedBearModelConfig +from app.db import get_db_context +from app.repositories.neo4j.graph_search import search_graph, search_graph_by_embedding +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.services.memory_config_service import MemoryConfigService # 参考 evaluation/locomo/evaluation.py 的 F1 计算逻辑（移除外部依赖，内联实现） @@ -327,9 +343,13 @@ async def run_locomo_eval( await ingest_contexts_via_full_pipeline(contents, group_id, save_chunk_output=True) # 使用异步LLM客户端 - llm_client = get_llm_client(SELECTED_LLM_ID) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client(SELECTED_LLM_ID) # 初始化embedder用于直接调用 - cfg_dict = get_embedder_config(SELECTED_EMBEDDING_ID) + with get_db_context() as db: + config_service = MemoryConfigService(db) + cfg_dict = config_service.get_embedder_config(SELECTED_EMBEDDING_ID) embedder = OpenAIEmbedderClient( model_config=RedBearModelConfig.model_validate(cfg_dict) ) diff --git a/api/app/core/memory/evaluation/longmemeval/qwen_search_eval.py b/api/app/core/memory/evaluation/longmemeval/qwen_search_eval.py index d7cd711b..58652033 100644 --- a/api/app/core/memory/evaluation/longmemeval/qwen_search_eval.py +++ b/api/app/core/memory/evaluation/longmemeval/qwen_search_eval.py @@ -2,11 +2,11 @@ import argparse import asyncio import json import os -import time import re import statistics +import time from datetime import datetime, timedelta -from typing import List, Dict, Any +from typing import Any, Dict, List try: from dotenv import load_dotenv @@ -16,6 +16,7 @@ except Exception: # 确保可以找到 src 及项目根路径 import sys + _THIS_DIR = os.path.dirname(os.path.abspath(__file__)) _PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(_THIS_DIR))) _SRC_DIR = os.path.join(_PROJECT_ROOT, "src") @@ -25,19 +26,33 @@ for _p in (_SRC_DIR, _PROJECT_ROOT): # 与现有评估脚本保持一致的导入方式 from app.repositories.neo4j.neo4j_connector import Neo4jConnector + try: # 优先从 extraction_utils1 导入 - from app.core.memory.evaluation.extraction_utils import ingest_contexts_via_full_pipeline # type: ignore + from app.core.memory.evaluation.extraction_utils import ( + ingest_contexts_via_full_pipeline, # type: ignore + ) except Exception: ingest_contexts_via_full_pipeline = None # 在运行时做兜底检查 -from app.repositories.neo4j.graph_search import search_graph, search_graph_by_embedding -from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient -from app.core.models.base import RedBearModelConfig -from app.core.memory.utils.config.config_utils import get_embedder_config -from app.core.memory.utils.llm.llm_utils import get_llm_client +from app.core.memory.client_factory import MemoryClientFactory +from app.core.memory.evaluation.common.metrics import ( + avg_context_tokens, + jaccard, + latency_stats, +) +from app.core.memory.evaluation.common.metrics import f1_score as common_f1 from app.core.memory.evaluation.dialogue_queries import SEARCH_ENTITIES_BY_NAME -from app.core.memory.utils.config.definitions import PROJECT_ROOT, SELECTED_LLM_ID, SELECTED_EMBEDDING_ID -from app.core.memory.evaluation.common.metrics import f1_score as common_f1, jaccard, latency_stats, avg_context_tokens +from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient +from app.core.memory.utils.config.definitions import ( + PROJECT_ROOT, + SELECTED_EMBEDDING_ID, + SELECTED_LLM_ID, +) +from app.core.models.base import RedBearModelConfig +from app.db import get_db_context +from app.repositories.neo4j.graph_search import search_graph, search_graph_by_embedding +from app.services.memory_config_service import MemoryConfigService + try: from app.core.memory.evaluation.common.metrics import exact_match except Exception: @@ -686,9 +701,13 @@ async def run_longmemeval_test( ) # 初始化组件（摄入后再初始化连接器）- 使用异步LLM客户端 - llm_client = get_llm_client(SELECTED_LLM_ID) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client(SELECTED_LLM_ID) connector = Neo4jConnector() - cfg_dict = get_embedder_config(SELECTED_EMBEDDING_ID) + with get_db_context() as db: + config_service = MemoryConfigService(db) + cfg_dict = config_service.get_embedder_config(SELECTED_EMBEDDING_ID) embedder = OpenAIEmbedderClient( model_config=RedBearModelConfig.model_validate(cfg_dict) ) @@ -748,10 +767,10 @@ async def run_longmemeval_test( if stmt_text: contexts_all.append(stmt_text) - for sm in summaries: - summary_text = str(sm.get("summary", "")).strip() - if summary_text: - contexts_all.append(summary_text) + # for sm in summaries: + # summary_text = str(sm.get("summary", "")).strip() + # if summary_text: + # contexts_all.append(summary_text) # 实体摘要（最多3个） scored = [e for e in entities if e.get("score") is not None] diff --git a/api/app/core/memory/evaluation/longmemeval/test_eval.py b/api/app/core/memory/evaluation/longmemeval/test_eval.py index 550de2d2..a0038260 100644 --- a/api/app/core/memory/evaluation/longmemeval/test_eval.py +++ b/api/app/core/memory/evaluation/longmemeval/test_eval.py @@ -2,11 +2,11 @@ import argparse import asyncio import json import os -import time import re import statistics +import time from datetime import datetime, timedelta -from typing import List, Dict, Any +from typing import Any, Dict, List try: from dotenv import load_dotenv @@ -15,15 +15,26 @@ except Exception: return None # 与现有评估脚本保持一致的导入方式 -from app.repositories.neo4j.neo4j_connector import Neo4jConnector -from app.repositories.neo4j.graph_search import search_graph, search_graph_by_embedding -from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient -from app.core.models.base import RedBearModelConfig -from app.core.memory.utils.config_utils import get_embedder_config -from app.core.memory.utils.llm_utils import get_llm_client +from app.core.memory.client_factory import MemoryClientFactory +from app.core.memory.evaluation.common.metrics import ( + avg_context_tokens, + jaccard, + latency_stats, +) +from app.core.memory.evaluation.common.metrics import f1_score as common_f1 from app.core.memory.evaluation.dialogue_queries import SEARCH_ENTITIES_BY_NAME -from app.core.memory.utils.config.definitions import PROJECT_ROOT, SELECTED_LLM_ID, SELECTED_EMBEDDING_ID -from app.core.memory.evaluation.common.metrics import f1_score as common_f1, jaccard, latency_stats, avg_context_tokens +from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient +from app.core.memory.utils.config.definitions import ( + PROJECT_ROOT, + SELECTED_EMBEDDING_ID, + SELECTED_LLM_ID, +) +from app.core.models.base import RedBearModelConfig +from app.db import get_db_context +from app.repositories.neo4j.graph_search import search_graph, search_graph_by_embedding +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.services.memory_config_service import MemoryConfigService + try: from app.core.memory.evaluation.common.metrics import exact_match except Exception: @@ -647,9 +658,13 @@ async def run_longmemeval_test( items = qa_list[start_index:start_index + sample_size] # 初始化组件 - 使用异步LLM客户端 - llm_client = get_llm_client(SELECTED_LLM_ID) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client(SELECTED_LLM_ID) connector = Neo4jConnector() - cfg_dict = get_embedder_config(SELECTED_EMBEDDING_ID) + with get_db_context() as db: + config_service = MemoryConfigService(db) + cfg_dict = config_service.get_embedder_config(SELECTED_EMBEDDING_ID) embedder = OpenAIEmbedderClient( model_config=RedBearModelConfig.model_validate(cfg_dict) ) diff --git a/api/app/core/memory/evaluation/memsciqa/evaluate_qa.py b/api/app/core/memory/evaluation/memsciqa/evaluate_qa.py index f41d8f10..3e6a1216 100644 --- a/api/app/core/memory/evaluation/memsciqa/evaluate_qa.py +++ b/api/app/core/memory/evaluation/memsciqa/evaluate_qa.py @@ -4,19 +4,35 @@ import json import os import time from datetime import datetime -from typing import List, Dict, Any +from typing import TYPE_CHECKING, Any, Dict, List + +if TYPE_CHECKING: + from app.schemas.memory_config_schema import MemoryConfig + try: from dotenv import load_dotenv except Exception: def load_dotenv(): return None -from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.core.memory.client_factory import MemoryClientFactory +from app.core.memory.evaluation.common.metrics import ( + avg_context_tokens, + exact_match, + latency_stats, +) +from app.core.memory.evaluation.extraction_utils import ( + ingest_contexts_via_full_pipeline, +) from app.core.memory.storage_services.search import run_hybrid_search -from app.core.memory.utils.config.definitions import PROJECT_ROOT, SELECTED_GROUP_ID, SELECTED_EMBEDDING_ID, SELECTED_LLM_ID -from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.core.memory.evaluation.extraction_utils import ingest_contexts_via_full_pipeline -from app.core.memory.evaluation.common.metrics import exact_match, latency_stats, avg_context_tokens +from app.core.memory.utils.config.definitions import ( + PROJECT_ROOT, + SELECTED_EMBEDDING_ID, + SELECTED_GROUP_ID, + SELECTED_LLM_ID, +) +from app.db import get_db_context +from app.repositories.neo4j.neo4j_connector import Neo4jConnector def smart_context_selection(contexts: List[str], question: str, max_chars: int = 4000) -> str: @@ -119,7 +135,7 @@ def _combine_dialogues_for_hybrid(results: Dict[str, Any]) -> List[Dict[str, Any return merged -async def run_memsciqa_eval(sample_size: int = 1, group_id: str | None = None, search_limit: int = 8, context_char_budget: int = 4000, llm_temperature: float = 0.0, llm_max_tokens: int = 64, search_type: str = "hybrid") -> Dict[str, Any]: +async def run_memsciqa_eval(sample_size: int = 1, group_id: str | None = None, search_limit: int = 8, context_char_budget: int = 4000, llm_temperature: float = 0.0, llm_max_tokens: int = 64, search_type: str = "hybrid", memory_config: "MemoryConfig" = None) -> Dict[str, Any]: group_id = group_id or SELECTED_GROUP_ID # Load data data_path = os.path.join(PROJECT_ROOT, "data", "msc_self_instruct.jsonl") @@ -134,7 +150,9 @@ async def run_memsciqa_eval(sample_size: int = 1, group_id: str | None = None, s await ingest_contexts_via_full_pipeline(contexts, group_id) # LLM client (使用异步调用) - llm_client = get_llm_client(SELECTED_LLM_ID) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client(SELECTED_LLM_ID) # Evaluate each item connector = Neo4jConnector() @@ -159,6 +177,7 @@ async def run_memsciqa_eval(sample_size: int = 1, group_id: str | None = None, s limit=search_limit, include=["dialogues", "statements", "entities"], output_path=None, + memory_config=memory_config, ) except Exception: results = None @@ -242,7 +261,11 @@ async def run_memsciqa_eval(sample_size: int = 1, group_id: str | None = None, s pred = resp.content.strip() if hasattr(resp, 'content') else (resp["choices"][0]["message"]["content"].strip() if isinstance(resp, dict) else str(resp).strip()) # Metrics: F1, BLEU-1, Jaccard; keep exact match for reference correct_flags.append(exact_match(pred, reference)) - from app.core.memory.evaluation.common.metrics import f1_score, bleu1, jaccard + from app.core.memory.evaluation.common.metrics import ( + bleu1, + f1_score, + jaccard, + ) f1s.append(f1_score(str(pred), str(reference))) b1s.append(bleu1(str(pred), str(reference))) jss.append(jaccard(str(pred), str(reference))) diff --git a/api/app/core/memory/evaluation/memsciqa/memsciqa-test.py b/api/app/core/memory/evaluation/memsciqa/memsciqa-test.py index c8d89a4d..ebbe6e7e 100644 --- a/api/app/core/memory/evaluation/memsciqa/memsciqa-test.py +++ b/api/app/core/memory/evaluation/memsciqa/memsciqa-test.py @@ -2,10 +2,10 @@ import argparse import asyncio import json import os +import re import time from datetime import datetime -from typing import List, Dict, Any -import re +from typing import Any, Dict, List try: from dotenv import load_dotenv @@ -15,6 +15,7 @@ except Exception: # 路径与模块导入保持与现有评估脚本一致 import sys + _THIS_DIR = os.path.dirname(os.path.abspath(__file__)) _PROJECT_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR)) _SRC_DIR = os.path.join(_PROJECT_ROOT, "src") @@ -23,17 +24,27 @@ for _p in (_SRC_DIR, _PROJECT_ROOT): sys.path.insert(0, _p) # 对齐 locomo_test 的检索逻辑：直接使用 graph_search 与 Neo4jConnector/Embedder1 -from app.repositories.neo4j.neo4j_connector import Neo4jConnector -from app.repositories.neo4j.graph_search import search_graph, search_graph_by_embedding +from app.core.memory.client_factory import MemoryClientFactory +from app.core.memory.evaluation.common.metrics import ( + avg_context_tokens, + exact_match, + latency_stats, +) from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient +from app.core.memory.utils.config.definitions import ( + PROJECT_ROOT, + SELECTED_EMBEDDING_ID, + SELECTED_GROUP_ID, + SELECTED_LLM_ID, +) from app.core.models.base import RedBearModelConfig -from app.core.memory.utils.config_utils import get_embedder_config +from app.db import get_db_context +from app.repositories.neo4j.graph_search import search_graph, search_graph_by_embedding +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.services.memory_config_service import MemoryConfigService -from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.core.memory.utils.config.definitions import PROJECT_ROOT, SELECTED_GROUP_ID, SELECTED_EMBEDDING_ID, SELECTED_LLM_ID -from app.core.memory.evaluation.common.metrics import exact_match, latency_stats, avg_context_tokens try: - from app.core.memory.evaluation.common.metrics import f1_score, bleu1, jaccard + from app.core.memory.evaluation.common.metrics import bleu1, f1_score, jaccard except Exception: # 兜底：简单实现（必要时） def f1_score(pred: str, ref: str) -> float: @@ -226,13 +237,17 @@ async def run_memsciqa_test( items = all_items[start_index:start_index + sample_size] # 初始化 LLM（纯测试：不进行摄入） - llm = get_llm_client(SELECTED_LLM_ID) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm = factory.get_llm_client(SELECTED_LLM_ID) # 初始化 Neo4j 连接与向量检索 Embedder（对齐 locomo_test） connector = Neo4jConnector() embedder = None if search_type in ("embedding", "hybrid"): - cfg_dict = get_embedder_config(SELECTED_EMBEDDING_ID) + with get_db_context() as db: + config_service = MemoryConfigService(db) + cfg_dict = config_service.get_embedder_config(SELECTED_EMBEDDING_ID) embedder = OpenAIEmbedderClient( model_config=RedBearModelConfig.model_validate(cfg_dict) ) diff --git a/api/app/core/memory/llm_tools/openai_client.py b/api/app/core/memory/llm_tools/openai_client.py index bcaa52c2..dce7b495 100644 --- a/api/app/core/memory/llm_tools/openai_client.py +++ b/api/app/core/memory/llm_tools/openai_client.py @@ -5,18 +5,17 @@ OpenAI LLM 客户端实现 """ import asyncio -from typing import List, Dict, Any import json import logging +from typing import Any, Dict, List -from pydantic import BaseModel -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.output_parsers import PydanticOutputParser - +from app.core.config import settings +from app.core.memory.llm_tools.llm_client import LLMClient, LLMClientException from app.core.models.base import RedBearModelConfig from app.core.models.llm import RedBearLLM -from app.core.memory.llm_tools.llm_client import LLMClient, LLMClientException -from app.core.memory.utils.config.definitions import LANGFUSE_ENABLED +from langchain_core.output_parsers import PydanticOutputParser +from langchain_core.prompts import ChatPromptTemplate +from pydantic import BaseModel logger = logging.getLogger(__name__) @@ -43,7 +42,7 @@ class OpenAIClient(LLMClient): # 初始化 Langfuse 回调处理器（如果启用） self.langfuse_handler = None - if LANGFUSE_ENABLED: + if settings.LANGFUSE_ENABLED: try: from langfuse.langchain import CallbackHandler self.langfuse_handler = CallbackHandler() diff --git a/api/app/core/memory/main.py b/api/app/core/memory/main.py deleted file mode 100644 index 68bb1de9..00000000 --- a/api/app/core/memory/main.py +++ /dev/null @@ -1,430 +0,0 @@ -""" -MemSci 记忆系统主入口 - 重构版本 - -该模块是重构后的记忆系统主入口，使用新的模块化架构。 -旧版本入口（app/core/memory/src/main.py）已删除。 - -主要功能： -1. 协调整个知识提取流水线 -2. 支持试运行模式和正常运行模式 -3. 使用重构后的 storage_services 模块 -4. 提供统一的配置管理和日志记录 - -作者：Lance77 -日期：2025-11-22 -""" - -# 必须在最开始禁用 LangSmith 追踪，避免速率限制错误 -import os -os.environ["LANGCHAIN_TRACING_V2"] = "false" -os.environ["LANGCHAIN_TRACING"] = "false" -import asyncio -import time -from datetime import datetime -from typing import Optional, Callable, Awaitable -from dotenv import load_dotenv - -# 导入重构后的模块 -from app.repositories.neo4j.neo4j_connector import Neo4jConnector -from app.core.memory.storage_services.extraction_engine.extraction_orchestrator import ExtractionOrchestrator -from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.core.memory.utils.config.config_utils import get_embedder_config -from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient -from app.core.memory.models.message_models import ConversationMessage, ConversationContext, DialogData -from app.core.memory.models.variate_config import ExtractionPipelineConfig - -# 导入数据加载函数 -from app.core.memory.storage_services.extraction_engine.extraction_orchestrator import ( - get_chunked_dialogs_with_preprocessing, - get_chunked_dialogs_from_preprocessed, -) -# 导入配置模块（而不是直接导入变量） -from app.core.memory.utils.config import definitions as config_defs -from app.core.logging_config import get_memory_logger, log_time - -load_dotenv() - -logger = get_memory_logger(__name__) - - - - - -async def main( - # Required configuration parameters (no longer from global variables) - chunker_strategy: str, - group_id: str, - user_id: str, - apply_id: str, - llm_model_id: str, - embedding_model_id: str, - # Optional parameters - dialogue_text: Optional[str] = None, - is_pilot_run: bool = False, - progress_callback: Optional[Callable[[str, str, Optional[dict]], Awaitable[None]]] = None -): - """ - 记忆系统主流程 - 重构版本 (Updated to eliminate global variables) - - 该函数是重构后的主入口，使用新的模块化架构。 - Global variables have been eliminated in favor of explicit parameters. - - Args: - chunker_strategy: Chunking strategy to use (required) - group_id: Group ID for the operation (required) - user_id: User ID for the operation (required) - apply_id: Application ID for the operation (required) - llm_model_id: LLM model ID to use (required) - embedding_model_id: Embedding model ID to use (required) - dialogue_text: 输入的对话文本（可选，用于试运行模式） - is_pilot_run: 是否为试运行模式 - - True: 试运行模式，不保存到 Neo4j - - False: 正常运行模式，保存到 Neo4j - progress_callback: 可选的进度回调函数 - - 类型: Callable[[str, str, Optional[dict]], Awaitable[None]] - - 参数1 (stage): 当前处理阶段标识符 - - 参数2 (message): 人类可读的进度消息 - - 参数3 (data): 可选的附加数据字典，包含详细的进度信息或结果 - - 在管线关键点调用以报告进度和结果数据 - - 工作流程： - 1. 初始化客户端和配置 - 2. 加载或准备数据 - 3. 执行知识提取流水线 - 4. 保存结果（正常模式）或输出结果（试运行模式） - """ - print("=" * 60) - print("MemSci 知识提取流水线 - 重构版本") - print("=" * 60) - print(f"运行模式: {'试运行（不保存到Neo4j）' if is_pilot_run else '正常运行（保存到Neo4j）'}") - print("Using chunker strategy:", chunker_strategy) - print("Using group ID:", group_id) - print("Using model ID:", llm_model_id) - print("Using embedding model ID:", embedding_model_id) - print("=" * 60) - - # 初始化日志 - log_file = "logs/time.log" - os.makedirs(os.path.dirname(log_file), exist_ok=True) - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - with open(log_file, "a", encoding="utf-8") as f: - f.write(f"\n=== Pipeline Run Started: {timestamp} ({'Pilot Run' if is_pilot_run else 'Normal Run'}) ===\n") - - pipeline_start = time.time() - - try: - # 步骤 1: 初始化客户端 - logger.info("Initializing clients...") - step_start = time.time() - - llm_client = get_llm_client(llm_model_id) - - # 获取 embedder 配置并转换为 RedBearModelConfig 对象 - from app.core.models.base import RedBearModelConfig - embedder_config_dict = get_embedder_config(embedding_model_id) - embedder_config = RedBearModelConfig(**embedder_config_dict) - embedder_client = OpenAIEmbedderClient(embedder_config) - - neo4j_connector = Neo4jConnector() - - log_time("Client Initialization", time.time() - step_start, log_file) - - # 步骤 2: 加载或准备数据 - logger.info("Loading data...") - logger.info(f"[MAIN] dialogue_text type={type(dialogue_text)}, length={len(dialogue_text) if dialogue_text else 0}, is_pilot_run={is_pilot_run}") - logger.info(f"[MAIN] dialogue_text preview: {repr(dialogue_text)[:200] if dialogue_text else 'None'}") - logger.info(f"[MAIN] Condition check: dialogue_text={bool(dialogue_text)}, isinstance={isinstance(dialogue_text, str) if dialogue_text else False}, strip={bool(dialogue_text.strip()) if dialogue_text and isinstance(dialogue_text, str) else False}") - step_start = time.time() - - if dialogue_text and isinstance(dialogue_text, str) and dialogue_text.strip(): - # 试运行模式：处理前端传入的对话文本 - logger.info("[MAIN] ✓ Using frontend dialogue text (pilot run mode)") - import re - - # 解析对话文本，支持 "用户:" 和 "AI:" 格式 - pattern = r"(用户|AI)[：:]\s*([^\n]+(?:\n(?!(?:用户|AI)[：:])[^\n]*)*?)" - matches = re.findall(pattern, dialogue_text, re.MULTILINE | re.DOTALL) - messages = [ - ConversationMessage(role=r, msg=c.strip()) - for r, c in matches if c.strip() - ] - - # 如果没有匹配到格式化的对话，将整个文本作为用户消息 - if not messages: - messages = [ConversationMessage(role="用户", msg=dialogue_text.strip())] - - # 创建对话上下文和对话数据 - context = ConversationContext(msgs=messages) - dialog = DialogData( - context=context, - ref_id="pilot_dialog_1", - group_id=group_id, - user_id=user_id, - apply_id=apply_id, - metadata={"source": "pilot_run", "input_type": "frontend_text"} - ) - - # 进度回调：开始预处理文本 - if progress_callback: - await progress_callback("text_preprocessing", "开始预处理文本...") - - # 对前端传入的对话进行分块处理 - chunked_dialogs = await get_chunked_dialogs_from_preprocessed( - data=[dialog], - chunker_strategy=chunker_strategy, - llm_client=llm_client, - ) - logger.info(f"Processed frontend dialogue text: {len(messages)} messages") - - # 进度回调：输出每个分块的结果 - if progress_callback: - for dialog in chunked_dialogs: - for i, chunk in enumerate(dialog.chunks): - chunk_result = { - "chunk_index": i + 1, - "content": chunk.content[:200] + "..." if len(chunk.content) > 200 else chunk.content, - "full_length": len(chunk.content), - "dialog_id": dialog.id, - "chunker_strategy": chunker_strategy - } - await progress_callback("text_preprocessing_result", f"分块 {i + 1} 处理完成", chunk_result) - - # 进度回调：预处理文本完成 - preprocessing_summary = { - "total_chunks": sum(len(dialog.chunks) for dialog in chunked_dialogs), - "total_dialogs": len(chunked_dialogs), - "chunker_strategy": chunker_strategy - } - await progress_callback("text_preprocessing_complete", "预处理文本完成", preprocessing_summary) - else: - # 正常运行模式：从 testdata.json 文件加载 - logger.warning("[MAIN] ✗ Falling back to testdata.json (dialogue_text not provided or empty)") - logger.info("Loading data from testdata.json...") - test_data_path = os.path.join( - os.path.dirname(__file__), "data", "testdata.json" - ) - - if not os.path.exists(test_data_path): - raise FileNotFoundError(f"Test data file not found: {test_data_path}") - - # 进度回调：开始预处理文本 - if progress_callback: - await progress_callback("text_preprocessing", "开始预处理文本...") - - chunked_dialogs = await get_chunked_dialogs_with_preprocessing( - chunker_strategy=chunker_strategy, - group_id=group_id, - user_id=user_id, - apply_id=apply_id, - indices=None, - input_data_path=test_data_path, - llm_client=llm_client, - skip_cleaning=True, - ) - logger.info(f"Loaded {len(chunked_dialogs)} dialogues from testdata.json") - - # 进度回调：输出每个分块的结果 - if progress_callback: - for dialog in chunked_dialogs: - for i, chunk in enumerate(dialog.chunks): - chunk_result = { - "chunk_index": i + 1, - "content": chunk.content[:200] + "..." if len(chunk.content) > 200 else chunk.content, - "full_length": len(chunk.content), - "dialog_id": dialog.id, - "chunker_strategy": chunker_strategy - } - await progress_callback("text_preprocessing_result", f"分块 {i + 1} 处理完成", chunk_result) - - # 进度回调：预处理文本完成 - preprocessing_summary = { - "total_chunks": sum(len(dialog.chunks) for dialog in chunked_dialogs), - "total_dialogs": len(chunked_dialogs), - "chunker_strategy": chunker_strategy - } - await progress_callback("text_preprocessing_complete", "预处理文本完成", preprocessing_summary) - - log_time("Data Loading & Chunking", time.time() - step_start, log_file) - - # 步骤 3: 初始化流水线编排器 - logger.info("Initializing extraction orchestrator...") - step_start = time.time() - - # 从 runtime.json 加载配置（已经过数据库覆写） - from app.core.memory.utils.config.config_utils import get_pipeline_config - config = get_pipeline_config() - - logger.info(f"Pipeline config loaded: enable_llm_dedup_blockwise={config.deduplication.enable_llm_dedup_blockwise}, enable_llm_disambiguation={config.deduplication.enable_llm_disambiguation}") - - orchestrator = ExtractionOrchestrator( - llm_client=llm_client, - embedder_client=embedder_client, - connector=neo4j_connector, - config=config, - progress_callback=progress_callback, # 传递进度回调 - embedding_id=embedding_model_id, # 传递嵌入模型ID - ) - - log_time("Orchestrator Initialization", time.time() - step_start, log_file) - - # 步骤 4: 执行知识提取流水线 - logger.info("Running extraction pipeline...") - step_start = time.time() - - - # 进度回调：正在知识抽取 - if progress_callback: - await progress_callback("knowledge_extraction", "正在知识抽取...") - - extraction_result = await orchestrator.run( - dialog_data_list=chunked_dialogs, - is_pilot_run=is_pilot_run, # 传递试运行模式标志 - ) - - # 解包 extraction_result tuple - # extraction_result 是一个包含 7 个元素的 tuple: - # (dialogue_nodes, chunk_nodes, statement_nodes, entity_nodes, - # statement_chunk_edges, statement_entity_edges, entity_edges) - ( - dialogue_nodes, - chunk_nodes, - statement_nodes, - entity_nodes, - statement_chunk_edges, - statement_entity_edges, - entity_edges, - ) = extraction_result - - log_time("Extraction Pipeline", time.time() - step_start, log_file) - - # 进度回调：生成结果 - if progress_callback: - await progress_callback("generating_results", "正在生成结果...") - - - # 步骤 5: 保存结果或输出结果 - if is_pilot_run: - logger.info("Pilot run mode: Skipping Neo4j save") - print("\n试运行模式：跳过 Neo4j 保存，流水线处理完成。") - print("提取结果已生成，可在相关输出中查看。") - else: - logger.info("Normal mode: Saving to Neo4j...") - step_start = time.time() - - # 创建索引和约束 - try: - from app.repositories.neo4j.create_indexes import ( - create_fulltext_indexes, - create_unique_constraints, - ) - await create_fulltext_indexes() - await create_unique_constraints() - logger.info("Successfully created indexes and constraints") - except Exception as e: - logger.error(f"Error creating indexes/constraints: {e}") - - # 保存数据到 Neo4j - try: - from app.repositories.neo4j.graph_saver import ( - save_dialog_and_statements_to_neo4j, - ) - - success = await save_dialog_and_statements_to_neo4j( - dialogue_nodes=dialogue_nodes, - chunk_nodes=chunk_nodes, - statement_nodes=statement_nodes, - entity_nodes=entity_nodes, - statement_chunk_edges=statement_chunk_edges, - statement_entity_edges=statement_entity_edges, - entity_edges=entity_edges, - connector=neo4j_connector, - ) - - if success: - logger.info("Successfully saved all data to Neo4j") - print("\n✓ 成功保存所有数据到 Neo4j") - else: - logger.warning("Failed to save some data to Neo4j") - print("\n⚠ 部分数据保存到 Neo4j 失败") - except Exception as e: - logger.error(f"Error saving to Neo4j: {e}", exc_info=True) - print(f"\n✗ 保存到 Neo4j 失败: {e}") - - log_time("Neo4j Database Save", time.time() - step_start, log_file) - - # 步骤 6: 生成记忆摘要（可选） - try: - logger.info("Generating memory summaries...") - step_start = time.time() - - from app.core.memory.storage_services.extraction_engine.knowledge_extraction.memory_summary import ( - Memory_summary_generation, - ) - from app.repositories.neo4j.add_nodes import add_memory_summary_nodes - from app.repositories.neo4j.add_edges import ( - add_memory_summary_statement_edges, - ) - - summaries = await Memory_summary_generation( - chunked_dialogs, llm_client=llm_client, embedding_id=embedding_model_id - ) - - if not is_pilot_run: - # 保存记忆摘要到 Neo4j - ms_connector = Neo4jConnector() - try: - await add_memory_summary_nodes(summaries, ms_connector) - await add_memory_summary_statement_edges(summaries, ms_connector) - finally: - await ms_connector.close() - - log_time("Memory Summary Generation", time.time() - step_start, log_file) - except Exception as e: - logger.error(f"Memory summary step failed: {e}", exc_info=True) - - except Exception as e: - logger.error(f"Pipeline execution failed: {e}", exc_info=True) - print(f"\n✗ 流水线执行失败: {e}") - raise - finally: - # 清理资源 - try: - await neo4j_connector.close() - except Exception: - pass - - # 记录总时间 - total_time = time.time() - pipeline_start - log_time("TOTAL PIPELINE TIME", total_time, log_file) - - # 添加完成标记 - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - with open(log_file, "a", encoding="utf-8") as f: - f.write(f"=== Pipeline Run Completed: {timestamp} ===\n\n") - - logger.info("=== Pipeline Complete ===") - logger.info(f"Total execution time: {total_time:.2f} seconds") - logger.info(f"Timing details saved to: {log_file}") - - print("\n" + "=" * 60) - print("✓ 流水线执行完成") - print(f"✓ 总耗时: {total_time:.2f} 秒") - print(f"✓ 详细日志: {log_file}") - print("=" * 60) - - -if __name__ == "__main__": - print("⚠️ Warning: This script now requires explicit configuration parameters.") - print("Global variables have been removed. Please provide configuration parameters.") - print("Example usage:") - print(" asyncio.run(main(") - print(" chunker_strategy='RecursiveChunker',") - print(" group_id='your_group_id',") - print(" user_id='your_user_id',") - print(" apply_id='your_apply_id',") - print(" llm_model_id='your_llm_id',") - print(" embedding_model_id='your_embedding_id'") - print(" ))") - - # This will fail because global variables are removed - raise RuntimeError("Global variables removed. Please provide explicit configuration parameters.") diff --git a/api/app/core/memory/models/graph_models.py b/api/app/core/memory/models/graph_models.py index a8c3f7b0..5977a2d7 100644 --- a/api/app/core/memory/models/graph_models.py +++ b/api/app/core/memory/models/graph_models.py @@ -20,14 +20,14 @@ Classes: MemorySummaryNode: Node representing a memory summary """ -from uuid import uuid4 +import re from datetime import datetime, timezone from typing import List, Optional -from pydantic import BaseModel, Field, field_validator -import re +from uuid import uuid4 -from app.core.memory.utils.data.ontology import TemporalInfo from app.core.memory.utils.alias_utils import validate_aliases +from app.core.memory.utils.data.ontology import TemporalInfo +from pydantic import BaseModel, Field, field_validator def parse_historical_datetime(v): @@ -361,7 +361,7 @@ class ExtractedEntityNode(Node): description="Entity aliases - alternative names for this entity" ) name_embedding: Optional[List[float]] = Field(default_factory=list, description="Name embedding vector") - fact_summary: str = Field(..., description="Summary of the fact about this entity") + fact_summary: str = Field(default="", description="Summary of the fact about this entity") connect_strength: str = Field(..., description="Strong VS Weak about this entity") config_id: Optional[int | str] = Field(None, description="Configuration ID used to process this entity (integer or string)") diff --git a/api/app/core/memory/models/triplet_models.py b/api/app/core/memory/models/triplet_models.py index 2325f3bd..b0a062a3 100644 --- a/api/app/core/memory/models/triplet_models.py +++ b/api/app/core/memory/models/triplet_models.py @@ -10,9 +10,10 @@ Classes: """ from typing import List, Optional -from pydantic import BaseModel, Field, ConfigDict from uuid import uuid4 +from pydantic import BaseModel, ConfigDict, Field + class Entity(BaseModel): """Represents an extracted entity from dialogue. diff --git a/api/app/core/memory/src/search.py b/api/app/core/memory/src/search.py index 597a4789..9353f00e 100644 --- a/api/app/core/memory/src/search.py +++ b/api/app/core/memory/src/search.py @@ -5,7 +5,10 @@ import math import os import time from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +if TYPE_CHECKING: + from app.schemas.memory_config_schema import MemoryConfig from app.core.logging_config import get_memory_logger from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient @@ -14,16 +17,14 @@ from app.core.memory.models.variate_config import ForgettingEngineConfig from app.core.memory.storage_services.forgetting_engine.forgetting_engine import ( ForgettingEngine, ) -from app.core.memory.utils.config import definitions as config_defs from app.core.memory.utils.config.config_utils import ( - get_embedder_config, get_pipeline_config, ) -from app.core.memory.utils.config.definitions import CONFIG, RUNTIME_CONFIG from app.core.memory.utils.data.text_utils import extract_plain_query from app.core.memory.utils.data.time_utils import normalize_date_safe from app.core.memory.utils.llm.llm_utils import get_reranker_client from app.core.models.base import RedBearModelConfig +from app.db import get_db_context from app.repositories.neo4j.graph_search import ( search_graph, search_graph_by_chunk_id, @@ -34,6 +35,7 @@ from app.repositories.neo4j.graph_search import ( # 使用新的仓储层 from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.services.memory_config_service import MemoryConfigService from dotenv import load_dotenv load_dotenv() @@ -324,229 +326,229 @@ def apply_reranker_placeholder( If config enables reranker, annotate items with a final_score equal to combined_score and keep ordering. This is a no-op reranker to be replaced later. """ - try: - rc = (RUNTIME_CONFIG.get("reranker", {}) or CONFIG.get("reranker", {})) - except Exception as e: - logger.debug(f"Failed to load reranker config: {e}") - rc = {} - if not rc or not rc.get("enabled", False): - return results + # try: + # rc = (RUNTIME_CONFIG.get("reranker", {}) or CONFIG.get("reranker", {})) + # except Exception as e: + # logger.debug(f"Failed to load reranker config: {e}") + # rc = {} + # if not rc or not rc.get("enabled", False): + # return results - top_k = int(rc.get("top_k", 100)) - model_name = rc.get("model", "placeholder") + # top_k = int(rc.get("top_k", 100)) + # model_name = rc.get("model", "placeholder") - for cat, items in results.items(): - head = items[:top_k] - for it in head: - base = float(it.get("combined_score", it.get("score", 0.0)) or 0.0) - it["final_score"] = base - it["reranker_model"] = model_name - # Keep overall order by final_score if present, otherwise combined/score - results[cat] = sorted( - items, - key=lambda x: float(x.get("final_score", x.get("combined_score", x.get("score", 0.0)) or 0.0)), - reverse=True, - ) + # for cat, items in results.items(): + # head = items[:top_k] + # for it in head: + # base = float(it.get("combined_score", it.get("score", 0.0)) or 0.0) + # it["final_score"] = base + # it["reranker_model"] = model_name + # # Keep overall order by final_score if present, otherwise combined/score + # results[cat] = sorted( + # items, + # key=lambda x: float(x.get("final_score", x.get("combined_score", x.get("score", 0.0)) or 0.0)), + # reverse=True, + # ) return results -async def apply_llm_reranker( - results: Dict[str, List[Dict[str, Any]]], - query_text: str, - reranker_client: Optional[Any] = None, - llm_weight: Optional[float] = None, - top_k: Optional[int] = None, - batch_size: Optional[int] = None, -) -> Dict[str, List[Dict[str, Any]]]: - """ - Apply LLM-based reranking to search results. +# async def apply_llm_reranker( +# results: Dict[str, List[Dict[str, Any]]], +# query_text: str, +# reranker_client: Optional[Any] = None, +# llm_weight: Optional[float] = None, +# top_k: Optional[int] = None, +# batch_size: Optional[int] = None, +# ) -> Dict[str, List[Dict[str, Any]]]: +# """ +# Apply LLM-based reranking to search results. - Args: - results: Search results organized by category - query_text: Original search query - reranker_client: Optional pre-initialized reranker client - llm_weight: Weight for LLM score (0.0-1.0, higher favors LLM) - top_k: Maximum number of items to rerank per category - batch_size: Number of items to process concurrently +# Args: +# results: Search results organized by category +# query_text: Original search query +# reranker_client: Optional pre-initialized reranker client +# llm_weight: Weight for LLM score (0.0-1.0, higher favors LLM) +# top_k: Maximum number of items to rerank per category +# batch_size: Number of items to process concurrently - Returns: - Reranked results with final_score and reranker_model fields - """ - # Load reranker configuration from runtime.json - try: - rc = RUNTIME_CONFIG.get("reranker", {}) or CONFIG.get("reranker", {}) - except Exception as e: - logger.debug(f"Failed to load reranker config: {e}") - rc = {} +# Returns: +# Reranked results with final_score and reranker_model fields +# """ +# # Load reranker configuration from runtime.json +# # try: +# # rc = RUNTIME_CONFIG.get("reranker", {}) or CONFIG.get("reranker", {}) +# # except Exception as e: +# # logger.debug(f"Failed to load reranker config: {e}") +# # rc = {} - # Check if reranking is enabled - enabled = rc.get("enabled", False) - if not enabled: - logger.debug("LLM reranking is disabled in configuration") - return results +# # Check if reranking is enabled +# enabled = rc.get("enabled", False) +# if not enabled: +# logger.debug("LLM reranking is disabled in configuration") +# return results - # Load configuration parameters with defaults - llm_weight = llm_weight if llm_weight is not None else rc.get("llm_weight", 0.5) - top_k = top_k if top_k is not None else rc.get("top_k", 20) - batch_size = batch_size if batch_size is not None else rc.get("batch_size", 5) +# # Load configuration parameters with defaults +# llm_weight = llm_weight if llm_weight is not None else rc.get("llm_weight", 0.5) +# top_k = top_k if top_k is not None else rc.get("top_k", 20) +# batch_size = batch_size if batch_size is not None else rc.get("batch_size", 5) - # Initialize reranker client if not provided - if reranker_client is None: - try: - reranker_client = get_reranker_client() - except Exception as e: - logger.warning(f"Failed to initialize reranker client: {e}, skipping LLM reranking") - return results +# # Initialize reranker client if not provided +# if reranker_client is None: +# try: +# reranker_client = get_reranker_client() +# except Exception as e: +# logger.warning(f"Failed to initialize reranker client: {e}, skipping LLM reranking") +# return results - # Get model name for metadata - model_name = getattr(reranker_client, 'model_name', 'unknown') +# # Get model name for metadata +# model_name = getattr(reranker_client, 'model_name', 'unknown') - # Process each category - reranked_results = {} - for category in ["statements", "chunks", "entities", "summaries"]: - items = results.get(category, []) - if not items: - reranked_results[category] = [] - continue +# # Process each category +# reranked_results = {} +# for category in ["statements", "chunks", "entities", "summaries"]: +# items = results.get(category, []) +# if not items: +# reranked_results[category] = [] +# continue - # Select top K items by combined_score for reranking - sorted_items = sorted( - items, - key=lambda x: float(x.get("combined_score", x.get("score", 0.0)) or 0.0), - reverse=True - ) +# # Select top K items by combined_score for reranking +# sorted_items = sorted( +# items, +# key=lambda x: float(x.get("combined_score", x.get("score", 0.0)) or 0.0), +# reverse=True +# ) - top_items = sorted_items[:top_k] - remaining_items = sorted_items[top_k:] +# top_items = sorted_items[:top_k] +# remaining_items = sorted_items[top_k:] - # Extract text content from each item - def extract_text(item: Dict[str, Any]) -> str: - """Extract text content from a result item.""" - # Try different text fields based on category - text = item.get("text") or item.get("content") or item.get("statement") or item.get("name") or "" - return str(text).strip() +# # Extract text content from each item +# def extract_text(item: Dict[str, Any]) -> str: +# """Extract text content from a result item.""" +# # Try different text fields based on category +# text = item.get("text") or item.get("content") or item.get("statement") or item.get("name") or "" +# return str(text).strip() - # Batch items for concurrent processing - batches = [] - for i in range(0, len(top_items), batch_size): - batch = top_items[i:i + batch_size] - batches.append(batch) +# # Batch items for concurrent processing +# batches = [] +# for i in range(0, len(top_items), batch_size): +# batch = top_items[i:i + batch_size] +# batches.append(batch) - # Process batches concurrently - async def process_batch(batch: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Process a batch of items with LLM relevance scoring.""" - scored_batch = [] +# # Process batches concurrently +# async def process_batch(batch: List[Dict[str, Any]]) -> List[Dict[str, Any]]: +# """Process a batch of items with LLM relevance scoring.""" +# scored_batch = [] - for item in batch: - item_text = extract_text(item) +# for item in batch: +# item_text = extract_text(item) - # Skip items with no text - if not item_text: - item_copy = item.copy() - combined_score = float(item.get("combined_score", item.get("score", 0.0)) or 0.0) - item_copy["final_score"] = combined_score - item_copy["llm_relevance_score"] = 0.0 - item_copy["reranker_model"] = model_name - scored_batch.append(item_copy) - continue +# # Skip items with no text +# if not item_text: +# item_copy = item.copy() +# combined_score = float(item.get("combined_score", item.get("score", 0.0)) or 0.0) +# item_copy["final_score"] = combined_score +# item_copy["llm_relevance_score"] = 0.0 +# item_copy["reranker_model"] = model_name +# scored_batch.append(item_copy) +# continue - # Create relevance scoring prompt - prompt = f"""Given the search query and a result item, rate the relevance of the item to the query on a scale from 0.0 to 1.0. +# # Create relevance scoring prompt +# prompt = f"""Given the search query and a result item, rate the relevance of the item to the query on a scale from 0.0 to 1.0. -Query: {query_text} +# Query: {query_text} -Result: {item_text} +# Result: {item_text} -Respond with only a number between 0.0 and 1.0, where: -- 0.0 means completely irrelevant -- 1.0 means perfectly relevant +# Respond with only a number between 0.0 and 1.0, where: +# - 0.0 means completely irrelevant +# - 1.0 means perfectly relevant -Relevance score:""" +# Relevance score:""" - # Send request to LLM - try: - messages = [{"role": "user", "content": prompt}] - response = await reranker_client.chat(messages) +# # Send request to LLM +# try: +# messages = [{"role": "user", "content": prompt}] +# response = await reranker_client.chat(messages) - # Parse LLM response to extract relevance score - response_text = str(response.content if hasattr(response, 'content') else response).strip() +# # Parse LLM response to extract relevance score +# response_text = str(response.content if hasattr(response, 'content') else response).strip() - # Try to extract a float from the response - try: - # Remove any non-numeric characters except decimal point - import re - score_match = re.search(r'(\d+\.?\d*)', response_text) - if score_match: - llm_score = float(score_match.group(1)) - # Clamp to [0.0, 1.0] - llm_score = max(0.0, min(1.0, llm_score)) - else: - raise ValueError("No numeric score found in response") - except (ValueError, AttributeError) as e: - logger.warning(f"Invalid LLM score format: {response_text}, using combined_score. Error: {e}") - llm_score = None +# # Try to extract a float from the response +# try: +# # Remove any non-numeric characters except decimal point +# import re +# score_match = re.search(r'(\d+\.?\d*)', response_text) +# if score_match: +# llm_score = float(score_match.group(1)) +# # Clamp to [0.0, 1.0] +# llm_score = max(0.0, min(1.0, llm_score)) +# else: +# raise ValueError("No numeric score found in response") +# except (ValueError, AttributeError) as e: +# logger.warning(f"Invalid LLM score format: {response_text}, using combined_score. Error: {e}") +# llm_score = None - # Calculate final score - item_copy = item.copy() - combined_score = float(item.get("combined_score", item.get("score", 0.0)) or 0.0) +# # Calculate final score +# item_copy = item.copy() +# combined_score = float(item.get("combined_score", item.get("score", 0.0)) or 0.0) - if llm_score is not None: - final_score = (1 - llm_weight) * combined_score + llm_weight * llm_score - item_copy["llm_relevance_score"] = llm_score - else: - # Use combined_score as fallback - final_score = combined_score - item_copy["llm_relevance_score"] = combined_score +# if llm_score is not None: +# final_score = (1 - llm_weight) * combined_score + llm_weight * llm_score +# item_copy["llm_relevance_score"] = llm_score +# else: +# # Use combined_score as fallback +# final_score = combined_score +# item_copy["llm_relevance_score"] = combined_score - item_copy["final_score"] = final_score - item_copy["reranker_model"] = model_name - scored_batch.append(item_copy) - except Exception as e: - logger.warning(f"Error processing item in LLM reranking: {e}, using combined_score") - item_copy = item.copy() - combined_score = float(item.get("combined_score", item.get("score", 0.0)) or 0.0) - item_copy["final_score"] = combined_score - item_copy["llm_relevance_score"] = combined_score - item_copy["reranker_model"] = model_name - scored_batch.append(item_copy) +# item_copy["final_score"] = final_score +# item_copy["reranker_model"] = model_name +# scored_batch.append(item_copy) +# except Exception as e: +# logger.warning(f"Error processing item in LLM reranking: {e}, using combined_score") +# item_copy = item.copy() +# combined_score = float(item.get("combined_score", item.get("score", 0.0)) or 0.0) +# item_copy["final_score"] = combined_score +# item_copy["llm_relevance_score"] = combined_score +# item_copy["reranker_model"] = model_name +# scored_batch.append(item_copy) - return scored_batch +# return scored_batch - # Process all batches concurrently - try: - batch_tasks = [process_batch(batch) for batch in batches] - batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) +# # Process all batches concurrently +# try: +# batch_tasks = [process_batch(batch) for batch in batches] +# batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) - # Merge batch results - scored_items = [] - for result in batch_results: - if isinstance(result, Exception): - logger.warning(f"Batch processing failed: {result}") - continue - scored_items.extend(result) +# # Merge batch results +# scored_items = [] +# for result in batch_results: +# if isinstance(result, Exception): +# logger.warning(f"Batch processing failed: {result}") +# continue +# scored_items.extend(result) - # Add remaining items (not in top K) with their combined_score as final_score - for item in remaining_items: - item_copy = item.copy() - combined_score = float(item.get("combined_score", item.get("score", 0.0)) or 0.0) - item_copy["final_score"] = combined_score - item_copy["reranker_model"] = model_name - scored_items.append(item_copy) +# # Add remaining items (not in top K) with their combined_score as final_score +# for item in remaining_items: +# item_copy = item.copy() +# combined_score = float(item.get("combined_score", item.get("score", 0.0)) or 0.0) +# item_copy["final_score"] = combined_score +# item_copy["reranker_model"] = model_name +# scored_items.append(item_copy) - # Sort all items by final_score in descending order - scored_items.sort(key=lambda x: float(x.get("final_score", 0.0) or 0.0), reverse=True) - reranked_results[category] = scored_items +# # Sort all items by final_score in descending order +# scored_items.sort(key=lambda x: float(x.get("final_score", 0.0) or 0.0), reverse=True) +# reranked_results[category] = scored_items - except Exception as e: - logger.error(f"Error in LLM reranking for category {category}: {e}, returning original results") - # Return original items with combined_score as final_score - for item in items: - combined_score = float(item.get("combined_score", item.get("score", 0.0)) or 0.0) - item["final_score"] = combined_score - item["reranker_model"] = model_name - reranked_results[category] = items +# except Exception as e: +# logger.error(f"Error in LLM reranking for category {category}: {e}, returning original results") +# # Return original items with combined_score as final_score +# for item in items: +# combined_score = float(item.get("combined_score", item.get("score", 0.0)) or 0.0) +# item["final_score"] = combined_score +# item["reranker_model"] = model_name +# reranked_results[category] = items - return reranked_results +# return reranked_results async def run_hybrid_search( @@ -556,7 +558,7 @@ async def run_hybrid_search( limit: int, include: List[str], output_path: str | None, - embedding_id: str, + memory_config: "MemoryConfig", rerank_alpha: float = 0.6, use_forgetting_rerank: bool = False, use_llm_rerank: bool = False, @@ -564,11 +566,14 @@ async def run_hybrid_search( """ Run search with specified type: 'keyword', 'embedding', or 'hybrid' + + Args: + memory_config: MemoryConfig object containing embedding_model_id and config_id """ # Start overall timing search_start_time = time.time() latency_metrics = {} - logger.info(f"using embedding_id:{embedding_id}...") + logger.info(f"using embedding_id:{memory_config.embedding_model_id}...") # Clean and normalize the incoming query before use/logging query_text = extract_plain_query(query_text) @@ -621,7 +626,9 @@ async def run_hybrid_search( # 从数据库读取嵌入器配置（按 ID）并构建 RedBearModelConfig config_load_start = time.time() - embedder_config_dict = get_embedder_config(embedding_id) + with get_db_context() as db: + config_service = MemoryConfigService(db) + embedder_config_dict = config_service.get_embedder_config(str(memory_config.embedding_model_id)) rb_config = RedBearModelConfig( model_name=embedder_config_dict["model_name"], provider=embedder_config_dict["provider"], @@ -683,7 +690,7 @@ async def run_hybrid_search( if use_forgetting_rerank: # Load forgetting parameters from pipeline config try: - pc = get_pipeline_config() + pc = get_pipeline_config(memory_config) forgetting_cfg = pc.forgetting_engine except Exception as e: logger.debug(f"Failed to load forgetting config, using defaults: {e}") @@ -711,16 +718,16 @@ async def run_hybrid_search( # Apply LLM reranking if enabled llm_rerank_applied = False - if use_llm_rerank: - try: - reranked_results = await apply_llm_reranker( - results=reranked_results, - query_text=query_text, - ) - llm_rerank_applied = True - logger.info("LLM reranking applied successfully") - except Exception as e: - logger.warning(f"LLM reranking failed: {e}, using previous scores") + # if use_llm_rerank: + # try: + # reranked_results = await apply_llm_reranker( + # results=reranked_results, + # query_text=query_text, + # ) + # llm_rerank_applied = True + # logger.info("LLM reranking applied successfully") + # except Exception as e: + # logger.warning(f"LLM reranking failed: {e}, using previous scores") results["reranked_results"] = reranked_results results["combined_summary"] = { @@ -896,90 +903,95 @@ async def search_chunk_by_chunk_id( return {"chunks": chunks} -def main(): - """Main entry point for the hybrid graph search CLI. +# def main(): +# """Main entry point for the hybrid graph search CLI. - Parses command line arguments and executes search with specified parameters. - Supports keyword, embedding, and hybrid search modes. - """ - parser = argparse.ArgumentParser(description="Hybrid graph search with keyword and embedding options") - parser.add_argument( - "--query", "-q", required=True, help="Free-text query to search" - ) - parser.add_argument( - "--search-type", - "-t", - choices=["keyword", "embedding", "hybrid"], - default="hybrid", - help="Search type: keyword (text matching), embedding (semantic), or hybrid (both) (default: hybrid)" - ) - parser.add_argument( - "--embedding-name", - "-m", - default="openai/nomic-embed-text:v1.5", - help="Embedding config name from config.json (default: openai/nomic-embed-text:v1.5)", - ) - parser.add_argument( - "--group-id", - "-g", - default=None, - help="Optional group_id to filter results (default: None)", - ) - parser.add_argument( - "--limit", - "-k", - type=int, - default=5, - help="Max number of results per type (default: 5)", - ) - parser.add_argument( - "--include", - "-i", - nargs="+", - default=["statements", "chunks", "entities", "summaries"], - choices=["statements", "chunks", "entities", "summaries"], - help="Which targets to search for embedding search (default: statements chunks entities summaries)" - ) - parser.add_argument( - "--output", - "-o", - default="search_results.json", - help="Path to save the search results JSON (default: search_results.json)", - ) - parser.add_argument( - "--rerank-alpha", - "-a", - type=float, - default=0.6, - help="Weight for BM25 scores in reranking (0.0-1.0, higher values favor keyword search) (default: 0.6)", - ) - parser.add_argument( - "--forgetting-rerank", - action="store_true", - help="Apply forgetting curve during reranking for hybrid search.", - ) - parser.add_argument( - "--llm-rerank", - action="store_true", - help="Apply LLM-based reranking for hybrid search.", - ) - args = parser.parse_args() +# Parses command line arguments and executes search with specified parameters. +# Supports keyword, embedding, and hybrid search modes. +# """ +# parser = argparse.ArgumentParser(description="Hybrid graph search with keyword and embedding options") +# parser.add_argument( +# "--query", "-q", required=True, help="Free-text query to search" +# ) +# parser.add_argument( +# "--search-type", +# "-t", +# choices=["keyword", "embedding", "hybrid"], +# default="hybrid", +# help="Search type: keyword (text matching), embedding (semantic), or hybrid (both) (default: hybrid)" +# ) +# parser.add_argument( +# "--config-id", +# "-c", +# type=int, +# required=True, +# help="Database configuration ID (required)", +# ) +# parser.add_argument( +# "--group-id", +# "-g", +# default=None, +# help="Optional group_id to filter results (default: None)", +# ) +# parser.add_argument( +# "--limit", +# "-k", +# type=int, +# default=5, +# help="Max number of results per type (default: 5)", +# ) +# parser.add_argument( +# "--include", +# "-i", +# nargs="+", +# default=["statements", "chunks", "entities", "summaries"], +# choices=["statements", "chunks", "entities", "summaries"], +# help="Which targets to search for embedding search (default: statements chunks entities summaries)" +# ) +# parser.add_argument( +# "--output", +# "-o", +# default="search_results.json", +# help="Path to save the search results JSON (default: search_results.json)", +# ) +# parser.add_argument( +# "--rerank-alpha", +# "-a", +# type=float, +# default=0.6, +# help="Weight for BM25 scores in reranking (0.0-1.0, higher values favor keyword search) (default: 0.6)", +# ) +# parser.add_argument( +# "--forgetting-rerank", +# action="store_true", +# help="Apply forgetting curve during reranking for hybrid search.", +# ) +# parser.add_argument( +# "--llm-rerank", +# action="store_true", +# help="Apply LLM-based reranking for hybrid search.", +# ) +# args = parser.parse_args() - asyncio.run( - run_hybrid_search( - query_text=args.query, - search_type=args.search_type, - group_id=args.group_id, - limit=args.limit, - include=args.include, - output_path=args.output, - embedding_id=config_defs.SELECTED_EMBEDDING_ID, - rerank_alpha=args.rerank_alpha, - use_forgetting_rerank=args.forgetting_rerank, - use_llm_rerank=args.llm_rerank, - ) - ) +# # Load memory config from database +# from app.services.memory_config_service import MemoryConfigService +# memory_config = MemoryConfigService.load_memory_config(args.config_id) + +# asyncio.run( +# run_hybrid_search( +# query_text=args.query, +# search_type=args.search_type, +# group_id=args.group_id, +# limit=args.limit, +# include=args.include, +# output_path=args.output, +# memory_config=memory_config, +# rerank_alpha=args.rerank_alpha, +# use_forgetting_rerank=args.forgetting_rerank, +# use_llm_rerank=args.llm_rerank, +# ) +# ) -if __name__ == "__main__": - main() +# if __name__ == "__main__": +# main() diff --git a/api/app/core/memory/storage_services/extraction_engine/deduplication/deduped_and_disamb.py b/api/app/core/memory/storage_services/extraction_engine/deduplication/deduped_and_disamb.py index 9088a300..62b656b0 100644 --- a/api/app/core/memory/storage_services/extraction_engine/deduplication/deduped_and_disamb.py +++ b/api/app/core/memory/storage_services/extraction_engine/deduplication/deduped_and_disamb.py @@ -1,19 +1,22 @@ """ 去重功能函数 """ -from app.core.memory.models.variate_config import DedupConfig -from typing import List, Dict, Tuple, Any -from app.core.memory.models.graph_models import( - StatementEntityEdge, - EntityEntityEdge, - ExtractedEntityNode -) -import os -from datetime import datetime -import difflib # 提供字符串相似度计算工具 import asyncio +import difflib # 提供字符串相似度计算工具 import importlib +import os import re +from datetime import datetime +from typing import Any, Dict, List, Tuple + +from app.core.memory.models.graph_models import ( + EntityEntityEdge, + ExtractedEntityNode, + StatementEntityEdge, +) +from app.core.memory.models.variate_config import DedupConfig + + # 模块级类型统一工具函数 def _unify_entity_type(canonical: ExtractedEntityNode, losing: ExtractedEntityNode, suggested_type: str = None) -> None: """统一实体类型：基于LLM建议或启发式规则选择最合适的类型。 @@ -705,7 +708,8 @@ async def LLM_decision( # 决策中包含去重和消歧的功能 statement_entity_edges: List[StatementEntityEdge], entity_entity_edges: List[EntityEntityEdge], id_redirect: Dict[str, str], - config: DedupConfig | None = None, + config: DedupConfig, + llm_client = None, ) -> Tuple[List[ExtractedEntityNode], Dict[str, str], List[str]]: """ 基于迭代分块并发的 LLM 判定，生成实体重定向并在本地应用融合。 @@ -717,26 +721,13 @@ async def LLM_decision( # 决策中包含去重和消歧的功能 """ llm_records: List[str] = [] try: - # 优先使用运行时配置；若未提供配置，使用模型默认值，不再回退到环境变量 - enable_switch = ( - bool(config.enable_llm_dedup_blockwise) if config is not None else DedupConfig().enable_llm_dedup_blockwise - ) - if not enable_switch: - return deduped_entities, id_redirect, llm_records - # 从配置读取 LLM 迭代参数；若无配置则使用 DedupConfig 的默认值 - _defaults = DedupConfig() - block_size = (config.llm_block_size if config is not None else _defaults.llm_block_size) - block_concurrency = (config.llm_block_concurrency if config is not None else _defaults.llm_block_concurrency) - pair_concurrency = (config.llm_pair_concurrency if config is not None else _defaults.llm_pair_concurrency) - max_rounds = (config.llm_max_rounds if config is not None else _defaults.llm_max_rounds) - - # 动态导入 llm 客户端（修正导入路径） - try: - llm_utils_mod = importlib.import_module("app.core.memory.utils.llm.llm_utils") - get_llm_client_fn = llm_utils_mod.get_llm_client - except Exception as e: - llm_records.append(f"[LLM错误] 无法导入 llm_utils 模块: {e}") + if not bool(config.enable_llm_dedup_blockwise): return deduped_entities, id_redirect, llm_records + # 从配置读取 LLM 迭代参数 + block_size = config.llm_block_size + block_concurrency = config.llm_block_concurrency + pair_concurrency = config.llm_pair_concurrency + max_rounds = config.llm_max_rounds try: llm_mod = importlib.import_module("app.core.memory.storage_services.extraction_engine.deduplication.entity_dedup_llm") @@ -745,14 +736,9 @@ async def LLM_decision( # 决策中包含去重和消歧的功能 llm_records.append(f"[LLM错误] 无法导入 entity_dedup_llm 模块: {e}") return deduped_entities, id_redirect, llm_records - # 获取 LLM 客户端 - try: - llm_client = get_llm_client_fn() - if llm_client is None: - llm_records.append("[LLM错误] LLM 客户端初始化失败：返回 None") - return deduped_entities, id_redirect, llm_records - except Exception as e: - llm_records.append(f"[LLM错误] 获取 LLM 客户端失败: {e}") + # 验证 LLM 客户端 + if llm_client is None: + llm_records.append("[LLM错误] LLM 客户端未提供") return deduped_entities, id_redirect, llm_records llm_redirect, llm_records = await llm_fn( @@ -813,7 +799,8 @@ async def LLM_disamb_decision( statement_entity_edges: List[StatementEntityEdge], entity_entity_edges: List[EntityEntityEdge], id_redirect: Dict[str, str], - config: DedupConfig | None = None, + config: DedupConfig, + llm_client = None, ) -> Tuple[List[ExtractedEntityNode], Dict[str, str], set[tuple[str, str]], List[str]]: """ 预消歧阶段：对“同名但类型不同”的实体对调用LLM进行消歧， @@ -824,22 +811,16 @@ async def LLM_disamb_decision( disamb_records: List[str] = [] blocked_pairs: set[tuple[str, str]] = set() try: - enable_switch = ( - config.enable_llm_disambiguation - if config is not None - else DedupConfig().enable_llm_disambiguation - ) - if not bool(enable_switch): + if not bool(config.enable_llm_disambiguation): return deduped_entities, id_redirect, blocked_pairs, disamb_records - from app.core.memory.utils.llm.llm_utils import get_llm_client - from app.core.memory.storage_services.extraction_engine.deduplication.entity_dedup_llm import llm_disambiguate_pairs_iterative - from app.core.memory.utils.config import definitions as config_defs + from app.core.memory.storage_services.extraction_engine.deduplication.entity_dedup_llm import ( + llm_disambiguate_pairs_iterative, + ) - # 获取 LLM 客户端并验证 - llm_client = get_llm_client(config_defs.SELECTED_LLM_ID) + # 验证 LLM 客户端 if llm_client is None: - disamb_records.append("[DISAMB错误] LLM 客户端初始化失败：返回 None") + disamb_records.append("[DISAMB错误] LLM 客户端未提供") return deduped_entities, id_redirect, blocked_pairs, disamb_records merge_redirect, block_list, disamb_records = await llm_disambiguate_pairs_iterative( @@ -895,6 +876,7 @@ async def deduplicate_entities_and_edges( report_append: bool = False, report_stage_notes: List[str] | None = None, dedup_config: DedupConfig | None = None, + llm_client = None, ) -> Tuple[ List[ExtractedEntityNode], List[StatementEntityEdge], @@ -911,7 +893,7 @@ async def deduplicate_entities_and_edges( # 1.5) LLM 决策消歧：阻断同名不同类型的高相似对，并应用必要的合并 deduped_entities, id_redirect, blocked_pairs, disamb_records = await LLM_disamb_decision( - deduped_entities, statement_entity_edges, entity_entity_edges, id_redirect, config=dedup_config + deduped_entities, statement_entity_edges, entity_entity_edges, id_redirect, config=dedup_config, llm_client=llm_client ) # 2) 模糊匹配（本地规则） @@ -936,7 +918,7 @@ async def deduplicate_entities_and_edges( if should_trigger_llm: deduped_entities, id_redirect, llm_decision_records = await LLM_decision( - deduped_entities, statement_entity_edges, entity_entity_edges, id_redirect, config=dedup_config + deduped_entities, statement_entity_edges, entity_entity_edges, id_redirect, config=dedup_config, llm_client=llm_client ) else: llm_decision_records = [] diff --git a/api/app/core/memory/storage_services/extraction_engine/deduplication/second_layer_dedup.py b/api/app/core/memory/storage_services/extraction_engine/deduplication/second_layer_dedup.py index 04aa6cb6..b41f35a4 100644 --- a/api/app/core/memory/storage_services/extraction_engine/deduplication/second_layer_dedup.py +++ b/api/app/core/memory/storage_services/extraction_engine/deduplication/second_layer_dedup.py @@ -10,15 +10,27 @@ from __future__ import annotations -from typing import List, Dict, Any, Tuple from datetime import datetime +from typing import Any, Dict, List, Tuple + +from app.core.memory.models.graph_models import ( + EntityEntityEdge, + ExtractedEntityNode, + StatementEntityEdge, +) +from app.core.memory.models.variate_config import DedupConfig +from app.core.memory.storage_services.extraction_engine.deduplication.deduped_and_disamb import ( # 导入报告写入以在跳过时追加说明 + _write_dedup_fusion_report, + deduplicate_entities_and_edges, +) +from app.repositories.neo4j.graph_search import ( + get_dedup_candidates_for_entities, # 导入ge函数，用于从 Neo4j 中检索与输入实体可能重复的候选实体（去重的核心检索逻辑）。 +) # 使用新的仓储层 -from app.repositories.neo4j.neo4j_connector import Neo4jConnector # 导入 Neo4j 数据库连接器类，用于与 Neo4j 数据库进行交互 -from app.repositories.neo4j.graph_search import get_dedup_candidates_for_entities # 导入ge函数，用于从 Neo4j 中检索与输入实体可能重复的候选实体（去重的核心检索逻辑）。 -from app.core.memory.models.graph_models import ExtractedEntityNode, StatementEntityEdge, EntityEntityEdge -from app.core.memory.storage_services.extraction_engine.deduplication.deduped_and_disamb import deduplicate_entities_and_edges, _write_dedup_fusion_report # 导入报告写入以在跳过时追加说明 -from app.core.memory.models.variate_config import DedupConfig +from app.repositories.neo4j.neo4j_connector import ( + Neo4jConnector, # 导入 Neo4j 数据库连接器类，用于与 Neo4j 数据库进行交互 +) def _parse_dt(val: Any) -> datetime: # 定义内部辅助函数_parse_dt，用于将任意类型的输入值解析为datetime对象（处理实体节点中的时间字段） @@ -72,6 +84,7 @@ async def second_layer_dedup_and_merge_with_neo4j( # 二层去重的核心逻辑 statement_entity_edges: List[StatementEntityEdge], # 输入的语句实体边列表，用于处理实体之间的关系 entity_entity_edges: List[EntityEntityEdge], # 输入的实体实体边列表，用于处理实体之间的关系 dedup_config: DedupConfig | None = None, + llm_client = None, ) -> Tuple[List[ExtractedEntityNode], List[StatementEntityEdge], List[EntityEntityEdge]]: """ 第二层去重消歧： @@ -137,13 +150,14 @@ async def second_layer_dedup_and_merge_with_neo4j( # 二层去重的核心逻辑 union_entities: List[ExtractedEntityNode] = db_candidate_models + list(entity_nodes) # 融合（内部执行精确/模糊/LLM 决策；随后再做边重定向与去重） - fused_entities, fused_stmt_entity_edges, fused_entity_entity_edges = await deduplicate_entities_and_edges( + fused_entities, fused_stmt_entity_edges, fused_entity_entity_edges, _ = await deduplicate_entities_and_edges( union_entities, statement_entity_edges, entity_entity_edges, report_stage="第二层去重消歧", report_append=True, dedup_config=dedup_config, + llm_client=llm_client, ) return fused_entities, fused_stmt_entity_edges, fused_entity_entity_edges diff --git a/api/app/core/memory/storage_services/extraction_engine/deduplication/two_stage_dedup.py b/api/app/core/memory/storage_services/extraction_engine/deduplication/two_stage_dedup.py index e4857ff3..11845d7d 100644 --- a/api/app/core/memory/storage_services/extraction_engine/deduplication/two_stage_dedup.py +++ b/api/app/core/memory/storage_services/extraction_engine/deduplication/two_stage_dedup.py @@ -1,23 +1,27 @@ from __future__ import annotations -from typing import List, Tuple, Optional +from typing import List, Optional, Tuple -from app.core.memory.models.variate_config import ExtractionPipelineConfig -from app.core.memory.utils.config.config_utils import get_pipeline_config -from app.core.memory.storage_services.extraction_engine.deduplication.deduped_and_disamb import deduplicate_entities_and_edges -from app.core.memory.storage_services.extraction_engine.deduplication.second_layer_dedup import second_layer_dedup_and_merge_with_neo4j -# 使用新的仓储层 -from app.repositories.neo4j.neo4j_connector import Neo4jConnector from app.core.memory.models.graph_models import ( - DialogueNode, ChunkNode, - StatementNode, + DialogueNode, + EntityEntityEdge, ExtractedEntityNode, StatementChunkEdge, StatementEntityEdge, - EntityEntityEdge, + StatementNode, ) from app.core.memory.models.message_models import DialogData +from app.core.memory.models.variate_config import ExtractionPipelineConfig +from app.core.memory.storage_services.extraction_engine.deduplication.deduped_and_disamb import ( + deduplicate_entities_and_edges, +) +from app.core.memory.storage_services.extraction_engine.deduplication.second_layer_dedup import ( + second_layer_dedup_and_merge_with_neo4j, +) + +# 使用新的仓储层 +from app.repositories.neo4j.neo4j_connector import Neo4jConnector async def dedup_layers_and_merge_and_return( @@ -29,8 +33,9 @@ async def dedup_layers_and_merge_and_return( statement_entity_edges: List[StatementEntityEdge], entity_entity_edges: List[EntityEntityEdge], dialog_data_list: List[DialogData], - pipeline_config: Optional[ExtractionPipelineConfig] = None, + pipeline_config: ExtractionPipelineConfig, connector: Optional[Neo4jConnector] = None, + llm_client = None, ) -> Tuple[ List[DialogueNode], List[ChunkNode], @@ -48,12 +53,9 @@ async def dedup_layers_and_merge_and_return( 返回融合后的实体与边，同时保留原始的对话、片段与语句节点与边。 """ - # 默认从 runtime.json 加载管线配置，避免回退到环境变量 + # pipeline_config is required - caller must provide it if pipeline_config is None: - try: - pipeline_config = get_pipeline_config() - except Exception: - pipeline_config = None + raise ValueError("pipeline_config is required for dedup_layers_and_merge_and_return") # 先探测 group_id，决定报告写入策略 group_id: Optional[str] = None @@ -70,6 +72,7 @@ async def dedup_layers_and_merge_and_return( report_stage="第一层去重消歧", report_append=False, dedup_config=(pipeline_config.deduplication if pipeline_config else None), + llm_client=llm_client, ) # 初始化第二层融合结果为第一层结果 @@ -88,6 +91,7 @@ async def dedup_layers_and_merge_and_return( statement_entity_edges=dedup_statement_entity_edges, entity_entity_edges=dedup_entity_entity_edges, dedup_config=(pipeline_config.deduplication if pipeline_config else None), + llm_client=llm_client, ) else: print("Skip second-layer dedup: missing connector") diff --git a/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py b/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py index fa079c97..7c2ed5f4 100644 --- a/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py +++ b/api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py @@ -1253,6 +1253,7 @@ class ExtractionOrchestrator: report_stage="第一层去重消歧（试运行）", report_append=False, dedup_config=self.config.deduplication, + llm_client=self.llm_client, ) # 保存去重消歧的详细记录到实例变量 @@ -1284,6 +1285,7 @@ class ExtractionOrchestrator: dialog_data_list, self.config, self.connector, + llm_client=self.llm_client, ) # 解包返回值 diff --git a/api/app/core/memory/storage_services/extraction_engine/knowledge_extraction/embedding_generation.py b/api/app/core/memory/storage_services/extraction_engine/knowledge_extraction/embedding_generation.py index 396c1e9e..72f3641e 100644 --- a/api/app/core/memory/storage_services/extraction_engine/knowledge_extraction/embedding_generation.py +++ b/api/app/core/memory/storage_services/extraction_engine/knowledge_extraction/embedding_generation.py @@ -5,11 +5,13 @@ """ import asyncio -from typing import List, Dict, Any, Tuple -from app.core.memory.models.message_models import DialogData +from typing import Any, Dict, List, Tuple + from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient -from app.core.memory.utils.config.config_utils import get_embedder_config +from app.core.memory.models.message_models import DialogData from app.core.models.base import RedBearModelConfig +from app.db import get_db_context +from app.services.memory_config_service import MemoryConfigService class EmbeddingGenerator: @@ -21,7 +23,9 @@ class EmbeddingGenerator: Args: embedding_id: 嵌入模型 ID """ - embedder_config = get_embedder_config(embedding_id) + with get_db_context() as db: + config_service = MemoryConfigService(db) + embedder_config = config_service.get_embedder_config(embedding_id) self.embedder_client = OpenAIEmbedderClient( model_config=RedBearModelConfig.model_validate(embedder_config), ) diff --git a/api/app/core/memory/storage_services/extraction_engine/knowledge_extraction/memory_summary.py b/api/app/core/memory/storage_services/extraction_engine/knowledge_extraction/memory_summary.py index ffd4ed12..70c1ceb3 100644 --- a/api/app/core/memory/storage_services/extraction_engine/knowledge_extraction/memory_summary.py +++ b/api/app/core/memory/storage_services/extraction_engine/knowledge_extraction/memory_summary.py @@ -1,21 +1,17 @@ -import os import asyncio from datetime import datetime from typing import List, Optional - -from pydantic import Field, field_validator +from uuid import uuid4 from app.core.logging_config import get_memory_logger +from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient +from app.core.memory.models.base_response import RobustLLMResponse +from app.core.memory.models.graph_models import MemorySummaryNode from app.core.memory.models.message_models import DialogData +from app.core.memory.utils.prompt.prompt_utils import render_memory_summary_prompt +from pydantic import Field logger = get_memory_logger(__name__) -from app.core.memory.models.graph_models import MemorySummaryNode -from app.core.memory.models.base_response import RobustLLMResponse -from app.core.models.base import RedBearModelConfig -from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient -from app.core.memory.utils.config.config_utils import get_embedder_config -from app.core.memory.utils.prompt.prompt_utils import render_memory_summary_prompt -from uuid import uuid4 class MemorySummaryResponse(RobustLLMResponse): @@ -91,22 +87,17 @@ async def _process_chunk_summary( return None -async def Memory_summary_generation( +async def memory_summary_generation( chunked_dialogs: List[DialogData], llm_client, - embedding_id, + embedder_client: OpenAIEmbedderClient, ) -> List[MemorySummaryNode]: """Generate memory summaries per chunk, embed them, and return nodes.""" - embedder_cfg_dict = get_embedder_config(embedding_id) - embedder = OpenAIEmbedderClient( - model_config=RedBearModelConfig.model_validate(embedder_cfg_dict), - ) - # Collect all tasks for parallel processing tasks = [] for dialog in chunked_dialogs: for chunk in dialog.chunks: - tasks.append(_process_chunk_summary(dialog, chunk, llm_client, embedder)) + tasks.append(_process_chunk_summary(dialog, chunk, llm_client, embedder_client)) # Process all chunks in parallel results = await asyncio.gather(*tasks, return_exceptions=False) diff --git a/api/app/core/memory/storage_services/extraction_engine/knowledge_extraction/statement_extraction.py b/api/app/core/memory/storage_services/extraction_engine/knowledge_extraction/statement_extraction.py index 1e79c339..17f76b17 100644 --- a/api/app/core/memory/storage_services/extraction_engine/knowledge_extraction/statement_extraction.py +++ b/api/app/core/memory/storage_services/extraction_engine/knowledge_extraction/statement_extraction.py @@ -1,17 +1,21 @@ -import os import asyncio import logging -from typing import List, Optional, Dict, Any -from pydantic import BaseModel, Field +import os from datetime import datetime +from typing import Any, Dict, List, Optional from app.core.memory.models.message_models import DialogData, Statement -#避免在测试收集阶段因为 OpenAIClient 间接引入 langfuse 导致 ModuleNotFoundError 。这只是类型注解与导入时机的调整，不改变实现。 -from app.core.memory.utils.data.ontology import LABEL_DEFINITIONS, StatementType, TemporalInfo +#避免在测试收集阶段因为 OpenAIClient 间接引入 langfuse 导致 ModuleNotFoundError 。这只是类型注解与导入时机的调整，不改变实现。 from app.core.memory.models.variate_config import StatementExtractionConfig +from app.core.memory.utils.data.ontology import ( + LABEL_DEFINITIONS, + RelevenceInfo, + StatementType, + TemporalInfo, +) from app.core.memory.utils.prompt.prompt_utils import render_statement_extraction_prompt -from app.core.memory.utils.data.ontology import LABEL_DEFINITIONS, StatementType, TemporalInfo, RelevenceInfo +from pydantic import BaseModel, Field logger = logging.getLogger(__name__) diff --git a/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py b/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py index 6ccec500..6f537916 100644 --- a/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py +++ b/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py @@ -8,21 +8,25 @@ 4. 反思结果应用 - 更新记忆库 """ +import asyncio import json import logging -import asyncio import os import time -from typing import List, Dict, Any, Optional -from enum import Enum import uuid - -from pydantic import BaseModel +from enum import Enum +from typing import Any, Dict, List, Optional from app.core.response_utils import success -from app.repositories.neo4j.cypher_queries import neo4j_query_part, neo4j_statement_part, neo4j_query_all, neo4j_statement_all -from app.repositories.neo4j.neo4j_update import neo4j_data +from app.repositories.neo4j.cypher_queries import ( + neo4j_query_all, + neo4j_query_part, + neo4j_statement_all, + neo4j_statement_part, +) from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.repositories.neo4j.neo4j_update import neo4j_data +from pydantic import BaseModel # 配置日志 _root_logger = logging.getLogger() @@ -135,14 +139,20 @@ class ReflectionEngine: self.neo4j_connector = Neo4jConnector() if self.llm_client is None: - from app.core.memory.utils.llm.llm_utils import get_llm_client from app.core.memory.utils.config import definitions as config_defs - self.llm_client = get_llm_client(config_defs.SELECTED_LLM_ID) + from app.core.memory.utils.llm.llm_utils import MemoryClientFactory + from app.db import get_db_context + with get_db_context() as db: + factory = MemoryClientFactory(db) + self.llm_client = factory.get_llm_client(config_defs.SELECTED_LLM_ID) elif isinstance(self.llm_client, str): # 如果 llm_client 是字符串（model_id），则用它初始化客户端 - from app.core.memory.utils.llm.llm_utils import get_llm_client + from app.core.memory.utils.llm.llm_utils import MemoryClientFactory + from app.db import get_db_context model_id = self.llm_client - self.llm_client = get_llm_client(model_id) + with get_db_context() as db: + factory = MemoryClientFactory(db) + self.llm_client = factory.get_llm_client(model_id) if self.get_data_func is None: from app.core.memory.utils.config.get_data import get_data @@ -154,11 +164,15 @@ class ReflectionEngine: self.get_data_statement = get_data_statement if self.render_evaluate_prompt_func is None: - from app.core.memory.utils.prompt.template_render import render_evaluate_prompt + from app.core.memory.utils.prompt.template_render import ( + render_evaluate_prompt, + ) self.render_evaluate_prompt_func = render_evaluate_prompt if self.render_reflexion_prompt_func is None: - from app.core.memory.utils.prompt.template_render import render_reflexion_prompt + from app.core.memory.utils.prompt.template_render import ( + render_reflexion_prompt, + ) self.render_reflexion_prompt_func = render_reflexion_prompt if self.conflict_schema is None: @@ -170,7 +184,9 @@ class ReflectionEngine: self.reflexion_schema = ReflexionResultSchema if self.update_query is None: - from app.repositories.neo4j.cypher_queries import UPDATE_STATEMENT_INVALID_AT + from app.repositories.neo4j.cypher_queries import ( + UPDATE_STATEMENT_INVALID_AT, + ) self.update_query = UPDATE_STATEMENT_INVALID_AT self._lazy_init_done = True diff --git a/api/app/core/memory/storage_services/search/__init__.py b/api/app/core/memory/storage_services/search/__init__.py index 04a7a4c2..2bec5bf1 100644 --- a/api/app/core/memory/storage_services/search/__init__.py +++ b/api/app/core/memory/storage_services/search/__init__.py @@ -4,10 +4,20 @@ 本模块提供统一的搜索服务接口，支持关键词搜索、语义搜索和混合搜索。 """ -from app.core.memory.storage_services.search.search_strategy import SearchStrategy, SearchResult -from app.core.memory.storage_services.search.keyword_search import KeywordSearchStrategy -from app.core.memory.storage_services.search.semantic_search import SemanticSearchStrategy +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from app.schemas.memory_config_schema import MemoryConfig + from app.core.memory.storage_services.search.hybrid_search import HybridSearchStrategy +from app.core.memory.storage_services.search.keyword_search import KeywordSearchStrategy +from app.core.memory.storage_services.search.search_strategy import ( + SearchResult, + SearchStrategy, +) +from app.core.memory.storage_services.search.semantic_search import ( + SemanticSearchStrategy, +) __all__ = [ "SearchStrategy", @@ -34,7 +44,7 @@ async def run_hybrid_search( include: list[str] | None = None, alpha: float = 0.6, use_forgetting_curve: bool = False, - embedding_id: str | None = None, + memory_config: "MemoryConfig" = None, **kwargs ) -> dict: """运行混合搜索（向后兼容的函数式API） @@ -51,24 +61,26 @@ async def run_hybrid_search( include: 要包含的搜索类别列表 alpha: BM25分数权重（0.0-1.0） use_forgetting_curve: 是否使用遗忘曲线 - embedding_id: 嵌入模型ID + memory_config: MemoryConfig object containing embedding_model_id **kwargs: 其他参数 Returns: dict: 搜索结果字典，格式与旧API兼容 """ - from app.repositories.neo4j.neo4j_connector import Neo4jConnector from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient - from app.core.memory.utils.config.config_utils import get_embedder_config - from app.core.memory.utils.config import definitions as config_defs from app.core.models.base import RedBearModelConfig + from app.db import get_db_context + from app.repositories.neo4j.neo4j_connector import Neo4jConnector + from app.services.memory_config_service import MemoryConfigService - # 使用提供的embedding_id或默认值 - emb_id = embedding_id or config_defs.SELECTED_EMBEDDING_ID + if not memory_config: + raise ValueError("memory_config is required for search") # 初始化客户端 connector = Neo4jConnector() - embedder_config_dict = get_embedder_config(emb_id) + with get_db_context() as db: + config_service = MemoryConfigService(db) + embedder_config_dict = config_service.get_embedder_config(str(memory_config.embedding_model_id)) embedder_config = RedBearModelConfig(**embedder_config_dict) embedder_client = OpenAIEmbedderClient(embedder_config) diff --git a/api/app/core/memory/storage_services/search/semantic_search.py b/api/app/core/memory/storage_services/search/semantic_search.py index 363ff1aa..b20f90a5 100644 --- a/api/app/core/memory/storage_services/search/semantic_search.py +++ b/api/app/core/memory/storage_services/search/semantic_search.py @@ -5,15 +5,20 @@ 使用余弦相似度进行语义匹配。 """ -from typing import List, Dict, Any, Optional +from typing import Any, Dict, List, Optional + from app.core.logging_config import get_memory_logger -from app.repositories.neo4j.neo4j_connector import Neo4jConnector -from app.core.memory.storage_services.search.search_strategy import SearchStrategy, SearchResult -from app.repositories.neo4j.graph_search import search_graph_by_embedding from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient -from app.core.memory.utils.config.config_utils import get_embedder_config +from app.core.memory.storage_services.search.search_strategy import ( + SearchResult, + SearchStrategy, +) from app.core.memory.utils.config import definitions as config_defs from app.core.models.base import RedBearModelConfig +from app.db import get_db_context +from app.repositories.neo4j.graph_search import search_graph_by_embedding +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.services.memory_config_service import MemoryConfigService logger = get_memory_logger(__name__) @@ -62,7 +67,9 @@ class SemanticSearchStrategy(SearchStrategy): """ try: # 从数据库读取嵌入器配置 - embedder_config_dict = get_embedder_config(config_defs.SELECTED_EMBEDDING_ID) + with get_db_context() as db: + config_service = MemoryConfigService(db) + embedder_config_dict = config_service.get_embedder_config(config_defs.SELECTED_EMBEDDING_ID) rb_config = RedBearModelConfig( model_name=embedder_config_dict["model_name"], provider=embedder_config_dict["provider"], diff --git a/api/app/core/memory/utils/config/__init__.py b/api/app/core/memory/utils/config/__init__.py index c2a8c6ca..f69c13a2 100644 --- a/api/app/core/memory/utils/config/__init__.py +++ b/api/app/core/memory/utils/config/__init__.py @@ -9,7 +9,6 @@ from .config_utils import ( get_chunker_config, get_embedder_config, get_model_config, - get_neo4j_config, get_picture_config, get_pipeline_config, get_pruning_config, @@ -41,7 +40,6 @@ __all__ = [ # config_utils "get_model_config", "get_embedder_config", - "get_neo4j_config", "get_chunker_config", "get_pipeline_config", "get_pruning_config", diff --git a/api/app/core/memory/utils/config/config_utils.py b/api/app/core/memory/utils/config/config_utils.py index b05e176c..7edb2a09 100644 --- a/api/app/core/memory/utils/config/config_utils.py +++ b/api/app/core/memory/utils/config/config_utils.py @@ -1,90 +1,74 @@ -from app.core.memory.models.variate_config import ( - DedupConfig, - ExtractionPipelineConfig, - ForgettingEngineConfig, - StatementExtractionConfig, -) -from app.core.memory.utils.config.definitions import CONFIG -from app.db import get_db -from app.models.models_model import ModelApiKey -from app.services.model_service import ModelConfigService -from fastapi import status -from fastapi.exceptions import HTTPException -from sqlalchemy.orm import Session +""" +Configuration utilities - Backward compatibility layer + +DEPRECATED: These functions now require a db session parameter. +New code should use MemoryConfigService(db) instance directly. + +For functions that don't require db (get_pipeline_config, get_pruning_config), +they are still re-exported here. +""" + +import warnings + +from app.services.memory_config_service import MemoryConfigService + +# These functions don't require db - safe to re-export as static methods +get_pipeline_config = MemoryConfigService.get_pipeline_config +get_pruning_config = MemoryConfigService.get_pruning_config -def get_model_config(model_id: str, db: Session | None = None) -> dict: +def get_model_config(model_id: str, db=None): + """DEPRECATED: Use MemoryConfigService(db).get_model_config(model_id) directly.""" if db is None: - db_gen = get_db() # get_db 通常是一个生成器 - db = next(db_gen) # 取到真正的 Session + raise ValueError( + "get_model_config now requires a db session. " + "Use MemoryConfigService(db).get_model_config(model_id) directly." + ) + return MemoryConfigService(db).get_model_config(model_id) - config = ModelConfigService.get_model_by_id(db=db, model_id=model_id) - if not config: - print(f"模型ID {model_id} 不存在") - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="模型ID不存在") - apiConfig: ModelApiKey = config.api_keys[0] - - # 从环境变量读取超时和重试配置 - from app.core.config import settings - - model_config = { - "model_name": apiConfig.model_name, - "provider": apiConfig.provider, - "api_key": apiConfig.api_key, - "base_url": apiConfig.api_base, - "model_config_id":apiConfig.model_config_id, - "type": config.type, - # 添加超时和重试配置，避免 LLM 请求超时 - "timeout": settings.LLM_TIMEOUT, # 从环境变量读取，默认120秒 - "max_retries": settings.LLM_MAX_RETRIES, # 从环境变量读取，默认2次 - } - # 写入model_config.log文件中 - with open("logs/model_config.log", "a", encoding="utf-8") as f: - f.write(f"模型ID: {model_id}\n") - f.write(f"模型配置信息:\n{model_config}\n") - f.write("=============================\n\n") - return model_config -def get_embedder_config(embedding_id: str, db: Session | None = None) -> dict: +def get_embedder_config(embedding_id: str, db=None): + """DEPRECATED: Use MemoryConfigService(db).get_embedder_config(embedding_id) directly.""" if db is None: - db_gen = get_db() # get_db 通常是一个生成器 - db = next(db_gen) # 取到真正的 Session + raise ValueError( + "get_embedder_config now requires a db session. " + "Use MemoryConfigService(db).get_embedder_config(embedding_id) directly." + ) + return MemoryConfigService(db).get_embedder_config(embedding_id) - config = ModelConfigService.get_model_by_id(db=db, model_id=embedding_id) - if not config: - print(f"嵌入模型ID {embedding_id} 不存在") - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="嵌入模型ID不存在") - apiConfig: ModelApiKey = config.api_keys[0] - model_config = { - "model_name": apiConfig.model_name, - "provider": apiConfig.provider, - "api_key": apiConfig.api_key, - "base_url": apiConfig.api_base, - "model_config_id":apiConfig.model_config_id, - # Ensure required field for RedBearModelConfig validation - "type": config.type, - # 添加超时和重试配置，避免嵌入服务请求超时 - "timeout": 120.0, # 嵌入服务超时时间（秒） - "max_retries": 5, # 最大重试次数 - } - # 写入embedder_config.log文件中 - with open("logs/embedder_config.log", "a", encoding="utf-8") as f: - f.write(f"嵌入模型ID: {embedding_id}\n") - f.write(f"嵌入模型配置信息:\n{model_config}\n") - f.write("=============================\n\n") - return model_config -def get_neo4j_config() -> dict: - """Retrieves the Neo4j configuration from the config file.""" - return CONFIG.get("neo4j", {}) def get_picture_config(llm_name: str) -> dict: - """Retrieves the configuration for a specific model from the config file.""" + """Retrieves the configuration for a specific model from the config file. + + .. deprecated:: + This function is deprecated and will be removed in a future version. + Use database-backed model configuration instead. + """ + warnings.warn( + "get_picture_config is deprecated and will be removed in a future version. " + "Use database-backed model configuration instead.", + DeprecationWarning, + stacklevel=2 + ) for model_config in CONFIG.get("picture_recognition", []): if model_config["llm_name"] == llm_name: return model_config raise ValueError(f"Model '{llm_name}' not found in config.json") + + def get_voice_config(llm_name: str) -> dict: - """Retrieves the configuration for a specific model from the config file.""" + """Retrieves the configuration for a specific model from the config file. + + .. deprecated:: + This function is deprecated and will be removed in a future version. + Use database-backed model configuration instead. + """ + warnings.warn( + "get_voice_config is deprecated and will be removed in a future version. " + "Use database-backed model configuration instead.", + DeprecationWarning, + stacklevel=2 + ) for model_config in CONFIG.get("voice_recognition", []): if model_config["llm_name"] == llm_name: return model_config @@ -92,19 +76,8 @@ def get_voice_config(llm_name: str) -> dict: def get_chunker_config(chunker_strategy: str) -> dict: - """Retrieves the configuration for a specific chunker strategy. + """Retrieves the configuration for a specific chunker strategy.""" - Enhancements: - - Supports default configs for `LLMChunker` and `HybridChunker` if not present. - - Falls back to the first available chunker config when the requested one is missing. - """ - # 1) Try to find exact match in config - chunker_list = CONFIG.get("chunker_list", []) - for chunker_config in chunker_list: - if chunker_config.get("chunker_strategy") == chunker_strategy: - return chunker_config - - # 2) Provide sane defaults for newer strategies default_configs = { "RecursiveChunker": { "chunker_strategy": "RecursiveChunker", @@ -112,7 +85,6 @@ def get_chunker_config(chunker_strategy: str) -> dict: "chunk_size": 512, "min_characters_per_chunk": 50 }, - "LLMChunker": { "chunker_strategy": "LLMChunker", "embedding_model": "BAAI/bge-m3", @@ -137,127 +109,6 @@ def get_chunker_config(chunker_strategy: str) -> dict: if chunker_strategy in default_configs: return default_configs[chunker_strategy] - # 3) Fallback: use first available config but tag with requested strategy - if chunker_list: - fallback = chunker_list[0].copy() - fallback["chunker_strategy"] = chunker_strategy - # Non-fatal notice for visibility in logs if any - print(f"Warning: Using first available chunker config as fallback for '{chunker_strategy}'") - return fallback - - # 4) If no configs available at all raise ValueError( - f"Chunker '{chunker_strategy}' not found in config.json and no default or fallback available" + f"Chunker '{chunker_strategy}' not found " ) - -#TODO: Fix this - -def get_pipeline_config( - config_id: int, - db: Session | None = None, -) -> ExtractionPipelineConfig: - """Build ExtractionPipelineConfig from database. - - Args: - config_id: Database configuration ID (required). Loads pipeline - settings from the DataConfig table. - db: Optional database session. If not provided, a new session - will be created. - - Returns: - ExtractionPipelineConfig with deduplication, statement extraction, - and forgetting engine settings loaded from database. - - Raises: - ValueError: If config_id not found in database. - """ - from app.repositories.data_config_repository import DataConfigRepository - - # Load from database - if db is None: - db_gen = get_db() - db = next(db_gen) - - db_config = DataConfigRepository.get_by_id(db, config_id) - if db_config is None: - raise ValueError(f"Configuration {config_id} not found in database") - - # Build DedupConfig from database - dedup_kwargs = { - "enable_llm_dedup_blockwise": bool(db_config.enable_llm_dedup_blockwise) if db_config.enable_llm_dedup_blockwise is not None else False, - "enable_llm_disambiguation": bool(db_config.enable_llm_disambiguation) if db_config.enable_llm_disambiguation is not None else False, - } - - # Fuzzy thresholds - if db_config.t_name_strict is not None: - dedup_kwargs["fuzzy_name_threshold_strict"] = db_config.t_name_strict - if db_config.t_type_strict is not None: - dedup_kwargs["fuzzy_type_threshold_strict"] = db_config.t_type_strict - if db_config.t_overall is not None: - dedup_kwargs["fuzzy_overall_threshold"] = db_config.t_overall - - dedup_config = DedupConfig(**dedup_kwargs) - - # Build StatementExtractionConfig from database - stmt_kwargs = {} - if db_config.statement_granularity is not None: - stmt_kwargs["statement_granularity"] = db_config.statement_granularity - if db_config.include_dialogue_context is not None: - stmt_kwargs["include_dialogue_context"] = bool(db_config.include_dialogue_context) - if db_config.max_context is not None: - stmt_kwargs["max_dialogue_context_chars"] = db_config.max_context - - stmt_config = StatementExtractionConfig(**stmt_kwargs) - - # Build ForgettingEngineConfig from database - forget_kwargs = {} - if db_config.offset is not None: - forget_kwargs["offset"] = db_config.offset - if db_config.lambda_time is not None: - forget_kwargs["lambda_time"] = db_config.lambda_time - if db_config.lambda_mem is not None: - forget_kwargs["lambda_mem"] = db_config.lambda_mem - - forget_config = ForgettingEngineConfig(**forget_kwargs) - - return ExtractionPipelineConfig( - statement_extraction=stmt_config, - deduplication=dedup_config, - forgetting_engine=forget_config, - ) - - -def get_pruning_config( - config_id: int, - db: Session | None = None, -) -> dict: - """Retrieve semantic pruning config from database. - - Args: - config_id: Database configuration ID (required). - db: Optional database session. - - Returns: - Dict suitable for PruningConfig.model_validate with keys: - - pruning_switch: bool - - pruning_scene: str ("education" | "online_service" | "outbound") - - pruning_threshold: float (0-0.9) - - Raises: - ValueError: If config_id not found in database. - """ - from app.repositories.data_config_repository import DataConfigRepository - - if db is None: - db_gen = get_db() - db = next(db_gen) - - db_config = DataConfigRepository.get_by_id(db, config_id) - if db_config is None: - raise ValueError(f"Configuration {config_id} not found in database") - - return { - "pruning_switch": bool(db_config.pruning_enabled) if db_config.pruning_enabled is not None else False, - "pruning_scene": db_config.pruning_scene or "education", - "pruning_threshold": float(db_config.pruning_threshold) if db_config.pruning_threshold is not None else 0.5, - } diff --git a/api/app/core/memory/utils/config/definitions.py b/api/app/core/memory/utils/config/definitions.py index cc1aef66..fc07c2cc 100644 --- a/api/app/core/memory/utils/config/definitions.py +++ b/api/app/core/memory/utils/config/definitions.py @@ -1,269 +1,268 @@ -""" -配置加载模块 - DEPRECATED +# """ +# 配置加载模块 - DEPRECATED -⚠️ DEPRECATION NOTICE ⚠️ -This module is deprecated and will be removed in a future version. -Global configuration variables have been eliminated in favor of dependency injection. +# ⚠️ DEPRECATION NOTICE ⚠️ +# This module is deprecated and will be removed in a future version. +# Global configuration variables have been eliminated in favor of dependency injection. -Use the new MemoryConfig system instead: -- app.core.memory_config.config.MemoryConfig for configuration objects -- app.services.memory_agent_service.MemoryAgentService.load_memory_config() -- app.services.memory_storage_service.MemoryStorageService.load_memory_config() +# Use the new MemoryConfig system instead: +# - app.schemas.memory_config_schema.MemoryConfig for configuration objects +# - config_service = MemoryConfigService(db); config_service.load_memory_config(config_id) -阶段 1: 从 runtime.json 加载配置（路径 A）- DEPRECATED -阶段 2: 从数据库加载配置（路径 B，基于 dbrun.json 中的 config_id）- DEPRECATED -阶段 3: 暴露配置常量供项目使用（路径 A 和 B 的汇合点）- DEPRECATED -""" -import json -import os -import threading -from datetime import datetime, timedelta -from typing import Any, Dict, Optional +# 阶段 1: 从 runtime.json 加载配置（路径 A）- DEPRECATED +# 阶段 2: 从数据库加载配置（路径 B，基于 dbrun.json 中的 config_id）- DEPRECATED +# 阶段 3: 暴露配置常量供项目使用（路径 A 和 B 的汇合点）- DEPRECATED +# """ +# import json +# import os +# import threading +# from datetime import datetime, timedelta +# from typing import Any, Dict, Optional -#TODO: Fix this +# #TODO: Fix this -try: - from dotenv import load_dotenv - load_dotenv() -except Exception: - pass +# try: +# from dotenv import load_dotenv +# load_dotenv() +# except Exception: +# pass -# Import unified configuration system -try: - from app.core.config import settings - USE_UNIFIED_CONFIG = True -except ImportError: - USE_UNIFIED_CONFIG = False - settings = None +# # Import unified configuration system +# try: +# from app.core.config import settings +# USE_UNIFIED_CONFIG = True +# except ImportError: +# USE_UNIFIED_CONFIG = False +# settings = None -# PROJECT_ROOT 应该指向 app/core/memory/ 目录 -# __file__ = app/core/memory/utils/config/definitions.py -# os.path.dirname(__file__) = app/core/memory/utils/config -# os.path.dirname(...) = app/core/memory/utils -# os.path.dirname(...) = app/core/memory -PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# # PROJECT_ROOT 应该指向 app/core/memory/ 目录 +# # __file__ = app/core/memory/utils/config/definitions.py +# # os.path.dirname(__file__) = app/core/memory/utils/config +# # os.path.dirname(...) = app/core/memory/utils +# # os.path.dirname(...) = app/core/memory +# PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -# DEPRECATED: Global configuration lock removed -# Use MemoryConfig objects with dependency injection instead +# # DEPRECATED: Global configuration lock removed +# # Use MemoryConfig objects with dependency injection instead -# DEPRECATED: Legacy config.json loading removed -# Use MemoryConfig objects with dependency injection instead -CONFIG = {} +# # DEPRECATED: Legacy config.json loading removed +# # Use MemoryConfig objects with dependency injection instead +# CONFIG = {} -DEFAULT_VALUES = { - "llm_name": "openai/qwen-plus", - "embedding_name": "openai/nomic-embed-text:v1.5", - "chunker_strategy": "RecursiveChunker", - "group_id": "group_123", - "user_id": "default_user", - "apply_id": "default_apply", - "llm_agent_name": "openai/qwen-plus", - "llm_verify_name": "openai/qwen-plus", - "llm_image_recognition": "openai/qwen-plus", - "llm_voice_recognition": "openai/qwen-plus", - "prompt_level": "DEBUG", - "reflexion_iteration_period": "3", - "reflexion_range": "retrieval", - "reflexion_baseline": "TIME", -} +# DEFAULT_VALUES = { +# "llm_name": "openai/qwen-plus", +# "embedding_name": "openai/nomic-embed-text:v1.5", +# "chunker_strategy": "RecursiveChunker", +# "group_id": "group_123", +# "user_id": "default_user", +# "apply_id": "default_apply", +# "llm_agent_name": "openai/qwen-plus", +# "llm_verify_name": "openai/qwen-plus", +# "llm_image_recognition": "openai/qwen-plus", +# "llm_voice_recognition": "openai/qwen-plus", +# "prompt_level": "DEBUG", +# "reflexion_iteration_period": "3", +# "reflexion_range": "retrieval", +# "reflexion_baseline": "TIME", +# } -# DEPRECATED: Legacy global variables for backward compatibility only -# These will be removed in a future version -# Use MemoryConfig objects with dependency injection instead -LANGFUSE_ENABLED = os.getenv("LANGFUSE_ENABLED", "false").lower() == "true" -SELECTED_LLM_ID = os.getenv("SELECTED_LLM_ID", DEFAULT_VALUES["llm_name"]) +# # DEPRECATED: Legacy global variables for backward compatibility only +# # These will be removed in a future version +# # Use MemoryConfig objects with dependency injection instead +# # LANGFUSE_ENABLED = os.getenv("LANGFUSE_ENABLED", "false").lower() == "true" +# # SELECTED_LLM_ID = os.getenv("SELECTED_LLM_ID", DEFAULT_VALUES["llm_name"]) -# 阶段 1: 从 runtime.json 加载配置（路径 A） -def _load_from_runtime_json() -> Dict[str, Any]: - """ - DEPRECATED: Legacy runtime.json loading +# # 阶段 1: 从 runtime.json 加载配置（路径 A） +# def _load_from_runtime_json() -> Dict[str, Any]: +# """ +# DEPRECATED: Legacy runtime.json loading - ⚠️ This function is deprecated and will be removed in a future version. - Use MemoryConfig objects with dependency injection instead. +# ⚠️ This function is deprecated and will be removed in a future version. +# Use MemoryConfig objects with dependency injection instead. - Returns: - Dict[str, Any]: Empty configuration (legacy support only) - """ - import warnings - warnings.warn( - "Runtime JSON loading is deprecated. Use MemoryConfig objects with dependency injection instead.", - DeprecationWarning, - stacklevel=2 - ) - return {"selections": {}} +# Returns: +# Dict[str, Any]: Empty configuration (legacy support only) +# """ +# import warnings +# warnings.warn( +# "Runtime JSON loading is deprecated. Use MemoryConfig objects with dependency injection instead.", +# DeprecationWarning, +# stacklevel=2 +# ) +# return {"selections": {}} -# 阶段 2: 从数据库加载配置（路径 B）- 已整合到统一加载器 -# 注意：此函数已被 _load_from_runtime_json 中的统一配置加载器替代 -# 保留此函数仅为向后兼容 -def _load_from_database() -> Optional[Dict[str, Any]]: - """ - DEPRECATED: Legacy database configuration loading +# # 阶段 2: 从数据库加载配置（路径 B）- 已整合到统一加载器 +# # 注意：此函数已被 _load_from_runtime_json 中的统一配置加载器替代 +# # 保留此函数仅为向后兼容 +# def _load_from_database() -> Optional[Dict[str, Any]]: +# """ +# DEPRECATED: Legacy database configuration loading - ⚠️ This function is deprecated and will be removed in a future version. - Use MemoryConfig objects with dependency injection instead. +# ⚠️ This function is deprecated and will be removed in a future version. +# Use MemoryConfig objects with dependency injection instead. - Returns: - Optional[Dict[str, Any]]: None (deprecated functionality) - """ - import warnings - warnings.warn( - "Database configuration loading is deprecated. Use MemoryConfig objects with dependency injection instead.", - DeprecationWarning, - stacklevel=2 - ) - return None +# Returns: +# Optional[Dict[str, Any]]: None (deprecated functionality) +# """ +# import warnings +# warnings.warn( +# "Database configuration loading is deprecated. Use MemoryConfig objects with dependency injection instead.", +# DeprecationWarning, +# stacklevel=2 +# ) +# return None -# 阶段 3: 暴露配置常量（路径 A 和 B 的汇合点）- DEPRECATED -def _expose_runtime_constants(runtime_cfg: Dict[str, Any]) -> None: - """ - DEPRECATED: 将运行时配置暴露为全局常量供项目使用 +# # 阶段 3: 暴露配置常量（路径 A 和 B 的汇合点）- DEPRECATED +# def _expose_runtime_constants(runtime_cfg: Dict[str, Any]) -> None: +# """ +# DEPRECATED: 将运行时配置暴露为全局常量供项目使用 - ⚠️ This function is deprecated and will be removed in a future version. - Global configuration variables have been eliminated in favor of dependency injection. +# ⚠️ This function is deprecated and will be removed in a future version. +# Global configuration variables have been eliminated in favor of dependency injection. - Use the new MemoryConfig system instead: - - app.core.memory_config.config.MemoryConfig for configuration objects - - Pass configuration objects as parameters instead of using global variables +# Use the new MemoryConfig system instead: +# - app.core.memory_config.config.MemoryConfig for configuration objects +# - Pass configuration objects as parameters instead of using global variables - Args: - runtime_cfg: 运行时配置字典 - """ - import warnings - warnings.warn( - "Global configuration variables are deprecated. Use MemoryConfig objects with dependency injection instead.", - DeprecationWarning, - stacklevel=2 - ) +# Args: +# runtime_cfg: 运行时配置字典 +# """ +# import warnings +# warnings.warn( +# "Global configuration variables are deprecated. Use MemoryConfig objects with dependency injection instead.", +# DeprecationWarning, +# stacklevel=2 +# ) - # Keep minimal global state for backward compatibility only - # These will be removed in a future version - global RUNTIME_CONFIG, SELECTIONS +# # Keep minimal global state for backward compatibility only +# # These will be removed in a future version +# global RUNTIME_CONFIG, SELECTIONS - RUNTIME_CONFIG = runtime_cfg - SELECTIONS = RUNTIME_CONFIG.get("selections", {}) +# RUNTIME_CONFIG = runtime_cfg +# SELECTIONS = RUNTIME_CONFIG.get("selections", {}) - # All other global variables have been removed - # Use MemoryConfig objects instead +# # All other global variables have been removed +# # Use MemoryConfig objects instead -# 初始化：使用统一配置加载器 -def _initialize_configuration() -> None: - """ - DEPRECATED: Legacy configuration initialization +# # 初始化：使用统一配置加载器 +# def _initialize_configuration() -> None: +# """ +# DEPRECATED: Legacy configuration initialization - ⚠️ This function is deprecated and will be removed in a future version. - Use MemoryConfig objects with dependency injection instead. - """ - import warnings - warnings.warn( - "Global configuration initialization is deprecated. Use MemoryConfig objects with dependency injection instead.", - DeprecationWarning, - stacklevel=2 - ) - # Initialize with empty configuration for backward compatibility - _expose_runtime_constants({"selections": {}}) +# ⚠️ This function is deprecated and will be removed in a future version. +# Use MemoryConfig objects with dependency injection instead. +# """ +# import warnings +# warnings.warn( +# "Global configuration initialization is deprecated. Use MemoryConfig objects with dependency injection instead.", +# DeprecationWarning, +# stacklevel=2 +# ) +# # Initialize with empty configuration for backward compatibility +# _expose_runtime_constants({"selections": {}}) -# 模块加载时自动初始化配置 -_initialize_configuration() +# # 模块加载时自动初始化配置 +# _initialize_configuration() -# DEPRECATED: Global variables removed -# These variables have been eliminated in favor of dependency injection -# Use MemoryConfig objects instead of accessing global variables +# # DEPRECATED: Global variables removed +# # These variables have been eliminated in favor of dependency injection +# # Use MemoryConfig objects instead of accessing global variables -# 公共 API：动态重新加载配置 -def reload_configuration_from_database(config_id, force_reload: bool = False) -> bool: - """ - DEPRECATED: Legacy configuration reloading +# # 公共 API：动态重新加载配置 +# def reload_configuration_from_database(config_id, force_reload: bool = False) -> bool: +# """ +# DEPRECATED: Legacy configuration reloading - ⚠️ This function is deprecated and will be removed in a future version. - Use MemoryConfig objects with dependency injection instead. +# ⚠️ This function is deprecated and will be removed in a future version. +# Use MemoryConfig objects with dependency injection instead. - For new code, use: - - app.services.memory_agent_service.MemoryAgentService.load_memory_config() - - app.services.memory_storage_service.MemoryStorageService.load_memory_config() +# For new code, use: +# - app.services.memory_agent_service.MemoryAgentService.load_memory_config() +# - app.services.memory_storage_service.MemoryStorageService.load_memory_config() - Args: - config_id: Configuration ID (deprecated) - force_reload: Force reload flag (deprecated) +# Args: +# config_id: Configuration ID (deprecated) +# force_reload: Force reload flag (deprecated) - Returns: - bool: Always returns False (deprecated functionality) - """ - import logging - import warnings +# Returns: +# bool: Always returns False (deprecated functionality) +# """ +# import logging +# import warnings - logger = logging.getLogger(__name__) +# logger = logging.getLogger(__name__) - warnings.warn( - "reload_configuration_from_database is deprecated. Use MemoryConfig objects with dependency injection instead.", - DeprecationWarning, - stacklevel=2 - ) +# warnings.warn( +# "reload_configuration_from_database is deprecated. Use MemoryConfig objects with dependency injection instead.", +# DeprecationWarning, +# stacklevel=2 +# ) - logger.warning(f"Deprecated function reload_configuration_from_database called with config_id={config_id}. " - "Use MemoryConfig objects with dependency injection instead.") +# logger.warning(f"Deprecated function reload_configuration_from_database called with config_id={config_id}. " +# "Use MemoryConfig objects with dependency injection instead.") - return False +# return False -def get_current_config_id() -> Optional[str]: - """ - DEPRECATED: Legacy config ID retrieval +# def get_current_config_id() -> Optional[str]: +# """ +# DEPRECATED: Legacy config ID retrieval - ⚠️ This function is deprecated and will be removed in a future version. - Use MemoryConfig objects with dependency injection instead. +# ⚠️ This function is deprecated and will be removed in a future version. +# Use MemoryConfig objects with dependency injection instead. - Returns: - Optional[str]: None (deprecated functionality) - """ - import warnings - warnings.warn( - "get_current_config_id is deprecated. Use MemoryConfig objects with dependency injection instead.", - DeprecationWarning, - stacklevel=2 - ) - return None +# Returns: +# Optional[str]: None (deprecated functionality) +# """ +# import warnings +# warnings.warn( +# "get_current_config_id is deprecated. Use MemoryConfig objects with dependency injection instead.", +# DeprecationWarning, +# stacklevel=2 +# ) +# return None -def ensure_fresh_config(config_id = None) -> bool: - """ - DEPRECATED: Legacy configuration freshness check +# def ensure_fresh_config(config_id = None) -> bool: +# """ +# DEPRECATED: Legacy configuration freshness check - ⚠️ This function is deprecated and will be removed in a future version. - Use MemoryConfig objects with dependency injection instead. +# ⚠️ This function is deprecated and will be removed in a future version. +# Use MemoryConfig objects with dependency injection instead. - For new code, use: - - app.services.memory_agent_service.MemoryAgentService.load_memory_config() - - app.services.memory_storage_service.MemoryStorageService.load_memory_config() +# For new code, use: +# - app.services.memory_agent_service.MemoryAgentService.load_memory_config() +# - app.services.memory_storage_service.MemoryStorageService.load_memory_config() - Args: - config_id: Configuration ID (deprecated) +# Args: +# config_id: Configuration ID (deprecated) - Returns: - bool: Always returns False (deprecated functionality) - """ - import logging - import warnings +# Returns: +# bool: Always returns False (deprecated functionality) +# """ +# import logging +# import warnings - logger = logging.getLogger(__name__) +# logger = logging.getLogger(__name__) - warnings.warn( - "ensure_fresh_config is deprecated. Use MemoryConfig objects with dependency injection instead.", - DeprecationWarning, - stacklevel=2 - ) +# warnings.warn( +# "ensure_fresh_config is deprecated. Use MemoryConfig objects with dependency injection instead.", +# DeprecationWarning, +# stacklevel=2 +# ) - logger.warning(f"Deprecated function ensure_fresh_config called with config_id={config_id}. " - "Use MemoryConfig objects with dependency injection instead.") +# logger.warning(f"Deprecated function ensure_fresh_config called with config_id={config_id}. " +# "Use MemoryConfig objects with dependency injection instead.") - return False +# return False diff --git a/api/app/core/memory/utils/embedder/embedder_utils.py b/api/app/core/memory/utils/embedder/embedder_utils.py index 86899e30..0a384a87 100644 --- a/api/app/core/memory/utils/embedder/embedder_utils.py +++ b/api/app/core/memory/utils/embedder/embedder_utils.py @@ -6,8 +6,9 @@ This module provides centralized functions for creating embedder clients. from typing import TYPE_CHECKING from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient -from app.core.memory.utils.config.config_utils import get_embedder_config from app.core.models.base import RedBearModelConfig +from app.db import get_db_context +from app.services.memory_config_service import MemoryConfigService if TYPE_CHECKING: from app.schemas.memory_config_schema import MemoryConfig @@ -66,7 +67,8 @@ def get_embedder_client(embedding_id: str) -> OpenAIEmbedderClient: raise ValueError("Embedding ID is required but was not provided") try: - embedder_config_dict = get_embedder_config(embedding_id) + with get_db_context() as db: + embedder_config_dict = MemoryConfigService(db).get_embedder_config(embedding_id) except Exception as e: raise ValueError(f"Invalid embedding ID '{embedding_id}': {str(e)}") from e diff --git a/api/app/core/memory/utils/llm/llm_utils.py b/api/app/core/memory/utils/llm/llm_utils.py index a4b327da..19d76d68 100644 --- a/api/app/core/memory/utils/llm/llm_utils.py +++ b/api/app/core/memory/utils/llm/llm_utils.py @@ -1,9 +1,9 @@ from typing import TYPE_CHECKING from app.core.memory.llm_tools.openai_client import OpenAIClient -from app.core.memory.utils.config.config_utils import get_model_config from app.core.models.base import RedBearModelConfig from pydantic import BaseModel +from sqlalchemy.orm import Session if TYPE_CHECKING: from app.schemas.memory_config_schema import MemoryConfig @@ -13,105 +13,225 @@ async def handle_response(response: type[BaseModel]) -> dict: return response.model_dump() -def get_llm_client_from_config(memory_config: "MemoryConfig") -> OpenAIClient: +class MemoryClientFactory: """ - Get LLM client from MemoryConfig object. + Factory for creating LLM, embedder, and reranker clients. - **PREFERRED METHOD**: Use this function in production code when you have a MemoryConfig object. - This ensures proper configuration management and multi-tenant support. + Initialize once with db session, then call methods without passing db each time. - Args: - memory_config: MemoryConfig object containing llm_model_id - - Returns: - OpenAIClient: Initialized LLM client - - Raises: - ValueError: If LLM model ID is not configured or client initialization fails - Example: - >>> llm_client = get_llm_client_from_config(memory_config) + >>> factory = MemoryClientFactory(db) + >>> llm_client = factory.get_llm_client(model_id) + >>> embedder_client = factory.get_embedder_client(embedding_id) """ - if not memory_config.llm_model_id: - raise ValueError( - f"Configuration {memory_config.config_id} has no LLM model configured" - ) - return get_llm_client(str(memory_config.llm_model_id)) + + def __init__(self, db: Session): + from app.services.memory_config_service import MemoryConfigService + self._config_service = MemoryConfigService(db) + + def get_llm_client(self, llm_id: str) -> OpenAIClient: + """Get LLM client by model ID.""" + if not llm_id: + raise ValueError("LLM ID is required") + + try: + model_config = self._config_service.get_model_config(llm_id) + except Exception as e: + raise ValueError(f"Invalid LLM ID '{llm_id}': {str(e)}") from e + + try: + return OpenAIClient( + RedBearModelConfig( + model_name=model_config.get("model_name"), + provider=model_config.get("provider"), + api_key=model_config.get("api_key"), + base_url=model_config.get("base_url") + ), + type_=model_config.get("type") + ) + except Exception as e: + model_name = model_config.get('model_name', 'unknown') + raise ValueError(f"Failed to initialize LLM client for model '{model_name}': {str(e)}") from e + + def get_embedder_client(self, embedding_id: str): + """Get embedder client by model ID.""" + from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient + + if not embedding_id: + raise ValueError("Embedding ID is required") + + try: + embedder_config = self._config_service.get_embedder_config(embedding_id) + except Exception as e: + raise ValueError(f"Invalid embedding ID '{embedding_id}': {str(e)}") from e + + try: + return OpenAIEmbedderClient( + RedBearModelConfig( + model_name=embedder_config.get("model_name"), + provider=embedder_config.get("provider"), + api_key=embedder_config.get("api_key"), + base_url=embedder_config.get("base_url") + ) + ) + except Exception as e: + model_name = embedder_config.get('model_name', 'unknown') + raise ValueError(f"Failed to initialize embedder client for model '{model_name}': {str(e)}") from e + + def get_reranker_client(self, rerank_id: str) -> OpenAIClient: + """Get reranker client by model ID.""" + if not rerank_id: + raise ValueError("Rerank ID is required") + + try: + model_config = self._config_service.get_model_config(rerank_id) + except Exception as e: + raise ValueError(f"Invalid rerank ID '{rerank_id}': {str(e)}") from e + + try: + return OpenAIClient( + RedBearModelConfig( + model_name=model_config.get("model_name"), + provider=model_config.get("provider"), + api_key=model_config.get("api_key"), + base_url=model_config.get("base_url") + ), + type_=model_config.get("type") + ) + except Exception as e: + model_name = model_config.get('model_name', 'unknown') + raise ValueError(f"Failed to initialize reranker client for model '{model_name}': {str(e)}") from e + + def get_llm_client_from_config(self, memory_config: "MemoryConfig") -> OpenAIClient: + """Get LLM client from MemoryConfig object. + + Args: + memory_config: Configuration containing llm_model_id + + Returns: + OpenAIClient configured for the LLM model + + Raises: + ValueError: If memory_config has no LLM model configured + """ + if not memory_config.llm_model_id: + raise ValueError( + f"Configuration {memory_config.config_id} has no LLM model configured" + ) + return self.get_llm_client(str(memory_config.llm_model_id)) + + def get_embedder_client_from_config(self, memory_config: "MemoryConfig"): + """Get embedder client from MemoryConfig object. + + Args: + memory_config: Configuration containing embedding_model_id + + Returns: + OpenAIEmbedderClient configured for the embedding model + + Raises: + ValueError: If memory_config has no embedding model configured + """ + if not memory_config.embedding_model_id: + raise ValueError( + f"Configuration {memory_config.config_id} has no embedding model configured" + ) + return self.get_embedder_client(str(memory_config.embedding_model_id)) + + def get_reranker_client_from_config(self, memory_config: "MemoryConfig") -> OpenAIClient: + """Get reranker client from MemoryConfig object. + + Args: + memory_config: Configuration containing rerank_model_id + + Returns: + OpenAIClient configured for the reranker model + + Raises: + ValueError: If memory_config has no rerank model configured + """ + if not memory_config.rerank_model_id: + raise ValueError( + f"Configuration {memory_config.config_id} has no rerank model configured" + ) + return self.get_reranker_client(str(memory_config.rerank_model_id)) -def get_llm_client(llm_id: str): - """ - Get LLM client by model ID. +# Legacy functions for backward compatibility +def get_llm_client_from_config(memory_config: "MemoryConfig", db: Session) -> OpenAIClient: + """Get LLM client from MemoryConfig object. - **LEGACY/TEST METHOD**: Use this function only for: - - Test/evaluation code where you have a model ID directly - - Legacy code that hasn't been migrated to MemoryConfig yet + DEPRECATED: Use MemoryClientFactory(db).get_llm_client_from_config(memory_config) instead. - For production code with MemoryConfig, use get_llm_client_from_config() instead. + This function is maintained for backward compatibility during migration to the + factory pattern. New code should create a MemoryClientFactory instance and use + its get_llm_client_from_config method directly. Args: - llm_id: LLM model ID (required) + memory_config: Configuration containing llm_model_id + db: Database session Returns: - OpenAIClient: Initialized LLM client + OpenAIClient configured for the LLM model Raises: - ValueError: If llm_id is not provided or client initialization fails - - Example: - >>> # For tests/evaluations only - >>> llm_client = get_llm_client("model-uuid-string") + ValueError: If memory_config has no LLM model configured """ - if not llm_id: - raise ValueError("LLM ID is required but was not provided") - - try: - model_config = get_model_config(llm_id) - except Exception as e: - raise ValueError(f"Invalid LLM ID '{llm_id}': {str(e)}") from e - - try: - llm_client = OpenAIClient(RedBearModelConfig( - model_name=model_config.get("model_name"), - provider=model_config.get("provider"), - api_key=model_config.get("api_key"), - base_url=model_config.get("base_url") - ),type_=model_config.get("type")) - return llm_client - except Exception as e: - model_name = model_config.get('model_name', 'unknown') - raise ValueError(f"Failed to initialize LLM client for model '{model_name}': {str(e)}") from e + return MemoryClientFactory(db).get_llm_client_from_config(memory_config) -def get_reranker_client(rerank_id: str): - """ - Get an LLM client configured for reranking. +def get_llm_client(llm_id: str, db: Session) -> OpenAIClient: + """Get LLM client by model ID. + + DEPRECATED: Use MemoryClientFactory(db).get_llm_client(llm_id) instead. + + This function is maintained for backward compatibility during migration to the + factory pattern. New code should create a MemoryClientFactory instance and use + its get_llm_client method directly. Args: - rerank_id: Reranker model ID (required) + llm_id: LLM model ID + db: Database session Returns: - OpenAIClient: Initialized client for the reranker model - - Raises: - ValueError: If rerank_id is not provided or client initialization fails + OpenAIClient configured for the LLM model """ - if not rerank_id: - raise ValueError("Rerank ID is required but was not provided") + return MemoryClientFactory(db).get_llm_client(llm_id) + + +def get_embedder_client(embedding_id: str, db: Session): + """Get embedder client by model ID. - try: - model_config = get_model_config(rerank_id) - except Exception as e: - raise ValueError(f"Invalid rerank ID '{rerank_id}': {str(e)}") from e + DEPRECATED: Use MemoryClientFactory(db).get_embedder_client(embedding_id) instead. - try: - reranker_client = OpenAIClient(RedBearModelConfig( - model_name=model_config.get("model_name"), - provider=model_config.get("provider"), - api_key=model_config.get("api_key"), - base_url=model_config.get("base_url") - ),type_=model_config.get("type")) - return reranker_client - except Exception as e: - model_name = model_config.get('model_name', 'unknown') - raise ValueError(f"Failed to initialize reranker client for model '{model_name}': {str(e)}") from e \ No newline at end of file + This function is maintained for backward compatibility during migration to the + factory pattern. New code should create a MemoryClientFactory instance and use + its get_embedder_client method directly. + + Args: + embedding_id: Embedding model ID + db: Database session + + Returns: + OpenAIEmbedderClient configured for the embedding model + """ + return MemoryClientFactory(db).get_embedder_client(embedding_id) + + +def get_reranker_client(rerank_id: str, db: Session) -> OpenAIClient: + """Get reranker client by model ID. + + DEPRECATED: Use MemoryClientFactory(db).get_reranker_client(rerank_id) instead. + + This function is maintained for backward compatibility during migration to the + factory pattern. New code should create a MemoryClientFactory instance and use + its get_reranker_client method directly. + + Args: + rerank_id: Reranker model ID + db: Database session + + Returns: + OpenAIClient configured for the reranker model + """ + return MemoryClientFactory(db).get_reranker_client(rerank_id) diff --git a/api/app/core/memory/utils/self_reflexion_utils/evaluate.py b/api/app/core/memory/utils/self_reflexion_utils/evaluate.py index 0ea68461..4d1835cd 100644 --- a/api/app/core/memory/utils/self_reflexion_utils/evaluate.py +++ b/api/app/core/memory/utils/self_reflexion_utils/evaluate.py @@ -6,11 +6,12 @@ """ import logging -from typing import List, Any import time +from typing import Any, List +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory from app.core.memory.utils.prompt.template_render import render_evaluate_prompt -from app.core.memory.utils.llm.llm_utils import get_llm_client +from app.db import get_db_context from app.schemas.memory_storage_schema import ConflictResultSchema from pydantic import BaseModel @@ -25,7 +26,9 @@ async def conflict(evaluate_data: List[Any]) -> List[Any]: 冲突记忆列表（JSON 数组）。 """ from app.core.memory.utils.config import definitions as config_defs - client = get_llm_client(config_defs.SELECTED_LLM_ID) + with get_db_context() as db: + factory = MemoryClientFactory(db) + client = factory.get_llm_client(config_defs.SELECTED_LLM_ID) rendered_prompt = await render_evaluate_prompt(evaluate_data, ConflictResultSchema) messages = [{"role": "user", "content": rendered_prompt}] print(f"提示词长度: {len(rendered_prompt)}") diff --git a/api/app/core/memory/utils/self_reflexion_utils/reflexion.py b/api/app/core/memory/utils/self_reflexion_utils/reflexion.py index 6835b868..1b915118 100644 --- a/api/app/core/memory/utils/self_reflexion_utils/reflexion.py +++ b/api/app/core/memory/utils/self_reflexion_utils/reflexion.py @@ -6,11 +6,12 @@ """ import logging -from typing import List, Any import time +from typing import Any, List +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory from app.core.memory.utils.prompt.template_render import render_reflexion_prompt -from app.core.memory.utils.llm.llm_utils import get_llm_client +from app.db import get_db_context from app.schemas.memory_storage_schema import ReflexionResultSchema from pydantic import BaseModel @@ -25,7 +26,9 @@ async def reflexion(ref_data: List[Any]) -> List[Any]: 反思结果列表（JSON 数组）。 """ from app.core.memory.utils.config import definitions as config_defs - client = get_llm_client(config_defs.SELECTED_LLM_ID) + with get_db_context() as db: + factory = MemoryClientFactory(db) + client = factory.get_llm_client(config_defs.SELECTED_LLM_ID) rendered_prompt = await render_reflexion_prompt(ref_data, ReflexionResultSchema) messages = [{"role": "user", "content": rendered_prompt}] print(f"提示词长度: {len(rendered_prompt)}") diff --git a/api/app/core/rag_utils/chunk_insight.py b/api/app/core/rag_utils/chunk_insight.py index 2c96160e..e904e53d 100644 --- a/api/app/core/rag_utils/chunk_insight.py +++ b/api/app/core/rag_utils/chunk_insight.py @@ -5,16 +5,24 @@ This module provides functionality to analyze chunk content and generate insight """ import asyncio -from typing import List, Dict, Any from collections import Counter +from typing import Any, Dict, List + +from app.core.logging_config import get_business_logger +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.db import get_db_context from pydantic import BaseModel, Field -from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.core.logging_config import get_business_logger - business_logger = get_business_logger() +def _get_llm_client(): + """Get LLM client using db context.""" + with get_db_context() as db: + factory = MemoryClientFactory(db) + return factory.get_llm_client(None) # Uses default LLM + + class ChunkInsight(BaseModel): """Pydantic model for chunk insight.""" insight: str = Field(..., description="对chunk内容的深度洞察分析") @@ -40,7 +48,7 @@ async def classify_chunk_domain(chunk: str) -> str: Domain name """ try: - llm_client = get_llm_client() + llm_client = _get_llm_client() prompt = f"""请将以下文本内容归类到最合适的领域中。 @@ -177,7 +185,7 @@ async def generate_chunk_insight(chunks: List[str], max_chunks: int = 15) -> str ] # 调用LLM生成洞察 - llm_client = get_llm_client() + llm_client = _get_llm_client() response = await llm_client.chat(messages=messages) insight = response.content.strip() diff --git a/api/app/core/rag_utils/chunk_summary.py b/api/app/core/rag_utils/chunk_summary.py index 971d6907..7f69af88 100644 --- a/api/app/core/rag_utils/chunk_summary.py +++ b/api/app/core/rag_utils/chunk_summary.py @@ -5,15 +5,23 @@ This module provides functionality to summarize chunk content using LLM. """ import asyncio -from typing import List, Dict, Any +from typing import Any, Dict, List + +from app.core.logging_config import get_business_logger +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.db import get_db_context from pydantic import BaseModel, Field -from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.core.logging_config import get_business_logger - business_logger = get_business_logger() +def _get_llm_client(): + """Get LLM client using db context.""" + with get_db_context() as db: + factory = MemoryClientFactory(db) + return factory.get_llm_client(None) # Uses default LLM + + class ChunkSummary(BaseModel): """Pydantic model for chunk summary.""" summary: str = Field(..., description="简洁的chunk内容摘要") @@ -59,7 +67,7 @@ async def generate_chunk_summary(chunks: List[str], max_chunks: int = 10) -> str ] # 调用LLM生成摘要 - llm_client = get_llm_client() + llm_client = _get_llm_client() response = await llm_client.chat(messages=messages) summary = response.content.strip() diff --git a/api/app/core/rag_utils/chunk_tags.py b/api/app/core/rag_utils/chunk_tags.py index 719f97e6..2057f8ac 100644 --- a/api/app/core/rag_utils/chunk_tags.py +++ b/api/app/core/rag_utils/chunk_tags.py @@ -7,14 +7,22 @@ This module provides functionality to extract meaningful tags from chunk content import asyncio from collections import Counter from typing import List, Tuple + +from app.core.logging_config import get_business_logger +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.db import get_db_context from pydantic import BaseModel, Field -from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.core.logging_config import get_business_logger - business_logger = get_business_logger() +def _get_llm_client(): + """Get LLM client using db context.""" + with get_db_context() as db: + factory = MemoryClientFactory(db) + return factory.get_llm_client(None) # Uses default LLM + + class ExtractedTags(BaseModel): """Pydantic model for extracted tags.""" tags: List[str] = Field(..., description="从文本中提取的关键标签列表") @@ -56,7 +64,7 @@ async def extract_chunk_tags(chunks: List[str], max_tags: int = 10, max_chunks: "标签应该是名词或名词短语，能够准确概括文本的核心内容。" ) - llm_client = get_llm_client() + llm_client = _get_llm_client() # 为每个chunk单独提取标签，然后统计频率 all_tags = [] @@ -151,7 +159,7 @@ async def extract_chunk_persona(chunks: List[str], max_personas: int = 5, max_ch ] # 调用LLM提取人物形象 - llm_client = get_llm_client() + llm_client = _get_llm_client() structured_response = await llm_client.response_structured( messages=messages, response_model=ExtractedPersona diff --git a/api/app/schemas/memory_config_schema.py b/api/app/schemas/memory_config_schema.py index 2ea24be8..171abb7a 100644 --- a/api/app/schemas/memory_config_schema.py +++ b/api/app/schemas/memory_config_schema.py @@ -391,6 +391,29 @@ class MemoryConfig: embedding_params: Dict[str, Any] = field(default_factory=dict) config_version: str = "2.0" + # Pipeline config: Deduplication + enable_llm_dedup_blockwise: bool = False + enable_llm_disambiguation: bool = False + deep_retrieval: bool = True + t_type_strict: float = 0.8 + t_name_strict: float = 0.8 + t_overall: float = 0.8 + + # Pipeline config: Statement extraction + statement_granularity: int = 2 + include_dialogue_context: bool = False + max_dialogue_context_chars: int = 1000 + + # Pipeline config: Forgetting engine + lambda_time: float = 0.5 + lambda_mem: float = 0.5 + offset: float = 0.0 + + # Pipeline config: Pruning + pruning_enabled: bool = False + pruning_scene: Optional[str] = "education" + pruning_threshold: float = 0.5 + def __post_init__(self): """Validate configuration after initialization.""" if not self.config_name or not self.config_name.strip(): diff --git a/api/app/services/draft_run_service.py b/api/app/services/draft_run_service.py index 489ffe4b..c0d2e3ff 100644 --- a/api/app/services/draft_run_service.py +++ b/api/app/services/draft_run_service.py @@ -3,26 +3,26 @@ 提供 Agent 试运行功能，允许用户在不发布应用的情况下测试配置。 """ -import time -import uuid -import json import asyncio import datetime -from typing import Dict, Any, Optional, List, AsyncGenerator -from langchain.tools import tool -from pydantic import BaseModel, Field -from sqlalchemy.orm import Session -from sqlalchemy import select +import json +import time +import uuid +from typing import Any, AsyncGenerator, Dict, List, Optional -from app.models import AgentConfig, ModelConfig, ModelApiKey -from app.core.exceptions import BusinessException from app.core.error_codes import BizCode +from app.core.exceptions import BusinessException from app.core.logging_config import get_business_logger -from app.schemas.prompt_schema import render_prompt_message, PromptMessageRole +from app.core.rag.nlp.search import knowledge_retrieval +from app.models import AgentConfig, ModelApiKey, ModelConfig +from app.schemas.prompt_schema import PromptMessageRole, render_prompt_message +from app.services.langchain_tool_server import Search from app.services.memory_agent_service import MemoryAgentService from app.services.model_parameter_merger import ModelParameterMerger -from app.core.rag.nlp.search import knowledge_retrieval -from app.services.langchain_tool_server import Search +from langchain.tools import tool +from pydantic import BaseModel, Field +from sqlalchemy import select +from sqlalchemy.orm import Session logger = get_business_logger() class KnowledgeRetrievalInput(BaseModel): @@ -83,17 +83,23 @@ def create_long_term_memory_tool(memory_config: Dict[str, Any], end_user_id: str """ logger.info(f" 长期记忆工具被调用！question={question}, user={end_user_id}") try: - memory_content = asyncio.run( - MemoryAgentService().read_memory( - group_id=end_user_id, - message=question, - history=[], - search_switch="1", - config_id=config_id, - storage_type=storage_type, - user_rag_memory_id=user_rag_memory_id + from app.db import get_db + db = next(get_db()) + try: + memory_content = asyncio.run( + MemoryAgentService().read_memory( + group_id=end_user_id, + message=question, + history=[], + search_switch="1", + config_id=config_id, + db=db, + storage_type=storage_type, + user_rag_memory_id=user_rag_memory_id + ) ) - ) + finally: + db.close() logger.info(f'用户ID：Agent:{end_user_id}') logger.debug("调用长期记忆 API", extra={"question": question, "end_user_id": end_user_id}) @@ -713,9 +719,9 @@ class DraftRunService: Raises: BusinessException: 当指定的会话不存在时 """ - from app.services.conversation_service import ConversationService - from app.schemas.conversation_schema import ConversationCreate from app.models import Conversation as ConversationModel + from app.schemas.conversation_schema import ConversationCreate + from app.services.conversation_service import ConversationService conversation_service = ConversationService(self.db) diff --git a/api/app/services/emotion_analytics_service.py b/api/app/services/emotion_analytics_service.py index 6952256e..2e63eeb0 100644 --- a/api/app/services/emotion_analytics_service.py +++ b/api/app/services/emotion_analytics_service.py @@ -7,14 +7,15 @@ Classes: EmotionAnalyticsService: 情绪分析服务，提供各种情绪分析功能 """ -from typing import Dict, Any, Optional, List -import statistics import json -from pydantic import BaseModel, Field +import statistics +from typing import Any, Dict, List, Optional +from app.core.logging_config import get_business_logger from app.repositories.neo4j.emotion_repository import EmotionRepository from app.repositories.neo4j.neo4j_connector import Neo4jConnector -from app.core.logging_config import get_business_logger +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session logger = get_business_logger() @@ -454,7 +455,7 @@ class EmotionAnalyticsService: async def generate_emotion_suggestions( self, end_user_id: str, - config_id: Optional[int] = None + db: Session, ) -> Dict[str, Any]: """生成个性化情绪建议 @@ -462,7 +463,7 @@ class EmotionAnalyticsService: Args: end_user_id: 宿主ID（用户组ID） - config_id: 配置ID（可选，用于从数据库加载LLM配置） + db: 数据库会话 Returns: Dict: 包含个性化建议的响应： @@ -470,14 +471,32 @@ class EmotionAnalyticsService: - suggestions: 建议列表（3-5条） """ try: - logger.info(f"生成个性化情绪建议: user={end_user_id}, config_id={config_id}") + logger.info(f"生成个性化情绪建议: user={end_user_id}") - # 1. 如果提供了 config_id，从数据库加载配置 - if config_id is not None: - from app.core.memory.utils.config.definitions import reload_configuration_from_database - config_loaded = reload_configuration_from_database(config_id) - if not config_loaded: - logger.warning(f"无法加载配置 config_id={config_id}，将使用默认配置") + # 1. 从 end_user_id 获取关联的 memory_config_id + llm_client = None + try: + from app.services.memory_agent_service import ( + get_end_user_connected_config, + ) + + connected_config = get_end_user_connected_config(end_user_id, db) + config_id = connected_config.get("memory_config_id") + + if config_id is not None: + from app.services.memory_config_service import ( + MemoryConfigService, + ) + config_service = MemoryConfigService(db) + memory_config = config_service.load_memory_config( + config_id=int(config_id), + service_name="EmotionAnalyticsService.generate_emotion_suggestions" + ) + from app.core.memory.client_factory import MemoryClientFactory + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client(str(memory_config.llm_model_id)) + except Exception as e: + logger.warning(f"无法获取 end_user {end_user_id} 的配置，将使用默认配置: {e}") # 2. 获取情绪健康数据 health_data = await self.calculate_emotion_health_index(end_user_id, time_range="30d") @@ -498,8 +517,9 @@ class EmotionAnalyticsService: prompt = await self._build_suggestion_prompt(health_data, patterns, user_profile) # 7. 调用LLM生成建议（使用配置中的LLM） - from app.core.memory.utils.llm.llm_utils import get_llm_client - llm_client = get_llm_client() + if llm_client is None: + # 无法获取配置时，抛出错误而不是使用默认配置 + raise ValueError("无法获取LLM配置，请确保end_user关联了有效的memory_config") # 将 prompt 转换为 messages 格式 messages = [ @@ -598,7 +618,9 @@ class EmotionAnalyticsService: Returns: str: LLM prompt """ - from app.core.memory.utils.prompt.prompt_utils import render_emotion_suggestions_prompt + from app.core.memory.utils.prompt.prompt_utils import ( + render_emotion_suggestions_prompt, + ) prompt = await render_emotion_suggestions_prompt( health_data=health_data, diff --git a/api/app/services/emotion_extraction_service.py b/api/app/services/emotion_extraction_service.py index b3172df1..d134251d 100644 --- a/api/app/services/emotion_extraction_service.py +++ b/api/app/services/emotion_extraction_service.py @@ -9,10 +9,12 @@ Classes: import logging from typing import Optional -from app.core.memory.models.emotion_models import EmotionExtraction -from app.models.data_config_model import DataConfig -from app.core.memory.utils.llm.llm_utils import get_llm_client + from app.core.memory.llm_tools.llm_client import LLMClientException +from app.core.memory.models.emotion_models import EmotionExtraction +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.db import get_db_context +from app.models.data_config_model import DataConfig logger = logging.getLogger(__name__) @@ -50,7 +52,9 @@ class EmotionExtractionService: """ if self.llm_client is None or model_id: effective_model_id = model_id or self.llm_id - self.llm_client = get_llm_client(effective_model_id) + with get_db_context() as db: + factory = MemoryClientFactory(db) + self.llm_client = factory.get_llm_client(effective_model_id) return self.llm_client async def extract_emotion( @@ -142,7 +146,9 @@ class EmotionExtractionService: Returns: Formatted prompt string for LLM """ - from app.core.memory.utils.prompt.prompt_utils import render_emotion_extraction_prompt + from app.core.memory.utils.prompt.prompt_utils import ( + render_emotion_extraction_prompt, + ) prompt = await render_emotion_extraction_prompt( statement=statement, diff --git a/api/app/services/memory_agent_service.py b/api/app/services/memory_agent_service.py index 27fdfa48..e23f9471 100644 --- a/api/app/services/memory_agent_service.py +++ b/api/app/services/memory_agent_service.py @@ -21,8 +21,8 @@ from app.core.memory.agent.logger_file.log_streamer import LogStreamer from app.core.memory.agent.utils.mcp_tools import get_mcp_server_config from app.core.memory.agent.utils.type_classifier import status_typle from app.core.memory.analytics.hot_memory_tags import get_hot_memory_tags -from app.core.memory.utils.llm.llm_utils import get_llm_client -from app.db import get_db +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.db import get_db_context from app.models.knowledge_model import Knowledge, KnowledgeType from app.repositories.neo4j.neo4j_connector import Neo4jConnector from app.schemas.memory_config_schema import ConfigurationError, MemoryConfig @@ -45,8 +45,7 @@ config_logger = get_config_logger() # Initialize Neo4j connector for analytics functions _neo4j_connector = Neo4jConnector() -db_gen = get_db() -db = next(db_gen) + class MemoryAgentService: """Service for memory agent operations""" @@ -55,27 +54,6 @@ class MemoryAgentService: self.user_locks: Dict[str, Lock] = {} self.locks_lock = Lock() - def load_memory_config(self, config_id: int) -> MemoryConfig: - """ - Load memory configuration from database by config_id. - - This method delegates to the centralized MemoryConfigService to avoid - code duplication with other services. - - Args: - config_id: Configuration ID from database - - Returns: - MemoryConfig: Immutable configuration object - - Raises: - ConfigurationError: If validation fails - """ - return MemoryConfigService.load_memory_config( - config_id=config_id, - service_name="MemoryAgentService" - ) - def writer_messages_deal(self,messages,start_time,group_id,config_id,message): messages = str(messages).replace("'", '"').replace('\\n', '').replace('\n', '').replace('\\', '') countext = re.findall(r'"status": "(.*?)",', messages)[0] @@ -277,14 +255,17 @@ class MemoryAgentService: logger.info("Log streaming completed, cleaning up resources") # LogStreamer uses context manager for file handling, so cleanup is automatic - async def write_memory(self, group_id: str, message: str, config_id: str,storage_type:str,user_rag_memory_id:str) -> str: + async def write_memory(self, group_id: str, message: str, config_id: str, db: Session, storage_type: str, user_rag_memory_id: str) -> str: """ Process write operation with config_id Args: - group_id: Group identifier + group_id: Group identifier (also used as end_user_id) message: Message to write config_id: Configuration ID from database + db: SQLAlchemy database session + storage_type: Storage type (neo4j or rag) + user_rag_memory_id: User RAG memory ID Returns: Write operation result status @@ -292,14 +273,24 @@ class MemoryAgentService: Raises: ValueError: If config loading fails or write operation fails """ - if config_id==None: - config_id = os.getenv("config_id") + # Resolve config_id if None using end_user's connected config + if config_id is None: + try: + connected_config = get_end_user_connected_config(group_id, db) + config_id = connected_config.get("memory_config_id") + except Exception as e: + logger.warning(f"Failed to get connected config for end_user {group_id}: {e}") + import time start_time = time.time() # Load configuration from database only try: - memory_config = self.load_memory_config(config_id) + config_service = MemoryConfigService(db) + memory_config = config_service.load_memory_config( + config_id=config_id, + service_name="MemoryAgentService" + ) logger.info(f"Configuration loaded successfully: {memory_config.config_name}") except ConfigurationError as e: error_msg = f"Failed to load configuration for config_id: {config_id}: {e}" @@ -366,6 +357,7 @@ class MemoryAgentService: history: List[Dict], search_switch: str, config_id: str, + db: Session, storage_type: str, user_rag_memory_id: str ) -> Dict: @@ -378,11 +370,14 @@ class MemoryAgentService: - "2": Direct answer based on context Args: - group_id: Group identifier + group_id: Group identifier (also used as end_user_id) message: User message history: Conversation history search_switch: Search mode switch config_id: Configuration ID from database + db: SQLAlchemy database session + storage_type: Storage type (neo4j or rag) + user_rag_memory_id: User RAG memory ID Returns: Dict with 'answer' and 'intermediate_outputs' keys @@ -394,8 +389,13 @@ class MemoryAgentService: import time start_time = time.time() - if config_id==None: - config_id = os.getenv("config_id") + # Resolve config_id if None using end_user's connected config + if config_id is None: + try: + connected_config = get_end_user_connected_config(group_id, db) + config_id = connected_config.get("memory_config_id") + except Exception as e: + logger.warning(f"Failed to get connected config for end_user {group_id}: {e}") logger.info(f"Read operation for group {group_id} with config_id {config_id}") @@ -411,7 +411,11 @@ class MemoryAgentService: with group_lock: # Step 1: Load configuration from database only try: - memory_config = self.load_memory_config(config_id) + config_service = MemoryConfigService(db) + memory_config = config_service.load_memory_config( + config_id=config_id, + service_name="MemoryAgentService" + ) logger.info(f"Configuration loaded successfully: {memory_config.config_name}") except ConfigurationError as e: error_msg = f"Failed to load configuration for config_id: {config_id}: {e}" @@ -696,7 +700,11 @@ class MemoryAgentService: logger.info("Classifying message type") # Load configuration to get LLM model ID - memory_config = self.load_memory_config(config_id) + config_service = MemoryConfigService(db) + memory_config = config_service.load_memory_config( + config_id=config_id, + service_name="MemoryAgentService" + ) status = await status_typle(message, memory_config.llm_model_id) logger.debug(f"Message type: {status}") @@ -865,7 +873,8 @@ class MemoryAgentService: self, end_user_id: Optional[str] = None, current_user_id: Optional[str] = None, - llm_id: Optional[str] = None + llm_id: Optional[str] = None, + db: Session = None ) -> Dict[str, Any]: """ 获取用户详情，包含： @@ -877,6 +886,7 @@ class MemoryAgentService: - end_user_id: 用户ID（可选） - current_user_id: 当前登录用户的ID（保留参数） - llm_id: LLM模型ID（用于生成标签，可选，如果不提供则跳过标签生成） + - db: 数据库会话（可选） 返回格式： { @@ -893,7 +903,7 @@ class MemoryAgentService: # 1. 根据 end_user_id 获取 end_user_name try: - if end_user_id: + if end_user_id and db: from app.repositories import end_user_repository from app.schemas.end_user_schema import EndUser as EndUserSchema @@ -948,7 +958,9 @@ class MemoryAgentService: logger.debug(f"User data found: {len(statements)} statements, {len(entities)} entities") # 使用LLM提取标签 - llm_client = get_llm_client(llm_id) + with get_db_context() as db: + factory = MemoryClientFactory(db) + llm_client = factory.get_llm_client(llm_id) # 定义标签提取的结构 class UserTags(BaseModel): @@ -1110,4 +1122,69 @@ class MemoryAgentService: # "msg": "解析失败", # "error_code": "DOC_PARSE_ERROR", # "data": {"error": str(e)} -# } \ No newline at end of file +# } + + +def get_end_user_connected_config(end_user_id: str, db: Session) -> Dict[str, Any]: + """ + 获取终端用户关联的记忆配置 + + 通过以下流程获取配置： + 1. 根据 end_user_id 获取用户的 app_id + 2. 获取该应用的最新发布版本 + 3. 从发布版本的 config 字段中提取 memory_config_id + + Args: + end_user_id: 终端用户ID + db: 数据库会话 + + Returns: + 包含 memory_config_id 和相关信息的字典 + + Raises: + ValueError: 当终端用户不存在或应用未发布时 + """ + from app.models.app_release_model import AppRelease + from app.models.end_user_model import EndUser + from sqlalchemy import select + + logger.info(f"Getting connected config for end_user: {end_user_id}") + + # 1. 获取 end_user 及其 app_id + end_user = db.query(EndUser).filter(EndUser.id == end_user_id).first() + if not end_user: + logger.warning(f"End user not found: {end_user_id}") + raise ValueError(f"终端用户不存在: {end_user_id}") + + app_id = end_user.app_id + logger.debug(f"Found end_user app_id: {app_id}") + + # 2. 获取该应用的最新发布版本 + stmt = ( + select(AppRelease) + .where(AppRelease.app_id == app_id, AppRelease.is_active.is_(True)) + .order_by(AppRelease.version.desc()) + ) + latest_release = db.scalars(stmt).first() + + if not latest_release: + logger.warning(f"No active release found for app: {app_id}") + raise ValueError(f"应用未发布: {app_id}") + + logger.debug(f"Found latest release: version={latest_release.version}, id={latest_release.id}") + + # 3. 从 config 中提取 memory_config_id + config = latest_release.config or {} + memory_obj = config.get('memory', {}) + memory_config_id = memory_obj.get('memory_content') if isinstance(memory_obj, dict) else None + + result = { + "end_user_id": str(end_user_id), + "app_id": str(app_id), + "release_id": str(latest_release.id), + "release_version": latest_release.version, + "memory_config_id": memory_config_id + } + + logger.info(f"Successfully retrieved connected config: memory_config_id={memory_config_id}") + return result \ No newline at end of file diff --git a/api/app/services/memory_config_service.py b/api/app/services/memory_config_service.py index d19eb02a..3413ebd6 100644 --- a/api/app/services/memory_config_service.py +++ b/api/app/services/memory_config_service.py @@ -3,7 +3,6 @@ Memory Configuration Service Centralized configuration loading and management for memory services. This service eliminates code duplication between MemoryAgentService and MemoryStorageService. -Database session management is handled internally. """ import time @@ -57,7 +56,7 @@ def _validate_config_id(config_id): invalid_value=config_id, ) return parsed_id - except ValueError as e: + except ValueError: raise InvalidConfigError( f"Invalid configuration ID format: '{config_id}'", field_name="config_id", @@ -77,19 +76,29 @@ class MemoryConfigService: This class provides a single implementation of configuration loading logic that can be shared across multiple services, eliminating code duplication. - Database session management is handled internally. + + Usage: + config_service = MemoryConfigService(db) + memory_config = config_service.load_memory_config(config_id) + model_config = config_service.get_model_config(model_id) """ - @staticmethod + def __init__(self, db: Session): + """Initialize the service with a database session. + + Args: + db: SQLAlchemy database session + """ + self.db = db + def load_memory_config( + self, config_id: int, service_name: str = "MemoryConfigService", ) -> MemoryConfig: """ Load memory configuration from database by config_id. - This method manages its own database session internally. - Args: config_id: Configuration ID from database service_name: Name of the calling service (for logging purposes) @@ -100,27 +109,6 @@ class MemoryConfigService: Raises: ConfigurationError: If validation fails """ - from app.db import get_db - - db_gen = get_db() - db = next(db_gen) - - try: - return MemoryConfigService._load_memory_config_with_db( - config_id=config_id, - db=db, - service_name=service_name, - ) - finally: - db.close() - - @staticmethod - def _load_memory_config_with_db( - config_id: int, - db: Session, - service_name: str = "MemoryConfigService", - ) -> MemoryConfig: - """Internal method that loads memory configuration with an existing db session.""" start_time = time.time() config_logger.info( @@ -137,7 +125,7 @@ class MemoryConfigService: try: validated_config_id = _validate_config_id(config_id) - result = DataConfigRepository.get_config_with_workspace(db, validated_config_id) + result = DataConfigRepository.get_config_with_workspace(self.db, validated_config_id) if not result: elapsed_ms = (time.time() - start_time) * 1000 config_logger.error( @@ -160,7 +148,7 @@ class MemoryConfigService: embedding_uuid = validate_embedding_model( validated_config_id, memory_config.embedding_id, - db, + self.db, workspace.tenant_id, workspace.id, ) @@ -169,7 +157,7 @@ class MemoryConfigService: llm_uuid, llm_name = validate_and_resolve_model_id( memory_config.llm_id, "llm", - db, + self.db, workspace.tenant_id, required=True, config_id=validated_config_id, @@ -183,7 +171,7 @@ class MemoryConfigService: rerank_uuid, rerank_name = validate_and_resolve_model_id( memory_config.rerank_id, "rerank", - db, + self.db, workspace.tenant_id, required=False, config_id=validated_config_id, @@ -194,7 +182,7 @@ class MemoryConfigService: embedding_name, _ = validate_model_exists_and_active( embedding_uuid, "embedding", - db, + self.db, workspace.tenant_id, config_id=validated_config_id, workspace_id=workspace.id, @@ -220,6 +208,25 @@ class MemoryConfigService: reflexion_range=memory_config.reflexion_range or "retrieval", reflexion_baseline=memory_config.baseline or "time", loaded_at=datetime.now(), + # Pipeline config: Deduplication + enable_llm_dedup_blockwise=bool(memory_config.enable_llm_dedup_blockwise) if memory_config.enable_llm_dedup_blockwise is not None else False, + enable_llm_disambiguation=bool(memory_config.enable_llm_disambiguation) if memory_config.enable_llm_disambiguation is not None else False, + deep_retrieval=bool(memory_config.deep_retrieval) if memory_config.deep_retrieval is not None else True, + t_type_strict=float(memory_config.t_type_strict) if memory_config.t_type_strict is not None else 0.8, + t_name_strict=float(memory_config.t_name_strict) if memory_config.t_name_strict is not None else 0.8, + t_overall=float(memory_config.t_overall) if memory_config.t_overall is not None else 0.8, + # Pipeline config: Statement extraction + statement_granularity=int(memory_config.statement_granularity) if memory_config.statement_granularity is not None else 2, + include_dialogue_context=bool(memory_config.include_dialogue_context) if memory_config.include_dialogue_context is not None else False, + max_dialogue_context_chars=int(memory_config.max_context) if memory_config.max_context is not None else 1000, + # Pipeline config: Forgetting engine + lambda_time=float(memory_config.lambda_time) if memory_config.lambda_time is not None else 0.5, + lambda_mem=float(memory_config.lambda_mem) if memory_config.lambda_mem is not None else 0.5, + offset=float(memory_config.offset) if memory_config.offset is not None else 0.0, + # Pipeline config: Pruning + pruning_enabled=bool(memory_config.pruning_enabled) if memory_config.pruning_enabled is not None else False, + pruning_scene=memory_config.pruning_scene or "education", + pruning_threshold=float(memory_config.pruning_threshold) if memory_config.pruning_threshold is not None else 0.5, ) elapsed_ms = (time.time() - start_time) * 1000 @@ -262,3 +269,131 @@ class MemoryConfigService: raise else: raise ConfigurationError(f"Failed to load configuration {config_id}: {e}") + + def get_model_config(self, model_id: str) -> dict: + """Get LLM model configuration by ID. + + Args: + model_id: Model ID to look up + + Returns: + Dict with model configuration including api_key, base_url, etc. + """ + from app.core.config import settings + from app.models.models_model import ModelApiKey + from app.services.model_service import ModelConfigService as ModelSvc + from fastapi import status + from fastapi.exceptions import HTTPException + + config = ModelSvc.get_model_by_id(db=self.db, model_id=model_id) + if not config: + logger.warning(f"Model ID {model_id} not found") + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="模型ID不存在") + + api_config: ModelApiKey = config.api_keys[0] + + return { + "model_name": api_config.model_name, + "provider": api_config.provider, + "api_key": api_config.api_key, + "base_url": api_config.api_base, + "model_config_id": api_config.model_config_id, + "type": config.type, + "timeout": settings.LLM_TIMEOUT, + "max_retries": settings.LLM_MAX_RETRIES, + } + + def get_embedder_config(self, embedding_id: str) -> dict: + """Get embedding model configuration by ID. + + Args: + embedding_id: Embedding model ID to look up + + Returns: + Dict with embedder configuration including api_key, base_url, etc. + """ + from app.models.models_model import ModelApiKey + from app.services.model_service import ModelConfigService as ModelSvc + from fastapi import status + from fastapi.exceptions import HTTPException + + config = ModelSvc.get_model_by_id(db=self.db, model_id=embedding_id) + if not config: + logger.warning(f"Embedding model ID {embedding_id} not found") + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="嵌入模型ID不存在") + + api_config: ModelApiKey = config.api_keys[0] + + return { + "model_name": api_config.model_name, + "provider": api_config.provider, + "api_key": api_config.api_key, + "base_url": api_config.api_base, + "model_config_id": api_config.model_config_id, + "type": config.type, + "timeout": 120.0, + "max_retries": 5, + } + + @staticmethod + def get_pipeline_config(memory_config: MemoryConfig): + """Build ExtractionPipelineConfig from MemoryConfig. + + Args: + memory_config: MemoryConfig object containing all pipeline settings. + + Returns: + ExtractionPipelineConfig with deduplication, statement extraction, + and forgetting engine settings. + """ + from app.core.memory.models.variate_config import ( + DedupConfig, + ExtractionPipelineConfig, + ForgettingEngineConfig, + StatementExtractionConfig, + ) + + dedup_config = DedupConfig( + enable_llm_dedup_blockwise=memory_config.enable_llm_dedup_blockwise, + enable_llm_disambiguation=memory_config.enable_llm_disambiguation, + fuzzy_name_threshold_strict=memory_config.t_name_strict, + fuzzy_type_threshold_strict=memory_config.t_type_strict, + fuzzy_overall_threshold=memory_config.t_overall, + ) + + stmt_config = StatementExtractionConfig( + statement_granularity=memory_config.statement_granularity, + include_dialogue_context=memory_config.include_dialogue_context, + max_dialogue_context_chars=memory_config.max_dialogue_context_chars, + ) + + forget_config = ForgettingEngineConfig( + offset=memory_config.offset, + lambda_time=memory_config.lambda_time, + lambda_mem=memory_config.lambda_mem, + ) + + return ExtractionPipelineConfig( + statement_extraction=stmt_config, + deduplication=dedup_config, + forgetting_engine=forget_config, + ) + + @staticmethod + def get_pruning_config(memory_config: MemoryConfig) -> dict: + """Retrieve semantic pruning config from MemoryConfig. + + Args: + memory_config: MemoryConfig object containing pruning settings. + + Returns: + Dict suitable for PruningConfig.model_validate with keys: + - pruning_switch: bool + - pruning_scene: str + - pruning_threshold: float + """ + return { + "pruning_switch": memory_config.pruning_enabled, + "pruning_scene": memory_config.pruning_scene, + "pruning_threshold": memory_config.pruning_threshold, + } diff --git a/api/app/services/memory_storage_service.py b/api/app/services/memory_storage_service.py index bee3d22a..c88dd1d5 100644 --- a/api/app/services/memory_storage_service.py +++ b/api/app/services/memory_storage_service.py @@ -49,27 +49,6 @@ class MemoryStorageService: def __init__(self): logger.info("MemoryStorageService initialized") - - def load_memory_config(self, config_id: int, db: Session) -> MemoryConfig: - """ - Load memory configuration from database by config_id. - - This method delegates to the centralized MemoryConfigService to avoid - code duplication with other services. - - Args: - config_id: Configuration ID from database - - Returns: - MemoryConfig: Immutable configuration object - - Raises: - ConfigurationError: If validation fails - """ - return MemoryConfigService.load_memory_config( - config_id=config_id, - service_name="MemoryStorageService" - ) async def get_storage_info(self) -> dict: """ @@ -293,7 +272,8 @@ class DataConfigService: # 数据配置服务类（PostgreSQL） # Load configuration from database only using centralized manager try: - memory_config = MemoryConfigService.load_memory_config( + config_service = MemoryConfigService(self.db) + memory_config = config_service.load_memory_config( config_id=int(cid), service_name="MemoryStorageService.pilot_run_stream" ) @@ -320,13 +300,14 @@ class DataConfigService: # 数据配置服务类（PostgreSQL） async def run_pipeline(): """在后台执行管线并捕获异常""" try: - from app.core.memory.main import main as pipeline_main + from app.services.pilot_run_service import run_pilot_extraction - logger.info(f"[PILOT_RUN_STREAM] Calling pipeline_main with dialogue_text length: {len(dialogue_text)}, is_pilot_run=True") - await pipeline_main( - dialogue_text=dialogue_text, - is_pilot_run=True, - progress_callback=progress_callback + logger.info(f"[PILOT_RUN_STREAM] Calling run_pilot_extraction with dialogue_text length: {len(dialogue_text)}") + await run_pilot_extraction( + memory_config=memory_config, + dialogue_text=dialogue_text, + db=self.db, + progress_callback=progress_callback, ) logger.info("[PILOT_RUN_STREAM] pipeline_main completed") diff --git a/api/app/services/pilot_run_service.py b/api/app/services/pilot_run_service.py new file mode 100644 index 00000000..17dfd7eb --- /dev/null +++ b/api/app/services/pilot_run_service.py @@ -0,0 +1,219 @@ +""" +Pilot Run Service - 试运行服务 + +用于执行记忆系统的试运行流程，不保存到 Neo4j。 +""" + +import os +import re +import time +from datetime import datetime +from typing import Awaitable, Callable, Optional + +from app.core.logging_config import get_memory_logger, log_time +from app.core.memory.models.message_models import ( + ConversationContext, + ConversationMessage, + DialogData, +) +from app.core.memory.storage_services.extraction_engine.extraction_orchestrator import ( + ExtractionOrchestrator, + get_chunked_dialogs_from_preprocessed, +) +from app.core.memory.utils.config.config_utils import ( + get_pipeline_config, +) +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from app.schemas.memory_config_schema import MemoryConfig +from sqlalchemy.orm import Session + +logger = get_memory_logger(__name__) + + +async def run_pilot_extraction( + memory_config: MemoryConfig, + dialogue_text: str, + db: Session, + progress_callback: Optional[Callable[[str, str, Optional[dict]], Awaitable[None]]] = None, +) -> None: + """ + 执行试运行模式的知识提取流水线。 + + Args: + memory_config: 从数据库加载的内存配置对象 + dialogue_text: 输入的对话文本 + progress_callback: 可选的进度回调函数 + - 参数1 (stage): 当前处理阶段标识符 + - 参数2 (message): 人类可读的进度消息 + - 参数3 (data): 可选的附加数据字典 + """ + log_file = "logs/time.log" + os.makedirs(os.path.dirname(log_file), exist_ok=True) + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + with open(log_file, "a", encoding="utf-8") as f: + f.write(f"\n=== Pilot Run Started: {timestamp} ===\n") + + pipeline_start = time.time() + neo4j_connector = None + + try: + # 步骤 1: 初始化客户端 + logger.info("Initializing clients...") + step_start = time.time() + + client_factory = MemoryClientFactory(db) + llm_client = client_factory.get_llm_client(str(memory_config.llm_model_id)) + embedder_client = client_factory.get_embedder_client(str(memory_config.embedding_model_id)) + + neo4j_connector = Neo4jConnector() + + log_time("Client Initialization", time.time() - step_start, log_file) + + # 步骤 2: 解析对话文本 + logger.info("Parsing dialogue text...") + step_start = time.time() + + # 解析对话文本，支持 "用户:" 和 "AI:" 格式 + pattern = r"(用户|AI)[：:]\s*([^\n]+(?:\n(?!(?:用户|AI)[：:])[^\n]*)*?)" + matches = re.findall(pattern, dialogue_text, re.MULTILINE | re.DOTALL) + messages = [ + ConversationMessage(role=r, msg=c.strip()) + for r, c in matches + if c.strip() + ] + + # 如果没有匹配到格式化的对话，将整个文本作为用户消息 + if not messages: + messages = [ConversationMessage(role="用户", msg=dialogue_text.strip())] + + context = ConversationContext(msgs=messages) + dialog = DialogData( + context=context, + ref_id="pilot_dialog_1", + group_id=str(memory_config.workspace_id), + user_id=str(memory_config.tenant_id), + apply_id=str(memory_config.config_id), + metadata={"source": "pilot_run", "input_type": "frontend_text"}, + ) + + if progress_callback: + await progress_callback("text_preprocessing", "开始预处理文本...") + + chunked_dialogs = await get_chunked_dialogs_from_preprocessed( + data=[dialog], + chunker_strategy=memory_config.chunker_strategy, + llm_client=llm_client, + ) + logger.info(f"Processed dialogue text: {len(messages)} messages") + + # 进度回调：输出每个分块的结果 + if progress_callback: + for dlg in chunked_dialogs: + for i, chunk in enumerate(dlg.chunks): + chunk_result = { + "chunk_index": i + 1, + "content": chunk.content[:200] + "..." if len(chunk.content) > 200 else chunk.content, + "full_length": len(chunk.content), + "dialog_id": dlg.id, + "chunker_strategy": memory_config.chunker_strategy, + } + await progress_callback("text_preprocessing_result", f"分块 {i + 1} 处理完成", chunk_result) + + preprocessing_summary = { + "total_chunks": sum(len(dlg.chunks) for dlg in chunked_dialogs), + "total_dialogs": len(chunked_dialogs), + "chunker_strategy": memory_config.chunker_strategy, + } + await progress_callback("text_preprocessing_complete", "预处理文本完成", preprocessing_summary) + + log_time("Data Loading & Chunking", time.time() - step_start, log_file) + + # 步骤 3: 初始化流水线编排器 + logger.info("Initializing extraction orchestrator...") + step_start = time.time() + + config = get_pipeline_config(memory_config) + logger.info( + f"Pipeline config loaded: enable_llm_dedup_blockwise={config.deduplication.enable_llm_dedup_blockwise}, " + f"enable_llm_disambiguation={config.deduplication.enable_llm_disambiguation}" + ) + + orchestrator = ExtractionOrchestrator( + llm_client=llm_client, + embedder_client=embedder_client, + connector=neo4j_connector, + config=config, + progress_callback=progress_callback, + embedding_id=str(memory_config.embedding_model_id), + ) + + log_time("Orchestrator Initialization", time.time() - step_start, log_file) + + # 步骤 4: 执行知识提取流水线 + logger.info("Running extraction pipeline...") + step_start = time.time() + + if progress_callback: + await progress_callback("knowledge_extraction", "正在知识抽取...") + + extraction_result = await orchestrator.run( + dialog_data_list=chunked_dialogs, + is_pilot_run=True, + ) + + # 解包 extraction_result tuple (与 main.py 保持一致) + ( + dialogue_nodes, + chunk_nodes, + statement_nodes, + entity_nodes, + statement_chunk_edges, + statement_entity_edges, + entity_edges, + ) = extraction_result + + log_time("Extraction Pipeline", time.time() - step_start, log_file) + + if progress_callback: + await progress_callback("generating_results", "正在生成结果...") + + # 步骤 5: 生成记忆摘要（与 main.py 保持一致） + try: + logger.info("Generating memory summaries...") + step_start = time.time() + + from app.core.memory.storage_services.extraction_engine.knowledge_extraction.memory_summary import ( + memory_summary_generation, + ) + + summaries = await memory_summary_generation( + chunked_dialogs, + llm_client=llm_client, + embedder_client=embedder_client, + ) + + log_time("Memory Summary Generation", time.time() - step_start, log_file) + except Exception as e: + logger.error(f"Memory summary step failed: {e}", exc_info=True) + + logger.info("Pilot run completed: Skipping Neo4j save") + + except Exception as e: + logger.error(f"Pilot run failed: {e}", exc_info=True) + raise + finally: + if neo4j_connector: + try: + await neo4j_connector.close() + except Exception: + pass + + total_time = time.time() - pipeline_start + log_time("TOTAL PILOT RUN TIME", total_time, log_file) + + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + with open(log_file, "a", encoding="utf-8") as f: + f.write(f"=== Pilot Run Completed: {timestamp} ===\n\n") + + logger.info(f"Pilot run complete. Total time: {total_time:.2f}s") diff --git a/api/app/tasks.py b/api/app/tasks.py index c4d9fc10..362172f0 100644 --- a/api/app/tasks.py +++ b/api/app/tasks.py @@ -176,7 +176,7 @@ def read_message_task(self, group_id: str, message: str, history: List[Dict[str, """Celery task to process a read message via MemoryAgentService. Args: - group_id: Group ID for the memory agent + group_id: Group ID for the memory agent (also used as end_user_id) message: User message to process history: Conversation history search_switch: Search switch parameter @@ -190,9 +190,28 @@ def read_message_task(self, group_id: str, message: str, history: List[Dict[str, """ start_time = time.time() + # Resolve config_id if None + actual_config_id = config_id + if actual_config_id is None: + try: + from app.services.memory_agent_service import get_end_user_connected_config + db = next(get_db()) + try: + connected_config = get_end_user_connected_config(group_id, db) + actual_config_id = connected_config.get("memory_config_id") + finally: + db.close() + except Exception as e: + # Log but continue - will fail later with proper error + pass + async def _run() -> str: - service = MemoryAgentService() - return await service.read_memory(group_id, message, history, search_switch, config_id,storage_type,user_rag_memory_id) + db = next(get_db()) + try: + service = MemoryAgentService() + return await service.read_memory(group_id, message, history, search_switch, actual_config_id, db, storage_type, user_rag_memory_id) + finally: + db.close() try: # 使用 nest_asyncio 来避免事件循环冲突 @@ -246,7 +265,7 @@ def write_message_task(self, group_id: str, message: str, config_id: str,storage """Celery task to process a write message via MemoryAgentService. Args: - group_id: Group ID for the memory agent + group_id: Group ID for the memory agent (also used as end_user_id) message: Message to write config_id: Optional configuration ID @@ -258,9 +277,28 @@ def write_message_task(self, group_id: str, message: str, config_id: str,storage """ start_time = time.time() + # Resolve config_id if None + actual_config_id = config_id + if actual_config_id is None: + try: + from app.services.memory_agent_service import get_end_user_connected_config + db = next(get_db()) + try: + connected_config = get_end_user_connected_config(group_id, db) + actual_config_id = connected_config.get("memory_config_id") + finally: + db.close() + except Exception as e: + # Log but continue - will fail later with proper error + pass + async def _run() -> str: - service = MemoryAgentService() - return await service.write_memory(group_id, message, config_id,storage_type,user_rag_memory_id) + db = next(get_db()) + try: + service = MemoryAgentService() + return await service.write_memory(group_id, message, actual_config_id, db, storage_type, user_rag_memory_id) + finally: + db.close() try: # 使用 nest_asyncio 来避免事件循环冲突 From b515e76efb444b4b5a47db9485b05280722489cb Mon Sep 17 00:00:00 2001 From: Ke Sun Date: Tue, 23 Dec 2025 18:27:15 +0800 Subject: [PATCH 03/12] refactor(memory): reorganize imports and move MemoryClientFactory to utils - Move MemoryClientFactory from app.core.memory.client_factory to app.core.memory.utils.llm.llm_utils - Update all evaluation modules to import MemoryClientFactory from new location (locomo, longmemeval, memsciqa) - Move GenerateCacheRequest from memory_storage_service to memory_storage_schema - Update memory_storage_controller imports to reflect schema reorganization - Add analytics_user_summary import to memory_storage_controller from user_memory_service - Consolidate utility imports across evaluation test files for consistency - Improve code organization by centralizing LLM utilities in dedicated utils module --- api/app/controllers/memory_storage_controller.py | 3 ++- .../memory/evaluation/locomo/locomo_benchmark.py | 2 +- api/app/core/memory/evaluation/locomo/locomo_test.py | 2 +- .../memory/evaluation/locomo/qwen_search_eval.py | 2 +- .../evaluation/longmemeval/qwen_search_eval.py | 2 +- .../core/memory/evaluation/longmemeval/test_eval.py | 2 +- .../core/memory/evaluation/memsciqa/evaluate_qa.py | 2 +- .../core/memory/evaluation/memsciqa/memsciqa-test.py | 2 +- .../reflection_engine/self_reflexion.py | 12 ++++++++---- api/app/services/emotion_analytics_service.py | 2 +- api/app/services/user_memory_service.py | 11 +++++------ 11 files changed, 23 insertions(+), 19 deletions(-) diff --git a/api/app/controllers/memory_storage_controller.py b/api/app/controllers/memory_storage_controller.py index 022615ff..e8e725d8 100644 --- a/api/app/controllers/memory_storage_controller.py +++ b/api/app/controllers/memory_storage_controller.py @@ -23,11 +23,11 @@ from app.schemas.memory_storage_schema import ( ConfigUpdate, ConfigUpdateExtracted, ConfigUpdateForget, + GenerateCacheRequest, ) from app.schemas.response_schema import ApiResponse from app.services.memory_storage_service import ( DataConfigService, - GenerateCacheRequest, MemoryStorageService, analytics_hot_memory_tags, analytics_recent_activity_stats, @@ -41,6 +41,7 @@ from app.services.memory_storage_service import ( search_entity_graph, search_statement, ) +from app.services.user_memory_service import analytics_user_summary from fastapi import APIRouter, Depends from fastapi.responses import StreamingResponse from sqlalchemy.orm import Session diff --git a/api/app/core/memory/evaluation/locomo/locomo_benchmark.py b/api/app/core/memory/evaluation/locomo/locomo_benchmark.py index 4992aa29..b7d988c5 100644 --- a/api/app/core/memory/evaluation/locomo/locomo_benchmark.py +++ b/api/app/core/memory/evaluation/locomo/locomo_benchmark.py @@ -23,7 +23,6 @@ except ImportError: def load_dotenv(): pass -from app.core.memory.client_factory import MemoryClientFactory from app.core.memory.evaluation.common.metrics import ( avg_context_tokens, bleu1, @@ -51,6 +50,7 @@ from app.core.memory.utils.definitions import ( SELECTED_GROUP_ID, SELECTED_LLM_ID, ) +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory from app.core.models.base import RedBearModelConfig from app.db import get_db_context from app.repositories.neo4j.neo4j_connector import Neo4jConnector diff --git a/api/app/core/memory/evaluation/locomo/locomo_test.py b/api/app/core/memory/evaluation/locomo/locomo_test.py index a8fa1820..b5ad5820 100644 --- a/api/app/core/memory/evaluation/locomo/locomo_test.py +++ b/api/app/core/memory/evaluation/locomo/locomo_test.py @@ -435,12 +435,12 @@ async def run_enhanced_evaluation(): return None # 修正导入路径：使用 app.core.memory.src 前缀 - from app.core.memory.client_factory import MemoryClientFactory from app.core.memory.llm_tools.openai_embedder import OpenAIEmbedderClient from app.core.memory.utils.config.definitions import ( SELECTED_EMBEDDING_ID, SELECTED_LLM_ID, ) + from app.core.memory.utils.llm.llm_utils import MemoryClientFactory from app.core.models.base import RedBearModelConfig from app.db import get_db_context from app.repositories.neo4j.graph_search import search_graph_by_embedding diff --git a/api/app/core/memory/evaluation/locomo/qwen_search_eval.py b/api/app/core/memory/evaluation/locomo/qwen_search_eval.py index e7a95e71..87a70a29 100644 --- a/api/app/core/memory/evaluation/locomo/qwen_search_eval.py +++ b/api/app/core/memory/evaluation/locomo/qwen_search_eval.py @@ -15,7 +15,6 @@ except Exception: import re -from app.core.memory.client_factory import MemoryClientFactory from app.core.memory.evaluation.common.metrics import ( avg_context_tokens, bleu1, @@ -34,6 +33,7 @@ from app.core.memory.utils.config.definitions import ( SELECTED_GROUP_ID, SELECTED_LLM_ID, ) +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory from app.core.models.base import RedBearModelConfig from app.db import get_db_context from app.repositories.neo4j.graph_search import search_graph, search_graph_by_embedding diff --git a/api/app/core/memory/evaluation/longmemeval/qwen_search_eval.py b/api/app/core/memory/evaluation/longmemeval/qwen_search_eval.py index 58652033..53c5ce19 100644 --- a/api/app/core/memory/evaluation/longmemeval/qwen_search_eval.py +++ b/api/app/core/memory/evaluation/longmemeval/qwen_search_eval.py @@ -34,7 +34,6 @@ try: ) except Exception: ingest_contexts_via_full_pipeline = None # 在运行时做兜底检查 -from app.core.memory.client_factory import MemoryClientFactory from app.core.memory.evaluation.common.metrics import ( avg_context_tokens, jaccard, @@ -48,6 +47,7 @@ from app.core.memory.utils.config.definitions import ( SELECTED_EMBEDDING_ID, SELECTED_LLM_ID, ) +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory from app.core.models.base import RedBearModelConfig from app.db import get_db_context from app.repositories.neo4j.graph_search import search_graph, search_graph_by_embedding diff --git a/api/app/core/memory/evaluation/longmemeval/test_eval.py b/api/app/core/memory/evaluation/longmemeval/test_eval.py index a0038260..08a763e3 100644 --- a/api/app/core/memory/evaluation/longmemeval/test_eval.py +++ b/api/app/core/memory/evaluation/longmemeval/test_eval.py @@ -15,7 +15,6 @@ except Exception: return None # 与现有评估脚本保持一致的导入方式 -from app.core.memory.client_factory import MemoryClientFactory from app.core.memory.evaluation.common.metrics import ( avg_context_tokens, jaccard, @@ -29,6 +28,7 @@ from app.core.memory.utils.config.definitions import ( SELECTED_EMBEDDING_ID, SELECTED_LLM_ID, ) +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory from app.core.models.base import RedBearModelConfig from app.db import get_db_context from app.repositories.neo4j.graph_search import search_graph, search_graph_by_embedding diff --git a/api/app/core/memory/evaluation/memsciqa/evaluate_qa.py b/api/app/core/memory/evaluation/memsciqa/evaluate_qa.py index 3e6a1216..6efb66ff 100644 --- a/api/app/core/memory/evaluation/memsciqa/evaluate_qa.py +++ b/api/app/core/memory/evaluation/memsciqa/evaluate_qa.py @@ -15,7 +15,6 @@ except Exception: def load_dotenv(): return None -from app.core.memory.client_factory import MemoryClientFactory from app.core.memory.evaluation.common.metrics import ( avg_context_tokens, exact_match, @@ -31,6 +30,7 @@ from app.core.memory.utils.config.definitions import ( SELECTED_GROUP_ID, SELECTED_LLM_ID, ) +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory from app.db import get_db_context from app.repositories.neo4j.neo4j_connector import Neo4jConnector diff --git a/api/app/core/memory/evaluation/memsciqa/memsciqa-test.py b/api/app/core/memory/evaluation/memsciqa/memsciqa-test.py index ebbe6e7e..279f4042 100644 --- a/api/app/core/memory/evaluation/memsciqa/memsciqa-test.py +++ b/api/app/core/memory/evaluation/memsciqa/memsciqa-test.py @@ -24,7 +24,6 @@ for _p in (_SRC_DIR, _PROJECT_ROOT): sys.path.insert(0, _p) # 对齐 locomo_test 的检索逻辑：直接使用 graph_search 与 Neo4jConnector/Embedder1 -from app.core.memory.client_factory import MemoryClientFactory from app.core.memory.evaluation.common.metrics import ( avg_context_tokens, exact_match, @@ -37,6 +36,7 @@ from app.core.memory.utils.config.definitions import ( SELECTED_GROUP_ID, SELECTED_LLM_ID, ) +from app.core.memory.utils.llm.llm_utils import MemoryClientFactory from app.core.models.base import RedBearModelConfig from app.db import get_db_context from app.repositories.neo4j.graph_search import search_graph, search_graph_by_embedding diff --git a/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py b/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py index 34fcaf79..ebbb97f7 100644 --- a/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py +++ b/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py @@ -19,7 +19,6 @@ from typing import Any, Dict, List, Optional from app.core.memory.llm_tools.openai_client import OpenAIClient from app.core.memory.utils.config import definitions as config_defs -from app.core.memory.utils.config import get_model_config from app.core.memory.utils.config.get_data import ( extract_and_process_changes, get_data, @@ -169,12 +168,18 @@ class ReflectionEngine: self.llm_client = factory.get_llm_client(config_defs.SELECTED_LLM_ID) elif isinstance(self.llm_client, str): # 如果 llm_client 是字符串（model_id），则用它初始化客户端 - # from app.core.memory.utils.llm.llm_utils import MemoryClientFactory + from app.core.memory.utils.llm.llm_utils import MemoryClientFactory from app.db import get_db_context - # model_id = self.llm_client + from app.services.memory_config_service import MemoryConfigService + model_id = self.llm_client with get_db_context() as db: factory = MemoryClientFactory(db) # self.llm_client = factory.get_llm_client(model_id) + + # Use MemoryConfigService to get model config + config_service = MemoryConfigService(db) + model_config = config_service.get_model_config(model_id) + extra_params={ "temperature": 0.2, # 降低温度提高响应速度和一致性 "max_tokens": 600, # 限制最大token数 @@ -182,7 +187,6 @@ class ReflectionEngine: "stream": False, # 确保非流式输出以获得最快响应 } - model_config = get_model_config(self.llm_client) self.llm_client = OpenAIClient(RedBearModelConfig( model_name=model_config.get("model_name"), provider=model_config.get("provider"), diff --git a/api/app/services/emotion_analytics_service.py b/api/app/services/emotion_analytics_service.py index 2e63eeb0..4109c91b 100644 --- a/api/app/services/emotion_analytics_service.py +++ b/api/app/services/emotion_analytics_service.py @@ -492,7 +492,7 @@ class EmotionAnalyticsService: config_id=int(config_id), service_name="EmotionAnalyticsService.generate_emotion_suggestions" ) - from app.core.memory.client_factory import MemoryClientFactory + from app.core.memory.utils.llm.llm_utils import MemoryClientFactory factory = MemoryClientFactory(db) llm_client = factory.get_llm_client(str(memory_config.llm_model_id)) except Exception as e: diff --git a/api/app/services/user_memory_service.py b/api/app/services/user_memory_service.py index a69c776e..e1b4b6eb 100644 --- a/api/app/services/user_memory_service.py +++ b/api/app/services/user_memory_service.py @@ -4,15 +4,15 @@ User Memory Service 处理用户记忆相关的业务逻辑，包括记忆洞察、用户摘要、节点统计和图数据等。 """ -from typing import Dict, List, Optional, Any import uuid -from sqlalchemy.orm import Session +from typing import Any, Dict, List, Optional from app.core.logging_config import get_logger -from app.repositories.end_user_repository import EndUserRepository -from app.repositories.neo4j.neo4j_connector import Neo4jConnector from app.core.memory.analytics.memory_insight import MemoryInsight from app.core.memory.analytics.user_summary import generate_user_summary +from app.repositories.end_user_repository import EndUserRepository +from app.repositories.neo4j.neo4j_connector import Neo4jConnector +from sqlalchemy.orm import Session logger = get_logger(__name__) @@ -284,8 +284,7 @@ class UserMemoryService: # 使用 end_user_id 调用分析函数 try: logger.info(f"使用 end_user_id={end_user_id} 生成用户摘要") - result = await analytics_user_summary(end_user_id) - summary = result.get("summary", "") + summary = await generate_user_summary(end_user_id) if not summary: logger.warning(f"end_user_id {end_user_id} 的用户摘要生成结果为空") From 3afe5475592b6447811f64dc9aaacfd6be2b4398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=B0=E6=9C=88?= Date: Wed, 24 Dec 2025 06:59:22 +0000 Subject: [PATCH 04/12] Merge #37 into develop from fix/memory_reflection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 反思输出输入格式统一 * fix/memory_reflection: (60 commits squashed) - 新增反思功能（功能配置接口+反思celery后台检测反思的迭代周期） - 新增反思功能（功能配置接口+反思celery后台检测反思的迭代周期） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 新增反思功能（检测代码/规范化程序） - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - 反思优化 - Merge branch develop into fix/memory_reflection (Conflict resolved online) # Conflicts: # api/app/controllers/memory_reflection_controller.py # api/app/schemas/memory_reflection_schemas.py - 反思优化 - Merge remote-tracking branch 'origin/fix/memory_reflection' into fix/memory_reflection - 统一输出 - 统一输出 - 统一输出 - Merge branch develop into fix/memory_reflection (Conflict resolved online) # Conflicts: # api/app/controllers/memory_reflection_controller.py - 统一输出 - Merge remote-tracking branch 'origin/fix/memory_reflection' into fix/memory_reflection - 统一输出 - 反思速度提升，从4分钟优化成1分10-40秒 - 反思速度提升，从4分钟优化成1分10-40秒 - 反思速度提升，从4分钟优化成1分10-40秒 - Merge branch develop into fix/memory_reflection (Conflict resolved online) # Conflicts: # api/app/core/memory/storage_services/reflection_engine/self_reflexion.py - 反思速度提升，从4分钟优化成1分10-40秒 - Merge remote-tracking branch 'origin/fix/memory_reflection' into fix/memory_reflection # Conflicts: # api/app/core/memory/storage_services/reflection_engine/self_reflexion.py - 更新 self_reflexion.py - 反思图谱添加边的修改 - Merge remote-tracking branch 'origin/fix/memory_reflection' into fix/memory_reflection # Conflicts: # api/app/core/memory/storage_services/reflection_engine/self_reflexion.py - 反思图谱添加边的修改 - 反思图谱添加边的修改 - 反思图谱添加边的修改 - 反思图谱添加边的修改 - 反思图谱添加边的修改 - update # Conflicts: # api/app/core/memory/storage_services/reflection_engine/self_reflexion.py # api/app/core/memory/utils/prompt/prompts/reflexion.jinja2 - 反思BUG修复 - Merge remote-tracking branch 'origin/fix/memory_reflection' into fix/memory_reflection - 反思BUG修复 - Merge branch develop into fix/memory_reflection (Conflict resolved online) # Conflicts: # api/app/core/memory/storage_services/reflection_engine/self_reflexion.py - 反思BUG修复 - Merge remote-tracking branch 'origin/fix/memory_reflection' into fix/memory_reflection - 反思输出输入格式统一 - Merge branch develop into fix/memory_reflection (Conflict resolved online) # Conflicts: # api/app/core/memory/utils/prompt/template_render.py - 反思优化提示词，提升速度，删除多余LOG日志 - Merge remote-tracking branch 'origin/fix/memory_reflection' into fix/memory_reflection Signed-off-by: aliyun8644380055 Reviewed-by: aliyun6762716068 Merged-by: aliyun6762716068 CR-link: https://codeup.aliyun.com/redbearai/python/redbear-mem-open/change/37 --- .../reflection_engine/example/example.json | 24 ++++--------- .../reflection_engine/self_reflexion.py | 34 ++----------------- .../utils/prompt/prompts/evaluate.jinja2 | 2 ++ .../utils/prompt/prompts/reflexion.jinja2 | 1 - .../memory/utils/prompt/template_render.py | 29 ++++++++++++---- api/app/schemas/memory_storage_schema.py | 10 +++++- 6 files changed, 42 insertions(+), 58 deletions(-) diff --git a/api/app/core/memory/storage_services/reflection_engine/example/example.json b/api/app/core/memory/storage_services/reflection_engine/example/example.json index fe7a3816..18a2b185 100644 --- a/api/app/core/memory/storage_services/reflection_engine/example/example.json +++ b/api/app/core/memory/storage_services/reflection_engine/example/example.json @@ -50,9 +50,7 @@ "entity2_name": "用户", "entity2": { "description": "叙述者，讲述个人工作与生活经历的个体", - "statement_id": "62beac695b1346f4871740a45db88782", - "name": "用户", - "id": "3d3896797b334572a80d57590026063d" + "name": "用户" } }, { @@ -62,9 +60,7 @@ "entity2_name": "身份信息", "entity2": { "description": "用于个人身份识别的数据", - "statement_id": "030afd362e9b4110b139e68e5d3e7143", - "name": "身份信息", - "id": "aa766a517e82490599a9b3af54cfd933" + "name": "身份信息" } }, { @@ -74,9 +70,7 @@ "entity2_name": "6222023847595898", "entity2": { "description": "用户的银行卡号码", - "statement_id": "6c7567cd1f3c478bb42d1b65383e6f2f", - "name": "6222023847595898", - "id": "610ba361918f4e68a65ce6ad06e5c7a0" + "name": "6222023847595898" } }, { @@ -88,9 +82,7 @@ "entity_idx": 1, "aliases": ["上海办"], "description": "位于上海的工作办公场所", - "statement_id": "8b1b12e23b844b8088dfeb67da6ad669", - "name": "上海办公室", - "id": "fb702ef695c14e14af3e56786bc8815b" + "name": "上海办公室" } }, { @@ -101,9 +93,7 @@ "entity2": { "aliases": ["京", "京城", "北平"], "description": "中国的首都城市，用户主要工作和生活所在地", - "statement_id": "62beac695b1346f4871740a45db88782", - "name": "北京", - "id": "81b2d1a571bb46a08a2d7a1e87efb945" + "name": "北京" } }, { @@ -113,9 +103,7 @@ "entity2_name": "身份证号", "entity2": { "description": "中华人民共和国公民的身份号码", - "statement_id": "030afd362e9b4110b139e68e5d3e7143", - "name": "身份证号", - "id": "3e5f920645b2404fadb0e9ff60d1306e" + "name": "身份证号" } } ] diff --git a/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py b/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py index 224a9560..3a4db30d 100644 --- a/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py +++ b/api/app/core/memory/storage_services/reflection_engine/self_reflexion.py @@ -239,8 +239,6 @@ class ReflectionEngine: # # 检查是否真的有冲突 conflicts_found='' - # 记录冲突数据 - await self._log_data("conflict", conflict_data) conflicts_found='' # 3. 解决冲突 solved_data = await self._resolve_conflicts(conflict_data, statement_databasets) @@ -258,8 +256,6 @@ class ReflectionEngine: conflicts_resolved = len(solved_data) logging.info(f"解决了 {conflicts_resolved} 个冲突") - # 记录解决方案 - await self._log_data("solved_data", solved_data) # 4. 应用反思结果（更新记忆库） memories_updated=await self._apply_reflection_results(solved_data) @@ -360,14 +356,7 @@ class ReflectionEngine: memory_verifies.append(item['memory_verify']) result_data['memory_verifies'] = memory_verifies result_data['quality_assessments'] = quality_assessments - - # 检查是否真的有冲突 - has_conflict = conflict_data[0].get('conflict', False) - conflicts_found = len(conflict_data[0]['data']) if has_conflict else 0 - logging.info(f"冲突状态: {has_conflict}, 发现 {conflicts_found} 个冲突") - - # 记录冲突数据 - await self._log_data("conflict", conflict_data) + conflicts_found='' # Clearn conflict_data，And memory_verify和quality_assessment cleaned_conflict_data = [] @@ -377,6 +366,7 @@ class ReflectionEngine: 'conflict': item['conflict'] } cleaned_conflict_data.append(cleaned_item) + # 3. 解决冲突 solved_data = await self._resolve_conflicts(cleaned_conflict_data, source_data) if not solved_data: @@ -615,26 +605,7 @@ class ReflectionEngine: success_count = await neo4j_data(changes) return success_count - async def _log_data(self, label: str, data: Any) -> None: - """ - 记录数据到文件 - Args: - label: 数据标签 - data: 要记录的数据 - """ - - def _write(): - try: - with open("reflexion_data.json", "a", encoding="utf-8") as f: - f.write(f"### {label} ###\n") - json.dump(data, f, ensure_ascii=False, indent=4) - f.write("\n\n") - except Exception as e: - logging.warning(f"记录数据失败: {e}") - - # 在后台线程中执行写入，避免阻塞事件循环 - await asyncio.to_thread(_write) # 基于时间的反思方法 async def time_based_reflection( @@ -723,4 +694,3 @@ class ReflectionEngine: raise ValueError(f"未知的反思基线: {self.config.baseline}") - diff --git a/api/app/core/memory/utils/prompt/prompts/evaluate.jinja2 b/api/app/core/memory/utils/prompt/prompts/evaluate.jinja2 index b292c804..200f2667 100644 --- a/api/app/core/memory/utils/prompt/prompts/evaluate.jinja2 +++ b/api/app/core/memory/utils/prompt/prompts/evaluate.jinja2 @@ -17,10 +17,12 @@ - **日期属性冲突**: 同一人的生日等单值属性出现多值 - **先后约束违反**: 存在A→B约束但t(A)>t(B)（如入学>毕业） - **互斥重叠**: 同一时间出现在不同地点等互斥事件 +- **隐私审核**: 存在隐私信息也作为冲突输出当{{ memory_verify }}是true的时候 ### 事实冲突 - **属性互斥**: 同一实体的相反属性（喜欢↔不喜欢） - **关系矛盾**: 同一实体在相同语境下的不同关系描述 - **身份冲突**: 同一实体被赋予不同类型或角色 +- **隐私审核**: 存在隐私信息也作为冲突输出当{{ memory_verify }}是true的时候 ### 混合冲突 检测所有逻辑不一致或相互矛盾的记录。 **检测原则**: diff --git a/api/app/core/memory/utils/prompt/prompts/reflexion.jinja2 b/api/app/core/memory/utils/prompt/prompts/reflexion.jinja2 index 36474d91..99476c82 100644 --- a/api/app/core/memory/utils/prompt/prompts/reflexion.jinja2 +++ b/api/app/core/memory/utils/prompt/prompts/reflexion.jinja2 @@ -171,7 +171,6 @@ ] } ``` - **输出要求**: - 只输出JSON，不添加解释文本 - 使用标准双引号，必要时转义 diff --git a/api/app/core/memory/utils/prompt/template_render.py b/api/app/core/memory/utils/prompt/template_render.py index 46bb64e8..68e0ffe4 100644 --- a/api/app/core/memory/utils/prompt/template_render.py +++ b/api/app/core/memory/utils/prompt/template_render.py @@ -7,7 +7,7 @@ from typing import List, Dict, Any prompt_dir = os.path.join(os.path.dirname(__file__), "prompts") prompt_env = Environment(loader=FileSystemLoader(prompt_dir)) -async def render_evaluate_prompt(evaluate_data: List[Any], schema: Dict[str, Any], +async def render_evaluate_prompt(evaluate_data: List[Any], schema: Any, baseline: str = "TIME", memory_verify: bool = False,quality_assessment:bool = False, statement_databasets: List[str] = [],language_type:str = "zh") -> str: @@ -16,7 +16,7 @@ async def render_evaluate_prompt(evaluate_data: List[Any], schema: Dict[str, Any Args: evaluate_data: The data to evaluate - schema: The JSON schema to use for the output. + schema: The Pydantic model class or JSON schema to use for the output. baseline: The baseline type for conflict detection (TIME/FACT/TIME-FACT) memory_verify: Whether to enable memory verification for privacy detection @@ -25,9 +25,17 @@ async def render_evaluate_prompt(evaluate_data: List[Any], schema: Dict[str, Any """ template = prompt_env.get_template("evaluate.jinja2") + # Convert Pydantic model to JSON schema if needed + if hasattr(schema, 'model_json_schema'): + json_schema = schema.model_json_schema() + elif hasattr(schema, 'schema'): + json_schema = schema.schema() + else: + json_schema = schema + rendered_prompt = template.render( evaluate_data=evaluate_data, - json_schema=schema, + json_schema=json_schema, baseline=baseline, memory_verify=memory_verify, quality_assessment=quality_assessment, @@ -36,14 +44,15 @@ async def render_evaluate_prompt(evaluate_data: List[Any], schema: Dict[str, Any ) return rendered_prompt -async def render_reflexion_prompt(data: Dict[str, Any], schema: Dict[str, Any], baseline: str, memory_verify: bool = False, + +async def render_reflexion_prompt(data: Dict[str, Any], schema: Any, baseline: str, memory_verify: bool = False, statement_databasets: List[str] = [],language_type:str = "zh") -> str: """ Renders the reflexion prompt using the reflexion_optimized.jinja2 template. Args: data: The data to reflex on. - schema: The JSON schema to use for the output. + schema: The Pydantic model class or JSON schema to use for the output. baseline: The baseline type for conflict resolution. Returns: @@ -51,7 +60,15 @@ async def render_reflexion_prompt(data: Dict[str, Any], schema: Dict[str, Any], """ template = prompt_env.get_template("reflexion.jinja2") - rendered_prompt = template.render(data=data, json_schema=schema, + # Convert Pydantic model to JSON schema if needed + if hasattr(schema, 'model_json_schema'): + json_schema = schema.model_json_schema() + elif hasattr(schema, 'schema'): + json_schema = schema.schema() + else: + json_schema = schema + + rendered_prompt = template.render(data=data, json_schema=json_schema, baseline=baseline,memory_verify=memory_verify, statement_databasets=statement_databasets,language_type=language_type) diff --git a/api/app/schemas/memory_storage_schema.py b/api/app/schemas/memory_storage_schema.py index df70ec77..33d0d097 100644 --- a/api/app/schemas/memory_storage_schema.py +++ b/api/app/schemas/memory_storage_schema.py @@ -31,7 +31,7 @@ class BaseDataSchema(BaseModel): # 保持原有必需字段为可选，以兼容不同数据源 id: Optional[str] = Field(None, description="The unique identifier for the data entry.") statement: Optional[str] = Field(None, description="The statement text.") - created_at: str = Field(..., description="The creation timestamp in ISO 8601 format.") + created_at: Optional[str] = Field(None, description="The creation timestamp in ISO 8601 format.") expired_at: Optional[str] = Field(None, description="The expiration timestamp in ISO 8601 format.") description: Optional[str] = Field(None, description="The description of the data entry.") @@ -46,6 +46,14 @@ class BaseDataSchema(BaseModel): relationship: Optional[Union[str, Dict[str, Any]]] = Field(None, description="The relationship object or string.") entity2: Optional[Dict[str, Any]] = Field(None, description="The second entity object.") + @model_validator(mode="before") + def _set_default_created_at(cls, v): + """Set default created_at if missing""" + if isinstance(v, dict) and v.get("created_at") is None: + from datetime import datetime + v["created_at"] = datetime.now().isoformat() + return v + class QualityAssessmentSchema(BaseModel): """Schema for memory quality assessment results.""" From 0fff23d2a2afe2512062f02888b3cb73c63e24a9 Mon Sep 17 00:00:00 2001 From: Ke Sun Date: Wed, 24 Dec 2025 16:06:54 +0800 Subject: [PATCH 05/12] feat(memory): make config_id optional and improve configuration validation - Make config_id optional in UserInput and Write_UserInput schemas - Update write_memory and read_memory method signatures to accept Optional[str] for config_id - Implement automatic config retrieval from end_user connection when config_id is not provided - Add explicit error handling for missing memory configurations with descriptive error messages - Improve emotion_controller to validate config_id using MemoryConfigService when provided - Fallback to get_end_user_connected_config when config_id is None - Distinguish between specific "no configuration found" errors and other exceptions for better debugging - Ensures users can operate without explicitly providing config_id if they have a connected configuration --- api/app/controllers/emotion_controller.py | 23 +++++++++++++++++++---- api/app/schemas/memory_agent_schema.py | 4 ++-- api/app/services/memory_agent_service.py | 18 ++++++++++++++---- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/api/app/controllers/emotion_controller.py b/api/app/controllers/emotion_controller.py index 144aa281..90465c95 100644 --- a/api/app/controllers/emotion_controller.py +++ b/api/app/controllers/emotion_controller.py @@ -210,13 +210,28 @@ async def get_emotion_suggestions( """ try: # 验证 config_id（如果提供） + # 获取终端用户关联的配置 config_id = request.config_id - if config_id is not None: - from app.controllers.memory_agent_controller import validate_config_id + if config_id is None: + # 如果没有提供 config_id，尝试获取用户关联的配置 try: - config_id = validate_config_id(config_id, db) + from app.services.memory_agent_service import ( + get_end_user_connected_config, + ) + connected_config = get_end_user_connected_config(request.group_id, db) + config_id = connected_config.get("memory_config_id") except ValueError as e: - return fail(BizCode.INVALID_PARAMETER, "配置ID无效", str(e)) + return fail(BizCode.INVALID_PARAMETER, "无法获取用户关联的配置", str(e)) + else: + # 如果提供了 config_id，验证其有效性 + from app.services.memory_config_service import MemoryConfigService + try: + config_service = MemoryConfigService(db) + config = config_service.get_config_by_id(config_id) + if not config: + return fail(BizCode.INVALID_PARAMETER, "配置ID无效", f"配置 {config_id} 不存在") + except Exception as e: + return fail(BizCode.INVALID_PARAMETER, "配置ID验证失败", str(e)) api_logger.info( f"用户 {current_user.username} 请求获取个性化情绪建议", diff --git a/api/app/schemas/memory_agent_schema.py b/api/app/schemas/memory_agent_schema.py index e6fed0e5..47dc6b2a 100644 --- a/api/app/schemas/memory_agent_schema.py +++ b/api/app/schemas/memory_agent_schema.py @@ -8,13 +8,13 @@ class UserInput(BaseModel): history: list[dict] search_switch: str group_id: str - config_id: str + config_id: Optional[str] = None class Write_UserInput(BaseModel): message: str group_id: str - config_id: str + config_id: Optional[str] = None class End_User_Information(BaseModel): end_user_name: str # 这是要更新的用户名 diff --git a/api/app/services/memory_agent_service.py b/api/app/services/memory_agent_service.py index e23f9471..8193da8a 100644 --- a/api/app/services/memory_agent_service.py +++ b/api/app/services/memory_agent_service.py @@ -255,7 +255,7 @@ class MemoryAgentService: logger.info("Log streaming completed, cleaning up resources") # LogStreamer uses context manager for file handling, so cleanup is automatic - async def write_memory(self, group_id: str, message: str, config_id: str, db: Session, storage_type: str, user_rag_memory_id: str) -> str: + async def write_memory(self, group_id: str, message: str, config_id: Optional[str], db: Session, storage_type: str, user_rag_memory_id: str) -> str: """ Process write operation with config_id @@ -278,8 +278,13 @@ class MemoryAgentService: try: connected_config = get_end_user_connected_config(group_id, db) config_id = connected_config.get("memory_config_id") + if config_id is None: + raise ValueError(f"No memory configuration found for end_user {group_id}. Please ensure the user has a connected memory configuration.") except Exception as e: - logger.warning(f"Failed to get connected config for end_user {group_id}: {e}") + if "No memory configuration found" in str(e): + raise # Re-raise our specific error + logger.error(f"Failed to get connected config for end_user {group_id}: {e}") + raise ValueError(f"Unable to determine memory configuration for end_user {group_id}: {e}") import time start_time = time.time() @@ -356,7 +361,7 @@ class MemoryAgentService: message: str, history: List[Dict], search_switch: str, - config_id: str, + config_id: Optional[str], db: Session, storage_type: str, user_rag_memory_id: str @@ -394,8 +399,13 @@ class MemoryAgentService: try: connected_config = get_end_user_connected_config(group_id, db) config_id = connected_config.get("memory_config_id") + if config_id is None: + raise ValueError(f"No memory configuration found for end_user {group_id}. Please ensure the user has a connected memory configuration.") except Exception as e: - logger.warning(f"Failed to get connected config for end_user {group_id}: {e}") + if "No memory configuration found" in str(e): + raise # Re-raise our specific error + logger.error(f"Failed to get connected config for end_user {group_id}: {e}") + raise ValueError(f"Unable to determine memory configuration for end_user {group_id}: {e}") logger.info(f"Read operation for group {group_id} with config_id {config_id}") From d18c2a491a1c10cdc49d96a9b9a5dad777107965 Mon Sep 17 00:00:00 2001 From: lanceyq <1982376970@qq.com> Date: Wed, 24 Dec 2025 17:13:22 +0800 Subject: [PATCH 06/12] [fix]Delete the unnecessary "analytics_user_summary" --- api/app/controllers/memory_storage_controller.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/api/app/controllers/memory_storage_controller.py b/api/app/controllers/memory_storage_controller.py index e8e725d8..bb53b833 100644 --- a/api/app/controllers/memory_storage_controller.py +++ b/api/app/controllers/memory_storage_controller.py @@ -41,7 +41,6 @@ from app.services.memory_storage_service import ( search_entity_graph, search_statement, ) -from app.services.user_memory_service import analytics_user_summary from fastapi import APIRouter, Depends from fastapi.responses import StreamingResponse from sqlalchemy.orm import Session @@ -510,18 +509,6 @@ async def get_recent_activity_stats_api( return fail(BizCode.INTERNAL_ERROR, "最近活动统计失败", str(e)) -@router.get("/analytics/user_summary", response_model=ApiResponse) -async def get_user_summary_api( - end_user_id: Optional[str] = None, - current_user: User = Depends(get_current_user), - ) -> dict: - api_logger.info(f"User summary requested for end_user_id: {end_user_id}") - try: - result = await analytics_user_summary(end_user_id) - return success(data=result, msg="查询成功") - except Exception as e: - api_logger.error(f"User summary failed: {str(e)}") - return fail(BizCode.INTERNAL_ERROR, "用户摘要生成失败", str(e)) @router.get("/self_reflexion") From bb666eef232caf3021084883f7de33d80fceca97 Mon Sep 17 00:00:00 2001 From: lanceyq <1982376970@qq.com> Date: Wed, 24 Dec 2025 17:18:43 +0800 Subject: [PATCH 07/12] [fix]Fix the log error --- api/app/controllers/user_memory_controllers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/app/controllers/user_memory_controllers.py b/api/app/controllers/user_memory_controllers.py index 5ff34d21..fbefe5ba 100644 --- a/api/app/controllers/user_memory_controllers.py +++ b/api/app/controllers/user_memory_controllers.py @@ -198,8 +198,8 @@ async def get_node_statistics_api( api_logger.info(f"成功获取节点统计: end_user_id={end_user_id}, total={result['total']}") return success(data=result, msg="查询成功") except Exception as e: - api_logger.error(f"用户摘要查询失败: end_user_id={end_user_id}, error={str(e)}") - return fail(BizCode.INTERNAL_ERROR, "用户摘要查询失败", str(e)) + api_logger.error(f"节点统计查询失败: end_user_id={end_user_id}, error={str(e)}") + return fail(BizCode.INTERNAL_ERROR, "节点统计查询失败", str(e)) @router.get("/analytics/graph_data", response_model=ApiResponse) async def get_graph_data_api( From 6338edda11e951e259b441a08da3424097e295a3 Mon Sep 17 00:00:00 2001 From: lixiangcheng1 Date: Wed, 24 Dec 2025 17:50:03 +0800 Subject: [PATCH 08/12] [ADD]Support parsing of unstructured data MP3, MP4, etc --- api/app/core/rag/app/audio.py | 2 +- api/app/core/rag/app/naive.py | 25 +++++++++++++++++++--- api/app/core/rag/app/picture.py | 3 +-- api/app/core/rag/llm/cv_model.py | 2 +- api/app/core/rag/llm/sequence2txt_model.py | 14 +++++++++--- api/app/tasks.py | 18 ++++++++++++++++ api/env.example | 5 +++++ 7 files changed, 59 insertions(+), 10 deletions(-) diff --git a/api/app/core/rag/app/audio.py b/api/app/core/rag/app/audio.py index 1bddc048..6ce422e5 100644 --- a/api/app/core/rag/app/audio.py +++ b/api/app/core/rag/app/audio.py @@ -26,7 +26,7 @@ def chunk(filename, binary, lang, callback=None, seq2txt_mdl=None, **kwargs): tmp_path = os.path.abspath(tmpf.name) callback(0.1, "USE Sequence2Txt LLM to transcription the audio") - ans = seq2txt_mdl.transcription(tmp_path) + ans, ans_num_tokens = seq2txt_mdl.transcription(tmp_path) callback(0.8, "Sequence2Txt LLM respond: %s ..." % ans[:32]) tokenize(doc, ans, eng) diff --git a/api/app/core/rag/app/naive.py b/api/app/core/rag/app/naive.py index 5430e38f..6d6b933a 100644 --- a/api/app/core/rag/app/naive.py +++ b/api/app/core/rag/app/naive.py @@ -13,6 +13,7 @@ from PIL import Image import copy from app.core.rag.llm.cv_model import AzureGptV4, QWenCV +from app.core.rag.llm.sequence2txt_model import QWenSeq2txt from app.core.rag.common.file_utils import get_project_base_directory from app.core.rag.utils.file_utils import extract_embed_file, extract_links_from_pdf, extract_links_from_docx, extract_html from app.core.rag.utils.libre_office import convert_to_pdf, async_convert_to_pdf @@ -809,13 +810,31 @@ if __name__ == "__main__": # chunk(sys.argv[1], from_page=0, to_page=10, callback=dummy) # Prepare to configure vision_model information - vision_model = QWenCV( - key="sk-8e9e40cd171749858ce2d3722ea75669", - model_name="qwen-vl-max", + # 文字+图片 + # vision_model = QWenCV( + # key="", + # model_name="qwen-vl-max", + # lang="chinese", # 默认使用中文 + # base_url="https://dashscope.aliyuncs.com/compatible-mode/v1" + # ) + + # 音频 + vision_model = QWenSeq2txt( + key="", + model_name="qwen3-omni-flash", lang="chinese", # 默认使用中文 base_url="https://dashscope.aliyuncs.com/compatible-mode/v1" ) + + # 视频 + # vision_model = QWenCV( + # key="", + # model_name="qwen3-omni-flash", + # lang="chinese", # 默认使用中文 + # base_url="https://dashscope.aliyuncs.com/compatible-mode/v1" + # ) + def progress_callback(prog=None, msg=None): print(f"prog: {prog} msg: {msg}\n") diff --git a/api/app/core/rag/app/picture.py b/api/app/core/rag/app/picture.py index addc7d9b..da133c27 100644 --- a/api/app/core/rag/app/picture.py +++ b/api/app/core/rag/app/picture.py @@ -24,9 +24,8 @@ def chunk(filename, binary, lang, callback=None, vision_model=None, **kwargs): if any(filename.lower().endswith(ext) for ext in VIDEO_EXTS): try: doc.update({"doc_type_kwd": "video"}) - ans = vision_model.chat(system="", history=[], gen_conf={}, video_bytes=binary, filename=filename) + ans, ans_num_tokens = vision_model.chat(system="", history=[], gen_conf={}, video_bytes=binary, filename=filename) callback(0.8, "CV LLM respond: %s ..." % ans[:32]) - ans += "\n" + ans tokenize(doc, ans, eng) return [doc] except Exception as e: diff --git a/api/app/core/rag/llm/cv_model.py b/api/app/core/rag/llm/cv_model.py index 663272ce..5f841433 100644 --- a/api/app/core/rag/llm/cv_model.py +++ b/api/app/core/rag/llm/cv_model.py @@ -252,7 +252,7 @@ class QWenCV(GptV4): "fps": 2, }, { - "text": "Please summarize this video in proper sentences.", + "text": "视频的内容是什么?,并且，请用恰当的句子总结这个视频。" if self.lang.lower() == "chinese" else "What is the content of the video? And please summarize this video in proper sentences.", }, ], } diff --git a/api/app/core/rag/llm/sequence2txt_model.py b/api/app/core/rag/llm/sequence2txt_model.py index dcea9346..be4d3649 100644 --- a/api/app/core/rag/llm/sequence2txt_model.py +++ b/api/app/core/rag/llm/sequence2txt_model.py @@ -46,11 +46,12 @@ class GPTSeq2txt(Base): class QWenSeq2txt(Base): _FACTORY_NAME = "Tongyi-Qianwen" - def __init__(self, key, model_name="qwen-audio-asr", **kwargs): + def __init__(self, key, model_name="qwen-audio-asr", lang="Chinese", **kwargs): import dashscope dashscope.api_key = key self.model_name = model_name + self.lang = lang def transcription(self, audio_path): if "paraformer" in self.model_name or "sensevoice" in self.model_name: @@ -62,14 +63,21 @@ class QWenSeq2txt(Base): messages = [ { "role": "user", - "content": [{"audio": audio_path}], + "content": [ + { + "audio": audio_path + }, + { + "text": "这段音频在说什么?" if self.lang.lower() == "chinese" else "What is this audio saying?", + }, + ], } ] response = None full_content = "" try: - response = MultiModalConversation.call(model="qwen-audio-asr", messages=messages, result_format="message", stream=True) + response = MultiModalConversation.call(model=self.model_name, messages=messages, result_format="message", stream=True) for response in response: try: full_content += response["output"]["choices"][0]["message"].content[0]["text"] diff --git a/api/app/tasks.py b/api/app/tasks.py index 907fae00..00bc7237 100644 --- a/api/app/tasks.py +++ b/api/app/tasks.py @@ -6,6 +6,7 @@ import uuid from datetime import datetime, timezone from math import ceil from typing import Any, Dict, List, Optional +import re import redis import requests @@ -16,6 +17,7 @@ from app.core.config import settings from app.core.rag.graphrag.utils import get_llm_cache, set_llm_cache from app.core.rag.llm.chat_model import Base from app.core.rag.llm.cv_model import QWenCV +from app.core.rag.llm.sequence2txt_model import QWenSeq2txt from app.core.rag.models.chunk import DocumentChunk from app.core.rag.prompts.generator import question_proposal from app.core.rag.vdb.elasticsearch.elasticsearch_vector import ( @@ -83,6 +85,22 @@ def parse_document(file_path: str, document_id: uuid.UUID): lang="Chinese", base_url=db_knowledge.image2text.api_keys[0].api_base ) + if re.search(r"\.(da|wave|wav|mp3|aac|flac|ogg|aiff|au|midi|wma|realaudio|vqf|oggvorbis|ape?)$", file_path, re.IGNORECASE): + vision_model = QWenSeq2txt( + key=os.getenv("QWEN3_OMNI_API_KEY", ""), + model_name=os.getenv("QWEN3_OMNI_MODEL_NAME", "qwen3-omni-flash"), + lang="Chinese", + base_url=os.getenv("QWEN3_OMNI_BASE_URL", "https://dashscope.aliyuncs.com/compatible-mode/v1"), + ) + elif re.search(r"\.(png|jpeg|jpg|gif|bmp|svg|mp4|mov|avi|flv|mpeg|mpg|webm|wmv|3gp|3gpp|mkv?)$", file_path, re.IGNORECASE): + vision_model = QWenCV( + key=os.getenv("QWEN3_OMNI_API_KEY", ""), + model_name=os.getenv("QWEN3_OMNI_MODEL_NAME", "qwen3-omni-flash"), + lang="Chinese", + base_url=os.getenv("QWEN3_OMNI_BASE_URL", "https://dashscope.aliyuncs.com/compatible-mode/v1"), + ) + else: + print(file_path) from app.core.rag.app.naive import chunk res = chunk(filename=file_path, from_page=0, diff --git a/api/env.example b/api/env.example index 8ceb3934..371a0561 100644 --- a/api/env.example +++ b/api/env.example @@ -83,6 +83,11 @@ TEXTLN_APISERVER=https://api.textin.com/ai/service/v1/pdf_to_markdown TEXTLN_APP_ID= TEXTLN_SECRET_CODE= +# vision model +QWEN3_OMNI_API_KEY= +QWEN3_OMNI_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 +QWEN3_OMNI_MODEL_NAME=qwen3-omni-flash + # VOLC ASR VOLC_APP_KEY= VOLC_ACCESS_KEY= From 9cf8d5cb0a5402b633beb1b466e68904cabb94b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=90=E5=8A=9B=E9=BD=90?= Date: Wed, 24 Dec 2025 10:11:31 +0000 Subject: [PATCH 09/12] Merge #47 into develop from fix/othername-name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [fix]Fix the bug that affects user memory. * fix/othername-name: (11 commits squashed) - [fix]Fix the issue with the display of the user's memory list - [fix]Ensure the six dimensions of emotional expression - [fix]Fix the issue with the display of the user's memory list - [fix]Ensure the six dimensions of emotional expression - Merge branch 'fix/othername-name' of codeup.aliyun.com:redbearai/python/redbear-mem-open into fix/othername-name - [fix]Restore the display of memory types - [fix]Fix the issue with the display of the user's memory list - [fix]Ensure the six dimensions of emotional expression - [fix]Restore the display of memory types - Merge branch 'fix/othername-name' of codeup.aliyun.com:redbearai/python/redbear-mem-open into fix/othername-name - [updated]Update the title of the "analytics/node_statistics" log Signed-off-by: 乐力齐 Reviewed-by: aliyun6762716068 Merged-by: aliyun6762716068 CR-link: https://codeup.aliyun.com/redbearai/python/redbear-mem-open/change/47 --- .../controllers/user_memory_controllers.py | 21 ++--- api/app/models/end_user_model.py | 1 - api/app/schemas/end_user_schema.py | 5 +- api/app/services/emotion_analytics_service.py | 48 +++++++---- api/app/services/memory_dashboard_service.py | 2 +- api/app/services/user_memory_service.py | 85 +++++++++++++++++++ 6 files changed, 129 insertions(+), 33 deletions(-) diff --git a/api/app/controllers/user_memory_controllers.py b/api/app/controllers/user_memory_controllers.py index fbefe5ba..8afb0863 100644 --- a/api/app/controllers/user_memory_controllers.py +++ b/api/app/controllers/user_memory_controllers.py @@ -14,6 +14,7 @@ from app.core.error_codes import BizCode from app.services.user_memory_service import ( UserMemoryService, analytics_node_statistics, + analytics_memory_types, analytics_graph_data, ) from app.schemas.response_schema import ApiResponse @@ -185,21 +186,17 @@ async def get_node_statistics_api( api_logger.warning(f"用户 {current_user.username} 尝试查询节点统计但未选择工作空间") return fail(BizCode.INVALID_PARAMETER, "请先切换到一个工作空间", "current_workspace_id is None") - api_logger.info(f"节点统计请求: end_user_id={end_user_id}, user={current_user.username}, workspace={workspace_id}") + api_logger.info(f"记忆类型统计请求: end_user_id={end_user_id}, user={current_user.username}, workspace={workspace_id}") try: - result = await analytics_node_statistics(db, end_user_id) + # 调用新的记忆类型统计函数 + result = await analytics_memory_types(db, end_user_id) - # 检查是否有错误消息 - if "message" in result and result["total"] == 0: - api_logger.warning(f"节点统计查询返回空结果: {result.get('message')}") - return success(data=result, msg=result.get("message", "查询成功")) - - api_logger.info(f"成功获取节点统计: end_user_id={end_user_id}, total={result['total']}") + api_logger.info(f"成功获取记忆类型统计: end_user_id={end_user_id}, 感知记忆={result.get('感知记忆', 0)}") return success(data=result, msg="查询成功") except Exception as e: - api_logger.error(f"节点统计查询失败: end_user_id={end_user_id}, error={str(e)}") - return fail(BizCode.INTERNAL_ERROR, "节点统计查询失败", str(e)) + api_logger.error(f"记忆类型查询失败: end_user_id={end_user_id}, error={str(e)}") + return fail(BizCode.INTERNAL_ERROR, "记忆类型查询失败", str(e)) @router.get("/analytics/graph_data", response_model=ApiResponse) async def get_graph_data_api( @@ -293,7 +290,7 @@ async def get_end_user_profile( # 构建响应数据 profile_data = EndUserProfileResponse( id=end_user.id, - name=end_user.name, + other_name=end_user.other_name, position=end_user.position, department=end_user.department, contact=end_user.contact, @@ -364,7 +361,7 @@ async def update_end_user_profile( # 构建响应数据 profile_data = EndUserProfileResponse( id=end_user.id, - name=end_user.name, + other_name=end_user.other_name, position=end_user.position, department=end_user.department, contact=end_user.contact, diff --git a/api/app/models/end_user_model.py b/api/app/models/end_user_model.py index 0ef11ffa..1e1ce4f3 100644 --- a/api/app/models/end_user_model.py +++ b/api/app/models/end_user_model.py @@ -19,7 +19,6 @@ class EndUser(Base): updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now) # 用户基本信息字段 - name = Column(String, nullable=True, comment="姓名") position = Column(String, nullable=True, comment="职位") department = Column(String, nullable=True, comment="部门") contact = Column(String, nullable=True, comment="联系方式") diff --git a/api/app/schemas/end_user_schema.py b/api/app/schemas/end_user_schema.py index 939d2d3e..07188096 100644 --- a/api/app/schemas/end_user_schema.py +++ b/api/app/schemas/end_user_schema.py @@ -18,7 +18,6 @@ class EndUser(BaseModel): updated_at: datetime.datetime = Field(description="更新时间", default_factory=datetime.datetime.now) # 用户基本信息字段 - name: Optional[str] = Field(description="姓名", default=None) position: Optional[str] = Field(description="职位", default=None) department: Optional[str] = Field(description="部门", default=None) contact: Optional[str] = Field(description="联系方式", default=None) @@ -32,7 +31,7 @@ class EndUserProfileResponse(BaseModel): model_config = ConfigDict(from_attributes=True) id: uuid.UUID = Field(description="终端用户ID") - name: Optional[str] = Field(description="姓名", default=None) + other_name: Optional[str] = Field(description="其他名称", default="") position: Optional[str] = Field(description="职位", default=None) department: Optional[str] = Field(description="部门", default=None) contact: Optional[str] = Field(description="联系方式", default=None) @@ -44,7 +43,7 @@ class EndUserProfileResponse(BaseModel): class EndUserProfileUpdate(BaseModel): """终端用户基本信息更新请求模型""" end_user_id: str = Field(description="终端用户ID") - name: Optional[str] = Field(description="姓名", default=None) + other_name: Optional[str] = Field(description="其他名称", default="") position: Optional[str] = Field(description="职位", default=None) department: Optional[str] = Field(description="部门", default=None) contact: Optional[str] = Field(description="联系方式", default=None) diff --git a/api/app/services/emotion_analytics_service.py b/api/app/services/emotion_analytics_service.py index 4109c91b..e4b5bed6 100644 --- a/api/app/services/emotion_analytics_service.py +++ b/api/app/services/emotion_analytics_service.py @@ -65,19 +65,9 @@ class EmotionAnalyticsService: """获取情绪标签统计 查询指定用户的情绪类型分布，包括计数、百分比和平均强度。 - - Args: - end_user_id: 宿主ID（用户组ID） - emotion_type: 可选的情绪类型过滤 - start_date: 可选的开始日期（ISO格式） - end_date: 可选的结束日期（ISO格式） - limit: 返回结果的最大数量 - - Returns: - Dict: 包含情绪标签统计的响应数据： - - tags: 情绪标签列表 - - total_count: 总情绪数量 - - time_range: 时间范围信息 + 确保返回所有6个情绪维度（joy、sadness、anger、fear、surprise、neutral）， + 即使某些维度没有数据也会返回count=0的记录。 + """ try: logger.info(f"获取情绪标签统计: user={end_user_id}, type={emotion_type}, " @@ -92,8 +82,34 @@ class EmotionAnalyticsService: limit=limit ) + # 定义所有6个情绪维度 + all_emotion_types = ['joy', 'sadness', 'anger', 'fear', 'surprise', 'neutral'] + + # 将查询结果转换为字典，方便查找 + tags_dict = {tag["emotion_type"]: tag for tag in tags} + + # 补全缺失的情绪维度 + complete_tags = [] + for emotion in all_emotion_types: + if emotion in tags_dict: + complete_tags.append(tags_dict[emotion]) + else: + # 如果该情绪类型不存在，添加默认值 + complete_tags.append({ + "emotion_type": emotion, + "count": 0, + "percentage": 0.0, + "avg_intensity": 0.0 + }) + # 计算总数 - total_count = sum(tag["count"] for tag in tags) + total_count = sum(tag["count"] for tag in complete_tags) + + # 如果有数据，重新计算百分比（因为补全了0值项） + if total_count > 0: + for tag in complete_tags: + if tag["count"] > 0: + tag["percentage"] = round((tag["count"] / total_count) * 100, 2) # 构建时间范围信息 time_range = {} @@ -104,12 +120,12 @@ class EmotionAnalyticsService: # 格式化响应 response = { - "tags": tags, + "tags": complete_tags, "total_count": total_count, "time_range": time_range if time_range else None } - logger.info(f"情绪标签统计完成: total_count={total_count}, tags_count={len(tags)}") + logger.info(f"情绪标签统计完成: total_count={total_count}, tags_count={len(complete_tags)}") return response except Exception as e: diff --git a/api/app/services/memory_dashboard_service.py b/api/app/services/memory_dashboard_service.py index 6acc699a..9a8ae9aa 100644 --- a/api/app/services/memory_dashboard_service.py +++ b/api/app/services/memory_dashboard_service.py @@ -272,7 +272,7 @@ async def get_workspace_total_memory_count( from app.repositories.end_user_repository import EndUserRepository repo = EndUserRepository(db) end_user = repo.get_by_id(uuid.UUID(end_user_id)) - user_name = end_user.name if end_user else None + user_name = end_user.other_name if end_user else None return { "total_memory_count": search_result.get("total", 0), diff --git a/api/app/services/user_memory_service.py b/api/app/services/user_memory_service.py index e1b4b6eb..f728fa6f 100644 --- a/api/app/services/user_memory_service.py +++ b/api/app/services/user_memory_service.py @@ -534,6 +534,91 @@ async def analytics_node_statistics( return data +async def analytics_memory_types( + db: Session, + end_user_id: Optional[str] = None +) -> Dict[str, Any]: + """ + 统计8种记忆类型的数量 + + 计算规则： + 1. 感知记忆 = statement + entity + 2. 工作记忆 = chunk + entity + 3. 短期记忆 = chunk + 4. 长期记忆 = entity + 5. 显性记忆 = 1/2 * entity + 6. 隐形记忆 = 1/3 * entity + 7. 情绪记忆 = statement + 8. 情景记忆 = memory_summary + + Args: + db: 数据库会话 + end_user_id: 可选的终端用户ID (UUID)，用于过滤特定用户的节点 + + Returns: + { + "感知记忆": int, + "工作记忆": int, + "短期记忆": int, + "长期记忆": int, + "显性记忆": int, + "隐形记忆": int, + "情绪记忆": int, + "情景记忆": int + } + """ + # 定义需要查询的节点类型 + node_types = { + "Statement": "Statement", + "Entity": "ExtractedEntity", + "Chunk": "Chunk", + "MemorySummary": "MemorySummary" + } + + # 存储每种节点类型的计数 + node_counts = {} + + # 查询每种节点类型的数量 + for key, node_type in node_types.items(): + if end_user_id: + query = f""" + MATCH (n:{node_type}) + WHERE n.group_id = $group_id + RETURN count(n) as count + """ + result = await _neo4j_connector.execute_query(query, group_id=end_user_id) + else: + query = f""" + MATCH (n:{node_type}) + RETURN count(n) as count + """ + result = await _neo4j_connector.execute_query(query) + + # 提取计数结果 + count = result[0]["count"] if result and len(result) > 0 else 0 + node_counts[key] = count + + # 获取各节点类型的数量 + statement_count = node_counts.get("Statement", 0) + entity_count = node_counts.get("Entity", 0) + chunk_count = node_counts.get("Chunk", 0) + memory_summary_count = node_counts.get("MemorySummary", 0) + + # 按规则计算8种记忆类型 + memory_types = { + "感知记忆": statement_count + entity_count, + "工作记忆": chunk_count + entity_count, + "短期记忆": chunk_count, + "长期记忆": entity_count, + "显性记忆": entity_count // 2, # 1/2 entity，使用整除 + "隐形记忆": entity_count // 3, # 1/3 entity，使用整除 + "情绪记忆": statement_count, + "情景记忆": memory_summary_count + } + + return memory_types + + async def analytics_graph_data( db: Session, end_user_id: str, From caf9b8a9dae1c410fe6f08a361cd70c97767f817 Mon Sep 17 00:00:00 2001 From: Ke Sun Date: Wed, 24 Dec 2025 18:21:36 +0800 Subject: [PATCH 10/12] feat(workspace): add workspace models configuration update endpoint - Add PUT endpoint to update workspace LLM, embedding, and rerank model configurations - Create WorkspaceModelsUpdate schema for model configuration update requests - Create WorkspaceModelsConfig schema for model configuration responses with proper validation - Implement update_workspace_models_configs service method to persist model configuration changes - Update workspace_models_configs GET endpoint to return validated WorkspaceModelsConfig response - Reorganize imports across controller, schema, and service files for consistency and readability - Add proper logging for model configuration updates with user and workspace context --- api/app/controllers/workspace_controller.py | 68 +++++++++--- api/app/schemas/workspace_schema.py | 32 +++++- api/app/services/workspace_service.py | 112 ++++++++++++++------ 3 files changed, 163 insertions(+), 49 deletions(-) diff --git a/api/app/controllers/workspace_controller.py b/api/app/controllers/workspace_controller.py index f4390568..eb6065e0 100644 --- a/api/app/controllers/workspace_controller.py +++ b/api/app/controllers/workspace_controller.py @@ -1,25 +1,38 @@ -from fastapi import APIRouter, Depends, HTTPException, status, Query -from sqlalchemy.orm import Session -from typing import List, Optional import uuid +from typing import List, Optional +from app.core.logging_config import get_api_logger from app.core.response_utils import success from app.db import get_db -from app.dependencies import get_current_superuser, get_current_user, get_current_tenant, workspace_access_guard, cur_workspace_access_guard -from app.models.user_model import User +from app.dependencies import ( + cur_workspace_access_guard, + get_current_superuser, + get_current_tenant, + get_current_user, + workspace_access_guard, +) from app.models.tenant_model import Tenants -from app.models.workspace_model import Workspace, InviteStatus +from app.models.user_model import User +from app.models.workspace_model import InviteStatus, Workspace +from app.schemas import knowledge_schema from app.schemas.response_schema import ApiResponse from app.schemas.workspace_schema import ( - WorkspaceCreate, WorkspaceUpdate, WorkspaceResponse, - WorkspaceInviteCreate, WorkspaceInviteResponse, - InviteValidateResponse, InviteAcceptRequest, - WorkspaceMemberUpdate, WorkspaceMemberItem + InviteAcceptRequest, + InviteValidateResponse, + WorkspaceCreate, + WorkspaceInviteCreate, + WorkspaceInviteResponse, + WorkspaceMemberItem, + WorkspaceMemberUpdate, + WorkspaceModelsConfig, + WorkspaceModelsUpdate, + WorkspaceResponse, + WorkspaceUpdate, ) -from app.schemas import knowledge_schema -from app.services import workspace_service -from app.core.logging_config import get_api_logger -from app.services import knowledge_service, document_service +from app.services import document_service, knowledge_service, workspace_service +from fastapi import APIRouter, Depends, HTTPException, Query, status +from sqlalchemy.orm import Session + # 获取API专用日志器 api_logger = get_api_logger() # 需要认证的路由器 @@ -338,5 +351,30 @@ def workspace_models_configs( f"成功获取工作空间 {workspace_id} 的模型配置: " f"llm={configs.get('llm')}, embedding={configs.get('embedding')}, rerank={configs.get('rerank')}" ) - return success(data=configs, msg="模型配置获取成功") + return success(data=WorkspaceModelsConfig.model_validate(configs), msg="模型配置获取成功") + + +@router.put("/workspace_models", response_model=ApiResponse) +@cur_workspace_access_guard() +def update_workspace_models_configs( + models_update: WorkspaceModelsUpdate, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """更新当前工作空间的模型配置（llm, embedding, rerank）""" + workspace_id = current_user.current_workspace_id + api_logger.info(f"用户 {current_user.username} 请求更新工作空间 {workspace_id} 的模型配置") + + updated_workspace = workspace_service.update_workspace_models_configs( + db=db, + workspace_id=workspace_id, + models_update=models_update, + user=current_user + ) + + api_logger.info( + f"成功更新工作空间 {workspace_id} 的模型配置: " + f"llm={updated_workspace.llm}, embedding={updated_workspace.embedding}, rerank={updated_workspace.rerank}" + ) + return success(data=WorkspaceModelsConfig.model_validate(updated_workspace), msg="模型配置更新成功") diff --git a/api/app/schemas/workspace_schema.py b/api/app/schemas/workspace_schema.py index eb3e31e2..1fdfe426 100644 --- a/api/app/schemas/workspace_schema.py +++ b/api/app/schemas/workspace_schema.py @@ -1,9 +1,17 @@ -import email -from pydantic import BaseModel, Field, EmailStr, field_serializer, computed_field, ConfigDict import datetime +import email import uuid -from typing import Literal -from app.models.workspace_model import WorkspaceRole, InviteStatus +from typing import Literal, Optional + +from app.models.workspace_model import InviteStatus, WorkspaceRole +from pydantic import ( + BaseModel, + ConfigDict, + EmailStr, + Field, + computed_field, + field_serializer, +) class WorkspaceBase(BaseModel): @@ -170,3 +178,19 @@ class InviteValidateResponse(BaseModel): class InviteAcceptRequest(BaseModel): token: str = Field(..., description="邀请令牌") + + +class WorkspaceModelsUpdate(BaseModel): + """工作空间模型配置更新请求""" + llm: Optional[uuid.UUID] = Field(default=None, description="LLM模型ID") + embedding: Optional[uuid.UUID] = Field(default=None, description="嵌入模型ID") + rerank: Optional[uuid.UUID] = Field(default=None, description="重排序模型ID") + + +class WorkspaceModelsConfig(BaseModel): + """工作空间模型配置响应""" + model_config = ConfigDict(from_attributes=True) + + llm: Optional[str] = Field(default=None, description="LLM模型ID") + embedding: Optional[str] = Field(default=None, description="嵌入模型ID") + rerank: Optional[str] = Field(default=None, description="重排序模型ID") diff --git a/api/app/services/workspace_service.py b/api/app/services/workspace_service.py index 04ee647c..bb6c53dc 100644 --- a/api/app/services/workspace_service.py +++ b/api/app/services/workspace_service.py @@ -1,36 +1,38 @@ -from sqlalchemy.orm import Session -from typing import List, Optional -import uuid -import secrets -import hashlib import datetime -from fastapi import HTTPException, status +import hashlib +import secrets +import uuid +from os import getenv +from typing import List, Optional + +from app.core.config import settings from app.core.error_codes import BizCode from app.core.exceptions import BusinessException, PermissionDeniedException -from app.models.tenant_model import Tenants +from app.core.logging_config import get_business_logger from app.models.user_model import User -from app.models.app_model import App -from app.models.end_user_model import EndUser -from app.models.workspace_model import Workspace, WorkspaceRole, WorkspaceInvite, InviteStatus, WorkspaceMember -from app.schemas.workspace_schema import ( - WorkspaceCreate, - WorkspaceUpdate, - WorkspaceInviteCreate, - WorkspaceInviteResponse, - InviteValidateResponse, - InviteAcceptRequest, - WorkspaceMemberUpdate +from app.models.workspace_model import ( + InviteStatus, + Workspace, + WorkspaceMember, + WorkspaceRole, ) from app.repositories import workspace_repository from app.repositories.workspace_invite_repository import WorkspaceInviteRepository -from app.core.logging_config import get_business_logger -from app.core.config import settings -from app.services import user_service -from os import getenv +from app.schemas.workspace_schema import ( + InviteAcceptRequest, + InviteValidateResponse, + WorkspaceCreate, + WorkspaceInviteCreate, + WorkspaceInviteResponse, + WorkspaceMemberUpdate, + WorkspaceModelsUpdate, + WorkspaceUpdate, +) +from dotenv import load_dotenv +from sqlalchemy.orm import Session + # 获取业务逻辑专用日志器 business_logger = get_business_logger() -import os # -from dotenv import load_dotenv load_dotenv() def switch_workspace( db: Session, @@ -134,10 +136,9 @@ def create_workspace( f"{db_workspace.id} 创建知识库" ) try: - import os - from app.schemas.knowledge_schema import KnowledgeCreate from app.models.knowledge_model import KnowledgeType, PermissionType from app.repositories import knowledge_repository + from app.schemas.knowledge_schema import KnowledgeCreate # 创建知识库数据 knowledge_data = KnowledgeCreate( @@ -232,7 +233,7 @@ def get_workspace_members( ) # 权限检查：工作空间成员或超级管理员可以查看成员列表 - from app.core.permissions import permission_service, Subject, Resource, Action + from app.core.permissions import Action, Resource, Subject, permission_service member = workspace_repository.get_member_in_workspace( db=db, user_id=user.id, workspace_id=workspace_id ) @@ -287,7 +288,7 @@ def _check_workspace_member_permission(db: Session, workspace_id: uuid.UUID, use ) # 使用统一权限服务检查访问权限 - from app.core.permissions import permission_service, Subject, Resource, Action + from app.core.permissions import Action, Resource, Subject, permission_service # 获取用户的工作空间成员关系 member = workspace_repository.get_member_in_workspace( @@ -325,7 +326,7 @@ def _check_workspace_admin_permission(db: Session, workspace_id: uuid.UUID, user ) # 使用统一权限服务检查管理权限 - from app.core.permissions import permission_service, Subject, Resource, Action + from app.core.permissions import Action, Resource, Subject, permission_service # 获取用户的工作空间成员关系 member = workspace_repository.get_member_in_workspace( @@ -801,4 +802,55 @@ def get_workspace_models_configs( f"成功获取工作空间 {workspace_id} 的模型配置: " f"llm={configs.get('llm')}, embedding={configs.get('embedding')}, rerank={configs.get('rerank')}" ) - return configs \ No newline at end of file + return configs + + +def update_workspace_models_configs( + db: Session, + workspace_id: uuid.UUID, + models_update: WorkspaceModelsUpdate, + user: User, +) -> Workspace: + """更新工作空间的模型配置（llm, embedding, rerank） + + Args: + db: 数据库会话 + workspace_id: 工作空间ID + models_update: 模型配置更新对象 + user: 当前用户 + + Returns: + Workspace: 更新后的工作空间对象 + """ + business_logger.info(f"用户 {user.username} 请求更新工作空间 {workspace_id} 的模型配置") + + # 检查用户是否有管理员权限 + db_workspace = _check_workspace_admin_permission(db, workspace_id, user) + + try: + if models_update.llm is not None: + db_workspace.llm = str(models_update.llm) if models_update.llm else None + business_logger.debug(f"更新LLM配置: {models_update.llm}") + + if models_update.embedding is not None: + db_workspace.embedding = str(models_update.embedding) if models_update.embedding else None + business_logger.debug(f"更新嵌入模型配置: {models_update.embedding}") + + if models_update.rerank is not None: + db_workspace.rerank = str(models_update.rerank) if models_update.rerank else None + business_logger.debug(f"更新重排序模型配置: {models_update.rerank}") + + db.add(db_workspace) + db.commit() + db.refresh(db_workspace) + + business_logger.info( + f"工作空间模型配置更新成功: workspace_id={workspace_id}, " + f"llm={db_workspace.llm}, embedding={db_workspace.embedding}, rerank={db_workspace.rerank}" + ) + return db_workspace + + except Exception as e: + business_logger.error(f"工作空间模型配置更新失败: workspace_id={workspace_id} - {str(e)}") + db.rollback() + raise BusinessException(f"更新模型配置失败: {str(e)}", BizCode.INTERNAL_ERROR) \ No newline at end of file From d8336503bcb91985f6eabde65acfa4aaf9d7d1da Mon Sep 17 00:00:00 2001 From: Ke Sun Date: Wed, 24 Dec 2025 18:21:36 +0800 Subject: [PATCH 11/12] feat(workspace): add workspace models configuration update endpoint - Add PUT endpoint to update workspace LLM, embedding, and rerank model configurations - Create WorkspaceModelsUpdate schema for model configuration update requests - Create WorkspaceModelsConfig schema for model configuration responses with proper validation - Implement update_workspace_models_configs service method to persist model configuration changes - Update workspace_models_configs GET endpoint to return validated WorkspaceModelsConfig response - Reorganize imports across controller, schema, and service files for consistency and readability - Add proper logging for model configuration updates with user and workspace context --- api/app/controllers/workspace_controller.py | 68 +++++++++--- api/app/schemas/workspace_schema.py | 32 +++++- api/app/services/workspace_service.py | 112 ++++++++++++++------ 3 files changed, 163 insertions(+), 49 deletions(-) diff --git a/api/app/controllers/workspace_controller.py b/api/app/controllers/workspace_controller.py index f4390568..eb6065e0 100644 --- a/api/app/controllers/workspace_controller.py +++ b/api/app/controllers/workspace_controller.py @@ -1,25 +1,38 @@ -from fastapi import APIRouter, Depends, HTTPException, status, Query -from sqlalchemy.orm import Session -from typing import List, Optional import uuid +from typing import List, Optional +from app.core.logging_config import get_api_logger from app.core.response_utils import success from app.db import get_db -from app.dependencies import get_current_superuser, get_current_user, get_current_tenant, workspace_access_guard, cur_workspace_access_guard -from app.models.user_model import User +from app.dependencies import ( + cur_workspace_access_guard, + get_current_superuser, + get_current_tenant, + get_current_user, + workspace_access_guard, +) from app.models.tenant_model import Tenants -from app.models.workspace_model import Workspace, InviteStatus +from app.models.user_model import User +from app.models.workspace_model import InviteStatus, Workspace +from app.schemas import knowledge_schema from app.schemas.response_schema import ApiResponse from app.schemas.workspace_schema import ( - WorkspaceCreate, WorkspaceUpdate, WorkspaceResponse, - WorkspaceInviteCreate, WorkspaceInviteResponse, - InviteValidateResponse, InviteAcceptRequest, - WorkspaceMemberUpdate, WorkspaceMemberItem + InviteAcceptRequest, + InviteValidateResponse, + WorkspaceCreate, + WorkspaceInviteCreate, + WorkspaceInviteResponse, + WorkspaceMemberItem, + WorkspaceMemberUpdate, + WorkspaceModelsConfig, + WorkspaceModelsUpdate, + WorkspaceResponse, + WorkspaceUpdate, ) -from app.schemas import knowledge_schema -from app.services import workspace_service -from app.core.logging_config import get_api_logger -from app.services import knowledge_service, document_service +from app.services import document_service, knowledge_service, workspace_service +from fastapi import APIRouter, Depends, HTTPException, Query, status +from sqlalchemy.orm import Session + # 获取API专用日志器 api_logger = get_api_logger() # 需要认证的路由器 @@ -338,5 +351,30 @@ def workspace_models_configs( f"成功获取工作空间 {workspace_id} 的模型配置: " f"llm={configs.get('llm')}, embedding={configs.get('embedding')}, rerank={configs.get('rerank')}" ) - return success(data=configs, msg="模型配置获取成功") + return success(data=WorkspaceModelsConfig.model_validate(configs), msg="模型配置获取成功") + + +@router.put("/workspace_models", response_model=ApiResponse) +@cur_workspace_access_guard() +def update_workspace_models_configs( + models_update: WorkspaceModelsUpdate, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """更新当前工作空间的模型配置（llm, embedding, rerank）""" + workspace_id = current_user.current_workspace_id + api_logger.info(f"用户 {current_user.username} 请求更新工作空间 {workspace_id} 的模型配置") + + updated_workspace = workspace_service.update_workspace_models_configs( + db=db, + workspace_id=workspace_id, + models_update=models_update, + user=current_user + ) + + api_logger.info( + f"成功更新工作空间 {workspace_id} 的模型配置: " + f"llm={updated_workspace.llm}, embedding={updated_workspace.embedding}, rerank={updated_workspace.rerank}" + ) + return success(data=WorkspaceModelsConfig.model_validate(updated_workspace), msg="模型配置更新成功") diff --git a/api/app/schemas/workspace_schema.py b/api/app/schemas/workspace_schema.py index eb3e31e2..1fdfe426 100644 --- a/api/app/schemas/workspace_schema.py +++ b/api/app/schemas/workspace_schema.py @@ -1,9 +1,17 @@ -import email -from pydantic import BaseModel, Field, EmailStr, field_serializer, computed_field, ConfigDict import datetime +import email import uuid -from typing import Literal -from app.models.workspace_model import WorkspaceRole, InviteStatus +from typing import Literal, Optional + +from app.models.workspace_model import InviteStatus, WorkspaceRole +from pydantic import ( + BaseModel, + ConfigDict, + EmailStr, + Field, + computed_field, + field_serializer, +) class WorkspaceBase(BaseModel): @@ -170,3 +178,19 @@ class InviteValidateResponse(BaseModel): class InviteAcceptRequest(BaseModel): token: str = Field(..., description="邀请令牌") + + +class WorkspaceModelsUpdate(BaseModel): + """工作空间模型配置更新请求""" + llm: Optional[uuid.UUID] = Field(default=None, description="LLM模型ID") + embedding: Optional[uuid.UUID] = Field(default=None, description="嵌入模型ID") + rerank: Optional[uuid.UUID] = Field(default=None, description="重排序模型ID") + + +class WorkspaceModelsConfig(BaseModel): + """工作空间模型配置响应""" + model_config = ConfigDict(from_attributes=True) + + llm: Optional[str] = Field(default=None, description="LLM模型ID") + embedding: Optional[str] = Field(default=None, description="嵌入模型ID") + rerank: Optional[str] = Field(default=None, description="重排序模型ID") diff --git a/api/app/services/workspace_service.py b/api/app/services/workspace_service.py index 04ee647c..bb6c53dc 100644 --- a/api/app/services/workspace_service.py +++ b/api/app/services/workspace_service.py @@ -1,36 +1,38 @@ -from sqlalchemy.orm import Session -from typing import List, Optional -import uuid -import secrets -import hashlib import datetime -from fastapi import HTTPException, status +import hashlib +import secrets +import uuid +from os import getenv +from typing import List, Optional + +from app.core.config import settings from app.core.error_codes import BizCode from app.core.exceptions import BusinessException, PermissionDeniedException -from app.models.tenant_model import Tenants +from app.core.logging_config import get_business_logger from app.models.user_model import User -from app.models.app_model import App -from app.models.end_user_model import EndUser -from app.models.workspace_model import Workspace, WorkspaceRole, WorkspaceInvite, InviteStatus, WorkspaceMember -from app.schemas.workspace_schema import ( - WorkspaceCreate, - WorkspaceUpdate, - WorkspaceInviteCreate, - WorkspaceInviteResponse, - InviteValidateResponse, - InviteAcceptRequest, - WorkspaceMemberUpdate +from app.models.workspace_model import ( + InviteStatus, + Workspace, + WorkspaceMember, + WorkspaceRole, ) from app.repositories import workspace_repository from app.repositories.workspace_invite_repository import WorkspaceInviteRepository -from app.core.logging_config import get_business_logger -from app.core.config import settings -from app.services import user_service -from os import getenv +from app.schemas.workspace_schema import ( + InviteAcceptRequest, + InviteValidateResponse, + WorkspaceCreate, + WorkspaceInviteCreate, + WorkspaceInviteResponse, + WorkspaceMemberUpdate, + WorkspaceModelsUpdate, + WorkspaceUpdate, +) +from dotenv import load_dotenv +from sqlalchemy.orm import Session + # 获取业务逻辑专用日志器 business_logger = get_business_logger() -import os # -from dotenv import load_dotenv load_dotenv() def switch_workspace( db: Session, @@ -134,10 +136,9 @@ def create_workspace( f"{db_workspace.id} 创建知识库" ) try: - import os - from app.schemas.knowledge_schema import KnowledgeCreate from app.models.knowledge_model import KnowledgeType, PermissionType from app.repositories import knowledge_repository + from app.schemas.knowledge_schema import KnowledgeCreate # 创建知识库数据 knowledge_data = KnowledgeCreate( @@ -232,7 +233,7 @@ def get_workspace_members( ) # 权限检查：工作空间成员或超级管理员可以查看成员列表 - from app.core.permissions import permission_service, Subject, Resource, Action + from app.core.permissions import Action, Resource, Subject, permission_service member = workspace_repository.get_member_in_workspace( db=db, user_id=user.id, workspace_id=workspace_id ) @@ -287,7 +288,7 @@ def _check_workspace_member_permission(db: Session, workspace_id: uuid.UUID, use ) # 使用统一权限服务检查访问权限 - from app.core.permissions import permission_service, Subject, Resource, Action + from app.core.permissions import Action, Resource, Subject, permission_service # 获取用户的工作空间成员关系 member = workspace_repository.get_member_in_workspace( @@ -325,7 +326,7 @@ def _check_workspace_admin_permission(db: Session, workspace_id: uuid.UUID, user ) # 使用统一权限服务检查管理权限 - from app.core.permissions import permission_service, Subject, Resource, Action + from app.core.permissions import Action, Resource, Subject, permission_service # 获取用户的工作空间成员关系 member = workspace_repository.get_member_in_workspace( @@ -801,4 +802,55 @@ def get_workspace_models_configs( f"成功获取工作空间 {workspace_id} 的模型配置: " f"llm={configs.get('llm')}, embedding={configs.get('embedding')}, rerank={configs.get('rerank')}" ) - return configs \ No newline at end of file + return configs + + +def update_workspace_models_configs( + db: Session, + workspace_id: uuid.UUID, + models_update: WorkspaceModelsUpdate, + user: User, +) -> Workspace: + """更新工作空间的模型配置（llm, embedding, rerank） + + Args: + db: 数据库会话 + workspace_id: 工作空间ID + models_update: 模型配置更新对象 + user: 当前用户 + + Returns: + Workspace: 更新后的工作空间对象 + """ + business_logger.info(f"用户 {user.username} 请求更新工作空间 {workspace_id} 的模型配置") + + # 检查用户是否有管理员权限 + db_workspace = _check_workspace_admin_permission(db, workspace_id, user) + + try: + if models_update.llm is not None: + db_workspace.llm = str(models_update.llm) if models_update.llm else None + business_logger.debug(f"更新LLM配置: {models_update.llm}") + + if models_update.embedding is not None: + db_workspace.embedding = str(models_update.embedding) if models_update.embedding else None + business_logger.debug(f"更新嵌入模型配置: {models_update.embedding}") + + if models_update.rerank is not None: + db_workspace.rerank = str(models_update.rerank) if models_update.rerank else None + business_logger.debug(f"更新重排序模型配置: {models_update.rerank}") + + db.add(db_workspace) + db.commit() + db.refresh(db_workspace) + + business_logger.info( + f"工作空间模型配置更新成功: workspace_id={workspace_id}, " + f"llm={db_workspace.llm}, embedding={db_workspace.embedding}, rerank={db_workspace.rerank}" + ) + return db_workspace + + except Exception as e: + business_logger.error(f"工作空间模型配置更新失败: workspace_id={workspace_id} - {str(e)}") + db.rollback() + raise BusinessException(f"更新模型配置失败: {str(e)}", BizCode.INTERNAL_ERROR) \ No newline at end of file From cf900219a387609147edbfdf61e5f2ed833d651e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=90=E5=8A=9B=E9=BD=90?= Date: Wed, 24 Dec 2025 10:55:05 +0000 Subject: [PATCH 12/12] Merge #48 into develop from fix/othername-name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [fix]Fix the data structure + implement memory dashboard display * fix/othername-name: (18 commits squashed) - [fix]Fix the issue with the display of the user's memory list - [fix]Ensure the six dimensions of emotional expression - [fix]Fix the issue with the display of the user's memory list - [fix]Ensure the six dimensions of emotional expression - Merge branch 'fix/othername-name' of codeup.aliyun.com:redbearai/python/redbear-mem-open into fix/othername-name - [fix]Restore the display of memory types - [fix]Fix the issue with the display of the user's memory list - [fix]Ensure the six dimensions of emotional expression - [fix]Restore the display of memory types - Merge branch 'fix/othername-name' of codeup.aliyun.com:redbearai/python/redbear-mem-open into fix/othername-name - [updated]Update the title of the "analytics/node_statistics" log - [fix]Fix the data structure + implement memory dashboard display - [fix]Fix the issue with the display of the user's memory list - [fix]Restore the display of memory types - [fix]Fix the issue with the display of the user's memory list - [fix]Restore the display of memory types - [fix]Fix the data structure + implement memory dashboard display - Merge branch 'fix/othername-name' of codeup.aliyun.com:redbearai/python/redbear-mem-open into fix/othername-name Signed-off-by: 乐力齐 Reviewed-by: aliyun6762716068 Merged-by: aliyun6762716068 CR-link: https://codeup.aliyun.com/redbearai/python/redbear-mem-open/change/48 --- .../controllers/user_memory_controllers.py | 4 +- api/app/services/memory_dashboard_service.py | 6 +- api/app/services/user_memory_service.py | 81 ++++++++++++------- 3 files changed, 57 insertions(+), 34 deletions(-) diff --git a/api/app/controllers/user_memory_controllers.py b/api/app/controllers/user_memory_controllers.py index 8afb0863..d254992b 100644 --- a/api/app/controllers/user_memory_controllers.py +++ b/api/app/controllers/user_memory_controllers.py @@ -192,7 +192,9 @@ async def get_node_statistics_api( # 调用新的记忆类型统计函数 result = await analytics_memory_types(db, end_user_id) - api_logger.info(f"成功获取记忆类型统计: end_user_id={end_user_id}, 感知记忆={result.get('感知记忆', 0)}") + # 计算总数用于日志 + total_count = sum(item["count"] for item in result) + api_logger.info(f"成功获取记忆类型统计: end_user_id={end_user_id}, 总记忆数={total_count}, 类型数={len(result)}") return success(data=result, msg="查询成功") except Exception as e: api_logger.error(f"记忆类型查询失败: end_user_id={end_user_id}, error={str(e)}") diff --git a/api/app/services/memory_dashboard_service.py b/api/app/services/memory_dashboard_service.py index 9a8ae9aa..a774647e 100644 --- a/api/app/services/memory_dashboard_service.py +++ b/api/app/services/memory_dashboard_service.py @@ -298,10 +298,10 @@ async def get_workspace_total_memory_count( details.append({ "end_user_id": end_user_id_str, "count": host_total, - "name": host.name # 添加 name 字段 + "name": host.other_name # 使用 other_name 字段 }) - business_logger.debug(f"EndUser {end_user_id_str} ({host.name}) 记忆数: {host_total}") + business_logger.debug(f"EndUser {end_user_id_str} ({host.other_name}) 记忆数: {host_total}") except Exception as e: business_logger.warning(f"获取 end_user {host.id} 记忆数失败: {str(e)}") @@ -309,7 +309,7 @@ async def get_workspace_total_memory_count( details.append({ "end_user_id": str(host.id), "count": 0, - "name": host.name # 添加 name 字段 + "name": host.other_name # 使用 other_name 字段 }) result = { diff --git a/api/app/services/user_memory_service.py b/api/app/services/user_memory_service.py index f728fa6f..6a444f27 100644 --- a/api/app/services/user_memory_service.py +++ b/api/app/services/user_memory_service.py @@ -537,35 +537,43 @@ async def analytics_node_statistics( async def analytics_memory_types( db: Session, end_user_id: Optional[str] = None -) -> Dict[str, Any]: +) -> List[Dict[str, Any]]: """ - 统计8种记忆类型的数量 + 统计8种记忆类型的数量和百分比 计算规则： - 1. 感知记忆 = statement + entity - 2. 工作记忆 = chunk + entity - 3. 短期记忆 = chunk - 4. 长期记忆 = entity - 5. 显性记忆 = 1/2 * entity - 6. 隐形记忆 = 1/3 * entity - 7. 情绪记忆 = statement - 8. 情景记忆 = memory_summary + 1. 感知记忆 (PERCEPTUAL_MEMORY) = statement + entity + 2. 工作记忆 (WORKING_MEMORY) = chunk + entity + 3. 短期记忆 (SHORT_TERM_MEMORY) = chunk + 4. 长期记忆 (LONG_TERM_MEMORY) = entity + 5. 显性记忆 (EXPLICIT_MEMORY) = 1/2 * entity + 6. 隐性记忆 (IMPLICIT_MEMORY) = 1/3 * entity + 7. 情绪记忆 (EMOTIONAL_MEMORY) = statement + 8. 情景记忆 (EPISODIC_MEMORY) = memory_summary Args: db: 数据库会话 end_user_id: 可选的终端用户ID (UUID)，用于过滤特定用户的节点 Returns: - { - "感知记忆": int, - "工作记忆": int, - "短期记忆": int, - "长期记忆": int, - "显性记忆": int, - "隐形记忆": int, - "情绪记忆": int, - "情景记忆": int - } + [ + { + "type": str, # 记忆类型枚举值 (如 PERCEPTUAL_MEMORY, WORKING_MEMORY 等) + "count": int, # 该类型的数量 + "percentage": float # 该类型在所有记忆中的占比 + }, + ... + ] + + 记忆类型枚举值： + - PERCEPTUAL_MEMORY: 感知记忆 + - WORKING_MEMORY: 工作记忆 + - SHORT_TERM_MEMORY: 短期记忆 + - LONG_TERM_MEMORY: 长期记忆 + - EXPLICIT_MEMORY: 显性记忆 + - IMPLICIT_MEMORY: 隐性记忆 + - EMOTIONAL_MEMORY: 情绪记忆 + - EPISODIC_MEMORY: 情景记忆 """ # 定义需要查询的节点类型 node_types = { @@ -604,18 +612,31 @@ async def analytics_memory_types( chunk_count = node_counts.get("Chunk", 0) memory_summary_count = node_counts.get("MemorySummary", 0) - # 按规则计算8种记忆类型 - memory_types = { - "感知记忆": statement_count + entity_count, - "工作记忆": chunk_count + entity_count, - "短期记忆": chunk_count, - "长期记忆": entity_count, - "显性记忆": entity_count // 2, # 1/2 entity，使用整除 - "隐形记忆": entity_count // 3, # 1/3 entity，使用整除 - "情绪记忆": statement_count, - "情景记忆": memory_summary_count + # 按规则计算8种记忆类型的数量（使用英文枚举作为key） + memory_counts = { + "PERCEPTUAL_MEMORY": statement_count + entity_count, # 感知记忆 + "WORKING_MEMORY": chunk_count + entity_count, # 工作记忆 + "SHORT_TERM_MEMORY": chunk_count, # 短期记忆 + "LONG_TERM_MEMORY": entity_count, # 长期记忆 + "EXPLICIT_MEMORY": entity_count // 2, # 显性记忆 (1/2 entity) + "IMPLICIT_MEMORY": entity_count // 3, # 隐性记忆 (1/3 entity) + "EMOTIONAL_MEMORY": statement_count, # 情绪记忆 + "EPISODIC_MEMORY": memory_summary_count # 情景记忆 } + # 计算总数 + total = sum(memory_counts.values()) + + # 构建返回数据，包含 type、count 和 percentage + memory_types = [] + for memory_type, count in memory_counts.items(): + percentage = round((count / total * 100), 2) if total > 0 else 0.0 + memory_types.append({ + "type": memory_type, + "count": count, + "percentage": percentage + }) + return memory_types