Fix/memory celery fix (#168)

* refactor(celery): optimize task routing and worker configuration

- Simplify Celery queue configuration with single default 'io_tasks' queue
- Implement task routing strategy separating IO-bound and CPU-bound tasks
- Add Flower monitoring support with task event tracking enabled
- Add summary node search optimization to only retrieve summary nodes
- Clean up unused imports and reorganize import statements for consistency
- Update docker-compose configuration to support multi-queue worker setup

* chore(celery): simplify flower configuration and add gevent dependency

* chore(dependencies): add gevent dependency to requirements

- Add gevent==24.11.1 to api/requirements.txt
- Gevent is required for async worker support in Celery
- Complements existing flower and celery configuration

* refactor(celery): simplify async event loop handling and reorganize task queues

- Replace complex nest_asyncio and manual event loop management with asyncio.run() in read_message_task, write_message_task, regenerate_memory_cache, and workspace_reflection_task
- Rename task queues from io_tasks/cpu_tasks to memory_tasks/document_tasks for better semantic clarity
- Update task routing configuration to reflect new queue names for memory agent tasks and document processing tasks
- Remove redundant exception handling comments and simplify error handling logic
- Update README with improved community support section including GitHub Issues, Pull Requests, Discussions, and WeChat community links
- Simplifies event loop management by leveraging asyncio.run() which handles loop creation and cleanup automatically, reducing code complexity and potential race conditions

api/app/services/draft_run_service.py

@@ -10,11 +10,6 @@ import time
 import uuid
 from typing import Any, AsyncGenerator, Dict, List, Optional
 
-from langchain.tools import tool
-from pydantic import BaseModel, Field
-from sqlalchemy import select
-from sqlalchemy.orm import Session
-
 from app.celery_app import celery_app
 from app.core.error_codes import BizCode
 from app.core.exceptions import BusinessException
@@ -28,6 +23,10 @@ from app.services.langchain_tool_server import Search
 from app.services.memory_agent_service import MemoryAgentService
 from app.services.model_parameter_merger import ModelParameterMerger
 from app.services.tool_service import ToolService
+from langchain.tools import tool
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+from sqlalchemy.orm import Session
 
 logger = get_business_logger()
 class KnowledgeRetrievalInput(BaseModel):
@@ -107,9 +106,9 @@ def create_long_term_memory_tool(memory_config: Dict[str, Any], end_user_id: str
                     "app.core.memory.agent.read_message",
                     args=[end_user_id, question, [], "1", config_id, storage_type, user_rag_memory_id]
                 )
-                result = task_service.get_task_memory_read_result(task.id)
-                status = result.get("status")
-                logger.info(f"读取任务状态：{status}")
+                # result = task_service.get_task_memory_read_result(task.id)
+                # status = result.get("status")
+                # logger.info(f"读取任务状态：{status}")
 
             finally:
                 db.close()

api/app/services/memory_agent_service.py

@@ -10,15 +10,17 @@ import re
 import time
 import uuid
 from typing import Any, AsyncGenerator, Dict, List, Optional
-import redis
-from langchain_core.messages import HumanMessage
 
+import redis
 from app.core.config import settings
 from app.core.logging_config import get_config_logger, get_logger
 from app.core.memory.agent.langgraph_graph.read_graph import make_read_graph
 from app.core.memory.agent.langgraph_graph.write_graph import make_write_graph
 from app.core.memory.agent.logger_file.log_streamer import LogStreamer
-from app.core.memory.agent.utils.messages_tools import merge_multiple_search_results, reorder_output_results
+from app.core.memory.agent.utils.messages_tools import (
+    merge_multiple_search_results,
+    reorder_output_results,
+)
 from app.core.memory.agent.utils.type_classifier import status_typle
 from app.core.memory.agent.utils.write_tools import write  # 新增：直接导入 write 函数
 from app.core.memory.analytics.hot_memory_tags import get_hot_memory_tags
@@ -33,6 +35,7 @@ from app.services.memory_config_service import MemoryConfigService
 from app.services.memory_konwledges_server import (
     write_rag,
 )
+from langchain_core.messages import HumanMessage
 from pydantic import BaseModel, Field
 from sqlalchemy import func
 from sqlalchemy.orm import Session
@@ -404,6 +407,7 @@ class MemoryAgentService:
 
         import time
         start_time = time.time()
+        logger.info(f"[PERF] read_memory started for group_id={group_id}, search_switch={search_switch}")
 
         # Resolve config_id if None using end_user's connected config
         if config_id is None:
@@ -427,13 +431,15 @@ class MemoryAgentService:
             audit_logger = None
 
 
+        config_load_start = time.time()
         try:
             config_service = MemoryConfigService(db)
             memory_config = config_service.load_memory_config(
                 config_id=config_id,
                 service_name="MemoryAgentService"
             )
-            logger.info(f"Configuration loaded successfully: {memory_config.config_name}")
+            config_load_time = time.time() - config_load_start
+            logger.info(f"[PERF] Configuration loaded in {config_load_time:.4f}s: {memory_config.config_name}")
         except ConfigurationError as e:
             error_msg = f"Failed to load configuration for config_id: {config_id}: {e}"
             logger.error(error_msg)
@@ -457,6 +463,7 @@ class MemoryAgentService:
         logger.debug(f"Group ID:{group_id}, Message:{message}, History:{history}, Config ID:{config_id}")
 
         # Step 3: Initialize MCP client and execute read workflow
+        graph_exec_start = time.time()
         try:
             async with make_read_graph() as graph:
                 config = {"configurable": {"thread_id": group_id}}
@@ -513,6 +520,9 @@ class MemoryAgentService:
                         if summary_n and summary_n != [] and summary_n != {}:
                             _intermediate_outputs.append(summary_n)
 
+                graph_exec_time = time.time() - graph_exec_start
+                logger.info(f"[PERF] Graph execution completed in {graph_exec_time:.4f}s")
+
                 _intermediate_outputs = [item for item in _intermediate_outputs if item and item != [] and item != {}]
 
                 optimized_outputs = merge_multiple_search_results(_intermediate_outputs)
@@ -570,6 +580,8 @@ class MemoryAgentService:
                     logger.error(f"保存短期记忆失败: {str(save_error)}", exc_info=True)
 
                 # Log successful operation
+                total_time = time.time() - start_time
+                logger.info(f"[PERF] read_memory completed successfully in {total_time:.4f}s (config: {config_load_time:.4f}s, graph: {graph_exec_time:.4f}s)")
                 if audit_logger:
                     duration = time.time() - start_time
                     audit_logger.log_operation(
@@ -587,7 +599,8 @@ class MemoryAgentService:
         except Exception as e:
             # Ensure proper error handling and logging
             error_msg = f"Read operation failed: {str(e)}"
-            logger.error(error_msg)
+            total_time = time.time() - start_time
+            logger.error(f"[PERF] read_memory failed after {total_time:.4f}s: {error_msg}")
             if audit_logger:
                 duration = time.time() - start_time
                 audit_logger.log_operation(

api/app/services/memory_config_service.py

@@ -125,7 +125,11 @@ class MemoryConfigService:
         try:
             validated_config_id = _validate_config_id(config_id)
             
+            # Step 1: Get config and workspace
+            db_query_start = time.time()
             result = DataConfigRepository.get_config_with_workspace(self.db, validated_config_id)
+            db_query_time = time.time() - db_query_start
+            logger.info(f"[PERF] Config+Workspace query: {db_query_time:.4f}s")
             if not result:
                 elapsed_ms = (time.time() - start_time) * 1000
                 config_logger.error(
@@ -144,16 +148,20 @@ class MemoryConfigService:
             
             memory_config, workspace = result
             
-            # Validate embedding model
-            embedding_uuid = validate_embedding_model(
+            # Step 2: Validate embedding model (returns both UUID and name)
+            embed_start = time.time()
+            embedding_uuid, embedding_name = validate_embedding_model(
                 validated_config_id,
                 memory_config.embedding_id,
                 self.db,
                 workspace.tenant_id,
                 workspace.id,
             )
+            embed_time = time.time() - embed_start
+            logger.info(f"[PERF] Embedding validation: {embed_time:.4f}s")
             
-            # Resolve LLM model
+            # Step 3: Resolve LLM model
+            llm_start = time.time()
             llm_uuid, llm_name = validate_and_resolve_model_id(
                 memory_config.llm_id,
                 "llm",
@@ -163,8 +171,11 @@ class MemoryConfigService:
                 config_id=validated_config_id,
                 workspace_id=workspace.id,
             )
+            llm_time = time.time() - llm_start
+            logger.info(f"[PERF] LLM validation: {llm_time:.4f}s")
             
-            # Resolve optional rerank model
+            # Step 4: Resolve optional rerank model
+            rerank_start = time.time()
             rerank_uuid = None
             rerank_name = None
             if memory_config.rerank_id:
@@ -177,16 +188,12 @@ class MemoryConfigService:
                     config_id=validated_config_id,
                     workspace_id=workspace.id,
                 )
+            rerank_time = time.time() - rerank_start
+            if memory_config.rerank_id:
+                logger.info(f"[PERF] Rerank validation: {rerank_time:.4f}s")
             
-            # Get embedding model name
-            embedding_name, _ = validate_model_exists_and_active(
-                embedding_uuid,
-                "embedding",
-                self.db,
-                workspace.tenant_id,
-                config_id=validated_config_id,
-                workspace_id=workspace.id,
-            )
+            # Note: embedding_name is now returned from validate_embedding_model above
+            # No need for redundant query!
             
             # Create immutable MemoryConfig object
             config = MemoryConfig(