adai/MemoryBear
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(memory): remove legacy extraction pipeline and add dialog_at temporal grounding

Browse Source 
- Delete ExtractionOrchestrator (~2500 lines) and write_tools legacy path;
  MemoryService/WritePipeline is now the sole write path
- Remove NEW_PIPELINE_ENABLED feature flag from memory_agent_service
- Simplify pilot_run_service to always use PilotWritePipeline
- Add dialog_at field to statement and triplet extraction prompts as the
  primary reference time for resolving relative temporal expressions
- Rewrite relative time phrases (e.g. 昨天, 下周) into concrete dates
  directly in statement_text when stably resolvable from dialog_at
- Rename extracat_Pruning.jinja2 to extracat_pruning.jinja2; expand
  few-shot examples and update memory type enum (drop NULL, add
  agreement/repetition/other)
This commit is contained in:
lanceyq
2026-04-30 15:58:08 +08:00
parent cf389bb978
commit 9dc9b7aee7
16 changed files with 386 additions and 3327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
api/app/core/config.py
View File

@@ -272,12 +272,6 @@ class Settings:
MEMORY_OUTPUT_DIR: str = os.getenv("MEMORY_OUTPUT_DIR", "logs/memory-output")
MEMORY_CONFIG_DIR: str = os.getenv("MEMORY_CONFIG_DIR", "app/core/memory")
# Pilot run pipeline switch:
# true -> use refactored PilotWritePipeline
# false -> use legacy ExtractionOrchestrator pipeline
PILOT_RUN_USE_REFACTORED_PIPELINE: bool = (
os.getenv("PILOT_RUN_USE_REFACTORED_PIPELINE", "true").lower() == "true"
)
# Tool Management Configuration
TOOL_CONFIG_DIR: str = os.getenv("TOOL_CONFIG_DIR", "app/core/tools")

67
api/app/core/memory/agent/langgraph_graph/nodes/write_nodes.py
View File

@@ -1,67 +0,0 @@
from app.cache.memory.interest_memory import InterestMemoryCache
from app.core.memory.agent.utils.llm_tools import WriteState
from app.core.memory.agent.utils.write_tools import write
from app.core.logging_config import get_agent_logger
logger = get_agent_logger(__name__)
async def write_node(state: WriteState) -> WriteState:
"""
Write data to the database/file system.
Args:
state: WriteState containing messages, end_user_id, memory_config, and language
Returns:
dict: Contains 'write_result' with status and data fields
"""
messages = state.get('messages', [])
end_user_id = state.get('end_user_id', '')
memory_config = state.get('memory_config', '')
language = state.get('language', 'zh') # 默认中文
# Convert LangChain messages to structured format expected by write()
structured_messages = []
for msg in messages:
if hasattr(msg, 'type') and hasattr(msg, 'content'):
# Map LangChain message types to role names
role = 'user' if msg.type == 'human' else 'assistant' if msg.type == 'ai' else msg.type
structured_messages.append({
"role": role,
"content": msg.content # content is now guaranteed to be a string
})
try:
result = await write(
messages=structured_messages,
end_user_id=end_user_id,
memory_config=memory_config,
language=language,
)
logger.info(f"Write completed successfully! Config: {memory_config.config_name}")
# 写入 neo4j 成功后，删除该用户的兴趣分布缓存，确保下次请求重新生成
for lang in ["zh", "en"]:
deleted = await InterestMemoryCache.delete_interest_distribution(
end_user_id=end_user_id,
language=lang,
)
if deleted:
logger.info(f"Invalidated interest distribution cache: end_user_id={end_user_id}, language={lang}")
write_result = {
"status": "success",
"data": structured_messages,
"config_id": memory_config.config_id,
"config_name": memory_config.config_name,
}
return {"write_result": write_result}
except Exception as e:
logger.error(f"Data_write failed: {e}", exc_info=True)
write_result = {
"status": "error",
"message": str(e),
}
return {"write_result": write_result}

413
api/app/core/memory/agent/utils/write_tools.py
View File

@@ -1,413 +0,0 @@
"""
Write Tools for Memory Knowledge Extraction Pipeline
This module provides the main write function for executing the knowledge extraction
pipeline. Only MemoryConfig is needed - clients are constructed internally.
"""
import asyncio
import time
import uuid
from datetime import datetime
from typing import List, Optional
from dotenv import load_dotenv
from app.core.logging_config import get_agent_logger
from app.core.memory.agent.utils.get_dialogs import get_chunked_dialogs
from app.core.memory.storage_services.extraction_engine.deduplication.deduped_and_disamb import _USER_PLACEHOLDER_NAMES
from app.core.memory.storage_services.extraction_engine.extraction_orchestrator import ExtractionOrchestrator
from app.core.memory.storage_services.extraction_engine.knowledge_extraction.memory_summary import \
memory_summary_generation
from app.core.memory.utils.llm.llm_utils import MemoryClientFactory
from app.core.memory.utils.log.logging_utils import log_time
from app.db import get_db_context
from app.repositories.neo4j.add_edges import add_memory_summary_statement_edges
from app.repositories.neo4j.add_nodes import add_memory_summary_nodes
from app.repositories.neo4j.graph_saver import save_dialog_and_statements_to_neo4j
from app.repositories.neo4j.neo4j_connector import Neo4jConnector
from app.schemas.memory_config_schema import MemoryConfig
load_dotenv()
logger = get_agent_logger(__name__)
async def write(
end_user_id: str,
memory_config: MemoryConfig,
messages: list,
ref_id: str = "",
language: str = "zh",
) -> None:
"""
Execute the complete knowledge extraction pipeline.
Args:
end_user_id: Group identifier
memory_config: MemoryConfig object containing all configuration
messages: Structured message list [{"role": "user", "content": "..."}, ...]
ref_id: Reference ID, defaults to ""
language: 语言类型 ("zh" 中文, "en" 英文)，默认中文
"""
if not ref_id:
ref_id = uuid.uuid4().hex
# Extract config values
embedding_model_id = str(memory_config.embedding_model_id)
chunker_strategy = memory_config.chunker_strategy
config_id = str(memory_config.config_id)
logger.info("=== MemSci Knowledge Extraction Pipeline ===")
logger.info(f"Config: {memory_config.config_name} (ID: {config_id})")
logger.info(f"Workspace: {memory_config.workspace_name}")
logger.info(f"LLM model: {memory_config.llm_model_name}")
logger.info(f"Embedding model: {memory_config.embedding_model_name}")
logger.info(f"Chunker strategy: {chunker_strategy}")
logger.info(f"end_user_id ID: {end_user_id}")
# Construct clients from memory_config using factory pattern with db session
with get_db_context() as db:
factory = MemoryClientFactory(db)
llm_client = factory.get_llm_client_from_config(memory_config)
embedder_client = factory.get_embedder_client_from_config(memory_config)
logger.info("LLM and embedding clients constructed")
# Initialize timing log
log_file = "logs/time.log"
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
with open(log_file, "a", encoding="utf-8") as f:
f.write(f"\n=== Pipeline Run Started: {timestamp} ===\n")
f.write(f"Config: {memory_config.config_name} (ID: {config_id})\n")
pipeline_start = time.time()
# Initialize Neo4j connector
neo4j_connector = Neo4jConnector()
# Step 1: Load and chunk data
step_start = time.time()
chunked_dialogs = await get_chunked_dialogs(
chunker_strategy=chunker_strategy,
end_user_id=end_user_id,
messages=messages,
ref_id=ref_id,
config_id=config_id,
)
log_time("Data Loading & Chunking", time.time() - step_start, log_file)
# Step 2: Initialize and run ExtractionOrchestrator
step_start = time.time()
from app.core.memory.utils.config.config_utils import get_pipeline_config
pipeline_config = get_pipeline_config(memory_config)
# Fetch ontology types if scene_id is configured
ontology_types = None
if memory_config.scene_id:
try:
from app.core.memory.ontology_services.ontology_type_loader import load_ontology_types_for_scene
with get_db_context() as db:
ontology_types = load_ontology_types_for_scene(
scene_id=memory_config.scene_id,
workspace_id=memory_config.workspace_id,
db=db
)
if ontology_types:
logger.info(
f"Loaded {len(ontology_types.types)} ontology types for scene_id: {memory_config.scene_id}"
)
else:
logger.info(f"No ontology classes found for scene_id: {memory_config.scene_id}")
except Exception as e:
logger.warning(
f"Failed to fetch ontology types for scene_id {memory_config.scene_id}: {e}",
exc_info=True
)
orchestrator = ExtractionOrchestrator(
llm_client=llm_client,
embedder_client=embedder_client,
connector=neo4j_connector,
config=pipeline_config,
embedding_id=embedding_model_id,
language=language,
ontology_types=ontology_types,
)
# Run the complete extraction pipeline
(
all_dialogue_nodes,
all_chunk_nodes,
all_statement_nodes,
all_entity_nodes,
all_perceptual_nodes,
all_statement_chunk_edges,
all_statement_entity_edges,
all_entity_entity_edges,
all_perceptual_edges,
all_dedup_details,
) = await orchestrator.run(chunked_dialogs, is_pilot_run=False)
# region TODO 乐力齐 重构流水线切换至生产环境稳定后，移除快照对比代码
# ── Snapshot: 旧流水线萃取结果（按 phase2_step_io_schema_v1.md 格式） ──
from app.core.memory.utils.debug.pipeline_snapshot import PipelineSnapshot
snapshot = PipelineSnapshot("legacy")
# Statement 输出（从 dialog_data_list 中提取）
stmt_snapshot = []
for d in all_dedup_details:
if not hasattr(d, "chunks"):
continue
for c in d.chunks:
for s in c.statements:
stmt_snapshot.append({
"statement_id": s.id,
"statement_text": s.statement,
"statement_type": str(getattr(s, "stmt_type", "")),
"temporal_type": str(getattr(s, "temporal_info", "")),
"relevance": str(getattr(s, "relevence_info", "RELEVANT")),
"speaker": getattr(s, "speaker", "user") or "user",
"valid_at": s.temporal_validity.valid_at if s.temporal_validity else "NULL",
"invalid_at": s.temporal_validity.invalid_at if s.temporal_validity else "NULL",
})
snapshot.save_stage("2_statement_outputs", stmt_snapshot)
# Triplet 输出（从 dialog_data_list 中提取）
triplet_snapshot = {}
for d in all_dedup_details:
if not hasattr(d, "chunks"):
continue
for c in d.chunks:
for s in c.statements:
if s.triplet_extraction_info:
triplet_snapshot[s.id] = {
"entities": [
{
"entity_idx": e.entity_idx, "name": e.name,
"type": e.type, "type_description": getattr(e, "type_description", ""),
"description": e.description,
"is_explicit_memory": getattr(e, "is_explicit_memory", False),
}
for e in s.triplet_extraction_info.entities
],
"triplets": [
{
"subject_name": t.subject_name, "subject_id": t.subject_id,
"predicate": t.predicate,
"predicate_description": getattr(t, "predicate_description", ""),
"object_name": t.object_name, "object_id": t.object_id,
}
for t in s.triplet_extraction_info.triplets
],
}
snapshot.save_stage("3_triplet_outputs", triplet_snapshot)
# 图节点和边（去重后）
snapshot.save_stage("6_nodes_edges_after_dedup", {
"dialogue_nodes_count": len(all_dialogue_nodes),
"chunk_nodes_count": len(all_chunk_nodes),
"statement_nodes_count": len(all_statement_nodes),
"entity_nodes": [
{"id": e.id, "name": e.name, "entity_type": e.entity_type, "type_description": e.type_description, "description": e.description}
for e in all_entity_nodes
],
"entity_entity_edges": [
{
"source": e.source, "target": e.target,
"relation_type": e.relation_type, "relation_type_description": e.relation_type_description, "statement": e.statement,
}
for e in all_entity_entity_edges
],
})
snapshot.save_summary({
"dialogue_count": len(all_dialogue_nodes),
"chunk_count": len(all_chunk_nodes),
"statement_count": len(all_statement_nodes),
"entity_count": len(all_entity_nodes),
"relation_count": len(all_entity_entity_edges),
})
# endregion
log_time("Extraction Pipeline", time.time() - step_start, log_file)
# Step 3: Save all data to Neo4j database
step_start = time.time()
# Neo4j 写入前：清洗用户/AI助手实体之间的别名交叉污染
# 从 Neo4j 查询已有的 AI 助手别名，与本轮实体中的 AI 助手别名合并，
# 确保用户实体的 aliases 不包含 AI 助手的名字
try:
from app.core.memory.storage_services.extraction_engine.deduplication.deduped_and_disamb import (
clean_cross_role_aliases,
fetch_neo4j_assistant_aliases,
)
neo4j_assistant_aliases = set()
if all_entity_nodes:
_eu_id = all_entity_nodes[0].end_user_id
if _eu_id:
neo4j_assistant_aliases = await fetch_neo4j_assistant_aliases(neo4j_connector, _eu_id)
clean_cross_role_aliases(all_entity_nodes, external_assistant_aliases=neo4j_assistant_aliases)
logger.info(f"Neo4j 写入前别名清洗完成，AI助手别名排除集大小: {len(neo4j_assistant_aliases)}")
except Exception as e:
logger.warning(f"Neo4j 写入前别名清洗失败（不影响主流程）: {e}")
# 添加死锁重试机制
max_retries = 3
retry_delay = 1 # 秒
for attempt in range(max_retries):
try:
success = await save_dialog_and_statements_to_neo4j(
dialogue_nodes=all_dialogue_nodes,
chunk_nodes=all_chunk_nodes,
statement_nodes=all_statement_nodes,
entity_nodes=all_entity_nodes,
perceptual_nodes=all_perceptual_nodes,
statement_chunk_edges=all_statement_chunk_edges,
statement_entity_edges=all_statement_entity_edges,
entity_edges=all_entity_entity_edges,
perceptual_edges=all_perceptual_edges,
connector=neo4j_connector,
)
if success:
logger.info("Successfully saved all data to Neo4j")
if all_entity_nodes:
end_user_id = all_entity_nodes[0].end_user_id
# Neo4j 写入完成后，用 PgSQL 权威 aliases 覆盖 Neo4j 用户实体
try:
from app.repositories.end_user_info_repository import EndUserInfoRepository
if end_user_id:
with get_db_context() as db_session:
info = EndUserInfoRepository(db_session).get_by_end_user_id(uuid.UUID(end_user_id))
pg_aliases = info.aliases if info and info.aliases else []
if info is not None:
# 将 Python 侧占位名集合作为参数传入，避免 Cypher 硬编码
placeholder_names = list(_USER_PLACEHOLDER_NAMES)
await neo4j_connector.execute_query(
"""
MATCH (e:ExtractedEntity)
WHERE e.end_user_id = $end_user_id AND toLower(e.name) IN $placeholder_names
SET e.aliases = $aliases
""",
end_user_id=end_user_id, aliases=pg_aliases,
placeholder_names=placeholder_names,
)
logger.info(f"[AliasSync] Neo4j 用户实体 aliases 已用 PgSQL 权威源覆盖: {pg_aliases}")
except Exception as sync_err:
logger.warning(f"[AliasSync] PgSQL→Neo4j aliases 同步失败（不影响主流程）: {sync_err}")
# 使用 Celery 异步任务触发聚类（不阻塞主流程）
try:
from app.tasks import run_incremental_clustering
new_entity_ids = [e.id for e in all_entity_nodes]
task = run_incremental_clustering.apply_async(
kwargs={
"end_user_id": end_user_id,
"new_entity_ids": new_entity_ids,
"llm_model_id": str(memory_config.llm_model_id) if memory_config.llm_model_id else None,
"embedding_model_id": str(memory_config.embedding_model_id) if memory_config.embedding_model_id else None,
},
priority=3,
)
logger.info(
f"[Clustering] 增量聚类任务已提交到 Celery - "
f"task_id={task.id}, end_user_id={end_user_id}, entity_count={len(new_entity_ids)}"
)
except Exception as e:
logger.error(f"[Clustering] 提交聚类任务失败（不影响主流程）: {e}", exc_info=True)
break
else:
logger.warning("Failed to save some data to Neo4j")
if attempt < max_retries - 1:
logger.info(f"Retrying... (attempt {attempt + 2}/{max_retries})")
await asyncio.sleep(retry_delay * (attempt + 1)) # 指数退避
except Exception as e:
error_msg = str(e)
# 检查是否是死锁错误
if "DeadlockDetected" in error_msg or "deadlock" in error_msg.lower():
if attempt < max_retries - 1:
logger.warning(f"Deadlock detected, retrying... (attempt {attempt + 2}/{max_retries})")
await asyncio.sleep(retry_delay * (attempt + 1)) # 指数退避
else:
logger.error(f"Failed after {max_retries} attempts due to deadlock: {e}")
raise
else:
# 非死锁错误，直接抛出
raise
try:
await neo4j_connector.close()
except Exception as e:
logger.error(f"Error closing Neo4j connector: {e}")
log_time("Neo4j Database Save", time.time() - step_start, log_file)
# Step 4: Generate Memory summaries and save to Neo4j
step_start = time.time()
try:
summaries = await memory_summary_generation(
chunked_dialogs, llm_client=llm_client, embedder_client=embedder_client, language=language
)
ms_connector = Neo4jConnector()
try:
await add_memory_summary_nodes(summaries, ms_connector)
await add_memory_summary_statement_edges(summaries, ms_connector)
finally:
try:
await ms_connector.close()
except Exception:
pass
except Exception as e:
logger.error(f"Memory summary step failed: {e}", exc_info=True)
finally:
log_time("Memory Summary (Neo4j)", time.time() - step_start, log_file)
# Log total pipeline time
total_time = time.time() - pipeline_start
log_time("TOTAL PIPELINE TIME", total_time, log_file)
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
with open(log_file, "a", encoding="utf-8") as f:
f.write(f"=== Pipeline Run Completed: {timestamp} ===\n\n")
# 将提取统计写入 Redis，按 workspace_id 存储
try:
from app.cache.memory.activity_stats_cache import ActivityStatsCache
stats_to_cache = {
"chunk_count": len(all_chunk_nodes) if all_chunk_nodes else 0,
"statements_count": len(all_statement_nodes) if all_statement_nodes else 0,
"triplet_entities_count": len(all_entity_nodes) if all_entity_nodes else 0,
"triplet_relations_count": len(all_entity_entity_edges) if all_entity_entity_edges else 0,
"temporal_count": 0,
}
await ActivityStatsCache.set_activity_stats(
workspace_id=str(memory_config.workspace_id),
stats=stats_to_cache,
)
logger.info(f"[WRITE] 活动统计已写入 Redis: workspace_id = {memory_config.workspace_id}")
except Exception as cache_err:
logger.warning(f"[WRITE] 写入活动统计缓存失败（不影响主流程）: {cache_err}", exc_info=True)
# Close LLM/Embedder underlying httpx clients to prevent
# 'RuntimeError: Event loop is closed' during garbage collection
for client_obj in (llm_client, embedder_client):
try:
underlying = getattr(client_obj, 'client', None) or getattr(client_obj, 'model', None)
if underlying is None:
continue
# Unwrap RedBearLLM / RedBearEmbeddings to get the LangChain model
inner = getattr(underlying, '_model', underlying)
# LangChain OpenAI models expose async_client (httpx.AsyncClient)
http_client = getattr(inner, 'async_client', None)
if http_client is not None and hasattr(http_client, 'aclose'):
await http_client.aclose()
except Exception:
pass
logger.info("=== Pipeline Complete ===")
logger.info(f"Total execution time: {total_time:.2f} seconds")

6
api/app/core/memory/storage_services/extraction_engine/data_preprocessing/data_pruning.py
View File

@@ -98,7 +98,7 @@ class SemanticPruner:
self._snapshot = snapshot # PipelineSnapshot 实例，用于输出剪枝快照
# 加载 Jinja2 模板
self.template = prompt_env.get_template("extracat_Pruning.jinja2")
self.template = prompt_env.get_template("extracat_pruning.jinja2")
# LRU 缓存：避免对相同消息对重复调用 LLM
self._cache: OrderedDict[str, AssistantPruningResponse] = OrderedDict()
@@ -360,7 +360,7 @@ class SemanticPruner:
) -> AssistantPruningResponse:
"""调用 LLM 从 User-Assistant 消息对中提取 Assistant 记忆摘要。
使用 extracat_Pruning.jinja2 模板，输入格式：
使用 extracat_pruning.jinja2 模板，输入格式：
{"msgs": [{"role": "User", "msg": "..."}, {"role": "Assistant", "msg": "..."}]}
"""
# 构建模板输入
@@ -387,7 +387,7 @@ class SemanticPruner:
# 渲染模板
rendered = self.template.render(dialog_text=dialog_text)
log_template_rendering("extracat_Pruning.jinja2", {
log_template_rendering("extracat_pruning.jinja2", {
"language": self.language,
})
log_prompt_rendering("pruning-assistant-hint", rendered)

2
api/app/core/memory/storage_services/extraction_engine/data_preprocessing/scene_config.py
View File

@@ -1,7 +1,7 @@
"""
场景特定配置 - 统一填充词库
重要性判断已完全交由 extracat_Pruning.jinja2 提示词 + LLM preserve_tokens 机制承担。
重要性判断已完全交由 extracat_pruning.jinja2 提示词 + LLM preserve_tokens 机制承担。
本模块仅保留统一填充词库（filler_phrases），用于识别无意义寒暄/表情/口头禅。
所有场景共用同一份词库，场景差异由 LLM 语义判断处理。
"""

2425
api/app/core/memory/storage_services/extraction_engine/extraction_orchestrator.py
View File

File diff suppressed because it is too large Load Diff

16
api/app/core/memory/storage_services/extraction_engine/extraction_pipeline_orchestrator.py
View File

@@ -223,6 +223,7 @@ class NewExtractionOrchestrator:
temporal_type=stmt_out.temporal_type,
supporting_context=supporting_context,
speaker=stmt_out.speaker,
dialog_at=stmt_out.dialog_at or "",
valid_at=stmt_out.valid_at,
invalid_at=stmt_out.invalid_at,
has_unsolved_reference=stmt_out.has_unsolved_reference,
@@ -494,10 +495,9 @@ class NewExtractionOrchestrator:
else None
)
for chunk in dialog.chunks:
# 仅对 speaker="user" 的 chunk 进行陈述句抽取；assistant 内容交给
# 上游预处理/剪枝阶段处理，避免浪费 LLM 调用。
chunk_speaker = getattr(chunk, "speaker", "user")
if chunk_speaker != "user":
# 仅跳过明确标记为 assistant 的 chunk；speaker=None（混合分块）正常处理。
chunk_speaker = getattr(chunk, "speaker", None)
if chunk_speaker == "assistant":
continue
inp = StatementStepInput(
chunk_id=chunk.id,
@@ -506,6 +506,7 @@ class NewExtractionOrchestrator:
target_message_date=str(
getattr(dialog, "created_at", "") or ""
),
dialog_at=getattr(chunk, "dialog_at", "") or "",
supporting_context=ctx,
)
tasks.append(self.statement_temporal_step.run(inp))
@@ -561,10 +562,9 @@ class NewExtractionOrchestrator:
chunk_stmts = all_stmt_results.get(dialog.id, {})
for _chunk_id, stmts in chunk_stmts.items():
for stmt in stmts:
# 防御性过滤：三元组抽取仅针对 user statement。
# 上游 _extract_all_statements 已过滤 chunk.speaker，此处再做
# 一次 statement.speaker 的二次校验，防止外部注入或 legacy 数据脱漏。
if getattr(stmt, "speaker", "user") != "user":
# 防御性过滤：跳过明确标记为 assistant 的 statement。
# speaker=None（混合分块）正常处理。
if getattr(stmt, "speaker", None) == "assistant":
continue
inp = self._convert_to_triplet_input(stmt, ctx)
tasks.append(self.triplet_step.run(inp))

3
api/app/core/memory/storage_services/extraction_engine/steps/schema/extraction_step_schema.py
View File

@@ -34,6 +34,7 @@ class StatementStepInput(BaseModel):
end_user_id: str
target_content: str
target_message_date: str
dialog_at: str = "" # ISO 8601 timestamp of the source message; used as "now" for relative time resolution
supporting_context: SupportingContext
@@ -50,6 +51,7 @@ class StatementStepOutput(BaseModel):
valid_at: str # ISO 8601 or "NULL"
invalid_at: str # ISO 8601 or "NULL"
has_unsolved_reference: bool = False # Whether the statement has unresolved references
dialog_at: str = "" # Passed through from input; carried into TripletStepInput
# ── Triplet extraction ──
@@ -62,6 +64,7 @@ class TripletStepInput(BaseModel):
temporal_type: str
supporting_context: SupportingContext
speaker: str
dialog_at: str = "" # ISO 8601 timestamp of the source message; helps LLM ground entity descriptions in time
valid_at: str
invalid_at: str
has_unsolved_reference: bool = False # From upstream statement extraction

3
api/app/core/memory/storage_services/extraction_engine/steps/statement_temporal_step.py
View File

@@ -38,6 +38,7 @@ class _ExtractedStatement(BaseModel):
False,
description="Whether the statement reflects user's emotional state",
)
dialog_at: str = Field("", description="ISO 8601 session timestamp, copied verbatim from input")
valid_at: str = Field("NULL", description="ISO 8601 or NULL")
invalid_at: str = Field("NULL", description="ISO 8601 or NULL")
has_unsolved_reference: bool = Field(False, description="Whether the statement has unresolved references")
@@ -106,6 +107,7 @@ class StatementTemporalExtractionStep(ExtractionStep[StatementStepInput, List[St
input_json = {
"chunk_id": input_data.chunk_id,
"end_user_id": input_data.end_user_id,
"dialog_at": input_data.dialog_at or "",
"target_content": input_data.target_content,
"target_message_date": input_data.target_message_date,
"supporting_context": {
@@ -160,6 +162,7 @@ class StatementTemporalExtractionStep(ExtractionStep[StatementStepInput, List[St
# relevance=stmt.relevance.strip().upper(),
speaker="user", # default; orchestrator overrides from chunk metadata
has_emotional_state=getattr(stmt, "has_emotional_state", False),
dialog_at=input_data.dialog_at or "", # carry through from input
valid_at=stmt.valid_at or "NULL",
invalid_at=stmt.invalid_at or "NULL",
has_unsolved_reference=getattr(stmt, "has_unsolved_reference", False),

1
api/app/core/memory/storage_services/extraction_engine/steps/triplet_step.py
View File

@@ -68,6 +68,7 @@ class TripletExtractionStep(ExtractionStep[TripletStepInput, TripletStepOutput])
]
},
"speaker": input_data.speaker,
"dialog_at": input_data.dialog_at or "",
"valid_at": input_data.valid_at,
"invalid_at": input_data.invalid_at,
"has_unsolved_reference": input_data.has_unsolved_reference,

130
api/app/core/memory/utils/prompt/prompts/extracat_Pruning.jinja2
View File

@@ -1,130 +0,0 @@
你是一个面向记忆存储的 Assistant 辅助信息提取器。
任务：
- 输入是一个 JSON，对话放在 `msgs` 数组里，且数组中只有两条消息：第一条是 `User`，第二条是 `Assistant`。
- 你只处理第二条消息里的 `Assistant.msg`。
- 第一条消息里的 `User.msg` 只用于理解上下文，不允许出现在输出里。
- 你的输出必须包含两个字段：
1. `assistant_memory_hint`
2. `assistant_memory_type`
目标：
- 从 `Assistant.msg` 中提取一条适合后续检索的极短辅助摘要。
- 删除冗长解释、寒暄、礼貌话术、重复复述和空泛铺垫。
- 允许做摘要式改写，但只能保留原消息中已经出现的建议、推荐、提醒、安慰、步骤或其他对后续记忆有帮助的核心内容。
- 如果没有值得保留的信息，`assistant_memory_hint` 输出 `"NULL"`，`assistant_memory_type` 也输出 `"NULL"`。
硬约束：
- 不得改写、复述或输出 `User.msg`。
- 不得捏造新事实、新建议、新步骤、新材料。
- 不得改变 `Assistant` 原始语义和立场。
- 可以压缩、合并、重写 `Assistant.msg`，但必须忠于原内容。
- `assistant_memory_type` 只能从以下枚举中选择：
`comfort | suggestion | recommendation | warning | instruction | NULL`
- 只输出严格 JSON，不要输出解释。
压缩原则：
- 优先保留具体建议、推荐、提醒、操作步骤、风险提示、安慰动作。
- 优先删除长背景解释、寒暄、礼貌收尾、对用户原话的重复复述。
- 如果原文是长说明、长步骤、长菜谱，输出更短的概要版本，但不要丢掉核心意图。
- 优先保留最短但仍有信息密度的版本。
- `assistant_memory_hint` 尽量写成完整句，不要只写零散词组或标签。
- 优先使用显式主语来写结果，例如：
`安慰了用户……`
`建议用户……`
`推荐用户……`
`提醒用户……`
Few-shot 示例 1
输入：
{
"msgs": [
{
"role": "User",
"msg": "我室友小雯这学期一直在准备毕业论文，这两周都在改答辩 PPT。她下周三答辩，我有点担心她会紧张。"
},
{
"role": "Assistant",
"msg": "听起来你很关心小雯，也希望她答辩顺利。她现在紧张其实很正常，很多人在答辩前都会这样。"
}
]
}
输出：
{
"assistant_memory_hint": "安慰了用户对室友答辩状态的担忧。",
"assistant_memory_type": "comfort"
}
Few-shot 示例 2
输入：
{
"msgs": [
{
"role": "User",
"msg": "我最近总失眠，已经两周了，想先自己调一调。"
},
{
"role": "Assistant",
"msg": "如果你想先自己调整，可以先减少咖啡因摄入，尤其下午和晚上尽量不要再喝咖啡或浓茶，同时把睡前刷手机的时间压缩一些，尽量固定上床时间，先连续观察几天。"
}
]
}
输出：
{
"assistant_memory_hint": "建议用户减少咖啡因摄入、减少睡前刷手机时间并固定上床时间。",
"assistant_memory_type": "suggestion"
}
Few-shot 示例 3
输入：
{
"msgs": [
{
"role": "User",
"msg": "我晚上想做个简单点的减脂餐，最好二十分钟左右能搞定。"
},
{
"role": "Assistant",
"msg": "你可以做一个鸡胸肉沙拉碗，主要用鸡胸肉、生菜、黄瓜和圣女果。鸡胸肉简单煎熟切块后和蔬菜拌在一起，调味尽量用橄榄油加一点醋，不要放太多沙拉酱。"
}
]
}
输出：
{
"assistant_memory_hint": "推荐用户做鸡胸肉沙拉碗，并提醒用户调味时少放沙拉酱。",
"assistant_memory_type": "recommendation"
}
Few-shot 示例 4
输入：
{
"msgs": [
{
"role": "User",
"msg": "剪枝引擎和萃取引擎我都想先做，但是估计都会比较花时间。"
},
{
"role": "Assistant",
"msg": "这两个模块都涉及比较多的设计和实现细节。如果你想先推进，我建议先拆需求，再分别评估开发量。"
}
]
}
输出：
{
"assistant_memory_hint": "建议用户先拆需求，再分别评估两个模块的开发量。",
"assistant_memory_type": "suggestion"
}
现在处理下面这个输入。
输入：
{{ dialog_text }}
只输出严格 JSON：
{
"assistant_memory_hint": "<string or NULL>",
"assistant_memory_type": "comfort | suggestion | recommendation | warning | instruction | NULL"
}

180
api/app/core/memory/utils/prompt/prompts/extracat_pruning.jinja2 Normal file
View File

@@ -0,0 +1,180 @@
你是一个面向记忆存储的 Assistant 辅助信息压缩器。
任务：
- 输入是一个 JSON，对话放在 `msgs` 数组里。
- 你只处理 `Assistant.msg`。
- `User.msg` 只用于理解上下文，不允许出现在输出里，也不允许被复述成用户摘要。
- 你的输出必须包含两个字段：
1. `assistant_memory_hint`
2. `assistant_memory_type`
目标：
- 把较长的 `Assistant.msg` 压缩成一条更短、便于检索的辅助摘要。
- 保留建议、推荐、提醒、说明、提问、附和、重复等核心动作。
- 删除冗长解释、寒暄、礼貌套话和低价值铺垫，但不要漏掉真正有用的信息。
硬约束：
- 不得输出或复述 `User.msg`。
- 不得捏造新事实、新建议、新步骤、新材料或新限制。
- 不得改变 `Assistant` 原始语义和立场。
- 可以压缩、合并、重写 `Assistant.msg`，但必须忠于原内容。
- `assistant_memory_hint` 必须是简短的完整句，尽量包含清晰主谓宾，不要只写零散词组。
- 如果 `assistant_memory_hint` 里出现"室友""老师""朋友""同事""这件事"这类泛称，而上下文中存在清晰、稳定、唯一的指代对象，则优先改写成那个清晰指代对象。
- 只有在当前两条消息里无法稳定落到唯一对象时，才保留泛称或模糊表达。
- 如果对象本身已经足够清晰，例如"数据库作业""鸡胸肉沙拉""李教授"，则不要为了"更具体"而做不必要的过度展开。
- `assistant_memory_type` 只能从以下枚举中选择：
`comfort | suggestion | recommendation | warning | instruction | question | agreement | repetition | other`
- 如果 `Assistant.msg` 同时包含多个动作，`assistant_memory_hint` 可以保留多个动作，但 `assistant_memory_type` 只标记其中最主要、最值得检索的主动作。
- 不再输出 `NULL`。即使内容价值较低，也要尽量压成一条最短的辅助摘要。
- 如果 `Assistant.msg` 含有提问、追问或反问，`assistant_memory_hint` 必须保留提问的具体内容，不能只写"询问了用户"。
- 如果提问里给出了明确选项、候选分支或对比项，`assistant_memory_hint` 应尽量保留这些选项，而不是只保留上位概括。
- `question` 只在"提问/追问/反问"是这条消息的主推进动作时使用；如果消息里同时有建议和提问，但建议明显更核心，则类型标为 `suggestion`，并在 hint 里按需保留提问内容。
- 对 `question` 类型，优先保留：
1. 问题的核心主题
2. 明确给出的选项或分支
3. 必要的限定条件
- 对 `question` 类型，不要只保留寒暄式前缀，例如"听起来不错""如果方便的话"；应保留真正要用户回答的部分。
- 只输出严格 JSON，不要输出解释。
压缩原则：
- 优先保留具体建议、推荐、提醒、操作步骤、风险提示和问题内容。
- 对纯附和内容，压成极短摘要，例如"附和了用户对某事的看法。"
- 对明显重复用户内容的回复，压成极短摘要，例如"重复了用户关于某事的说法。"
- 对泛泛回应、空泛鼓励、礼貌性延展，压成最短可理解摘要，并标为 `other`。
- 如果上下文里能确定人名、关系对象或具体事物，优先在摘要里写出明确对象，不要无必要地保留"室友""那个老师""这件事"这类泛称。
- 如果原文里的对象已经明确且自然，就直接保留该对象，不要改写成更绕或更长的表达。
- 如果问题中存在"是 A、B 还是 C"这类显式选项，优先保留 A、B、C，而不是只写成"询问用户偏好"。
- 如果原文既有建议又有提问，允许在 hint 里同时保留；但 type 只标主动作。若提问是核心推进动作，则 type 标为 `question`；若建议更核心，则 type 标为 `suggestion`。
- 优先使用显式主语来写结果，例如：
`安慰了用户……`
`建议用户……`
`推荐用户……`
`提醒用户……`
`询问用户……`
`附和了用户……`
`重复了用户……`
类型判断补充：
- `question`：主动作是向用户提问、追问、澄清、确认选项或收集偏好。
- `suggestion`：主动作是给用户建议；即使末尾顺带问一句，也仍以建议为主。
- `recommendation`：主动作是推荐某个方案、菜谱、产品或选择。
- `warning`：主动作是提醒风险、限制、禁忌或后果。
- `instruction`：主动作是说明操作顺序、步骤或执行流程。
- `comfort`：主动作是安慰、理解、支持用户情绪。
- `agreement`：主动作是附和、认同用户说法。
- `repetition`：主动作是重复、转述用户已有内容，没有新增有效信息。
- `other`：不适合归入以上类型，但仍值得压成一条短摘要。
Few-shot 示例 1
输入：
{
"msgs": [
{
"role": "User",
"msg": "我室友小雯这学期一直在准备毕业论文，这两周都在改答辩PPT。她下周三答辩，我有点担心她会紧张。"
},
{
"role": "Assistant",
"msg": "听起来你很关心小雯，也希望她答辩顺利。她现在紧张其实很正常，很多人在答辩前都会这样。"
}
]
}
输出：
{
"assistant_memory_hint": "安慰了用户对室友小雯答辩状态的担忧。",
"assistant_memory_type": "comfort"
}
Few-shot 示例 2
输入：
{
"msgs": [
{
"role": "User",
"msg": "我最近总失眠，已经两周了，想先自己调一调。"
},
{
"role": "Assistant",
"msg": "如果你想先自己调整，可以先减少咖啡因摄入，尤其下午和晚上尽量不要再喝咖啡或浓茶，同时把睡前刷手机的时间压缩一些，尽量固定上床时间，先连续观察几天。"
}
]
}
输出：
{
"assistant_memory_hint": "建议用户减少咖啡因摄入、减少睡前刷手机时间并固定上床时间。",
"assistant_memory_type": "suggestion"
}
Few-shot 示例 3
输入：
{
"msgs": [
{
"role": "User",
"msg": "我晚上想做个简单点的减脂餐，最好二十分钟左右能搞定。"
},
{
"role": "Assistant",
"msg": "你可以做一个鸡胸肉沙拉碗，主要用鸡胸肉、生菜、黄瓜和圣女果。鸡胸肉简单煎熟切块后和蔬菜拌在一起，调味尽量用橄榄油加一点醋，不要放太多沙拉酱。"
}
]
}
输出：
{
"assistant_memory_hint": "推荐用户做鸡胸肉沙拉碗，并提醒用户调味时少放沙拉酱。",
"assistant_memory_type": "suggestion"
}
Few-shot 示例 4
输入：
{
"msgs": [
{
"role": "User",
"msg": "听起来不错！"
},
{
"role": "Assistant",
"msg": "听起来不错！你最喜欢吃什么类型的沙拉呢？是蔬菜沙拉、水果沙拉还是其他的？如果有任何特定的食材是你最喜欢的，也可以告诉我哦。"
}
]
}
输出：
{
"assistant_memory_hint": "询问用户更喜欢蔬菜沙拉、水果沙拉还是其他类型的沙拉，以及是否有偏好的食材。",
"assistant_memory_type": "question"
}
Few-shot 示例 5
输入：
{
"msgs": [
{
"role": "User",
"msg": "我最近总失眠，白天特别困，想先自己调一调。"
},
{
"role": "Assistant",
"msg": "你可以先减少下午和晚上的咖啡因摄入，睡前也尽量少看手机。如果方便的话，我还想了解一下，你通常晚上大概几点上床、几点真正睡着？"
}
]
}
输出：
{
"assistant_memory_hint": "建议用户减少下午和晚上的咖啡因摄入并减少睡前看手机，同时询问用户通常几点上床和几点入睡。",
"assistant_memory_type": "suggestion"
}
现在处理下面这个输入。
输入：{{ dialog_text }}
只输出严格 JSON：
{
"assistant_memory_hint": "<string>",
"assistant_memory_type": "comfort | suggestion | recommendation | warning | instruction | question | agreement | repetition | other"
}

81
api/app/core/memory/utils/prompt/prompts/extract_statement_temporal.jinja2
View File

@@ -13,6 +13,7 @@
- temporal_type
- has_emotional_state
- has_unsolved_reference
- dialog_at
- valid_at
- invalid_at
@@ -26,6 +27,7 @@ Your task is to identify and extract declarative statements from the provided ta
- temporal_type
- has_emotional_state
- has_unsolved_reference
- dialog_at
- valid_at
- invalid_at
@@ -37,15 +39,19 @@ Each output item should be a structured candidate memory statement.
- chunk_id: chunk 唯一 ID
- end_user_id: 终端用户 ID
- dialog_at: 会话时间，必须是 ISO 8601 时间点
- target_content: 当前要处理的对话片段文本，也是唯一允许被抽取的目标文本
- target_message_date: 目标文本对应的时间，用于解析相对时间表达
- dialog_at: 会话时间，优先作为解析相对时间表达的参考时间
- target_message_date: 目标文本对应的时间，可作为辅助时间背景；当与 dialog_at 同时存在时，优先使用 dialog_at 解析相对时间表达
- supporting_context: 完整对话上下文，仅用于辅助理解 target_content，不能单独贡献新的可抽取事实
- supporting_context.msgs: 按顺序提供的上下文消息，可包含 User 和 Assistant
{% else %}
- chunk_id: unique chunk identifier
- end_user_id: end-user identifier
- dialog_at: session time, which must be an ISO 8601 timestamp
- target_content: the current dialogue fragment to process, and the only text span that may be extracted from
- target_message_date: the reference time for the target content, used for resolving relative temporal expressions
- dialog_at: session time, used as the primary reference for resolving relative temporal expressions
- target_message_date: the time associated with the target content and may serve as supporting temporal context; when both exist, prefer `dialog_at` for resolving relative expressions
- supporting_context: full dialogue context used only to help interpret target_content and must not independently contribute new extractable facts
- supporting_context.msgs: ordered contextual messages, which may include User and Assistant messages
{% endif %}
@@ -114,9 +120,18 @@ statement_type：
时间规则：
- 仅使用目标文本中明确陈述或可由 `target_message_date` 直接解析的时间信息；不要使用外部知识补时间。
- 使用 `target_message_date` 作为“现在”来解释相对时间，例如“昨天”“上周五”“下个月”。
- 优先使用 `dialog_at` 作为“现在”来解释相对时间，例如“昨天”“上周五”“下个月”；只有在 `dialog_at` 缺失时才退回 `target_message_date`。
- 如果相对时间可以稳定落到更具体的中文时间表达，就应直接改写进 `statement_text`，而不要保留原始模糊表达。
- 可稳定具体化的示例包括：
- “昨天” -> “2026年4月29日”
- “前天晚上” -> “2026年4月28日晚上”
- “上周三” -> “2026年4月22日”
- “上个月” -> “2026年3月”
- “下周” -> “2026年5月4日至2026年5月10日”
- 如果相对时间只能粗粒度定位，保留该粗粒度但仍尽量具体化；例如“去年冬天”可以保留为“去年冬天”，不要强行伪精确到具体日期。
- `valid_at` 表示陈述开始成立或生效的时间。
- `invalid_at` 表示陈述结束或不再成立的时间；如果仍在持续，填 `"NULL"`。
- `dialog_at` 表示当前会话时间，每条 statement 都必须原样复制输入中的 `dialog_at`。
- 时间格式优先使用 ISO 8601。
- 对于只有日期没有时分秒的时间，默认使用整天边界，便于后续检索。
- 如果没有明确时间，不要编造时间。
@@ -185,10 +200,19 @@ statement_type:
Temporal rules:
- Use only temporal information explicitly stated in the target text or directly resolvable from `target_message_date`; do not add dates from external knowledge.
- Use `target_message_date` as “now” when interpreting relative expressions such as “yesterday,” “last Friday,” or “next month.”
- Use only temporal information explicitly stated in the target text or directly resolvable from `dialog_at` / `target_message_date`; do not add dates from external knowledge.
- Prefer `dialog_at` as “now” when interpreting relative expressions such as “yesterday,” “last Friday,” or “next month”; only fall back to `target_message_date` when `dialog_at` is unavailable.
- If a relative time can be stably grounded to a more concrete Chinese time phrase, rewrite it directly into `statement_text` rather than keeping the vague source phrase.
- Examples of stable concretization:
- “yesterday” -> “2026年4月29日”
- “the night before last” -> “2026年4月28日晚上”
- “last Wednesday” -> “2026年4月22日”
- “last month” -> “2026年3月”
- “next week” -> “2026年5月4日至2026年5月10日”
- If the relative time can only be grounded coarsely, keep that coarse granularity while still making it as concrete as reasonably possible; for example, “last winter” may stay as “去年冬天” instead of being forced into fake exact dates.
- `valid_at` means when the statement became valid or started to hold.
- `invalid_at` means when the statement ended or stopped being valid; use `"NULL"` if it is still ongoing.
- `dialog_at` is the session timestamp, and every statement must copy the input `dialog_at` verbatim.
- Prefer ISO 8601 for time values.
- When only a date can be resolved, default to full-day boundaries for retrieval use.
- If no explicit time is available, do not invent one.
@@ -213,6 +237,9 @@ temporal_type:
Rewrite boundary:
- Minimal rewriting is allowed only to resolve reference, ellipsis, and temporal ambiguity.
- For resolvable relative time expressions, rewrite them into grounded Chinese time phrases directly inside `statement_text`.
- Do not keep both the vague source phrase and the grounded phrase together; output only the rewritten concrete form.
- Do not fake precision for time expressions that cannot be grounded reliably from `dialog_at`.
- Do not introduce unsupported facts, extra inference, or stylistic summarization.
{% endif %}
@@ -222,6 +249,7 @@ Rewrite boundary:
示例输入: {
"chunk_id": "chunk_a1b2c3d4",
"end_user_id": "eu_12345678",
"dialog_at": "2023-09-04T18:00:00Z",
"target_content": "老李这学期要求还是一如既往地严，不过他讲课确实清晰透彻，而且每节课的结构都特别清楚。就是气场实在太吓人了，我每次被他点名都有点发怵。",
"target_message_date": "2023-09-04T18:00:00",
"supporting_context": {
@@ -247,6 +275,7 @@ Rewrite boundary:
"temporal_type": "DYNAMIC",
"has_emotional_state": false,
"has_unsolved_reference": false,
"dialog_at": "2023-09-04T18:00:00Z",
"valid_at": "2023-09-04T18:00:00",
"invalid_at": "NULL"
},
@@ -257,6 +286,7 @@ Rewrite boundary:
"temporal_type": "ATEMPORAL",
"has_emotional_state": false,
"has_unsolved_reference": false,
"dialog_at": "2023-09-04T18:00:00Z",
"valid_at": "NULL",
"invalid_at": "NULL"
},
@@ -267,6 +297,7 @@ Rewrite boundary:
"temporal_type": "DYNAMIC",
"has_emotional_state": true,
"has_unsolved_reference": false,
"dialog_at": "2023-09-04T18:00:00Z",
"valid_at": "2023-09-04T18:00:00",
"invalid_at": "NULL"
}
@@ -277,6 +308,7 @@ Rewrite boundary:
示例输入: {
"chunk_id": "chunk_b2c3d4e5",
"end_user_id": "eu_12345678",
"dialog_at": "2026-04-01T00:00:00Z",
"target_content": "我最近在学Python，每天晚上都会练一个小时。这周还打算先把基础语法和函数部分过一遍。",
"target_message_date": "2026-04-01T00:00:00",
"supporting_context": {
@@ -302,6 +334,7 @@ Rewrite boundary:
"temporal_type": "DYNAMIC",
"has_emotional_state": false,
"has_unsolved_reference": false,
"dialog_at": "2026-04-01T00:00:00Z",
"valid_at": "2026-04-01T00:00:00",
"invalid_at": "NULL"
},
@@ -312,16 +345,18 @@ Rewrite boundary:
"temporal_type": "DYNAMIC",
"has_emotional_state": false,
"has_unsolved_reference": false,
"dialog_at": "2026-04-01T00:00:00Z",
"valid_at": "2026-04-01T00:00:00",
"invalid_at": "NULL"
},
{
"statement_id": "stmt_u1v2w3x4",
"statement_text": "用户这周打算先复习Python的基础语法和函数部分。",
"statement_text": "用户计划在2026年3月30日至2026年4月5日先复习Python的基础语法和函数部分。",
"statement_type": "FACT",
"temporal_type": "DYNAMIC",
"has_emotional_state": false,
"has_unsolved_reference": false,
"dialog_at": "2026-04-01T00:00:00Z",
"valid_at": "2026-04-01T00:00:00",
"invalid_at": "NULL"
}
@@ -332,6 +367,7 @@ Rewrite boundary:
示例输入: {
"chunk_id": "chunk_c3d4e5f6",
"end_user_id": "eu_12345678",
"dialog_at": "2026-04-01T00:00:00Z",
"target_content": "这周老师新布置的那两个我觉得有点难，而且我昨晚看了半天还是没太搞明白。要是周末再弄不出来，我可能就得去问助教了。",
"target_message_date": "2026-04-01T00:00:00",
"supporting_context": {
@@ -352,31 +388,34 @@ Rewrite boundary:
"statements": [
{
"statement_id": "stmt_y5z6a7b8",
"statement_text": "用户觉得那两个有点难。",
"statement_text": "用户觉得2026年3月30日至2026年4月5日老师新布置的那两个内容有点难。",
"statement_type": "OPINION",
"temporal_type": "DYNAMIC",
"has_emotional_state": true,
"has_unsolved_reference": true,
"dialog_at": "2026-04-01T00:00:00Z",
"valid_at": "2026-04-01T00:00:00",
"invalid_at": "NULL"
},
{
"statement_id": "stmt_c9d0e1f2",
"statement_text": "用户昨晚看了半天那两个还是没太搞明白。",
"statement_text": "用户2026年3月31日晚上看了半天那两个内容还是没太搞明白。",
"statement_type": "FACT",
"temporal_type": "DYNAMIC",
"has_emotional_state": false,
"has_unsolved_reference": true,
"dialog_at": "2026-04-01T00:00:00Z",
"valid_at": "2026-03-31T00:00:00",
"invalid_at": "2026-03-31T23:59:59"
},
{
"statement_id": "stmt_g3h4i5j6",
"statement_text": "如果周末还弄不出来，用户可能会去问助教。",
"statement_text": "如果到2026年4月4日至2026年4月5日还弄不出来，用户可能会去问助教。",
"statement_type": "OTHER",
"temporal_type": "DYNAMIC",
"has_emotional_state": false,
"has_unsolved_reference": true,
"dialog_at": "2026-04-01T00:00:00Z",
"valid_at": "2026-04-01T00:00:00",
"invalid_at": "NULL"
}
@@ -387,6 +426,7 @@ Example 1:
Example Input: {
"chunk_id": "chunk_a1b2c3d4",
"end_user_id": "eu_12345678",
"dialog_at": "2023-09-04T18:00:00Z",
"target_content": "Old Li is just as strict as ever this semester, but he really explains things clearly and the structure of every class is extremely clear. His presence is honestly kind of intimidating, and I get nervous every time he calls on me.",
"target_message_date": "2023-09-04T18:00:00",
"supporting_context": {
@@ -412,6 +452,7 @@ Example Output: {
"temporal_type": "DYNAMIC",
"has_emotional_state": false,
"has_unsolved_reference": false,
"dialog_at": "2023-09-04T18:00:00Z",
"valid_at": "2023-09-04T18:00:00",
"invalid_at": "NULL"
},
@@ -422,6 +463,7 @@ Example Output: {
"temporal_type": "ATEMPORAL",
"has_emotional_state": false,
"has_unsolved_reference": false,
"dialog_at": "2023-09-04T18:00:00Z",
"valid_at": "NULL",
"invalid_at": "NULL"
},
@@ -432,6 +474,7 @@ Example Output: {
"temporal_type": "DYNAMIC",
"has_emotional_state": true,
"has_unsolved_reference": false,
"dialog_at": "2023-09-04T18:00:00Z",
"valid_at": "2023-09-04T18:00:00",
"invalid_at": "NULL"
}
@@ -442,6 +485,7 @@ Example 2:
Example Input: {
"chunk_id": "chunk_b2c3d4e5",
"end_user_id": "eu_12345678",
"dialog_at": "2026-04-01T00:00:00Z",
"target_content": "I've been learning Python recently, and I practice for an hour every night. This week I also plan to review basic syntax and functions first.",
"target_message_date": "2026-04-01T00:00:00",
"supporting_context": {
@@ -467,6 +511,7 @@ Example Output: {
"temporal_type": "DYNAMIC",
"has_emotional_state": false,
"has_unsolved_reference": false,
"dialog_at": "2026-04-01T00:00:00Z",
"valid_at": "2026-04-01T00:00:00",
"invalid_at": "NULL"
},
@@ -477,16 +522,18 @@ Example Output: {
"temporal_type": "DYNAMIC",
"has_emotional_state": false,
"has_unsolved_reference": false,
"dialog_at": "2026-04-01T00:00:00Z",
"valid_at": "2026-04-01T00:00:00",
"invalid_at": "NULL"
},
{
"statement_id": "stmt_u1v2w3x4",
"statement_text": "The user plans to review Python basic syntax and functions first this week.",
"statement_text": "The user plans to review Python basic syntax and functions first during 2026-03-30 to 2026-04-05.",
"statement_type": "FACT",
"temporal_type": "DYNAMIC",
"has_emotional_state": false,
"has_unsolved_reference": false,
"dialog_at": "2026-04-01T00:00:00Z",
"valid_at": "2026-04-01T00:00:00",
"invalid_at": "NULL"
}
@@ -497,6 +544,7 @@ Example 3:
Example Input: {
"chunk_id": "chunk_c3d4e5f6",
"end_user_id": "eu_12345678",
"dialog_at": "2026-04-01T00:00:00Z",
"target_content": "The two things the teacher assigned this week seem hard to me, and even after looking at them for a long time last night I still didn't really understand them. If I still can't finish them by the weekend, I may have to ask the TA.",
"target_message_date": "2026-04-01T00:00:00",
"supporting_context": {
@@ -517,31 +565,34 @@ Example Output: {
"statements": [
{
"statement_id": "stmt_y5z6a7b8",
"statement_text": "The user thinks those two things are difficult.",
"statement_text": "The user thinks the two items assigned during 2026-03-30 to 2026-04-05 are difficult.",
"statement_type": "OPINION",
"temporal_type": "DYNAMIC",
"has_emotional_state": true,
"has_unsolved_reference": true,
"dialog_at": "2026-04-01T00:00:00Z",
"valid_at": "2026-04-01T00:00:00",
"invalid_at": "NULL"
},
{
"statement_id": "stmt_c9d0e1f2",
"statement_text": "The user spent a long time last night looking at those two things but still did not really understand them.",
"statement_text": "The user spent a long time on the evening of 2026-03-31 looking at those two items but still did not really understand them.",
"statement_type": "FACT",
"temporal_type": "DYNAMIC",
"has_emotional_state": false,
"has_unsolved_reference": true,
"dialog_at": "2026-04-01T00:00:00Z",
"valid_at": "2026-03-31T00:00:00",
"invalid_at": "2026-03-31T23:59:59"
},
{
"statement_id": "stmt_g3h4i5j6",
"statement_text": "If the user still cannot finish them by the weekend, the user may ask the TA.",
"statement_text": "If the user still cannot finish them by 2026-04-04 to 2026-04-05, the user may ask the TA.",
"statement_type": "OTHER",
"temporal_type": "DYNAMIC",
"has_emotional_state": false,
"has_unsolved_reference": true,
"dialog_at": "2026-04-01T00:00:00Z",
"valid_at": "2026-04-01T00:00:00",
"invalid_at": "NULL"
}
@@ -557,6 +608,7 @@ Example Output: {
- 如果主语是用户，是否统一写“用户”
- 非用户主体是否尽量写成具体名称；若无法做到，是否已正确标记 `has_unsolved_reference = true`
- 如果最终 `statement_text` 已经落到具体实体名，`has_unsolved_reference` 是否已经改为 `false`
- 如果 `statement_text` 中出现可由 `dialog_at` 稳定解析的相对时间，是否已经改写成更具体的日期、月份或日期区间表达
- statement_type 是否合法，且没有把一般事实机械标成 `OPINION`
- `has_emotional_state` 是否仅用于判断是否存在情感状态，而没有被当作情绪分类字段
- temporal_type 是否与 valid_at / invalid_at 一致
@@ -567,6 +619,7 @@ Example Output: {
- If the subject is the user, render it as “the user”
- Render non-user subjects as concrete names when possible; otherwise mark `has_unsolved_reference = true`
- If the final `statement_text` already resolves the reference to a concrete named entity, ensure `has_unsolved_reference = false`
- If `statement_text` contains relative time expressions that can be stably resolved from `dialog_at`, rewrite them into more concrete date, month, or date-range expressions
- Ensure statement_type is valid and do not mechanically label ordinary facts as `OPINION`
- Ensure `has_emotional_state` is used only for emotional-state presence detection, not emotion classification
- Ensure temporal_type is consistent with valid_at and invalid_at
@@ -584,6 +637,7 @@ Example Output: {
**ISO 8601 HARD CONSTRAINT:**
- `dialog_at` must be ISO 8601.
- `target_message_date` must be ISO 8601.
- `valid_at` and `invalid_at` must be ISO 8601, or `"NULL"` when no time is available.
- Do not output non-ISO values such as `2026/04/01`, `2026-04-01 00:00:00`, `yesterday evening`, or `下周三`.
@@ -615,6 +669,7 @@ Return only a JSON object matching the schema below:
"temporal_type": "STATIC | DYNAMIC | ATEMPORAL",
"has_emotional_state": "boolean",
"has_unsolved_reference": "boolean",
"dialog_at": "string",
"valid_at": "string | NULL",
"invalid_at": "string | NULL"
}

10
api/app/core/memory/utils/prompt/prompts/extract_triplet.jinja2
View File

@@ -39,6 +39,7 @@ Extract entities and knowledge triplets from the given statement.
- `supporting_context.msgs[].role`: `User` / `Assistant`
- `supporting_context.msgs[].msg`: 消息文本
- `speaker`: `user` / `assistant`
- `dialog_at`: 会话时间，ISO 8601 时间点；可用于在 `description` 中标注实体的时间背景
- `valid_at`: ISO 8601 时间点，或 `NULL`
- `invalid_at`: ISO 8601 时间点，或 `NULL`
- `has_unsolved_reference`: 布尔值
@@ -53,6 +54,7 @@ Extract entities and knowledge triplets from the given statement.
- `supporting_context.msgs[].role`: `User` / `Assistant`
- `supporting_context.msgs[].msg`: message text
- `speaker`: `user` / `assistant`
- `dialog_at`: session time as an ISO 8601 timestamp; may be used to anchor temporal context in entity `description`
- `valid_at`: ISO 8601 timestamp or `NULL`
- `invalid_at`: ISO 8601 timestamp or `NULL`
- `has_unsolved_reference`: boolean
@@ -113,6 +115,7 @@ Primary statement to analyze:
- 如果某条信息只出现在 `supporting_context.msgs` 中，而没有出现在 `statement_text` 中，就不要输出它。
- 如果 `supporting_context.msgs` 中的 Assistant 消息包含总结、猜测、解释或改写，这些内容只能作为理解辅助，不能直接作为抽取来源。
- `statement_type`、`temporal_type` 是辅助理解字段，不是抽取目标。
- `dialog_at` 是辅助时间上下文字段，不是抽取目标。
- `valid_at`、`invalid_at` 不用于决定是否创建实体或关系，但如果产生 triplet，必须原样复制到每个 triplet 的同名字段中。
- 对 `statement_text` 中的用户自指表达，要统一规范成实体 `用户`。
- 对其他可稳定解析的代词或指示表达，要替换为具体指代实体名后再抽取。
@@ -124,6 +127,7 @@ Primary statement to analyze:
- If some information appears only in `supporting_context.msgs` but not in `statement_text`, do not include it in the output.
- If Assistant messages in `supporting_context.msgs` contain summary, guess, interpretation, or rephrasing, use them only as interpretive support and never as a direct extraction source.
- Treat `statement_type` and `temporal_type` as auxiliary context, not extraction targets.
- Treat `dialog_at` as auxiliary session-time context, not an extraction target.
- Do not use `valid_at` or `invalid_at` to decide whether to create entities or relations, but if any triplet is produced, copy them verbatim into every triplet field with the same names.
- Normalize user self-reference in `statement_text` to the entity `用户`.
- Replace other resolvable pronouns or demonstratives with their resolved entity names before extraction.
@@ -493,6 +497,8 @@ Do not let auxiliary fields drive the extraction process.
- 优先描述实体在当前陈述和必要上下文中的身份、作用或关系。
- `description` 只保留适合长期附着在该实体上的描述，例如稳定身份、稳定关系、长期偏好/兴趣/习惯、较稳定认知倾向或可用于区分实体的持久特征。
- 不要把短期状态、一次性事件、临时计划、当前情绪、具体时间锚点，或只在当前句子里短暂成立的信息写进 `description`。
- 但如果第一步已经把相对时间稳定改写成具体日期、月份或日期区间，且这段具体时间对识别当前实体有帮助，可以在 `description` 中沿用这段已经出现在 `statement_text` 里的具体时间表达。
- triplet 这一步不要自己新增时间推理；只允许复用 `statement_text` 中已经具体化的时间表述，不要把“上周三”“上个月”再次自行展开。
- 如果实体应保留，但当前 statement 中没有适合长期附着在该实体上的稳定描述，则 `description` 允许为空字符串 `""`；不要为了填充 `description` 而写入短期状态或临时信息。
- 避免使用“陈述中提到的人物”“陈述中提到的组织”“陈述中提到的物品”这类低信息量模板。
- 不要补充识别实体所不需要的外部知识。
@@ -501,6 +507,8 @@ Do not let auxiliary fields drive the extraction process.
- Prefer describing the entity's role, identity, or relation in the current statement and necessary supporting context.
- `description` should keep only information suitable to remain attached to the entity over time, such as stable identity, stable relations, long-term preferences/interests/habits, relatively stable beliefs, or persistent distinguishing traits.
- Do not put short-lived states, one-off events, temporary plans, current emotions, concrete time anchors, or information that only briefly holds in the current sentence into `description`.
- But if step 1 has already rewritten a relative time into a concrete date, month, or date range, and that concrete time phrase helps identify the current entity, you may reuse that already-grounded phrase in `description`.
- Do not perform new temporal inference in the triplet step; only reuse time wording that is already concretized in `statement_text`, and do not independently expand phrases like "last Wednesday" or "last month" again here.
- If an entity should be retained but the current statement does not provide any suitable stable description for it, `description` may be the empty string `""`; do not fill it with short-lived states or temporary information just to avoid emptiness.
- Avoid low-information templates such as "the person mentioned in the statement" or "the organization mentioned in the statement".
- Do not add extra world knowledge that is not needed for identifying the entity in context.
@@ -659,7 +667,7 @@ Output:
}
**示例 4**
Statement: "他上个月加入了这家公司。"
Statement: "他2026年3月加入了这家公司。"
Input condition: `"has_unsolved_reference": true`
Output:

33
api/app/services/memory_agent_service.py
View File

@@ -34,7 +34,6 @@ from app.core.memory.agent.utils.messages_tools import (
reorder_output_results,
)
from app.core.memory.agent.utils.type_classifier import status_typle
from app.core.memory.agent.utils.write_tools import write as write_neo4j
from app.core.memory.analytics.hot_memory_tags import get_interest_distribution
from app.core.memory.memory_service import MemoryService
from app.core.memory.utils.llm.llm_utils import MemoryClientFactory
@@ -447,32 +446,20 @@ class MemoryAgentService:
memory_config,
language: Language | str,
) -> None:
"""根据 NEW_PIPELINE_ENABLED 选择新旧流水线写入 Neo4j。"""
# 统一转换为 dict，下游流水线期望 list[dict]
"""使用新流水线（MemoryService → WritePipeline）写入 Neo4j。"""
messages_dict = [
msg if isinstance(msg, dict) else msg.model_dump(exclude_none=True)
for msg in messages
]
use_new_pipeline = os.getenv("NEW_PIPELINE_ENABLED", "false").lower() == "true"
if use_new_pipeline:
service = MemoryService(memory_config=memory_config, end_user_id=end_user_id)
result = await service.write(
messages=messages_dict, language=language, ref_id='',
)
logger.info(
f"[NewPipeline] 完成: status={result.status}, "
f"elapsed={result.elapsed_seconds:.2f}s, "
f"extraction={result.extraction}"
)
else:
await write_neo4j(
end_user_id=end_user_id,
messages=messages_dict,
memory_config=memory_config,
ref_id='',
language=language,
)
service = MemoryService(memory_config=memory_config, end_user_id=end_user_id)
result = await service.write(
messages=messages_dict, language=language, ref_id='',
)
logger.info(
f"[WritePipeline] 完成: status={result.status}, "
f"elapsed={result.elapsed_seconds:.2f}s, "
f"extraction={result.extraction}"
)
async def _invalidate_interest_cache(self, end_user_id: str) -> None:
"""写入完成后失效兴趣分布缓存。"""

337
api/app/services/pilot_run_service.py
View File

@@ -2,6 +2,11 @@
Pilot Run Service - 试运行服务
用于执行记忆系统的试运行流程，不保存到 Neo4j。
职责边界：
- 文本解析、语义剪枝、语义分块（预处理）
- 调用 PilotWritePipeline 执行萃取链路
- 输出结果文件
"""
import os
@@ -17,17 +22,10 @@ from app.core.memory.models.message_models import (
ConversationMessage,
DialogData,
)
from app.core.memory.storage_services.extraction_engine.extraction_orchestrator import (
ExtractionOrchestrator,
get_chunked_dialogs_from_preprocessed,
)
from app.core.memory.storage_services.extraction_engine.pipeline_help import (
_write_extracted_result_summary,
export_test_input_doc,
)
from app.core.memory.utils.config.config_utils import get_pipeline_config
from app.core.memory.utils.llm.llm_utils import MemoryClientFactory
from app.repositories.neo4j.neo4j_connector import Neo4jConnector
from app.schemas.memory_config_schema import MemoryConfig
from sqlalchemy.orm import Session
@@ -77,18 +75,19 @@ async def run_pilot_extraction(
progress_callback: Optional[Callable[[str, str, Optional[dict]], Awaitable[None]]] = None,
language: str = "zh",
) -> None:
"""
执行试运行模式的知识提取流水线。
"""执行试运行模式的知识提取流水线。
职责：
1. 文本解析 → 语义剪枝 → 语义分块（预处理，需要 llm_client）
2. 调用 PilotWritePipeline 执行萃取链路（Pipeline 自行管理客户端）
3. 将萃取结果写入输出文件
Args:
memory_config: 从数据库加载的内存配置对象
dialogue_text: 输入的对话文本
db: 数据库会话
progress_callback: 可选的进度回调函数
- 参数1 (stage): 当前处理阶段标识符
- 参数2 (message): 人类可读的进度消息
- 参数3 (data): 可选的附加数据字典
language: 语言类型 ("zh" 中文, "en" 英文)，默认中文
db: 数据库会话（用于初始化预处理所需的 LLM 客户端）
progress_callback: 可选的进度回调 (stage, message, data)
language: 语言类型 ("zh" | "en")
"""
log_file = "logs/time.log"
os.makedirs(os.path.dirname(log_file), exist_ok=True)
@@ -99,21 +98,16 @@ async def run_pilot_extraction(
pipeline_start = time.time()
try:
# 步骤 1: 初始化客户端
logger.info("Initializing clients...")
# ── 步骤 1: 初始化预处理所需的 LLM 客户端 ──────────────────────────
# 只用于语义剪枝和分块，PilotWritePipeline 内部会自行初始化萃取客户端
step_start = time.time()
client_factory = MemoryClientFactory(db)
llm_client = client_factory.get_llm_client(str(memory_config.llm_model_id))
embedder_client = client_factory.get_embedder_client(str(memory_config.embedding_model_id))
from app.core.memory.utils.llm.llm_utils import MemoryClientFactory
factory = MemoryClientFactory(db)
llm_client = factory.get_llm_client(str(memory_config.llm_model_id))
log_time("Client Initialization", time.time() - step_start, log_file)
# 步骤 2: 解析对话文本
logger.info("Parsing dialogue text...")
# ── 步骤 2: 文本解析 ────────────────────────────────────────────────
step_start = time.time()
# 解析对话文本，支持 "用户:" 和 "AI:" 格式
pattern = r"(用户|AI)[：:]\s*([^\n]+(?:\n(?!(?:用户|AI)[：:])[^\n]*)*?)"
matches = re.findall(pattern, dialogue_text, re.MULTILINE | re.DOTALL)
messages = [
@@ -121,14 +115,11 @@ async def run_pilot_extraction(
for r, c in matches
if c.strip()
]
# 如果没有匹配到格式化的对话，将整个文本作为用户消息
if not messages:
messages = [ConversationMessage(role="用户", msg=dialogue_text.strip())]
context = ConversationContext(msgs=messages)
dialog = DialogData(
context=context,
context=ConversationContext(msgs=messages),
ref_id="pilot_dialog_1",
end_user_id=str(memory_config.workspace_id),
user_id=str(memory_config.tenant_id),
@@ -139,267 +130,142 @@ async def run_pilot_extraction(
if progress_callback:
await progress_callback("text_preprocessing", "开始预处理文本（语义剪枝 + 语义分块）...")
# ========== 步骤 2.1: 语义剪枝 ==========
# ── 步骤 2.1: 语义剪枝 ─────────────────────────────────────────────
pruned_dialogs = [dialog]
deleted_messages = [] # 记录被删除的消息
pruning_stats = None # 保存剪枝统计信息，用于最终汇总
pruning_stats: dict = {"enabled": False}
if memory_config.pruning_enabled:
try:
from app.core.memory.storage_services.extraction_engine.data_preprocessing.data_pruning import (
SemanticPruner,
)
from app.core.memory.models.config_models import PruningConfig
# 构建剪枝配置
pruning_config_dict = {
"pruning_switch": memory_config.pruning_enabled,
"pruning_scene": memory_config.pruning_scene,
"pruning_threshold": memory_config.pruning_threshold,
"scene_id": str(memory_config.scene_id) if memory_config.scene_id else None,
"ontology_class_infos": memory_config.ontology_class_infos,
}
config = PruningConfig(**pruning_config_dict)
logger.info(f"[PILOT_RUN] 开始语义剪枝: scene={config.pruning_scene}, threshold={config.pruning_threshold}")
# 记录剪枝前的消息（用于对比）
original_messages = [{"role": msg.role, "content": msg.msg} for msg in dialog.context.msgs]
original_msg_count = len(original_messages)
# 执行剪枝
pruner = SemanticPruner(config=config, llm_client=llm_client)
pruned_dialogs = await pruner.prune_dataset([dialog])
# 计算剪枝结果并找出被删除的消息
config = PruningConfig(
pruning_switch=memory_config.pruning_enabled,
pruning_scene=memory_config.pruning_scene,
pruning_threshold=memory_config.pruning_threshold,
scene_id=str(memory_config.scene_id) if memory_config.scene_id else None,
ontology_class_infos=memory_config.ontology_class_infos,
)
original_msgs = [{"role": m.role, "content": m.msg} for m in dialog.context.msgs]
pruned_dialogs = await SemanticPruner(config=config, llm_client=llm_client).prune_dataset([dialog])
if pruned_dialogs and pruned_dialogs[0].context:
remaining_messages = [{"role": msg.role, "content": msg.msg} for msg in pruned_dialogs[0].context.msgs]
remaining_msg_count = len(remaining_messages)
deleted_msg_count = original_msg_count - remaining_msg_count
# 找出被删除的消息（基于索引精确匹配）
# 为剩余消息创建带索引的列表，用于精确追踪
remaining_with_index = []
remaining_idx = 0
for orig_idx, orig_msg in enumerate(original_messages):
if remaining_idx < len(remaining_messages) and \
orig_msg["role"] == remaining_messages[remaining_idx]["role"] and \
orig_msg["content"] == remaining_messages[remaining_idx]["content"]:
remaining_with_index.append(orig_idx)
remaining_idx += 1
# 找出未在保留列表中的消息索引
remaining = [{"role": m.role, "content": m.msg} for m in pruned_dialogs[0].context.msgs]
# 找出被删除的消息（顺序匹配）
kept_indices: list[int] = []
ri = 0
for oi, om in enumerate(original_msgs):
if ri < len(remaining) and om == remaining[ri]:
kept_indices.append(oi)
ri += 1
deleted_messages = [
{"index": idx, "role": msg["role"], "content": msg["content"]}
for idx, msg in enumerate(original_messages)
if idx not in remaining_with_index
{"index": i, "role": m["role"], "content": m["content"]}
for i, m in enumerate(original_msgs)
if i not in kept_indices
]
# 保存剪枝统计信息（用于最终汇总，只保留deleted_count）
pruning_stats = {
"enabled": True,
"scene": config.pruning_scene,
"threshold": config.pruning_threshold,
"deleted_count": deleted_msg_count,
"deleted_count": len(deleted_messages),
}
# 输出剪枝结果（显示删除的消息详情）
pruning_result = {
"type": "pruning",
"deleted_messages": deleted_messages,
}
logger.info(
f"[PILOT_RUN] 语义剪枝完成: 原始{original_msg_count}条 -> "
f"保留{remaining_msg_count}条 (删除{deleted_msg_count}条)"
f"[PILOT_RUN] 语义剪枝完成: {len(original_msgs)} → {len(remaining)} 条"
f"（删除 {len(deleted_messages)} 条）"
)
if progress_callback:
await progress_callback("text_preprocessing_result", "语义剪枝完成", pruning_result)
await progress_callback(
"text_preprocessing_result", "语义剪枝完成",
{"type": "pruning", "deleted_messages": deleted_messages},
)
else:
logger.warning("[PILOT_RUN] 剪枝后对话为空，使用原始对话")
pruned_dialogs = [dialog]
except Exception as e:
logger.error(f"[PILOT_RUN] 语义剪枝失败，使用原始对话: {e}", exc_info=True)
pruned_dialogs = [dialog]
if progress_callback:
error_result = {
"type": "pruning",
"error": str(e),
"fallback": "使用原始对话"
}
await progress_callback("text_preprocessing_result", "语义剪枝失败", error_result)
else:
logger.info("[PILOT_RUN] 语义剪枝已关闭，跳过")
pruning_stats = {
"enabled": False,
}
await progress_callback(
"text_preprocessing_result", "语义剪枝失败",
{"type": "pruning", "error": str(e), "fallback": "使用原始对话"},
)
# ========== 步骤 2.2: 语义分块 ==========
chunked_dialogs = await get_chunked_dialogs_from_preprocessed(
data=pruned_dialogs,
chunker_strategy=memory_config.chunker_strategy,
llm_client=llm_client,
# ── 步骤 2.2: 语义分块 ─────────────────────────────────────────────
from app.core.memory.storage_services.extraction_engine.knowledge_extraction.chunk_extraction import (
DialogueChunker,
)
remaining_msg_count = len(pruned_dialogs[0].context.msgs) if pruned_dialogs and pruned_dialogs[0].context else 0
logger.info(f"Processed dialogue text: {remaining_msg_count} messages after pruning")
chunked_dialogs = []
for dlg in pruned_dialogs:
dlg.chunks = await DialogueChunker(memory_config.chunker_strategy, llm_client=llm_client).process_dialogue(dlg)
chunked_dialogs.append(dlg)
# 进度回调：输出每个分块的结果
if progress_callback:
for dlg in chunked_dialogs:
if hasattr(dlg, 'chunks') and dlg.chunks:
for i, chunk in enumerate(dlg.chunks):
chunk_result = {
for i, chunk in enumerate(dlg.chunks or []):
await progress_callback(
"text_preprocessing_result", f"分块 {i + 1} 处理完成",
{
"type": "chunking",
"chunk_index": i + 1,
"content": chunk.content[:200] + "..." if len(chunk.content) > 200 else chunk.content,
"full_length": len(chunk.content),
"dialog_id": dlg.id,
"chunker_strategy": memory_config.chunker_strategy,
}
await progress_callback("text_preprocessing_result", f"分块 {i + 1} 处理完成", chunk_result)
# 构建预处理完成总结（包含剪枝统计）
preprocessing_summary = {
"total_chunks": sum(len(dlg.chunks) for dlg in chunked_dialogs if hasattr(dlg, 'chunks') and dlg.chunks),
"total_dialogs": len(chunked_dialogs),
"chunker_strategy": memory_config.chunker_strategy,
}
# 添加剪枝统计信息（始终包含 pruning 字段，确保前端不会因字段缺失报错）
preprocessing_summary["pruning"] = pruning_stats if pruning_stats else {
"enabled": memory_config.pruning_enabled,
"deleted_count": 0,
}
await progress_callback("text_preprocessing_complete", "预处理文本完成（剪枝 + 分块）", preprocessing_summary)
},
)
await progress_callback(
"text_preprocessing_complete", "预处理文本完成（剪枝 + 分块）",
{
"total_chunks": sum(len(dlg.chunks or []) for dlg in chunked_dialogs),
"total_dialogs": len(chunked_dialogs),
"chunker_strategy": memory_config.chunker_strategy,
"pruning": pruning_stats,
},
)
log_time("Data Loading & Chunking", time.time() - step_start, log_file)
# 步骤 3: 初始化并选择试运行流水线（环境变量可切换）
use_refactored = bool(settings.PILOT_RUN_USE_REFACTORED_PIPELINE)
logger.info(
"Selecting pilot pipeline by env: PILOT_RUN_USE_REFACTORED_PIPELINE=%s",
use_refactored,
)
logger.info(
"Initializing %s pilot pipeline...",
"refactored" if use_refactored else "legacy",
)
# ── 步骤 3: 萃取（PilotWritePipeline 自行管理客户端和本体加载）──────
step_start = time.time()
logger.info("Running pilot extraction pipeline...")
# 加载本体类型（如果配置了 scene_id），支持通用类型回退
ontology_types = None
try:
from app.core.memory.ontology_services.ontology_type_loader import load_ontology_types_with_fallback
ontology_types = load_ontology_types_with_fallback(
scene_id=memory_config.scene_id,
workspace_id=memory_config.workspace_id,
db=db,
enable_general_fallback=True
)
except Exception as e:
logger.warning(f"Failed to load ontology types: {e}", exc_info=True)
if progress_callback:
await progress_callback("knowledge_extraction", "正在知识抽取...")
if use_refactored:
from app.core.memory.memory_service import MemoryService
from app.core.memory.pipelines.pilot_write_pipeline import PilotWritePipeline
memory_service = MemoryService(
memory_config=memory_config,
end_user_id=str(memory_config.workspace_id),
)
log_time("Pilot Pipeline Initialization", time.time() - step_start, log_file)
# 步骤 4a: 执行重构后试运行短链路
# statement -> triplet -> graph_build -> 第一层去重消歧（结束）
logger.info("Running refactored pilot extraction short pipeline...")
step_start = time.time()
if progress_callback:
await progress_callback("knowledge_extraction", "正在知识抽取...")
pilot_result = await memory_service.pilot_write(
chunked_dialogs=chunked_dialogs,
language=language,
progress_callback=progress_callback,
)
dialog_data_list = pilot_result.dialog_data_list
graph = pilot_result.graph
chunk_nodes = graph.chunk_nodes
export_entity_nodes = graph.entity_nodes
export_stmt_entity_edges = graph.stmt_entity_edges
export_entity_edges = graph.entity_entity_edges
else:
# 步骤 4b: 执行旧试运行流水线
logger.info("Running legacy pilot extraction pipeline...")
step_start = time.time()
if progress_callback:
await progress_callback("knowledge_extraction", "正在知识抽取...")
neo4j_connector = Neo4jConnector()
try:
legacy_orchestrator = ExtractionOrchestrator(
llm_client=llm_client,
embedder_client=embedder_client,
connector=neo4j_connector,
config=get_pipeline_config(memory_config),
progress_callback=progress_callback,
embedding_id=str(memory_config.embedding_model_id),
language=language,
ontology_types=ontology_types,
)
extraction_result = await legacy_orchestrator.run(
dialog_data_list=chunked_dialogs,
is_pilot_run=True,
)
(
_dialogue_nodes,
chunk_nodes,
_statement_nodes,
entity_nodes,
_perceptual_nodes,
_statement_chunk_edges,
statement_entity_edges,
entity_edges,
_perceptual_edges,
_last_created_at,
) = extraction_result
dialog_data_list = chunked_dialogs
export_entity_nodes = entity_nodes
export_stmt_entity_edges = statement_entity_edges
export_entity_edges = entity_edges
finally:
try:
await neo4j_connector.close()
except Exception:
pass
pilot_result = await PilotWritePipeline(
memory_config=memory_config,
end_user_id=str(memory_config.workspace_id),
language=language,
progress_callback=progress_callback,
).run(chunked_dialogs)
log_time("Extraction Pipeline", time.time() - step_start, log_file)
# ── 步骤 4: 输出结果文件 ────────────────────────────────────────────
if progress_callback:
await progress_callback("generating_results", "正在生成结果...")
# 步骤 5: 输出试运行结果文件（保持 /pilot_run 返回契约）
graph = pilot_result.graph
settings.ensure_memory_output_dir()
export_test_input_doc(
entity_nodes=export_entity_nodes,
statement_entity_edges=export_stmt_entity_edges,
entity_entity_edges=export_entity_edges,
entity_nodes=graph.entity_nodes,
statement_entity_edges=graph.stmt_entity_edges,
entity_entity_edges=graph.entity_entity_edges,
)
_save_triplets_from_dialogs(
dialog_data_list=dialog_data_list,
dialog_data_list=pilot_result.dialog_data_list,
output_path=settings.get_memory_output_path("extracted_triplets.txt"),
)
_write_extracted_result_summary(
chunk_nodes=chunk_nodes,
chunk_nodes=graph.chunk_nodes,
pipeline_output_dir=settings.get_memory_output_path(),
)
logger.info("Pilot run completed: stop after layer-1 dedup (no layer-2 / no Neo4j write)")
logger.info("Pilot run completed: stop after layer-1 dedup (no Neo4j write)")
except Exception as e:
logger.error(f"Pilot run failed: {e}", exc_info=True)
@@ -407,9 +273,6 @@ async def run_pilot_extraction(
total_time = time.time() - pipeline_start
log_time("TOTAL PILOT RUN TIME", total_time, log_file)
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
with open(log_file, "a", encoding="utf-8") as f:
f.write(f"=== Pilot Run Completed: {timestamp} ===\n\n")
f.write(f"=== Pilot Run Completed: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ===\n\n")
logger.info(f"Pilot run complete. Total time: {total_time:.2f}s")