adai/MemoryBear
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/release/v0.2.9' into develop

Browse Source
This commit is contained in:
Ke Sun
2026-03-31 19:16:13 +08:00
parent 87c2419186 fe29141437
commit 3ea42ac27f
55 changed files with 1482 additions and 570 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
api/app/repositories/conversation_repository.py
View File

@@ -199,6 +199,96 @@ class ConversationRepository:
)
return conversations, total
def list_app_conversations(
self,
app_id: uuid.UUID,
workspace_id: uuid.UUID,
is_draft: Optional[bool] = None,
page: int = 1,
pagesize: int = 20
) -> tuple[list[Conversation], int]:
"""
查询应用日志会话列表（带分页和过滤）
Args:
app_id: 应用 ID
workspace_id: 工作空间 ID
is_draft: 是否草稿会话（None 表示不过滤）
page: 页码（从 1 开始）
pagesize: 每页数量
Returns:
Tuple[List[Conversation], int]: (会话列表，总数)
"""
stmt = select(Conversation).where(
Conversation.app_id == app_id,
Conversation.workspace_id == workspace_id,
Conversation.is_active.is_(True)
)
if is_draft is not None:
stmt = stmt.where(Conversation.is_draft == is_draft)
# Calculate total number of records
total = int(self.db.execute(
select(func.count()).select_from(stmt.subquery())
).scalar_one())
# Apply pagination
stmt = stmt.order_by(desc(Conversation.updated_at))
stmt = stmt.offset((page - 1) * pagesize).limit(pagesize)
conversations = list(self.db.scalars(stmt).all())
logger.info(
"Listed app conversations successfully",
extra={
"app_id": str(app_id),
"workspace_id": str(workspace_id),
"returned": len(conversations),
"total": total
}
)
return conversations, total
def get_conversation_for_app_log(
self,
conversation_id: uuid.UUID,
app_id: uuid.UUID,
workspace_id: uuid.UUID
) -> Conversation:
"""
查询应用日志的会话详情
Args:
conversation_id: 会话 ID
app_id: 应用 ID
workspace_id: 工作空间 ID
Returns:
Conversation: 会话对象
Raises:
ResourceNotFoundException: 当会话不存在时
"""
logger.info(f"Fetching conversation for app log: {conversation_id}")
stmt = select(Conversation).where(
Conversation.id == conversation_id,
Conversation.app_id == app_id,
Conversation.workspace_id == workspace_id,
Conversation.is_active.is_(True)
)
conversation = self.db.scalars(stmt).first()
if not conversation:
logger.warning(f"Conversation not found: {conversation_id}")
raise ResourceNotFoundException("会话", str(conversation_id))
logger.info(f"Conversation fetched successfully: {conversation_id}")
return conversation
def soft_delete_conversation_by_conversation_id(
self,
conversation_id: uuid.UUID,
@@ -290,6 +380,34 @@ class MessageRepository:
self.db.add(message)
return message
def get_messages_by_conversation(
self,
conversation_id: uuid.UUID
) -> list[Message]:
"""
查询会话的所有消息（按时间正序）
Args:
conversation_id: 会话 ID
Returns:
List[Message]: 消息列表
"""
stmt = select(Message).where(
Message.conversation_id == conversation_id
).order_by(Message.created_at)
messages = list(self.db.scalars(stmt).all())
logger.info(
"Fetched messages for conversation",
extra={
"conversation_id": str(conversation_id),
"message_count": len(messages)
}
)
return messages
def get_message_by_conversation_id(
self,
conversation_id: uuid.UUID,

121
api/app/repositories/end_user_repository.py
View File

@@ -132,6 +132,82 @@ class EndUserRepository:
db_logger.error(f"获取或创建终端用户时出错: {str(e)}")
raise
def get_or_create_end_user_with_config(
self,
app_id: Optional[uuid.UUID],
workspace_id: uuid.UUID,
other_id: str,
memory_config_id: Optional[uuid.UUID] = None,
other_name: Optional[str] = None
) -> EndUser:
"""获取或创建终端用户，并在单次事务中关联记忆配置。
与 get_or_create_end_user 类似，但额外支持在创建/获取时
一并设置 memory_config_id，避免多次提交。
Args:
app_id: 应用ID（可为 None）
workspace_id: 工作空间ID
other_id: 第三方ID
memory_config_id: 记忆配置ID（可选，仅在用户尚无配置时设置）
other_name: 用户名称（用于创建 EndUserInfo）
Returns:
EndUser: 终端用户对象（已关联记忆配置）
"""
try:
end_user = (
self.db.query(EndUser)
.filter(
EndUser.workspace_id == workspace_id,
EndUser.other_id == other_id
)
.order_by(EndUser.created_at.asc())
.first()
)
if end_user:
db_logger.debug(f"找到现有终端用户: workspace_id={workspace_id}, other_id={other_id}")
if app_id is not None:
end_user.app_id = app_id
if memory_config_id and not end_user.memory_config_id:
end_user.memory_config_id = memory_config_id
self.db.commit()
self.db.refresh(end_user)
return end_user
# 创建新用户
end_user = EndUser(
app_id=app_id,
workspace_id=workspace_id,
other_id=other_id,
memory_config_id=memory_config_id,
)
self.db.add(end_user)
self.db.flush()
end_user_info = EndUserInfo(
end_user_id=end_user.id,
other_name=other_name or "",
aliases=[],
meta_data={}
)
self.db.add(end_user_info)
self.db.commit()
self.db.refresh(end_user)
db_logger.info(
f"创建新终端用户及其信息: (other_id: {other_id}) for workspace {workspace_id}, "
f"memory_config_id={memory_config_id}"
)
return end_user
except Exception as e:
self.db.rollback()
db_logger.error(f"获取或创建终端用户(含配置)时出错: {str(e)}")
raise
def get_by_id(self, end_user_id: uuid.UUID) -> Optional[EndUser]:
"""根据ID获取终端用户（用于缓存操作）
@@ -515,6 +591,51 @@ class EndUserRepository:
)
raise
def batch_update_memory_config_id_by_app(
self,
app_id: uuid.UUID,
memory_config_id: uuid.UUID
) -> int:
"""批量更新应用下所有终端用户的 memory_config_id
Args:
app_id: 应用ID
memory_config_id: 新的记忆配置ID
Returns:
int: 更新的终端用户数量
Raises:
Exception: 数据库操作失败时抛出
"""
try:
from sqlalchemy import update
stmt = (
update(EndUser)
.where(EndUser.app_id == app_id)
.values(memory_config_id=memory_config_id)
)
result = self.db.execute(stmt)
self.db.commit()
updated_count = result.rowcount
db_logger.info(
f"批量更新终端用户记忆配置: app_id={app_id}, "
f"memory_config_id={memory_config_id}, updated_count={updated_count}"
)
return updated_count
except Exception as e:
self.db.rollback()
db_logger.error(
f"批量更新终端用户记忆配置时出错: app_id={app_id}, "
f"memory_config_id={memory_config_id}, error={str(e)}"
)
raise
def count_by_memory_config_id(
self,
memory_config_id: uuid.UUID

231
api/app/repositories/neo4j/create_indexes.py
View File

@@ -1,62 +1,47 @@
import asyncio
from app.repositories.neo4j.neo4j_connector import Neo4jConnector
async def create_fulltext_indexes():
"""Create full-text indexes for keyword search with BM25 scoring."""
connector = Neo4jConnector()
try:
print("\n" + "=" * 70)
print("Creating Full-Text Indexes (for keyword search)")
print("=" * 70)
# 创建 Statements 索引
await connector.execute_query("""
CREATE FULLTEXT INDEX statementsFulltext IF NOT EXISTS FOR (s:Statement) ON EACH [s.statement]
OPTIONS { indexConfig: { `fulltext.analyzer`: 'cjk' } }
""")
print("✓ Created: statementsFulltext")
""")
# # 创建 Dialogues 索引
# await connector.execute_query("""
# CREATE FULLTEXT INDEX dialoguesFulltext IF NOT EXISTS FOR (d:Dialogue) ON EACH [d.content]
# OPTIONS { indexConfig: { `fulltext.analyzer`: 'cjk' } }
# """)
# 创建 Entities 索引
await connector.execute_query("""
CREATE FULLTEXT INDEX entitiesFulltext IF NOT EXISTS FOR (e:ExtractedEntity) ON EACH [e.name]
OPTIONS { indexConfig: { `fulltext.analyzer`: 'cjk' } }
""")
print("✓ Created: entitiesFulltext")
""")
# 创建 Chunks 索引
await connector.execute_query("""
CREATE FULLTEXT INDEX chunksFulltext IF NOT EXISTS FOR (c:Chunk) ON EACH [c.content]
OPTIONS { indexConfig: { `fulltext.analyzer`: 'cjk' } }
""")
print("✓ Created: chunksFulltext")
""")
# 创建 MemorySummary 索引
await connector.execute_query("""
CREATE FULLTEXT INDEX summariesFulltext IF NOT EXISTS FOR (m:MemorySummary) ON EACH [m.content]
OPTIONS { indexConfig: { `fulltext.analyzer`: 'cjk' } }
""")
print("✓ Created: summariesFulltext")
""")
# 创建 Community 索引
await connector.execute_query("""
CREATE FULLTEXT INDEX communitiesFulltext IF NOT EXISTS FOR (c:Community) ON EACH [c.name, c.summary]
OPTIONS { indexConfig: { `fulltext.analyzer`: 'cjk' } }
""")
print("✓ Created: communitiesFulltext")
print("\nFull-text indexes created successfully with BM25 support.")
except Exception as e:
print(f"✗ Error creating full-text indexes: {e}")
finally:
await connector.close()
async def create_vector_indexes():
"""Create vector indexes for fast embedding similarity search.
@@ -65,12 +50,7 @@ async def create_vector_indexes():
"""
connector = Neo4jConnector()
try:
print("\n" + "=" * 70)
print("Creating Vector Indexes (for embedding search)")
print("=" * 70)
print("Note: Adjust vector.dimensions if using different embedding model")
print(" Current setting: 1024 dimensions (for bge-m3)")
print()
# Statement embedding index
await connector.execute_query("""
@@ -82,7 +62,7 @@ async def create_vector_indexes():
`vector.similarity_function`: 'cosine'
}}
""")
print("✓ Created: statement_embedding_index")
# Chunk embedding index
await connector.execute_query("""
@@ -94,7 +74,7 @@ async def create_vector_indexes():
`vector.similarity_function`: 'cosine'
}}
""")
print("✓ Created: chunk_embedding_index")
# Entity name embedding index
await connector.execute_query("""
@@ -106,7 +86,7 @@ async def create_vector_indexes():
`vector.similarity_function`: 'cosine'
}}
""")
print("✓ Created: entity_embedding_index")
# Memory summary embedding index
await connector.execute_query("""
@@ -118,8 +98,7 @@ async def create_vector_indexes():
`vector.similarity_function`: 'cosine'
}}
""")
print("✓ Created: summary_embedding_index")
# Community summary embedding index
await connector.execute_query("""
CREATE VECTOR INDEX community_summary_embedding_index IF NOT EXISTS
@@ -129,8 +108,7 @@ async def create_vector_indexes():
`vector.dimensions`: 1024,
`vector.similarity_function`: 'cosine'
}}
""")
print("✓ Created: community_summary_embedding_index")
""")
# Dialogue embedding index (optional)
await connector.execute_query("""
@@ -142,91 +120,15 @@ async def create_vector_indexes():
`vector.similarity_function`: 'cosine'
}}
""")
print("✓ Created: dialogue_embedding_index")
# Community summary embedding index
await connector.execute_query("""
CREATE VECTOR INDEX community_summary_embedding_index IF NOT EXISTS
FOR (c:Community)
ON c.summary_embedding
OPTIONS {indexConfig: {
`vector.dimensions`: 1024,
`vector.similarity_function`: 'cosine'
}}
""")
print("✓ Created: community_summary_embedding_index")
print("\nVector indexes created successfully!")
print("\nExpected performance improvement:")
print(" Before: ~1.4s for embedding search")
print(" After: ~0.05-0.2s for embedding search (10-30x faster!)")
except Exception as e:
print(f"✗ Error creating vector indexes: {e}")
finally:
await connector.close()
async def create_config_id_indexes():
"""Create indexes on config_id fields for improved query performance.
These indexes enable fast filtering of nodes by configuration ID,
which is essential for configuration isolation and multi-tenant scenarios.
"""
connector = Neo4jConnector()
try:
print("\n" + "=" * 70)
print("Creating Config ID Indexes")
print("=" * 70)
# Dialogue.config_id index
await connector.execute_query("""
CREATE INDEX dialogue_config_id_index IF NOT EXISTS
FOR (d:Dialogue) ON (d.config_id)
""")
print("✓ Created: dialogue_config_id_index")
# Statement.config_id index
await connector.execute_query("""
CREATE INDEX statement_config_id_index IF NOT EXISTS
FOR (s:Statement) ON (s.config_id)
""")
print("✓ Created: statement_config_id_index")
# ExtractedEntity.config_id index
await connector.execute_query("""
CREATE INDEX entity_config_id_index IF NOT EXISTS
FOR (e:ExtractedEntity) ON (e.config_id)
""")
print("✓ Created: entity_config_id_index")
# MemorySummary.config_id index
await connector.execute_query("""
CREATE INDEX summary_config_id_index IF NOT EXISTS
FOR (m:MemorySummary) ON (m.config_id)
""")
print("✓ Created: summary_config_id_index")
print("\nConfig ID indexes created successfully!")
print("These indexes enable fast filtering by configuration ID.")
except Exception as e:
print(f"✗ Error creating config_id indexes: {e}")
finally:
await connector.close()
async def create_unique_constraints():
"""Create uniqueness constraints for core node identifiers.
Ensures concurrent MERGE operations remain safe and prevents duplicates.
"""
connector = Neo4jConnector()
try:
print("\n" + "=" * 70)
print("Creating Unique Constraints")
print("=" * 70)
try:
# Dialogue.id unique
await connector.execute_query(
"""
@@ -234,8 +136,7 @@ async def create_unique_constraints():
FOR (d:Dialogue) REQUIRE d.id IS UNIQUE
"""
)
print("✓ Created: dialog_id_unique")
# Statement.id unique
await connector.execute_query(
"""
@@ -243,8 +144,7 @@ async def create_unique_constraints():
FOR (s:Statement) REQUIRE s.id IS UNIQUE
"""
)
print("✓ Created: statement_id_unique")
# Chunk.id unique
await connector.execute_query(
"""
@@ -252,112 +152,13 @@ async def create_unique_constraints():
FOR (c:Chunk) REQUIRE c.id IS UNIQUE
"""
)
print("✓ Created: chunk_id_unique")
print("\nUnique constraints ensured for Dialogue, Statement, and Chunk.")
except Exception as e:
print(f"✗ Error creating unique constraints: {e}")
finally:
await connector.close()
async def create_all_indexes():
"""Create all indexes and constraints in one go."""
print("\n" + "=" * 70)
print("Neo4j Index & Constraint Setup")
print("=" * 70)
print("This will create:")
print(" 1. Full-text indexes (for keyword/BM25 search)")
print(" 2. Vector indexes (for embedding similarity search)")
print(" 3. Config ID indexes (for configuration isolation)")
print(" 4. Unique constraints (for data integrity)")
print("=" * 70)
await create_fulltext_indexes()
await create_vector_indexes()
await create_config_id_indexes()
await create_unique_constraints()
print("\n" + "=" * 70)
print("✓ All indexes and constraints created successfully!")
print("=" * 70)
print("\nTo verify, run in Neo4j Browser:")
print(" SHOW INDEXES")
print(" SHOW CONSTRAINTS")
print()
async def check_indexes():
"""Check what indexes currently exist."""
connector = Neo4jConnector()
try:
print("\n" + "=" * 70)
print("Checking Existing Indexes")
print("=" * 70)
query = "SHOW INDEXES"
result = await connector.execute_query(query)
fulltext_indexes = [idx for idx in result if idx.get('type') == 'FULLTEXT']
vector_indexes = [idx for idx in result if idx.get('type') == 'VECTOR']
range_indexes = [idx for idx in result if idx.get('type') == 'RANGE']
print(f"\nFull-text indexes: {len(fulltext_indexes)}")
for idx in fulltext_indexes:
print(f" ✓ {idx.get('name')}")
print(f"\nVector indexes: {len(vector_indexes)}")
for idx in vector_indexes:
print(f" ✓ {idx.get('name')}")
print(f"\nRange indexes (including config_id): {len(range_indexes)}")
for idx in range_indexes:
print(f" ✓ {idx.get('name')}")
if not vector_indexes:
print("\n⚠️ WARNING: No vector indexes found!")
print(" Embedding search will be VERY SLOW (~1.4s)")
print(" Run: python create_indexes.py")
# Check for config_id indexes
config_id_indexes = [idx for idx in range_indexes if 'config_id' in idx.get('name', '')]
if len(config_id_indexes) < 4:
print("\n⚠️ WARNING: Not all config_id indexes found!")
print(f" Expected 4, found {len(config_id_indexes)}")
print(" Run: python create_indexes.py config_id")
print("=" * 70)
finally:
await connector.close()
if __name__ == "__main__":
import asyncio
import sys
if len(sys.argv) > 1:
command = sys.argv[1]
if command == "check":
asyncio.run(check_indexes())
elif command == "fulltext":
asyncio.run(create_fulltext_indexes())
elif command == "vector":
asyncio.run(create_vector_indexes())
elif command == "config_id":
asyncio.run(create_config_id_indexes())
elif command == "constraints":
asyncio.run(create_unique_constraints())
else:
print(f"Unknown command: {command}")
print("\nUsage:")
print(" python create_indexes.py # Create all indexes")
print(" python create_indexes.py check # Check existing indexes")
print(" python create_indexes.py fulltext # Create only full-text indexes")
print(" python create_indexes.py vector # Create only vector indexes")
print(" python create_indexes.py config_id # Create only config_id indexes")
print(" python create_indexes.py constraints # Create only constraints")
else:
asyncio.run(create_all_indexes())

23
api/app/repositories/neo4j/cypher_queries.py
View File

@@ -340,17 +340,22 @@ SEARCH_ENTITIES_BY_NAME_OR_ALIAS = """
CALL db.index.fulltext.queryNodes("entitiesFulltext", $q) YIELD node AS e, score
WHERE ($end_user_id IS NULL OR e.end_user_id = $end_user_id)
WITH e, score
UNION
MATCH (e:ExtractedEntity)
WHERE ($end_user_id IS NULL OR e.end_user_id = $end_user_id)
AND e.aliases IS NOT NULL
AND ANY(alias IN e.aliases WHERE toLower(alias) CONTAINS toLower($q))
WITH e,
WITH collect({entity: e, score: score}) AS fulltextResults
OPTIONAL MATCH (ae:ExtractedEntity)
WHERE ($end_user_id IS NULL OR ae.end_user_id = $end_user_id)
AND ae.aliases IS NOT NULL
AND ANY(alias IN ae.aliases WHERE toLower(alias) CONTAINS toLower($q))
WITH fulltextResults, collect(ae) AS aliasEntities
UNWIND (fulltextResults + [x IN aliasEntities | {entity: x, score:
CASE
WHEN ANY(alias IN e.aliases WHERE toLower(alias) = toLower($q)) THEN 1.0
WHEN ANY(alias IN e.aliases WHERE toLower(alias) STARTS WITH toLower($q)) THEN 0.9
WHEN ANY(alias IN x.aliases WHERE toLower(alias) = toLower($q)) THEN 1.0
WHEN ANY(alias IN x.aliases WHERE toLower(alias) STARTS WITH toLower($q)) THEN 0.9
ELSE 0.8
END AS score
END
}]) AS row
WITH row.entity AS e, row.score AS score
WITH DISTINCT e, MAX(score) AS score
OPTIONAL MATCH (s:Statement)-[:REFERENCES_ENTITY]->(e)
OPTIONAL MATCH (c:Chunk)-[:CONTAINS]->(s)

30
api/app/repositories/user_repository.py
View File

@@ -158,22 +158,26 @@ class UserRepository:
raise
def get_users_by_tenant(
self,
tenant_id: uuid.UUID,
skip: int = 0,
self,
tenant_id: uuid.UUID,
skip: int = 0,
limit: int = 100,
is_active: Optional[bool] = None,
is_superuser: Optional[bool] = None,
search: Optional[str] = None
) -> List[User]:
"""获取租户下的用户列表"""
db_logger.debug(f"查询租户用户: tenant_id={tenant_id}")
try:
query = self.db.query(User).options(joinedload(User.tenant)).filter(User.tenant_id == tenant_id)
if is_active is not None:
query = query.filter(User.is_active == is_active)
if is_superuser is not None:
query = query.filter(User.is_superuser == is_superuser)
if search:
query = query.filter(
or_(
@@ -181,7 +185,7 @@ class UserRepository:
User.email.ilike(f"%{search}%")
)
)
users = query.offset(skip).limit(limit).all()
db_logger.debug(f"租户用户查询成功: tenant_id={tenant_id}, count={len(users)}")
return users
@@ -190,18 +194,22 @@ class UserRepository:
raise
def count_users_by_tenant(
self,
self,
tenant_id: uuid.UUID,
is_active: Optional[bool] = None,
is_superuser: Optional[bool] = None,
search: Optional[str] = None
) -> int:
"""统计租户下的用户数量"""
try:
query = self.db.query(func.count(User.id)).filter(User.tenant_id == tenant_id)
if is_active is not None:
query = query.filter(User.is_active == is_active)
if is_superuser is not None:
query = query.filter(User.is_superuser == is_superuser)
if search:
query = query.filter(
or_(
@@ -209,7 +217,7 @@ class UserRepository:
User.email.ilike(f"%{search}%")
)
)
return query.scalar()
except Exception as e:
db_logger.error(f"统计租户用户失败: tenant_id={tenant_id} - {str(e)}")