adai/MemoryBear
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(memory): support perception-aware memory writing in workflow and Neo4j nodes

Browse Source
This commit is contained in:
Eternity
2026-03-23 16:33:25 +08:00
parent 31085ed678
commit 2ff81ba101
22 changed files with 820 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
api/app/services/memory_perceptual_service.py
View File

@@ -12,11 +12,12 @@ from app.core.error_codes import BizCode
from app.core.exceptions import BusinessException
from app.core.logging_config import get_business_logger
from app.core.models import RedBearLLM, RedBearModelConfig
from app.models import FileMetadata
from app.models import FileMetadata, ModelApiKey, ModelType
from app.models.memory_perceptual_model import PerceptualType, FileStorageService
from app.models.prompt_optimizer_model import RoleType
from app.repositories.memory_perceptual_repository import MemoryPerceptualRepository
from app.schemas import FileType
from app.schemas import FileType, FileInput
from app.schemas.memory_config_schema import MemoryConfig
from app.schemas.memory_perceptual_schema import (
PerceptualQuerySchema,
PerceptualTimelineResponse,
@@ -24,6 +25,8 @@ from app.schemas.memory_perceptual_schema import (
AudioModal, Content, VideoModal, TextModal
)
from app.schemas.model_schema import ModelInfo
from app.services.model_service import ModelApiKeyService
from app.services.multimodal_service import MultimodalService
business_logger = get_business_logger()
@@ -195,21 +198,58 @@ class MemoryPerceptualService:
business_logger.error(f"Failed to fetch perceptual memory timeline: {str(e)}")
raise BusinessException(f"Failed to fetch perceptual memory timeline: {str(e)}", BizCode.DB_ERROR)
def _get_mutlimodal_client(
self,
file_type: FileType,
config: MemoryConfig
) -> tuple[RedBearLLM | None, ModelApiKey | None]:
model_config = None
if file_type == FileType.AUDIO:
model_config = ModelApiKeyService.get_available_api_key(
self.db,
config.audio_model_id
)
elif file_type == FileType.VIDEO:
model_config = ModelApiKeyService.get_available_api_key(
self.db,
config.video_model_id
)
elif file_type == FileType.DOCUMENT:
model_config = ModelApiKeyService.get_available_api_key(
self.db,
config.llm_model_id
)
elif file_type == FileType.IMAGE:
model_config = ModelApiKeyService.get_available_api_key(
self.db,
config.vision_model_id
)
llm = None
if model_config:
llm = RedBearLLM(
RedBearModelConfig(
model_name=model_config.model_name,
provider=model_config.provider,
api_key=model_config.api_key,
base_url=model_config.api_base,
is_omni=model_config.is_omni
)
)
return llm, model_config
async def generate_perceptual_memory(
self,
end_user_id: str,
model_config: ModelInfo,
file_type: str,
file_url: str,
file_message: dict,
memory_config: MemoryConfig,
file: FileInput
):
memories = self.repository.get_by_url(file_url)
memories = self.repository.get_by_url(file.url)
if memories:
business_logger.info(f"Perceptual memory already exists: {file_url}")
business_logger.info(f"Perceptual memory already exists: {file.url}")
if end_user_id not in [memory.end_user_id for memory in memories]:
business_logger.info(f"Copy perceptual memory end_user_id: {end_user_id}")
memory_cache = memories[0]
self.repository.create_perceptual_memory(
memory = self.repository.create_perceptual_memory(
end_user_id=uuid.UUID(end_user_id),
perceptual_type=PerceptualType(memory_cache.perceptual_type),
file_path=memory_cache.file_path,
@@ -219,20 +259,31 @@ class MemoryPerceptualService:
meta_data=memory_cache.meta_data
)
self.db.commit()
return
llm = RedBearLLM(RedBearModelConfig(
return memory
else:
for memory in memories:
if memory.end_user_id == uuid.UUID(end_user_id):
return memory
llm, model_config = self._get_mutlimodal_client(file.type, memory_config)
multimodel_service = MultimodalService(self.db, ModelInfo(
model_name=model_config.model_name,
provider=model_config.provider,
api_key=model_config.api_key,
base_url=model_config.api_base,
is_omni=model_config.is_omni
), type=model_config.model_type)
api_base=model_config.api_base,
is_omni=model_config.is_omni,
capability=model_config.capability,
model_type=ModelType.LLM
))
file_message = await multimodel_service.process_files(
files=[file]
)
if file_message:
file_message = file_message[0]
try:
prompt_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'prompt')
with open(os.path.join(prompt_path, 'perceptual_summary_system.jinja2'), 'r', encoding='utf-8') as f:
opt_system_prompt = f.read()
rendered_system_message = Template(opt_system_prompt).render(file_type=file_type, language='zh')
rendered_system_message = Template(opt_system_prompt).render(file_type=file.type, language='zh')
except FileNotFoundError:
raise BusinessException(message="System prompt template not found", code=BizCode.NOT_FOUND)
messages = [
@@ -242,8 +293,22 @@ class MemoryPerceptualService:
]}
]
result = await llm.ainvoke(messages)
content = json_repair.repair_json(result.content, return_objects=True)
path = urlparse(file_url).path
content = result.content
final_output = ""
if isinstance(content, list):
for msg in content:
if isinstance(msg, dict):
final_output += msg.get("text", "")
elif isinstance(msg, str):
final_output += msg
elif isinstance(content, dict):
final_output += content.get("text", "")
elif isinstance(content, str):
final_output = content
else:
raise ValueError(f"Unexcept Model Output Type: {result.content}")
content = json_repair.repair_json(final_output, return_objects=True)
path = urlparse(file.url).path
filename = os.path.basename(path)
filename = unquote(filename)
file_ext = os.path.splitext(filename)[1]
@@ -260,13 +325,13 @@ class MemoryPerceptualService:
except ValueError:
business_logger.debug(f"Remote file, file_id={filename}")
if not file_ext:
if file_type == FileType.AUDIO:
if file.type == FileType.AUDIO:
file_ext = ".mp3"
elif file_type == FileType.VIDEO:
elif file.type == FileType.VIDEO:
file_ext = ".mp4"
elif file_type == FileType.DOCUMENT:
elif file.type == FileType.DOCUMENT:
file_ext = ".txt"
elif file_type == FileType.IMAGE:
elif file.type == FileType.IMAGE:
file_ext = ".jpg"
filename += file_ext
file_content = {
@@ -274,11 +339,11 @@ class MemoryPerceptualService:
"topic": content.get("topic"),
"domain": content.get("domain")
}
if file_type in [FileType.IMAGE, FileType.VIDEO]:
if file.type in [FileType.IMAGE, FileType.VIDEO]:
file_modalities = {
"scene": content.get("scene", [])
}
elif file_type in [FileType.DOCUMENT]:
elif file.type in [FileType.DOCUMENT]:
file_modalities = {
"section_count": content.get("section_count", 0),
"title": content.get("title", ""),
@@ -288,10 +353,10 @@ class MemoryPerceptualService:
file_modalities = {
"speaker_count": content.get("speaker_count", 0)
}
self.repository.create_perceptual_memory(
memory = self.repository.create_perceptual_memory(
end_user_id=uuid.UUID(end_user_id),
perceptual_type=PerceptualType.trans_from_file_type(file_type),
file_path=file_url,
perceptual_type=PerceptualType.trans_from_file_type(file.type),
file_path=file.url,
file_name=filename,
file_ext=file_ext,
summary=content.get('summary', ""),
@@ -301,3 +366,4 @@ class MemoryPerceptualService:
}
)
self.db.commit()
return memory