adai/MemoryBear
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(workflow): support multimodal context

Browse Source
This commit is contained in:
Eternity
2026-03-06 15:44:37 +08:00
parent e833db954a
commit ccc67df8df
4 changed files with 148 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
api/app/core/workflow/executor.py
View File

@@ -158,18 +158,36 @@ class WorkflowExecutor:
full_content += self.variable_pool.get_value(f"{end_id}.output", default="", strict=False)
# Append messages for user and assistant
result["messages"].extend(
[
{
"role": "user",
"content": input_data.get("message", '')
},
{
"role": "assistant",
"content": full_content
}
]
)
if input_data.get("files"):
result["messages"].extend(
[
{
"role": "user",
"content": input_data.get("message", '')
},
{
"role": "user",
"content": input_data.get("files")
},
{
"role": "assistant",
"content": full_content
}
]
)
else:
result["messages"].extend(
[
{
"role": "user",
"content": input_data.get("message", '')
},
{
"role": "assistant",
"content": full_content
}
]
)
# Calculate elapsed time
end_time = datetime.datetime.now()
elapsed_time = (end_time - start_time).total_seconds()
@@ -308,18 +326,36 @@ class WorkflowExecutor:
elapsed_time = (end_time - start_time).total_seconds()
# Append messages for user and assistant
result["messages"].extend(
[
{
"role": "user",
"content": input_data.get("message", '')
},
{
"role": "assistant",
"content": full_content
}
]
)
if input_data.get("files"):
result["messages"].extend(
[
{
"role": "user",
"content": input_data.get("message", '')
},
{
"role": "user",
"content": input_data.get("files")
},
{
"role": "assistant",
"content": full_content
}
]
)
else:
result["messages"].extend(
[
{
"role": "user",
"content": input_data.get("message", '')
},
{
"role": "assistant",
"content": full_content
}
]
)
logger.info(
f"Workflow execution completed (streaming), "
f"elapsed: {elapsed_time:.2f}ms, execution_id: {self.execution_context.execution_id}"

17
api/app/core/workflow/nodes/base_node.py
View File

@@ -617,10 +617,19 @@ class BaseNode(ABC):
return variable_pool.has(selector)
@staticmethod
async def process_message(provider: str, content: str | FileObject, enable_file=False) -> dict | str | None:
async def process_message(provider: str, content: str | dict | FileObject, enable_file=False) -> list | str | None:
if isinstance(content, dict):
content = FileObject(
type=content.get("type"),
url=content.get("url"),
transfer_method=content.get("transfer_method"),
origin_file_type=content.get("origin_file_type"),
file_id=content.get("file_id"),
is_file=True
)
if isinstance(content, str):
if enable_file:
return {"text": content}
return [{"text": content}]
return content
elif isinstance(content, FileObject):
@@ -639,8 +648,8 @@ class BaseNode(ABC):
)
if message:
content.content_cache[provider] = message[0]
return message[0]
content.content_cache[provider] = message
return message
return None
raise TypeError(f'Unexpect input value type - {type(content)}')

28
api/app/core/workflow/nodes/llm/node.py
View File

@@ -151,23 +151,23 @@ class LLMNode(BaseNode):
if role == "system":
messages.append({
"role": "system",
"content": content
"content": await self.process_message(provider, content, self.typed_config.vision)
})
elif role in ["user", "human"]:
messages.append({
"role": "user",
"content": content
"content": await self.process_message(provider, content, self.typed_config.vision)
})
elif role in ["ai", "assistant"]:
messages.append({
"role": "assistant",
"content": content
"content": await self.process_message(provider, content, self.typed_config.vision)
})
else:
logger.warning(f"未知的消息角色: {role}，默认使用 user")
messages.append({
"role": "user",
"content": content
"content": await self.process_message(provider, content, self.typed_config.vision)
})
if self.typed_config.vision_input and self.typed_config.vision:
@@ -176,14 +176,28 @@ class LLMNode(BaseNode):
for file in files.value:
content = await self.process_message(provider, file.value, self.typed_config.vision)
if content:
file_content.append(content)
file_content.extend(content)
if messages and messages[-1]["role"] == 'user':
messages[-1]['content'] = [messages[-1]["content"]] + file_content
messages[-1]['content'] = messages[-1]["content"] + file_content
else:
messages.append({"role": "user", "content": file_content})
if self.typed_config.memory.enable:
messages = messages[:-1] + state["messages"][-self.typed_config.memory.window_size:] + messages[-1:]
history_message = []
for message in state["messages"][-self.typed_config.memory.window_size:]:
if isinstance(message["content"], list):
file_content = []
for file in message["content"]:
content = await self.process_message(provider, file, self.typed_config.vision)
if content:
file_content.extend(content)
history_message.append(
{"role": message["role"], "content": file_content}
)
else:
message["content"] = await self.process_message(provider, message["content"], self.typed_config.vision)
history_message.append(message)
messages = messages[:-1] + history_message + messages[-1:]
self.messages = messages
else:
# 使用简单的 prompt 格式（向后兼容）