[modify] rag qa chunk
This commit is contained in:
@@ -131,18 +131,43 @@ def keyword_extraction(chat_mdl, content, topn=3):
|
||||
|
||||
|
||||
def question_proposal(chat_mdl, content, topn=3):
|
||||
"""生成问题(向后兼容,返回纯文本问题列表)"""
|
||||
pairs = qa_proposal(chat_mdl, content, topn)
|
||||
if not pairs:
|
||||
return ""
|
||||
return "\n".join([p["question"] for p in pairs])
|
||||
|
||||
|
||||
def qa_proposal(chat_mdl, content, topn=3):
|
||||
"""生成 QA 对,返回 [{"question": ..., "answer": ...}, ...]"""
|
||||
template = PROMPT_JINJA_ENV.from_string(QUESTION_PROMPT_TEMPLATE)
|
||||
rendered_prompt = template.render(content=content, topn=topn)
|
||||
|
||||
msg = [{"role": "system", "content": rendered_prompt}, {"role": "user", "content": "Output: "}]
|
||||
_, msg = message_fit_in(msg, getattr(chat_mdl, 'max_length', 8096))
|
||||
kwd = chat_mdl.chat(rendered_prompt, msg[1:], {"temperature": 0.2})
|
||||
if isinstance(kwd, tuple):
|
||||
kwd = kwd[0]
|
||||
kwd = re.sub(r"^.*</think>", "", kwd, flags=re.DOTALL)
|
||||
if kwd.find("**ERROR**") >= 0:
|
||||
return ""
|
||||
return kwd
|
||||
raw = chat_mdl.chat(rendered_prompt, msg[1:], {"temperature": 0.2})
|
||||
if isinstance(raw, tuple):
|
||||
raw = raw[0]
|
||||
raw = re.sub(r"^.*</think>", "", raw, flags=re.DOTALL)
|
||||
if raw.find("**ERROR**") >= 0:
|
||||
return []
|
||||
return parse_qa_pairs(raw)
|
||||
|
||||
|
||||
def parse_qa_pairs(text: str) -> list:
|
||||
"""解析 LLM 返回的 QA 对文本,格式: Q: xxx A: xxx"""
|
||||
pairs = []
|
||||
for line in text.strip().split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
# 匹配 Q: ... A: ... 格式
|
||||
match = re.match(r'^Q:\s*(.+?)\s+A:\s*(.+)$', line, re.IGNORECASE)
|
||||
if match:
|
||||
q, a = match.group(1).strip(), match.group(2).strip()
|
||||
if q and a:
|
||||
pairs.append({"question": q, "answer": a})
|
||||
return pairs
|
||||
|
||||
|
||||
def graph_entity_types(chat_mdl, scenario):
|
||||
|
||||
@@ -1,19 +1,24 @@
|
||||
## Role
|
||||
You are a text analyzer.
|
||||
You are a text analyzer and knowledge extraction expert.
|
||||
|
||||
## Task
|
||||
Propose {{ topn }} questions about a given piece of text content.
|
||||
Generate {{ topn }} question-answer pairs from the given text content.
|
||||
|
||||
## Requirements
|
||||
- Understand and summarize the text content, and propose the top {{ topn }} important questions.
|
||||
- Understand and summarize the text content, and generate the top {{ topn }} important question-answer pairs.
|
||||
- Each question-answer pair MUST be on a single line, formatted as: Q: <question> A: <answer>
|
||||
- The questions SHOULD NOT have overlapping meanings.
|
||||
- The questions SHOULD cover the main content of the text as much as possible.
|
||||
- The questions MUST be in the same language as the given piece of text content.
|
||||
- One question per line.
|
||||
- Output questions ONLY.
|
||||
- The answers MUST be concise, accurate, and directly derived from the text content.
|
||||
- The answers SHOULD be self-contained and understandable without additional context.
|
||||
- Both questions and answers MUST be in the same language as the given text content.
|
||||
- Output question-answer pairs ONLY, no extra explanation.
|
||||
|
||||
## Example Output
|
||||
Q: What is the capital of France? A: The capital of France is Paris.
|
||||
Q: When was the Eiffel Tower built? A: The Eiffel Tower was built in 1889.
|
||||
|
||||
---
|
||||
|
||||
## Text Content
|
||||
{{ content }}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user