adai/MemoryBear
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'refs/heads/develop' into fix/develop_kj_knowledge

Browse Source 
# Conflicts:
#	api/app/core/memory/utils/prompt/prompts/reflexion.jinja2
This commit is contained in:
lixinyue
2025-12-31 11:28:23 +08:00
parent b96a63fb22 6b0ee1b74a
commit 9935459b32
7 changed files with 74 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
api/app/core/rag/app/naive.py
View File

@@ -672,6 +672,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
excel_parser = ExcelParser()
if parser_config.get("html4excel"):
sections = [(_, "") for _ in excel_parser.html(binary, 12) if _]
parser_config["chunk_token_num"] = 0
else:
sections = [(_, "") for _ in excel_parser(binary) if _]
parser_config["chunk_token_num"] = 12800

74
api/app/core/rag/deepdoc/parser/excel_parser.py
View File

@@ -5,6 +5,7 @@ from io import BytesIO
import pandas as pd
from openpyxl import Workbook, load_workbook
from PIL import Image
from app.core.rag.nlp import find_codec
@@ -28,7 +29,7 @@ class RAGExcelParser:
try:
file_like_object.seek(0)
df = pd.read_csv(file_like_object)
df = pd.read_csv(file_like_object, on_bad_lines='skip')
return RAGExcelParser._dataframe_to_workbook(df)
except Exception as e_csv:
@@ -47,6 +48,7 @@ class RAGExcelParser:
logging.info(f"pandas with default engine load error: {ex}, try calamine instead")
file_like_object.seek(0)
df = pd.read_excel(file_like_object, engine="calamine")
print("lxc1")
return RAGExcelParser._dataframe_to_workbook(df)
except Exception as e_pandas:
raise Exception(f"pandas.read_excel error: {e_pandas}, original openpyxl error: {e}")
@@ -65,7 +67,6 @@ class RAGExcelParser:
# if contains multiple sheets use _dataframes_to_workbook
if isinstance(df, dict) and len(df) > 1:
return RAGExcelParser._dataframes_to_workbook(df)
df = RAGExcelParser._clean_dataframe(df)
wb = Workbook()
ws = wb.active
@@ -77,15 +78,14 @@ class RAGExcelParser:
for row_num, row in enumerate(df.values, 2):
for col_num, value in enumerate(row, 1):
ws.cell(row=row_num, column=col_num, value=value)
return wb
@staticmethod
def _dataframes_to_workbook(dfs: dict):
wb = Workbook()
default_sheet = wb.active
wb.remove(default_sheet)
for sheet_name, df in dfs.items():
df = RAGExcelParser._clean_dataframe(df)
ws = wb.create_sheet(title=sheet_name)
@@ -96,6 +96,52 @@ class RAGExcelParser:
ws.cell(row=row_num, column=col_num, value=value)
return wb
@staticmethod
def _extract_images_from_worksheet(ws, sheetname=None):
"""
Extract images from a worksheet and enrich them with vision-based descriptions.
Returns: List[dict]
"""
images = getattr(ws, "_images", [])
if not images:
return []
raw_items = []
for img in images:
try:
img_bytes = img._data()
pil_img = Image.open(BytesIO(img_bytes)).convert("RGB")
anchor = img.anchor
if hasattr(anchor, "_from") and hasattr(anchor, "_to"):
r1, c1 = anchor._from.row + 1, anchor._from.col + 1
r2, c2 = anchor._to.row + 1, anchor._to.col + 1
if r1 == r2 and c1 == c2:
span = "single_cell"
else:
span = "multi_cell"
else:
r1, c1 = anchor._from.row + 1, anchor._from.col + 1
r2, c2 = r1, c1
span = "single_cell"
item = {
"sheet": sheetname or ws.title,
"image": pil_img,
"image_description": "",
"row_from": r1,
"col_from": c1,
"row_to": r2,
"col_to": c2,
"span_type": span,
}
raw_items.append(item)
except Exception:
continue
return raw_items
def html(self, fnm, chunk_rows=256):
from html import escape
@@ -128,7 +174,7 @@ class RAGExcelParser:
tb = ""
tb += f"<table><caption>{sheetname}</caption>"
tb += tb_rows_0
for r in list(rows[1 + chunk_i * chunk_rows : min(1 + (chunk_i + 1) * chunk_rows, len(rows))]):
for r in list(rows[1 + chunk_i * chunk_rows: min(1 + (chunk_i + 1) * chunk_rows, len(rows))]):
tb += "<tr>"
for i, c in enumerate(r):
if c.value is None:
@@ -151,7 +197,7 @@ class RAGExcelParser:
except Exception as e:
logging.warning(f"Parse spreadsheet error: {e}, trying to interpret as CSV file")
file_like_object.seek(0)
df = pd.read_csv(file_like_object)
df = pd.read_csv(file_like_object, on_bad_lines='skip')
df = df.replace(r"^\s*$", "", regex=True)
return df.to_markdown(index=False)
@@ -189,14 +235,14 @@ class RAGExcelParser:
if fnm.split(".")[-1].lower().find("xls") >= 0:
wb = RAGExcelParser._load_excel_to_workbook(BytesIO(binary))
total = 0
for sheetname in wb.sheetnames:
try:
ws = wb[sheetname]
total += len(list(ws.rows))
except Exception as e:
logging.warning(f"Skip sheet '{sheetname}' due to rows access error: {e}")
continue
try:
ws = wb[sheetname]
total += len(list(ws.rows))
except Exception as e:
logging.warning(f"Skip sheet '{sheetname}' due to rows access error: {e}")
continue
return total
if fnm.split(".")[-1].lower() in ["csv", "txt"]:

2
api/app/core/rag/graphrag/entity_resolution.py
View File

@@ -196,7 +196,7 @@ class EntityResolution(Extractor):
ans_list = []
records = [r.strip() for r in results.split(record_delimiter)]
for record in records:
pattern_int = f"{re.escape(entity_index_delimiter)}(\d+){re.escape(entity_index_delimiter)}"
pattern_int = fr"{re.escape(entity_index_delimiter)}(\d+){re.escape(entity_index_delimiter)}"
match_int = re.search(pattern_int, record)
res_int = int(str(match_int.group(1) if match_int else '0'))
if res_int > records_length:

5
api/app/models/document_model.py
View File

@@ -26,6 +26,7 @@ class Document(Base):
"html4excel": False,
"graphrag": {
"use_graphrag": False,
"scene_name": "",
"entity_types": [
"organization",
"person",
@@ -33,7 +34,9 @@ class Document(Base):
"event",
"category"
],
"method": "general"
"method": "general",
"resolution": True,
"community": True
}
}, comment="default parser config")
chunk_num = Column(Integer, default=0, comment="chunk num")

5
api/app/models/knowledge_model.py
View File

@@ -65,6 +65,7 @@ class Knowledge(Base):
"html4excel": False,
"graphrag": {
"use_graphrag": False,
"scene_name": "",
"entity_types": [
"organization",
"person",
@@ -72,7 +73,9 @@ class Knowledge(Base):
"event",
"category"
],
"method": "general"
"method": "general",
"resolution": True,
"community": True
}
},
comment="default parser config")

2
api/pyproject.toml
View File

@@ -135,6 +135,8 @@ dependencies = [
"graspologic==3.4.5.dev2",
"markdown-to-json==2.1.1",
"valkey==6.0.2",
"python-calamine>=0.4.0",
"xlrd==2.0.2"
]
[tool.pytest.ini_options]

2
api/requirements.txt
View File

@@ -129,3 +129,5 @@ editdistance==0.8.1
graspologic==3.4.5.dev2
markdown-to-json==2.1.1
valkey==6.0.2
python-calamine>=0.4.0
xlrd==2.0.2