|
|
|
|
|
import enum
|
|
|
|
|
|
|
|
|
|
|
|
from magic_pdf.config.model_block_type import ModelBlockTypeEnum
|
|
|
|
|
|
from magic_pdf.config.ocr_content_type import CategoryId, ContentType
|
|
|
|
|
|
from magic_pdf.data.dataset import Dataset
|
|
|
|
|
|
from magic_pdf.libs.boxbase import (_is_in, bbox_distance, bbox_relative_pos,
|
|
|
|
|
|
calculate_iou)
|
|
|
|
|
|
from magic_pdf.libs.coordinate_transform import get_scale_ratio
|
|
|
|
|
|
from magic_pdf.pre_proc.remove_bbox_overlap import _remove_overlap_between_bbox
|
|
|
|
|
|
|
|
|
|
|
|
CAPATION_OVERLAP_AREA_RATIO = 0.6
|
|
|
|
|
|
MERGE_BOX_OVERLAP_AREA_RATIO = 1.1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class PosRelationEnum(enum.Enum):
|
|
|
|
|
|
LEFT = 'left'
|
|
|
|
|
|
RIGHT = 'right'
|
|
|
|
|
|
UP = 'up'
|
|
|
|
|
|
BOTTOM = 'bottom'
|
|
|
|
|
|
ALL = 'all'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class MagicModel:
|
|
|
|
|
|
"""每个函数没有得到元素的时候返回空list."""
|
|
|
|
|
|
|
|
|
|
|
|
def __fix_axis(self):
|
|
|
|
|
|
for model_page_info in self.__model_list:
|
|
|
|
|
|
need_remove_list = []
|
|
|
|
|
|
page_no = model_page_info['page_info']['page_no']
|
|
|
|
|
|
horizontal_scale_ratio, vertical_scale_ratio = get_scale_ratio(
|
|
|
|
|
|
model_page_info, self.__docs.get_page(page_no)
|
|
|
|
|
|
)
|
|
|
|
|
|
layout_dets = model_page_info['layout_dets']
|
|
|
|
|
|
for layout_det in layout_dets:
|
|
|
|
|
|
|
|
|
|
|
|
if layout_det.get('bbox') is not None:
|
|
|
|
|
|
# 兼容直接输出bbox的模型数据,如paddle
|
|
|
|
|
|
x0, y0, x1, y1 = layout_det['bbox']
|
|
|
|
|
|
else:
|
|
|
|
|
|
# 兼容直接输出poly的模型数据，如xxx
|
|
|
|
|
|
x0, y0, _, _, x1, y1, _, _ = layout_det['poly']
|
|
|
|
|
|
|
|
|
|
|
|
bbox = [
|
|
|
|
|
|
int(x0 / horizontal_scale_ratio),
|
|
|
|
|
|
int(y0 / vertical_scale_ratio),
|
|
|
|
|
|
int(x1 / horizontal_scale_ratio),
|
|
|
|
|
|
int(y1 / vertical_scale_ratio),
|
|
|
|
|
|
]
|
|
|
|
|
|
layout_det['bbox'] = bbox
|
|
|
|
|
|
# 删除高度或者宽度小于等于0的spans
|
|
|
|
|
|
if bbox[2] - bbox[0] <= 0 or bbox[3] - bbox[1] <= 0:
|
|
|
|
|
|
need_remove_list.append(layout_det)
|
|
|
|
|
|
for need_remove in need_remove_list:
|
|
|
|
|
|
layout_dets.remove(need_remove)
|
|
|
|
|
|
|
|
|
|
|
|
def __fix_by_remove_low_confidence(self):
|
|
|
|
|
|
for model_page_info in self.__model_list:
|
|
|
|
|
|
need_remove_list = []
|
|
|
|
|
|
layout_dets = model_page_info['layout_dets']
|
|
|
|
|
|
for layout_det in layout_dets:
|
|
|
|
|
|
if layout_det['score'] <= 0.05:
|
|
|
|
|
|
need_remove_list.append(layout_det)
|
|
|
|
|
|
else:
|
|
|
|
|
|
continue
|
|
|
|
|
|
for need_remove in need_remove_list:
|
|
|
|
|
|
layout_dets.remove(need_remove)
|
|
|
|
|
|
|
|
|
|
|
|
def __fix_by_remove_high_iou_and_low_confidence(self):
|
|
|
|
|
|
for model_page_info in self.__model_list:
|
|
|
|
|
|
need_remove_list = []
|
|
|
|
|
|
layout_dets = model_page_info['layout_dets']
|
|
|
|
|
|
for layout_det1 in layout_dets:
|
|
|
|
|
|
for layout_det2 in layout_dets:
|
|
|
|
|
|
if layout_det1 == layout_det2:
|
|
|
|
|
|
continue
|
|
|
|
|
|
if layout_det1['category_id'] in [
|
|
|
|
|
|
0,
|
|
|
|
|
|
1,
|
|
|
|
|
|
2,
|
|
|
|
|
|
3,
|
|
|
|
|
|
4,
|
|
|
|
|
|
5,
|
|
|
|
|
|
6,
|
|
|
|
|
|
7,
|
|
|
|
|
|
8,
|
|
|
|
|
|
9,
|
|
|
|
|
|
] and layout_det2['category_id'] in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]:
|
|
|
|
|
|
if (
|
|
|
|
|
|
calculate_iou(layout_det1['bbox'], layout_det2['bbox'])
|
|
|
|
|
|
> 0.9
|
|
|
|
|
|
):
|
|
|
|
|
|
if layout_det1['score'] < layout_det2['score']:
|
|
|
|
|
|
layout_det_need_remove = layout_det1
|
|
|
|
|
|
else:
|
|
|
|
|
|
layout_det_need_remove = layout_det2
|
|
|
|
|
|
|
|
|
|
|
|
if layout_det_need_remove not in need_remove_list:
|
|
|
|
|
|
need_remove_list.append(layout_det_need_remove)
|
|
|
|
|
|
else:
|
|
|
|
|
|
continue
|
|
|
|
|
|
else:
|
|
|
|
|
|
continue
|
|
|
|
|
|
for need_remove in need_remove_list:
|
|
|
|
|
|
layout_dets.remove(need_remove)
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self, model_list: list, docs: Dataset):
|
|
|
|
|
|
self.__model_list = model_list
|
|
|
|
|
|
self.__docs = docs
|
|
|
|
|
|
"""为所有模型数据添加bbox信息(缩放，poly->bbox)"""
|
|
|
|
|
|
self.__fix_axis()
|
|
|
|
|
|
"""删除置信度特别低的模型数据(<0.05),提高质量"""
|
|
|
|
|
|
self.__fix_by_remove_low_confidence()
|
|
|
|
|
|
"""删除高iou(>0.9)数据中置信度较低的那个"""
|
|
|
|
|
|
self.__fix_by_remove_high_iou_and_low_confidence()
|
|
|
|
|
|
self.__fix_footnote()
|
|
|
|
|
|
|
|
|
|
|
|
def _bbox_distance(self, bbox1, bbox2):
|
|
|
|
|
|
left, right, bottom, top = bbox_relative_pos(bbox1, bbox2)
|
|
|
|
|
|
flags = [left, right, bottom, top]
|
|
|
|
|
|
count = sum([1 if v else 0 for v in flags])
|
|
|
|
|
|
if count > 1:
|
|
|
|
|
|
return float('inf')
|
|
|
|
|
|
if left or right:
|
|
|
|
|
|
l1 = bbox1[3] - bbox1[1]
|
|
|
|
|
|
l2 = bbox2[3] - bbox2[1]
|
|
|
|
|
|
else:
|
|
|
|
|
|
l1 = bbox1[2] - bbox1[0]
|
|
|
|
|
|
l2 = bbox2[2] - bbox2[0]
|
|
|
|
|
|
|
|
|
|
|
|
if l2 > l1 and (l2 - l1) / l1 > 0.3:
|
|
|
|
|
|
return float('inf')
|
|
|
|
|
|
|
|
|
|
|
|
return bbox_distance(bbox1, bbox2)
|
|
|
|
|
|
|
|
|
|
|
|
def __fix_footnote(self):
|
|
|
|
|
|
# 3: figure, 5: table, 7: footnote
|
|
|
|
|
|
for model_page_info in self.__model_list:
|
|
|
|
|
|
footnotes = []
|
|
|
|
|
|
figures = []
|
|
|
|
|
|
tables = []
|
|
|
|
|
|
|
|
|
|
|
|
for obj in model_page_info['layout_dets']:
|
|
|
|
|
|
if obj['category_id'] == 7:
|
|
|
|
|
|
footnotes.append(obj)
|
|
|
|
|
|
elif obj['category_id'] == 3:
|
|
|
|
|
|
figures.append(obj)
|
|
|
|
|
|
elif obj['category_id'] == 5:
|
|
|
|
|
|
tables.append(obj)
|
|
|
|
|
|
if len(footnotes) * len(figures) == 0:
|
|
|
|
|
|
continue
|
|
|
|
|
|
dis_figure_footnote = {}
|
|
|
|
|
|
dis_table_footnote = {}
|
|
|
|
|
|
|
|
|
|
|
|
for i in range(len(footnotes)):
|
|
|
|
|
|
for j in range(len(figures)):
|
|
|
|
|
|
pos_flag_count = sum(
|
|
|
|
|
|
list(
|
|
|
|
|
|
map(
|
|
|
|
|
|
lambda x: 1 if x else 0,
|
|
|
|
|
|
bbox_relative_pos(
|
|
|
|
|
|
footnotes[i]['bbox'], figures[j]['bbox']
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
)
|
|
|
|
|
|
)
|
|
|
|
|
|
if pos_flag_count > 1:
|
|
|
|
|
|
continue
|
|
|
|
|
|
dis_figure_footnote[i] = min(
|
|
|
|
|
|
self._bbox_distance(figures[j]['bbox'], footnotes[i]['bbox']),
|
|
|
|
|
|
dis_figure_footnote.get(i, float('inf')),
|
|
|
|
|
|
)
|
|
|
|
|
|
for i in range(len(footnotes)):
|
|
|
|
|
|
for j in range(len(tables)):
|
|
|
|
|
|
pos_flag_count = sum(
|
|
|
|
|
|
list(
|
|
|
|
|
|
map(
|
|
|
|
|
|
lambda x: 1 if x else 0,
|
|
|
|
|
|
bbox_relative_pos(
|
|
|
|
|
|
footnotes[i]['bbox'], tables[j]['bbox']
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
)
|
|
|
|
|
|
)
|
|
|
|
|
|
if pos_flag_count > 1:
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
|
|
dis_table_footnote[i] = min(
|
|
|
|
|
|
self._bbox_distance(tables[j]['bbox'], footnotes[i]['bbox']),
|
|
|
|
|
|
dis_table_footnote.get(i, float('inf')),
|
|
|
|
|
|
)
|
|
|
|
|
|
for i in range(len(footnotes)):
|
|
|
|
|
|
if i not in dis_figure_footnote:
|
|
|
|
|
|
continue
|
|
|
|
|
|
if dis_table_footnote.get(i, float('inf')) > dis_figure_footnote[i]:
|
|
|
|
|
|
footnotes[i]['category_id'] = CategoryId.ImageFootnote
|
|
|
|
|
|
|
|
|
|
|
|
def __reduct_overlap(self, bboxes):
|
|
|
|
|
|
N = len(bboxes)
|
|
|
|
|
|
keep = [True] * N
|
|
|
|
|
|
for i in range(N):
|
|
|
|
|
|
for j in range(N):
|
|
|
|
|
|
if i == j:
|
|
|
|
|
|
continue
|
|
|
|
|
|
if _is_in(bboxes[i]['bbox'], bboxes[j]['bbox']):
|
|
|
|
|
|
keep[i] = False
|
|
|
|
|
|
return [bboxes[i] for i in range(N) if keep[i]]
|
|
|
|
|
|
|
|
|
|
|
|
def __tie_up_category_by_distance_v2(
|
|
|
|
|
|
self,
|
|
|
|
|
|
page_no: int,
|
|
|
|
|
|
subject_category_id: int,
|
|
|
|
|
|
object_category_id: int,
|
|
|
|
|
|
priority_pos: PosRelationEnum,
|
|
|
|
|
|
):
|
|
|
|
|
|
"""_summary_
|
|
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
|
page_no (int): _description_
|
|
|
|
|
|
subject_category_id (int): _description_
|
|
|
|
|
|
object_category_id (int): _description_
|
|
|
|
|
|
priority_pos (PosRelationEnum): _description_
|
|
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
|
_type_: _description_
|
|
|
|
|
|
"""
|
|
|
|
|
|
AXIS_MULPLICITY = 0.5
|
|
|
|
|
|
subjects = self.__reduct_overlap(
|
|
|
|
|
|
list(
|
|
|
|
|
|
map(
|
|
|
|
|
|
lambda x: {'bbox': x['bbox'], 'score': x['score']},
|
|
|
|
|
|
filter(
|
|
|
|
|
|
lambda x: x['category_id'] == subject_category_id,
|
|
|
|
|
|
self.__model_list[page_no]['layout_dets'],
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
)
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
objects = self.__reduct_overlap(
|
|
|
|
|
|
list(
|
|
|
|
|
|
map(
|
|
|
|
|
|
lambda x: {'bbox': x['bbox'], 'score': x['score']},
|
|
|
|
|
|
filter(
|
|
|
|
|
|
lambda x: x['category_id'] == object_category_id,
|
|
|
|
|
|
self.__model_list[page_no]['layout_dets'],
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
)
|
|
|
|
|
|
)
|
|
|
|
|
|
M = len(objects)
|
|
|
|
|
|
|
|
|
|
|
|
subjects.sort(key=lambda x: x['bbox'][0] ** 2 + x['bbox'][1] ** 2)
|
|
|
|
|
|
objects.sort(key=lambda x: x['bbox'][0] ** 2 + x['bbox'][1] ** 2)
|
|
|
|
|
|
|
|
|
|
|
|
sub_obj_map_h = {i: [] for i in range(len(subjects))}
|
|
|
|
|
|
|
|
|
|
|
|
dis_by_directions = {
|
|
|
|
|
|
'top': [[-1, float('inf')]] * M,
|
|
|
|
|
|
'bottom': [[-1, float('inf')]] * M,
|
|
|
|
|
|
'left': [[-1, float('inf')]] * M,
|
|
|
|
|
|
'right': [[-1, float('inf')]] * M,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
for i, obj in enumerate(objects):
|
|
|
|
|
|
l_x_axis, l_y_axis = (
|
|
|
|
|
|
obj['bbox'][2] - obj['bbox'][0],
|
|
|
|
|
|
obj['bbox'][3] - obj['bbox'][1],
|
|
|
|
|
|
)
|
|
|
|
|
|
axis_unit = min(l_x_axis, l_y_axis)
|
|
|
|
|
|
for j, sub in enumerate(subjects):
|
|
|
|
|
|
|
|
|
|
|
|
bbox1, bbox2, _ = _remove_overlap_between_bbox(
|
|
|
|
|
|
objects[i]['bbox'], subjects[j]['bbox']
|
|
|
|
|
|
)
|
|
|
|
|
|
left, right, bottom, top = bbox_relative_pos(bbox1, bbox2)
|
|
|
|
|
|
flags = [left, right, bottom, top]
|
|
|
|
|
|
if sum([1 if v else 0 for v in flags]) > 1:
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
|
|
if left:
|
|
|
|
|
|
if dis_by_directions['left'][i][1] > bbox_distance(
|
|
|
|
|
|
obj['bbox'], sub['bbox']
|
|
|
|
|
|
):
|
|
|
|
|
|
dis_by_directions['left'][i] = [
|
|
|
|
|
|
j,
|
|
|
|
|
|
bbox_distance(obj['bbox'], sub['bbox']),
|
|
|
|
|
|
]
|
|
|
|
|
|
if right:
|
|
|
|
|
|
if dis_by_directions['right'][i][1] > bbox_distance(
|
|
|
|
|
|
obj['bbox'], sub['bbox']
|
|
|
|
|
|
):
|
|
|
|
|
|
dis_by_directions['right'][i] = [
|
|
|
|
|
|
j,
|
|
|
|
|
|
bbox_distance(obj['bbox'], sub['bbox']),
|
|
|
|
|
|
]
|
|
|
|
|
|
if bottom:
|
|
|
|
|
|
if dis_by_directions['bottom'][i][1] > bbox_distance(
|
|
|
|
|
|
obj['bbox'], sub['bbox']
|
|
|
|
|
|
):
|
|
|
|
|
|
dis_by_directions['bottom'][i] = [
|
|
|
|
|
|
j,
|
|
|
|
|
|
bbox_distance(obj['bbox'], sub['bbox']),
|
|
|
|
|
|
]
|
|
|
|
|
|
if top:
|
|
|
|
|
|
if dis_by_directions['top'][i][1] > bbox_distance(
|
|
|
|
|
|
obj['bbox'], sub['bbox']
|
|
|
|
|
|
):
|
|
|
|
|
|
dis_by_directions['top'][i] = [
|
|
|
|
|
|
j,
|
|
|
|
|
|
bbox_distance(obj['bbox'], sub['bbox']),
|
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
|
|
if (
|
|
|
|
|
|
dis_by_directions['top'][i][1] != float('inf')
|
|
|
|
|
|
and dis_by_directions['bottom'][i][1] != float('inf')
|
|
|
|
|
|
and priority_pos in (PosRelationEnum.BOTTOM, PosRelationEnum.UP)
|
|
|
|
|
|
):
|
|
|
|
|
|
RATIO = 3
|
|
|
|
|
|
if (
|
|
|
|
|
|
abs(
|
|
|
|
|
|
dis_by_directions['top'][i][1]
|
|
|
|
|
|
- dis_by_directions['bottom'][i][1]
|
|
|
|
|
|
)
|
|
|
|
|
|
< RATIO * axis_unit
|
|
|
|
|
|
):
|
|
|
|
|
|
|
|
|
|
|
|
if priority_pos == PosRelationEnum.BOTTOM:
|
|
|
|
|
|
sub_obj_map_h[dis_by_directions['bottom'][i][0]].append(i)
|
|
|
|
|
|
else:
|
|
|
|
|
|
sub_obj_map_h[dis_by_directions['top'][i][0]].append(i)
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
|
|
if dis_by_directions['left'][i][1] != float('inf') or dis_by_directions[
|
|
|
|
|
|
'right'
|
|
|
|
|
|
][i][1] != float('inf'):
|
|
|
|
|
|
if dis_by_directions['left'][i][1] != float(
|
|
|
|
|
|
'inf'
|
|
|
|
|
|
) and dis_by_directions['right'][i][1] != float('inf'):
|
|
|
|
|
|
if AXIS_MULPLICITY * axis_unit >= abs(
|
|
|
|
|
|
dis_by_directions['left'][i][1]
|
|
|
|
|
|
- dis_by_directions['right'][i][1]
|
|
|
|
|
|
):
|
|
|
|
|
|
left_sub_bbox = subjects[dis_by_directions['left'][i][0]][
|
|
|
|
|
|
'bbox'
|
|
|
|
|
|
]
|
|
|
|
|
|
right_sub_bbox = subjects[dis_by_directions['right'][i][0]][
|
|
|
|
|
|
'bbox'
|
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
|
|
left_sub_bbox_y_axis = left_sub_bbox[3] - left_sub_bbox[1]
|
|
|
|
|
|
right_sub_bbox_y_axis = right_sub_bbox[3] - right_sub_bbox[1]
|
|
|
|
|
|
|
|
|
|
|
|
if (
|
|
|
|
|
|
abs(left_sub_bbox_y_axis - l_y_axis)
|
|
|
|
|
|
+ dis_by_directions['left'][i][0]
|
|
|
|
|
|
> abs(right_sub_bbox_y_axis - l_y_axis)
|
|
|
|
|
|
+ dis_by_directions['right'][i][0]
|
|
|
|
|
|
):
|
|
|
|
|
|
left_or_right = dis_by_directions['right'][i]
|
|
|
|
|
|
else:
|
|
|
|
|
|
left_or_right = dis_by_directions['left'][i]
|
|
|
|
|
|
else:
|
|
|
|
|
|
left_or_right = dis_by_directions['left'][i]
|
|
|
|
|
|
if left_or_right[1] > dis_by_directions['right'][i][1]:
|
|
|
|
|
|
left_or_right = dis_by_directions['right'][i]
|
|
|
|
|
|
else:
|
|
|
|
|
|
left_or_right = dis_by_directions['left'][i]
|
|
|
|
|
|
if left_or_right[1] == float('inf'):
|
|
|
|
|
|
left_or_right = dis_by_directions['right'][i]
|
|
|
|
|
|
else:
|
|
|
|
|
|
left_or_right = [-1, float('inf')]
|
|
|
|
|
|
|
|
|
|
|
|
if dis_by_directions['top'][i][1] != float('inf') or dis_by_directions[
|
|
|
|
|
|
'bottom'
|
|
|
|
|
|
][i][1] != float('inf'):
|
|
|
|
|
|
if dis_by_directions['top'][i][1] != float('inf') and dis_by_directions[
|
|
|
|
|
|
'bottom'
|
|
|
|
|
|
][i][1] != float('inf'):
|
|
|
|
|
|
if AXIS_MULPLICITY * axis_unit >= abs(
|
|
|
|
|
|
dis_by_directions['top'][i][1]
|
|
|
|
|
|
- dis_by_directions['bottom'][i][1]
|
|
|
|
|
|
):
|
|
|
|
|
|
top_bottom = subjects[dis_by_directions['bottom'][i][0]]['bbox']
|
|
|
|
|
|
bottom_top = subjects[dis_by_directions['top'][i][0]]['bbox']
|
|
|
|
|
|
|
|
|
|
|
|
top_bottom_x_axis = top_bottom[2] - top_bottom[0]
|
|
|
|
|
|
bottom_top_x_axis = bottom_top[2] - bottom_top[0]
|
|
|
|
|
|
if (
|
|
|
|
|
|
abs(top_bottom_x_axis - l_x_axis)
|
|
|
|
|
|
+ dis_by_directions['bottom'][i][1]
|
|
|
|
|
|
> abs(bottom_top_x_axis - l_x_axis)
|
|
|
|
|
|
+ dis_by_directions['top'][i][1]
|
|
|
|
|
|
):
|
|
|
|
|
|
top_or_bottom = dis_by_directions['top'][i]
|
|
|
|
|
|
else:
|
|
|
|
|
|
top_or_bottom = dis_by_directions['bottom'][i]
|
|
|
|
|
|
else:
|
|
|
|
|
|
top_or_bottom = dis_by_directions['top'][i]
|
|
|
|
|
|
if top_or_bottom[1] > dis_by_directions['bottom'][i][1]:
|
|
|
|
|
|
top_or_bottom = dis_by_directions['bottom'][i]
|
|
|
|
|
|
else:
|
|
|
|
|
|
top_or_bottom = dis_by_directions['top'][i]
|
|
|
|
|
|
if top_or_bottom[1] == float('inf'):
|
|
|
|
|
|
top_or_bottom = dis_by_directions['bottom'][i]
|
|
|
|
|
|
else:
|
|
|
|
|
|
top_or_bottom = [-1, float('inf')]
|
|
|
|
|
|
|
|
|
|
|
|
if left_or_right[1] != float('inf') or top_or_bottom[1] != float('inf'):
|
|
|
|
|
|
if left_or_right[1] != float('inf') and top_or_bottom[1] != float(
|
|
|
|
|
|
'inf'
|
|
|
|
|
|
):
|
|
|
|
|
|
if AXIS_MULPLICITY * axis_unit >= abs(
|
|
|
|
|
|
left_or_right[1] - top_or_bottom[1]
|
|
|
|
|
|
):
|
|
|
|
|
|
y_axis_bbox = subjects[left_or_right[0]]['bbox']
|
|
|
|
|
|
x_axis_bbox = subjects[top_or_bottom[0]]['bbox']
|
|
|
|
|
|
|
|
|
|
|
|
if (
|
|
|
|
|
|
abs((x_axis_bbox[2] - x_axis_bbox[0]) - l_x_axis) / l_x_axis
|
|
|
|
|
|
> abs((y_axis_bbox[3] - y_axis_bbox[1]) - l_y_axis)
|
|
|
|
|
|
/ l_y_axis
|
|
|
|
|
|
):
|
|
|
|
|
|
sub_obj_map_h[left_or_right[0]].append(i)
|
|
|
|
|
|
else:
|
|
|
|
|
|
sub_obj_map_h[top_or_bottom[0]].append(i)
|
|
|
|
|
|
else:
|
|
|
|
|
|
if left_or_right[1] > top_or_bottom[1]:
|
|
|
|
|
|
sub_obj_map_h[top_or_bottom[0]].append(i)
|
|
|
|
|
|
else:
|
|
|
|
|
|
sub_obj_map_h[left_or_right[0]].append(i)
|
|
|
|
|
|
else:
|
|
|
|
|
|
if left_or_right[1] != float('inf'):
|
|
|
|
|
|
sub_obj_map_h[left_or_right[0]].append(i)
|
|
|
|
|
|
else:
|
|
|
|
|
|
sub_obj_map_h[top_or_bottom[0]].append(i)
|
|
|
|
|
|
ret = []
|
|
|
|
|
|
for i in sub_obj_map_h.keys():
|
|
|
|
|
|
ret.append(
|
|
|
|
|
|
{
|
|
|
|
|
|
'sub_bbox': {
|
|
|
|
|
|
'bbox': subjects[i]['bbox'],
|
|
|
|
|
|
'score': subjects[i]['score'],
|
|
|
|
|
|
},
|
|
|
|
|
|
'obj_bboxes': [
|
|
|
|
|
|
{'score': objects[j]['score'], 'bbox': objects[j]['bbox']}
|
|
|
|
|
|
for j in sub_obj_map_h[i]
|
|
|
|
|
|
],
|
|
|
|
|
|
'sub_idx': i,
|
|
|
|
|
|
}
|
|
|
|
|
|
)
|
|
|
|
|
|
return ret
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def __tie_up_category_by_distance_v3(
|
|
|
|
|
|
self,
|
|
|
|
|
|
page_no: int,
|
|
|
|
|
|
subject_category_id: int,
|
|
|
|
|
|
object_category_id: int,
|
|
|
|
|
|
priority_pos: PosRelationEnum,
|
|
|
|
|
|
):
|
|
|
|
|
|
subjects = self.__reduct_overlap(
|
|
|
|
|
|
list(
|
|
|
|
|
|
map(
|
|
|
|
|
|
lambda x: {'bbox': x['bbox'], 'score': x['score']},
|
|
|
|
|
|
filter(
|
|
|
|
|
|
lambda x: x['category_id'] == subject_category_id,
|
|
|
|
|
|
self.__model_list[page_no]['layout_dets'],
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
)
|
|
|
|
|
|
)
|
|
|
|
|
|
objects = self.__reduct_overlap(
|
|
|
|
|
|
list(
|
|
|
|
|
|
map(
|
|
|
|
|
|
lambda x: {'bbox': x['bbox'], 'score': x['score']},
|
|
|
|
|
|
filter(
|
|
|
|
|
|
lambda x: x['category_id'] == object_category_id,
|
|
|
|
|
|
self.__model_list[page_no]['layout_dets'],
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
)
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
ret = []
|
|
|
|
|
|
N, M = len(subjects), len(objects)
|
|
|
|
|
|
subjects.sort(key=lambda x: x['bbox'][0] ** 2 + x['bbox'][1] ** 2)
|
|
|
|
|
|
objects.sort(key=lambda x: x['bbox'][0] ** 2 + x['bbox'][1] ** 2)
|
|
|
|
|
|
|
|
|
|
|
|
OBJ_IDX_OFFSET = 10000
|
|
|
|
|
|
SUB_BIT_KIND, OBJ_BIT_KIND = 0, 1
|
|
|
|
|
|
|
|
|
|
|
|
all_boxes_with_idx = [(i, SUB_BIT_KIND, sub['bbox'][0], sub['bbox'][1]) for i, sub in enumerate(subjects)] + [(i + OBJ_IDX_OFFSET , OBJ_BIT_KIND, obj['bbox'][0], obj['bbox'][1]) for i, obj in enumerate(objects)]
|
|
|
|
|
|
seen_idx = set()
|
|
|
|
|
|
seen_sub_idx = set()
|
|
|
|
|
|
|
|
|
|
|
|
while N > len(seen_sub_idx):
|
|
|
|
|
|
candidates = []
|
|
|
|
|
|
for idx, kind, x0, y0 in all_boxes_with_idx:
|
|
|
|
|
|
if idx in seen_idx:
|
|
|
|
|
|
continue
|
|
|
|
|
|
candidates.append((idx, kind, x0, y0))
|
|
|
|
|
|
|
|
|
|
|
|
if len(candidates) == 0:
|
|
|
|
|
|
break
|
|
|
|
|
|
left_x = min([v[2] for v in candidates])
|
|
|
|
|
|
top_y = min([v[3] for v in candidates])
|
|
|
|
|
|
|
|
|
|
|
|
candidates.sort(key=lambda x: (x[2]-left_x) ** 2 + (x[3] - top_y) ** 2)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fst_idx, fst_kind, left_x, top_y = candidates[0]
|
|
|
|
|
|
candidates.sort(key=lambda x: (x[2] - left_x) ** 2 + (x[3] - top_y)**2)
|
|
|
|
|
|
nxt = None
|
|
|
|
|
|
|
|
|
|
|
|
for i in range(1, len(candidates)):
|
|
|
|
|
|
if candidates[i][1] ^ fst_kind == 1:
|
|
|
|
|
|
nxt = candidates[i]
|
|
|
|
|
|
break
|
|
|
|
|
|
if nxt is None:
|
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
|
|
if fst_kind == SUB_BIT_KIND:
|
|
|
|
|
|
sub_idx, obj_idx = fst_idx, nxt[0] - OBJ_IDX_OFFSET
|
|
|
|
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
sub_idx, obj_idx = nxt[0], fst_idx - OBJ_IDX_OFFSET
|
|
|
|
|
|
|
|
|
|
|
|
pair_dis = bbox_distance(subjects[sub_idx]['bbox'], objects[obj_idx]['bbox'])
|
|
|
|
|
|
nearest_dis = float('inf')
|
|
|
|
|
|
for i in range(N):
|
|
|
|
|
|
if i in seen_idx or i == sub_idx:continue
|
|
|
|
|
|
nearest_dis = min(nearest_dis, bbox_distance(subjects[i]['bbox'], objects[obj_idx]['bbox']))
|
|
|
|
|
|
|
|
|
|
|
|
if pair_dis >= 3*nearest_dis:
|
|
|
|
|
|
seen_idx.add(sub_idx)
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
|
|
seen_idx.add(sub_idx)
|
|
|
|
|
|
seen_idx.add(obj_idx + OBJ_IDX_OFFSET)
|
|
|
|
|
|
seen_sub_idx.add(sub_idx)
|
|
|
|
|
|
|
|
|
|
|
|
ret.append(
|
|
|
|
|
|
{
|
|
|
|
|
|
'sub_bbox': {
|
|
|
|
|
|
'bbox': subjects[sub_idx]['bbox'],
|
|
|
|
|
|
'score': subjects[sub_idx]['score'],
|
|
|
|
|
|
},
|
|
|
|
|
|
'obj_bboxes': [
|
|
|
|
|
|
{'score': objects[obj_idx]['score'], 'bbox': objects[obj_idx]['bbox']}
|
|
|
|
|
|
],
|
|
|
|
|
|
'sub_idx': sub_idx,
|
|
|
|
|
|
}
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
for i in range(len(objects)):
|
|
|
|
|
|
j = i + OBJ_IDX_OFFSET
|
|
|
|
|
|
if j in seen_idx:
|
|
|
|
|
|
continue
|
|
|
|
|
|
seen_idx.add(j)
|
|
|
|
|
|
nearest_dis, nearest_sub_idx = float('inf'), -1
|
|
|
|
|
|
for k in range(len(subjects)):
|
|
|
|
|
|
dis = bbox_distance(objects[i]['bbox'], subjects[k]['bbox'])
|
|
|
|
|
|
if dis < nearest_dis:
|
|
|
|
|
|
nearest_dis = dis
|
|
|
|
|
|
nearest_sub_idx = k
|
|
|
|
|
|
|
|
|
|
|
|
for k in range(len(subjects)):
|
|
|
|
|
|
if k != nearest_sub_idx: continue
|
|
|
|
|
|
if k in seen_sub_idx:
|
|
|
|
|
|
for kk in range(len(ret)):
|
|
|
|
|
|
if ret[kk]['sub_idx'] == k:
|
|
|
|
|
|
ret[kk]['obj_bboxes'].append({'score': objects[i]['score'], 'bbox': objects[i]['bbox']})
|
|
|
|
|
|
break
|
|
|
|
|
|
else:
|
|
|
|
|
|
ret.append(
|
|
|
|
|
|
{
|
|
|
|
|
|
'sub_bbox': {
|
|
|
|
|
|
'bbox': subjects[k]['bbox'],
|
|
|
|
|
|
'score': subjects[k]['score'],
|
|
|
|
|
|
},
|
|
|
|
|
|
'obj_bboxes': [
|
|
|
|
|
|
{'score': objects[i]['score'], 'bbox': objects[i]['bbox']}
|
|
|
|
|
|
],
|
|
|
|
|
|
'sub_idx': k,
|
|
|
|
|
|
}
|
|
|
|
|
|
)
|
|
|
|
|
|
seen_sub_idx.add(k)
|
|
|
|
|
|
seen_idx.add(k)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for i in range(len(subjects)):
|
|
|
|
|
|
if i in seen_sub_idx:
|
|
|
|
|
|
continue
|
|
|
|
|
|
ret.append(
|
|
|
|
|
|
{
|
|
|
|
|
|
'sub_bbox': {
|
|
|
|
|
|
'bbox': subjects[i]['bbox'],
|
|
|
|
|
|
'score': subjects[i]['score'],
|
|
|
|
|
|
},
|
|
|
|
|
|
'obj_bboxes': [],
|
|
|
|
|
|
'sub_idx': i,
|
|
|
|
|
|
}
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return ret
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_imgs_v2(self, page_no: int):
|
|
|
|
|
|
with_captions = self.__tie_up_category_by_distance_v3(
|
|
|
|
|
|
page_no, 3, 4, PosRelationEnum.BOTTOM
|
|
|
|
|
|
)
|
|
|
|
|
|
with_footnotes = self.__tie_up_category_by_distance_v3(
|
|
|
|
|
|
page_no, 3, CategoryId.ImageFootnote, PosRelationEnum.ALL
|
|
|
|
|
|
)
|
|
|
|
|
|
ret = []
|
|
|
|
|
|
for v in with_captions:
|
|
|
|
|
|
record = {
|
|
|
|
|
|
'image_body': v['sub_bbox'],
|
|
|
|
|
|
'image_caption_list': v['obj_bboxes'],
|
|
|
|
|
|
}
|
|
|
|
|
|
filter_idx = v['sub_idx']
|
|
|
|
|
|
d = next(filter(lambda x: x['sub_idx'] == filter_idx, with_footnotes))
|
|
|
|
|
|
record['image_footnote_list'] = d['obj_bboxes']
|
|
|
|
|
|
ret.append(record)
|
|
|
|
|
|
return ret
|
|
|
|
|
|
|
|
|
|
|
|
def get_tables_v2(self, page_no: int) -> list:
|
|
|
|
|
|
with_captions = self.__tie_up_category_by_distance_v3(
|
|
|
|
|
|
page_no, 5, 6, PosRelationEnum.UP
|
|
|
|
|
|
)
|
|
|
|
|
|
with_footnotes = self.__tie_up_category_by_distance_v3(
|
|
|
|
|
|
page_no, 5, 7, PosRelationEnum.ALL
|
|
|
|
|
|
)
|
|
|
|
|
|
ret = []
|
|
|
|
|
|
for v in with_captions:
|
|
|
|
|
|
record = {
|
|
|
|
|
|
'table_body': v['sub_bbox'],
|
|
|
|
|
|
'table_caption_list': v['obj_bboxes'],
|
|
|
|
|
|
}
|
|
|
|
|
|
filter_idx = v['sub_idx']
|
|
|
|
|
|
d = next(filter(lambda x: x['sub_idx'] == filter_idx, with_footnotes))
|
|
|
|
|
|
record['table_footnote_list'] = d['obj_bboxes']
|
|
|
|
|
|
ret.append(record)
|
|
|
|
|
|
return ret
|
|
|
|
|
|
|
|
|
|
|
|
def get_imgs(self, page_no: int):
|
|
|
|
|
|
return self.get_imgs_v2(page_no)
|
|
|
|
|
|
|
|
|
|
|
|
def get_tables(
|
|
|
|
|
|
self, page_no: int
|
|
|
|
|
|
) -> list: # 3个坐标， caption, table主体，table-note
|
|
|
|
|
|
return self.get_tables_v2(page_no)
|
|
|
|
|
|
|
|
|
|
|
|
def get_equations(self, page_no: int) -> list: # 有坐标，也有字
|
|
|
|
|
|
inline_equations = self.__get_blocks_by_type(
|
|
|
|
|
|
ModelBlockTypeEnum.EMBEDDING.value, page_no, ['latex']
|
|
|
|
|
|
)
|
|
|
|
|
|
interline_equations = self.__get_blocks_by_type(
|
|
|
|
|
|
ModelBlockTypeEnum.ISOLATED.value, page_no, ['latex']
|
|
|
|
|
|
)
|
|
|
|
|
|
interline_equations_blocks = self.__get_blocks_by_type(
|
|
|
|
|
|
ModelBlockTypeEnum.ISOLATE_FORMULA.value, page_no
|
|
|
|
|
|
)
|
|
|
|
|
|
return inline_equations, interline_equations, interline_equations_blocks
|
|
|
|
|
|
|
|
|
|
|
|
def get_discarded(self, page_no: int) -> list: # 自研模型，只有坐标
|
|
|
|
|
|
blocks = self.__get_blocks_by_type(ModelBlockTypeEnum.ABANDON.value, page_no)
|
|
|
|
|
|
return blocks
|
|
|
|
|
|
|
|
|
|
|
|
def get_text_blocks(self, page_no: int) -> list: # 自研模型搞的，只有坐标，没有字
|
|
|
|
|
|
blocks = self.__get_blocks_by_type(ModelBlockTypeEnum.PLAIN_TEXT.value, page_no)
|
|
|
|
|
|
return blocks
|
|
|
|
|
|
|
|
|
|
|
|
def get_title_blocks(self, page_no: int) -> list: # 自研模型，只有坐标，没字
|
|
|
|
|
|
blocks = self.__get_blocks_by_type(ModelBlockTypeEnum.TITLE.value, page_no)
|
|
|
|
|
|
return blocks
|
|
|
|
|
|
|
|
|
|
|
|
def get_ocr_text(self, page_no: int) -> list: # paddle 搞的，有字也有坐标
|
|
|
|
|
|
text_spans = []
|
|
|
|
|
|
model_page_info = self.__model_list[page_no]
|
|
|
|
|
|
layout_dets = model_page_info['layout_dets']
|
|
|
|
|
|
for layout_det in layout_dets:
|
|
|
|
|
|
if layout_det['category_id'] == '15':
|
|
|
|
|
|
span = {
|
|
|
|
|
|
'bbox': layout_det['bbox'],
|
|
|
|
|
|
'content': layout_det['text'],
|
|
|
|
|
|
}
|
|
|
|
|
|
text_spans.append(span)
|
|
|
|
|
|
return text_spans
|
|
|
|
|
|
|
|
|
|
|
|
def get_all_spans(self, page_no: int) -> list:
|
|
|
|
|
|
|
|
|
|
|
|
def remove_duplicate_spans(spans):
|
|
|
|
|
|
new_spans = []
|
|
|
|
|
|
for span in spans:
|
|
|
|
|
|
if not any(span == existing_span for existing_span in new_spans):
|
|
|
|
|
|
new_spans.append(span)
|
|
|
|
|
|
return new_spans
|
|
|
|
|
|
|
|
|
|
|
|
all_spans = []
|
|
|
|
|
|
model_page_info = self.__model_list[page_no]
|
|
|
|
|
|
layout_dets = model_page_info['layout_dets']
|
|
|
|
|
|
allow_category_id_list = [3, 5, 13, 14, 15]
|
|
|
|
|
|
"""当成span拼接的"""
|
|
|
|
|
|
# 3: 'image', # 图片
|
|
|
|
|
|
# 5: 'table', # 表格
|
|
|
|
|
|
# 13: 'inline_equation', # 行内公式
|
|
|
|
|
|
# 14: 'interline_equation', # 行间公式
|
|
|
|
|
|
# 15: 'text', # ocr识别文本
|
|
|
|
|
|
for layout_det in layout_dets:
|
|
|
|
|
|
category_id = layout_det['category_id']
|
|
|
|
|
|
if category_id in allow_category_id_list:
|
|
|
|
|
|
span = {'bbox': layout_det['bbox'], 'score': layout_det['score']}
|
|
|
|
|
|
if category_id == 3:
|
|
|
|
|
|
span['type'] = ContentType.Image
|
|
|
|
|
|
elif category_id == 5:
|
|
|
|
|
|
# 获取table模型结果
|
|
|
|
|
|
latex = layout_det.get('latex', None)
|
|
|
|
|
|
html = layout_det.get('html', None)
|
|
|
|
|
|
if latex:
|
|
|
|
|
|
span['latex'] = latex
|
|
|
|
|
|
elif html:
|
|
|
|
|
|
span['html'] = html
|
|
|
|
|
|
span['type'] = ContentType.Table
|
|
|
|
|
|
elif category_id == 13:
|
|
|
|
|
|
span['content'] = layout_det['latex']
|
|
|
|
|
|
span['type'] = ContentType.InlineEquation
|
|
|
|
|
|
elif category_id == 14:
|
|
|
|
|
|
span['content'] = layout_det['latex']
|
|
|
|
|
|
span['type'] = ContentType.InterlineEquation
|
|
|
|
|
|
elif category_id == 15:
|
|
|
|
|
|
span['content'] = layout_det['text']
|
|
|
|
|
|
span['type'] = ContentType.Text
|
|
|
|
|
|
all_spans.append(span)
|
|
|
|
|
|
return remove_duplicate_spans(all_spans)
|
|
|
|
|
|
|
|
|
|
|
|
def get_page_size(self, page_no: int): # 获取页面宽高
|
|
|
|
|
|
# 获取当前页的page对象
|
|
|
|
|
|
page = self.__docs.get_page(page_no).get_page_info()
|
|
|
|
|
|
# 获取当前页的宽高
|
|
|
|
|
|
page_w = page.w
|
|
|
|
|
|
page_h = page.h
|
|
|
|
|
|
return page_w, page_h
|
|
|
|
|
|
|
|
|
|
|
|
def __get_blocks_by_type(
|
|
|
|
|
|
self, type: int, page_no: int, extra_col: list[str] = []
|
|
|
|
|
|
) -> list:
|
|
|
|
|
|
blocks = []
|
|
|
|
|
|
for page_dict in self.__model_list:
|
|
|
|
|
|
layout_dets = page_dict.get('layout_dets', [])
|
|
|
|
|
|
page_info = page_dict.get('page_info', {})
|
|
|
|
|
|
page_number = page_info.get('page_no', -1)
|
|
|
|
|
|
if page_no != page_number:
|
|
|
|
|
|
continue
|
|
|
|
|
|
for item in layout_dets:
|
|
|
|
|
|
category_id = item.get('category_id', -1)
|
|
|
|
|
|
bbox = item.get('bbox', None)
|
|
|
|
|
|
|
|
|
|
|
|
if category_id == type:
|
|
|
|
|
|
block = {
|
|
|
|
|
|
'bbox': bbox,
|
|
|
|
|
|
'score': item.get('score'),
|
|
|
|
|
|
}
|
|
|
|
|
|
for col in extra_col:
|
|
|
|
|
|
block[col] = item.get(col, None)
|
|
|
|
|
|
blocks.append(block)
|
|
|
|
|
|
return blocks
|
|
|
|
|
|
|
|
|
|
|
|
def get_model_list(self, page_no):
|
|
|
|
|
|
return self.__model_list[page_no]
|
