You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.
from loguru import logger
from magic_pdf . config . drop_reason import DropReason
def get_data_source ( jso : dict ) :
data_source = jso . get ( ' data_source ' )
if data_source is None :
data_source = jso . get ( ' file_source ' )
return data_source
def get_data_type ( jso : dict ) :
data_type = jso . get ( ' data_type ' )
if data_type is None :
data_type = jso . get ( ' file_type ' )
return data_type
def get_bookid ( jso : dict ) :
book_id = jso . get ( ' bookid ' )
if book_id is None :
book_id = jso . get ( ' original_file_id ' )
return book_id
def exception_handler ( jso : dict , e ) :
logger . exception ( e )
jso [ ' _need_drop ' ] = True
jso [ ' _drop_reason ' ] = DropReason . Exception
jso [ ' _exception ' ] = f ' ERROR: { e } '
return jso
def get_bookname ( jso : dict ) :
data_source = get_data_source ( jso )
file_id = jso . get ( ' file_id ' )
book_name = f ' { data_source } / { file_id } '
return book_name
def spark_json_extractor ( jso : dict ) - > dict :
""" 从json中提取数据， """
return {
' _pdf_type ' : jso [ ' _pdf_type ' ] ,
' model_list ' : jso [ ' doc_layout_result ' ] ,
}