You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
68 lines
2.2 KiB
Python
68 lines
2.2 KiB
Python
import json
|
|
import shutil
|
|
import os
|
|
|
|
import requests
|
|
from modelscope import snapshot_download
|
|
|
|
|
|
def download_json(url):
|
|
# 下载JSON文件
|
|
response = requests.get(url)
|
|
response.raise_for_status() # 检查请求是否成功
|
|
return response.json()
|
|
|
|
|
|
def download_and_modify_json(url, local_filename, modifications):
|
|
if os.path.exists(local_filename):
|
|
data = json.load(open(local_filename))
|
|
config_version = data.get('config_version', '0.0.0')
|
|
if config_version < '1.2.0':
|
|
data = download_json(url)
|
|
else:
|
|
data = download_json(url)
|
|
|
|
# 修改内容
|
|
for key, value in modifications.items():
|
|
data[key] = value
|
|
|
|
# 保存修改后的内容
|
|
with open(local_filename, 'w', encoding='utf-8') as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=4)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
mineru_patterns = [
|
|
# "models/Layout/LayoutLMv3/*",
|
|
"models/Layout/YOLO/*",
|
|
"models/MFD/YOLO/*",
|
|
"models/MFR/unimernet_hf_small_2503/*",
|
|
"models/OCR/paddleocr_torch/*",
|
|
# "models/TabRec/TableMaster/*",
|
|
# "models/TabRec/StructEqTable/*",
|
|
]
|
|
model_dir = snapshot_download('opendatalab/PDF-Extract-Kit-1.0', allow_patterns=mineru_patterns)
|
|
layoutreader_model_dir = snapshot_download('ppaanngggg/layoutreader')
|
|
model_dir = model_dir + '/models'
|
|
print(f'model_dir is: {model_dir}')
|
|
print(f'layoutreader_model_dir is: {layoutreader_model_dir}')
|
|
|
|
# paddleocr_model_dir = model_dir + '/OCR/paddleocr'
|
|
# user_paddleocr_dir = os.path.expanduser('~/.paddleocr')
|
|
# if os.path.exists(user_paddleocr_dir):
|
|
# shutil.rmtree(user_paddleocr_dir)
|
|
# shutil.copytree(paddleocr_model_dir, user_paddleocr_dir)
|
|
|
|
json_url = 'https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/magic-pdf.template.json'
|
|
config_file_name = 'magic-pdf.json'
|
|
home_dir = os.path.expanduser('~')
|
|
config_file = os.path.join(home_dir, config_file_name)
|
|
|
|
json_mods = {
|
|
'models-dir': model_dir,
|
|
'layoutreader-model-dir': layoutreader_model_dir,
|
|
}
|
|
|
|
download_and_modify_json(json_url, config_file, json_mods)
|
|
print(f'The configuration file has been configured successfully, the path is: {config_file}')
|