簡化合併
This commit is contained in:
101
pp.py
101
pp.py
@@ -1,4 +1,4 @@
|
||||
#!/usr/bin/env python3
|
||||
#!/usr/bin/env python3.10
|
||||
|
||||
import os
|
||||
import io
|
||||
@@ -94,41 +94,22 @@ def process_images(conn):
|
||||
if isinstance(image, Image.Image):
|
||||
image = np.array(image)
|
||||
print('---------------------', id, content)
|
||||
en = EN.ocr(image, cls=True)[0]
|
||||
ch = CH.ocr(image, cls=True)[0]
|
||||
jp = JP.ocr(image, cls=True)[0]
|
||||
kr = KR.ocr(image, cls=True)[0]
|
||||
ru = RU.ocr(image, cls=True)[0]
|
||||
|
||||
en = en if en is not None else []
|
||||
ch = ch if ch is not None else []
|
||||
jp = jp if jp is not None else []
|
||||
kr = kr if kr is not None else []
|
||||
ru = ru if ru is not None else []
|
||||
# 執行提取文字
|
||||
en = EN.ocr(image, cls=True)[0] or []
|
||||
ch = CH.ocr(image, cls=True)[0] or []
|
||||
jp = JP.ocr(image, cls=True)[0] or []
|
||||
kr = KR.ocr(image, cls=True)[0] or []
|
||||
ru = RU.ocr(image, cls=True)[0] or []
|
||||
|
||||
# 排除字符长度小于2的行
|
||||
jp = [x for x in jp if len(x[1][0]) > 1]
|
||||
kr = [x for x in kr if len(x[1][0]) > 1]
|
||||
ch = [x for x in ch if len(x[1][0]) > 1]
|
||||
en = [x for x in en if len(x[1][0]) > 1]
|
||||
ru = [x for x in ru if len(x[1][0]) > 1]
|
||||
|
||||
# 排除纯数字的行
|
||||
jp = [x for x in jp if not x[1][0].isdigit()]
|
||||
kr = [x for x in kr if not x[1][0].isdigit()]
|
||||
ch = [x for x in ch if not x[1][0].isdigit()]
|
||||
en = [x for x in en if not x[1][0].isdigit()]
|
||||
ru = [x for x in ru if not x[1][0].isdigit()]
|
||||
|
||||
# 排除置信度小于 0.8 的行
|
||||
jp = [x for x in jp if x[1][1] > 0.8]
|
||||
kr = [x for x in kr if x[1][1] > 0.8]
|
||||
ch = [x for x in ch if x[1][1] > 0.8]
|
||||
en = [x for x in en if x[1][1] > 0.8]
|
||||
ru = [x for x in ru if x[1][1] > 0.8]
|
||||
# 排除字符长度小于2的行, 排除纯数字的行, 排除置信度小于 0.8 的行
|
||||
jp = [x for x in jp if len(x[1][0]) > 1 and not x[1][0].isdigit() and x[1][1] > 0.8]
|
||||
kr = [x for x in kr if len(x[1][0]) > 1 and not x[1][0].isdigit() and x[1][1] > 0.8]
|
||||
ch = [x for x in ch if len(x[1][0]) > 1 and not x[1][0].isdigit() and x[1][1] > 0.8]
|
||||
en = [x for x in en if len(x[1][0]) > 1 and not x[1][0].isdigit() and x[1][1] > 0.8]
|
||||
ru = [x for x in ru if len(x[1][0]) > 1 and not x[1][0].isdigit() and x[1][1] > 0.8]
|
||||
print(f'置信度大于 0.8 的行: jp {len(jp)} kr {len(kr)} ch {len(ch)} en {len(en)} ru {len(ru)}')
|
||||
|
||||
|
||||
# 去除字符串中包含的数字和标点(不作计数)
|
||||
jp_ex = [[x[0], (x[1][0].translate(str.maketrans('', '', '0123456789.,，。！？：；“”‘’\'\"')), x[1][1])] for x in jp]
|
||||
kr_ex = [[x[0], (x[1][0].translate(str.maketrans('', '', '0123456789.,，。！？：；“”‘’\'\"')), x[1][1])] for x in kr]
|
||||
@@ -136,53 +117,25 @@ def process_images(conn):
|
||||
en_ex = [[x[0], (x[1][0].translate(str.maketrans('', '', '0123456789.,，。！？：；“”‘’\'\"')), x[1][1])] for x in en]
|
||||
ru_ex = [[x[0], (x[1][0].translate(str.maketrans('', '', '0123456789.,，。！？：；“”‘’\'\"')), x[1][1])] for x in ru]
|
||||
|
||||
# 计算置信度平均数
|
||||
jpx = np.mean([x[1][1] for x in jp_ex]) if len(jp_ex) > 0 else 0
|
||||
krx = np.mean([x[1][1] for x in kr_ex]) if len(kr_ex) > 0 else 0
|
||||
chx = np.mean([x[1][1] for x in ch_ex]) if len(ch_ex) > 0 else 0
|
||||
enx = np.mean([x[1][1] for x in en_ex]) if len(en_ex) > 0 else 0
|
||||
rux = np.mean([x[1][1] for x in ru_ex]) if len(ru_ex) > 0 else 0
|
||||
# 计算置信度平均值 x 计算总字数
|
||||
jpx = (np.mean([x[1][1] for x in jp_ex]) if jp_ex else 0) * len(''.join([x[1][0] for x in jp_ex]))
|
||||
krx = (np.mean([x[1][1] for x in kr_ex]) if kr_ex else 0) * len(''.join([x[1][0] for x in kr_ex]))
|
||||
chx = (np.mean([x[1][1] for x in ch_ex]) if ch_ex else 0) * len(''.join([x[1][0] for x in ch_ex]))
|
||||
enx = (np.mean([x[1][1] for x in en_ex]) if en_ex else 0) * len(''.join([x[1][0] for x in en_ex]))
|
||||
rux = (np.mean([x[1][1] for x in ru_ex]) if ru_ex else 0) * len(''.join([x[1][0] for x in ru_ex]))
|
||||
|
||||
# 计算总字数
|
||||
jpt = len(''.join([x[1][0] for x in jp_ex]))
|
||||
krt = len(''.join([x[1][0] for x in kr_ex]))
|
||||
cht = len(''.join([x[1][0] for x in ch_ex]))
|
||||
ent = len(''.join([x[1][0] for x in en_ex]))
|
||||
rut = len(''.join([x[1][0] for x in ru_ex]))
|
||||
|
||||
# 计算总字数 x 置信度平均数
|
||||
jpx = jpx * jpt
|
||||
krx = krx * krt
|
||||
chx = chx * cht
|
||||
enx = enx * ent
|
||||
rux = rux * rut
|
||||
|
||||
print('jp', jpx)
|
||||
print('kr', krx)
|
||||
print('ch', chx)
|
||||
print('en', enx)
|
||||
print('ru', rux)
|
||||
|
||||
# 创建一个新的字典，其中键是浮点数（置信度），值是语言
|
||||
confidence_dict = {jpx: 'jp', krx: 'kr', chx: 'ch', enx: 'en', rux: 'ru'}
|
||||
|
||||
# 找出置信度最高的语言
|
||||
max = np.max([jpx, krx, chx, enx, rux])
|
||||
max_confidence_language = confidence_dict[max]
|
||||
|
||||
# 结构化存储
|
||||
data = []
|
||||
|
||||
# 使用置信度最高的语言作为键来访问字典
|
||||
all = {'en': en, 'ch': ch, 'jp': jp, 'kr': kr, 'ru': ru}
|
||||
for 坐标, 文本 in all[max_confidence_language]:
|
||||
print(max_confidence_language, 坐标, 文本)
|
||||
data.append({'text': 文本[0], 'confidence': 文本[1], 'coordinate': 坐标 })
|
||||
# 找出置信度最高的语言, 结构化存储
|
||||
confidences = {'jp': jpx, 'kr': krx, 'ch': chx, 'en': enx, 'ru': rux}
|
||||
max_confidence_language = max(confidences, key=confidences.get)
|
||||
languages = {'en': en, 'ch': ch, 'jp': jp, 'kr': kr, 'ru': ru}
|
||||
data = [{'text': text[0], 'confidence': text[1], 'coordinate': coord} for coord, text in languages[max_confidence_language]]
|
||||
print("data:", data)
|
||||
|
||||
# 转换为字符串存储到索引库
|
||||
obj = { "_id": str(id), "text": ' '.join([x['text'] for x in data]) }
|
||||
print("转换为字符串存储到索引库:", obj)
|
||||
res = requests.put(zinc_url, headers=headers, data=json.dumps(obj), proxies={'http': '', 'https': ''})
|
||||
print("\033[1;32m{}\033[0m".format(id) if json.loads(res.text)['message'] == 'ok' else id, text)
|
||||
print("\033[1;32m{}\033[0m".format(id) if json.loads(res.text)['message'] == 'ok' else obj["id"], obj["text"])
|
||||
|
||||
# 转换为 JSON 存储到数据库
|
||||
with conn.cursor() as cursor:
|
||||
|
||||
Reference in New Issue
Block a user