diff --git a/pp.py b/pp.py index dcb4f06..fad84c3 100755 --- a/pp.py +++ b/pp.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.10 import os import io @@ -94,41 +94,22 @@ def process_images(conn): if isinstance(image, Image.Image): image = np.array(image) print('---------------------', id, content) - en = EN.ocr(image, cls=True)[0] - ch = CH.ocr(image, cls=True)[0] - jp = JP.ocr(image, cls=True)[0] - kr = KR.ocr(image, cls=True)[0] - ru = RU.ocr(image, cls=True)[0] - en = en if en is not None else [] - ch = ch if ch is not None else [] - jp = jp if jp is not None else [] - kr = kr if kr is not None else [] - ru = ru if ru is not None else [] + # 執行提取文字 + en = EN.ocr(image, cls=True)[0] or [] + ch = CH.ocr(image, cls=True)[0] or [] + jp = JP.ocr(image, cls=True)[0] or [] + kr = KR.ocr(image, cls=True)[0] or [] + ru = RU.ocr(image, cls=True)[0] or [] - # 排除字符长度小于2的行 - jp = [x for x in jp if len(x[1][0]) > 1] - kr = [x for x in kr if len(x[1][0]) > 1] - ch = [x for x in ch if len(x[1][0]) > 1] - en = [x for x in en if len(x[1][0]) > 1] - ru = [x for x in ru if len(x[1][0]) > 1] - - # 排除纯数字的行 - jp = [x for x in jp if not x[1][0].isdigit()] - kr = [x for x in kr if not x[1][0].isdigit()] - ch = [x for x in ch if not x[1][0].isdigit()] - en = [x for x in en if not x[1][0].isdigit()] - ru = [x for x in ru if not x[1][0].isdigit()] - - # 排除置信度小于 0.8 的行 - jp = [x for x in jp if x[1][1] > 0.8] - kr = [x for x in kr if x[1][1] > 0.8] - ch = [x for x in ch if x[1][1] > 0.8] - en = [x for x in en if x[1][1] > 0.8] - ru = [x for x in ru if x[1][1] > 0.8] + # 排除字符长度小于2的行, 排除纯数字的行, 排除置信度小于 0.8 的行 + jp = [x for x in jp if len(x[1][0]) > 1 and not x[1][0].isdigit() and x[1][1] > 0.8] + kr = [x for x in kr if len(x[1][0]) > 1 and not x[1][0].isdigit() and x[1][1] > 0.8] + ch = [x for x in ch if len(x[1][0]) > 1 and not x[1][0].isdigit() and x[1][1] > 0.8] + en = [x for x in en if len(x[1][0]) > 1 and not x[1][0].isdigit() and x[1][1] > 0.8] + ru = [x for x in ru if len(x[1][0]) > 1 and not x[1][0].isdigit() and x[1][1] > 0.8] print(f'置信度大于 0.8 的行: jp {len(jp)} kr {len(kr)} ch {len(ch)} en {len(en)} ru {len(ru)}') - # 去除字符串中包含的数字和标点(不作计数) jp_ex = [[x[0], (x[1][0].translate(str.maketrans('', '', '0123456789.,，。！？：；“”‘’\'\"')), x[1][1])] for x in jp] kr_ex = [[x[0], (x[1][0].translate(str.maketrans('', '', '0123456789.,，。！？：；“”‘’\'\"')), x[1][1])] for x in kr] @@ -136,53 +117,25 @@ def process_images(conn): en_ex = [[x[0], (x[1][0].translate(str.maketrans('', '', '0123456789.,，。！？：；“”‘’\'\"')), x[1][1])] for x in en] ru_ex = [[x[0], (x[1][0].translate(str.maketrans('', '', '0123456789.,，。！？：；“”‘’\'\"')), x[1][1])] for x in ru] - # 计算置信度平均数 - jpx = np.mean([x[1][1] for x in jp_ex]) if len(jp_ex) > 0 else 0 - krx = np.mean([x[1][1] for x in kr_ex]) if len(kr_ex) > 0 else 0 - chx = np.mean([x[1][1] for x in ch_ex]) if len(ch_ex) > 0 else 0 - enx = np.mean([x[1][1] for x in en_ex]) if len(en_ex) > 0 else 0 - rux = np.mean([x[1][1] for x in ru_ex]) if len(ru_ex) > 0 else 0 + # 计算置信度平均值 x 计算总字数 + jpx = (np.mean([x[1][1] for x in jp_ex]) if jp_ex else 0) * len(''.join([x[1][0] for x in jp_ex])) + krx = (np.mean([x[1][1] for x in kr_ex]) if kr_ex else 0) * len(''.join([x[1][0] for x in kr_ex])) + chx = (np.mean([x[1][1] for x in ch_ex]) if ch_ex else 0) * len(''.join([x[1][0] for x in ch_ex])) + enx = (np.mean([x[1][1] for x in en_ex]) if en_ex else 0) * len(''.join([x[1][0] for x in en_ex])) + rux = (np.mean([x[1][1] for x in ru_ex]) if ru_ex else 0) * len(''.join([x[1][0] for x in ru_ex])) - # 计算总字数 - jpt = len(''.join([x[1][0] for x in jp_ex])) - krt = len(''.join([x[1][0] for x in kr_ex])) - cht = len(''.join([x[1][0] for x in ch_ex])) - ent = len(''.join([x[1][0] for x in en_ex])) - rut = len(''.join([x[1][0] for x in ru_ex])) + # 找出置信度最高的语言, 结构化存储 + confidences = {'jp': jpx, 'kr': krx, 'ch': chx, 'en': enx, 'ru': rux} + max_confidence_language = max(confidences, key=confidences.get) + languages = {'en': en, 'ch': ch, 'jp': jp, 'kr': kr, 'ru': ru} + data = [{'text': text[0], 'confidence': text[1], 'coordinate': coord} for coord, text in languages[max_confidence_language]] + print("data:", data) - # 计算总字数 x 置信度平均数 - jpx = jpx * jpt - krx = krx * krt - chx = chx * cht - enx = enx * ent - rux = rux * rut - - print('jp', jpx) - print('kr', krx) - print('ch', chx) - print('en', enx) - print('ru', rux) - - # 创建一个新的字典，其中键是浮点数（置信度），值是语言 - confidence_dict = {jpx: 'jp', krx: 'kr', chx: 'ch', enx: 'en', rux: 'ru'} - - # 找出置信度最高的语言 - max = np.max([jpx, krx, chx, enx, rux]) - max_confidence_language = confidence_dict[max] - - # 结构化存储 - data = [] - - # 使用置信度最高的语言作为键来访问字典 - all = {'en': en, 'ch': ch, 'jp': jp, 'kr': kr, 'ru': ru} - for 坐标, 文本 in all[max_confidence_language]: - print(max_confidence_language, 坐标, 文本) - data.append({'text': 文本[0], 'confidence': 文本[1], 'coordinate': 坐标 }) - # 转换为字符串存储到索引库 obj = { "_id": str(id), "text": ' '.join([x['text'] for x in data]) } + print("转换为字符串存储到索引库:", obj) res = requests.put(zinc_url, headers=headers, data=json.dumps(obj), proxies={'http': '', 'https': ''}) - print("\033[1;32m{}\033[0m".format(id) if json.loads(res.text)['message'] == 'ok' else id, text) + print("\033[1;32m{}\033[0m".format(id) if json.loads(res.text)['message'] == 'ok' else obj["id"], obj["text"]) # 转换为 JSON 存储到数据库 with conn.cursor() as cursor: