test Elasticsearch

main.py

@@ -6,9 +6,12 @@ import oss2
 import plyvel
 
 from PIL import Image, ImageFile
+from pprint import pprint
+from dotenv import dotenv_values
 
+ImageFile.LOAD_TRUNCATED_IMAGES = True
 
-# 创建或打开一个数据库
+config = dotenv_values(".env")
 db = plyvel.DB('database', create_if_missing=True)
 
 '''
@@ -29,13 +32,6 @@ for key, value in db:
 db.close()
 '''
 
-
-# 读取 .env
-from dotenv import dotenv_values
-config = dotenv_values(".env")
-
-ImageFile.LOAD_TRUNCATED_IMAGES = True
-
 # 下载图片(使用OSS下载)
 def download_image(url:str) -> Image.Image:
     if url.startswith('http://image.gameuiux.cn/') or url.startswith('https://image.gameuiux.cn/'):
@@ -58,6 +54,8 @@ def download_image(url:str) -> Image.Image:
 import pymysql
 import pymysql.cursors
 import cnocr
+import json
+import numpy as np
 
 # 打开 mysql
 ocr = cnocr.CnOcr(rec_model_name='ch_PP-OCRv3')
@@ -65,6 +63,16 @@ conn = pymysql.connect(host=config['MYSQL_HOST'], user=config['MYSQL_USER'], pas
 cursor = conn.cursor()
 cursor.execute("SELECT id, content FROM web_images LIMIT 5")
 
+class MyEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.float32):
+            return int(obj)
+        if isinstance(obj, np.ndarray):
+            return obj.astype(int).tolist()
+        return super(MyEncoder, self).default(obj)
+
+dataset = []
+
 # 获取查询结果(跳过下载失败的)
 for item in cursor.fetchall():
     image = download_image(item['content'])
@@ -72,12 +80,21 @@ for item in cursor.fetchall():
         continue
     # 将只包含那些非空非纯数字且长度大于1的'text'值
     out = ocr.ocr(image)
-    texts = [x['text'] for x in out if x['text'] and not x['text'].isdigit() and len(x['text']) > 1]
-    print(item['id'], texts)
+    out = [x for x in out if x['text'] and not x['text'].isdigit() and len(x['text']) > 1]
+    #print(item['id'], json.dumps(out, ensure_ascii=False, cls=MyEncoder))
+    dataset.append({
+        'id': item['id'],
+        'content': json.dumps(out, ensure_ascii=False, cls=MyEncoder)
+    })
+    
+    #texts = [x['text'] for x in out if x['text'] and not x['text'].isdigit() and len(x['text']) > 1]
+    #print(item['id'], texts)
 
     # 将结果存入 leveldb
     # db.put(str(row['id']).encode(), ','.join(texts).encode())
 
+print(dataset)
+
 # 关闭游标和连接
 cursor.close()
 conn.close()