diff --git a/main.js b/main.js index 6793929..805f2fc 100644 --- a/main.js +++ b/main.js @@ -3,13 +3,33 @@ import { Tail } from 'tail' import { update } from './update.js' -const articles = new Map() // 文章统计 -const screenshots = new Map() // 截图统计 -const collections = new Map() // 收藏统计 +class 计数 { + 日 = new Map() + 周 = new Map() + 月 = new Map() + 年 = new Map() +} + +class 类型 { + 浏览数 = new 计数() + 评论数 = new 计数() + 点赞数 = new 计数() + 收藏数 = new 计数() +} + +const 文章 = new 类型() +const 作品 = new 类型() +const 游戏 = new 类型() +const 截图 = new 类型() +const 收藏 = new 类型() +const 综合 = new 类型() -// 日期格式化工具 -const dateFormat = new Intl.DateTimeFormat('zh-CN', { year: 'numeric', month: '2-digit', day: '2-digit' }) +const 日期格式 = new Intl.DateTimeFormat('zh-CN', { + year: 'numeric', + month: '2-digit', + day: '2-digit' +}) // 匹配文章和截图的正则表达式 @@ -18,43 +38,42 @@ const articleRegex = /\/web\/v1\/article\/get\?id=(\d+)&userId=(\d+)/ const collectionRegex = /\/web\/v1\/member\/explorer\/article\/get\?id=(\d+)/ -let currentDate = dateFormat.format(new Date()).replace(/\//g, '-') // 当前日期 -let isArchiving = false // 是否正在存档标记 +var 当前日期 = 日期格式.format(new Date()).replace(/\//g, '-') +var 正在存档 = false -// 存档函数:将数据存入文件 -const archive = async (message = '正在存档...') => { - if (isArchiving) return // 防止重复存档 - isArchiving = true +const 存档 = async (message = '正在存档...') => { + if (正在存档) return + 正在存档 = true console.log(message) - await fs.promises.writeFile(`./data/${currentDate}.json`, JSON.stringify({ - articles: Object.fromEntries(articles), - screenshots: Object.fromEntries(screenshots), - collections: Object.fromEntries(collections), + await fs.promises.writeFile(`./data/${当前日期}.json`, JSON.stringify({ + 文章: Object.fromEntries(文章), + 截图: Object.fromEntries(截图), + 收藏: Object.fromEntries(收藏), }, null, 2)) - isArchiving = false + 正在存档 = false } -// 主函数 + export default function () { - if (fs.existsSync(`./data/${currentDate}.json`)) { - console.log(currentDate, '启动时加载当日数据...') - const file = fs.readFileSync(`./data/${currentDate}.json`, 'utf8') + if (fs.existsSync(`./data/${当前日期}.json`)) { + console.log(当前日期, '启动时加载当日数据...') + const file = fs.readFileSync(`./data/${当前日期}.json`, 'utf8') if (file) { const data = JSON.parse(file) - Object.entries(data.articles).forEach(([key, value]) => articles.set(key, value)) - Object.entries(data.screenshots).forEach(([key, value]) => screenshots.set(key, value)) - Object.entries(data.collections).forEach(([key, value]) => collections.set(key, value)) + Object.entries(data.文章).forEach(([key, value]) => 文章.浏览数.日.set(key, value)) + Object.entries(data.截图).forEach(([key, value]) => 截图.浏览数.日.set(key, value)) + Object.entries(data.收藏).forEach(([key, value]) => 收藏.浏览数.日.set(key, value)) } } - console.log(currentDate, '开始收集日志...') + console.log(当前日期, '开始收集日志...') setInterval(() => { - archive('每10分钟自动存档...') - update('web_images', 'day_rank', Object.entries(Object.fromEntries(screenshots))), - update('web_article', 'day_rank', Object.entries(Object.fromEntries(articles))), - update('web_member_explorer', 'day_rank', Object.entries(Object.fromEntries(collections))) + 存档('每10分钟自动存档...') + update('web_images', 'day_rank', Object.entries(Object.fromEntries(screenshots))) + update('web_article', 'day_rank', Object.entries(Object.fromEntries(articles))) + //update('web_member_explorer', 'day_rank', Object.entries(Object.fromEntries(collections))) }, 600000) // 实时读取日志文件 @@ -65,13 +84,13 @@ export default function () { if (item.msg !== 'upstream roundtrip') return // 检查日志的日期是否跨天 - const logDate = dateFormat.format(new Date(item.ts * 1000)).replace(/\//g, '-') - if (currentDate !== logDate) { - await archive('跨日期存档...') - articles.clear() - screenshots.clear() - collections.clear() - currentDate = logDate + const logDate = 日期格式.format(new Date(item.ts * 1000)).replace(/\//g, '-') + if (当前日期 !== logDate) { + await 存档('跨日期存档...') + 文章.浏览数.日.clear() + 截图.浏览数.日.clear() + 收藏.浏览数.日.clear() + 当前日期 = logDate } // 处理文章日志 diff --git a/update.js b/update.js index 92e34d2..e0ae3d3 100644 --- a/update.js +++ b/update.js @@ -1,17 +1,40 @@ import fs from 'fs' import mysql from 'mysql2/promise' + // 更新指标到数据库 export async function update(table, rank, views) { if (!['day_rank', 'week_rank', 'month_rank', 'year_rank', 'aeon_rank'].includes(rank)) { throw new Error('字段类型错误') } + // 0=文章/游戏/作品 1=短视频 2=图片 + const TYPE = { + web_images: 2, + web_aritcle: 0 + } + + // 定义查询时间范围 + const WHERE = { + day_rank: `WHERE create_time >= CURDATE() AND type=${TYPE[table]}`, + week_rank: `WHERE create_time >= CURDATE() - INTERVAL 7 DAY AND type=${TYPE[table]}`, + month_rank: `WHERE create_time >= CURDATE() - INTERVAL 1 MONTH AND type=${TYPE[table]}`, + year_rank: `WHERE create_time >= CURDATE() - INTERVAL 1 YEAR AND type=${TYPE[table]}`, + aeon_rank: `type=${TYPE[table]}` + } + const { mysql: config } = JSON.parse(fs.readFileSync('config.json', 'utf8')) const conn = await mysql.createConnection(config) console.log('创建临时表') - await conn.query(`CREATE TEMPORARY TABLE temp_updates (id INT PRIMARY KEY, views_count INT UNSIGNED)`) + await conn.query(`CREATE TEMPORARY TABLE temp_updates ( + id INT PRIMARY KEY, + rank INT UNSIGNED, + views_count INT UNSIGNED, + praise_count INT UNSIGNED, + comment_count INT UNSIGNED, + collect_count INT UNSIGNED + )`) console.log(`数据总量 ${views.length} 条, 每次 10000 条写入 ${Math.ceil(views.length / 10000)} 次`) for (let i = 0; i < views.length; i += 10000) { @@ -20,38 +43,22 @@ export async function update(table, rank, views) { await conn.query(`INSERT INTO temp_updates (id, views_count) VALUES ?`, [values]) } + const JOIN = { + comment: `SELECT comment_id AS id, COUNT(*) AS comment_count FROM web_comment ${WHERE[rank]} GROUP BY id`, + praise: `SELECT praise_id AS id, COUNT(*) AS praise_count FROM web_praise ${WHERE[rank]} GROUP BY id`, + collect: `SELECT collect_id AS id, COUNT(*) AS collect_count FROM web_collect ${WHERE[rank]} GROUP BY id` + } + for (let name of ['comment', 'praise', 'collect']) { - console.log(`统计当天时间段内 ${name} 数到临时表`) - await conn.query(` - UPDATE temp_updates t - JOIN ( - SELECT - ${mame}_id AS article_id, - COUNT(*) AS ${name}_count - FROM - web_${name} - WHERE - DATE(create_time) = CURDATE() - GROUP BY - ${field}_id - ) stats - ON t.id = stats.article_id - SET t.${name}_count = stats.${name}_count; - `) + console.log(`统计 ${rank} 时间段内 ${name} 数到临时表`) + await conn.query(`UPDATE temp_updates t JOIN(${JOIN[name]}) stats ON t.id = stats.id SET t.${name}_count = stats.${name}_count`) } console.log('统计当天时间段内排行榜到临时表') - await conn.query(` - UPDATE temp_updates - SET day_rank = - collect_count * 35 + - praise_count * 20 + - comment_count * 20 + - views_count * 25; - `) + await conn.query("UPDATE temp_updates SET rank = collect_count * 35 + praise_count * 20 + comment_count * 20 + views_count * 25") console.log('更新数据从临时表转入生产库') - await conn.query(`UPDATE ${table} t LEFT JOIN temp_updates tu ON t.id = tu.id SET t.${rank} = IFNULL(tu.${rank}, 0)`) + await conn.query(`UPDATE ${table} t LEFT JOIN temp_updates tu ON t.id = tu.id SET t.${rank} = IFNULL(tu.rank, 0)`) await conn.end() console.log('更新完成') }