migrate: hustoj: fix bigint

packages/migrate/index.ts

@@ -28,6 +28,7 @@ export function apply(ctx: Context) {
             contestType: Schema.string().default('oi'),
             dataDir: Schema.string().required(),
             uploadDir: Schema.string().default('/home/judge/src/web/upload/'),
+            withContest: Schema.boolean().default(true),
         }),
         checkLock((...args) => require('./scripts/hustoj').run(...args)),
     );

packages/migrate/scripts/hustoj.ts

@@ -66,10 +66,31 @@ function fixFileName(fileName: string) {
     return fileName.replace(/[\\/?#~!|*]/g, '_');
 }
 
+async function iterate(
+    count: bigint | number, step: bigint | number, cb: (pageId: bigint) => Promise<void>,
+    reportOpts?: { every: number | bigint, namespace: string, report: (data: any) => void },
+) {
+    const _count = BigInt(count);
+    const _step = BigInt(step);
+    const { every, namespace, report } = reportOpts || {};
+    const _showProgress = every ? BigInt(every) : 0n;
+    const pageCount = _count / _step + (_count % _step === 0n ? 0n : 1n);
+    for (let pageId = 0n; pageId < pageCount; pageId++) {
+        await cb(pageId);
+        if (reportOpts && pageId % _showProgress === 0n) {
+            const progress = pageId * _step * 100n / _count;
+            report({
+                message: `${namespace} finished ${Number(pageId * _step)} / ${Number(count)} (${Number(progress)}%)`,
+            });
+        }
+    }
+}
+
 export async function run({
     host = 'localhost', port = 3306, name = 'jol',
     username, password, domainId, contestType = 'oi',
     dataDir, uploadDir = '/home/judge/src/web/upload/', rerun = true, randomMail = false,
+    withContest = true,
 }, report: (data: any) => void) {
     let remoteUsed = false;
     const src = await mariadb.createConnection({
@@ -172,10 +193,9 @@ export async function run({
     */
     const pidMap: Record<string, number> = {};
     const [{ 'count(*)': pcount }] = await query('SELECT count(*) FROM `problem`');
-    const step = 50;
+    const step = 50n;
-    const pageCount = Math.ceil(Number(pcount) / step);
+    await iterate(pcount, 50n, async (pageId: bigint) => {
-    for (let pageId = 0; pageId < pageCount; pageId++) {
+        const pdocs = await query(`SELECT * FROM \`problem\` LIMIT ${Number(pageId * step)}, ${Number(step)}`);
-        const pdocs = await query(`SELECT * FROM \`problem\` LIMIT ${pageId * step}, ${step}`);
         for (const pdoc of pdocs) {
             if (rerun) {
                 const opdoc = await ProblemModel.get(domainId, `P${pdoc.problem_id}`);
@@ -236,13 +256,8 @@ target: ybtbas/${+pdoc.id - 3000}
                 await SolutionModel.add(domainId, pidMap[pdoc.problem_id], 1, md);
             }
         }
-        if (pageId % 10 === 0) {
+    }, { every: 10n, namespace: 'problem', report });
-            const progress = Math.round(((pageId * step) / pcount) * 100);
+
-            report({
-                message: `problem finished ${pageId * step} / ${pcount} (${progress}%)`,
-            });
-        }
-    }
     if (remoteUsed) {
         MessageModel.sendNotification(`您导入的数据中使用了一本通编程启蒙远端测试题目。
 请在迁移脚本运行完成后，在终端中运行
@@ -266,6 +281,7 @@ hydrooj install https://hydro.ac/hydroac-client.zip
         user_id	char(48)			允许参加比赛用户列表
     */
     const tidMap: Record<string, string> = {};
+    if (withContest) {
         const tdocs = await query('SELECT * FROM `contest`');
         for (let tidx = 0; tidx < tdocs.length; tidx += 1) {
             const tdoc = tdocs[tidx];
@@ -314,6 +330,7 @@ hydrooj install https://hydro.ac/hydroac-client.zip
             }
         }
         report({ message: 'contest finished' });
+    }
     /*
         solution	程序运行结果记录
         字段名	类型	长度	是否允许为空	备注
@@ -336,10 +353,8 @@ hydrooj install https://hydro.ac/hydroac-client.zip
         judger	char(16)		N	判题机
     */
     // 测试运行 problem_id=0 导致非比赛的提交无法确定属于哪个题目,因此跳过测试运行
-    const [{ 'count(*)': _rcount }] = await query('SELECT count(*) FROM `solution` WHERE `problem_id` > 0');
+    const [{ 'count(*)': rcount }] = await query('SELECT count(*) FROM `solution` WHERE `problem_id` > 0');
-    const rcount = BigInt(_rcount);
+    await iterate(rcount, 50n, async (pageId: bigint) => {
-    const rpageCount = rcount / BigInt(step) + (rcount % BigInt(step) === 0n ? 0n : 1n);
-    for (let pageId = 0n; pageId < rpageCount; pageId++) {
         const rdocs = await query(`SELECT * FROM \`solution\` WHERE \`problem_id\` > 0 LIMIT ${pageId * BigInt(step)}, ${step}`);
         for (const rdoc of rdocs) {
             const data: RecordDoc = {
@@ -366,7 +381,7 @@ hydrooj install https://hydro.ac/hydroac-client.zip
             if (rtInfo[0]?.error) data.judgeTexts.push(rtInfo[0].error);
             const source = await query(`SELECT \`source\` FROM \`source_code\` WHERE \`solution_id\` = ${rdoc.solution_id}`);
             if (source[0]?.source) data.code = source[0].source;
-            if (rdoc.contest_id) {
+            if (rdoc.contest_id && withContest) {
                 if (!tidMap[rdoc.contest_id]) {
                     report({ message: `warning: contest_id ${rdoc.contest_id} for submission ${rdoc.solution_id} not found` });
                 } else {
@@ -377,13 +392,7 @@ hydrooj install https://hydro.ac/hydroac-client.zip
             await RecordModel.coll.insertOne(data);
             await postJudge(data).catch((err) => report({ message: err.message }));
         }
-        if (pageId % 10n === 0n) {
+    }, { every: 10n, namespace: 'record', report });
-            const progress = Math.round(((Number(pageId) * step) / Number(rcount)) * 100);
-            report({
-                message: `record finished ${Number(pageId * BigInt(step))} / ${Number(rcount)} (${progress}%)`,
-            });
-        }
-    }
     report({ message: 'record finished' });
 
     src.end();