migrate-hustoj: finish

packages/hydrooj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "hydrooj",
-    "version": "3.1.29",
+    "version": "3.1.30",
     "bin": "bin/hydrooj.js",
     "main": "src/loader",
     "module": "src/loader",

packages/hydrooj/src/handler/manage.ts

@@ -121,7 +121,7 @@ class SystemScriptHandler extends SystemHandler {
                     domainId,
                     rid,
                     status: STATUS.STATUS_SYSTEM_ERROR,
-                    message: `${err.message} \n${err.stack} `,
+                    message: `${err.message} \n${(err as any).params || []} \n${err.stack} `,
                     judger: 1,
                     time,
                     memory: 0,

packages/hydrooj/src/model/user.ts

@@ -129,7 +129,8 @@ class User implements _User {
     checkPassword(password: string) {
         const h = global.Hydro.lib[`hash.${this.hashType}`];
         if (!h) throw new Error('Unknown hash method');
-        if (h(password, this._salt, this) !== this._hash) {
+        const result = h(password, this._salt, this);
+        if (result !== true && result !== this._hash) {
             throw new LoginError(this.uname);
         } else if (this.hashType !== 'hydro') {
             // eslint-disable-next-line @typescript-eslint/no-use-before-define

packages/migrate-hustoj/lib.ts

@@ -1,11 +1,18 @@
 import { md5, sha1 } from 'hydrooj/src/lib/crypto';
 
-const RE_MD5 = /^[\da-fA-F]{32}$/;
+const RE_MD5 = /^[\da-f]{32}$/;
 
-function hash(password: string, stored: string) {
+function hash($password: string, $saved: string) {
-    if (RE_MD5.test(stored)) return md5(password) === stored;
+    $password = md5($password);
-    const salt = Buffer.from(stored, 'base64').toString().substr(0, 20);
+    if (RE_MD5.test($saved)) return $password === $saved;
-    return Buffer.from(sha1(password + salt).substr(0, 20) + salt).toString('base64') === stored;
+    const $svd = Buffer.from($saved, 'base64').toString();
+    const $salt = $svd.substr(20);
+    const $hash = Buffer.concat([
+        Buffer.from(sha1($password + $salt), 'hex'),
+        Buffer.from($salt),
+    ]).toString('base64');
+    if ($hash.trim() === $saved.trim()) return true;
+    return false;
 }
 
 global.Hydro.lib['hash.hust'] = hash;

packages/migrate-hustoj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hydrooj/migrate-hustoj",
-    "version": "0.0.2",
+    "version": "0.1.0",
     "main": "package.json",
     "repository": "git@github.com:hydro-dev/Hydro.git",
     "author": "undefined <i@undefined.moe>",
@@ -13,10 +13,8 @@
     "devDependencies": {
         "@types/mysql": "^2.15.19"
     },
-    "peerDependencies": {
-        "hydrooj": "^2.14.0"
-    },
     "dependencies": {
+        "@hydrooj/utils": "workspace:*",
         "mysql": "^2.18.1"
     }
 }

packages/migrate-hustoj/script.ts

@@ -1,13 +1,62 @@
 /* eslint-disable no-tabs */
 /* eslint-disable no-await-in-loop */
+import fs from 'fs-extra';
+import { ObjectID } from 'mongodb';
 import mysql from 'mysql';
+import { STATUS } from '@hydrooj/utils/lib/status';
+import { noop, Time } from '@hydrooj/utils/lib/utils';
+import { NotFoundError } from 'hydrooj/src/error';
+import { postJudge } from 'hydrooj/src/handler/judge';
+import { RecordDoc } from 'hydrooj/src/interface';
 import { buildContent } from 'hydrooj/src/lib/content';
+import * as contest from 'hydrooj/src/model/contest';
+import domain from 'hydrooj/src/model/domain';
 import problem from 'hydrooj/src/model/problem';
+import record from 'hydrooj/src/model/record';
 import * as system from 'hydrooj/src/model/system';
 import user from 'hydrooj/src/model/user';
 
+const statusMap = {
+    4: STATUS.STATUS_ACCEPTED,
+    5: STATUS.STATUS_WRONG_ANSWER,
+    6: STATUS.STATUS_WRONG_ANSWER,
+    7: STATUS.STATUS_TIME_LIMIT_EXCEEDED,
+    8: STATUS.STATUS_MEMORY_LIMIT_EXCEEDED,
+    9: STATUS.STATUS_OUTPUT_LIMIT_EXCEEDED,
+    10: STATUS.STATUS_RUNTIME_ERROR,
+    11: STATUS.STATUS_COMPILE_ERROR,
+};
+const langMap = {
+    0: 'c',
+    1: 'cc',
+    2: 'pas',
+    3: 'java',
+    4: 'rb',
+    5: 'bash',
+    6: 'py',
+    7: 'php',
+    8: 'perl',
+    9: 'cs',
+    10: 'oc',
+    11: 'fb',
+    12: 'sc',
+    13: 'cl',
+    14: 'cl++',
+    15: 'lua',
+    16: 'js',
+    17: 'go',
+};
+const nameMap = {
+    'sample.in': 'sample0.in',
+    'sample.out': 'sample0.out',
+    'test.in': 'test0.in',
+    'test.out': 'test0.out',
+};
+
 export async function run({
-    host = 'localhost', port = 27017, name = 'vijos4', username, password,
+    host = 'localhost', port = 3306, name = 'jol',
+    username, password, domainId, contestType = 'oi',
+    dataDir,
 }, report: Function) {
     const src = mysql.createConnection({
         host,
@@ -16,13 +65,15 @@ export async function run({
         password,
         database: name,
     });
-    await new Promise((resolve) => src.connect(resolve));
+    await new Promise((resolve, reject) => src.connect((err) => (err ? reject(err) : resolve(null))));
-    const query = (q: string | mysql.Query): Promise<[values: any[], fields: mysql.FieldInfo[]]> => new Promise((res, rej) => {
+    const query = (q: string | mysql.Query) => new Promise<[values: any[], fields: mysql.FieldInfo[]]>((res, rej) => {
         src.query(q, (err, val, fields) => {
             if (err) rej(err);
             else res([val, fields]);
         });
     });
+    const target = await domain.get(domainId);
+    if (!target) throw new NotFoundError(domainId);
     report({ message: 'Connected to database' });
     /*
         user_id     varchar 20	N	用户id（主键）
@@ -40,24 +91,39 @@ export async function run({
         school      varchar	100	N	用户所在学校
     */
     const uidMap: Record<string, number> = {};
-
     const [udocs] = await query('SELECT * FROM `users`');
+    const precheck = await user.getMulti({ unameLower: { $in: udocs.map((u) => u.user_id.toLowerCase()) } }).toArray();
+    if (precheck.length) throw new Error(`Conflict username: ${precheck.map((u) => u.unameLower).join(', ')}`);
     for (const udoc of udocs) {
-        const uid = await user.create(udoc.email, udoc.nick, '', null, udoc.ip, udoc.defunct === 'Y' ? 0 : system.get('default.priv'));
+        const current = await user.getByEmail(domainId, udoc.email || `${udoc.user_id}@hustoj.local`);
-        uidMap[udoc.uid] = uid;
+        if (current) {
+            report({ message: `duplicate user with email ${udoc.email}: ${current.uname},${udoc.user_id}` });
+            uidMap[udoc.user_id] = current._id;
+        } else {
+            const uid = await user.create(
+                udoc.email || `${udoc.user_id}@hustoj.local`, udoc.user_id, '',
+                null, udoc.ip, udoc.defunct === 'Y' ? 0 : system.get('default.priv'),
+            );
+            uidMap[udoc.user_id] = uid;
             await user.setById(uid, {
-            nSubmit: udoc.submit,
-            nSolve: udoc.solved,
                 loginat: udoc.accesstime,
                 regat: udoc.reg_time,
                 hash: udoc.password,
                 salt: udoc.password,
                 hashType: 'hust',
             });
+            await domain.setUserInDomain(domainId, uid, {
+                displayName: udoc.nick || '',
+                school: udoc.school || '',
+                nSubmit: udoc.submit,
+                nAccept: 0,
+            });
+        }
     }
 
-    const [admins] = await query("SELECT * FROM 'privilege' WHERE rightstr = 'administrator'");
+    const [admins] = await query("SELECT * FROM `privilege` WHERE `rightstr` = 'administrator'");
-    for (const admin of admins) await user.setSuperAdmin(uidMap[admin.user_id]);
+    for (const admin of admins) await domain.setUserRole(domainId, uidMap[admin.user_id], 'root');
+    const adminUids = admins.map((admin) => uidMap[admin.user_id]);
 
     /*
         problem_id	int	11	N	题目编号，主键
@@ -78,39 +144,150 @@ export async function run({
         submit	int	11	Y	总提交次数
         solved	int	11	Y	解答（未用）
     */
-    const [[pcount]] = await query('SELECT count(*) FROM `problem`');
+    const pidMap: Record<string, number> = {};
+    const [[{ 'count(*)': pcount }]] = await query('SELECT count(*) FROM `problem`');
     const step = 50;
     const pageCount = Math.ceil(pcount / step);
     for (let pageId = 0; pageId < pageCount; pageId++) {
         const [pdocs] = await query(`SELECT * FROM \`problem\` LIMIT ${pageId * step}, ${step}`);
         for (const pdoc of pdocs) {
             const pid = await problem.add(
-                'system', `P${pdoc.problem_id}`,
+                domainId, `P${pdoc.problem_id}`,
                 pdoc.title, buildContent({
                     description: pdoc.description,
                     input: pdoc.input,
                     output: pdoc.output,
-                    samples: [[pdoc.sample_input, pdoc.sample_output]],
+                    samples: [[pdoc.sample_input.trim(), pdoc.sample_output.trim()]],
                     hint: pdoc.hint,
                     source: pdoc.source,
-                }),
+                }, 'html'),
-                1, pdoc.source.split(' '), pdoc.defunct === 'Y',
+                1, pdoc.source.split(' ').map((i) => i.trim()).filter((i) => i), pdoc.defunct === 'Y',
             );
-            const [cdoc] = await query(`SELECT * FROM 'privilege' WHERE rightstr = 'p${pdoc.problem_id}'`);
+            pidMap[pdoc.problem_id] = pid;
-            await problem.edit('system', pid, {
+            const [cdoc] = await query(`SELECT * FROM \`privilege\` WHERE \`rightstr\` = 'p${pdoc.problem_id}'`);
-                nAccept: pdoc.accepted,
+            const maintainer = [];
+            for (let i = 1; i < cdoc.length; i++) maintainer.push(uidMap[cdoc[i].user_id]);
+            await problem.edit(domainId, pid, {
+                nAccept: 0,
                 nSubmit: pdoc.submit,
                 config: `time: ${pdoc.time_limit}s\nmemory: ${pdoc.memory_limit}m`,
-                owner: uidMap[cdoc[0].user_id],
+                owner: uidMap[cdoc[0]?.user_id] || 1,
+                maintainer,
+                html: true,
             });
         }
     }
+
+    /*
+        contest_id	int	11	N	竞赛id（主键）
+        title	varchar	255	Y	竞赛标题
+        start_time	datetime		Y	开始时间(年月日时分)
+        end_time	datatime		Y	结束时间(年月日时分)
+        defunct	char	1	N	是否屏蔽（Y/N）
+        description	text		Y	描述（在此版本中未用）
+        private	tinyint	4		公开/内部（0/1）
+        langmask	int	11		语言
+        password	char(16)			进入比赛的密码
+        user_id	char(48)			允许参加比赛用户列表
+    */
+    const tidMap: Record<string, string> = {};
+    const [tdocs] = await query('SELECT * FROM `contest`');
+    for (const tdoc of tdocs) {
+        const [pdocs] = await query(`SELECT * FROM \`contest_problem\` WHERE \`contest_id\` = ${tdoc.contest_id}`);
+        const pids = pdocs.map((i) => pidMap[i.problem_id]);
+        const tid = await contest.add(
+            domainId, tdoc.title, tdoc.description || 'Description',
+            adminUids[0], contestType, tdoc.start_time, tdoc.end_time, pids, true,
+            { _code: password },
+        );
+        tidMap[tdoc.contest_id] = tid.toHexString();
+    }
+
+    /*
+        solution	程序运行结果记录
+        字段名	类型	长度	是否允许为空	备注
+        solution_id	int	11	N	运行id（主键）
+        problem_id	int	11	N	问题id
+        user_id	char	20	N	用户id
+        time	int	11	N	用时（秒）
+        memory	int	11	N	所用空间（）
+        in_date	datetime		N	加入时间
+        result	smallint	6	N	结果（4：AC）
+        language	tinyint	4	N	语言
+        ip	char	15	N	用户ip
+        contest_id	int	11	Y	所属于竞赛组
+        valid	tinyint	4	N	是否有效？？？
+        num	tinyint	4	N	题目在竞赛中的顺序号
+        code_lenght	int	11	N	代码长度
+        judgetime	datetime		Y	判题时间
+        pass_rate	decimal	2	N	通过百分比（OI模式下可用）
+        lint_error	int		N	？？？
+        judger	char(16)		N	判题机
+    */
+    const [[{ 'count(*)': rcount }]] = await query('SELECT count(*) FROM `solution`');
+    const rpageCount = Math.ceil(rcount / step);
+    for (let pageId = 0; pageId < rpageCount; pageId++) {
+        const [rdocs] = await query(`SELECT * FROM \`solution\` LIMIT ${pageId * step}, ${step}`);
+        for (const rdoc of rdocs) {
+            const data: RecordDoc = {
+                status: statusMap[rdoc.result],
+                _id: Time.getObjectID(rdoc.in_date, false),
+                uid: uidMap[rdoc.user_id],
+                code: "HustOJ didn't provide user code",
+                lang: langMap[rdoc.language],
+                pid: pidMap[rdoc.problem_id],
+                domainId,
+                score: rdoc.pass_rate ? Math.ceil(rdoc.pass_rate * 100) : rdoc.result === 4 ? 100 : 0,
+                time: rdoc.time,
+                memory: rdoc.memory,
+                judgeTexts: [],
+                compilerTexts: [],
+                testCases: [],
+                judgeAt: new Date(),
+                rejudged: false,
+                judger: 1,
+            };
+            const [ceInfo] = await query(`SELECT \`error\` FROM \`compileinfo\` WHERE \`solution_id\` = ${rdoc.solution_id}`);
+            if (ceInfo[0]?.error) data.judgeTexts.push(ceInfo[0].error);
+            const [rtInfo] = await query(`SELECT \`error\` FROM \`runtimeinfo\` WHERE \`solution_id\` = ${rdoc.solution_id}`);
+            if (rtInfo[0]?.error) data.judgeTexts.push(rtInfo[0].error);
+            const [source] = await query(`SELECT \`source\` FROM \`source_code\` WHERE \`solution_id\` = ${rdoc.solution_id}`);
+            if (source[0]?.source) data.code = source[0].source;
+            if (rdoc.contest_id) {
+                data.contest = new ObjectID(tidMap[rdoc.contest_id]);
+                await contest.attend(domainId, data.contest, uidMap[rdoc.user_id]).catch(noop);
+            }
+            await record.coll.insertOne(data);
+            await postJudge(data);
+        }
+    }
+
+    src.end();
+
+    if (dataDir.endsWith('/')) dataDir = dataDir.slice(0, -1);
+    const files = await fs.readdir(dataDir, { withFileTypes: true });
+    for (const file of files) {
+        if (!file.isDirectory()) continue;
+        if (!pidMap[file.name]) continue;
+        const datas = await fs.readdir(`${dataDir}/${file.name}`);
+        for (const data of datas) {
+            const filename = nameMap[data] || data;
+            await problem.addTestdata(domainId, pidMap[file.name], filename, `${dataDir}/${file.name}/${data}`);
+        }
+    }
     return true;
 }
 
 export const description = 'migrate from hustoj';
 export const validate = {
-    host: 'string', port: 'number', name: 'string', username: 'string', password: 'string',
+    host: 'string',
+    port: 'number',
+    name: 'string',
+    username: 'string',
+    password: 'string',
+    domainId: 'string',
+    contestType: 'string',
+    dataDir: 'string',
 };
 
 global.Hydro.script.migrateHustoj = { run, description, validate };

packages/utils/lib/utils.ts

@@ -263,14 +263,13 @@ export namespace Time {
         return `${ms}ms`;
     }
 
-    export function getObjectID(timestamp: string | Date | Moment) {
+    export function getObjectID(timestamp: string | Date | Moment, allZero = true) {
         let _timestamp: number;
         if (typeof timestamp === 'string') _timestamp = new Date(timestamp).getTime();
         else if (isMoment(timestamp)) _timestamp = timestamp.toDate().getTime();
         else _timestamp = timestamp.getTime();
         const hexSeconds = Math.floor(_timestamp / 1000).toString(16);
-        const constructedObjectId = new ObjectID(`${hexSeconds}0000000000000000`);
+        return new ObjectID(`${hexSeconds}${allZero ? '0000000000000000' : new ObjectID().toHexString().substr(8, 12)}`);
-        return constructedObjectId;
     }
 }
 

packages/utils/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hydrooj/utils",
-    "version": "1.1.5",
+    "version": "1.1.6",
     "description": "hydrooj utils",
     "main": "package.json",
     "repository": "https://github.com/hydro-dev/Hydro.git",