From 803885330de5132896c1c090a7368eb58a7d3154 Mon Sep 17 00:00:00 2001 From: undefined Date: Sat, 13 Nov 2021 23:00:32 +0800 Subject: [PATCH] migrate-hustoj: finish --- packages/hydrooj/package.json | 2 +- packages/hydrooj/src/handler/manage.ts | 2 +- packages/hydrooj/src/model/user.ts | 3 +- packages/migrate-hustoj/lib.ts | 17 +- packages/migrate-hustoj/package.json | 6 +- packages/migrate-hustoj/script.ts | 245 +++++++++++++++++++++---- packages/utils/lib/utils.ts | 5 +- packages/utils/package.json | 2 +- 8 files changed, 232 insertions(+), 50 deletions(-) diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index 044bd86cc..da122bf95 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "3.1.29", + "version": "3.1.30", "bin": "bin/hydrooj.js", "main": "src/loader", "module": "src/loader", diff --git a/packages/hydrooj/src/handler/manage.ts b/packages/hydrooj/src/handler/manage.ts index 68bbf82fd..ee3dff8f8 100644 --- a/packages/hydrooj/src/handler/manage.ts +++ b/packages/hydrooj/src/handler/manage.ts @@ -121,7 +121,7 @@ class SystemScriptHandler extends SystemHandler { domainId, rid, status: STATUS.STATUS_SYSTEM_ERROR, - message: `${err.message} \n${err.stack} `, + message: `${err.message} \n${(err as any).params || []} \n${err.stack} `, judger: 1, time, memory: 0, diff --git a/packages/hydrooj/src/model/user.ts b/packages/hydrooj/src/model/user.ts index 17ad0b12e..ebbd0997e 100644 --- a/packages/hydrooj/src/model/user.ts +++ b/packages/hydrooj/src/model/user.ts @@ -129,7 +129,8 @@ class User implements _User { checkPassword(password: string) { const h = global.Hydro.lib[`hash.${this.hashType}`]; if (!h) throw new Error('Unknown hash method'); - if (h(password, this._salt, this) !== this._hash) { + const result = h(password, this._salt, this); + if (result !== true && result !== this._hash) { throw new LoginError(this.uname); } else if (this.hashType !== 'hydro') { // eslint-disable-next-line @typescript-eslint/no-use-before-define diff --git a/packages/migrate-hustoj/lib.ts b/packages/migrate-hustoj/lib.ts index 19f93e443..159601ae6 100644 --- a/packages/migrate-hustoj/lib.ts +++ b/packages/migrate-hustoj/lib.ts @@ -1,11 +1,18 @@ import { md5, sha1 } from 'hydrooj/src/lib/crypto'; -const RE_MD5 = /^[\da-fA-F]{32}$/; +const RE_MD5 = /^[\da-f]{32}$/; -function hash(password: string, stored: string) { - if (RE_MD5.test(stored)) return md5(password) === stored; - const salt = Buffer.from(stored, 'base64').toString().substr(0, 20); - return Buffer.from(sha1(password + salt).substr(0, 20) + salt).toString('base64') === stored; +function hash($password: string, $saved: string) { + $password = md5($password); + if (RE_MD5.test($saved)) return $password === $saved; + const $svd = Buffer.from($saved, 'base64').toString(); + const $salt = $svd.substr(20); + const $hash = Buffer.concat([ + Buffer.from(sha1($password + $salt), 'hex'), + Buffer.from($salt), + ]).toString('base64'); + if ($hash.trim() === $saved.trim()) return true; + return false; } global.Hydro.lib['hash.hust'] = hash; diff --git a/packages/migrate-hustoj/package.json b/packages/migrate-hustoj/package.json index 988726e02..ec43d7d0e 100644 --- a/packages/migrate-hustoj/package.json +++ b/packages/migrate-hustoj/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/migrate-hustoj", - "version": "0.0.2", + "version": "0.1.0", "main": "package.json", "repository": "git@github.com:hydro-dev/Hydro.git", "author": "undefined ", @@ -13,10 +13,8 @@ "devDependencies": { "@types/mysql": "^2.15.19" }, - "peerDependencies": { - "hydrooj": "^2.14.0" - }, "dependencies": { + "@hydrooj/utils": "workspace:*", "mysql": "^2.18.1" } } diff --git a/packages/migrate-hustoj/script.ts b/packages/migrate-hustoj/script.ts index 4c8c5c382..8e3cd0257 100644 --- a/packages/migrate-hustoj/script.ts +++ b/packages/migrate-hustoj/script.ts @@ -1,13 +1,62 @@ /* eslint-disable no-tabs */ /* eslint-disable no-await-in-loop */ +import fs from 'fs-extra'; +import { ObjectID } from 'mongodb'; import mysql from 'mysql'; +import { STATUS } from '@hydrooj/utils/lib/status'; +import { noop, Time } from '@hydrooj/utils/lib/utils'; +import { NotFoundError } from 'hydrooj/src/error'; +import { postJudge } from 'hydrooj/src/handler/judge'; +import { RecordDoc } from 'hydrooj/src/interface'; import { buildContent } from 'hydrooj/src/lib/content'; +import * as contest from 'hydrooj/src/model/contest'; +import domain from 'hydrooj/src/model/domain'; import problem from 'hydrooj/src/model/problem'; +import record from 'hydrooj/src/model/record'; import * as system from 'hydrooj/src/model/system'; import user from 'hydrooj/src/model/user'; +const statusMap = { + 4: STATUS.STATUS_ACCEPTED, + 5: STATUS.STATUS_WRONG_ANSWER, + 6: STATUS.STATUS_WRONG_ANSWER, + 7: STATUS.STATUS_TIME_LIMIT_EXCEEDED, + 8: STATUS.STATUS_MEMORY_LIMIT_EXCEEDED, + 9: STATUS.STATUS_OUTPUT_LIMIT_EXCEEDED, + 10: STATUS.STATUS_RUNTIME_ERROR, + 11: STATUS.STATUS_COMPILE_ERROR, +}; +const langMap = { + 0: 'c', + 1: 'cc', + 2: 'pas', + 3: 'java', + 4: 'rb', + 5: 'bash', + 6: 'py', + 7: 'php', + 8: 'perl', + 9: 'cs', + 10: 'oc', + 11: 'fb', + 12: 'sc', + 13: 'cl', + 14: 'cl++', + 15: 'lua', + 16: 'js', + 17: 'go', +}; +const nameMap = { + 'sample.in': 'sample0.in', + 'sample.out': 'sample0.out', + 'test.in': 'test0.in', + 'test.out': 'test0.out', +}; + export async function run({ - host = 'localhost', port = 27017, name = 'vijos4', username, password, + host = 'localhost', port = 3306, name = 'jol', + username, password, domainId, contestType = 'oi', + dataDir, }, report: Function) { const src = mysql.createConnection({ host, @@ -16,48 +65,65 @@ export async function run({ password, database: name, }); - await new Promise((resolve) => src.connect(resolve)); - const query = (q: string | mysql.Query): Promise<[values: any[], fields: mysql.FieldInfo[]]> => new Promise((res, rej) => { + await new Promise((resolve, reject) => src.connect((err) => (err ? reject(err) : resolve(null)))); + const query = (q: string | mysql.Query) => new Promise<[values: any[], fields: mysql.FieldInfo[]]>((res, rej) => { src.query(q, (err, val, fields) => { if (err) rej(err); else res([val, fields]); }); }); + const target = await domain.get(domainId); + if (!target) throw new NotFoundError(domainId); report({ message: 'Connected to database' }); /* - user_id varchar 20 N 用户id（主键） - email varchar 100 Y 用户E-mail - submit int 11 Y 用户提交次数 - solved int 11 Y 成功次数 - defunct char 1 N 是否屏蔽（Y/N） - ip varchar 20 N 用户注册ip + user_id varchar 20 N 用户id（主键） + email varchar 100 Y 用户E-mail + submit int 11 Y 用户提交次数 + solved int 11 Y 成功次数 + defunct char 1 N 是否屏蔽（Y/N） + ip varchar 20 N 用户注册ip accesstime datetime Y 用户注册时间 volume int 11 N 页码（表示用户上次看到第几页） language int 11 N 语言 password varchar 32 Y 密码（加密） - reg_time datetime Y 用户注册时间 + reg_time datetime Y 用户注册时间 nick varchar 100 N 昵称 school varchar 100 N 用户所在学校 */ const uidMap: Record = {}; - const [udocs] = await query('SELECT * FROM `users`'); + const precheck = await user.getMulti({ unameLower: { $in: udocs.map((u) => u.user_id.toLowerCase()) } }).toArray(); + if (precheck.length) throw new Error(`Conflict username: ${precheck.map((u) => u.unameLower).join(', ')}`); for (const udoc of udocs) { - const uid = await user.create(udoc.email, udoc.nick, '', null, udoc.ip, udoc.defunct === 'Y' ? 0 : system.get('default.priv')); - uidMap[udoc.uid] = uid; - await user.setById(uid, { - nSubmit: udoc.submit, - nSolve: udoc.solved, - loginat: udoc.accesstime, - regat: udoc.reg_time, - hash: udoc.password, - salt: udoc.password, - hashType: 'hust', - }); + const current = await user.getByEmail(domainId, udoc.email || `${udoc.user_id}@hustoj.local`); + if (current) { + report({ message: `duplicate user with email ${udoc.email}: ${current.uname},${udoc.user_id}` }); + uidMap[udoc.user_id] = current._id; + } else { + const uid = await user.create( + udoc.email || `${udoc.user_id}@hustoj.local`, udoc.user_id, '', + null, udoc.ip, udoc.defunct === 'Y' ? 0 : system.get('default.priv'), + ); + uidMap[udoc.user_id] = uid; + await user.setById(uid, { + loginat: udoc.accesstime, + regat: udoc.reg_time, + hash: udoc.password, + salt: udoc.password, + hashType: 'hust', + }); + await domain.setUserInDomain(domainId, uid, { + displayName: udoc.nick || '', + school: udoc.school || '', + nSubmit: udoc.submit, + nAccept: 0, + }); + } } - const [admins] = await query("SELECT * FROM 'privilege' WHERE rightstr = 'administrator'"); - for (const admin of admins) await user.setSuperAdmin(uidMap[admin.user_id]); + const [admins] = await query("SELECT * FROM `privilege` WHERE `rightstr` = 'administrator'"); + for (const admin of admins) await domain.setUserRole(domainId, uidMap[admin.user_id], 'root'); + const adminUids = admins.map((admin) => uidMap[admin.user_id]); /* problem_id int 11 N 题目编号，主键 @@ -78,39 +144,150 @@ export async function run({ submit int 11 Y 总提交次数 solved int 11 Y 解答（未用） */ - const [[pcount]] = await query('SELECT count(*) FROM `problem`'); + const pidMap: Record = {}; + const [[{ 'count(*)': pcount }]] = await query('SELECT count(*) FROM `problem`'); const step = 50; const pageCount = Math.ceil(pcount / step); for (let pageId = 0; pageId < pageCount; pageId++) { const [pdocs] = await query(`SELECT * FROM \`problem\` LIMIT ${pageId * step}, ${step}`); for (const pdoc of pdocs) { const pid = await problem.add( - 'system', `P${pdoc.problem_id}`, + domainId, `P${pdoc.problem_id}`, pdoc.title, buildContent({ description: pdoc.description, input: pdoc.input, output: pdoc.output, - samples: [[pdoc.sample_input, pdoc.sample_output]], + samples: [[pdoc.sample_input.trim(), pdoc.sample_output.trim()]], hint: pdoc.hint, source: pdoc.source, - }), - 1, pdoc.source.split(' '), pdoc.defunct === 'Y', + }, 'html'), + 1, pdoc.source.split(' ').map((i) => i.trim()).filter((i) => i), pdoc.defunct === 'Y', ); - const [cdoc] = await query(`SELECT * FROM 'privilege' WHERE rightstr = 'p${pdoc.problem_id}'`); - await problem.edit('system', pid, { - nAccept: pdoc.accepted, + pidMap[pdoc.problem_id] = pid; + const [cdoc] = await query(`SELECT * FROM \`privilege\` WHERE \`rightstr\` = 'p${pdoc.problem_id}'`); + const maintainer = []; + for (let i = 1; i < cdoc.length; i++) maintainer.push(uidMap[cdoc[i].user_id]); + await problem.edit(domainId, pid, { + nAccept: 0, nSubmit: pdoc.submit, config: `time: ${pdoc.time_limit}s\nmemory: ${pdoc.memory_limit}m`, - owner: uidMap[cdoc[0].user_id], + owner: uidMap[cdoc[0]?.user_id] || 1, + maintainer, + html: true, }); } } + + /* + contest_id int 11 N 竞赛id（主键） + title varchar 255 Y 竞赛标题 + start_time datetime Y 开始时间(年月日时分) + end_time datatime Y 结束时间(年月日时分) + defunct char 1 N 是否屏蔽（Y/N） + description text Y 描述（在此版本中未用） + private tinyint 4 公开/内部（0/1） + langmask int 11 语言 + password char(16) 进入比赛的密码 + user_id char(48) 允许参加比赛用户列表 + */ + const tidMap: Record = {}; + const [tdocs] = await query('SELECT * FROM `contest`'); + for (const tdoc of tdocs) { + const [pdocs] = await query(`SELECT * FROM \`contest_problem\` WHERE \`contest_id\` = ${tdoc.contest_id}`); + const pids = pdocs.map((i) => pidMap[i.problem_id]); + const tid = await contest.add( + domainId, tdoc.title, tdoc.description || 'Description', + adminUids[0], contestType, tdoc.start_time, tdoc.end_time, pids, true, + { _code: password }, + ); + tidMap[tdoc.contest_id] = tid.toHexString(); + } + + /* + solution 程序运行结果记录 + 字段名 类型 长度 是否允许为空 备注 + solution_id int 11 N 运行id（主键） + problem_id int 11 N 问题id + user_id char 20 N 用户id + time int 11 N 用时（秒） + memory int 11 N 所用空间（） + in_date datetime N 加入时间 + result smallint 6 N 结果（4：AC） + language tinyint 4 N 语言 + ip char 15 N 用户ip + contest_id int 11 Y 所属于竞赛组 + valid tinyint 4 N 是否有效？？？ + num tinyint 4 N 题目在竞赛中的顺序号 + code_lenght int 11 N 代码长度 + judgetime datetime Y 判题时间 + pass_rate decimal 2 N 通过百分比（OI模式下可用） + lint_error int N ？？？ + judger char(16) N 判题机 + */ + const [[{ 'count(*)': rcount }]] = await query('SELECT count(*) FROM `solution`'); + const rpageCount = Math.ceil(rcount / step); + for (let pageId = 0; pageId < rpageCount; pageId++) { + const [rdocs] = await query(`SELECT * FROM \`solution\` LIMIT ${pageId * step}, ${step}`); + for (const rdoc of rdocs) { + const data: RecordDoc = { + status: statusMap[rdoc.result], + _id: Time.getObjectID(rdoc.in_date, false), + uid: uidMap[rdoc.user_id], + code: "HustOJ didn't provide user code", + lang: langMap[rdoc.language], + pid: pidMap[rdoc.problem_id], + domainId, + score: rdoc.pass_rate ? Math.ceil(rdoc.pass_rate * 100) : rdoc.result === 4 ? 100 : 0, + time: rdoc.time, + memory: rdoc.memory, + judgeTexts: [], + compilerTexts: [], + testCases: [], + judgeAt: new Date(), + rejudged: false, + judger: 1, + }; + const [ceInfo] = await query(`SELECT \`error\` FROM \`compileinfo\` WHERE \`solution_id\` = ${rdoc.solution_id}`); + if (ceInfo[0]?.error) data.judgeTexts.push(ceInfo[0].error); + const [rtInfo] = await query(`SELECT \`error\` FROM \`runtimeinfo\` WHERE \`solution_id\` = ${rdoc.solution_id}`); + if (rtInfo[0]?.error) data.judgeTexts.push(rtInfo[0].error); + const [source] = await query(`SELECT \`source\` FROM \`source_code\` WHERE \`solution_id\` = ${rdoc.solution_id}`); + if (source[0]?.source) data.code = source[0].source; + if (rdoc.contest_id) { + data.contest = new ObjectID(tidMap[rdoc.contest_id]); + await contest.attend(domainId, data.contest, uidMap[rdoc.user_id]).catch(noop); + } + await record.coll.insertOne(data); + await postJudge(data); + } + } + + src.end(); + + if (dataDir.endsWith('/')) dataDir = dataDir.slice(0, -1); + const files = await fs.readdir(dataDir, { withFileTypes: true }); + for (const file of files) { + if (!file.isDirectory()) continue; + if (!pidMap[file.name]) continue; + const datas = await fs.readdir(`${dataDir}/${file.name}`); + for (const data of datas) { + const filename = nameMap[data] || data; + await problem.addTestdata(domainId, pidMap[file.name], filename, `${dataDir}/${file.name}/${data}`); + } + } return true; } export const description = 'migrate from hustoj'; export const validate = { - host: 'string', port: 'number', name: 'string', username: 'string', password: 'string', + host: 'string', + port: 'number', + name: 'string', + username: 'string', + password: 'string', + domainId: 'string', + contestType: 'string', + dataDir: 'string', }; global.Hydro.script.migrateHustoj = { run, description, validate }; diff --git a/packages/utils/lib/utils.ts b/packages/utils/lib/utils.ts index 319d448f3..ccb1223d4 100644 --- a/packages/utils/lib/utils.ts +++ b/packages/utils/lib/utils.ts @@ -263,14 +263,13 @@ export namespace Time { return `${ms}ms`; } - export function getObjectID(timestamp: string | Date | Moment) { + export function getObjectID(timestamp: string | Date | Moment, allZero = true) { let _timestamp: number; if (typeof timestamp === 'string') _timestamp = new Date(timestamp).getTime(); else if (isMoment(timestamp)) _timestamp = timestamp.toDate().getTime(); else _timestamp = timestamp.getTime(); const hexSeconds = Math.floor(_timestamp / 1000).toString(16); - const constructedObjectId = new ObjectID(`${hexSeconds}0000000000000000`); - return constructedObjectId; + return new ObjectID(`${hexSeconds}${allZero ? '0000000000000000' : new ObjectID().toHexString().substr(8, 12)}`); } } diff --git a/packages/utils/package.json b/packages/utils/package.json index e27b6fb05..6aa28baa6 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/utils", - "version": "1.1.5", + "version": "1.1.6", "description": "hydrooj utils", "main": "package.json", "repository": "https://github.com/hydro-dev/Hydro.git",