From 510d156383df735d1025398333599180593fbca0 Mon Sep 17 00:00:00 2001 From: panda Date: Fri, 10 May 2024 15:46:47 +0800 Subject: [PATCH] migrate: add poj & fix bug (#801) --- install/install.ts | 2 +- packages/migrate/index.ts | 28 ++- packages/migrate/scripts/hustoj.ts | 2 +- packages/migrate/scripts/poj.ts | 286 +++++++++++++++++++++++++++++ 4 files changed, 310 insertions(+), 8 deletions(-) create mode 100644 packages/migrate/scripts/poj.ts diff --git a/install/install.ts b/install/install.ts index 910f79354..2459aa204 100644 --- a/install/install.ts +++ b/install/install.ts @@ -558,7 +558,7 @@ ${nixConfBase}`); }; console.log(config); // TODO mail config - exec(`hydrooj cli script migrateUoj '${JSON.stringify(config)}'`, { stdio: 'inherit' }); + exec(`hydrooj cli script migrateUniversaloj '${JSON.stringify(config)}'`, { stdio: 'inherit' }); }, ], }, diff --git a/packages/migrate/index.ts b/packages/migrate/index.ts index 265f25ad3..7f8912ac9 100644 --- a/packages/migrate/index.ts +++ b/packages/migrate/index.ts @@ -9,13 +9,13 @@ export function apply(ctx: Context) { ctx.addScript( 'migrateHustoj', 'migrate from hustoj', Schema.object({ - host: Schema.string().required(), - port: Schema.number().required(), - name: Schema.string().required(), + host: Schema.string().default('localhost'), + port: Schema.number().default(3306), + name: Schema.string().default('jol'), username: Schema.string().required(), password: Schema.string().required(), - domainId: Schema.string().required(), - contestType: Schema.string().required(), + domainId: Schema.string().default('system'), + contestType: Schema.string().default('oi'), dataDir: Schema.string().required(), uploadDir: Schema.string().default('/home/judge/src/web/upload/'), }), @@ -46,7 +46,7 @@ export function apply(ctx: Context) { (...args) => require('./scripts/vijos').run(...args), ); ctx.addScript( - 'migrateuniversaloj', 'migrate from universaloj', + 'migrateUniversaloj', 'migrate from universaloj', Schema.object({ host: Schema.string().default('172.17.0.2'), port: Schema.number().default(3306), @@ -58,6 +58,21 @@ export function apply(ctx: Context) { }), (...args) => require('./scripts/universaloj').run(...args), ); + ctx.addScript( + 'migratePoj', 'migrate from poj', + Schema.object({ + host: Schema.string().required(), + port: Schema.number().default(3306), + name: Schema.string().default('judgeonline'), + username: Schema.string().required(), + password: Schema.string().required(), + domainId: Schema.string().default('system'), + contestType: Schema.string().default('oi'), + dataDir: Schema.string().required(), + imageDir: Schema.string().required(), + }), + (...args) => require('./scripts/poj').run(...args), + ); ctx.provideModule('hash', 'hust', ($password, $saved) => { $password = md5($password); @@ -84,5 +99,6 @@ export function apply(ctx: Context) { 'migrate from vijos': '从 Vijos 导入', 'migrate from syzoj': '从 SYZOJ 导入', 'migrate from universaloj': '从 UniversalOJ 导入', + 'migrate from poj': '从 POJ 导入', }); } diff --git a/packages/migrate/scripts/hustoj.ts b/packages/migrate/scripts/hustoj.ts index 542a1e780..7d0ea4a49 100644 --- a/packages/migrate/scripts/hustoj.ts +++ b/packages/migrate/scripts/hustoj.ts @@ -254,7 +254,7 @@ hydrooj install https://hydro.ac/hydroac-client.zip // WHY you allow contest with end time BEFORE start time? WHY??? const endAt = moment(tdoc.end_time).isSameOrBefore(tdoc.start_time) ? moment(tdoc.end_time).add(1, 'minute').toDate() : tdoc.end_time; const tid = await ContestModel.add( - domainId, tdoc.title, tdoc.description || 'Description', + domainId, tdoc.title, description || 'Description', adminUids[0], contestType, tdoc.start_time, endAt, pids, true, { _code: password }, ); diff --git a/packages/migrate/scripts/poj.ts b/packages/migrate/scripts/poj.ts new file mode 100644 index 000000000..295eb8a0a --- /dev/null +++ b/packages/migrate/scripts/poj.ts @@ -0,0 +1,286 @@ +/* eslint-disable no-tabs */ +/* eslint-disable no-await-in-loop */ +import path from 'path'; +import zlib from 'zlib'; +import mariadb from 'mariadb'; +import { + buildContent, ContestModel, DomainModel, fs, moment, noop, NotFoundError, ObjectId, postJudge, ProblemModel, + RecordDoc, RecordModel, SolutionModel, STATUS, SystemModel, Time, UserModel, +} from 'hydrooj'; + +const statusMap = { + 0: STATUS.STATUS_ACCEPTED, + 1: STATUS.STATUS_WRONG_ANSWER, + 2: STATUS.STATUS_TIME_LIMIT_EXCEEDED, + 3: STATUS.STATUS_MEMORY_LIMIT_EXCEEDED, + 4: STATUS.STATUS_WRONG_ANSWER, + 5: STATUS.STATUS_RUNTIME_ERROR, + 6: STATUS.STATUS_OUTPUT_LIMIT_EXCEEDED, + 7: STATUS.STATUS_COMPILE_ERROR, +}; +const langMap = { + 0: 'cc', + 1: 'c', + 2: 'pas', +}; +const nameMap: Record = { + 'sample.in': 'sample0.in', + 'sample.out': 'sample0.out', + 'test.in': 'test0.in', + 'test.out': 'test0.out', +}; + +export async function run({ + host = 'localhost', port = 3306, name, + username, password, domainId, contestType = 'oi', + dataDir, imageDir, rerun = true, randomMail = false, +}, report: Function) { + const src = await mariadb.createConnection({ + host, + port, + user: username, + password, + database: name, + // GBK + charset: 'GBK', + }); + const query = (q: string) => new Promise((res, rej) => { + src.query(q).then((r) => res(r)).catch((e) => rej(e)); + }); + const target = await DomainModel.get(domainId); + if (!target) throw new NotFoundError(domainId); + report({ message: 'Connected to database' }); + /* + user_id varchar 20 N 用户id（主键） + email varchar 100 Y 用户E-mail + submit int 11 Y 用户提交次数 + solved int 11 Y 成功次数 + defunct char 1 N 是否屏蔽（Y/N） + ip varchar 20 N 用户注册ip + accesstime datetime Y 用户注册时间 + volume int 11 N 页码（表示用户上次看到第几页） + language int 11 N 语言 + password varchar 32 Y 密码（加密） + reg_time datetime Y 用户注册时间 + nick varchar 100 N 昵称 + school varchar 100 N 用户所在学校 + */ + const uidMap: Record = {}; + const udocs = await query('SELECT *, DECODE(`password`, "PWDforJO2005") as `depass` FROM `users`'); + const precheck = await UserModel.getMulti({ unameLower: { $in: udocs.map((u) => u.user_id.toLowerCase()) } }).toArray(); + if (precheck.length) throw new Error(`Conflict username: ${precheck.map((u) => u.unameLower).join(', ')}`); + for (const udoc of udocs) { + if (randomMail) delete udoc.email; + let current = await UserModel.getByEmail(domainId, udoc.email || `${udoc.user_id}@poj.local`); + current ||= await UserModel.getByUname(domainId, udoc.user_id); + if (current) { + report({ message: `duplicate user with email ${udoc.email}: ${current.uname},${udoc.user_id}` }); + uidMap[udoc.user_id] = current._id; + } else { + const uid = await UserModel.create( + udoc.email || `${udoc.user_id}@poj.local`, udoc.user_id, '', + null, udoc.ip, udoc.defunct === 'Y' ? 0 : SystemModel.get('default.priv'), + ); + uidMap[udoc.user_id] = uid; + await UserModel.setById(uid, { + loginat: udoc.accesstime, + regat: udoc.reg_time, + school: udoc.school || '', + }); + await UserModel.setPassword(uid, udoc.depass); + await DomainModel.setUserInDomain(domainId, uid, { + displayName: udoc.nick || '', + school: udoc.school || '', + nSubmit: udoc.submit, + nAccept: 0, + }); + } + } + + const admins = await query("SELECT * FROM `privilege` WHERE `rightstr` = 'administrator'"); + for (const admin of admins) await DomainModel.setUserRole(domainId, uidMap[admin.user_id], 'root'); + const adminUids = admins.map((admin) => uidMap[admin.user_id]); + report({ message: 'user finished' }); + + /* + problem_id int 11 N 题目编号，主键 + title varchar 200 N 标题 + description text Y 题目描述 + inupt text Y 输入说明 + output text Y 输出说明 + input_path + output_path + sample_input text Y 输入参照 + sample_output text Y 输出参照 + sample_Program + hint text Y 暗示 + source varchar 100 Y 来源 + in_date datetime Y 加入时间 + time_limit int 11 N 限时（ms） + memory_limit int 11 N 空间限制(k) + defunct char 1 N 是否屏蔽（Y/N） + contest_id #ingore + accepted int 11 Y 总ac次数 + submit int 11 Y 总提交次数 + ratio + error + difficulty + submit_user + solved int 11 Y 解答（未用） + case_time_limit #ingore + */ + const pidMap: Record = {}; + const [{ 'count(*)': pcount }] = await query('SELECT count(*) FROM `problem`'); + const step = 50; + const pageCount = Math.ceil(Number(pcount) / step); + for (let pageId = 0; pageId < pageCount; pageId++) { + const pdocs = await query(`SELECT * FROM \`problem\` LIMIT ${pageId * step}, ${step}`); + for (const pdoc of pdocs) { + if (rerun) { + const opdoc = await ProblemModel.get(domainId, `P${pdoc.problem_id}`); + if (opdoc) pidMap[pdoc.problem_id] = opdoc.docId; + } + if (!pidMap[pdoc.problem_id]) { + const files = {}; + let content = buildContent({ + description: pdoc.description, + input: pdoc.input, + output: pdoc.output, + samples: [[pdoc.sample_input.trim(), pdoc.sample_output.trim()]], + hint: pdoc.hint, + source: pdoc.source, + }, 'html'); + const uploadFiles = content.matchAll(/(?:src|href)="\/images\/([^"]+\/([^"]+))"/g); + for (const file of uploadFiles) { + try { + files[file[2]] = await fs.readFile(path.join(imageDir, file[1])); + content = content.replace(`images/${file[1]}`, `file://${file[2]}`); + } catch (e) { + report({ message: `failed to read file: ${path.join(imageDir, file[1])}` }); + } + } + const pid = await ProblemModel.add( + domainId, `P${pdoc.problem_id}`, + pdoc.title, content, + 1, pdoc.source?.trim().length ? pdoc.source.split(' ').map((i) => i.trim()).filter((i) => i) : [], + { hidden: pdoc.defunct === 'Y' }, + ); + pidMap[pdoc.problem_id] = pid; + await Promise.all(Object.keys(files).map((filename) => ProblemModel.addAdditionalFile(domainId, pid, filename, files[filename]))); + if (Object.keys(files).length) report({ message: `move ${Object.keys(files).length} file for problem ${pid}` }); + } + await ProblemModel.edit(domainId, pidMap[pdoc.problem_id], { + nAccept: 0, + nSubmit: pdoc.submit, + config: `time: ${pdoc.time_limit}ms +memory: ${pdoc.memory_limit}k`, + owner: 1, + html: true, + }); + if (pdoc.sample_Program) { + await SolutionModel.add(domainId, pidMap[pdoc.problem_id], 1, pdoc.sample_Program); + } + } + } + report({ message: 'problem finished' }); + + /* + contest_id int 11 N 竞赛id（主键） + title varchar 255 Y 竞赛标题 + start_time datetime Y 开始时间(年月日时分) + end_time datatime Y 结束时间(年月日时分) + defunct char 1 N 是否屏蔽（Y/N） + description text Y 描述（在此版本中未用） + private tinyint 4 公开/内部（0/1） + */ + const tidMap: Record = {}; + const tdocs = await query('SELECT * FROM `contest`'); + for (const tdoc of tdocs) { + const pdocs = await query(`SELECT * FROM \`contest_problem\` WHERE \`contest_id\` = ${tdoc.contest_id} ORDER BY \`num\` ASC`); + const pids = pdocs.map((i) => pidMap[i.problem_id]).filter((i) => i); + const endAt = moment(tdoc.end_time).isSameOrBefore(tdoc.start_time) ? moment(tdoc.end_time).add(1, 'minute').toDate() : tdoc.end_time; + const tid = await ContestModel.add( + domainId, tdoc.title, tdoc.description || 'Description', + adminUids[0], contestType, tdoc.start_time, endAt, pids, true, + tdoc.private ? { _code: password } : {}, + ); + tidMap[tdoc.contest_id] = tid.toHexString(); + } + report({ message: 'contest finished' }); + /* + solution 程序运行结果记录 + 字段名 类型 长度 是否允许为空 备注 + solution_id int 11 N 运行id（主键） + problem_id int 11 N 问题id + user_id char 20 N 用户id + time int 11 N 用时（秒） + memory int 11 N 所用空间（） + className + in_date datetime N 加入时间 + result smallint 6 N 结果（0：AC） + language tinyint 4 N 语言 + ip char 15 N 用户ip + contest_id int 11 Y 所属于竞赛组 + valid tinyint 4 N 是否有效？？？ + num tinyint 4 N 题目在竞赛中的顺序号 + code_length int 11 N 代码长度 + */ + const [{ 'count(*)': rcount }] = await query('SELECT count(*) FROM `solution`'); + const rpageCount = Math.ceil(Number(rcount) / step); + for (let pageId = 0; pageId < rpageCount; pageId++) { + const rdocs = await query(`SELECT * FROM \`solution\` LIMIT ${pageId * step}, ${step}`); + for (const rdoc of rdocs) { + const data: RecordDoc = { + status: statusMap[rdoc.result] || 0, + _id: Time.getObjectID(rdoc.in_date, false), + uid: uidMap[rdoc.user_id] || 0, + code: "POJ didn't provide user code", + lang: langMap[rdoc.language || 0], + pid: pidMap[rdoc.problem_id] || 0, + domainId, + score: rdoc.result === 0 ? 100 : 0, + time: rdoc.time || 0, + memory: rdoc.memory || 0, + judgeTexts: [], + compilerTexts: [], + testCases: [], + judgeAt: new Date(), + rejudged: false, + judger: 1, + }; + const ceInfo = await query(`SELECT \`error\` FROM \`compileinfo\` WHERE \`solution_id\` = ${rdoc.solution_id}`); + if (ceInfo[0]?.error) data.judgeTexts.push(ceInfo[0].error); + const source = await query(`SELECT \`source\` FROM \`source_code\` WHERE \`solution_id\` = ${rdoc.solution_id}`); + // decompress mysql buffer, slice 4 then it can start at 78 9c (zlib header) + if (source[0]?.source) data.code = zlib.inflateSync(source[0].source.slice(4)).toString('utf-8'); + if (rdoc.contest_id) { + data.contest = new ObjectId(tidMap[rdoc.contest_id]); + await ContestModel.attend(domainId, data.contest, uidMap[rdoc.user_id]).catch(noop); + } + await RecordModel.coll.insertOne(data); + await postJudge(data).catch((err) => report({ message: err.message })); + } + } + report({ message: 'record finished' }); + + src.end(); + + if (!dataDir) return true; + if (dataDir.endsWith('/')) dataDir = dataDir.slice(0, -1); + const files = await fs.readdir(dataDir, { withFileTypes: true }); + for (const file of files) { + if (!file.isDirectory()) continue; + const datas = await fs.readdir(`${dataDir}/${file.name}`, { withFileTypes: true }); + const pdoc = await ProblemModel.get(domainId, `P${file.name}`, undefined, true); + if (!pdoc) continue; + report({ message: `Syncing testdata for ${file.name}` }); + for (const data of datas) { + if (data.isDirectory()) continue; + const filename = nameMap[data.name] || data.name; + const originalFile = fs.readFileSync(`${dataDir}/${file.name}/${data.name}`).toString(); + await ProblemModel.addTestdata(domainId, pdoc.docId, filename, originalFile.replace(/\r\n/g, '\n')); + } + await ProblemModel.addTestdata(domainId, pdoc.docId, 'config.yaml', Buffer.from(pdoc.config as string)); + } + return true; +}