From d20c405da95950da95f60c1920ac993930c1dc63 Mon Sep 17 00:00:00 2001 From: panda Date: Wed, 10 May 2023 19:25:05 +0800 Subject: [PATCH] migrate: add universaloj & hustoj upload file (#553) Co-authored-by: undefined --- packages/migrate/index.ts | 23 +- packages/migrate/package.json | 12 +- packages/migrate/scripts/hustoj.ts | 53 ++- packages/migrate/scripts/syzoj.ts | 6 +- packages/migrate/scripts/universaloj.ts | 545 ++++++++++++++++++++++++ 5 files changed, 617 insertions(+), 22 deletions(-) create mode 100644 packages/migrate/scripts/universaloj.ts diff --git a/packages/migrate/index.ts b/packages/migrate/index.ts index 04b2dab83..265f25ad3 100644 --- a/packages/migrate/index.ts +++ b/packages/migrate/index.ts @@ -1,3 +1,4 @@ +import crypto from 'crypto'; import { Context, md5, Schema, sha1, } from 'hydrooj'; @@ -16,15 +17,16 @@ export function apply(ctx: Context) { domainId: Schema.string().required(), contestType: Schema.string().required(), dataDir: Schema.string().required(), + uploadDir: Schema.string().default('/home/judge/src/web/upload/'), }), (...args) => require('./scripts/hustoj').run(...args), ); ctx.addScript( 'migrateSyzoj', 'migrate from syzoj', Schema.object({ - host: Schema.string().required(), - port: Schema.number().required(), - name: Schema.string().required(), + host: Schema.string().default('localhost'), + port: Schema.number().default(3306), + name: Schema.string().default('syzoj'), username: Schema.string().required(), password: Schema.string().required(), domainId: Schema.string().default('system'), @@ -43,6 +45,19 @@ export function apply(ctx: Context) { }), (...args) => require('./scripts/vijos').run(...args), ); + ctx.addScript( + 'migrateuniversaloj', 'migrate from universaloj', + Schema.object({ + host: Schema.string().default('172.17.0.2'), + port: Schema.number().default(3306), + name: Schema.string().default('app_uoj233'), + username: Schema.string().default('hydromigrate'), + password: Schema.string().required(), + domainId: Schema.string().default('system'), + dataDir: Schema.string().required(), + }), + (...args) => require('./scripts/universaloj').run(...args), + ); ctx.provideModule('hash', 'hust', ($password, $saved) => { $password = md5($password); @@ -62,10 +77,12 @@ export function apply(ctx: Context) { return `${Buffer.from(uname).toString('base64')}|${mixedSha1}`; }); ctx.provideModule('hash', 'syzoj', (password: string) => md5(`${password}syzoj2_xxx`)); + ctx.provideModule('hash', 'uoj', (password, salt, { uname }) => md5(`${uname}${crypto.createHmac('md5', salt).update(password).digest('hex')}`)); ctx.i18n.load('zh', { 'migrate from hustoj': '从 HustOJ 导入', 'migrate from vijos': '从 Vijos 导入', 'migrate from syzoj': '从 SYZOJ 导入', + 'migrate from universaloj': '从 UniversalOJ 导入', }); } diff --git a/packages/migrate/package.json b/packages/migrate/package.json index c3293bdfc..f16d5f4dd 100644 --- a/packages/migrate/package.json +++ b/packages/migrate/package.json @@ -5,12 +5,14 @@ "repository": "git@github.com:hydro-dev/Hydro.git", "author": "undefined ", "license": "AGPL-3.0-or-later", - "devDependencies": { - "@types/turndown": "^5.0.1" - }, "dependencies": { "mariadb": "^3.1.1", - "mongodb": "^5.3.0", - "turndown": "^7.1.2" + "mongodb": "^5.2.0", + "turndown": "^7.1.2", + "xml2js": "^0.4.23" + }, + "devDependencies": { + "@types/turndown": "^5.0.1", + "@types/xml2js": "^0.4.11" } } diff --git a/packages/migrate/scripts/hustoj.ts b/packages/migrate/scripts/hustoj.ts index ce8bf3a90..684b107f4 100644 --- a/packages/migrate/scripts/hustoj.ts +++ b/packages/migrate/scripts/hustoj.ts @@ -1,10 +1,11 @@ /* eslint-disable no-tabs */ /* eslint-disable no-await-in-loop */ +import path from 'path'; import mariadb from 'mariadb'; import TurndownService from 'turndown'; import { - buildContent, ContestModel, DomainModel, fs, noop, NotFoundError, ObjectId, postJudge, ProblemModel, - RecordDoc, RecordModel, SolutionModel, STATUS, SystemModel, Time, UserModel, + _, buildContent, ContestModel, DomainModel, fs, noop, NotFoundError, ObjectId, postJudge, ProblemModel, + RecordDoc, RecordModel, SolutionModel, STATUS, StorageModel, SystemModel, Time, UserModel, } from 'hydrooj'; const turndown = new TurndownService({ @@ -48,10 +49,21 @@ const nameMap: Record = { 'test.in': 'test0.in', 'test.out': 'test0.out', }; + +async function addContestFile(domainId: string, tid: ObjectId, filename: string, filepath: string) { + const tdoc = await ContestModel.get(domainId, tid); + await StorageModel.put(`contest/${domainId}/${tid}/${filename}`, filepath, 1); + const meta = await StorageModel.getMeta(`contest/${domainId}/${tid}/${filename}`); + const payload = { _id: filename, name: filename, ..._.pick(meta, ['size', 'lastModified', 'etag']) }; + if (!meta) return false; + await ContestModel.edit(domainId, tid, { files: [...(tdoc.files || []), payload] }); + return true; +} + export async function run({ host = 'localhost', port = 3306, name = 'jol', username, password, domainId, contestType = 'oi', - dataDir, rerun = true, randomMail = false, + dataDir, uploadDir = '/home/judge/src/web/upload/', rerun = true, randomMail = false, }, report: Function) { const src = await mariadb.createConnection({ host, @@ -153,19 +165,28 @@ export async function run({ if (opdoc) pidMap[pdoc.problem_id] = opdoc.docId; } if (!pidMap[pdoc.problem_id]) { + const files = {}; + let content = buildContent({ + description: pdoc.description, + input: pdoc.input, + output: pdoc.output, + samples: [[pdoc.sample_input.trim(), pdoc.sample_output.trim()]], + hint: pdoc.hint, + source: pdoc.source, + }, 'html'); + const uploadFiles = content.matchAll(/(?:src|href)="\/upload\/([^"]+\/([^"]+))"/g); + for (const file of uploadFiles) { + files[file[2]] = await fs.readFile(path.join(uploadDir, file[1])); + content = content.replace(`/upload/${file[1]}`, `file://${file[2]}`); + } const pid = await ProblemModel.add( domainId, `P${pdoc.problem_id}`, - pdoc.title, buildContent({ - description: pdoc.description, - input: pdoc.input, - output: pdoc.output, - samples: [[pdoc.sample_input.trim(), pdoc.sample_output.trim()]], - hint: pdoc.hint, - source: pdoc.source, - }, 'html'), + pdoc.title, content, 1, pdoc.source.split(' ').map((i) => i.trim()).filter((i) => i), pdoc.defunct === 'Y', ); pidMap[pdoc.problem_id] = pid; + await Promise.all(Object.keys(files).map((filename) => ProblemModel.addAdditionalFile(domainId, pid, filename, files[filename]))); + if (Object.keys(files).length) report({ message: `move ${Object.keys(files).length} file for problem ${pid}` }); } const cdoc = await query(`SELECT * FROM \`privilege\` WHERE \`rightstr\` = 'p${pdoc.problem_id}'`); const maintainer = []; @@ -203,15 +224,23 @@ export async function run({ for (const tdoc of tdocs) { const pdocs = await query(`SELECT * FROM \`contest_problem\` WHERE \`contest_id\` = ${tdoc.contest_id}`); const pids = pdocs.map((i) => pidMap[i.problem_id]).filter((i) => i); + const files = {}; + let description = tdoc.description; + const uploadFiles = description.matchAll(/(?:src|href)="\/upload\/([^"]+\/([^"]+))"/g); + for (const file of uploadFiles) { + files[file[2]] = await fs.readFile(path.join(uploadDir, file[1])); + description = description.replace(`/upload/${file[1]}`, `file://${file[2]}`); + } const tid = await ContestModel.add( domainId, tdoc.title, tdoc.description || 'Description', adminUids[0], contestType, tdoc.start_time, tdoc.end_time, pids, true, { _code: password }, ); tidMap[tdoc.contest_id] = tid.toHexString(); + await Promise.all(Object.keys(files).map((filename) => addContestFile(domainId, tid, filename, files[filename]))); + if (Object.keys(files).length) report({ message: `move ${Object.keys(files).length} file for contest ${tidMap[tdoc.contest_id]}` }); } report({ message: 'contest finished' }); - /* solution 程序运行结果记录 字段名 类型 长度 是否允许为空 备注 diff --git a/packages/migrate/scripts/syzoj.ts b/packages/migrate/scripts/syzoj.ts index dc82c9d98..12a4548ee 100644 --- a/packages/migrate/scripts/syzoj.ts +++ b/packages/migrate/scripts/syzoj.ts @@ -125,8 +125,10 @@ export async function run({ udoc.email || `${udoc.username}@syzoj.local`, udoc.username, '', null, udoc.ip, SystemModel.get('default.priv'), ); - if (udoc.is_admin) await UserModel.setSuperAdmin(uid); - superAdmin.push(uid); + if (udoc.is_admin) { + await UserModel.setSuperAdmin(uid); + superAdmin.push(uid); + } uidMap[udoc.id] = uid; await UserModel.setById(uid, { regat: new Date(udoc.register_time * 1000), diff --git a/packages/migrate/scripts/universaloj.ts b/packages/migrate/scripts/universaloj.ts new file mode 100644 index 000000000..32045e0d6 --- /dev/null +++ b/packages/migrate/scripts/universaloj.ts @@ -0,0 +1,545 @@ +/* eslint-disable no-await-in-loop */ +import mariadb from 'mariadb'; +import xml2js from 'xml2js'; +import { + AdmZip, ContestModel, DomainModel, fs, MessageModel, moment, + noop, NotFoundError, ObjectId, postJudge, ProblemConfigFile, ProblemModel, RecordDoc, RecordModel, + STATUS, SubtaskType, SystemModel, Time, UserModel, ValidationError, yaml, +} from 'hydrooj'; +const statusMap = { + Accepted: STATUS.STATUS_ACCEPTED, + 'Compile Error': STATUS.STATUS_COMPILE_ERROR, + 'Wrong Answer': STATUS.STATUS_WRONG_ANSWER, + 'Runtime Error': STATUS.STATUS_RUNTIME_ERROR, + 'Memory Limit Exceeded': STATUS.STATUS_MEMORY_LIMIT_EXCEEDED, + 'Time Limit Exceeded': STATUS.STATUS_TIME_LIMIT_EXCEEDED, + 'Output Limit Exceeded': STATUS.STATUS_OUTPUT_LIMIT_EXCEEDED, + 'Dangerous Syscalls': STATUS.STATUS_RUNTIME_ERROR, + 'Judgement Failed': STATUS.STATUS_SYSTEM_ERROR, + 'No Comment': STATUS.STATUS_SYSTEM_ERROR, + Skippped: STATUS.STATUS_CANCELED, + Judging: STATUS.STATUS_JUDGING, +}; +const sexMap = { + U: 3, + M: 1, + F: 2, +}; +const langMap = { + C: 'c', + 'C++': 'cc.cc98', + 'C++11': 'cc.cc11', + Java8: 'java', + Java11: 'java', + Pascal: 'pas', + Python2: 'py.py2', + Python3: 'py.py3', +}; +function handleMailLower(mail: string) { + let data = mail.trim().toLowerCase(); + if (data.endsWith('@googlemail.com')) data = data.replace('@googlemail.com', '@gmail.com'); + if (data.endsWith('@gmail.com')) { + const [prev] = data.split('@'); + data = `${prev.replace(/[.+]/g, '')}@gmail.com`; + } + return data; +} + +export async function run({ + host = '172.17.0.2', port = 3306, name = 'app_uoj233', + username, password, domainId, dataDir, + rerun = true, randomMail = false, +}, report: Function) { + const src = await mariadb.createConnection({ + host, + port, + user: username, + password, + database: name, + }); + const query = (q: string) => new Promise((res, rej) => { + src.query(q).then((r) => res(r)).catch((e) => rej(e)); + }); + const target = await DomainModel.get(domainId); + if (!target) throw new NotFoundError(domainId); + report({ message: 'Connected to database' }); + /* + CREATE TABLE `user_info` ( + `usergroup` char(1) NOT NULL DEFAULT 'U', + `username` varchar(20) NOT NULL, + `email` varchar(50) NOT NULL, + `password` char(32) NOT NULL, + `svn_password` char(10) NOT NULL, + `rating` int(11) NOT NULL DEFAULT '1500', + `qq` bigint(20) NOT NULL, + `sex` char(1) NOT NULL DEFAULT 'U', + `ac_num` int(11) NOT NULL, + `register_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `remote_addr` varchar(50) NOT NULL, + `http_x_forwarded_for` varchar(50) NOT NULL, + `remember_token` char(60) NOT NULL, + `motto` varchar(200) NOT NULL, + `cellphone` varchar(15) NOT NULL, + PRIMARY KEY (`username`), + KEY `rating` (`rating`,`username`), + KEY `ac_num` (`ac_num`,`username`) + ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; + */ + const configFile = await fs.readFile(`${dataDir}/opt/uoj/web/app/.config.php`, 'utf8'); + const salt = configFile.match(/'client_salt' => '(.+?)'/)[1]; + const uidMap: Record = {}; + const udocs = await query('SELECT * FROM `user_info`'); + const priv = await SystemModel.get('default.priv'); + report({ message: udocs.map((u) => u.username.toLowerCase()) }); + const precheck = await UserModel.getMulti({ unameLower: { $in: udocs.map((u) => u.username.toLowerCase()) } }).toArray(); + if (precheck.length) throw new Error(`Conflict username: ${precheck.map((u) => u.unameLower).join(', ')}`); + for (const udoc of udocs) { + if (randomMail) delete udoc.email; + let current = await UserModel.getByEmail(domainId, udoc.email || `${udoc.username}@universaloj.local`); + current ||= await UserModel.getByUname(domainId, udoc.username); + if (current) { + report({ message: `duplicate user with email ${udoc.email}: ${current.uname},${udoc.username}` }); + uidMap[udoc.username] = current._id; + } else { + const [u] = await UserModel.coll.find({}).sort({ _id: -1 }).limit(1).toArray(); + const uid = Math.max((u?._id || 0) + 1, 2); + await UserModel.coll.insertOne({ + _id: uid, + uname: udoc.username, + unameLower: udoc.username.toLowerCase(), + mail: udoc.email || `${udoc.username}@universaloj.local`, + mailLower: handleMailLower(udoc.email || `${udoc.username}@universaloj.local`), + regat: new Date(udoc.register_time), + hash: udoc.password, + salt, + hashType: 'uoj', + ip: [udoc.http_x_forwarded_for || udoc.remote_addr || '127.0.0.1'], + loginat: new Date(), + loginip: '127.0.0.1', + priv, + avatar: `gravatar:${udoc.email || `${udoc.username}@universaloj.local`}`, + bio: udoc.motto || '', + gender: sexMap[udoc.sex] || 3, + qq: udoc.qq.toString() || null, + phone: udoc.cellphone || null, + }); + if (udoc.usergroup === 'S') await UserModel.setSuperAdmin(uid); + uidMap[udoc.username] = uid; + await DomainModel.setUserInDomain(domainId, uid, { + displayName: udoc.nickname || '', + nAccept: udoc.ac_num, + }); + } + } + report({ message: 'user finished' }); + + /* + CREATE TABLE `user_msg` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `sender` varchar(20) NOT NULL, + `receiver` varchar(20) NOT NULL, + `message` varchar(5000) NOT NULL, + `send_time` datetime NOT NULL, + `read_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=MyISAM AUTO_INCREMENT=8299 DEFAULT CHARSET=utf8mb4; + + CREATE TABLE `user_system_msg` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `title` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL, + `content` varchar(300) COLLATE utf8mb4_unicode_ci NOT NULL, + `receiver` varchar(20) COLLATE utf8mb4_unicode_ci NOT NULL, + `send_time` datetime NOT NULL, + `read_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB AUTO_INCREMENT=16814 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + */ + const messages = await query('SELECT * FROM `user_msg`'); + await Promise.all(messages.map((msg) => MessageModel.coll.insertOne({ + _id: Time.getObjectID(new Date(msg.send_time), false), + from: uidMap[msg.sender], + to: uidMap[msg.receiver], + content: msg.message, + flag: 0, + }))); + const systemMessages = await query('SELECT * FROM `user_system_msg`'); + await Promise.all(systemMessages.map((msg) => MessageModel.coll.insertOne({ + _id: Time.getObjectID(new Date(msg.send_time), false), + from: 1, + to: uidMap[msg.receiver], + content: msg.content, + flag: 0, + }))); + report({ message: 'message finished' }); + + /* + CREATE TABLE `problems` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `title` text NOT NULL, + `is_hidden` tinyint(1) NOT NULL DEFAULT '0', + `submission_requirement` text, + `hackable` tinyint(1) NOT NULL DEFAULT '0', + `extra_config` varchar(500) NOT NULL DEFAULT '{"view_content_type":"ALL_AFTER_AC","view_details_type":"SELF","view_all_details_type":"SELF"}', + `zan` int(11) NOT NULL, + `ac_num` int(11) NOT NULL DEFAULT '0', + `submit_num` int(11) NOT NULL DEFAULT '0', + `difficulty` int(11) NOT NULL DEFAULT '9999', + PRIMARY KEY (`id`) + ) ENGINE=MyISAM AUTO_INCREMENT=1545 DEFAULT CHARSET=utf8mb4; + + CREATE TABLE `problems_contents` ( + `id` int(11) NOT NULL, + `statement` mediumtext NOT NULL, + `statement_md` mediumtext NOT NULL, + PRIMARY KEY (`id`) + ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; + + CREATE TABLE `problems_permissions` ( + `username` varchar(20) NOT NULL, + `problem_id` int(11) NOT NULL, + PRIMARY KEY (`username`,`problem_id`) + ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; + + CREATE TABLE `problems_tags` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `problem_id` int(11) NOT NULL, + `tag` varchar(30) NOT NULL, + PRIMARY KEY (`id`), + KEY `problem_id` (`problem_id`), + KEY `tag` (`tag`) + ) ENGINE=MyISAM AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4; + */ + const pidMap: Record = {}; + const [{ 'count(*)': pcount }] = await query('SELECT count(*) FROM `problems`'); + const step = 50; + const pageCount = Math.ceil(Number(pcount) / step); + for (let pageId = 0; pageId < pageCount; pageId++) { + const pdocs = await query(`SELECT * FROM \`problems\` LIMIT ${pageId * step}, ${step}`); + for (const pdoc of pdocs) { + if (rerun) { + const opdoc = await ProblemModel.get(domainId, `P${pdoc.id}`); + if (opdoc) pidMap[pdoc.id] = opdoc.docId; + } + if (!pidMap[pdoc.id]) { + const content = await query(`SELECT * FROM \`problems_contents\` WHERE \`id\` = ${pdoc.id}`); + const pid = await ProblemModel.add(domainId, `P${pdoc.id}`, pdoc.title, content[0].statement_md || '', 1); + pidMap[pdoc.id] = pid; + } + const [permissions, tags] = await Promise.all([ + query(`SELECT * FROM \`problems_permissions\` WHERE \`problem_id\` = ${pdoc.id}`), + query(`SELECT * FROM \`problems_tags\` WHERE \`problem_id\` = ${pdoc.id}`), + ]); + const maintainer = permissions.map((p) => uidMap[p.username]).slice(1); + await ProblemModel.edit(domainId, pidMap[pdoc.id], { + nAccept: pdoc.ac_num || 0, + nSubmit: pdoc.submit_num || 0, + hidden: !!pdoc.is_hidden, + tag: tags.map((t) => t.tag), + owner: uidMap[permissions[0]?.username] || 1, + maintainer, + }); + } + console.log({ message: `Synced ${pageId * step} / ${pcount} problems` }); + } + report({ message: 'problem finished' }); + + /* + CREATE TABLE `contests` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(50) NOT NULL, + `start_time` datetime NOT NULL, + `last_min` int(11) NOT NULL, + `player_num` int(11) NOT NULL, + `status` varchar(50) NOT NULL, + `extra_config` varchar(200) NOT NULL, + `zan` int(11) NOT NULL, + PRIMARY KEY (`id`) + ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; + + CREATE TABLE `contests_asks` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `contest_id` int(11) NOT NULL, + `username` varchar(20) NOT NULL, + `question` text NOT NULL, + `answer` text NOT NULL, + `post_time` datetime NOT NULL, + `reply_time` datetime NOT NULL, + `is_hidden` tinyint(1) DEFAULT '0', + PRIMARY KEY (`id`) + ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; + + CREATE TABLE `contests_notice` ( + `contest_id` int(11) NOT NULL, + `title` varchar(30) NOT NULL, + `content` varchar(500) NOT NULL, + `time` datetime NOT NULL, + KEY `contest_id` (`contest_id`) + ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; + + CREATE TABLE `contests_permissions` ( + `username` varchar(20) NOT NULL, + `contest_id` int(11) NOT NULL, + PRIMARY KEY (`username`,`contest_id`) + ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; + + CREATE TABLE `contests_problems` ( + `problem_id` int(11) NOT NULL, + `contest_id` int(11) NOT NULL, + PRIMARY KEY (`problem_id`,`contest_id`) + ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; + + CREATE TABLE `contests_registrants` ( + `username` varchar(20) NOT NULL, + `user_rating` int(11) NOT NULL, + `contest_id` int(11) NOT NULL, + `has_participated` tinyint(1) NOT NULL, + `rank` int(11) NOT NULL, + PRIMARY KEY (`contest_id`,`username`) + ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; + + CREATE TABLE `contests_submissions` ( + `contest_id` int(11) NOT NULL, + `submitter` varchar(20) NOT NULL, + `problem_id` int(11) NOT NULL, + `submission_id` int(11) NOT NULL, + `score` int(11) NOT NULL, + `penalty` int(11) NOT NULL, + PRIMARY KEY (`contest_id`,`submitter`,`problem_id`) + ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; + */ + + const tidMap: Record = {}; + const tdocs = await query('SELECT * FROM `contests`'); + for (const tdoc of tdocs) { + const [permissions, problems, notices] = await Promise.all([ + query(`SELECT * FROM \`contests_permissions\` WHERE \`contest_id\` = ${tdoc.id}`), + query(`SELECT * FROM \`contests_problems\` WHERE \`contest_id\` = ${tdoc.id} ORDER BY \`problem_rank\` ASC`), + // query(`SELECT * FROM \`contests_asks\` WHERE \`contest_id\` = ${tdoc.id}`), + query(`SELECT * FROM \`contests_notice\` WHERE \`contest_id\` = ${tdoc.id}`), + ]); + let content = ''; + for (const notice of notices) { + content += `## Notice: ${notice.title}\n${notice.content}\n${moment(notice.start_time).format('YYYY-MM-DD HH:mm:ss')}\n`; + } + const pids = problems.map((p) => pidMap[p.problem_id]); + const maintainer = permissions.map((p) => uidMap[p.username]).slice(1); + const info = JSON.parse(tdoc.extra_config) || {}; + const startAt = moment(tdoc.start_time); + const endAt = startAt.clone().add(tdoc.last_min, 'minutes'); + const tid = await ContestModel.add( + domainId, tdoc.name, content, uidMap[permissions[0]?.username] || 1, info.contest_type?.toLowerCase() || 'oi', + startAt.toDate(), endAt.toDate(), pids, !Object.keys(info).includes('unrated'), { maintainer }, + ); + tidMap[tdoc.id] = tid.toHexString(); + } + report({ message: 'contest finished' }); + + /* + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `problem_id` int(10) unsigned NOT NULL, + `contest_id` int(10) unsigned DEFAULT NULL, + `submit_time` datetime NOT NULL, + `submitter` varchar(20) NOT NULL, + `content` text NOT NULL, + `language` varchar(15) NOT NULL, + `tot_size` int(11) NOT NULL, + `judge_time` datetime DEFAULT NULL, + `result` blob NOT NULL, + `status` varchar(20) NOT NULL, + `result_error` varchar(20) DEFAULT NULL, + `score` int(11) DEFAULT NULL, + `used_time` int(11) NOT NULL DEFAULT '0', + `used_memory` int(11) NOT NULL DEFAULT '0', + `is_hidden` tinyint(1) NOT NULL, + `status_details` varchar(100) NOT NULL, + `contest_penalty` int(11) DEFAULT NULL, + */ + if (dataDir.endsWith('/')) dataDir = dataDir.slice(0, -1); + const [{ 'count(*)': rcount }] = await query('SELECT count(*) FROM `submissions`'); + const rpageCount = Math.ceil(Number(rcount) / step); + for (let pageId = 0; pageId < rpageCount; pageId++) { + const rdocs = await query(`SELECT * FROM \`submissions\` LIMIT ${pageId * step}, ${step}`); + for (const rdoc of rdocs) { + const data: RecordDoc = { + status: statusMap[rdoc.result_error] || STATUS.STATUS_WAITING, + _id: Time.getObjectID(new Date(rdoc.submit_time), false), + uid: uidMap[rdoc.submitter] || 1, + code: '', + lang: langMap[rdoc.language] || '', + pid: pidMap[rdoc.problem_id] || 0, + domainId, + score: rdoc.score || 0, + time: rdoc.used_time || 0, + memory: rdoc.used_memory || 0, + judgeTexts: [], + compilerTexts: [], + testCases: [], + judgeAt: new Date(rdoc.judge_time), + rejudged: false, + judger: 1, + }; + const content = JSON.parse(rdoc.content); + try { + let zip: AdmZip; + try { + zip = new AdmZip(await fs.readFile(`${dataDir}/opt/uoj/web/app/storage${content.file_name}`)); + } catch (e) { + throw new ValidationError('zip', null, e.message); + } + data.code = zip.getEntries().find((i) => i.name.endsWith('answer.code')).getData().toString(); + } catch { /* ignore no code */ } + const result = JSON.parse(Buffer.from(rdoc.result, 'base64').toString('utf8')); + if (result.error) { + if (data.status === STATUS.STATUS_COMPILE_ERROR) data.compilerTexts.push(result.error); + else data.judgeTexts.push(result.error); + } else { + // TODO final_result + if (!result.details) continue; + try { + const details = await xml2js.parseStringPromise(result.details); + if (details.tests.subtask) { + details.tests.subtask.forEach((subtask) => { + if (!subtask.test) { + data.testCases.push({ + subtaskId: subtask.$.num, + id: 1, + score: 0, + time: 0, + memory: 0, + message: 'Skipped', + status: STATUS.STATUS_CANCELED, + }); + return; + } + data.testCases.push(...subtask.test.map((curCase, caseIndex) => ({ + subtaskId: subtask.$.num, + id: caseIndex + 1, + score: curCase.$.score, + time: curCase.$.time === '-1' ? 0 : curCase.time, + memory: curCase.$.memory === '-1' ? 0 : curCase.memory, + message: curCase.res[0] || '', + status: statusMap[curCase.$.info] || STATUS.STATUS_WAITING, + }))); + }); + } else if (details.tests.test) { + data.testCases.push(...details.tests.test.map((curCase) => ({ + subtaskId: 1, + id: curCase.$.num, + score: curCase.$.score, + time: curCase.$.time === '-1' ? 0 : curCase.time, + memory: curCase.$.memory === '-1' ? 0 : curCase.memory, + message: curCase.res[0] || '', + status: statusMap[curCase.$.info] || STATUS.STATUS_WAITING, + }))); + } + data.status = Math.max(...data.testCases.map((x) => x.status)); + } catch (e) { console.log(rdoc.id, result); } + } + if (rdoc.contest_id) { + data.contest = new ObjectId(tidMap[rdoc.contest_id]); + await ContestModel.attend(domainId, data.contest, uidMap[rdoc.submitter]).catch(noop); + } + await RecordModel.coll.insertOne(data); + await postJudge(data).catch((err) => report({ message: err.message })); + } + console.log({ message: `Synced ${pageId * step} / ${rcount} records` }); + } + report({ message: 'record finished' }); + + // TODO: blog + + src.end(); + + const files = await fs.readdir(`${dataDir}/var/uoj_data/`, { withFileTypes: true }); + for (const file of files) { + if (!file.isDirectory()) continue; + const datas = await fs.readdir(`${dataDir}/var/uoj_data/${file.name}`, { withFileTypes: true }); + const pdoc = await ProblemModel.get(domainId, `P${file.name}`, undefined, true); + if (!pdoc) continue; + report({ message: `Syncing testdata for ${file.name}` }); + const filenames = datas.map((i) => i.name); + for (const data of datas) { + if (data.isDirectory()) continue; + await ProblemModel.addTestdata(domainId, pdoc.docId, data.name, `${dataDir}/var/uoj_data/${file.name}/${data.name}`); + if (data.name === 'problem.conf') { + const confInfo: any = { + subtask_end: {}, + subtask_score: {}, + point: {}, + }; + const conf = await fs.readFile(`${dataDir}/var/uoj_data/${file.name}/${data.name}`, 'utf8'); + const config: ProblemConfigFile = { + subtasks: [], + }; + const lines = conf.replace(/\r/g, '').split('\n').map((i) => i.trim()).filter((i) => i); + for (const line of lines) { + const [key, value] = line.split(' '); + if (key === 'use_builtin_checker') { + config.checker = value; + config.checker_type = 'testlib'; + } else if (key === 'time_limit') { + config.time = `${value}s`; + } else if (key === 'memory_limit') { + config.memory = `${value}mb`; + } else if (key.startsWith('subtask_end_')) { + confInfo.subtask_end[+key.slice(12)] = +value; + } else if (key.startsWith('subtask_score_')) { + confInfo.subtask_score[+key.slice(14)] = +value; + } else if (key.startsWith('point_score_')) { + confInfo.point[+key.slice(12)] = +value; + } else confInfo[key] = value; + } + if (!config.checker && filenames.includes('chk.cpp')) { + config.checker_type = 'testlib'; + config.checker = 'chk.cpp'; + } + if (filenames.includes('val.cpp')) config.validator = 'val.cpp'; + if (confInfo.n_tests && Object.keys(confInfo.point).length === +confInfo.n_tests) { + config.subtasks.push(...Object.keys(confInfo.point).map((i) => ({ + id: +i, + score: confInfo.point[i], + cases: [{ + input: `${confInfo.input_pre}${i}.${confInfo.input_suf}`, + output: `${confInfo.output_pre}${i}.${confInfo.output_suf}`, + }], + }))); + } + if (confInfo.n_subtasks + && Object.keys(confInfo.subtask_end).length === +confInfo.n_subtasks + && Object.keys(confInfo.subtask_score).length === +confInfo.n_subtasks) { + config.subtasks.push(...[...new Array(+confInfo.n_subtasks)].map((v, i) => i + 1).map((i) => ({ + id: +i, + score: confInfo.subtask_score[i], + cases: [...new Array(confInfo.subtask_end[i] - (confInfo.subtask_end[i - 1] || 0))].map((v, j) => ({ + input: `${confInfo.input_pre}${j + (confInfo.subtask_end[i - 1] || 0) + 1}.${confInfo.input_suf}`, + output: `${confInfo.output_pre}${j + (confInfo.subtask_end[i - 1] || 0) + 1}.${confInfo.output_suf}`, + })), + }))); + } + if (+confInfo.n_ex_tests) { + if (!config.subtasks.length) { + config.subtasks.push({ + id: 1, + score: 97, + type: 'sum' as SubtaskType, + cases: [...new Array(+confInfo.n_tests)].map((v, i) => i + 1).map((i) => ({ + input: `${confInfo.input_pre}${i}.${confInfo.input_suf}`, + output: `${confInfo.output_pre}${i}.${confInfo.output_suf}`, + })), + }); + } + config.subtasks.push({ + id: Math.max(...config.subtasks.map((i) => i.id)) + 1, + score: 3, + cases: [...new Array(+confInfo.n_ex_tests)].map((v, i) => i + 1).map((i) => ({ + input: `ex_${confInfo.input_pre}${i}.${confInfo.input_suf}`, + output: `ex_${confInfo.output_pre}${i}.${confInfo.output_suf}`, + })), + }); + } + await ProblemModel.addTestdata(domainId, pdoc.docId, 'config.yaml', Buffer.from(yaml.dump(config))); + } + } + } + return true; +}