mirror of
https://gitcode.com/gh_mirrors/hy/Hydro.git
synced 2026-10-20 01:38:19 +08:00
w: merge migration packages
This commit is contained in:
undefined committed
1 parent
b5e312b609
commit
74f4e44f0f
12 files changed
+28
-64
No files matched your search
@@ -0,0 +1,21 @@
|
||||
# migrate
|
||||
|
||||
|
||||
这个模块用于从其他系统批量导入数据。
|
||||
迁移过程中请保持网络畅通。
|
||||
|
||||
## migrate-vijos
|
||||
|
||||
从Vijos4.0的数据库导入数据。
|
||||
迁移前，您需要指定数据来源。
|
||||
**请不要**将 Hydro 正在使用的数据库设置为数据来源的数据库。
|
||||
迁移过程中，Hydro 的以下内容会被**清空**：
|
||||
题目列表，提交记录，用户列表，比赛列表，比赛成绩表，训练列表，训练进度，站内消息，题解，讨论
|
||||
以下内容将被迁移：
|
||||
题目与测试数据、题解，讨论与回复，比赛、训练、作业相关数据，提交记录，用户列表，站内消息。
|
||||
|
||||
## migrate-hustoj
|
||||
|
||||
从HustOJ的数据库导入数据。
|
||||
迁移过程中，Hydro 的内容会被**清空**，以下内容将被迁移：
|
||||
题目与测试数据，用户列表。
|
||||
@@ -0,0 +1,23 @@
|
||||
import { md5, sha1 } from 'hydrooj/src/lib/crypto';
|
||||
|
||||
const RE_MD5 = /^[\da-f]{32}$/;
|
||||
|
||||
global.Hydro.lib['hash.hust'] = ($password: string, $saved: string) => {
|
||||
$password = md5($password);
|
||||
if (RE_MD5.test($saved)) return $password === $saved;
|
||||
const $svd = Buffer.from($saved, 'base64').toString('hex');
|
||||
const $salt = Buffer.from($svd.substr(40), 'hex').toString();
|
||||
const $hash = Buffer.concat([
|
||||
Buffer.from(sha1($password + $salt), 'hex'),
|
||||
Buffer.from($salt),
|
||||
]).toString('base64');
|
||||
if ($hash.trim() === $saved.trim()) return true;
|
||||
return false;
|
||||
};
|
||||
|
||||
global.Hydro.lib['hash.vj2'] = (password: string, salt: string, udoc: { uname: string; unameLower: string }) => {
|
||||
const { uname, unameLower } = udoc;
|
||||
const pmd5 = md5(password);
|
||||
const mixedSha1 = sha1(md5(unameLower + pmd5) + salt + sha1(pmd5 + salt));
|
||||
return `${Buffer.from(uname).toString('base64')}|${mixedSha1}`;
|
||||
};
|
||||
@@ -0,0 +1,2 @@
|
||||
migrate from hustoj: 从 HustOJ 导入
|
||||
migrate from vijos: 从 Vijos 导入
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "@hydrooj/migrate",
|
||||
"version": "0.1.0",
|
||||
"main": "package.json",
|
||||
"repository": "git@github.com:hydro-dev/Hydro.git",
|
||||
"author": "undefined <i@undefined.moe>",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"devDependencies": {
|
||||
"@types/mongodb": "^3.6.20",
|
||||
"@types/mysql": "^2.15.21"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hydrooj/utils": "workspace:*",
|
||||
"mongodb": "^3.7.3",
|
||||
"mysql": "^2.18.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
import './scripts/hustoj';
|
||||
import './scripts/vijos';
|
||||
@@ -0,0 +1,305 @@
|
||||
/* eslint-disable no-tabs */
|
||||
/* eslint-disable no-await-in-loop */
|
||||
import fs from 'fs-extra';
|
||||
import { ObjectID } from 'mongodb';
|
||||
import mysql from 'mysql';
|
||||
import { STATUS } from '@hydrooj/utils/lib/status';
|
||||
import { noop, Time } from '@hydrooj/utils/lib/utils';
|
||||
import { NotFoundError } from 'hydrooj/src/error';
|
||||
import { postJudge } from 'hydrooj/src/handler/judge';
|
||||
import { RecordDoc } from 'hydrooj/src/interface';
|
||||
import { buildContent } from 'hydrooj/src/lib/content';
|
||||
import * as contest from 'hydrooj/src/model/contest';
|
||||
import domain from 'hydrooj/src/model/domain';
|
||||
import problem from 'hydrooj/src/model/problem';
|
||||
import record from 'hydrooj/src/model/record';
|
||||
import * as system from 'hydrooj/src/model/system';
|
||||
import user from 'hydrooj/src/model/user';
|
||||
|
||||
const statusMap = {
|
||||
4: STATUS.STATUS_ACCEPTED,
|
||||
5: STATUS.STATUS_WRONG_ANSWER,
|
||||
6: STATUS.STATUS_WRONG_ANSWER,
|
||||
7: STATUS.STATUS_TIME_LIMIT_EXCEEDED,
|
||||
8: STATUS.STATUS_MEMORY_LIMIT_EXCEEDED,
|
||||
9: STATUS.STATUS_OUTPUT_LIMIT_EXCEEDED,
|
||||
10: STATUS.STATUS_RUNTIME_ERROR,
|
||||
11: STATUS.STATUS_COMPILE_ERROR,
|
||||
};
|
||||
const langMap = {
|
||||
0: 'c',
|
||||
1: 'cc',
|
||||
2: 'pas',
|
||||
3: 'java',
|
||||
4: 'rb',
|
||||
5: 'bash',
|
||||
6: 'py',
|
||||
7: 'php',
|
||||
8: 'perl',
|
||||
9: 'cs',
|
||||
10: 'oc',
|
||||
11: 'fb',
|
||||
12: 'sc',
|
||||
13: 'cl',
|
||||
14: 'cl++',
|
||||
15: 'lua',
|
||||
16: 'js',
|
||||
17: 'go',
|
||||
};
|
||||
const nameMap = {
|
||||
'sample.in': 'sample0.in',
|
||||
'sample.out': 'sample0.out',
|
||||
'test.in': 'test0.in',
|
||||
'test.out': 'test0.out',
|
||||
};
|
||||
|
||||
export async function run({
|
||||
host = 'localhost', port = 3306, name = 'jol',
|
||||
username, password, domainId, contestType = 'oi',
|
||||
dataDir, rerun = false,
|
||||
}, report: Function) {
|
||||
const src = mysql.createConnection({
|
||||
host,
|
||||
port,
|
||||
user: username,
|
||||
password,
|
||||
database: name,
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
src.connect((err) => (err ? reject(err) : resolve(null)));
|
||||
});
|
||||
const query = (q: string | mysql.Query) => new Promise<[values: any[], fields: mysql.FieldInfo[]]>((res, rej) => {
|
||||
src.query(q, (err, val, fields) => {
|
||||
if (err) rej(err);
|
||||
else res([val, fields]);
|
||||
});
|
||||
});
|
||||
const target = await domain.get(domainId);
|
||||
if (!target) throw new NotFoundError(domainId);
|
||||
report({ message: 'Connected to database' });
|
||||
/*
|
||||
user_id varchar 20 N 用户id（主键）
|
||||
email varchar 100 Y 用户E-mail
|
||||
submit int 11 Y 用户提交次数
|
||||
solved int 11 Y 成功次数
|
||||
defunct char 1 N 是否屏蔽（Y/N）
|
||||
ip varchar 20 N 用户注册ip
|
||||
accesstime datetime Y 用户注册时间
|
||||
volume int 11 N 页码（表示用户上次看到第几页）
|
||||
language int 11 N 语言
|
||||
password varchar 32 Y 密码（加密）
|
||||
reg_time datetime Y 用户注册时间
|
||||
nick varchar 100 N 昵称
|
||||
school varchar 100 N 用户所在学校
|
||||
*/
|
||||
const uidMap: Record<string, number> = {};
|
||||
const [udocs] = await query('SELECT * FROM `users`');
|
||||
const precheck = await user.getMulti({ unameLower: { $in: udocs.map((u) => u.user_id.toLowerCase()) } }).toArray();
|
||||
if (precheck.length) throw new Error(`Conflict username: ${precheck.map((u) => u.unameLower).join(', ')}`);
|
||||
for (const udoc of udocs) {
|
||||
const current = await user.getByEmail(domainId, udoc.email || `${udoc.user_id}@hustoj.local`);
|
||||
if (current) {
|
||||
report({ message: `duplicate user with email ${udoc.email}: ${current.uname},${udoc.user_id}` });
|
||||
uidMap[udoc.user_id] = current._id;
|
||||
} else {
|
||||
const uid = await user.create(
|
||||
udoc.email || `${udoc.user_id}@hustoj.local`, udoc.user_id, '',
|
||||
null, udoc.ip, udoc.defunct === 'Y' ? 0 : system.get('default.priv'),
|
||||
);
|
||||
uidMap[udoc.user_id] = uid;
|
||||
await user.setById(uid, {
|
||||
loginat: udoc.accesstime,
|
||||
regat: udoc.reg_time,
|
||||
hash: udoc.password,
|
||||
salt: udoc.password,
|
||||
hashType: 'hust',
|
||||
});
|
||||
await domain.setUserInDomain(domainId, uid, {
|
||||
displayName: udoc.nick || '',
|
||||
school: udoc.school || '',
|
||||
nSubmit: udoc.submit,
|
||||
nAccept: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const [admins] = await query("SELECT * FROM `privilege` WHERE `rightstr` = 'administrator'");
|
||||
for (const admin of admins) await domain.setUserRole(domainId, uidMap[admin.user_id], 'root');
|
||||
const adminUids = admins.map((admin) => uidMap[admin.user_id]);
|
||||
|
||||
/*
|
||||
problem_id int 11 N 题目编号，主键
|
||||
title varchar 200 N 标题
|
||||
description text Y 题目描述
|
||||
inupt text Y 输入说明
|
||||
output text Y 输出说明
|
||||
sample_input text Y 输入参照
|
||||
sample_output text Y 输出参照
|
||||
spj char 1 N 是否为特别题目
|
||||
hint text Y 暗示
|
||||
source varchar 100 Y 来源
|
||||
in_date datetime Y 加入时间
|
||||
time_limit int 11 N 限时（秒）
|
||||
memory_limit int 11 N 空间限制(MByte)
|
||||
defunct char 1 N 是否屏蔽（Y/N）
|
||||
accepted int 11 Y 总ac次数
|
||||
submit int 11 Y 总提交次数
|
||||
solved int 11 Y 解答（未用）
|
||||
*/
|
||||
const pidMap: Record<string, number> = {};
|
||||
const [[{ 'count(*)': pcount }]] = await query('SELECT count(*) FROM `problem`');
|
||||
const step = 50;
|
||||
const pageCount = Math.ceil(pcount / step);
|
||||
for (let pageId = 0; pageId < pageCount; pageId++) {
|
||||
const [pdocs] = await query(`SELECT * FROM \`problem\` LIMIT ${pageId * step}, ${step}`);
|
||||
for (const pdoc of pdocs) {
|
||||
if (rerun) {
|
||||
const opdoc = await problem.get(domainId, `P${pdoc.problem_id}`);
|
||||
if (opdoc) pidMap[pdoc.problem_id] = opdoc.docId;
|
||||
}
|
||||
if (!pidMap[pdoc.problem_id]) {
|
||||
const pid = await problem.add(
|
||||
domainId, `P${pdoc.problem_id}`,
|
||||
pdoc.title, buildContent({
|
||||
description: pdoc.description,
|
||||
input: pdoc.input,
|
||||
output: pdoc.output,
|
||||
samples: [[pdoc.sample_input.trim(), pdoc.sample_output.trim()]],
|
||||
hint: pdoc.hint,
|
||||
source: pdoc.source,
|
||||
}, 'html'),
|
||||
1, pdoc.source.split(' ').map((i) => i.trim()).filter((i) => i), pdoc.defunct === 'Y',
|
||||
);
|
||||
pidMap[pdoc.problem_id] = pid;
|
||||
}
|
||||
const [cdoc] = await query(`SELECT * FROM \`privilege\` WHERE \`rightstr\` = 'p${pdoc.problem_id}'`);
|
||||
const maintainer = [];
|
||||
for (let i = 1; i < cdoc.length; i++) maintainer.push(uidMap[cdoc[i].user_id]);
|
||||
await problem.edit(domainId, pidMap[pdoc.problem_id], {
|
||||
nAccept: 0,
|
||||
nSubmit: pdoc.submit,
|
||||
config: `time: ${pdoc.time_limit}s\nmemory: ${pdoc.memory_limit}m`,
|
||||
owner: uidMap[cdoc[0]?.user_id] || 1,
|
||||
maintainer,
|
||||
html: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
contest_id int 11 N 竞赛id（主键）
|
||||
title varchar 255 Y 竞赛标题
|
||||
start_time datetime Y 开始时间(年月日时分)
|
||||
end_time datatime Y 结束时间(年月日时分)
|
||||
defunct char 1 N 是否屏蔽（Y/N）
|
||||
description text Y 描述（在此版本中未用）
|
||||
private tinyint 4 公开/内部（0/1）
|
||||
langmask int 11 语言
|
||||
password char(16) 进入比赛的密码
|
||||
user_id char(48) 允许参加比赛用户列表
|
||||
*/
|
||||
const tidMap: Record<string, string> = {};
|
||||
const [tdocs] = await query('SELECT * FROM `contest`');
|
||||
for (const tdoc of tdocs) {
|
||||
const [pdocs] = await query(`SELECT * FROM \`contest_problem\` WHERE \`contest_id\` = ${tdoc.contest_id}`);
|
||||
const pids = pdocs.map((i) => pidMap[i.problem_id]);
|
||||
const tid = await contest.add(
|
||||
domainId, tdoc.title, tdoc.description || 'Description',
|
||||
adminUids[0], contestType, tdoc.start_time, tdoc.end_time, pids, true,
|
||||
{ _code: password },
|
||||
);
|
||||
tidMap[tdoc.contest_id] = tid.toHexString();
|
||||
}
|
||||
|
||||
/*
|
||||
solution 程序运行结果记录
|
||||
字段名 类型 长度 是否允许为空 备注
|
||||
solution_id int 11 N 运行id（主键）
|
||||
problem_id int 11 N 问题id
|
||||
user_id char 20 N 用户id
|
||||
time int 11 N 用时（秒）
|
||||
memory int 11 N 所用空间（）
|
||||
in_date datetime N 加入时间
|
||||
result smallint 6 N 结果（4：AC）
|
||||
language tinyint 4 N 语言
|
||||
ip char 15 N 用户ip
|
||||
contest_id int 11 Y 所属于竞赛组
|
||||
valid tinyint 4 N 是否有效？？？
|
||||
num tinyint 4 N 题目在竞赛中的顺序号
|
||||
code_lenght int 11 N 代码长度
|
||||
judgetime datetime Y 判题时间
|
||||
pass_rate decimal 2 N 通过百分比（OI模式下可用）
|
||||
lint_error int N ？？？
|
||||
judger char(16) N 判题机
|
||||
*/
|
||||
const [[{ 'count(*)': rcount }]] = await query('SELECT count(*) FROM `solution`');
|
||||
const rpageCount = Math.ceil(rcount / step);
|
||||
for (let pageId = 0; pageId < rpageCount; pageId++) {
|
||||
const [rdocs] = await query(`SELECT * FROM \`solution\` LIMIT ${pageId * step}, ${step}`);
|
||||
for (const rdoc of rdocs) {
|
||||
const data: RecordDoc = {
|
||||
status: statusMap[rdoc.result],
|
||||
_id: Time.getObjectID(rdoc.in_date, false),
|
||||
uid: uidMap[rdoc.user_id],
|
||||
code: "HustOJ didn't provide user code",
|
||||
lang: langMap[rdoc.language],
|
||||
pid: pidMap[rdoc.problem_id],
|
||||
domainId,
|
||||
score: rdoc.pass_rate ? Math.ceil(rdoc.pass_rate * 100) : rdoc.result === 4 ? 100 : 0,
|
||||
time: rdoc.time,
|
||||
memory: rdoc.memory,
|
||||
judgeTexts: [],
|
||||
compilerTexts: [],
|
||||
testCases: [],
|
||||
judgeAt: new Date(),
|
||||
rejudged: false,
|
||||
judger: 1,
|
||||
};
|
||||
const [ceInfo] = await query(`SELECT \`error\` FROM \`compileinfo\` WHERE \`solution_id\` = ${rdoc.solution_id}`);
|
||||
if (ceInfo[0]?.error) data.judgeTexts.push(ceInfo[0].error);
|
||||
const [rtInfo] = await query(`SELECT \`error\` FROM \`runtimeinfo\` WHERE \`solution_id\` = ${rdoc.solution_id}`);
|
||||
if (rtInfo[0]?.error) data.judgeTexts.push(rtInfo[0].error);
|
||||
const [source] = await query(`SELECT \`source\` FROM \`source_code\` WHERE \`solution_id\` = ${rdoc.solution_id}`);
|
||||
if (source[0]?.source) data.code = source[0].source;
|
||||
if (rdoc.contest_id) {
|
||||
data.contest = new ObjectID(tidMap[rdoc.contest_id]);
|
||||
await contest.attend(domainId, data.contest, uidMap[rdoc.user_id]).catch(noop);
|
||||
}
|
||||
await record.coll.insertOne(data);
|
||||
await postJudge(data);
|
||||
}
|
||||
}
|
||||
|
||||
src.end();
|
||||
|
||||
if (!dataDir) return true;
|
||||
if (dataDir.endsWith('/')) dataDir = dataDir.slice(0, -1);
|
||||
const files = await fs.readdir(dataDir, { withFileTypes: true });
|
||||
for (const file of files) {
|
||||
if (!file.isDirectory()) continue;
|
||||
const datas = await fs.readdir(`${dataDir}/${file.name}`);
|
||||
const pdoc = await problem.get(domainId, `P${file.name}`, undefined, true);
|
||||
if (!pdoc) continue;
|
||||
report({ message: `Syncing testdata for ${file.name}` });
|
||||
for (const data of datas) {
|
||||
const filename = nameMap[data] || data;
|
||||
await problem.addTestdata(domainId, pdoc.docId, filename, `${dataDir}/${file.name}/${data}`);
|
||||
}
|
||||
await problem.addTestdata(domainId, pdoc.docId, 'config.yaml', Buffer.from(pdoc.config as string));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export const description = 'migrate from hustoj';
|
||||
export const validate = {
|
||||
host: 'string',
|
||||
port: 'number',
|
||||
name: 'string',
|
||||
username: 'string',
|
||||
password: 'string',
|
||||
domainId: 'string',
|
||||
contestType: 'string',
|
||||
dataDir: 'string',
|
||||
};
|
||||
|
||||
global.Hydro.script.migrateHustoj = { run, description, validate };
|
||||
@@ -0,0 +1,510 @@
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
/* eslint-disable no-await-in-loop */
|
||||
import mongodb, { Cursor, Db } from 'mongodb';
|
||||
import {
|
||||
DiscussionTailReplyDoc, MessageDoc, RecordDoc, TestCase, TrainingNode,
|
||||
} from 'hydrooj';
|
||||
|
||||
const dst = global.Hydro.service.db;
|
||||
const { discussion } = global.Hydro.model;
|
||||
const map = {};
|
||||
|
||||
const pid = (id) => {
|
||||
if (map[id.toString()]) return map[id.toString()];
|
||||
return id;
|
||||
};
|
||||
|
||||
const tasks = {
|
||||
user: async (doc) => ({
|
||||
_id: doc._id,
|
||||
uname: doc.uname,
|
||||
unameLower: doc.uname_lower,
|
||||
salt: doc.salt,
|
||||
hash: doc.hash.split('|')[1],
|
||||
hashType: doc.hash.split('|')[0] === 'vj4' ? 'hydro' : doc.hash.split('|')[0],
|
||||
priv: doc.priv,
|
||||
avatar: `gravatar:${doc.gravatar}`,
|
||||
mail: doc.mail,
|
||||
mailLower: doc.mail_lower,
|
||||
regat: doc.regat,
|
||||
regip: doc.regip,
|
||||
loginat: doc.loginat,
|
||||
loginip: doc.loginip,
|
||||
codeLang: doc.code_lang,
|
||||
codeTemplate: doc.code_template,
|
||||
timezone: doc.timezone,
|
||||
viewLang: doc.viewLang,
|
||||
bio: doc.bio,
|
||||
gender: doc.gender,
|
||||
qq: doc.qq,
|
||||
}),
|
||||
document: {
|
||||
_id: '_id',
|
||||
doc_id: 'docId',
|
||||
doc_type: 'docType',
|
||||
num_submit: 'nSubmit',
|
||||
num_accept: 'nAccept',
|
||||
difficulty: 'difficulty',
|
||||
pname: 'pid',
|
||||
title: 'title',
|
||||
content: 'content',
|
||||
owner_uid: 'owner',
|
||||
category: 'category',
|
||||
hidden: 'hidden',
|
||||
data: 'data',
|
||||
tag: 'tag',
|
||||
vote: 'vote',
|
||||
reply: {
|
||||
field: 'reply',
|
||||
processer: (reply) => {
|
||||
const res = [];
|
||||
for (const r of reply) {
|
||||
const drrdoc: DiscussionTailReplyDoc = {
|
||||
_id: r._id,
|
||||
content: r.content,
|
||||
owner: r.owner_uid,
|
||||
ip: r.ip,
|
||||
history: [],
|
||||
};
|
||||
res.push(drrdoc);
|
||||
}
|
||||
return res;
|
||||
},
|
||||
},
|
||||
parent_doc_id: {
|
||||
field: 'parentId',
|
||||
processer: (parentId, doc) => {
|
||||
if (doc.parent_doc_type === global.Hydro.model.document.TYPE_PROBLEM) {
|
||||
return pid(parentId);
|
||||
}
|
||||
return parentId;
|
||||
},
|
||||
},
|
||||
parent_doc_type: 'parentType',
|
||||
num_replies: 'nReply',
|
||||
views: 'views',
|
||||
highlight: 'highlight',
|
||||
ip: 'ip',
|
||||
domain_id: 'domainId',
|
||||
update_at: 'updateAt',
|
||||
begin_at: 'beginAt',
|
||||
end_at: 'endAt',
|
||||
penalty_since: 'penaltySince',
|
||||
penalty_rules: {
|
||||
field: 'penaltyRules',
|
||||
processer: (rule) => {
|
||||
const n = {};
|
||||
for (const key in rule) {
|
||||
n[parseInt(key, 10) / 3600] = rule;
|
||||
}
|
||||
return n;
|
||||
},
|
||||
},
|
||||
rule: {
|
||||
field: 'rule',
|
||||
processer: (rule: number) => {
|
||||
const rules = {
|
||||
2: 'oi',
|
||||
3: 'acm',
|
||||
11: 'homework',
|
||||
};
|
||||
return rules[rule];
|
||||
},
|
||||
},
|
||||
pids: {
|
||||
field: 'pids',
|
||||
processer: (pids) => pids.map((p) => pid(p)),
|
||||
},
|
||||
attend: 'attend',
|
||||
desc: 'description',
|
||||
enroll: 'attend',
|
||||
rated: 'rated',
|
||||
dag: {
|
||||
field: 'dag',
|
||||
processer: (dag) => {
|
||||
const r: TrainingNode[] = [];
|
||||
for (const t of dag) {
|
||||
r.push({
|
||||
_id: t._id,
|
||||
title: t.title,
|
||||
requireNids: t.require_nids,
|
||||
pids: t.pids.map((id) => pid(id)),
|
||||
});
|
||||
}
|
||||
return r;
|
||||
},
|
||||
},
|
||||
},
|
||||
'document.status': {
|
||||
_id: '_id',
|
||||
doc_id: 'docId',
|
||||
doc_type: 'docType',
|
||||
uid: 'uid',
|
||||
domain_id: 'domainId',
|
||||
num_accept: 'nAccept',
|
||||
num_submit: 'nSubmit',
|
||||
status: 'status',
|
||||
accept: 'accept',
|
||||
vote: 'vote',
|
||||
star: 'star',
|
||||
enroll: 'enroll',
|
||||
rid: 'rid',
|
||||
rev: 'rev',
|
||||
attend: 'attend',
|
||||
journal: {
|
||||
field: 'journal',
|
||||
processer: (journal) => {
|
||||
const r = [];
|
||||
for (const i of journal) {
|
||||
r.push({ ...i, pid: pid(i.pid) });
|
||||
}
|
||||
return r;
|
||||
},
|
||||
},
|
||||
penalty_score: 'penaltyScore',
|
||||
detail: {
|
||||
field: 'detail',
|
||||
processer: (detail) => {
|
||||
const r = [];
|
||||
for (const i of detail) {
|
||||
r.push({ ...i, pid: pid(i.pid) });
|
||||
}
|
||||
return r;
|
||||
},
|
||||
},
|
||||
score: 'score',
|
||||
time: 'time',
|
||||
done: 'done',
|
||||
done_nids: 'doneNids',
|
||||
done_pids: {
|
||||
field: 'donePids',
|
||||
processer: (pids) => pids.map((id) => pid(id)),
|
||||
},
|
||||
rp: null,
|
||||
},
|
||||
'domain.user': {
|
||||
_id: '_id',
|
||||
domain_id: 'domainId',
|
||||
uid: 'uid',
|
||||
num_problems: 'nProblem',
|
||||
num_submit: 'nSubmit',
|
||||
num_accept: 'nAccept',
|
||||
num_liked: 'nLike',
|
||||
level: 'level',
|
||||
role: 'role',
|
||||
join_at: 'joinAt',
|
||||
display_name: 'displayName',
|
||||
rp: null,
|
||||
rank: null,
|
||||
userfile_usage: null,
|
||||
},
|
||||
record: async (doc) => {
|
||||
if (doc.hidden) return null;
|
||||
const testCases: TestCase[] = [];
|
||||
for (const c of doc.cases || []) {
|
||||
testCases.push({
|
||||
status: c.status,
|
||||
time: c.time_ms || c.time,
|
||||
memory: c.memory_kb || c.memory,
|
||||
message: (c.judge_text || '') + (c.message || ''),
|
||||
});
|
||||
}
|
||||
const rdoc: RecordDoc = {
|
||||
_id: doc._id,
|
||||
status: doc.status,
|
||||
score: doc.score,
|
||||
time: doc.time_ms,
|
||||
memory: doc.memory_kb,
|
||||
code: doc.code,
|
||||
lang: doc.lang,
|
||||
uid: doc.uid,
|
||||
source: `${doc.domainId}/${pid(doc.pid)}`,
|
||||
pid: pid(doc.pid),
|
||||
domainId: doc.domain_id,
|
||||
judger: doc.judge_uid || 1,
|
||||
judgeAt: doc.judge_at || new Date(),
|
||||
judgeTexts: doc.judge_texts || [],
|
||||
compilerTexts: doc.compiler_texts || [],
|
||||
testCases,
|
||||
rejudged: !!doc.rejudged,
|
||||
};
|
||||
if (doc.rejudged) rdoc.rejudged = true;
|
||||
if (doc.tid) rdoc.contest = doc.tid;
|
||||
return rdoc;
|
||||
},
|
||||
domain: {
|
||||
_id: '_id',
|
||||
doc_id: 'docId',
|
||||
doc_type: 'docType',
|
||||
owner_uid: 'owner',
|
||||
owner: 'owner',
|
||||
name: 'name',
|
||||
roles: {
|
||||
field: 'roles',
|
||||
processer: (roles) => {
|
||||
const res = {};
|
||||
for (const role in roles) {
|
||||
res[role] = roles[role].toString();
|
||||
}
|
||||
return res;
|
||||
},
|
||||
},
|
||||
gravatar: {
|
||||
field: 'avatar',
|
||||
processer: (gravatar) => `gavatar:${gravatar}`,
|
||||
},
|
||||
bulletin: 'bulletin',
|
||||
},
|
||||
'fs.files': async (doc) => doc,
|
||||
'fs.chunks': async (doc) => doc,
|
||||
file: async (doc) => ({
|
||||
_id: doc._id,
|
||||
count: 0,
|
||||
secret: doc.metadata.secret,
|
||||
size: doc.length,
|
||||
md5: doc.md5,
|
||||
}),
|
||||
};
|
||||
|
||||
type CursorGetter = (s: Db) => Cursor<any>;
|
||||
|
||||
const cursor: NodeJS.Dict<CursorGetter> = {
|
||||
user: (s) => s.collection('user').find(),
|
||||
document: (s) => s.collection('document').find({ doc_type: { $ne: 20 } }),
|
||||
'document.status': (s) => s.collection('document.status').find(),
|
||||
'domain.user': (s) => s.collection('domain.user').find(),
|
||||
record: (s) => s.collection('record').find(),
|
||||
domain: (s) => s.collection('domain').find(),
|
||||
'fs.files': (s) => s.collection('fs.files').find(),
|
||||
'fs.chunks': (s) => s.collection('fs.chunks').find(),
|
||||
file: (s) => s.collection('fs.files').find(),
|
||||
};
|
||||
|
||||
async function discussionNode(src: Db, report: Function) {
|
||||
const count = await src.collection('document').find({ doc_type: 20 }).count();
|
||||
await report({ progress: 1, message: `discussion.node: ${count}` });
|
||||
const total = Math.floor(count / 5);
|
||||
for (let i = 0; i <= total; i++) {
|
||||
const docs = await src.collection('document')
|
||||
.find({ doc_type: 20 }).skip(i * 5).limit(5)
|
||||
.toArray();
|
||||
for (const doc of docs) {
|
||||
const t = [];
|
||||
for (const item of doc.content || []) {
|
||||
const category = item[0];
|
||||
const nodes = item[1];
|
||||
for (const node of nodes || []) {
|
||||
if (node.pic) {
|
||||
t.push(discussion.addNode(
|
||||
doc.domain_id, node.name, category, { pic: node.pic },
|
||||
));
|
||||
} else {
|
||||
t.push(discussion.addNode(doc.domain_id, node.name, category, {}));
|
||||
}
|
||||
}
|
||||
}
|
||||
await Promise.all(t).catch((e) => e);
|
||||
}
|
||||
await report({ progress: Math.round(100 * ((i + 1) / (total + 1))) });
|
||||
}
|
||||
}
|
||||
|
||||
function addSpace(content: string) {
|
||||
const lines = content.split('\r\n');
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (lines[i].endsWith('|')) continue; // Markdown table;
|
||||
if (!lines[i].endsWith(' ')) lines[i] = `${lines[i]} `;
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
async function fix(doc) {
|
||||
await dst.collection('document').updateOne(
|
||||
{ _id: doc._id },
|
||||
{ $set: { pid: doc.pid || doc.docId.toString(), content: addSpace(doc.content) } },
|
||||
);
|
||||
}
|
||||
|
||||
async function fixProblem(report: Function) {
|
||||
const count = await dst.collection('document').find({ docType: 10 }).count();
|
||||
await report({ progress: 1, message: `Fix pid: ${count}` });
|
||||
const total = Math.floor(count / 50);
|
||||
for (let i = 0; i <= total; i++) {
|
||||
const docs = await dst.collection('document')
|
||||
.find({ docType: 10 }).skip(i * 50).limit(50)
|
||||
.toArray();
|
||||
for (const doc of docs) {
|
||||
await fix(doc).catch((e) => report({ message: `${e.toString()}\n${e.stack}` }));
|
||||
}
|
||||
await report({ progress: Math.round(100 * ((i + 1) / (total + 1))) });
|
||||
}
|
||||
}
|
||||
|
||||
function objid(ts: Date) {
|
||||
const p = Math.floor(ts.getTime() / 1000).toString(16);
|
||||
const id = new mongodb.ObjectID();
|
||||
return new mongodb.ObjectID(p + id.toHexString().slice(8, 8 + 6 + 4 + 6));
|
||||
}
|
||||
|
||||
// FIXME this seems not working
|
||||
async function message(src: Db, report: Function) {
|
||||
const count = await src.collection('message').find().count();
|
||||
await report({ progress: 1, message: `Messages: ${count}` });
|
||||
const total = Math.floor(count / 50);
|
||||
for (let i = 0; i <= total; i++) {
|
||||
const docs = await src.collection('message')
|
||||
.find().skip(i * 50).limit(50)
|
||||
.toArray();
|
||||
for (const doc of docs) {
|
||||
for (const msg of doc.reply) {
|
||||
const mdoc: MessageDoc = {
|
||||
_id: objid(msg.at),
|
||||
from: msg.sender_uid,
|
||||
to: msg.sender_uid === doc.sender_uid ? doc.sendee_uid : doc.sender_uid,
|
||||
content: msg.content,
|
||||
// Mark all as read
|
||||
flag: 0,
|
||||
};
|
||||
await dst.collection('message').insertOne(mdoc);
|
||||
}
|
||||
}
|
||||
await report({ progress: Math.round(100 * ((i + 1) / (total + 1))) });
|
||||
}
|
||||
}
|
||||
|
||||
async function removeInvalidPid(report: Function) {
|
||||
const count = await dst.collection('document').find({ docType: 10 }).count();
|
||||
const bulk = dst.collection('document').initializeUnorderedBulkOp();
|
||||
await report({ progress: 1, message: `Remove pid: ${count}` });
|
||||
const total = Math.floor(count / 50);
|
||||
for (let i = 0; i <= total; i++) {
|
||||
const docs = await dst.collection('document')
|
||||
.find({ docType: 10 }).skip(i * 50).limit(50)
|
||||
.toArray();
|
||||
for (const doc of docs) {
|
||||
const id = parseInt(doc.pid, 10);
|
||||
if (Number.isSafeInteger(id)) {
|
||||
bulk.find({ _id: doc._id }).updateOne({ $unset: { pid: '' } });
|
||||
}
|
||||
}
|
||||
await bulk.execute();
|
||||
}
|
||||
}
|
||||
|
||||
async function task(name: any, src: Db, report: Function) {
|
||||
const count = await cursor[name](src).count();
|
||||
await report({ progress: 1, message: `${name}: ${count}` });
|
||||
const total = Math.floor(count / 50);
|
||||
let lastProgress = -1;
|
||||
for (let i = 0; i <= total; i++) {
|
||||
const docs = await cursor[name](src).skip(i * 50).limit(50).toArray();
|
||||
const res = [];
|
||||
for (const doc of docs) {
|
||||
let d: any = {};
|
||||
if (typeof tasks[name].call === 'function') {
|
||||
d = await tasks[name](doc);
|
||||
} else {
|
||||
const mapper = tasks[name];
|
||||
for (const key in doc) {
|
||||
if (typeof mapper[key] === 'string') {
|
||||
d[mapper[key]] = doc[key];
|
||||
} else if (mapper[key] === null) {
|
||||
// Ignore this key
|
||||
} else if (typeof mapper[key] === 'object') {
|
||||
d[mapper[key].field] = mapper[key].processer(doc[key], doc);
|
||||
} else {
|
||||
await report({ message: `Unknown key ${key} in collection ${name}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
if (d) {
|
||||
if (d.rule) d.rated = true; // Hack contest rated field
|
||||
const docWithoutId = {};
|
||||
const docWithoutDid = {};
|
||||
for (const key in d) {
|
||||
if (key !== '_id') {
|
||||
if (key !== 'domainId' && key !== 'docId' && key !== 'docType' && key !== 'uid') {
|
||||
docWithoutDid[key] = d[key];
|
||||
}
|
||||
docWithoutId[key] = d[key];
|
||||
}
|
||||
}
|
||||
if (d.domainId && d.docId && d.docType) {
|
||||
const query: any = { domainId: d.domainId, docId: d.docId, docType: d.docType };
|
||||
if (d.uid) query.uid = d.uid;
|
||||
res.push((async () => {
|
||||
const data = await dst.collection(name).findOne(query);
|
||||
if (data) {
|
||||
await dst.collection(name).updateOne(query, { $set: docWithoutDid });
|
||||
} else if (d._id) {
|
||||
const dat = await dst.collection(name).findOne({ _id: d._id });
|
||||
if (dat) {
|
||||
await dst.collection(name).updateOne({
|
||||
_id: d._id,
|
||||
}, { $set: docWithoutId });
|
||||
} else {
|
||||
await dst.collection(name).insertOne(d);
|
||||
}
|
||||
} else {
|
||||
await dst.collection(name).insertOne(d);
|
||||
}
|
||||
})());
|
||||
} else if (d._id) {
|
||||
res.push(dst.collection(name).updateOne({
|
||||
_id: d._id,
|
||||
}, { $set: docWithoutId }, { upsert: true }));
|
||||
} else res.push(dst.collection(name).insertOne(d));
|
||||
}
|
||||
}
|
||||
await Promise.all(res).catch((e) => report({ message: `${e}\n${e.stack}` }));
|
||||
const progress = Math.round(100 * ((i + 1) / (total + 1)));
|
||||
if (progress > lastProgress) {
|
||||
await report({ progress });
|
||||
lastProgress = progress;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function run({
|
||||
host = 'localhost', port = 27017, name = 'vijos4', username, password,
|
||||
}, report: Function) {
|
||||
let mongourl = 'mongodb://';
|
||||
if (username) mongourl += `${username}:${password}@`;
|
||||
mongourl += `${host}:${port}/${name}`;
|
||||
const Database = await mongodb.MongoClient.connect(mongourl, {
|
||||
useNewUrlParser: true, useUnifiedTopology: true,
|
||||
});
|
||||
const src = Database.db(name);
|
||||
await report({ progress: 0, message: 'Database connected.' });
|
||||
const userCounter = await src.collection('system').findOne({ _id: 'user_counter' });
|
||||
if (!userCounter) {
|
||||
report({ message: 'No valid installation found' });
|
||||
return false;
|
||||
}
|
||||
await report({ progress: 1, message: 'Collection:system done.' });
|
||||
if (!await dst.collection('system').findOne({ _id: 'migrateVijosFs' })) {
|
||||
const f = ['fs.files', 'fs.chunks'] as any;
|
||||
for (const i of f) {
|
||||
await dst.collection(i).deleteMany({});
|
||||
await task(i, src, report);
|
||||
}
|
||||
await dst.collection('system').insertOne({ _id: 'migrateVijosFs', value: 1 });
|
||||
}
|
||||
await dst.collection('user').deleteMany({ _id: { $nin: [0, 1] } });
|
||||
await dst.collection('message').deleteMany({});
|
||||
const d = ['domain', 'user', 'document', 'document.status', 'domain.user', 'record', 'file'];
|
||||
for (const i of d) await task(i, src, report);
|
||||
await fixProblem(report);
|
||||
await discussionNode(src, report);
|
||||
await message(src, report);
|
||||
await removeInvalidPid(report);
|
||||
await global.Hydro.model.system.set('db.ver', 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
export const description = 'migrate from vijos';
|
||||
export const validate = {
|
||||
host: 'string', port: 'number', name: 'string', username: 'string', password: 'string',
|
||||
};
|
||||
|
||||
global.Hydro.script.migrateVijos = { run, description, validate };
|
||||
Reference in new issue
Block a user