优化rating计算速度

This commit is contained in:
masnn committed 2020-06-09 13:22:04 +08:00
1 parent 45f1184d68
commit 55912b087d
22 files changed
+309 -178

No files matched your search

+4
View File
@@ -28,6 +28,10 @@ jobs:
with:
name: full.js
path: .build/full.js
- uses: actions/upload-artifact@v1
with:
name: install.js
path: .build/install.js
- uses: actions/upload-artifact@v1
with:
name: module
+2 -2
View File
@@ -282,7 +282,7 @@ class ContestCreateHandler extends ContestHandler {
}
async post({
domainId, title, content, rule, beginAtDate, beginAtTime, duration, pids,
domainId, title, content, rule, beginAtDate, beginAtTime, duration, pids, rated = false,
}) {
let beginAt;
try {
@@ -294,7 +294,7 @@ class ContestCreateHandler extends ContestHandler {
pids = await this.verifyProblems(domainId, pids);
const tid = await contest.add(
domainId, title, content,
this.user._id, rule, beginAt, endAt, pids,
this.user._id, rule, beginAt, endAt, pids, rated,
);
this.response.body = { tid };
this.response.redirect = `/c/${tid}`;
+7 -2
View File
@@ -95,7 +95,12 @@ class DomainUserHandler extends ManageHandler {
const uids = [];
const rudocs = {};
const [udocs, roles] = await Promise.all([
user.getInDomain(domainId),
user.getMultiInDomain(domainId, {
$and: [
{ role: { $ne: 'default' } },
{ role: { $ne: null } },
],
}).toArray(),
user.getRoles(domainId),
]);
for (const role of roles) rudocs[role._id] = [];
@@ -104,7 +109,7 @@ class DomainUserHandler extends ManageHandler {
// TODO Improve here
// eslint-disable-next-line no-await-in-loop
const ud = await user.getById(domainId, udoc.uid);
rudocs[udoc.role].push({ ...ud, role: udoc.role });
rudocs[ud.role].push(ud);
}
const rolesSelect = roles.map((role) => [role._id, role._id]);
const udict = await user.getList(domainId, uids);
+2 -2
View File
@@ -197,7 +197,7 @@ class HomeworkCreateHandler extends HomeworkHandler {
async post({
domainId, title, content, beginAtDate, beginAtTime,
penaltySinceDate, penaltySinceTime, extensionDays,
penaltyRules, pids,
penaltyRules, pids, rated = false,
}) {
let beginAt;
let penaltySince;
@@ -216,7 +216,7 @@ class HomeworkCreateHandler extends HomeworkHandler {
if (penaltySince > endAt) throw new ValidationError('extensionDays');
await this.verifyProblems(domainId, pids);
const tid = await contest.add(domainId, title, content, this.user._id,
'homework', beginAt, endAt, pids, { penaltySince, penaltyRules }, document.TYPE_HOMEWORK);
'homework', beginAt, endAt, pids, rated, { penaltySince, penaltyRules }, document.TYPE_HOMEWORK);
this.response.body = { tid };
this.response.redirect = `/homework/${tid}`;
}
+2 -2
View File
@@ -8,7 +8,7 @@ const { PermissionError } = require('../error');
class SystemHandler extends Handler {
async prepare() {
if (!this.user.priv) throw new PermissionError('???');
this.body = {
this.response.body = {
path: [
['Hydro', '/'],
['manage_main', '/manage'],
@@ -58,7 +58,7 @@ class SystemScriptHandler extends SystemHandler {
async post({ id, args }) {
// TODO Do not use console.log
await global.Hydro.script[id].run(args, console.log);
await global.Hydro.script[id].run(JSON.parse(args), console.log);
this.back();
}
}
+1 -1
View File
@@ -176,7 +176,7 @@ class ProblemSubmitHandler extends ProblemDetailHandler {
});
await Promise.all([
record.judge(domainId, rid),
user.inc(this.user._id, 'nSubmit'),
user.incDomain(domainId, this.user._id, 'nSubmit'),
]);
this.response.body = { rid };
this.response.redirect = `/r/${rid}`;
+10
View File
@@ -0,0 +1,10 @@
const crypto = require('crypto');
/**
* @param {any} content
* @return {string}
*/
global.Hydro.lib.sha1 = module.exports = (content) => {
const hash = crypto.createHash('sha1');
hash.update(content);
return hash.digest('hex');
};
+2 -2
View File
@@ -216,7 +216,7 @@ const builtinHandler = [
];
const builtinScript = [
'install', 'uninstall', 'rating', 'recalcRating',
'install', 'uninstall', 'rating', 'recalcRating', 'register',
];
async function loadAsMaster() {
@@ -385,7 +385,7 @@ async function load() {
cluster.on('online', (worker) => {
console.log(`Worker ${worker.process.pid} is online`);
});
for (let i = 0; i < numCPUs; i++) {
for (let i = 1; i < numCPUs; i++) {
cluster.fork();
}
} else {
+3 -2
View File
@@ -207,7 +207,8 @@ const RULES = {
* @returns {ObjectID} tid
*/
function add(domainId, title, content, owner, rule,
beginAt = new Date(), endAt = new Date(), pids = [], data = {}, type = document.TYPE_CONTEST) {
beginAt = new Date(), endAt = new Date(), pids = [], rated = false,
data = {}, type = document.TYPE_CONTEST) {
validator.checkTitle(title);
validator.checkContent(content);
if (!this.RULES[rule]) throw new ValidationError('rule');
@@ -217,7 +218,7 @@ function add(domainId, title, content, owner, rule,
});
this.RULES[rule].check(data);
return document.add(domainId, content, owner, type, null, null, null, {
...data, title, rule, beginAt, endAt, pids, attend: 0,
...data, title, rule, beginAt, endAt, pids, attend: 0, rated,
});
}
+16 -8
View File
@@ -29,7 +29,7 @@ class USER {
this.nSubmit = dudoc.nSubmit || 0;
this.rating = dudoc.rating || 1500;
this.perm = dudoc.perm;
this.role = dudoc.role;
this.role = dudoc.role || 'default';
}
hasPerm(p) {
@@ -39,7 +39,7 @@ class USER {
checkPassword(password) {
const h = global.Hydro.lib[`hash.${this.hashType || 'hydro'}`];
if (!h) throw new Error('Unknown hash method');
return h.check(password, this.salt(), this.hash());
return h.check(password, this.salt(), this.hash(), this);
}
}
@@ -120,6 +120,17 @@ async function inc(_id, field, n = 1) {
return udoc;
}
function setInDomain(domainId, uid, params) {
return document.setStatus(domainId, document.TYPE_DOMAIN_USER, 0, uid, params);
}
async function incDomain(domainId, uid, field, n = 1) {
const dudoc = await getInDomain(domainId, { _id: uid });
dudoc[field] = dudoc[field] + n || n;
await setInDomain(domainId, uid, { [field]: dudoc[field] });
return dudoc;
}
async function create({
uid, mail, uname, password, regip = '127.0.0.1', priv = perm.PRIV_NONE,
}) {
@@ -151,10 +162,6 @@ function getMulti(params) {
return coll.find(params);
}
function setInDomain(domainId, uid, params) {
return document.setStatus(domainId, document.TYPE_DOMAIN_USER, 0, uid, params);
}
function setMultiInDomain(domainId, query, params) {
return document.setMultiStatus(domainId, document.TYPE_DOMAIN_USER, query, params);
}
@@ -193,8 +200,8 @@ function getRole(domainId, name) {
return document.get(domainId, document.TYPE_DOMAIN_USER, name);
}
function getMultiInDomain(domainId) {
return document.getMultiStatus(domainId, document.TYPE_DOMAIN_USER);
function getMultiInDomain(domainId, query) {
return document.getMultiStatus(domainId, document.TYPE_DOMAIN_USER, query);
}
function addRole(domainId, name, permission) {
@@ -223,6 +230,7 @@ global.Hydro.model.user = module.exports = {
getByUname,
getMulti,
inc,
incDomain,
setById,
setEmail,
setPassword,
+1 -1
View File
@@ -56,7 +56,7 @@ async function run({ username, password } = {}) {
}
await Promise.all(tasks);
tasks = [
allowFail(domain.add, 'system', 0),
domain.add('system', 0, 'Hydro').catch(() => domain.edit('system', { owner: 0, name: 'Hydro' })),
collUser.updateOne({ _id: 0 }, {
$set: defaults({
_id: 0,
+9 -7
View File
@@ -14,28 +14,30 @@ async function run({ domainId, isSub = false }, report) {
}
return true;
}
const contests = await contest.getMulti(domainId).sort('endAt', -1).toArray();
const contests = await contest.getMulti(domainId, { rated: true }).sort('endAt', -1).toArray();
await report({ total: contests.length });
await user.setMultiInDomain(domainId, {}, { rating: 1500 });
const udict = {};
for (const i in contests) {
const tdoc = contests[i];
const tsdocs = await contest.getMultiStatus(domainId, { docId: tdoc.docId })
.sort(contest.RULES[tdoc.rule].statusSort).toArray();
const udict = await user.getList(domainId, tsdocs.map((tsdoc) => tsdoc.uid));
const rankedTsdocs = contest.RULES[tdoc.rule].rank(tsdocs);
const users = [];
for (const result of rankedTsdocs) {
users.push({ ...result[1], rank: result[0], old: udict[result[1].uid].rating || 1500 });
users.push({ ...result[1], rank: result[0], old: udict[result[1].uid] || 1500 });
}
const rated = rating(users);
const tasks = [];
for (const udoc of rated) {
tasks.push(user.setInDomain(domainId, udoc.uid, { rating: udoc.new }));
udict[udoc.uid] = udoc.new;
}
await Promise.all(tasks);
if (isSub) await report({ subProgress: (i + 1) / contests.length });
else await report({ current: parseInt(i) + 1 });
}
const tasks = [];
for (const uid in udict) {
tasks.push(user.setInDomain(domainId, uid, { rating: udict[uid] }));
}
await Promise.all(tasks);
return true;
}
+14
View File
@@ -0,0 +1,14 @@
const user = require('../model/user');
const system = require('../model/system');
async function run({
uname, password, mail, uid,
}) {
if (!uid) uid = await system.inc('user');
await user.create({
uid, uname, password, mail, regip: '127.0.0.1',
});
return uid;
}
global.Hydro.script.register = module.exports = { run };
+2 -2
View File
@@ -302,7 +302,7 @@ function Route(route, RouteHandler, permission = null) {
const method = ctx.method.toLowerCase();
// TODO(masnn) domainId
const args = {
...ctx.params, ...ctx.query, ...ctx.request.body, domainId: 'system',
domainId: 'system', ...ctx.params, ...ctx.query, ...ctx.request.body,
};
if (h.request.host !== await system.get('server.host')) {
args.domainId = h.request.url.split('//')[1].split(`.${await system.get('server.host')}`)[0] || '';
@@ -399,7 +399,7 @@ function Connection(prefix, RouteConnHandler) {
sock.on('connection', async (conn) => {
const h = new RouteConnHandler(conn);
try {
const args = { ...h.request.params, domainId: 'system' };
const args = { domainId: 'system', ...h.request.params };
if (args.uid) args.uid = parseInt(validator.checkUid(args.uid));
if (args.page) args.page = parseInt(args.page);
+23
View File
@@ -0,0 +1,23 @@
const { md5, sha1 } = global.Hydro.lib;
global.Hydro.lib['hash.vj2'] = module.exports = {
/**
* @param {string} password
* @param {string} salt
* @param {string} hash
*/
check(password, salt, hash, udoc) {
return hash === this.hash(password, salt, udoc);
},
/**
* @param {string} password
* @param {string} salt
*/
hash(password, salt, udoc) {
const { uname, unameLower } = udoc;
const pmd5 = md5(password);
const mixedSha1 = sha1(md5(unameLower + pmd5) + salt + sha1(pmd5 + salt));
return `${Buffer.from(uname).toString('base64')}|${mixedSha1}`;
},
};
+194 -140
View File
@@ -34,151 +34,123 @@ const tasks = {
gender: doc.gender,
qq: doc.qq,
}),
document: async (doc) => {
const res = {};
const mapper = {
_id: '_id',
doc_id: 'docId',
doc_type: 'docType',
num_submit: 'nSubmit',
num_accept: 'nAccept',
difficulty: 'difficulty',
pname: 'pid',
title: 'title',
content: 'content',
owner_uid: 'owner',
category: 'category',
hidden: 'hidden',
data: 'data',
tag: 'tag',
vote: 'vote',
reply: 'reply',
parent_doc_id: 'parentId',
parent_doc_type: 'parentType',
num_replies: 'nReply',
views: 'views',
highlight: 'highlight',
ip: 'ip',
domain_id: 'domainId',
update_at: 'updateAt',
begin_at: 'beginAt',
end_at: 'endAt',
penalty_since: 'penaltySince',
penalty_rules: {
field: 'penaltyRules',
processer: (rule) => {
const n = {};
for (const key in rule) {
n[key / 3600] = rule;
}
return n;
},
document: {
_id: '_id',
doc_id: 'docId',
doc_type: 'docType',
num_submit: 'nSubmit',
num_accept: 'nAccept',
difficulty: 'difficulty',
pname: 'pid',
title: 'title',
content: 'content',
owner_uid: 'owner',
category: 'category',
hidden: 'hidden',
data: 'data',
tag: 'tag',
vote: 'vote',
reply: 'reply',
parent_doc_id: 'parentId',
parent_doc_type: 'parentType',
num_replies: 'nReply',
views: 'views',
highlight: 'highlight',
ip: 'ip',
domain_id: 'domainId',
update_at: 'updateAt',
begin_at: 'beginAt',
end_at: 'endAt',
penalty_since: 'penaltySince',
penalty_rules: {
field: 'penaltyRules',
processer: (rule) => {
const n = {};
for (const key in rule) {
n[key / 3600] = rule;
}
return n;
},
rule: {
field: 'rule',
processer: (rule) => {
const rules = {
2: 'oi',
3: 'acm',
11: 'homework',
};
return rules[rule];
},
},
rule: {
field: 'rule',
processer: (rule) => {
const rules = {
2: 'oi',
3: 'acm',
11: 'homework',
};
return rules[rule];
},
pids: {
field: 'pids',
processer: (pids) => pids.map((p) => pid(p)),
},
pids: {
field: 'pids',
processer: (pids) => pids.map((p) => pid(p)),
},
attend: 'attend',
desc: 'description',
enroll: 'enroll',
dag: {
field: 'dag',
processer: (dag) => {
const r = [];
for (const t of dag) {
r.push({
_id: t._id,
title: t.title,
requireNids: t.require_nids,
pids: t.pids.map((id) => pid(id)),
});
}
return r;
},
attend: 'attend',
desc: 'description',
enroll: 'enroll',
dag: {
field: 'dag',
processer: (dag) => {
const r = [];
for (const t of dag) {
r.push({
_id: t._id,
title: t.title,
requireNids: t.require_nids,
pids: t.pids.map((id) => pid(id)),
});
}
return r;
},
},
};
for (const key in doc) {
if (typeof mapper[key] === 'string') {
res[mapper[key]] = doc[key];
} else if (typeof mapper[key] === 'object') {
res[mapper[key].field] = mapper[key].processer(doc[key]);
} else if (mapper[key] === null) {
// Ignore this key
} else {
console.log('Unknown key:', key);
}
}
return res;
},
},
'document.status': async (doc) => {
const res = {};
const mapper = {
_id: '_id',
doc_id: 'docId',
doc_type: 'docType',
uid: 'uid',
domain_id: 'domainId',
num_accept: 'nAccept',
num_submit: 'nSubmit',
rp: 'rp',
status: 'status',
accept: 'accept',
vote: 'vote',
star: 'star',
enroll: 'enroll',
rid: 'rid',
rev: 'rev',
attend: 'attend',
journal: {
field: 'journal',
processer: (journal) => {
const r = [];
for (const i of journal) {
r.push({ ...i, pid: pid(i.pid) });
}
return r;
},
'document.status': {
_id: '_id',
doc_id: 'docId',
doc_type: 'docType',
uid: 'uid',
domain_id: 'domainId',
num_accept: 'nAccept',
num_submit: 'nSubmit',
rp: 'rp',
status: 'status',
accept: 'accept',
vote: 'vote',
star: 'star',
enroll: 'enroll',
rid: 'rid',
rev: 'rev',
attend: 'attend',
journal: {
field: 'journal',
processer: (journal) => {
const r = [];
for (const i of journal) {
r.push({ ...i, pid: pid(i.pid) });
}
return r;
},
detail: {
field: 'detail',
processer: (detail) => {
const r = [];
for (const i of detail) {
r.push({ ...i, pid: pid(i.pid) });
}
return r;
},
},
detail: {
field: 'detail',
processer: (detail) => {
const r = [];
for (const i of detail) {
r.push({ ...i, pid: pid(i.pid) });
}
return r;
},
score: 'score',
time: 'time',
done: 'done',
done_nids: 'doneNids',
done_pids: {
field: 'donePids',
processer: (pids) => pids.map((id) => pid(id)),
},
};
for (const key in doc) {
if (typeof mapper[key] === 'string') {
res[mapper[key]] = doc[key];
} else if (typeof mapper[key] === 'object') {
res[mapper[key].field] = mapper[key].processer(doc[key]);
} else {
console.log('Unknown key:', key);
}
}
return res;
},
score: 'score',
time: 'time',
done: 'done',
done_nids: 'doneNids',
done_pids: {
field: 'donePids',
processer: (pids) => pids.map((id) => pid(id)),
},
},
record: async (doc) => {
const testCases = [];
@@ -211,6 +183,16 @@ const tasks = {
testCases,
};
},
domain: {
_id: '_id',
doc_id: 'docId',
doc_type: 'docType',
owner_uid: 'owner',
name: 'name',
gravatar: 'gravatar',
bulletin: 'bulletin',
pid_counter: 'pidCounter',
},
'fs.files': async (doc) => doc,
'fs.chunks': async (doc) => doc,
file: async (doc) => ({
@@ -227,6 +209,7 @@ const cursor = {
document: (s) => s.collection('document').find({ doc_type: { $ne: 20 } }),
'document.status': (s) => s.collection('document.status').find(),
record: (s) => s.collection('record').find(),
domain: (s) => s.collection('domain').find(),
'fs.files': (s) => s.collection('fs.files').find(),
'fs.chunks': (s) => s.collection('fs.chunks').find(),
file: (s) => s.collection('fs.files').find(),
@@ -255,6 +238,59 @@ async function discussionNode(src, report) {
}
}
async function domainUser(src, report) {
const count = await src.collection('domain.user').find().count();
await report({ progress: 1, message: `domain.user: ${count}` });
const total = Math.floor(count / 50);
for (let i = 0; i <= total; i++) {
const docs = await src.collection('domain.user')
.find().skip(i * 50).limit(50)
.toArray();
for (const doc of docs) {
const mapper = {
_id: '_id',
uid: 'uid',
join_at: 'joinAt',
role: 'role',
num_liked: 'nLike',
num_submit: 'nSubmit',
num_accept: 'nAccept',
num_problems: 'nProblems',
domain_id: 'domainId',
level: 'level',
};
const d = {};
for (const key in doc) {
if (typeof mapper[key] === 'string') {
d[mapper[key]] = doc[key];
} else if (typeof mapper[key] === 'object') {
d[mapper[key].field] = mapper[key].processer(doc[key]);
} else if (mapper[key] === null) {
// Ignore this key
} else {
console.log('Unknown key:', key);
}
}
const docWithoutId = {};
for (const key in d) {
if (key !== '_id') {
docWithoutId[key] = d[key];
}
}
d.push(dst.collection('document').updateOne(
{ _id: d._id },
{
$set: {
...docWithoutId, docType: 0, docId: 1, role: docWithoutId.role || 'default',
},
},
{ upsert: true },
));
}
await report({ progress: Math.round(100 * ((i + 1) / (total + 1))) });
}
}
async function fixPid(report) {
const count = await dst.collection('document').find({ docType: 10 }).count();
await report({ progress: 1, message: `Fix pid: ${count}` });
@@ -298,8 +334,25 @@ async function task(name, src, report) {
const docs = await cursor[name](src).skip(i * 50).limit(50).toArray();
const res = [];
for (const doc of docs) {
const d = await tasks[name](doc);
let d = {};
if (typeof tasks[name].call === 'function') {
d = await tasks[name](doc);
} else {
const mapper = tasks[name];
for (const key in doc) {
if (typeof mapper[key] === 'string') {
res[mapper[key]] = doc[key];
} else if (typeof mapper[key] === 'object') {
res[mapper[key].field] = mapper[key].processer(doc[key]);
} else if (mapper[key] === null) {
// Ignore this key
} else {
console.log('Unknown key:', key);
}
}
}
if (d) {
if (d.rule) d.rated = true; // Hack contest rated field
const docWithoutId = {};
const docWithoutDid = {};
for (const key in d) {
@@ -380,6 +433,7 @@ async function migrateVijos({
for (const i of d) await task(i, src, report);
await fixPid(report);
await discussionNode(src, report);
await domainUser(src, report);
return true;
}
+4 -2
View File
@@ -1,6 +1,6 @@
{
"name": "hydro",
"version": "2.0.0",
"version": "2.0.1",
"main": "hydro/development.js",
"repository": "https://github.com/hydro-dev/Hydro.git",
"author": "masnn",
@@ -35,8 +35,10 @@
"build": "node build/index.js",
"build:development": "node build/development.js",
"build:production": "node build/production.js",
"build:all": "cd ui && yarn && yarn build:production && cd .. && yarn && yarn build:production",
"start": "node hydro/loader.js",
"pack": "pkg .",
"lint": "eslint build hydro module --fix"
"lint": "eslint build hydro module tool --fix"
},
"pkg": {
"scripts": [
+9 -1
View File
@@ -15,7 +15,7 @@
row:false
}) }}
{{ form.form_text({
columns:8,
columns:4,
label:'Title',
name:'title',
placeholder:_('title'),
@@ -23,6 +23,14 @@
autofocus:true,
row:false
}) }}
{{ form.form_checkbox({
columns:4,
label:'Rated',
name:'rated',
placeholder:_('rated'),
value:true,
row:false
}) }}
</div><div class="row">
{{ form.form_text({
columns:4,
+1 -1
View File
@@ -17,7 +17,7 @@
<tr>
<th class="col--description">{{ _('Permissions') }}</th>
{% for role in roles %}
<th class="col--p"><span data-tooltip="{{ role }}">{{ role._id }}</span></th>
<th class="col--p"><span data-tooltip="{{ role._id }}">{{ role._id }}</span></th>
{% endfor %}
</tr>
</thead>
+1 -1
View File
@@ -1,4 +1,4 @@
.page--manage_permission
.page--domain_permission
.col--p
width: 80px
text-align: center
+1 -1
View File
@@ -1,4 +1,4 @@
.page--manage_role
.page--domain_role
.col--checkbox
width: 60px
+1 -1
View File
@@ -1,4 +1,4 @@
.page--manage_user
.page--domain_user
.col--checkbox
width: 60px