From 5d7f57334f51a72547a5ef549f556aced64e212c Mon Sep 17 00:00:00 2001 From: undefined Date: Fri, 8 May 2020 12:02:19 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8D=87=E7=BA=A7queue=E4=B8=BAtask?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 15 ------- build/buildModule.js | 30 ++++++++----- build/index.js | 1 + hydro/handler/judge.js | 72 ++++++------------------------- hydro/handler/problem.js | 7 +-- hydro/handler/record.js | 6 +-- hydro/loader.js | 26 ++++++++--- hydro/model/index.js | 3 ++ hydro/model/record.js | 35 ++++++++++++++- hydro/model/task.js | 30 +++++++++++++ hydro/service/queue.js | 27 ------------ module/html2md/hydro.json | 5 +++ module/html2md/lib.js | 16 +++++++ module/module-pastebin/handler.js | 2 +- 14 files changed, 145 insertions(+), 130 deletions(-) create mode 100644 hydro/model/task.js delete mode 100644 hydro/service/queue.js create mode 100644 module/html2md/hydro.json create mode 100644 module/html2md/lib.js diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cac58e4a8..b491fed7a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,21 +28,6 @@ jobs: with: name: .uibuild path: .uibuild - - run: | - yarn add pkg -D - yarn run pack - - uses: actions/upload-artifact@v1 - with: - name: Hydro_linux_amd64 - path: hydro-linux - - uses: actions/upload-artifact@v1 - with: - name: Hydro_macos_amd64 - path: hydro-macos - - uses: actions/upload-artifact@v1 - with: - name: Hydro_win_amd64.exe - path: hydro-win.exe - uses: elgohr/Publish-Docker-Github-Action@master with: name: hydrooj/hydro:alpine diff --git a/build/buildModule.js b/build/buildModule.js index 0f3e95efe..93500253d 100644 --- a/build/buildModule.js +++ b/build/buildModule.js @@ -35,11 +35,14 @@ const build = async (type) => { }; for (const i of modules) { if (!i.startsWith('.')) { - if (exist(`module/${i}/model.js`)) { - config.entry[`${i}/model`] = root(`module/${i}/model.js`); - } - if (exist(`module/${i}/handler.js`)) { - config.entry[`${i}/handler`] = root(`module/${i}/handler.js`); + const prepare = ['model', 'lib', 'handler', 'service']; + for (const j of prepare) { + if (exist(`module/${i}/${j}.js`)) { + const file = fs.readFileSync(root(`module/${i}/${j}.js`)); + if (file.includes('require')) { + config.entry[`${i}/${j}`] = root(`module/${i}/${j}.js`); + } + } } } } @@ -64,6 +67,17 @@ const build = async (type) => { for (const i of modules) { if (!i.startsWith('.')) { const current = {}; + const prepare = ['model', 'lib', 'handler', 'service']; + for (const j of prepare) { + if (exist(`module/${i}/${j}.js`)) { + const file = fs.readFileSync(root(`module/${i}/${j}.js`)); + if (file.includes('require')) { + current[j] = fs.readFileSync(root(`.build/module/${i}/${j}.js`)).toString(); + } else { + current[j] = fs.readFileSync(root(`module/${i}/${j}.js`)).toString(); + } + } + } if (exist(`module/${i}/locale`)) { const locales = fs.readdirSync(root(`module/${i}/locale`)); const lang = {}; @@ -76,12 +90,6 @@ const build = async (type) => { if (exist(`module/${i}/template`)) { current.template = template(`module/${i}/template`); } - if (exist(`module/${i}/model.js`)) { - current.model = fs.readFileSync(root(`.build/module/${i}/model.js`)).toString(); - } - if (exist(`module/${i}/handler.js`)) { - current.handler = fs.readFileSync(root(`.build/module/${i}/handler.js`)).toString(); - } const m = require(root(`module/${i}/hydro.json`)); current.description = m.description || ''; current.requirements = m.requirements || []; diff --git a/build/index.js b/build/index.js index 3e2b80090..07fa99ad9 100644 --- a/build/index.js +++ b/build/index.js @@ -8,6 +8,7 @@ const ignoreFailure = require('./ignoreFailure'); async function build(type) { if (!['development', 'production'].includes(type)) throw new Error(`Unknown type: ${type}`); ignoreFailure(fs.mkdirSync, root('.build')); + ignoreFailure(fs.mkdirSync, root('.build/module')); const langs = fs.readdirSync(root('locales')); const lang = {}; for (const i of langs) { diff --git a/hydro/handler/judge.js b/hydro/handler/judge.js index 081f1ad38..aed47e278 100644 --- a/hydro/handler/judge.js +++ b/hydro/handler/judge.js @@ -4,14 +4,12 @@ const problem = require('../model/problem'); const builtin = require('../model/builtin'); const contest = require('../model/contest'); const user = require('../model/user'); +const task = require('../model/task'); const bus = require('../service/bus'); -const queue = require('../service/queue'); const { Route, Handler, Connection, ConnectionHandler, } = require('../service/server'); -queue.assert('judge'); - async function _postJudge(rdoc) { const accept = rdoc.status === builtin.STATUS_ACCEPTED; bus.publish('record_change', rdoc); @@ -57,6 +55,7 @@ async function next(body) { rdoc = await record.update(body.rid, $set, $push); bus.publish('record_change', rdoc); } + async function end(body) { let rdoc = await record.get(body.rid); const $set = {}; @@ -92,34 +91,10 @@ class JudgeHandler extends Handler { async get({ check = false }) { if (check) return; const tasks = []; - let rid = await queue.get('judge', false); - while (rid) { - const rdoc = await record.get(rid); // eslint-disable-line no-await-in-loop - let task; - if (!rdoc.input) { - // eslint-disable-next-line no-await-in-loop - const pdoc = await problem.getById(rdoc.pid); - task = { - event: 'judge', - rid, - type: 0, - pid: rdoc.pid, - data: pdoc.data, - lang: rdoc.lang, - code: rdoc.code, - }; - } else { - task = { - event: 'pretest', - rid, - type: 0, - input: rdoc.input, - lang: rdoc.lang, - code: rdoc.code, - }; - } - tasks.push(task); - rid = await queue.get('judge', false); // eslint-disable-line no-await-in-loop + let t = await task.getFirst({ type: 'judge' }); + while (t) { + tasks.push(t); + t = await task.getFirst({ type: 'judge' }); // eslint-disable-line no-await-in-loop } this.response.body = { tasks }; } @@ -133,6 +108,7 @@ class JudgeHandler extends Handler { await end(this.request.body); } } + class JudgeConnectionHandler extends ConnectionHandler { async prepare() { this.checkPerm(PERM_JUDGE); @@ -143,40 +119,16 @@ class JudgeConnectionHandler extends ConnectionHandler { else if (msg.key === 'end') { await end({ judger: this.user._id, ...msg }); this.processing = null; - const rid = await queue.get('judge'); - const rdoc = await record.get(rid); - let task; - if (!rdoc.input) { - // eslint-disable-next-line no-await-in-loop - const pdoc = await problem.getById(rdoc.pid); - task = { - event: 'judge', - rid, - type: 0, - pid: rdoc.pid, - data: pdoc.data, - lang: rdoc.lang, - code: rdoc.code, - }; - } else { - task = { - event: 'pretest', - rid, - type: 0, - input: rdoc.input, - lang: rdoc.lang, - code: rdoc.code, - }; - } - this.send({ task }); - this.processing = task.rid; + const t = await task.getFirst({ type: 'judge' }); + this.send({ task: t }); + this.processing = t; } } async cleanup() { if (this.processing) { await record.reset(this.processing); - queue.push('judge', this.processing); + task.add(this.processing); } } } @@ -187,5 +139,5 @@ async function apply() { } global.Hydro.handler.judge = module.exports = { - JudgeHandler, JudgeConnectionHandler, apply, + JudgeHandler, JudgeConnectionHandler, apply, next, end, }; diff --git a/hydro/handler/problem.js b/hydro/handler/problem.js index 559baa653..8d811d232 100644 --- a/hydro/handler/problem.js +++ b/hydro/handler/problem.js @@ -11,7 +11,6 @@ const { Route, Connection, Handler, ConnectionHandler, } = require('../service/server'); const gridfs = require('../service/gridfs'); -const queue = require('../service/queue'); const { NoProblemError, ProblemDataNotFoundError, BadRequestError, SolutionNotFoundError, @@ -25,8 +24,6 @@ const { PERM_REPLY_PROBLEM_SOLUTION, PERM_LOGGEDIN, } = require('../permission'); -queue.assert('judge'); - class ProblemHandler extends Handler { async _prepare() { this.checkPerm(PERM_VIEW_PROBLEM); @@ -122,7 +119,7 @@ class ProblemSubmitHandler extends ProblemDetailHandler { const rid = await record.add({ uid: this.uid, lang, code, pid: this.pdoc._id, }); - await queue.push('judge', rid); + await record.judge(rid); this.user.nSubmit++; this.response.body = { rid }; this.response.redirect = `/r/${rid}`; @@ -135,7 +132,7 @@ class ProblemPretestHandler extends ProblemDetailHandler { const rid = await record.add({ uid: this.uid, lang, code, pid: this.pdoc._id, input, }); - await queue.push('judge', rid); + await record.judge(rid); this.response.body = { rid }; } } diff --git a/hydro/handler/record.js b/hydro/handler/record.js index 488acc7d5..58e9d24e7 100644 --- a/hydro/handler/record.js +++ b/hydro/handler/record.js @@ -8,7 +8,6 @@ const record = require('../model/record'); const contest = require('../model/contest'); const user = require('../model/user'); const bus = require('../service/bus'); -const queue = require('../service/queue'); const { Route, Handler, Connection, ConnectionHandler, } = require('../service/server'); @@ -69,10 +68,7 @@ class RecordRejudgeHandler extends Handler { async post({ rid }) { this.checkPerm(PERM_REJUDGE); const rdoc = await record.get(rid); - if (rdoc) { - await record.reset(rid); - await queue.push('judge', rid); - } + if (rdoc) await record.rejudge(rid); this.back(); } } diff --git a/hydro/loader.js b/hydro/loader.js index f2250a0a4..eec96535e 100644 --- a/hydro/loader.js +++ b/hydro/loader.js @@ -26,8 +26,8 @@ async function handler() { for (const i of active) { if (i.handler) { console.time(`Handler init: ${i.id}`); - const module = {}; // eslint-disable-line no-unused-vars - const exports = {}; // eslint-disable-line no-unused-vars + const exports = {}; + const module = { exports }; // eslint-disable-line no-unused-vars eval(i.handler); console.timeEnd(`Handler init: ${i.id}`); } @@ -56,15 +56,28 @@ async function model() { for (const i of active) { if (i.model) { console.time(`Model init: ${i.id}`); - const module = {}; const exports = {}; // eslint-disable-line no-unused-vars - eval(i.handler); + const module = { exports }; + eval(i.model); if ((module.exports || {}).index) await module.exports.index(); console.timeEnd(`Model init: ${i.id}`); } } } +async function service() { + for (const i of active) { + if (i.service) { + console.time(`Service init: ${i.id}`); + const exports = {}; // eslint-disable-line no-unused-vars + const module = { exports }; + eval(i.service); + if ((module.exports || {}).init) await module.exports.init(); + console.timeEnd(`Service init: ${i.id}`); + } + } +} + async function load() { global.Hydro = { handler: {}, @@ -99,8 +112,8 @@ async function load() { for (const i of builtinLib) require(`./lib/${i}`); await locale(); require('./service/gridfs'); - require('./service/queue'); const server = require('./service/server'); + await service(); const builtinModel = [ 'blacklist', 'builtin', 'contest', 'discussion', 'message', 'opcount', 'problem', 'record', 'setting', 'solution', @@ -122,6 +135,9 @@ async function load() { } const notfound = require('./handler/notfound'); await notfound.apply(); + for (const i in global.Hydro.service) { + if (global.Hydro.service[i].postInit) await global.Hydro.service[i].postInit(); + } server.start(); } diff --git a/hydro/model/index.js b/hydro/model/index.js index 2b3efe767..0765ec387 100644 --- a/hydro/model/index.js +++ b/hydro/model/index.js @@ -10,4 +10,7 @@ module.exports = { token: require('./token'), training: require('./training'), user: require('./user'), + task: require('./task'), + message: require('./message'), + setting: require('./setting'), }; diff --git a/hydro/model/record.js b/hydro/model/record.js index 3bb5ab59d..f85063744 100644 --- a/hydro/model/record.js +++ b/hydro/model/record.js @@ -1,7 +1,9 @@ const _ = require('lodash'); const { ObjectID } = require('bson'); -const { RecordNotFoundError } = require('../error'); const { STATUS_WAITING } = require('./builtin').STATUS; +const task = require('./task'); +const problem = require('./problem'); +const { RecordNotFoundError } = require('../error'); const db = require('../service/db.js'); const coll = db.collection('record'); @@ -75,6 +77,35 @@ function getUserInProblemMulti(uid, pid) { return coll.find({ owner: uid, pid }); } +async function judge(rid) { + const rdoc = await get(rid); + const pdoc = await problem.getById(rdoc.pid); + await task.add({ + event: 'judge', + rid, + type: 0, + pid: rdoc.pid, + data: pdoc.data, + lang: rdoc.lang, + code: rdoc.code, + }); +} + +async function rejudge(rid) { + await reset(rid); + const rdoc = await get(rid); + const pdoc = await problem.getById(rdoc.pid); + await task.add({ + event: 'judge', + rid, + type: 0, + pid: rdoc.pid, + data: pdoc.data, + lang: rdoc.lang, + code: rdoc.code, + }); +} + module.exports = { add, get, @@ -84,4 +115,6 @@ module.exports = { reset, getList, getUserInProblemMulti, + judge, + rejudge, }; diff --git a/hydro/model/task.js b/hydro/model/task.js new file mode 100644 index 000000000..0769e0d9b --- /dev/null +++ b/hydro/model/task.js @@ -0,0 +1,30 @@ +const db = require('../service/db.js'); + +const coll = db.collection('task'); + +async function add(task) { + const res = await coll.insertOne(task); + return res.insertedId; +} + +function get(_id) { + return coll.findOne({ _id }); +} + +function count(query) { + return coll.find(query).count(); +} + +function del(_id) { + return coll.deleteOne({ _id }); +} + +async function getFirst(query) { + const res = await coll.find(query).sort('_id', 1).limit(1).toArray(); + await coll.deleteOne({ _id: res[0]._id }); + return res[0]; +} + +module.exports = { + add, get, del, count, getFirst, +}; diff --git a/hydro/service/queue.js b/hydro/service/queue.js deleted file mode 100644 index 8fd1e0972..000000000 --- a/hydro/service/queue.js +++ /dev/null @@ -1,27 +0,0 @@ -const q = {}; -module.exports = { - async connect() { }, // eslint-disable-line no-empty-function - async assert(queue) { - q[queue] = q[queue] || []; - }, - async push(queue, data) { - q[queue].push(data); - }, - async get(queue, wait = true) { - if (wait) { - while (!q[queue].length) { - await new Promise((resolve) => { // eslint-disable-line no-await-in-loop - setTimeout(resolve, 100); - }); - } - const data = q[queue][0]; - q[queue].splice(0, 1); - return data; - } - if (!q[queue].length) return null; - - const data = q[queue][0]; - q[queue].splice(0, 1); - return data; - }, -}; diff --git a/module/html2md/hydro.json b/module/html2md/hydro.json new file mode 100644 index 000000000..3b6df7bd9 --- /dev/null +++ b/module/html2md/hydro.json @@ -0,0 +1,5 @@ +{ + "name": "html2md", + "version": "0.0.1", + "description": "Convert html to markdown" +} \ No newline at end of file diff --git a/module/html2md/lib.js b/module/html2md/lib.js new file mode 100644 index 000000000..d1551c62c --- /dev/null +++ b/module/html2md/lib.js @@ -0,0 +1,16 @@ +global.Hydro.lib.html2md = module.exports = function html2md(str) { + return str.replace(/

/gmi, '').replace(/<\/p>/gmi, ' ') + .replace(/

/gmi, '# ') + .replace(/

/gmi, '## ') + .replace(/

/gmi, '### ') + .replace(/

/gmi, '#### ') + .replace(/

/gmi, '##### ') + .replace(/
/gmi, '###### ') + .replace(/<\/?code>/, '\n```\n') + .replace(/.*?<\/img>/gmi, (substr) => { + const RE_IMG = /(.*?)<\/img>/i; + const res = RE_IMG.exec(substr); + return `[${res[2]}](${res[1]})`; + }) + .replace(/<\/h[0-6]>/, '\n'); +}; diff --git a/module/module-pastebin/handler.js b/module/module-pastebin/handler.js index 36205fb07..65acdc1cf 100644 --- a/module/module-pastebin/handler.js +++ b/module/module-pastebin/handler.js @@ -47,6 +47,6 @@ async function apply() { nav('/paste/create', 'pastebin', 'pastebin', PERM_LOGGEDIN); } -global.Hydro.handler.paste = module.exports = { +global.Hydro.handler.pastebin = module.exports = { PasteCreateHandler, PasteShowHandler, apply, };