From 6134a16f8c6ccf566194f56bb2ca7637d4a51bb6 Mon Sep 17 00:00:00 2001 From: undefined Date: Fri, 8 May 2020 12:28:04 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B5=8B=E8=AF=95=EF=BC=9Ajudger=E5=B0=81?= =?UTF-8?q?=E8=A3=85=E4=B8=BA=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/buildModule.js | 14 ++- build/webpack.js | 4 +- hydro/loader.js | 112 +++++++++++++++++----- hydro/model/record.js | 6 +- module/judger/hydro.json | 5 + module/judger/service.js | 165 ++++++++++++++++++++++++++++++++ module/sandbox-linux/hydro.json | 6 ++ module/sandbox-linux/service.js | 23 +++++ 8 files changed, 305 insertions(+), 30 deletions(-) create mode 100644 module/judger/hydro.json create mode 100644 module/judger/service.js create mode 100644 module/sandbox-linux/hydro.json create mode 100644 module/sandbox-linux/service.js diff --git a/build/buildModule.js b/build/buildModule.js index 93500253d..0d02756cf 100644 --- a/build/buildModule.js +++ b/build/buildModule.js @@ -30,7 +30,9 @@ const build = async (type) => { module: {}, plugins: [ new webpack.ProgressPlugin(), - new FriendlyErrorsPlugin(), + new FriendlyErrorsPlugin({ + clearConsole: false, + }), ], }; for (const i of modules) { @@ -90,11 +92,19 @@ const build = async (type) => { if (exist(`module/${i}/template`)) { current.template = template(`module/${i}/template`); } + if (exist(`module/${i}/file`)) { + const files = fs.readdirSync(root(`module/${i}/file`)); + current.file = {}; + for (const file of files) { + current.file[file] = fs.readFileSync(root(`module/${i}/file/${file}`)).toString('base64'); + } + } const m = require(root(`module/${i}/hydro.json`)); current.description = m.description || ''; current.requirements = m.requirements || []; current.version = m.version || 'unknown'; - const data = zlib.gzipSync(Buffer.from(yaml.safeDump(current)), { level: 3 }); + if (m.os) current.os = m.os; + const data = zlib.gzipSync(Buffer.from(yaml.safeDump(current)), { level: -1 }); fs.writeFileSync(root(`.build/module/${i}.hydro`), data); } } diff --git a/build/webpack.js b/build/webpack.js index 871a1c71d..63b2ae09d 100644 --- a/build/webpack.js +++ b/build/webpack.js @@ -34,7 +34,9 @@ const build = async (type) => { module: {}, plugins: [ new webpack.ProgressPlugin(), - new FriendlyErrorsPlugin(), + new FriendlyErrorsPlugin({ + clearConsole: false, + }), ], }; const compiler = webpack(config); diff --git a/hydro/loader.js b/hydro/loader.js index eec96535e..ea5f01391 100644 --- a/hydro/loader.js +++ b/hydro/loader.js @@ -2,6 +2,7 @@ /* eslint-disable no-await-in-loop */ /* eslint-disable no-eval */ const fs = require('fs'); +const os = require('os'); const zlib = require('zlib'); const path = require('path'); const yaml = require('js-yaml'); @@ -9,6 +10,14 @@ const yaml = require('js-yaml'); function root(name) { return path.resolve(process.cwd(), name); } +function ensureDir(dir) { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir); + } else if (!fs.statSync(dir).isDirectory()) { + fs.unlinkSync(dir); + fs.mkdirSync(dir); + } +} const active = []; @@ -16,8 +25,25 @@ async function preload() { const files = fs.readdirSync(root('.build/module')); for (const file of files) { if (file.endsWith('.hydro')) { - const f = fs.readFileSync(root(`.build/module/${file}`)); - active.push({ ...yaml.safeLoad(zlib.gunzipSync(f)), id: file.split('.')[0] }); + try { + const f = fs.readFileSync(root(`.build/module/${file}`)); + const m = { ...yaml.safeLoad(zlib.gunzipSync(f)), id: file.split('.')[0] }; + if (m.os) { + if (!m.os.includes(os.platform())) throw new Error('Unsupported OS'); + } + if (m.file) { + m.files = {}; + ensureDir(path.resolve(os.tmpdir(), 'hydro', file)); + for (const n in m.file) { + const e = path.resolve(os.tmpdir(), 'hydro', file, n); + fs.writeFileSync(e, Buffer.from(m.file[n], 'base64'), { mode: 755 }); + m.files[n] = e; + } + } + active.push(m); + } catch (e) { + console.error(`Module Load Fail: ${file}`); + } } } } @@ -25,11 +51,16 @@ async function preload() { async function handler() { for (const i of active) { if (i.handler) { - console.time(`Handler init: ${i.id}`); - const exports = {}; - const module = { exports }; // eslint-disable-line no-unused-vars - eval(i.handler); - console.timeEnd(`Handler init: ${i.id}`); + try { + console.time(`Handler init: ${i.id}`); + const exports = {}; + // eslint-disable-next-line no-unused-vars + const module = { exports, file: i.files || {} }; + eval(i.handler); + console.timeEnd(`Handler init: ${i.id}`); + } catch (e) { + console.error(`Handler Load Fail: ${i.id}`); + } } } } @@ -37,8 +68,12 @@ async function handler() { async function locale() { for (const i of active) { if (i.locale) { - global.Hydro.lib.i18n(i.locale); - console.log(`Locale init: ${i.id}`); + try { + global.Hydro.lib.i18n(i.locale); + console.log(`Locale init: ${i.id}`); + } catch (e) { + console.error(`Locale Load Fail: ${i.id}`); + } } } } @@ -46,8 +81,12 @@ async function locale() { async function template() { for (const i of active) { if (i.template) { - Object.assign(global.Hydro.template, i.template); - console.log(`Template init: ${i.id}`); + try { + Object.assign(global.Hydro.template, i.template); + console.log(`Template init: ${i.id}`); + } catch (e) { + console.error(`Template Load Fail: ${i.id}`); + } } } } @@ -55,12 +94,33 @@ async function template() { async function model() { for (const i of active) { if (i.model) { - console.time(`Model init: ${i.id}`); - const exports = {}; // eslint-disable-line no-unused-vars - const module = { exports }; - eval(i.model); - if ((module.exports || {}).index) await module.exports.index(); - console.timeEnd(`Model init: ${i.id}`); + try { + console.time(`Model init: ${i.id}`); + const exports = {}; + const module = { exports, file: i.files || {} }; + eval(i.model); + if ((module.exports || {}).index) await module.exports.index(); + console.timeEnd(`Model init: ${i.id}`); + } catch (e) { + console.error(`Model Load Fail: ${i.id}`); + } + } + } +} + +async function lib() { + for (const i of active) { + if (i.lib) { + try { + console.time(`Lib init: ${i.id}`); + const exports = {}; + // eslint-disable-next-line no-unused-vars + const module = { exports, file: i.files || {} }; + eval(i.lib); + console.timeEnd(`Lib init: ${i.id}`); + } catch (e) { + console.error(`Lib Load Fail: ${i.id}`); + } } } } @@ -68,17 +128,22 @@ async function model() { async function service() { for (const i of active) { if (i.service) { - console.time(`Service init: ${i.id}`); - const exports = {}; // eslint-disable-line no-unused-vars - const module = { exports }; - eval(i.service); - if ((module.exports || {}).init) await module.exports.init(); - console.timeEnd(`Service init: ${i.id}`); + try { + console.time(`Service init: ${i.id}`); + const exports = {}; // eslint-disable-line no-unused-vars + const module = { exports }; + eval(i.service); + if ((module.exports || {}).init) await module.exports.init(); + console.timeEnd(`Service init: ${i.id}`); + } catch (e) { + console.error(`Service Load Fail: ${i.id}`); + } } } } async function load() { + ensureDir(path.resolve(os.tmpdir(), 'hydro')); global.Hydro = { handler: {}, service: {}, @@ -110,6 +175,7 @@ async function load() { 'template', 'validator', 'nav', ]; for (const i of builtinLib) require(`./lib/${i}`); + await lib(); await locale(); require('./service/gridfs'); const server = require('./service/server'); diff --git a/hydro/model/record.js b/hydro/model/record.js index f85063744..809831ca1 100644 --- a/hydro/model/record.js +++ b/hydro/model/record.js @@ -81,9 +81,8 @@ async function judge(rid) { const rdoc = await get(rid); const pdoc = await problem.getById(rdoc.pid); await task.add({ - event: 'judge', + type: 'judge', rid, - type: 0, pid: rdoc.pid, data: pdoc.data, lang: rdoc.lang, @@ -96,9 +95,8 @@ async function rejudge(rid) { const rdoc = await get(rid); const pdoc = await problem.getById(rdoc.pid); await task.add({ - event: 'judge', + type: 'judge', rid, - type: 0, pid: rdoc.pid, data: pdoc.data, lang: rdoc.lang, diff --git a/module/judger/hydro.json b/module/judger/hydro.json new file mode 100644 index 000000000..ff1c9906e --- /dev/null +++ b/module/judger/hydro.json @@ -0,0 +1,5 @@ +{ + "name": "judger", + "version": "1.0.0", + "description": "HydroJudger" +} \ No newline at end of file diff --git a/module/judger/service.js b/module/judger/service.js new file mode 100644 index 000000000..86f905850 --- /dev/null +++ b/module/judger/service.js @@ -0,0 +1,165 @@ +// Webpack Required +const fs = require('fs'); +const path = require('path'); +const child = require('child_process'); + +async function postInit() { + const { mkdirp, rmdir, compilerText } = require('./HydroJudger/judger/utils'); + const log = require('./HydroJudger/judger/log'); + const { CACHE_DIR, TEMP_DIR } = require('./HydroJudger/judger/config'); + const tmpfs = require('./HydroJudger/judger/tmpfs'); + const { FormatError, CompileError, SystemError } = require('./HydroJudger/judger/error'); + const { STATUS_COMPILE_ERROR, STATUS_SYSTEM_ERROR } = require('./HydroJudger/judger/status'); + const readCases = require('./HydroJudger/judger/cases'); + const judger = require('./HydroJudger/judger/judger'); + + const fsp = fs.promises; + const { problem, task } = global.Hydro.model; + const { judge } = global.Hydro.handler; + + async function problemData(pid, savePath) { + const tmpFilePath = path.resolve(CACHE_DIR, `download_${pid}`); + const res = await problem.getData(pid); + const w = await fs.createWriteStream(tmpFilePath); + res.data.pipe(w); + await new Promise((resolve, reject) => { + w.on('finish', resolve); + w.on('error', reject); + }); + mkdirp(path.dirname(savePath)); + await new Promise((resolve, reject) => { + child.exec(`unzip ${tmpFilePath} -d ${savePath}`, (e) => { + if (e) reject(e); + else resolve(); + }); + }); + await fsp.unlink(tmpFilePath); + await this.processData(savePath).catch(); + return savePath; + } + + async function cacheOpen(pid, version) { + const filePath = path.join(CACHE_DIR, pid); + if (fs.existsSync(filePath)) { + let ver; + try { + ver = fs.readFileSync(path.join(filePath, 'version')).toString(); + } catch (e) { /* ignore */ } + if (version === ver) return filePath; + rmdir(filePath); + } + mkdirp(filePath); + await problemData(pid, filePath); + fs.writeFileSync(path.join(filePath, 'version'), version); + return filePath; + } + + function getNext(that) { + that.nextId = 1; + that.nextWaiting = []; + return (data, id) => { + data.key = 'next'; + data.rid = that.rid; + if (id) { + if (id === that.nextId) { + judge.next(data); + that.nextId++; + let t = true; + while (t) { + t = false; + for (const i in that.nextWaiting) { + if (that.nextId === that.nextWaiting[i].id) { + judge.next(that.nextWaiting[i].data); + that.nextId++; + that.nextWaiting.splice(i, 1); + t = true; + } + } + } + } else that.nextWaiting.push({ data, id }); + } else judge.next(data); + }; + } + + function getEnd(rid) { + return (data) => { + data.key = 'end'; + data.rid = rid; + log.log({ + status: data.status, + score: data.score, + time_ms: data.time_ms, + memory_kb: data.memory_kb, + }); + judge.end(data); + }; + } + + class JudgeTask { + constructor(request) { + this.stat = {}; + this.stat.receive = new Date(); + this.request = request; + } + + async handle() { + try { + this.stat.handle = new Date(); + this.event = this.request.event; + this.pid = this.request.pid; + this.rid = this.request.rid; + this.lang = this.request.lang; + this.code = this.request.code; + this.data = this.request.data; + this.next = getNext(this); + this.end = getEnd(this.rid); + this.tmpdir = path.resolve(TEMP_DIR, 'tmp', this.rid); + this.clean = []; + mkdirp(this.tmpdir); + tmpfs.mount(this.tmpdir, '64m'); + log.submission(`${this.rid}`, { pid: this.pid }); + if (!this.event) await this.submission(); + else throw new SystemError(`Unsupported type: ${this.type}`); + } catch (e) { + if (e instanceof CompileError) { + this.next({ compiler_text: compilerText(e.stdout, e.stderr) }); + this.end({ + status: STATUS_COMPILE_ERROR, score: 0, time_ms: 0, memory_kb: 0, + }); + } else if (e instanceof FormatError) { + this.next({ judge_text: `${e.message}\n${JSON.stringify(e.params)}` }); + this.end({ + status: STATUS_SYSTEM_ERROR, score: 0, time_ms: 0, memory_kb: 0, + }); + } else { + log.error(e); + this.next({ judge_text: `${e.message}\n${e.stack}\n${JSON.stringify(e.params)}` }); + this.end({ + status: STATUS_SYSTEM_ERROR, score: 0, time_ms: 0, memory_kb: 0, + }); + } + } + for (const clean of this.clean) await clean().catch(); + tmpfs.umount(this.tmpdir); + await rmdir(this.tmpdir); + } + + async submission() { + this.stat.cache_start = new Date(); + this.folder = await cacheOpen(this.pid, this.data); + this.stat.read_cases = new Date(); + this.config = await readCases( + this.folder, + { detail: true }, + { next: this.next }, + ); + this.stat.judge = new Date(); + await judger[this.config.type || 'default'].judge(this); + } + } + task.consume((t) => { + (new JudgeTask(t)).handle(); + }); +} + +module.exports = { postInit }; diff --git a/module/sandbox-linux/hydro.json b/module/sandbox-linux/hydro.json new file mode 100644 index 000000000..f4061dfe3 --- /dev/null +++ b/module/sandbox-linux/hydro.json @@ -0,0 +1,6 @@ +{ + "name": "sandbox-linux", + "version": "1.0.0", + "os": "Linux", + "description": "Sandbox service for linux" +} \ No newline at end of file diff --git a/module/sandbox-linux/service.js b/module/sandbox-linux/service.js new file mode 100644 index 000000000..ff600f5e8 --- /dev/null +++ b/module/sandbox-linux/service.js @@ -0,0 +1,23 @@ +const os = require('os'); +const child = require('child_process'); + +const p = child.spawn(path.resolve(os.tmpdir(), 'hydro', 'sandbox-linux', 'executorserver', ['-silent']); +if (!p.stdout) throw new Error('Cannot start executorserver'); +else { + p.stdout.on('data', (data) => { + const s = data.toString(); + console.log(s.substr(0, s.length - 1)); + }); + p.stderr.on('data', (data) => { + const s = data.toString(); + console.log(s.substr(0, s.length - 1)); + }); +} +p.on('error', (error) => console.error(error)); + +async function stop() { + p.emit('exit'); +} + + +module.exports = { stop };