From 26f780ddeecd0fc61f86fcf6c0016ec5f0f75c6b Mon Sep 17 00:00:00 2001 From: masnn Date: Sat, 13 Jun 2020 12:26:04 +0800 Subject: [PATCH] multi entry --- hydro/entry/common.js | 158 ++++++++++++++++ hydro/entry/master.js | 89 +++++++++ hydro/{service => entry}/setup.js | 8 +- hydro/{ => entry}/unzip.js | 4 +- hydro/entry/worker.js | 59 ++++++ hydro/loader.js | 292 +----------------------------- hydro/model/contest.js | 2 +- module/migrate-vijos/handler.js | 2 +- 8 files changed, 324 insertions(+), 290 deletions(-) create mode 100644 hydro/entry/common.js create mode 100644 hydro/entry/master.js rename hydro/{service => entry}/setup.js (97%) rename hydro/{ => entry}/unzip.js (98%) create mode 100644 hydro/entry/worker.js diff --git a/hydro/entry/common.js b/hydro/entry/common.js new file mode 100644 index 000000000..88f3acf51 --- /dev/null +++ b/hydro/entry/common.js @@ -0,0 +1,158 @@ +/* eslint-disable no-eval */ +const os = require('os'); +const fs = require('fs'); + +const builtinLib = [ + 'axios', 'download', 'i18n', 'mail', 'markdown', + 'md5', 'misc', 'paginate', 'hash.hydro', 'rank', + 'template', 'validator', 'nav', 'sysinfo', +]; + +const builtinModel = [ + 'builtin', 'document', 'domain', 'blacklist', 'opcount', + 'setting', 'token', 'user', 'problem', 'record', + 'contest', 'message', 'solution', 'training', 'file', + 'discussion', 'system', +]; + +const builtinHandler = [ + 'home', 'problem', 'record', 'judge', 'user', + 'contest', 'training', 'discussion', 'manage', 'import', + 'misc', 'homework', 'domain', 'wiki', +]; + +const builtinScript = [ + 'install', 'uninstall', 'rating', 'recalcRating', 'register', + 'blacklist', 'setSuperadmin', +]; + +async function handler(pending, fail) { + for (const i of pending) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/handler.js`; + if (fs.existsSync(p) && !fail.includes(i)) { + try { + console.log(`Handler init: ${i}`); + console.time(`Handler init: ${i}`); + eval('require')(p); + console.timeEnd(`Handler init: ${i}`); + } catch (e) { + fail.push(i); + console.error(`Handler Load Fail: ${i}`); + } + } + } +} + +async function locale(pending, fail) { + for (const i of pending) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/locale.json`; + if (fs.existsSync(p) && !fail.includes(i)) { + try { + global.Hydro.lib.i18n(eval('require')(p)); + console.log(`Locale init: ${i}`); + } catch (e) { + fail.push(i); + console.error(`Locale Load Fail: ${i}`); + } + } + } +} + +async function template(pending, fail) { + for (const i of pending) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/template.json`; + if (fs.existsSync(p) && !fail.includes(i)) { + try { + Object.assign(global.Hydro.template, eval('require')(p)); + console.log(`Template init: ${i}`); + } catch (e) { + fail.push(i); + console.error(`Template Load Fail: ${i}`); + } + } + } +} + +async function model(pending, fail) { + for (const i of pending) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/model.js`; + if (fs.existsSync(p) && !fail.includes(i)) { + try { + console.log(`Model init: ${i}`); + console.time(`Model init: ${i}`); + eval('require')(p); + console.timeEnd(`Model init: ${i}`); + } catch (e) { + fail.push(i); + console.error(`Model Load Fail: ${i}`); + } + } + } +} + +async function lib(pending, fail) { + for (const i of pending) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/lib.js`; + if (fs.existsSync(p) && !fail.includes(i)) { + try { + console.log(`Lib init: ${i}`); + console.time(`Lib init: ${i}`); + eval('require')(p); + console.timeEnd(`Lib init: ${i}`); + } catch (e) { + fail.push(i); + console.error(`Lib Load Fail: ${i}`); + } + } + } +} + +async function service(pending, fail) { + for (const i of pending) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/service.js`; + if (fs.existsSync(p) && !fail.includes(i)) { + try { + console.log(`Service init: ${i}`); + console.time(`Service init: ${i}`); + eval('require')(p); + console.timeEnd(`Service init: ${i}`); + } catch (e) { + fail.push(i); + console.error(`Service Load Fail: ${i}`); + console.error(e); + } + } + } +} + +async function script(pending, fail, active) { + for (const i of pending) { + const p = `${os.tmpdir()}/hydro/tmp/${i}/script.js`; + if (fs.existsSync(p) && !fail.includes(i)) { + try { + console.time(`Script init: ${i}`); + eval('require')(p); + console.timeEnd(`Script init: ${i}`); + } catch (e) { + fail.push(i); + console.error(`Script Load Fail: ${i}`); + console.error(e); + } + } + active.push(i); + } +} + +module.exports = { + builtinLib, + builtinModel, + builtinHandler, + builtinScript, + handler, + lib, + locale, + model, + script, + service, + template, +}; diff --git a/hydro/entry/master.js b/hydro/entry/master.js new file mode 100644 index 000000000..b8063490d --- /dev/null +++ b/hydro/entry/master.js @@ -0,0 +1,89 @@ +/* eslint-disable no-await-in-loop */ +/* eslint-disable import/no-dynamic-require */ +const os = require('os'); +const fs = require('fs'); +const path = require('path'); +const { + lib, service, model, handler, script, + builtinLib, builtinScript, builtinHandler, builtinModel, +} = require('./common'); + +function ensureDir(dir) { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir); + } else if (!fs.statSync(dir).isDirectory()) { + fs.unlinkSync(dir); + fs.mkdirSync(dir); + } +} + +async function load(call) { + ensureDir(path.resolve(os.tmpdir(), 'hydro')); + ensureDir(path.resolve(os.tmpdir(), 'hydro', 'tmp')); + ensureDir(path.resolve(os.tmpdir(), 'hydro', 'public')); + // TODO better run in another process as this needs lots of memory + await call({ entry: 'unzip', newProcess: true }); + let pending = await require('../lib/hpm').getInstalled(); + const fail = []; + const active = []; + require('../lib/i18n'); + require('../utils'); + require('../error'); + require('../permission'); + try { + require('../options'); + } catch (e) { + await call({ entry: 'install' }); + require('../options'); + } + const bus = require('../service/bus'); + await new Promise((resolve) => { + const h = () => { + console.log('Database connected'); + bus.unsubscribe(['system_database_connected'], h); + resolve(); + }; + bus.subscribe(['system_database_connected'], h); + require('../service/db'); + }); + for (const i of builtinLib) require(`../lib/${i}`); + await lib(pending, fail); + require('../service/gridfs'); + require('../service/monitor'); + const server = require('../service/server'); + await server.prepare(); + await service(pending, fail); + for (const i of builtinModel) require(`../model/${i}`); + for (const i of builtinHandler) require(`../handler/${i}`); + await model(pending, fail); + for (const m in global.Hydro.model) { + if (global.Hydro.model[m].ensureIndexes) { + await global.Hydro.model[m].ensureIndexes(); + } + } + const system = require('../model/system'); + const dbVer = await system.get('db.ver'); + if (dbVer !== 1) { + const ins = require('../script/install'); + await ins.run({ username: 'Root', password: 'rootroot' }); + } + await handler(pending, fail); + for (const i in global.Hydro.handler) { + await global.Hydro.handler[i](); + } + for (const i in global.Hydro.service) { + if (global.Hydro.service[i].postInit) { + try { + await global.Hydro.service[i].postInit(); + } catch (e) { + console.error(e); + } + } + } + for (const i of builtinScript) require(`../script/${i}`); + await script(pending, fail, active); + pending = []; + return { fail, active }; +} + +module.exports = load; diff --git a/hydro/service/setup.js b/hydro/entry/setup.js similarity index 97% rename from hydro/service/setup.js rename to hydro/entry/setup.js index 4bda33c58..b9842005f 100644 --- a/hydro/service/setup.js +++ b/hydro/entry/setup.js @@ -93,12 +93,12 @@ function render(name) { }); } -async function setup() { +async function load() { const app = new Koa(); const server = http.createServer(app.callback()); const router = new Router(); app.keys = ['Hydro']; - app.use(cache(path.join(os.tmpdir(), 'hydro', 'builtin'), { + app.use(cache(path.join(os.tmpdir(), 'hydro', 'public'), { maxAge: 365 * 24 * 60 * 60, })); app.use(Body()); @@ -156,7 +156,7 @@ async function setup() { else resolve(); }); }); - console.log('Done! Restarting...'); + return 'Done! Restarting...'; } -module.exports = { setup }; +module.exports = load; diff --git a/hydro/unzip.js b/hydro/entry/unzip.js similarity index 98% rename from hydro/unzip.js rename to hydro/entry/unzip.js index 82dd1487d..efd3f731a 100644 --- a/hydro/unzip.js +++ b/hydro/entry/unzip.js @@ -30,7 +30,7 @@ for (const i of moduleRoots) { break; } } -function exec() { +async function load() { const files = fs.readdirSync(moduleRoot); const t = ['service', 'lib', 'model', 'handler', 'script']; ensureDir(`${os.tmpdir()}/hydro`); @@ -111,4 +111,4 @@ function exec() { } } -module.exports = exec; +module.exports = load; diff --git a/hydro/entry/worker.js b/hydro/entry/worker.js new file mode 100644 index 000000000..211ef7171 --- /dev/null +++ b/hydro/entry/worker.js @@ -0,0 +1,59 @@ +/* eslint-disable no-await-in-loop */ +/* eslint-disable import/no-dynamic-require */ +const { + locale, template, lib, service, model, handler, script, + builtinLib, builtinScript, builtinHandler, builtinModel, +} = require('./common'); + +async function load() { + let pending = await require('../lib/hpm').getInstalled(); + const fail = []; + const active = []; + require('../lib/i18n'); + require('../utils'); + require('../error'); + require('../permission'); + require('../options'); + await Promise.all([locale(pending, fail), template(pending, fail)]); + const bus = require('../service/bus'); + await new Promise((resolve) => { + const h = () => { + console.log('Database connected'); + bus.unsubscribe(['system_database_connected'], h); + resolve(); + }; + bus.subscribe(['system_database_connected'], h); + require('../service/db'); + }); + for (const i of builtinLib) require(`../lib/${i}`); + await lib(pending, fail); + require('../service/gridfs'); + const server = require('../service/server'); + await server.prepare(); + await service(pending, fail); + for (const i of builtinModel) require(`../model/${i}`); + for (const i of builtinHandler) require(`../handler/${i}`); + await model(pending, fail); + await handler(pending, fail); + for (const i in global.Hydro.handler) { + await global.Hydro.handler[i](); + } + const notfound = require('../handler/notfound'); + await notfound(); + for (const i in global.Hydro.service) { + if (global.Hydro.service[i].postInit) { + try { + await global.Hydro.service[i].postInit(); + } catch (e) { + console.error(e); + } + } + } + for (const i of builtinScript) require(`../script/${i}`); + await script(pending, fail, active); + pending = []; + await server.start(); + return { active, fail }; +} + +module.exports = load; diff --git a/hydro/loader.js b/hydro/loader.js index 10521e3be..a02e0326e 100644 --- a/hydro/loader.js +++ b/hydro/loader.js @@ -1,285 +1,9 @@ /* eslint-disable import/no-dynamic-require */ /* eslint-disable no-await-in-loop */ /* eslint-disable no-eval */ -const fs = require('fs'); -const os = require('os'); -const path = require('path'); const cluster = require('cluster'); const numCPUs = require('os').cpus().length; -function ensureDir(dir) { - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir); - } else if (!fs.statSync(dir).isDirectory()) { - fs.unlinkSync(dir); - fs.mkdirSync(dir); - } -} - -let pending = []; -const active = []; -const fail = []; - -async function handler() { - for (const i of pending) { - const p = `${os.tmpdir()}/hydro/tmp/${i}/handler.js`; - if (fs.existsSync(p) && !fail.includes(i)) { - try { - console.log(`Handler init: ${i}`); - console.time(`Handler init: ${i}`); - eval('require')(p); - console.timeEnd(`Handler init: ${i}`); - } catch (e) { - fail.push(i); - console.error(`Handler Load Fail: ${i}`); - } - } - } -} - -async function locale() { - for (const i of pending) { - const p = `${os.tmpdir()}/hydro/tmp/${i}/locale.json`; - if (fs.existsSync(p) && !fail.includes(i)) { - try { - global.Hydro.lib.i18n(eval('require')(p)); - console.log(`Locale init: ${i}`); - } catch (e) { - fail.push(i); - console.error(`Locale Load Fail: ${i}`); - } - } - } -} - -async function template() { - for (const i of pending) { - const p = `${os.tmpdir()}/hydro/tmp/${i}/template.json`; - if (fs.existsSync(p) && !fail.includes(i)) { - try { - Object.assign(global.Hydro.template, eval('require')(p)); - console.log(`Template init: ${i}`); - } catch (e) { - fail.push(i); - console.error(`Template Load Fail: ${i}`); - } - } - } -} - -async function model() { - for (const i of pending) { - const p = `${os.tmpdir()}/hydro/tmp/${i}/model.js`; - if (fs.existsSync(p) && !fail.includes(i)) { - try { - console.log(`Model init: ${i}`); - console.time(`Model init: ${i}`); - eval('require')(p); - console.timeEnd(`Model init: ${i}`); - } catch (e) { - fail.push(i); - console.error(`Model Load Fail: ${i}`); - } - } - } -} - -async function lib() { - for (const i of pending) { - const p = `${os.tmpdir()}/hydro/tmp/${i}/lib.js`; - if (fs.existsSync(p) && !fail.includes(i)) { - try { - console.log(`Lib init: ${i}`); - console.time(`Lib init: ${i}`); - eval('require')(p); - console.timeEnd(`Lib init: ${i}`); - } catch (e) { - fail.push(i); - console.error(`Lib Load Fail: ${i}`); - } - } - } -} - -async function service() { - for (const i of pending) { - const p = `${os.tmpdir()}/hydro/tmp/${i}/service.js`; - if (fs.existsSync(p) && !fail.includes(i)) { - try { - console.log(`Service init: ${i}`); - console.time(`Service init: ${i}`); - eval('require')(p); - console.timeEnd(`Service init: ${i}`); - } catch (e) { - fail.push(i); - console.error(`Service Load Fail: ${i}`); - console.error(e); - } - } - } -} - -async function script() { - for (const i of pending) { - const p = `${os.tmpdir()}/hydro/tmp/${i}/script.js`; - if (fs.existsSync(p) && !fail.includes(i)) { - try { - console.time(`Script init: ${i}`); - eval('require')(p); - console.timeEnd(`Script init: ${i}`); - } catch (e) { - fail.push(i); - console.error(`Script Load Fail: ${i}`); - console.error(e); - } - } - active.push(i); - } -} - -async function install() { - const setup = require('./service/setup'); - await setup.setup(); -} - -const builtinLib = [ - 'axios', 'download', 'i18n', 'mail', 'markdown', - 'md5', 'misc', 'paginate', 'hash.hydro', 'rank', - 'template', 'validator', 'nav', 'sysinfo', -]; - -const builtinModel = [ - 'builtin', 'document', 'domain', 'blacklist', 'opcount', - 'setting', 'token', 'user', 'problem', 'record', - 'contest', 'message', 'solution', 'training', 'file', - 'discussion', 'system', -]; - -const builtinHandler = [ - 'home', 'problem', 'record', 'judge', 'user', - 'contest', 'training', 'discussion', 'manage', 'import', - 'misc', 'homework', 'domain', 'wiki', -]; - -const builtinScript = [ - 'install', 'uninstall', 'rating', 'recalcRating', 'register', - 'blacklist', 'setSuperadmin', -]; - -async function loadAsMaster() { - ensureDir(path.resolve(os.tmpdir(), 'hydro')); - ensureDir(path.resolve(os.tmpdir(), 'hydro', 'tmp')); - ensureDir(path.resolve(os.tmpdir(), 'hydro', 'public')); - // TODO better run in another process as this needs lots of memory - require('./unzip')(); - pending = await require('./lib/hpm').getInstalled(); - require('./lib/i18n'); - require('./utils'); - require('./error'); - require('./permission'); - try { - require('./options'); - } catch (e) { - await install(); - require('./options'); - } - const bus = require('./service/bus'); - await new Promise((resolve) => { - const h = () => { - console.log('Database connected'); - bus.unsubscribe(['system_database_connected'], h); - resolve(); - }; - bus.subscribe(['system_database_connected'], h); - require('./service/db'); - }); - for (const i of builtinLib) require(`./lib/${i}`); - await lib(); - require('./service/gridfs'); - require('./service/monitor'); - const server = require('./service/server'); - await server.prepare(); - await service(); - for (const i of builtinModel) require(`./model/${i}`); - for (const i of builtinHandler) require(`./handler/${i}`); - await model(); - for (const m in global.Hydro.model) { - if (global.Hydro.model[m].ensureIndexes) { - await global.Hydro.model[m].ensureIndexes(); - } - } - const system = require('./model/system'); - const dbVer = await system.get('db.ver'); - if (dbVer !== 1) { - const ins = require('./script/install'); - await ins.run({ username: 'Root', password: 'rootroot' }); - } - await handler(); - for (const i in global.Hydro.handler) { - await global.Hydro.handler[i](); - } - for (const i in global.Hydro.service) { - if (global.Hydro.service[i].postInit) { - try { - await global.Hydro.service[i].postInit(); - } catch (e) { - console.error(e); - } - } - } - for (const i of builtinScript) require(`./script/${i}`); - await script(); - pending = []; -} - -async function loadAsWorker() { - pending = await require('./lib/hpm').getInstalled(); - require('./lib/i18n'); - require('./utils'); - require('./error'); - require('./permission'); - require('./options'); - await Promise.all([locale(), template()]); - const bus = require('./service/bus'); - await new Promise((resolve) => { - const h = () => { - console.log('Database connected'); - bus.unsubscribe(['system_database_connected'], h); - resolve(); - }; - bus.subscribe(['system_database_connected'], h); - require('./service/db'); - }); - for (const i of builtinLib) require(`./lib/${i}`); - await lib(); - require('./service/gridfs'); - const server = require('./service/server'); - await server.prepare(); - await service(); - for (const i of builtinModel) require(`./model/${i}`); - for (const i of builtinHandler) require(`./handler/${i}`); - await model(); - await handler(); - for (const i in global.Hydro.handler) { - await global.Hydro.handler[i](); - } - const notfound = require('./handler/notfound'); - await notfound(); - for (const i in global.Hydro.service) { - if (global.Hydro.service[i].postInit) { - try { - await global.Hydro.service[i].postInit(); - } catch (e) { - console.error(e); - } - } - } - for (const i of builtinScript) require(`./script/${i}`); - await script(); - pending = []; - await server.start(); -} - async function terminate() { for (const task of global.onDestory) { // eslint-disable-next-line no-await-in-loop @@ -288,6 +12,14 @@ async function terminate() { process.exit(0); } +async function entry(config) { + if (config.entry) { + // TODO newProcess + const loader = require(`./entry/${config.entry}`); + await loader(entry); + } +} + async function load() { global.nodeModules = { bson: require('bson'), @@ -320,7 +52,7 @@ async function load() { console.warn(e); } }); - await loadAsMaster(); + await entry({ entry: 'master' }); cluster.on('exit', (worker, code, signal) => { console.log(`Worker ${worker.process.pid} exit: ${code} ${signal}`); }); @@ -338,16 +70,12 @@ async function load() { } } else { console.log(`Worker ${process.pid} Starting`); - await loadAsWorker(); + await entry({ entry: 'worker' }); console.log(`Worker ${process.pid} Started`); } if (global.gc) global.gc(); } -module.exports = { - load, pending, active, fail, -}; - if (!module.parent) { load().catch((e) => { console.error(e); diff --git a/hydro/model/contest.js b/hydro/model/contest.js index 28a5c98d2..d4536be6e 100644 --- a/hydro/model/contest.js +++ b/hydro/model/contest.js @@ -365,7 +365,7 @@ function count(domainId, query, type = document.TYPE_CONTEST) { } function getMulti(domainId, query = {}, type = document.TYPE_CONTEST) { - return document.getMulti(domainId, type, query).sort('beginAt', -1); + return document.getMulti(domainId, type, query); } function _getStatusJournal(tsdoc) { diff --git a/module/migrate-vijos/handler.js b/module/migrate-vijos/handler.js index 033971b36..75a3818d8 100644 --- a/module/migrate-vijos/handler.js +++ b/module/migrate-vijos/handler.js @@ -9,7 +9,7 @@ class MigrateVijosHandler extends Handler { async get() { const path = [ - ['Hydro', '/'], + ['Hydro', 'homepage'], ['migrate_vijos', null], ]; this.response.body = { path };