From 605f8fc65dd2f397db9e732b54f8fe52199d30a9 Mon Sep 17 00:00:00 2001 From: masnn Date: Wed, 10 Jun 2020 13:08:59 +0800 Subject: [PATCH] fix --- .github/workflows/build.yml | 4 ---- .npmignore | 6 ++++++ build/hack.js | 24 +++++++++++++++++++++++ build/index.js | 14 +++++++++----- build/webpack.js | 15 --------------- hydro/handler/manage.js | 2 +- hydro/handler/misc.js | 2 +- hydro/lib/hpm.js | 33 ++++++++++++++++++++++---------- hydro/loader.js | 38 +++++++++++++++++++++++++------------ hydro/script/blacklist.js | 30 ++++++++++++++--------------- hydro/service/server.js | 2 +- hydro/unzip.js | 13 ++++++++++++- templates/status.html | 2 +- 13 files changed, 119 insertions(+), 66 deletions(-) create mode 100644 .npmignore create mode 100644 build/hack.js diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 22dd51f0b..ff1418b8b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,10 +24,6 @@ jobs: with: name: app.js path: .build/app.js - - uses: actions/upload-artifact@v1 - with: - name: full.js - path: .build/full.js - uses: actions/upload-artifact@v1 with: name: install.js diff --git a/.npmignore b/.npmignore new file mode 100644 index 000000000..d9917bee2 --- /dev/null +++ b/.npmignore @@ -0,0 +1,6 @@ +node_modules +config.json +docker-compose.yml +Dockerfile +.vscode +.github \ No newline at end of file diff --git a/build/hack.js b/build/hack.js new file mode 100644 index 000000000..99ad122fd --- /dev/null +++ b/build/hack.js @@ -0,0 +1,24 @@ +const fs = require('fs'); +const { root } = require('./utils'); + +module.exports = { + formidable: () => { + const tasks = ['incoming_form', 'file', 'json_parser', 'querystring_parser']; + for (const task of tasks) { + let file = fs.readFileSync(root(`node_modules/formidable/lib/${task}.js`)).toString(); + if (file.startsWith('if (global.GENTLY) require = GENTLY.hijack(require);')) { + file = file.split('\n'); + file[0] = ''; + file = file.join('\n'); + fs.writeFileSync(root(`node_modules/formidable/lib/${task}.js`), file); + } + } + }, + sockjs: () => { + const file = fs.readFileSync(root('node_modules/sockjs/lib/sockjs.js')).toString().split('\n'); + if (file[139].includes('this.app.log')) { + file[139] = ''; + fs.writeFileSync(root('node_modules/sockjs/lib/sockjs.js'), file.join('\n')); + } + }, +}; diff --git a/build/index.js b/build/index.js index 9bdd53a1a..e82a93900 100644 --- a/build/index.js +++ b/build/index.js @@ -5,6 +5,7 @@ const zlib = require('zlib'); const yaml = require('js-yaml'); const { root, ignoreFailure, rmdir } = require('./utils'); const template = require('./template'); +const hack = require('./hack'); const fsp = fs.promises; @@ -24,6 +25,11 @@ async function build(type) { if (!['development', 'production'].includes(type)) throw new Error(`Unknown type: ${type}`); ignoreFailure(fs.mkdirSync, root('.build')); ignoreFailure(fs.mkdirSync, root('.build/module')); + + for (const task in hack) { + hack[task](); + } + const langs = fs.readdirSync(root('locales')); const locale = {}; for (const i of langs) { @@ -34,14 +40,14 @@ async function build(type) { id: 'builtin', locale, template: template('templates'), - file: {}, + public: {}, }; const files = getFiles('.uibuild'); for (const f of files) { if (fs.statSync(root(`.uibuild/${f}`)).isDirectory()) { - builtin.file[f] = null; + builtin.public[f] = null; } else { - builtin.file[f] = fs.readFileSync(root(`.uibuild/${f}`)).toString('base64'); + builtin.public[f] = fs.readFileSync(root(`.uibuild/${f}`)).toString('base64'); } } const data = zlib.gzipSync(Buffer.from(yaml.safeDump(builtin)), { level: -1 }); @@ -68,8 +74,6 @@ async function build(type) { } const f = fs.readFileSync(root('.build/app.js')).toString(); const installer = fs.readFileSync(root('tool/install.js')).toString(); - const file = `global._hydroModule=${JSON.stringify(j)};${f}`; - fs.writeFileSync(root('.build/full.js'), file); const hydro = {}; hydro.app = Buffer.from(f).toString('base64'); hydro.modules = j; diff --git a/build/webpack.js b/build/webpack.js index 1bfe8f318..3f4436c11 100644 --- a/build/webpack.js +++ b/build/webpack.js @@ -1,24 +1,9 @@ /* eslint-disable import/no-extraneous-dependencies */ -const fs = require('fs'); const webpack = require('webpack'); const FriendlyErrorsPlugin = require('friendly-errors-webpack-plugin'); const { root } = require('./utils'); const build = async (type) => { - function hackNodeModuleFormidable() { - const tasks = ['incoming_form', 'file', 'json_parser', 'querystring_parser']; - for (const task of tasks) { - let file = fs.readFileSync(root(`node_modules/formidable/lib/${task}.js`)).toString(); - if (file.startsWith('if (global.GENTLY) require = GENTLY.hijack(require);')) { - file = file.split('\n'); - file[0] = ''; - file = file.join('\n'); - fs.writeFileSync(root(`node_modules/formidable/lib/${task}.js`), file); - } - } - } - hackNodeModuleFormidable(); - const config = { mode: type, entry: { diff --git a/hydro/handler/manage.js b/hydro/handler/manage.js index 6bb1b2264..c4607afa6 100644 --- a/hydro/handler/manage.js +++ b/hydro/handler/manage.js @@ -33,7 +33,7 @@ class SystemDashboardHandler extends SystemHandler { class SystemModuleHandler extends SystemHandler { async get() { this.response.body.path.push(['manage_module', null]); - this.response.body.installed = await hpm.getInstalled(); + this.response.body.installed = await hpm.getDetail(); this.response.active = loader.active; this.response.template = 'manage_module.html'; } diff --git a/hydro/handler/misc.js b/hydro/handler/misc.js index 573a7e39a..54f1499f7 100644 --- a/hydro/handler/misc.js +++ b/hydro/handler/misc.js @@ -22,9 +22,9 @@ class StatusHandler extends Handler { const online = new Date(stats[i].updateAt).getTime() > new Date().getTime() - 300000; if (!online) desc = 'Offline'; desc = desc || 'Online'; + stats[i].isOnline = online; stats[i].status = desc; } - console.log(stats); const path = [ ['Hydro', '/'], ]; diff --git a/hydro/lib/hpm.js b/hydro/lib/hpm.js index f3ab414f6..4e5435f63 100644 --- a/hydro/lib/hpm.js +++ b/hydro/lib/hpm.js @@ -30,15 +30,26 @@ async function getInstalled() { for (const file of files) { const info = `${os.tmpdir()}/hydro/tmp/${file}/hydro.json`; if (fs.existsSync(info)) { - try { - const m = file; - const t = JSON.parse(fs.readFileSync(info).toString()); - modules.push(`${m}|${t.id}`); - } catch (e) { - if (e.code === 'Z_DATA_ERROR') { - console.error(`Module Load Fail: ${file} (File Corrupted)`); - } else console.error(`Module Load Fail: ${file} ${e}`); - } + modules.push(file); + } + } + return modules; +} + +async function getDetail() { + const modules = []; + const files = fs.readdirSync(`${os.tmpdir()}/hydro/tmp`); + for (const file of files) { + const info = `${os.tmpdir()}/hydro/tmp/${file}/hydro.json`; + if (fs.existsSync(info)) { + const i = JSON.parse(fs.readFileSync(info).toString()); + const f = fs.statSync(info); + modules.push({ + id: i.id, + version: i.version, + description: i.description, + size: f.size, + }); } } return modules; @@ -52,4 +63,6 @@ async function install(url) { await download(url, root(`${moduleRoot}/${String.random(16)}.hydro`)); } -global.Hydro.lib.hpm = module.exports = { getInstalled, del, install }; +global.Hydro.lib.hpm = module.exports = { + getInstalled, getDetail, del, install, +}; diff --git a/hydro/loader.js b/hydro/loader.js index b9f51fcdb..f7f67ed52 100644 --- a/hydro/loader.js +++ b/hydro/loader.js @@ -31,7 +31,7 @@ const fail = []; async function handler() { for (const i of pending) { const p = `${os.tmpdir()}/hydro/tmp/${i}/handler.js`; - if (fs.existsSync(p) && i.fail) { + if (fs.existsSync(p) && !fail.includes(i)) { try { console.log(`Handler init: ${i}`); console.time(`Handler init: ${i}`); @@ -48,7 +48,7 @@ async function handler() { async function locale() { for (const i of pending) { const p = `${os.tmpdir()}/hydro/tmp/${i}/locale.json`; - if (fs.existsSync(p) && i.fail) { + if (fs.existsSync(p) && !fail.includes(i)) { try { global.Hydro.lib.i18n(superRequire(p)); console.log(`Locale init: ${i}`); @@ -63,7 +63,7 @@ async function locale() { async function template() { for (const i of pending) { const p = `${os.tmpdir()}/hydro/tmp/${i}/template.json`; - if (fs.existsSync(p) && !i.fail) { + if (fs.existsSync(p) && !fail.includes(i)) { try { Object.assign(global.Hydro.template, superRequire(p)); console.log(`Template init: ${i}`); @@ -78,7 +78,7 @@ async function template() { async function model() { for (const i of pending) { const p = `${os.tmpdir()}/hydro/tmp/${i}/model.js`; - if (fs.existsSync(p) && !i.fail) { + if (fs.existsSync(p) && !fail.includes(i)) { try { console.log(`Model init: ${i}`); console.time(`Model init: ${i}`); @@ -95,7 +95,7 @@ async function model() { async function lib() { for (const i of pending) { const p = `${os.tmpdir()}/hydro/tmp/${i}/lib.js`; - if (fs.existsSync(p) && !i.fail) { + if (fs.existsSync(p) && !fail.includes(i)) { try { console.log(`Lib init: ${i}`); console.time(`Lib init: ${i}`); @@ -112,7 +112,7 @@ async function lib() { async function service() { for (const i of pending) { const p = `${os.tmpdir()}/hydro/tmp/${i}/service.js`; - if (fs.existsSync(p) && !i.fail) { + if (fs.existsSync(p) && !fail.includes(i)) { try { console.log(`Service init: ${i}`); console.time(`Service init: ${i}`); @@ -130,7 +130,7 @@ async function service() { async function script() { for (const i of pending) { const p = `${os.tmpdir()}/hydro/tmp/${i}/script.js`; - if (fs.existsSync(p) && !i.fail) { + if (fs.existsSync(p) && !fail.includes(i)) { try { console.time(`Script init: ${i}`); superRequire(p); @@ -141,6 +141,7 @@ async function script() { console.error(e); } } + active.push(i); } } @@ -175,6 +176,8 @@ const builtinScript = [ async function loadAsMaster() { ensureDir(path.resolve(os.tmpdir(), 'hydro')); + ensureDir(path.resolve(os.tmpdir(), 'hydro', 'tmp')); + ensureDir(path.resolve(os.tmpdir(), 'hydro', 'public')); // TODO better run in another process as this needs lots of memory require('./unzip')(); pending = await require('./lib/hpm').getInstalled(); @@ -182,7 +185,6 @@ async function loadAsMaster() { require('./utils'); require('./error'); require('./permission'); - await Promise.all([locale(), template()]); try { require('./options'); } catch (e) { @@ -201,11 +203,18 @@ async function loadAsMaster() { }); for (const i of builtinLib) require(`./lib/${i}`); await lib(); + require('./service/gridfs'); require('./service/monitor'); + const server = require('./service/server'); + await server.prepare(); await service(); - for (const i of builtinModel) { - const m = require(`./model/${i}`); - if (m.ensureIndexes) await m.ensureIndexes(); + for (const i of builtinModel) require(`./model/${i}`); + for (const i of builtinHandler) require(`./handler/${i}`); + await model(); + for (const m in global.Hydro.model) { + if (global.Hydro.model[m].ensureIndexes) { + await global.Hydro.model[m].ensureIndexes(); + } } const system = require('./model/system'); const dbVer = await system.get('db.ver'); @@ -213,7 +222,10 @@ async function loadAsMaster() { const ins = require('./script/install'); await ins.run({ username: 'Root', password: 'rootroot' }); } - await model(); + await handler(); + for (const i in global.Hydro.handler) { + await global.Hydro.handler[i](); + } for (const i in global.Hydro.service) { if (global.Hydro.service[i].postInit) { try { @@ -223,6 +235,8 @@ async function loadAsMaster() { } } } + for (const i of builtinScript) require(`./script/${i}`); + await script(); pending = []; } diff --git a/hydro/script/blacklist.js b/hydro/script/blacklist.js index 5b26d5713..956ea4f70 100644 --- a/hydro/script/blacklist.js +++ b/hydro/script/blacklist.js @@ -7,58 +7,58 @@ const user = require('../model/user'); const blacklist = require('../model/blacklist'); const db = require('../service/db.js'); -async function _address(ip, bset, uset, dset, dryrun) { +async function _address(ip, bset, uset, dset, dryrun, report) { if (bset.has(ip)) return bset.add(ip) - console.log('ip %s', ip) + report({ message: `ip ${ip}` }) const users = await db.collection('user').find({ loginip: ip }).toArray() const tasks = [] for (const udoc of users) { - tasks.push(_user(udoc._id, bset, uset, dset, dryrun)) } + tasks.push(_user(udoc._id, bset, uset, dset, dryrun, report)) } await Promise.all(tasks) if (!dryrun) await blacklist.add(ip) } -async function _discussion(domainId, did, bset, uset, dset, dryrun) { +async function _discussion(domainId, did, bset, uset, dset, dryrun, report) { if (dset.has(did)) return dset.add(did) const ddoc = await discussion.get(domainId, did) if (!ddoc) return - console.log('discussion %s', ddoc['title']) - await _user(ddoc['owner'], bset, uset, dset, dryrun) + report({ message: `discussion ${ddoc['title']}` }) + await _user(ddoc['owner'], bset, uset, dset, dryrun, report) if (ddoc.ip) - await _address(ddoc['ip'], bset, uset, dset, dryrun) + await _address(ddoc['ip'], bset, uset, dset, dryrun, report) if (!dryrun) await discussion.delete(domainId, ddoc['docId']) } -async function _user(uid, bset, uset, dset, dryrun) { +async function _user(uid, bset, uset, dset, dryrun, report) { if (uset.has(uid)) return uset.add(uid) const udoc = await user.getById(uid) if (!udoc) return - console.log('user %s %s', udoc['_id'], udoc['uname']) - await _address(udoc['loginip'], bset, uset, dset, dryrun) + report({ message: `user ${udoc['_id']} ${udoc['uname']}` }) + await _address(udoc['loginip'], bset, uset, dset, dryrun, report) const ddocs = await db.collection('document') .find({ docType: document.TYPE_DISCUSSION }).sort({ domainId: 1, docId: 1 }).toArray() const tasks = [] for (const ddoc of ddocs) - tasks.push(_discussion(ddoc.domainId, ddoc.docId, bset, uset, dset, dryrun)) + tasks.push(_discussion(ddoc.domainId, ddoc.docId, bset, uset, dset, dryrun, report)) await Promise.all(tasks) if (!dryrun) await user.ban(uid) } -async function run({address = null, discuss = null, user = null, dryrun = true}) { +async function run({address = null, discuss = null, user = null, dryrun = true}, report) { if (address) - await _address(address, new Set(), new Set(), new Set(), dryrun) + await _address(address, new Set(), new Set(), new Set(), dryrun, report) if (discuss) await _discussion(discuss.domainId, new ObjectID(discuss.did), - new Set(), new Set(), new Set(), dryrun) + new Set(), new Set(), new Set(), dryrun, report) if (user) - await _user(user, new Set(), new Set(), new Set(), dryrun) } + await _user(user, new Set(), new Set(), new Set(), dryrun, report) } global.Hydro.script.blacklist = module.exports = { run }; diff --git a/hydro/service/server.js b/hydro/service/server.js index de67dd371..f3768a221 100644 --- a/hydro/service/server.js +++ b/hydro/service/server.js @@ -81,7 +81,7 @@ const validate = { async function prepare() { server = http.createServer(app.callback()); app.keys = await system.get('session.keys'); - app.use(cache(path.join(os.tmpdir(), 'hydro', 'builtin'), { + app.use(cache(path.join(os.tmpdir(), 'hydro', 'public'), { maxAge: 365 * 24 * 60 * 60, })); app.use(Body({ diff --git a/hydro/unzip.js b/hydro/unzip.js index 757d2e1fd..82dd1487d 100644 --- a/hydro/unzip.js +++ b/hydro/unzip.js @@ -66,7 +66,7 @@ function exec() { if (m[i]) { fs.writeFileSync( `${os.tmpdir()}/hydro/tmp/${id}/${i}.js`, - f[i], + m[i], ); } } @@ -81,6 +81,17 @@ function exec() { } } } + if (m.public) { + ensureDir(path.resolve(os.tmpdir(), 'hydro', 'public')); + for (const n in m.public) { + if (m.public[n] === null) { + ensureDir(path.resolve(os.tmpdir(), 'hydro', 'public', n)); + } else { + const e = path.resolve(os.tmpdir(), 'hydro', 'public', n); + fs.writeFileSync(e, Buffer.from(m.public[n], 'base64'), { mode: 755 }); + } + } + } fs.writeFileSync( root(`${os.tmpdir()}/hydro/tmp/${id}/hydro.json`), JSON.stringify({ diff --git a/templates/status.html b/templates/status.html index f766fac23..793511e4e 100644 --- a/templates/status.html +++ b/templates/status.html @@ -26,7 +26,7 @@ {% for doc in stats %} {{ doc._id }} - {{ doc.status }} + {% if doc.isOnline %}{{ doc.status }}{% else %}{{ datetimeSpan(doc.updateAt)|safe }}{% endif %} {{ doc.osinfo }} {{ doc.cpu }} {{ doc.memory }}