简化模块加载

deprecated hydro-build

hydro/entry/common.ts

@@ -2,6 +2,7 @@
 /* eslint-disable no-eval */
 import os from 'os';
 import fs from 'fs';
+import path from 'path';
 import yaml from 'js-yaml';
 
 export const builtinLib = [
@@ -29,9 +30,22 @@ export const builtinScript = [
     'blacklist', 'setSuperadmin', 'deleteUser',
 ];
 
+function getFiles(folder, base = '') {
+    const files = [];
+    const f = fs.readdirSync(folder);
+    for (const i of f) {
+        if (!i.startsWith('.')) {
+            if (fs.statSync(path.join(folder, i)).isDirectory()) {
+                files.push(...getFiles(path.join(folder, i), path.join(base, i)));
+            } else files.push(path.join(base, i));
+        }
+    }
+    return files;
+}
+
 export async function handler(pending, fail) {
     for (const i of pending) {
-        const p = `${os.tmpdir()}/hydro/tmp/${i}/handler.js`;
+        const p = path.resolve(i, 'handler.js');
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 console.log(`Handler init: ${i}`);
@@ -48,10 +62,17 @@ export async function handler(pending, fail) {
 
 export async function locale(pending, fail) {
     for (const i of pending) {
-        const p = `${os.tmpdir()}/hydro/tmp/${i}/locale.json`;
-        if (fs.existsSync(p) && !fail.includes(i)) {
+        let p = path.resolve(i, 'locales');
+        if (!fs.existsSync(p)) p = path.resolve(i, 'locale');
+        if (fs.existsSync(p) && fs.statSync(p).isDirectory() && !fail.includes(i)) {
             try {
-                global.Hydro.lib.i18n(eval('require')(p));
+                const files = fs.readdirSync(p);
+                const locales = {};
+                for (const file of files) {
+                    const content = fs.readFileSync(path.resolve(p, file)).toString();
+                    locales[file.split('.')[0]] = yaml.safeLoad(content);
+                }
+                global.Hydro.lib.i18n(locales);
                 console.log(`Locale init: ${i}`);
             } catch (e) {
                 fail.push(i);
@@ -61,22 +82,19 @@ export async function locale(pending, fail) {
     }
 }
 
-export async function setting(pending, fail, modelSetting, modelSystem = null) {
+export async function setting(pending, fail, modelSetting: typeof import('../model/setting')) {
     const map = {
         system: modelSetting.SystemSetting,
         account: modelSetting.AccountSetting,
         preference: modelSetting.PreferenceSetting,
     };
     for (const i of pending) {
-        const p = `${os.tmpdir()}/hydro/tmp/${i}/setting.yaml`;
+        let p = path.resolve(i, 'setting.yaml');
+        if (!fs.existsSync(p)) p = path.resolve(i, 'settings.yaml');
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 const cfg: any = yaml.safeLoad(fs.readFileSync(p).toString());
                 for (const key in cfg) {
-                    if (cfg[key].default && modelSystem) {
-                        const current = await modelSystem.get(`${i}.${key}`);
-                        if (!current) await modelSystem.set(`${i}.${key}`, cfg[key].default);
-                    }
                     map[cfg[key].category || 'system'](
                         modelSetting.Setting(
                             i, `${i}.${key}`, cfg[key].range, cfg[key].default,
@@ -93,10 +111,14 @@ export async function setting(pending, fail, modelSetting, modelSystem = null) {
 
 export async function template(pending, fail) {
     for (const i of pending) {
-        const p = `${os.tmpdir()}/hydro/tmp/${i}/template.json`;
-        if (fs.existsSync(p) && !fail.includes(i)) {
+        let p = path.resolve(i, 'templates');
+        if (!fs.existsSync(p)) p = path.resolve(i, 'template');
+        if (fs.existsSync(p) && fs.statSync(p).isDirectory() && !fail.includes(i)) {
             try {
-                Object.assign(global.Hydro.template, eval('require')(p));
+                const files = getFiles(p);
+                for (const file of files) {
+                    global.Hydro.template[file] = fs.readFileSync(path.resolve(p, file)).toString();
+                }
                 console.log(`Template init: ${i}`);
             } catch (e) {
                 fail.push(i);
@@ -108,7 +130,7 @@ export async function template(pending, fail) {
 
 export async function model(pending, fail) {
     for (const i of pending) {
-        const p = `${os.tmpdir()}/hydro/tmp/${i}/model.js`;
+        const p = path.resolve(i, 'model.js');
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 console.log(`Model init: ${i}`);
@@ -125,7 +147,7 @@ export async function model(pending, fail) {
 
 export async function lib(pending, fail) {
     for (const i of pending) {
-        const p = `${os.tmpdir()}/hydro/tmp/${i}/lib.js`;
+        const p = path.resolve(i, 'lib.js');
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 console.log(`Lib init: ${i}`);
@@ -143,7 +165,7 @@ export async function lib(pending, fail) {
 
 export async function service(pending, fail) {
     for (const i of pending) {
-        const p = `${os.tmpdir()}/hydro/tmp/${i}/service.js`;
+        const p = path.resolve(i, 'service.js');
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 console.log(`Service init: ${i}`);
@@ -161,7 +183,7 @@ export async function service(pending, fail) {
 
 export async function script(pending, fail, active) {
     for (const i of pending) {
-        const p = `${os.tmpdir()}/hydro/tmp/${i}/script.js`;
+        const p = path.resolve(i, 'script.js');
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 console.time(`Script init: ${i}`);
@@ -176,3 +198,43 @@ export async function script(pending, fail, active) {
         active.push(i);
     }
 }
+
+export async function wiki(pending, fail) {
+    for (const i of pending) {
+        const p = path.resolve(i, 'wiki');
+        if (fs.existsSync(p) && fs.statSync(p).isDirectory() && !fail.includes(i)) {
+            try {
+                const r = {};
+                const categories = fs.readdirSync(p);
+                for (const category of categories) {
+                    if (!fs.statSync(p).isDirectory()) return;
+                    const pages = fs.readdirSync(path.join(p, category));
+                    const res = {};
+                    for (const page of pages) {
+                        const c = fs.readFileSync(path.join(p, category, page)).toString().split('\n');
+                        const pagename = page.split('.')[0];
+                        res[pagename] = [];
+                        let content = null;
+                        for (let j = 0; j < c.length; j++) {
+                            const line = c[j];
+                            if (line.startsWith('# ')) {
+                                if (content) res[pagename].push(content);
+                                content = {};
+                                const t = line.split('# ')[1].split('|');
+                                [content.title, content.id] = t.map((q) => q.trim());
+                                content.content = '';
+                            } else {
+                                content.content = `${content.content}${line}\n`;
+                            }
+                        }
+                    }
+                    r[category] = res;
+                }
+                console.log(`Wiki init: ${i}`);
+            } catch (e) {
+                fail.push(i);
+                console.error(`Wiki Load Fail: ${i}`);
+            }
+        }
+    }
+}

hydro/entry/master.ts

@@ -12,10 +12,9 @@ export async function load(call) {
     require('../utils');
     require('../error');
     try {
-        require('../options');
+        require('../options').default();
     } catch (e) {
-        await call({ entry: 'setup' });
-        require('../options');
+        await call({ entry: 'setup', newProcess: true });
     }
     const bus = require('../service/bus');
     await new Promise((resolve) => {
@@ -49,7 +48,7 @@ export async function load(call) {
         const ins = require('../script/install');
         await ins.run({ username: 'Root', password: 'rootroot' });
     }
-    await setting(pending, fail, modelSetting, modelSystem);
+    await setting(pending, fail, modelSetting);
     for (const i in global.Hydro.service) {
         if (global.Hydro.service[i].postInit) {
             try {

hydro/entry/setup.ts

@@ -8,6 +8,9 @@ import Router from 'koa-router';
 import cache from 'koa-static-cache';
 import nunjucks from 'nunjucks';
 import mongodb from 'mongodb';
+import parse from 'yargs-parser';
+
+const argv = parse(process.argv.slice(2));
 
 class Loader {
     getSource(name: string) { // eslint-disable-line class-methods-use-this
@@ -94,7 +97,7 @@ function render(name) {
     });
 }
 
-export async function load() {
+export function load() {
     const app = new Koa();
     const server = http.createServer(app.callback());
     const router = new Router();
@@ -107,55 +110,56 @@ export async function load() {
         ctx.body = await render('setup.html');
         ctx.response.type = 'text/html';
     });
-    const p = new Promise((resolve) => {
-        router.post('/', async (ctx) => {
-            const {
+    const listenPort = argv.port || 8888;
+    router.post('/', async (ctx) => {
+        const {
+            host, port, name, username, password,
+        } = ctx.request.body;
+        let mongourl = 'mongodb://';
+        if (username) mongourl += `${username}:${password}@`;
+        mongourl += `${host}:${port}/${name}`;
+        try {
+            const Database = await mongodb.MongoClient.connect(mongourl, {
+                useNewUrlParser: true, useUnifiedTopology: true,
+            });
+            const db = Database.db(name);
+            await Promise.all([
+                db.collection('system').updateOne(
+                    { _id: 'server.host' },
+                    { $set: { value: ctx.request.host } },
+                    { upsert: true },
+                ),
+                db.collection('system').updateOne(
+                    { _id: 'server.hostname' },
+                    { $set: { value: ctx.request.hostname } },
+                    { upsert: true },
+                ),
+                db.collection('system').updateOne(
+                    { _id: 'server.url' },
+                    { $set: { value: ctx.request.href } },
+                    { upsert: true },
+                ),
+                db.collection('system').updateOne(
+                    { _id: 'server.port' },
+                    { $set: { value: listenPort } },
+                    { upsert: true },
+                ),
+            ]);
+            fs.writeFileSync(path.resolve(process.cwd(), 'config.json'), JSON.stringify({
                 host, port, name, username, password,
-            } = ctx.request.body;
-            let mongourl = 'mongodb://';
-            if (username) mongourl += `${username}:${password}@`;
-            mongourl += `${host}:${port}/${name}`;
-            try {
-                const Database = await mongodb.MongoClient.connect(mongourl, {
-                    useNewUrlParser: true, useUnifiedTopology: true,
-                });
-                const db = Database.db(name);
-                await Promise.all([
-                    db.collection('system').updateOne(
-                        { _id: 'server.host' },
-                        { $set: { value: ctx.request.host } },
-                        { upsert: true },
-                    ),
-                    db.collection('system').updateOne(
-                        { _id: 'server.hostname' },
-                        { $set: { value: ctx.request.hostname } },
-                        { upsert: true },
-                    ),
-                    db.collection('system').updateOne(
-                        { _id: 'server.url' },
-                        { $set: { value: ctx.request.href } },
-                        { upsert: true },
-                    ),
-                ]);
-                fs.writeFileSync(path.resolve(process.cwd(), 'config.json'), JSON.stringify({
-                    host, port, name, username, password,
-                }));
-                ctx.redirect('/');
-                resolve();
-            } catch (e) {
-                ctx.body = `Error connecting to database: ${e.message}\n${e.stack}`;
-            }
-        });
+            }));
+            ctx.body = `<h1>This page will reload in 3 secs.</h1>
+                <script>
+                setTimeout(function (){
+                    window.location.href = '/';
+                }, 3000);
+                </script>`;
+            setTimeout(() => process.exit(0), 500);
+        } catch (e) {
+            ctx.body = `Error connecting to database: ${e.message}\n${e.stack}`;
+        }
     });
     app.use(router.routes()).use(router.allowedMethods());
-    server.listen(8888);
-    console.log('Server listening at: 8888');
-    await p;
-    await new Promise((resolve, reject) => {
-        server.close((err) => {
-            if (err) reject(err);
-            else resolve();
-        });
-    });
-    return 'Done! Restarting...';
+    server.listen(listenPort);
+    console.log('Server listening at:', listenPort);
 }

hydro/entry/unzip.ts

@@ -1,123 +1,68 @@
-import fs from 'fs';
+/* eslint-disable no-continue */
 import os from 'os';
 import path from 'path';
-import zlib from 'zlib';
-import yaml from 'js-yaml';
+import AdmZip from 'adm-zip';
+import fs from 'fs-extra';
 
-function ensureDir(dir) {
-    if (!fs.existsSync(dir)) {
-        fs.mkdirSync(dir);
-    } else if (!fs.statSync(dir).isDirectory()) {
-        fs.unlinkSync(dir);
-        fs.mkdirSync(dir);
-    }
-}
+const moduleRoots = Array.from(new Set([
+    path.resolve(process.cwd(), 'node_modules', '@hydrooj'),
+    path.resolve(process.cwd(), 'modules'),
+    path.resolve(process.cwd(), 'module'),
+    path.resolve(process.cwd(), '.build'),
+    process.cwd(),
+    path.resolve(__dirname, 'node_modules', '@hydrooj'),
+    path.resolve(__dirname, 'modules'),
+    path.resolve(__dirname, 'module'),
+    path.resolve(__dirname, '.build'),
+    __dirname,
+    path.resolve(os.homedir(), '.hydro', 'module'),
+]));
 
-function root(name) {
-    return path.resolve(process.cwd(), name);
-}
-
-const moduleRoots = [
-    root('.build/module'),
-    root('module'),
-    root(path.resolve(os.homedir(), '.hydro', 'module')),
-    root('.'),
-];
-let moduleRoot;
-for (const i of moduleRoots) {
-    if (fs.existsSync(i) && fs.statSync(i).isDirectory()) {
-        moduleRoot = i;
-        break;
-    }
-}
+const moduleTemp = path.resolve(os.tmpdir(), 'hydro', 'module');
+const publicTemp = path.resolve(os.tmpdir(), 'hydro', 'public');
+const tmp = path.resolve(os.tmpdir(), 'hydro', '__');
 
 export async function load() {
     process.on('unhandledRejection', (e) => {
         console.error(e);
         process.exit(1);
     });
-    ensureDir(path.resolve(os.tmpdir(), 'hydro'));
-    ensureDir(path.resolve(os.tmpdir(), 'hydro', 'tmp'));
-    ensureDir(path.resolve(os.tmpdir(), 'hydro', 'public'));
-    const files = fs.readdirSync(moduleRoot);
-    const t = ['service', 'lib', 'model', 'handler', 'script'];
-    for (const file of files) {
-        if (file.endsWith('.hydro')) {
-            try {
-                const f = fs.readFileSync(root(`${moduleRoot}/${file}`));
-                const s = fs.statSync(root(`${moduleRoot}/${file}`));
-                const m: any = yaml.safeLoad(zlib.gunzipSync(f).toString());
-                const { id } = m;
-                if (m.os) {
-                    if (!m.os.includes(os.platform().toLowerCase())) {
-                        console.error(`Module load fail: ${id} ${file.split('.')[0]} Unsupported OS`);
-                        // eslint-disable-next-line no-continue
-                        continue;
-                    }
-                }
-                ensureDir(`${os.tmpdir()}/hydro/tmp/${id}`);
-                if (m.locale) {
-                    fs.writeFileSync(
-                        `${os.tmpdir()}/hydro/tmp/${id}/locale.json`,
-                        JSON.stringify(m.locale),
-                    );
-                }
-                if (m.setting) {
-                    fs.writeFileSync(
-                        `${os.tmpdir()}/hydro/tmp/${id}/setting.yaml`,
-                        m.setting,
-                    );
-                }
-                if (m.template) {
-                    fs.writeFileSync(
-                        `${os.tmpdir()}/hydro/tmp/${id}/template.json`,
-                        JSON.stringify(m.template),
-                    );
-                }
-                for (const i of t) {
-                    if (m[i]) {
-                        fs.writeFileSync(
-                            `${os.tmpdir()}/hydro/tmp/${id}/${i}.js`,
-                            m[i],
-                        );
-                    }
-                }
-                if (m.file) {
-                    ensureDir(path.resolve(os.tmpdir(), 'hydro', m.id));
-                    for (const n in m.file) {
-                        if (m.file[n] === null) {
-                            ensureDir(path.resolve(os.tmpdir(), 'hydro', m.id, n));
-                        } else {
-                            const e = path.resolve(os.tmpdir(), 'hydro', m.id, n);
-                            fs.writeFileSync(e, Buffer.from(m.file[n], 'base64'), { mode: 755 });
+    fs.ensureDirSync(moduleTemp);
+    fs.ensureDirSync(publicTemp);
+    for (const moduleRoot of moduleRoots) {
+        if (fs.existsSync(moduleRoot)) {
+            const files = fs.readdirSync(moduleRoot);
+            for (const file of files) {
+                try {
+                    // This fixs a mistake.
+                    // Markdown-it-katex is not a hydro module but a dependency.
+                    if (file === 'markdown-it-katex') continue;
+                    const modulePath = path.join(moduleRoot, file);
+                    const packagejson = path.join(modulePath, 'package.json');
+                    if (fs.statSync(modulePath).isFile() && file.endsWith('.hydro')) {
+                        // Is *.hydro module
+                        const zip = new AdmZip(modulePath);
+                        const targetPath = path.resolve(moduleTemp, file.split('.')[0]);
+                        zip.extractAllTo(targetPath, true);
+                        const content = fs.readdirSync(targetPath);
+                        const ipath = path.join(targetPath, content[0]);
+                        if (content.length === 1 && fs.statSync(ipath).isDirectory()) {
+                            fs.moveSync(ipath, tmp);
+                            fs.rmdirSync(targetPath);
+                            fs.moveSync(tmp, targetPath);
                         }
-                    }
+                        const publicPath = path.resolve(moduleTemp, file.split('.')[0], 'public');
+                        if (fs.existsSync(publicPath)) fs.copySync(publicPath, publicTemp);
+                    } else if (fs.existsSync(packagejson)) {
+                        // Is a npm package
+                        const q = fs.readFileSync(packagejson).toString();
+                        if (!JSON.parse(q).name.startsWith('@hydrooj/')) continue;
+                        const publicPath = path.resolve(modulePath, 'public');
+                        if (fs.existsSync(publicPath)) fs.copySync(publicPath, publicTemp);
+                    } else continue;
+                } catch (e) {
+                    console.error('Module extract fail:', file, e);
                 }
-                if (m.public) {
-                    ensureDir(path.resolve(os.tmpdir(), 'hydro', 'public'));
-                    for (const n in m.public) {
-                        if (m.public[n] === null) {
-                            ensureDir(path.resolve(os.tmpdir(), 'hydro', 'public', n));
-                        } else {
-                            const e = path.resolve(os.tmpdir(), 'hydro', 'public', n);
-                            fs.writeFileSync(e, Buffer.from(m.public[n], 'base64'), { mode: 755 });
-                        }
-                    }
-                }
-                fs.writeFileSync(
-                    root(`${os.tmpdir()}/hydro/tmp/${id}/hydro.json`),
-                    JSON.stringify({
-                        size: s.size,
-                        version: m.version,
-                        id: m.id,
-                        name: m.name,
-                        description: m.description,
-                    }),
-                );
-            } catch (e) {
-                if (e.code === 'Z_DATA_ERROR') {
-                    console.error(`Module Load Fail: ${file} (File Corrupted)`);
-                } else console.error(`Module Load Fail: ${file} ${e}`);
             }
         }
     }