core: add ui

packages/hydrooj/package.json

@@ -27,14 +27,12 @@
         "moment-timezone": "^0.5.32",
         "mongodb": "^3.6.3",
         "nodemailer": "^6.4.17",
-        "ps-list": "^7.2.0",
         "serialize-javascript": "^5.0.1",
         "sockjs": "^0.3.20",
         "superagent": "^6.1.0",
         "supports-color": "^8.0.0",
         "systeminformation": "^4.31.1",
         "terminal-kit": "^1.44.0",
-        "wtfnode": "^0.8.4",
         "yargs": "^16.2.0"
     },
     "devDependencies": {
@@ -54,7 +52,6 @@
         "@types/superagent": "^4.1.8",
         "@types/supports-color": "^7.2.0",
         "@types/terminal-kit": "^1.28.2",
-        "@types/wtfnode": "^0.7.0",
         "@types/yargs": "^15.0.11"
     },
     "scripts": {

packages/hydrooj/src/entry/common.ts

@@ -18,8 +18,8 @@ export const builtinLib = [
 export const builtinModel = [
     'builtin', 'document', 'domain', 'blacklist', 'opcount',
     'setting', 'token', 'user', 'problem', 'record',
-    'contest', 'message', 'solution', 'training', 'file',
-    'discussion', 'system', 'oplog',
+    'contest', 'message', 'solution', 'training', 'oplog',
+    'discussion', 'system',
 ];
 
 export const builtinHandler = [
@@ -53,7 +53,7 @@ export async function handler(pending: string[], fail: string[]) {
             } catch (e) {
                 fail.push(i);
                 logger.error('Handler Load Fail: %s', i);
-                logger.error('%o', e);
+                logger.error(e);
             }
         }
     }
@@ -77,7 +77,7 @@ export async function locale(pending: string[], fail: string[]) {
             } catch (e) {
                 fail.push(i);
                 logger.error('Locale Load Fail: %s', i);
-                logger.error('%o', e);
+                logger.error(e);
             }
         }
     }

packages/hydrooj/src/entry/master.ts

@@ -3,7 +3,6 @@
 import os from 'os';
 import path from 'path';
 import fs from 'fs-extra';
-import pslist from 'ps-list';
 import { argv } from 'yargs';
 import { builtinModel } from './common';
 import { Entry } from '../loader';
@@ -22,13 +21,11 @@ export async function load(call: Entry) {
     if (fs.existsSync(lockfile) && !argv.ignorelock) {
         try {
             const file = require(lockfile);
-            const plist = await pslist();
-            if (file.pid && plist.map((i) => i.pid).includes(file.pid)) {
-                logger.error(`Lockfile exists, pid=${file.pid}`);
-                process.exit(1);
-            }
+            process.kill(file.pid, 0);
+            logger.error(`Lockfile exists, pid=${file.pid}`);
+            process.exit(1);
         } catch {
-            // Invalid lockfile. ignore.
+            // Invalid lockfile or process not exist
         }
     }
     const context = {

packages/hydrooj/src/handler/manage.ts

@@ -79,7 +79,7 @@ class SystemDashboardHandler extends SystemHandler {
 
     async postRestart() {
         const count = system.get('server.worker');
-        process.send({ event: 'restart', count });
+        process.send({ event: 'restart', payload: [count] });
         this.back();
     }
 }

packages/hydrooj/src/interface.ts

@@ -598,7 +598,7 @@ declare global {
                 stat: any,
                 ui: UI,
                 error: typeof import('./error'),
-                Logger: typeof import('./logger'),
+                Logger: typeof import('./logger').Logger,
                 locales: Dict<Dict<string>>,
                 isFirstWorker: boolean,
             },

packages/hydrooj/src/loader.ts

@@ -1,3 +1,4 @@
+/* eslint-disable import/first */
 /* eslint-disable no-continue */
 /* eslint-disable import/no-dynamic-require */
 /* eslint-disable no-await-in-loop */
@@ -6,10 +7,8 @@ import './interface';
 import os from 'os';
 import path from 'path';
 import cluster from 'cluster';
-import wtfnode from 'wtfnode';
 import fs from 'fs-extra';
 import { argv } from 'yargs';
-import AdmZip from 'adm-zip';
 import './utils';
 
 export * from './interface';
@@ -42,27 +41,13 @@ if (!global.Hydro) {
     global.addons = [];
 }
 
-// eslint-disable-next-line import/first
 import { Logger } from './logger';
-// eslint-disable-next-line import/first
+import './ui';
 import * as bus from './service/bus';
 
 const logger = new Logger('loader');
-wtfnode.setLogger('error', logger.error.bind(logger));
-wtfnode.setLogger('warn', logger.warn.bind(logger));
-wtfnode.setLogger('info', logger.info.bind(logger));
 logger.debug('%o', argv);
 
-async function terminate() {
-    if (argv.debug) wtfnode.dump();
-    try {
-        await bus.parallel('app/exit');
-    } catch (e) {
-        process.exit(1);
-    }
-    process.exit(0);
-}
-
 async function fork(args: string[] = []) {
     const _args = process.argv.slice(2);
     _args.push(...args, `--addons=${Buffer.from(JSON.stringify(global.addons)).toString('base64')}`);
@@ -108,16 +93,6 @@ async function startWorker(cnt: number) {
     for (let i = 1; i < cnt; i++) await fork();
 }
 
-async function executeCommand(input: string) {
-    try {
-        const t = eval(input.toString().trim());
-        if (t instanceof Promise) console.log(await t);
-        else console.log(t);
-    } catch (e) {
-        console.warn(e);
-    }
-}
-
 async function reload(count = 1) {
     logger.info('Reloading');
     await stopWorker();
@@ -125,30 +100,30 @@ async function reload(count = 1) {
     await startWorker(count);
 }
 
-async function messageHandler(worker: cluster.Worker, msg: any) {
-    if (!msg) msg = worker;
-    if (msg.event) {
-        if (msg.event === 'bus') {
-            if (cluster.isMaster) {
-                for (const i in cluster.workers) {
-                    cluster.workers[i].send(msg);
-                }
-            } else {
-                global.Hydro.service.bus.emit(msg.eventName, ...msg.payload);
-            }
-        } else if (msg.event === 'stat') {
-            global.Hydro.stat.reqCount += msg.count;
-        } else if (msg.event === 'restart') {
-            await reload(msg.count);
-        } else if (msg.event === 'run') {
-            await executeCommand(msg.command);
+const shell = new Logger('shell');
+async function executeCommand(input: string) {
+    // Clear the stack
+    setImmediate(async () => {
+        if (input === 'restart') return reload();
+        if (input === 'exit' || input === 'quit' || input === 'shutdown') {
+            return process.kill(process.pid, 'SIGINT');
         }
-    }
+        try {
+            // eslint-disable-next-line no-eval
+            shell.info(await eval(input.trim()));
+        } catch (e) {
+            shell.warn(e);
+        }
+        return true;
+    });
 }
 
-const moduleTemp = path.resolve(os.tmpdir(), 'hydro', 'module');
+bus.on('message/reload', reload);
+bus.on('message/run', executeCommand);
+process.on('unhandledRejection', logger.error);
+process.on('uncaughtException', logger.error);
+
 const publicTemp = path.resolve(os.tmpdir(), 'hydro', 'public');
-const tmp = path.resolve(os.tmpdir(), 'hydro', '__');
 
 export function addon(addonPath: string) {
     let modulePath = path.resolve(process.cwd(), addonPath);
@@ -163,53 +138,14 @@ export function addon(addonPath: string) {
         } catch (e) {
             logger.error(`Addon not found: ${addonPath}`);
         }
-    } else if (modulePath.endsWith('.hydro')) {
-        try {
-            // Is *.hydro module
-            const t = modulePath.split(path.sep);
-            const name = t[t.length - 1].split('.')[0];
-            const zip = new AdmZip(modulePath);
-            const targetPath = path.resolve(moduleTemp, name);
-            zip.extractAllTo(targetPath, true);
-            const content = fs.readdirSync(targetPath);
-            const ipath = path.join(targetPath, content[0]);
-            if (content.length === 1 && fs.statSync(ipath).isDirectory()) {
-                fs.moveSync(ipath, tmp);
-                fs.rmdirSync(targetPath);
-                fs.moveSync(tmp, targetPath);
-            }
-            const publicPath = path.resolve(targetPath, 'public');
-            if (fs.existsSync(publicPath)) fs.copySync(publicPath, publicTemp);
-            global.addons.push(targetPath);
-        } catch (e) {
-            logger.error('Addon load fail: ', e);
-            throw e;
-        }
     } else logger.error(`Addon not found: ${addonPath}`);
 }
 
-process.on('unhandledRejection', logger.error);
-process.on('SIGINT', terminate);
-
 export async function load() {
     addon(path.resolve(__dirname, '..'));
     Error.stackTraceLimit = 50;
-    process.on('message', messageHandler);
-    cluster.on('message', messageHandler);
     if (cluster.isMaster || argv.startAsMaster) {
         logger.info(`Master ${process.pid} Starting`);
-        process.stdin.setEncoding('utf8');
-        process.stdin.on('data', (buf) => {
-            const input = buf.toString();
-            if (input[0] === '@') {
-                for (const i in cluster.workers) {
-                    cluster.workers[i].send({ event: 'run', command: input.substr(1, input.length - 1) });
-                    break;
-                }
-            } else {
-                executeCommand(input);
-            }
-        });
         const cnt = await entry({ entry: 'master' });
         logger.info('Master started');
         cluster.on('exit', (worker, code, signal) => {
@@ -245,10 +181,10 @@ export async function load() {
 
 export async function loadCli() {
     await entry({ entry: 'cli' });
-    return terminate();
+    process.kill(process.pid, 'SIGINT');
 }
 
-if (argv.pandora || !module.parent) {
+if (argv.pandora || require.main === module) {
     const func = argv._[0] === 'cli' ? load : loadCli;
     func().catch((e) => {
         logger.error(e);

packages/hydrooj/src/logger.ts

@@ -1,85 +1,25 @@
-import { inspect, InspectOptions, format } from 'util';
+import { inspect, format } from 'util';
 import { argv } from 'yargs';
-import { stderr } from 'supports-color';
-import { } from './loader';
-
-namespace Time {
-    export const second = 1000;
-    export const minute = second * 60;
-    export const hour = minute * 60;
-    export const day = hour * 24;
-    export const week = day * 7;
-    export function formatTimeShort(ms: number) {
-        const abs = Math.abs(ms);
-        if (abs >= day - hour / 2) {
-            return `${Math.round(ms / day)}d`;
-        } if (abs >= hour - minute / 2) {
-            return `${Math.round(ms / hour)}h`;
-        } if (abs >= minute - second / 2) {
-            return `${Math.round(ms / minute)}m`;
-        } if (abs >= second) {
-            return `${Math.round(ms / second)}s`;
-        }
-        return `${ms}ms`;
-    }
-}
-
-const colors = stderr.level < 2 ? [6, 2, 3, 4, 5, 1] : [
-    20, 21, 26, 27, 32, 33, 38, 39, 40, 41, 42, 43, 44, 45, 56, 57, 62,
-    63, 68, 69, 74, 75, 76, 77, 78, 79, 80, 81, 92, 93, 98, 99, 112, 113,
-    129, 134, 135, 148, 149, 160, 161, 162, 163, 164, 165, 166, 167, 168,
-    169, 170, 171, 172, 173, 178, 179, 184, 185, 196, 197, 198, 199, 200,
-    201, 202, 203, 204, 205, 206, 207, 208, 209, 214, 215, 220, 221,
-];
+import { Time } from './utils';
 
 const instances: Record<string, Logger> = {};
-
 type LogFunction = (format: any, ...param: any[]) => void
-
 type LogType = 'success' | 'error' | 'info' | 'warn' | 'debug'
-
 export interface Logger extends Record<LogType, LogFunction> { }
-
 export class Logger {
     static readonly SUCCESS = 1
-
     static readonly ERROR = 1
-
     static readonly INFO = 2
-
     static readonly WARN = 2
-
     static readonly DEBUG = 3
-
     static baseLevel = argv.debug ? 3 : 2
-
     static showDiff = false
-
     static levels: Record<string, number> = {}
-
     static lastTime = 0
-
-    static options: InspectOptions = {
-        colors: stderr.hasBasic,
-    }
-
     static formatters: Record<string, (this: Logger, value: any) => string> = {
-        c: Logger.prototype.color,
-        C: (value) => Logger.color(15, value, ';1'),
-        o: (value) => inspect(value, Logger.options).replace(/\s*\n\s*/g, ' '),
+        o: (value) => inspect(value).replace(/\s*\n\s*/g, ' '),
     }
 
-    static color(code: number, value: any, decoration = '') {
-        if (!Logger.options.colors) return `${value}`;
-        return `\u001B[3${code < 8 ? code : `8;5;${code}`}${decoration}m${value}\u001B[0m`;
-    }
-
-    private code: number
-
-    private displayName: string
-
-    public stream: NodeJS.WritableStream = process.stderr
-
     constructor(public name: string, private showDiff = false) {
         if (name in instances) return instances[name];
         let hash = 0;
@@ -88,8 +28,6 @@ export class Logger {
             hash |= 0;
         }
         instances[name] = this;
-        this.code = colors[Math.abs(hash) % colors.length];
-        this.displayName = name ? this.color(`${name} `, ';1') : '';
         this.createMethod('success', '[S] ', Logger.SUCCESS);
         this.createMethod('error', '[E] ', Logger.ERROR);
         this.createMethod('info', '[I] ', Logger.INFO);
@@ -97,14 +35,12 @@ export class Logger {
         this.createMethod('debug', '[D] ', Logger.DEBUG);
     }
 
-    private color(value: any, decoration = '') {
-        return Logger.color(this.code, value, decoration);
-    }
-
     private createMethod(name: LogType, prefix: string, minLevel: number) {
         this[name] = (...args: [any, ...any[]]) => {
             if (this.level < minLevel) return;
-            this.stream.write(`${prefix + this.displayName + this.format(...args)}\n`);
+            const msg = `${prefix} ${this.name} ${this.format(...args)}`;
+            if (process.send) process.send({ event: 'message/log', payload: [msg] });
+            else global.Hydro.service.bus.parallel('message/log', msg);
         };
     }
 
@@ -131,15 +67,11 @@ export class Logger {
         }).split('\n').join('\n    ');
         if (Logger.showDiff || this.showDiff) {
             const now = Date.now();
-            if (Logger.lastTime) {
-                args.push(this.color(`+${Time.formatTimeShort(now - Logger.lastTime)}`));
-            }
+            if (Logger.lastTime) args.push(`+${Time.formatTimeShort(now - Logger.lastTime)}`);
             Logger.lastTime = now;
         }
         return format(...args);
     }
 }
 
-// This fails in jest
-// @ts-ignore
-global.Hydro.Logger = { Logger };
+global.Hydro.Logger = Logger;

packages/hydrooj/src/options.ts

@@ -1,6 +1,9 @@
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
+import { Logger } from './logger';
+
+const logger = new Logger('options');
 
 export = function load() {
     let f = path.resolve(process.cwd(), 'config.json');
@@ -13,7 +16,7 @@ export = function load() {
     try {
         result = JSON.parse(fs.readFileSync(f).toString());
     } catch (e) {
-        console.error('Cannot read config file', e);
+        logger.error('Cannot read config file %o', e);
         result = {};
     }
     return result;

packages/hydrooj/src/service/bus.ts

@@ -3,12 +3,11 @@ import cluster from 'cluster';
 import { Db, FilterQuery, OnlyFieldsOfType } from 'mongodb';
 import { argv } from 'yargs';
 import { Logger } from '../logger';
-import {
+import type {
     Mdoc, Pdoc, Rdoc, TrainingDoc, User,
 } from '../interface';
-import { DomainDoc } from '../loader';
-
-type DocType = import('../model/document').DocType;
+import type { DomainDoc } from '../loader';
+import type { DocType } from '../model/document';
 
 const _hooks: Record<keyof any, Array<(...args: any[]) => any>> = {};
 const logger = new Logger('bus', true);
@@ -32,6 +31,10 @@ export interface EventMap {
     'app/load/service': () => VoidReturn
     'app/exit': () => VoidReturn
 
+    'message/log': (message: string) => VoidReturn
+    'message/reload': (count: number) => VoidReturn
+    'message/run': (command: string) => VoidReturn
+
     'database/connect': (db: Db) => void
     'database/config': () => void
 
@@ -152,6 +155,24 @@ export function boardcast<K extends keyof EventMap>(event: K, ...payload: Parame
     } else parallel(event, ...payload);
 }
 
+async function messageHandler(worker: cluster.Worker, msg: any) {
+    if (!msg) msg = worker;
+    if (msg.event) {
+        if (msg.event === 'bus') {
+            if (cluster.isMaster) {
+                for (const i in cluster.workers) {
+                    cluster.workers[i].send(msg);
+                }
+            } else global.Hydro.service.bus.emit(msg.eventName, ...msg.payload);
+        } else if (msg.event === 'stat') {
+            global.Hydro.stat.reqCount += msg.count;
+        } else await emit(msg.event, ...msg.payload);
+    }
+}
+
+process.on('message', messageHandler);
+cluster.on('message', messageHandler);
+
 global.Hydro.service.bus = {
     addListener, bail, boardcast, emit, on, off, once, parallel, prependListener, removeListener, serial,
 };

packages/hydrooj/src/ui.ts

@@ -0,0 +1,67 @@
+// @ts-nocheck
+import cluster from 'cluster';
+import {
+    terminal, TextBox, LabeledInput,
+} from 'terminal-kit';
+import * as bus from './service/bus';
+
+async function terminate() {
+    let hasError = false;
+    try {
+        await require('./service/bus').parallel('app/exit');
+    } catch (e) {
+        hasError = true;
+    }
+    if (cluster.isMaster) {
+        terminal.hideCursor(false);
+        terminal.styleReset();
+        terminal.resetScrollingRegion();
+        terminal.moveTo(terminal.width, terminal.height);
+        terminal('\n');
+    }
+    process.exit(hasError ? 1 : 0);
+}
+process.on('SIGINT', terminate);
+if (cluster.isMaster) {
+    terminal.clear();
+    terminal.grabInput();
+    terminal.hideCursor();
+    // Clipboard doesn't work well over ssh env, it throws an error.
+    terminal.getClipboard = () => { };
+    terminal.setClipboard = () => { };
+    const document = terminal.createDocument({});
+    const LogBox = new TextBox({
+        parent: document,
+        contentHasMarkup: true,
+        scrollable: true,
+        vScrollBar: true,
+        lineWrap: true,
+        x: 0,
+        y: 0,
+        width: terminal.width,
+        height: terminal.height - 1,
+    });
+    const ShellInput = new LabeledInput({
+        parent: document,
+        label: '>',
+        x: 1,
+        y: terminal.height - 1,
+        width: terminal.width,
+    });
+    bus.on('message/log', (message) => {
+        LogBox.appendLog(message);
+    });
+    ShellInput.on('submit', async (input) => {
+        ShellInput.input.setContent(' '.repeat(terminal.width), false, false);
+        ShellInput.input.setContent('', false, false);
+        if (input[0] === '@') {
+            for (const i in cluster.workers) {
+                cluster.workers[i].send({ event: 'message/run', payload: [input.substr(1, input.length - 1)] });
+                break;
+            }
+        } else bus.parallel('message/run', input);
+    });
+    terminal.on('key', (key, matches, data) => {
+        if (key === 'CTRL_C') terminate();
+    });
+}

packages/hydrooj/src/upgrade.ts

@@ -69,23 +69,19 @@ const scripts: UpgradeScript[] = [
                 collChunk.deleteMany({ files_id: _id }),
             ]);
         }
-        terminal.clear('Changing storage engine. This may take a long time.');
+        logger.info('Changing storage engine. This may take a long time.');
         // Problem file and User file
         try {
             let savedProgress = system.get('upgrade.file.progress.domain');
             if (savedProgress) savedProgress = JSON.parse(savedProgress);
             else savedProgress = { pdocs: [] };
-            const ddocs = await domain.getMulti().toArray();
-            const totalProgress = terminal.progressBar({
-                items: ddocs.length, title: 'Domain', y: 3, eta: true,
-            });
+            const ddocs = await domain.getMulti().project({ _id: 1 }).toArray();
+            logger.info('Total found %d domains.', ddocs.length);
             for (let i = 0; i < ddocs.length; i++) {
                 const ddoc = ddocs[i];
-                totalProgress.startItem(ddoc._id);
+                logger.info('Domain %s (%d/%d)', ddoc._id, i + 1, ddocs.length);
                 const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray();
-                const domainProgress = terminal.progressBar({
-                    items: pdocs.length, title: 'Problems', y: 4, eta: true,
-                });
+                const domainProgress = terminal.progressBar({ items: pdocs.length, title: 'Problems', inline: true });
                 for (let j = 0; j < pdocs.length; j++) {
                     const pdoc = pdocs[j];
                     domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`);
@@ -99,7 +95,7 @@ const scripts: UpgradeScript[] = [
                             ));
                         } catch (e) {
                             if (e.toString().includes('FileNotFound')) {
-                                logger.error('\n\n\n\n\nProblem data not found %s/%s', pdoc.domainId, pdoc.docId);
+                                logger.error('Problem data not found %s/%s', pdoc.domainId, pdoc.docId);
                             } else throw e;
                         }
                         savedProgress.pdocs.push(`${pdoc.domainId}/${pdoc.docId}`);
@@ -108,9 +104,7 @@ const scripts: UpgradeScript[] = [
                     domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`);
                 }
                 domainProgress.stop();
-                totalProgress.itemDone(ddoc._id);
             }
-            totalProgress.stop();
             const udocs = await user.getMulti().project({ _id: 1, uname: 1 }).toArray();
             const userfileProgress = terminal.progressBar({
                 items: udocs.length, title: 'Users', y: 3,
@@ -140,7 +134,7 @@ const scripts: UpgradeScript[] = [
             userfileProgress.stop();
             return true;
         } catch (e) {
-            logger.error('\n\n\n\n\n%o', e);
+            logger.error(e);
             return false;
         }
     },

packages/hydrooj/src/utils.ts

@@ -196,9 +196,7 @@ export function buildProjection(fields: string[]): Record<string, 1> {
 
 export function sleep(timeout: number) {
     return new Promise((resolve) => {
-        setTimeout(() => {
-            resolve();
-        }, timeout);
+        setTimeout(resolve, timeout);
     });
 }
 
@@ -214,9 +212,7 @@ function deepen(modifyString: (source: string) => string) {
     }
 
     return function t<T>(source: T): T {
-        if (typeof source === 'string') {
-            return modifyString(source) as any;
-        }
+        if (typeof source === 'string') return modifyString(source) as any;
         return modifyObject(source);
     };
 }
@@ -224,3 +220,24 @@ function deepen(modifyString: (source: string) => string) {
 export const camelCase = deepen((source) => source.replace(/[_-][a-z]/g, (str) => str.slice(1).toUpperCase()));
 export const paramCase = deepen((source) => source.replace(/_/g, '-').replace(/(?<!^)[A-Z]/g, (str) => `-${str.toLowerCase()}`));
 export const snakeCase = deepen((source) => source.replace(/-/g, '_').replace(/(?<!^)[A-Z]/g, (str) => `_${str.toLowerCase()}`));
+
+export namespace Time {
+    export const second = 1000;
+    export const minute = second * 60;
+    export const hour = minute * 60;
+    export const day = hour * 24;
+    export const week = day * 7;
+    export function formatTimeShort(ms: number) {
+        const abs = Math.abs(ms);
+        if (abs >= day - hour / 2) {
+            return `${Math.round(ms / day)}d`;
+        } if (abs >= hour - minute / 2) {
+            return `${Math.round(ms / hour)}h`;
+        } if (abs >= minute - second / 2) {
+            return `${Math.round(ms / minute)}m`;
+        } if (abs >= second) {
+            return `${Math.round(ms / second)}s`;
+        }
+        return `${ms}ms`;
+    }
+}