diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index 08bdf9709..c29b26624 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -27,14 +27,12 @@ "moment-timezone": "^0.5.32", "mongodb": "^3.6.3", "nodemailer": "^6.4.17", - "ps-list": "^7.2.0", "serialize-javascript": "^5.0.1", "sockjs": "^0.3.20", "superagent": "^6.1.0", "supports-color": "^8.0.0", "systeminformation": "^4.31.1", "terminal-kit": "^1.44.0", - "wtfnode": "^0.8.4", "yargs": "^16.2.0" }, "devDependencies": { @@ -54,7 +52,6 @@ "@types/superagent": "^4.1.8", "@types/supports-color": "^7.2.0", "@types/terminal-kit": "^1.28.2", - "@types/wtfnode": "^0.7.0", "@types/yargs": "^15.0.11" }, "scripts": { diff --git a/packages/hydrooj/src/entry/common.ts b/packages/hydrooj/src/entry/common.ts index e0e98cef3..8b6834c0c 100644 --- a/packages/hydrooj/src/entry/common.ts +++ b/packages/hydrooj/src/entry/common.ts @@ -18,8 +18,8 @@ export const builtinLib = [ export const builtinModel = [ 'builtin', 'document', 'domain', 'blacklist', 'opcount', 'setting', 'token', 'user', 'problem', 'record', - 'contest', 'message', 'solution', 'training', 'file', - 'discussion', 'system', 'oplog', + 'contest', 'message', 'solution', 'training', 'oplog', + 'discussion', 'system', ]; export const builtinHandler = [ @@ -53,7 +53,7 @@ export async function handler(pending: string[], fail: string[]) { } catch (e) { fail.push(i); logger.error('Handler Load Fail: %s', i); - logger.error('%o', e); + logger.error(e); } } } @@ -77,7 +77,7 @@ export async function locale(pending: string[], fail: string[]) { } catch (e) { fail.push(i); logger.error('Locale Load Fail: %s', i); - logger.error('%o', e); + logger.error(e); } } } diff --git a/packages/hydrooj/src/entry/master.ts b/packages/hydrooj/src/entry/master.ts index 77bddff8b..e69993a2f 100644 --- a/packages/hydrooj/src/entry/master.ts +++ b/packages/hydrooj/src/entry/master.ts @@ -3,7 +3,6 @@ import os from 'os'; import path from 'path'; import fs from 'fs-extra'; -import pslist from 'ps-list'; import { argv } from 'yargs'; import { builtinModel } from './common'; import { Entry } from '../loader'; @@ -22,13 +21,11 @@ export async function load(call: Entry) { if (fs.existsSync(lockfile) && !argv.ignorelock) { try { const file = require(lockfile); - const plist = await pslist(); - if (file.pid && plist.map((i) => i.pid).includes(file.pid)) { - logger.error(`Lockfile exists, pid=${file.pid}`); - process.exit(1); - } + process.kill(file.pid, 0); + logger.error(`Lockfile exists, pid=${file.pid}`); + process.exit(1); } catch { - // Invalid lockfile. ignore. + // Invalid lockfile or process not exist } } const context = { diff --git a/packages/hydrooj/src/handler/manage.ts b/packages/hydrooj/src/handler/manage.ts index 08b807732..69435774d 100644 --- a/packages/hydrooj/src/handler/manage.ts +++ b/packages/hydrooj/src/handler/manage.ts @@ -79,7 +79,7 @@ class SystemDashboardHandler extends SystemHandler { async postRestart() { const count = system.get('server.worker'); - process.send({ event: 'restart', count }); + process.send({ event: 'restart', payload: [count] }); this.back(); } } diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 440a7079e..30e7adcdd 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -598,7 +598,7 @@ declare global { stat: any, ui: UI, error: typeof import('./error'), - Logger: typeof import('./logger'), + Logger: typeof import('./logger').Logger, locales: Dict>, isFirstWorker: boolean, }, diff --git a/packages/hydrooj/src/loader.ts b/packages/hydrooj/src/loader.ts index 2468f2a4b..1296148b9 100644 --- a/packages/hydrooj/src/loader.ts +++ b/packages/hydrooj/src/loader.ts @@ -1,3 +1,4 @@ +/* eslint-disable import/first */ /* eslint-disable no-continue */ /* eslint-disable import/no-dynamic-require */ /* eslint-disable no-await-in-loop */ @@ -6,10 +7,8 @@ import './interface'; import os from 'os'; import path from 'path'; import cluster from 'cluster'; -import wtfnode from 'wtfnode'; import fs from 'fs-extra'; import { argv } from 'yargs'; -import AdmZip from 'adm-zip'; import './utils'; export * from './interface'; @@ -42,27 +41,13 @@ if (!global.Hydro) { global.addons = []; } -// eslint-disable-next-line import/first import { Logger } from './logger'; -// eslint-disable-next-line import/first +import './ui'; import * as bus from './service/bus'; const logger = new Logger('loader'); -wtfnode.setLogger('error', logger.error.bind(logger)); -wtfnode.setLogger('warn', logger.warn.bind(logger)); -wtfnode.setLogger('info', logger.info.bind(logger)); logger.debug('%o', argv); -async function terminate() { - if (argv.debug) wtfnode.dump(); - try { - await bus.parallel('app/exit'); - } catch (e) { - process.exit(1); - } - process.exit(0); -} - async function fork(args: string[] = []) { const _args = process.argv.slice(2); _args.push(...args, `--addons=${Buffer.from(JSON.stringify(global.addons)).toString('base64')}`); @@ -108,16 +93,6 @@ async function startWorker(cnt: number) { for (let i = 1; i < cnt; i++) await fork(); } -async function executeCommand(input: string) { - try { - const t = eval(input.toString().trim()); - if (t instanceof Promise) console.log(await t); - else console.log(t); - } catch (e) { - console.warn(e); - } -} - async function reload(count = 1) { logger.info('Reloading'); await stopWorker(); @@ -125,30 +100,30 @@ async function reload(count = 1) { await startWorker(count); } -async function messageHandler(worker: cluster.Worker, msg: any) { - if (!msg) msg = worker; - if (msg.event) { - if (msg.event === 'bus') { - if (cluster.isMaster) { - for (const i in cluster.workers) { - cluster.workers[i].send(msg); - } - } else { - global.Hydro.service.bus.emit(msg.eventName, ...msg.payload); - } - } else if (msg.event === 'stat') { - global.Hydro.stat.reqCount += msg.count; - } else if (msg.event === 'restart') { - await reload(msg.count); - } else if (msg.event === 'run') { - await executeCommand(msg.command); +const shell = new Logger('shell'); +async function executeCommand(input: string) { + // Clear the stack + setImmediate(async () => { + if (input === 'restart') return reload(); + if (input === 'exit' || input === 'quit' || input === 'shutdown') { + return process.kill(process.pid, 'SIGINT'); } - } + try { + // eslint-disable-next-line no-eval + shell.info(await eval(input.trim())); + } catch (e) { + shell.warn(e); + } + return true; + }); } -const moduleTemp = path.resolve(os.tmpdir(), 'hydro', 'module'); +bus.on('message/reload', reload); +bus.on('message/run', executeCommand); +process.on('unhandledRejection', logger.error); +process.on('uncaughtException', logger.error); + const publicTemp = path.resolve(os.tmpdir(), 'hydro', 'public'); -const tmp = path.resolve(os.tmpdir(), 'hydro', '__'); export function addon(addonPath: string) { let modulePath = path.resolve(process.cwd(), addonPath); @@ -163,53 +138,14 @@ export function addon(addonPath: string) { } catch (e) { logger.error(`Addon not found: ${addonPath}`); } - } else if (modulePath.endsWith('.hydro')) { - try { - // Is *.hydro module - const t = modulePath.split(path.sep); - const name = t[t.length - 1].split('.')[0]; - const zip = new AdmZip(modulePath); - const targetPath = path.resolve(moduleTemp, name); - zip.extractAllTo(targetPath, true); - const content = fs.readdirSync(targetPath); - const ipath = path.join(targetPath, content[0]); - if (content.length === 1 && fs.statSync(ipath).isDirectory()) { - fs.moveSync(ipath, tmp); - fs.rmdirSync(targetPath); - fs.moveSync(tmp, targetPath); - } - const publicPath = path.resolve(targetPath, 'public'); - if (fs.existsSync(publicPath)) fs.copySync(publicPath, publicTemp); - global.addons.push(targetPath); - } catch (e) { - logger.error('Addon load fail: ', e); - throw e; - } } else logger.error(`Addon not found: ${addonPath}`); } -process.on('unhandledRejection', logger.error); -process.on('SIGINT', terminate); - export async function load() { addon(path.resolve(__dirname, '..')); Error.stackTraceLimit = 50; - process.on('message', messageHandler); - cluster.on('message', messageHandler); if (cluster.isMaster || argv.startAsMaster) { logger.info(`Master ${process.pid} Starting`); - process.stdin.setEncoding('utf8'); - process.stdin.on('data', (buf) => { - const input = buf.toString(); - if (input[0] === '@') { - for (const i in cluster.workers) { - cluster.workers[i].send({ event: 'run', command: input.substr(1, input.length - 1) }); - break; - } - } else { - executeCommand(input); - } - }); const cnt = await entry({ entry: 'master' }); logger.info('Master started'); cluster.on('exit', (worker, code, signal) => { @@ -245,10 +181,10 @@ export async function load() { export async function loadCli() { await entry({ entry: 'cli' }); - return terminate(); + process.kill(process.pid, 'SIGINT'); } -if (argv.pandora || !module.parent) { +if (argv.pandora || require.main === module) { const func = argv._[0] === 'cli' ? load : loadCli; func().catch((e) => { logger.error(e); diff --git a/packages/hydrooj/src/logger.ts b/packages/hydrooj/src/logger.ts index 2583ca3a9..48529f35f 100644 --- a/packages/hydrooj/src/logger.ts +++ b/packages/hydrooj/src/logger.ts @@ -1,85 +1,25 @@ -import { inspect, InspectOptions, format } from 'util'; +import { inspect, format } from 'util'; import { argv } from 'yargs'; -import { stderr } from 'supports-color'; -import { } from './loader'; - -namespace Time { - export const second = 1000; - export const minute = second * 60; - export const hour = minute * 60; - export const day = hour * 24; - export const week = day * 7; - export function formatTimeShort(ms: number) { - const abs = Math.abs(ms); - if (abs >= day - hour / 2) { - return `${Math.round(ms / day)}d`; - } if (abs >= hour - minute / 2) { - return `${Math.round(ms / hour)}h`; - } if (abs >= minute - second / 2) { - return `${Math.round(ms / minute)}m`; - } if (abs >= second) { - return `${Math.round(ms / second)}s`; - } - return `${ms}ms`; - } -} - -const colors = stderr.level < 2 ? [6, 2, 3, 4, 5, 1] : [ - 20, 21, 26, 27, 32, 33, 38, 39, 40, 41, 42, 43, 44, 45, 56, 57, 62, - 63, 68, 69, 74, 75, 76, 77, 78, 79, 80, 81, 92, 93, 98, 99, 112, 113, - 129, 134, 135, 148, 149, 160, 161, 162, 163, 164, 165, 166, 167, 168, - 169, 170, 171, 172, 173, 178, 179, 184, 185, 196, 197, 198, 199, 200, - 201, 202, 203, 204, 205, 206, 207, 208, 209, 214, 215, 220, 221, -]; +import { Time } from './utils'; const instances: Record = {}; - type LogFunction = (format: any, ...param: any[]) => void - type LogType = 'success' | 'error' | 'info' | 'warn' | 'debug' - export interface Logger extends Record { } - export class Logger { static readonly SUCCESS = 1 - static readonly ERROR = 1 - static readonly INFO = 2 - static readonly WARN = 2 - static readonly DEBUG = 3 - static baseLevel = argv.debug ? 3 : 2 - static showDiff = false - static levels: Record = {} - static lastTime = 0 - - static options: InspectOptions = { - colors: stderr.hasBasic, - } - static formatters: Record string> = { - c: Logger.prototype.color, - C: (value) => Logger.color(15, value, ';1'), - o: (value) => inspect(value, Logger.options).replace(/\s*\n\s*/g, ' '), + o: (value) => inspect(value).replace(/\s*\n\s*/g, ' '), } - static color(code: number, value: any, decoration = '') { - if (!Logger.options.colors) return `${value}`; - return `\u001B[3${code < 8 ? code : `8;5;${code}`}${decoration}m${value}\u001B[0m`; - } - - private code: number - - private displayName: string - - public stream: NodeJS.WritableStream = process.stderr - constructor(public name: string, private showDiff = false) { if (name in instances) return instances[name]; let hash = 0; @@ -88,8 +28,6 @@ export class Logger { hash |= 0; } instances[name] = this; - this.code = colors[Math.abs(hash) % colors.length]; - this.displayName = name ? this.color(`${name} `, ';1') : ''; this.createMethod('success', '[S] ', Logger.SUCCESS); this.createMethod('error', '[E] ', Logger.ERROR); this.createMethod('info', '[I] ', Logger.INFO); @@ -97,14 +35,12 @@ export class Logger { this.createMethod('debug', '[D] ', Logger.DEBUG); } - private color(value: any, decoration = '') { - return Logger.color(this.code, value, decoration); - } - private createMethod(name: LogType, prefix: string, minLevel: number) { this[name] = (...args: [any, ...any[]]) => { if (this.level < minLevel) return; - this.stream.write(`${prefix + this.displayName + this.format(...args)}\n`); + const msg = `${prefix} ${this.name} ${this.format(...args)}`; + if (process.send) process.send({ event: 'message/log', payload: [msg] }); + else global.Hydro.service.bus.parallel('message/log', msg); }; } @@ -131,15 +67,11 @@ export class Logger { }).split('\n').join('\n '); if (Logger.showDiff || this.showDiff) { const now = Date.now(); - if (Logger.lastTime) { - args.push(this.color(`+${Time.formatTimeShort(now - Logger.lastTime)}`)); - } + if (Logger.lastTime) args.push(`+${Time.formatTimeShort(now - Logger.lastTime)}`); Logger.lastTime = now; } return format(...args); } } -// This fails in jest -// @ts-ignore -global.Hydro.Logger = { Logger }; +global.Hydro.Logger = Logger; diff --git a/packages/hydrooj/src/options.ts b/packages/hydrooj/src/options.ts index c0d27db05..4ee655c16 100644 --- a/packages/hydrooj/src/options.ts +++ b/packages/hydrooj/src/options.ts @@ -1,6 +1,9 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; +import { Logger } from './logger'; + +const logger = new Logger('options'); export = function load() { let f = path.resolve(process.cwd(), 'config.json'); @@ -13,7 +16,7 @@ export = function load() { try { result = JSON.parse(fs.readFileSync(f).toString()); } catch (e) { - console.error('Cannot read config file', e); + logger.error('Cannot read config file %o', e); result = {}; } return result; diff --git a/packages/hydrooj/src/service/bus.ts b/packages/hydrooj/src/service/bus.ts index 2b670f5e4..9f9123c12 100644 --- a/packages/hydrooj/src/service/bus.ts +++ b/packages/hydrooj/src/service/bus.ts @@ -3,12 +3,11 @@ import cluster from 'cluster'; import { Db, FilterQuery, OnlyFieldsOfType } from 'mongodb'; import { argv } from 'yargs'; import { Logger } from '../logger'; -import { +import type { Mdoc, Pdoc, Rdoc, TrainingDoc, User, } from '../interface'; -import { DomainDoc } from '../loader'; - -type DocType = import('../model/document').DocType; +import type { DomainDoc } from '../loader'; +import type { DocType } from '../model/document'; const _hooks: Record any>> = {}; const logger = new Logger('bus', true); @@ -32,6 +31,10 @@ export interface EventMap { 'app/load/service': () => VoidReturn 'app/exit': () => VoidReturn + 'message/log': (message: string) => VoidReturn + 'message/reload': (count: number) => VoidReturn + 'message/run': (command: string) => VoidReturn + 'database/connect': (db: Db) => void 'database/config': () => void @@ -152,6 +155,24 @@ export function boardcast(event: K, ...payload: Parame } else parallel(event, ...payload); } +async function messageHandler(worker: cluster.Worker, msg: any) { + if (!msg) msg = worker; + if (msg.event) { + if (msg.event === 'bus') { + if (cluster.isMaster) { + for (const i in cluster.workers) { + cluster.workers[i].send(msg); + } + } else global.Hydro.service.bus.emit(msg.eventName, ...msg.payload); + } else if (msg.event === 'stat') { + global.Hydro.stat.reqCount += msg.count; + } else await emit(msg.event, ...msg.payload); + } +} + +process.on('message', messageHandler); +cluster.on('message', messageHandler); + global.Hydro.service.bus = { addListener, bail, boardcast, emit, on, off, once, parallel, prependListener, removeListener, serial, }; diff --git a/packages/hydrooj/src/ui.ts b/packages/hydrooj/src/ui.ts new file mode 100644 index 000000000..beb36643a --- /dev/null +++ b/packages/hydrooj/src/ui.ts @@ -0,0 +1,67 @@ +// @ts-nocheck +import cluster from 'cluster'; +import { + terminal, TextBox, LabeledInput, +} from 'terminal-kit'; +import * as bus from './service/bus'; + +async function terminate() { + let hasError = false; + try { + await require('./service/bus').parallel('app/exit'); + } catch (e) { + hasError = true; + } + if (cluster.isMaster) { + terminal.hideCursor(false); + terminal.styleReset(); + terminal.resetScrollingRegion(); + terminal.moveTo(terminal.width, terminal.height); + terminal('\n'); + } + process.exit(hasError ? 1 : 0); +} +process.on('SIGINT', terminate); +if (cluster.isMaster) { + terminal.clear(); + terminal.grabInput(); + terminal.hideCursor(); + // Clipboard doesn't work well over ssh env, it throws an error. + terminal.getClipboard = () => { }; + terminal.setClipboard = () => { }; + const document = terminal.createDocument({}); + const LogBox = new TextBox({ + parent: document, + contentHasMarkup: true, + scrollable: true, + vScrollBar: true, + lineWrap: true, + x: 0, + y: 0, + width: terminal.width, + height: terminal.height - 1, + }); + const ShellInput = new LabeledInput({ + parent: document, + label: '>', + x: 1, + y: terminal.height - 1, + width: terminal.width, + }); + bus.on('message/log', (message) => { + LogBox.appendLog(message); + }); + ShellInput.on('submit', async (input) => { + ShellInput.input.setContent(' '.repeat(terminal.width), false, false); + ShellInput.input.setContent('', false, false); + if (input[0] === '@') { + for (const i in cluster.workers) { + cluster.workers[i].send({ event: 'message/run', payload: [input.substr(1, input.length - 1)] }); + break; + } + } else bus.parallel('message/run', input); + }); + terminal.on('key', (key, matches, data) => { + if (key === 'CTRL_C') terminate(); + }); +} diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index bbf51bbae..1ab0bf4e6 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -69,23 +69,19 @@ const scripts: UpgradeScript[] = [ collChunk.deleteMany({ files_id: _id }), ]); } - terminal.clear('Changing storage engine. This may take a long time.'); + logger.info('Changing storage engine. This may take a long time.'); // Problem file and User file try { let savedProgress = system.get('upgrade.file.progress.domain'); if (savedProgress) savedProgress = JSON.parse(savedProgress); else savedProgress = { pdocs: [] }; - const ddocs = await domain.getMulti().toArray(); - const totalProgress = terminal.progressBar({ - items: ddocs.length, title: 'Domain', y: 3, eta: true, - }); + const ddocs = await domain.getMulti().project({ _id: 1 }).toArray(); + logger.info('Total found %d domains.', ddocs.length); for (let i = 0; i < ddocs.length; i++) { const ddoc = ddocs[i]; - totalProgress.startItem(ddoc._id); + logger.info('Domain %s (%d/%d)', ddoc._id, i + 1, ddocs.length); const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray(); - const domainProgress = terminal.progressBar({ - items: pdocs.length, title: 'Problems', y: 4, eta: true, - }); + const domainProgress = terminal.progressBar({ items: pdocs.length, title: 'Problems', inline: true }); for (let j = 0; j < pdocs.length; j++) { const pdoc = pdocs[j]; domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`); @@ -99,7 +95,7 @@ const scripts: UpgradeScript[] = [ )); } catch (e) { if (e.toString().includes('FileNotFound')) { - logger.error('\n\n\n\n\nProblem data not found %s/%s', pdoc.domainId, pdoc.docId); + logger.error('Problem data not found %s/%s', pdoc.domainId, pdoc.docId); } else throw e; } savedProgress.pdocs.push(`${pdoc.domainId}/${pdoc.docId}`); @@ -108,9 +104,7 @@ const scripts: UpgradeScript[] = [ domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`); } domainProgress.stop(); - totalProgress.itemDone(ddoc._id); } - totalProgress.stop(); const udocs = await user.getMulti().project({ _id: 1, uname: 1 }).toArray(); const userfileProgress = terminal.progressBar({ items: udocs.length, title: 'Users', y: 3, @@ -140,7 +134,7 @@ const scripts: UpgradeScript[] = [ userfileProgress.stop(); return true; } catch (e) { - logger.error('\n\n\n\n\n%o', e); + logger.error(e); return false; } }, diff --git a/packages/hydrooj/src/utils.ts b/packages/hydrooj/src/utils.ts index 8553a19ba..e2afd3d4a 100644 --- a/packages/hydrooj/src/utils.ts +++ b/packages/hydrooj/src/utils.ts @@ -196,9 +196,7 @@ export function buildProjection(fields: string[]): Record { export function sleep(timeout: number) { return new Promise((resolve) => { - setTimeout(() => { - resolve(); - }, timeout); + setTimeout(resolve, timeout); }); } @@ -214,9 +212,7 @@ function deepen(modifyString: (source: string) => string) { } return function t(source: T): T { - if (typeof source === 'string') { - return modifyString(source) as any; - } + if (typeof source === 'string') return modifyString(source) as any; return modifyObject(source); }; } @@ -224,3 +220,24 @@ function deepen(modifyString: (source: string) => string) { export const camelCase = deepen((source) => source.replace(/[_-][a-z]/g, (str) => str.slice(1).toUpperCase())); export const paramCase = deepen((source) => source.replace(/_/g, '-').replace(/(? `-${str.toLowerCase()}`)); export const snakeCase = deepen((source) => source.replace(/-/g, '_').replace(/(? `_${str.toLowerCase()}`)); + +export namespace Time { + export const second = 1000; + export const minute = second * 60; + export const hour = minute * 60; + export const day = hour * 24; + export const week = day * 7; + export function formatTimeShort(ms: number) { + const abs = Math.abs(ms); + if (abs >= day - hour / 2) { + return `${Math.round(ms / day)}d`; + } if (abs >= hour - minute / 2) { + return `${Math.round(ms / hour)}h`; + } if (abs >= minute - second / 2) { + return `${Math.round(ms / minute)}m`; + } if (abs >= second) { + return `${Math.round(ms / second)}s`; + } + return `${ms}ms`; + } +}