From fd7c9cbeb17dd887690e27e41daa3cfa1952ffcc Mon Sep 17 00:00:00 2001 From: undefined Date: Tue, 20 Sep 2022 22:34:36 +0800 Subject: [PATCH] core: experimental hot reload support --- packages/center/handler.ts | 6 +- packages/elastic/script.ts | 57 ++--- packages/fps-importer/handler.ts | 7 +- packages/hydrooj/src/entry/common.ts | 7 +- packages/hydrooj/src/entry/worker.ts | 15 +- packages/hydrooj/src/handler/api.ts | 9 +- packages/hydrooj/src/handler/blog.ts | 12 +- packages/hydrooj/src/handler/compat.ts | 6 +- packages/hydrooj/src/handler/index.ts | 17 -- packages/hydrooj/src/handler/notfound.ts | 26 --- packages/hydrooj/src/loader.ts | 1 + packages/hydrooj/src/script/blacklist.ts | 13 +- packages/hydrooj/src/script/checkUpdate.ts | 3 +- packages/hydrooj/src/script/deleteUser.ts | 3 +- packages/hydrooj/src/script/problemStat.ts | 13 +- packages/hydrooj/src/script/rating.ts | 10 +- packages/hydrooj/src/script/storageUsage.ts | 5 +- packages/hydrooj/src/service/module.ts | 56 +++++ packages/hydrooj/src/service/router.ts | 9 +- packages/hydrooj/src/service/server.ts | 8 +- packages/hydrooj/src/service/watcher.ts | 235 ++++++++++++++++++++ packages/migrate/script.ts | 12 +- packages/migrate/scripts/hustoj.ts | 11 +- packages/migrate/scripts/syzoj.ts | 13 +- packages/migrate/scripts/vijos.ts | 11 +- packages/sonic/script.ts | 12 +- 26 files changed, 429 insertions(+), 148 deletions(-) delete mode 100644 packages/hydrooj/src/handler/index.ts delete mode 100644 packages/hydrooj/src/handler/notfound.ts create mode 100644 packages/hydrooj/src/service/module.ts create mode 100644 packages/hydrooj/src/service/watcher.ts diff --git a/packages/center/handler.ts b/packages/center/handler.ts index eb256bb23..feac6c147 100644 --- a/packages/center/handler.ts +++ b/packages/center/handler.ts @@ -63,8 +63,6 @@ class DataReportHandler extends Handler { } } -export function apply() { - Route('data_report', '/center/report', DataReportHandler); +export function apply(ctx) { + ctx.Route('data_report', '/center/report', DataReportHandler); } - -global.Hydro.handler.center = apply; diff --git a/packages/elastic/script.ts b/packages/elastic/script.ts index 43f08187f..e40954c69 100644 --- a/packages/elastic/script.ts +++ b/packages/elastic/script.ts @@ -1,6 +1,6 @@ import { Client } from '@elastic/elasticsearch'; import { omit } from 'lodash'; -import { addScript, Schema } from 'hydrooj'; +import { Schema } from 'hydrooj'; import DomainModel from 'hydrooj/src/model/domain'; import ProblemModel, { ProblemDoc } from 'hydrooj/src/model/problem'; import * as system from 'hydrooj/src/model/system'; @@ -24,27 +24,6 @@ const processDocument = (doc: Partial) => { return omit(doc, indexOmit); }; -bus.on('problem/add', async (doc, docId) => { - await client.index({ - index: 'problem', - id: `${doc.domainId}/${docId}`, - document: processDocument(doc), - }); -}); -bus.on('problem/edit', async (pdoc) => { - await client.index({ - index: 'problem', - id: `${pdoc.domainId}/${pdoc.docId}`, - document: processDocument(pdoc), - }); -}); -bus.on('problem/del', async (domainId, docId) => { - await client.delete({ - index: 'problem', - id: `${domainId}/${docId}`, - }); -}); - global.Hydro.lib.problemSearch = async (domainId, q, opts) => { const allowedSize = system.get('elasic-search.indexSize') || 10000; const size = opts?.limit || system.get('pagination.problem'); @@ -98,8 +77,32 @@ async function run({ domainId }, report) { return true; } -addScript('ensureElasticSearch', 'Elastic problem search re-index') - .args(Schema.object({ - domainId: Schema.string(), - })) - .action(run); +export const apply = (ctx) => { + bus.on('problem/add', async (doc, docId) => { + await client.index({ + index: 'problem', + id: `${doc.domainId}/${docId}`, + document: processDocument(doc), + }); + }); + bus.on('problem/edit', async (pdoc) => { + await client.index({ + index: 'problem', + id: `${pdoc.domainId}/${pdoc.docId}`, + document: processDocument(pdoc), + }); + }); + bus.on('problem/del', async (domainId, docId) => { + await client.delete({ + index: 'problem', + id: `${domainId}/${docId}`, + }); + }); + ctx.addScript( + 'ensureElasticSearch', 'Elastic problem search re-index', + Schema.object({ + domainId: Schema.string(), + }), + run, + ); +}; diff --git a/packages/fps-importer/handler.ts b/packages/fps-importer/handler.ts index f6569802e..e147eece6 100644 --- a/packages/fps-importer/handler.ts +++ b/packages/fps-importer/handler.ts @@ -129,9 +129,8 @@ class FpsProblemImportHandler extends Handler { } } -export async function apply() { - Route('problem_import_fps', '/problem/import/fps', FpsProblemImportHandler, PERM.PERM_CREATE_PROBLEM); +export const sideEffect = true; +export async function apply(ctx) { + ctx.Route('problem_import_fps', '/problem/import/fps', FpsProblemImportHandler, PERM.PERM_CREATE_PROBLEM); ProblemAdd('problem_import_fps', {}, 'copy', 'From FPS File'); } - -global.Hydro.handler.fpsImport = apply; diff --git a/packages/hydrooj/src/entry/common.ts b/packages/hydrooj/src/entry/common.ts index d03753c2c..40be86b18 100644 --- a/packages/hydrooj/src/entry/common.ts +++ b/packages/hydrooj/src/entry/common.ts @@ -8,6 +8,7 @@ import yaml from 'js-yaml'; import i18n from '../lib/i18n'; import { Logger } from '../logger'; import * as bus from '../service/bus'; +import { Runtime } from '../service/module'; const logger = new Logger('common'); @@ -36,7 +37,11 @@ const getLoader = (type: string, filename: string) => async function loader(pend if (p && !fail.includes(i)) { try { logger.info(`${type.replace(/^(.)/, (t) => t.toUpperCase())} init: %s`, i); - require(p); + const res = require(p); + if (res.apply) { + const runtime = new Runtime(p); + runtime.load(res); + } } catch (e) { fail.push(i); logger.info(`${type.replace(/^(.)/, (t) => t.toUpperCase())} load fail: %s`, i); diff --git a/packages/hydrooj/src/entry/worker.ts b/packages/hydrooj/src/entry/worker.ts index 9ba609163..afb22d891 100644 --- a/packages/hydrooj/src/entry/worker.ts +++ b/packages/hydrooj/src/entry/worker.ts @@ -8,12 +8,13 @@ import { Logger } from '../logger'; import options from '../options'; import * as bus from '../service/bus'; import db from '../service/db'; +import { Runtime } from '../service/module'; import { handler, lib, locale, model, script, service, setting, template, } from './common'; const logger = new Logger('worker'); -const detail = cac().parse().options.loaderDetail; +const { loaderDetail: detail, watch } = cac().parse().options; const tmpdir = path.resolve(os.tmpdir(), 'hydro'); export async function load() { @@ -53,11 +54,18 @@ export async function load() { const server = require('../service/server'); await server.prepare(); if (detail) logger.info('finish: server'); + if (watch) require('../service/watcher'); await service(pending, fail); if (detail) logger.info('finish: service.extra'); require('../model/index'); if (detail) logger.info('finish: model.builtin'); - require('../handler/index'); + const handlerDir = path.resolve(__dirname, '..', 'handler'); + const handlers = await fs.readdir(handlerDir); + for (const h of handlers) { + const f = path.resolve(handlerDir, h); + const m = require(f); + if (m.apply) new Runtime(f).load(m); + } if (detail) logger.info('finish: handler.builtin'); await model(pending, fail); if (detail) logger.info('finish: model.extra'); @@ -68,8 +76,6 @@ export async function load() { if (detail) logger.info('finish: handler.extra'); for (const i in global.Hydro.handler) await global.Hydro.handler[i](); if (detail) logger.info('finish: handler.apply'); - const notfound = require('../handler/notfound'); - await notfound.apply(); require('../script/index'); if (detail) logger.info('finish: script.builtin'); await script(pending, fail); @@ -98,5 +104,6 @@ export async function load() { } logger.success('Server started'); process.send?.('ready'); + await bus.serial('app/ready'); return { fail }; } diff --git a/packages/hydrooj/src/handler/api.ts b/packages/hydrooj/src/handler/api.ts index 26b08c49a..5dbce3213 100644 --- a/packages/hydrooj/src/handler/api.ts +++ b/packages/hydrooj/src/handler/api.ts @@ -2,7 +2,7 @@ import { makeExecutableSchema } from '@graphql-tools/schema'; import { graphql, GraphQLSchema } from 'graphql'; import { resolvers, typeDefs } from 'graphql-scalars'; import * as bus from '../service/bus'; -import { Handler, Route } from '../service/server'; +import { Handler } from '../service/server'; const types: Record> = {}; const unions: Record = {}; @@ -123,8 +123,7 @@ class ApiHandler extends Handler { } } -export function apply() { - Route('api', '/api', ApiHandler); +export const sideEffect = true; +export function apply(ctx) { + ctx.Route('api', '/api', ApiHandler); } - -global.Hydro.handler.api = apply; diff --git a/packages/hydrooj/src/handler/blog.ts b/packages/hydrooj/src/handler/blog.ts index f4fb74155..c0b838d58 100644 --- a/packages/hydrooj/src/handler/blog.ts +++ b/packages/hydrooj/src/handler/blog.ts @@ -119,11 +119,9 @@ class BlogEditHandler extends BlogHandler { } } -export async function apply() { - Route('blog_main', '/blog/:uid', BlogUserHandler); - Route('blog_create', '/blog/:uid/create', BlogEditHandler, PRIV.PRIV_USER_PROFILE); - Route('blog_detail', '/blog/:uid/:did', BlogDetailHandler); - Route('blog_edit', '/blog/:uid/:did/edit', BlogEditHandler, PRIV.PRIV_USER_PROFILE); +export async function apply(ctx) { + ctx.Route('blog_main', '/blog/:uid', BlogUserHandler); + ctx.Route('blog_create', '/blog/:uid/create', BlogEditHandler, PRIV.PRIV_USER_PROFILE); + ctx.Route('blog_detail', '/blog/:uid/:did', BlogDetailHandler); + ctx.Route('blog_edit', '/blog/:uid/:did/edit', BlogEditHandler, PRIV.PRIV_USER_PROFILE); } - -global.Hydro.handler.blog = apply; diff --git a/packages/hydrooj/src/handler/compat.ts b/packages/hydrooj/src/handler/compat.ts index f25fb23d6..1c178b1b1 100644 --- a/packages/hydrooj/src/handler/compat.ts +++ b/packages/hydrooj/src/handler/compat.ts @@ -1,4 +1,4 @@ -import { Handler, Route } from '../service/server'; +import { Handler } from '../service/server'; class ProblemCategoryCompatHandler extends Handler { async get({ category }) { @@ -6,6 +6,6 @@ class ProblemCategoryCompatHandler extends Handler { } } -global.Hydro.handler.compat = () => { - Route('problem_category_compat', '/p/category/:category', ProblemCategoryCompatHandler); +export const apply = (ctx) => { + ctx.Route('problem_category_compat', '/p/category/:category', ProblemCategoryCompatHandler); }; diff --git a/packages/hydrooj/src/handler/index.ts b/packages/hydrooj/src/handler/index.ts deleted file mode 100644 index d70cd464f..000000000 --- a/packages/hydrooj/src/handler/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -import './home'; -import './problem'; -import './record'; -import './judge'; -import './user'; -import './contest'; -import './training'; -import './discussion'; -import './manage'; -import './import'; -import './misc'; -import './homework'; -import './domain'; -import './status'; -import './api'; -import './blog'; -import './compat'; diff --git a/packages/hydrooj/src/handler/notfound.ts b/packages/hydrooj/src/handler/notfound.ts deleted file mode 100644 index 2cc9c326c..000000000 --- a/packages/hydrooj/src/handler/notfound.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* eslint-disable no-empty-function */ -/* eslint-disable class-methods-use-this */ -import { NotFoundError } from '../error'; -import { - Handler, param, Route, Types, -} from '../service/server'; - -class NotFoundHandler extends Handler { - @param('bsod', Types.Boolean) - prepare(domainId: string, bsod: boolean) { - if (bsod) throw new Error(this.request.path); - throw new NotFoundError(this.request.path); - } - - get() { } - post() { } - head() { } - put() { } - delete() { } -} - -export async function apply() { - Route('notfound', '(/.*)+', NotFoundHandler); -} - -global.Hydro.handler.notfound = apply; diff --git a/packages/hydrooj/src/loader.ts b/packages/hydrooj/src/loader.ts index d38920db3..87c9ad358 100644 --- a/packages/hydrooj/src/loader.ts +++ b/packages/hydrooj/src/loader.ts @@ -51,6 +51,7 @@ export function addon(addonPath: string, prepend = false) { } } +/** @deprecated */ export function addScript(name: string, description: string) { if (global.Hydro.script[name]) throw new Error(`duplicate script ${name} registered.`); return { diff --git a/packages/hydrooj/src/script/blacklist.ts b/packages/hydrooj/src/script/blacklist.ts index cb8d4e35c..7ba295244 100644 --- a/packages/hydrooj/src/script/blacklist.ts +++ b/packages/hydrooj/src/script/blacklist.ts @@ -1,7 +1,6 @@ /* eslint-disable @typescript-eslint/no-use-before-define */ import { ObjectID } from 'mongodb'; import Schema from 'schemastery'; -import { addScript } from '../loader'; import blacklist from '../model/blacklist'; import * as discussion from '../model/discussion'; import * as document from '../model/document'; @@ -61,8 +60,9 @@ async function _user( if (!dryrun) await UserModel.ban(uid); } -addScript('blacklist', 'Add blacklist by ip, uid') - .args(Schema.object({ +export const apply = (ctx) => ctx.addScript( + 'blacklist', 'Add blacklist by ip, uid', + Schema.object({ address: Schema.string(), discuss: Schema.object({ domainId: Schema.string(), @@ -70,8 +70,8 @@ addScript('blacklist', 'Add blacklist by ip, uid') }), user: Schema.number(), dryrun: Schema.boolean(), - })) - .action(async ({ + }), + async ({ address = null, discuss = null, user = null, dryrun = true, }, report) => { if (address) await _address(address, new Set(), new Set(), new Set(), dryrun, report); @@ -83,4 +83,5 @@ addScript('blacklist', 'Add blacklist by ip, uid') } if (user) await _user(user, new Set(), new Set(), new Set(), dryrun, report); return true; - }); + }, +); diff --git a/packages/hydrooj/src/script/checkUpdate.ts b/packages/hydrooj/src/script/checkUpdate.ts index af001b061..1a18a7b98 100644 --- a/packages/hydrooj/src/script/checkUpdate.ts +++ b/packages/hydrooj/src/script/checkUpdate.ts @@ -2,7 +2,6 @@ import Schema from 'schemastery'; import * as semver from 'semver'; import superagent from 'superagent'; -import { addScript } from '../loader'; import MessageModel from '../model/message'; import * as system from '../model/system'; @@ -38,4 +37,4 @@ export async function run(_: {}, report: Function) { return true; } -addScript('checkUpdate', 'Daily update check').args(Schema.any()).action(run); +export const apply = (ctx) => ctx.addScript('checkUpdate', 'Daily update check', Schema.any(), run); diff --git a/packages/hydrooj/src/script/deleteUser.ts b/packages/hydrooj/src/script/deleteUser.ts index fc99c2cc0..a096d875b 100644 --- a/packages/hydrooj/src/script/deleteUser.ts +++ b/packages/hydrooj/src/script/deleteUser.ts @@ -1,5 +1,4 @@ import Schema from 'schemastery'; -import { addScript } from '../loader'; import { PRIV } from '../model/builtin'; import user from '../model/user'; import db from '../service/db'; @@ -9,7 +8,7 @@ const collDomainUser = db.collection('domain.user'); const collRecord = db.collection('record'); const collMessage = db.collection('message'); -addScript('deleteUser', 'Delete a user') +export const apply = (ctx) => ctx.addScript('deleteUser', 'Delete a user') .args(Schema.object({ uid: Schema.number(), })) diff --git a/packages/hydrooj/src/script/problemStat.ts b/packages/hydrooj/src/script/problemStat.ts index 6b6e983ca..dcbe538db 100644 --- a/packages/hydrooj/src/script/problemStat.ts +++ b/packages/hydrooj/src/script/problemStat.ts @@ -2,7 +2,6 @@ /* eslint-disable no-await-in-loop */ import { ObjectID } from 'mongodb'; import Schema from 'schemastery'; -import { addScript } from '../loader'; import { STATUS } from '../model/builtin'; import * as document from '../model/document'; import db from '../service/db'; @@ -153,15 +152,17 @@ export async function pdoc(report) { if (bulk.length) await bulk.execute(); } -addScript('problemStat', 'Recalculates nSubmit and nAccept in problem status.') - .args(Schema.object({ +export const apply = (ctx) => ctx.addScript( + 'problemStat', 'Recalculates nSubmit and nAccept in problem status.', + Schema.object({ udoc: Schema.boolean(), pdoc: Schema.boolean(), psdoc: Schema.boolean(), - })) - .action(async (arg, report) => { + }), + async (arg, report) => { if (arg.pdoc === undefined || arg.pdoc) await pdoc(report); if (arg.udoc === undefined || arg.udoc) await udoc(report); if (arg.psdoc === undefined || arg.psdoc) await psdoc(report); return true; - }); + }, +); diff --git a/packages/hydrooj/src/script/rating.ts b/packages/hydrooj/src/script/rating.ts index 6e41dd9f0..7a6968681 100644 --- a/packages/hydrooj/src/script/rating.ts +++ b/packages/hydrooj/src/script/rating.ts @@ -6,7 +6,6 @@ import Schema from 'schemastery'; import { Tdoc, Udoc } from '../interface'; import difficultyAlgorithm from '../lib/difficulty'; import rating from '../lib/rating'; -import { addScript } from '../loader'; import { PRIV, STATUS } from '../model/builtin'; import * as contest from '../model/contest'; import domain from '../model/domain'; @@ -184,8 +183,7 @@ export async function run({ domainId }, report: Function) { return true; } -addScript('rp', 'Calculate rp of a domain, or all domains') - .args(Schema.object({ - domainId: Schema.string(), - })) - .action(run); +export const apply = (ctx) => ctx.addScript( + 'rp', 'Calculate rp of a domain, or all domains', + Schema.object({ domainId: Schema.string() }), run, +); diff --git a/packages/hydrooj/src/script/storageUsage.ts b/packages/hydrooj/src/script/storageUsage.ts index 3b9d6afa8..af1298cc4 100644 --- a/packages/hydrooj/src/script/storageUsage.ts +++ b/packages/hydrooj/src/script/storageUsage.ts @@ -1,6 +1,5 @@ /* eslint-disable no-await-in-loop */ import Schema from 'schemastery'; -import { addScript } from '../loader'; import { STATUS } from '../model/builtin'; import storage from '../model/storage'; @@ -37,6 +36,4 @@ export async function run(_, report) { return true; } -addScript('storageUsage', 'Calculate storage usage') - .args(Schema.any()) - .action(run); +export const apply = (ctx) => ctx.addScript('storageUsage', 'Calculate storage usage', Schema.any(), run); diff --git a/packages/hydrooj/src/service/module.ts b/packages/hydrooj/src/service/module.ts new file mode 100644 index 000000000..714989c0a --- /dev/null +++ b/packages/hydrooj/src/service/module.ts @@ -0,0 +1,56 @@ +import Schema from 'schemastery'; +import * as bus from './bus'; + +export const runtimes: Record = Object.create(null); + +function addScript(name: string, description: string, validate: Schema, run: (args: K, report: any) => boolean | Promise) { + if (global.Hydro.script[name]) throw new Error(`duplicate script ${name} registered.`); + global.Hydro.script[name] = { description, validate, run }; + return () => delete global.Hydro.script[name]; +} + +export interface Context { + Route: typeof import('./server').Route; + Connection: typeof import('./server').Connection; + on: typeof bus['on']; + once: typeof bus['once']; + off: typeof bus['off']; + addScript: typeof addScript; +} + +export class Runtime { + private disposables = []; + public sideEffect = false; + public loaded = false; + + constructor(public filename: string) { + runtimes[filename] = this; + } + + load(module: any) { + if (this.loaded) throw new Error(`Module ${this.filename} was already loaded`); + this.sideEffect = false; + if (typeof module.dispose === 'function') this.disposables.push(module.dispose); + const T = (origFunc) => (origFunc ? (...args) => { + const res = origFunc(...args); + this.disposables.push(res); + return res; + } : null); + module.apply({ + Route: T(global.Hydro.service.server?.Route), + Connection: T(global.Hydro.service.server?.Connection), + on: T(bus.on), + once: T(bus.once), + off: bus.off, + addScript: T(addScript), + } as Context); + if (module.sideEffect) this.sideEffect = true; + this.loaded = true; + } + + dispose() { + this.disposables.map((i) => i()); + this.disposables = []; + this.loaded = false; + } +} diff --git a/packages/hydrooj/src/service/router.ts b/packages/hydrooj/src/service/router.ts index ca74ad350..c4248b418 100644 --- a/packages/hydrooj/src/service/router.ts +++ b/packages/hydrooj/src/service/router.ts @@ -6,6 +6,10 @@ import type WebSocket from 'ws'; import type { KoaContext } from './server'; type WebSocketCallback = (socket: WebSocket, request: IncomingMessage, ctx: KoaContext) => void; +function remove(list: T[], item: T) { + const index = list.indexOf(item); + if (index >= 0) list.splice(index, 1); +} export class WebSocketLayer { clients = new Set(); @@ -34,20 +38,21 @@ export class WebSocketLayer { export class Router extends KoaRouter { wsStack: WebSocketLayer[] = []; + disposeLastOp = () => null; /** * hack into router methods to make sure that koa middlewares are disposable */ register(...args: Parameters) { const layer = super.register(...args); - // TODO handle unregister + this.disposeLastOp = () => remove(this.stack, layer); return layer; } ws(path: Parameters[0], callback?: WebSocketCallback) { const layer = new WebSocketLayer(this, path, callback); this.wsStack.push(layer); - // TODO handle unregistry + this.disposeLastOp = () => remove(this.wsStack, layer); return layer; } } diff --git a/packages/hydrooj/src/service/server.ts b/packages/hydrooj/src/service/server.ts index 6c2c16e11..9b34b6714 100644 --- a/packages/hydrooj/src/service/server.ts +++ b/packages/hydrooj/src/service/server.ts @@ -363,6 +363,7 @@ const Checker = (permPrivChecker) => { export function Route(name: string, path: string, RouteHandler: any, ...permPrivChecker) { router.all(name, path, (ctx) => handle(ctx as any, RouteHandler, Checker(permPrivChecker))); + return router.disposeLastOp; } export class ConnectionHandler extends HandlerCommon { @@ -429,11 +430,16 @@ export function Connection( let started = false; +class NotFoundHandler extends Handler { + prepare() { throw new NotFoundError(this.request.path); } + all() { } +} + // TODO use postInit? export async function start() { if (started) return; const port = system.get('server.port'); - app.use(router.routes()).use(router.allowedMethods()); + app.use(router.routes()).use(router.allowedMethods()).use((ctx) => handle(ctx, NotFoundHandler, () => true)); await new Promise((resolve) => { httpServer.listen(argv.options.port || port, () => { logger.success('Server listening at: %d', argv.options.port || port); diff --git a/packages/hydrooj/src/service/watcher.ts b/packages/hydrooj/src/service/watcher.ts new file mode 100644 index 000000000..423b428a7 --- /dev/null +++ b/packages/hydrooj/src/service/watcher.ts @@ -0,0 +1,235 @@ +/* eslint-disable consistent-return */ +/* eslint-disable import/no-dynamic-require */ +/* eslint-disable @typescript-eslint/no-shadow */ +import { relative, resolve } from 'path'; +import { FSWatcher, watch } from 'chokidar'; +import { debounce } from 'lodash'; +import { Logger } from '../logger'; +import * as bus from './bus'; +import { Runtime, runtimes } from './module'; + +function loadDependencies(filename: string, ignored: Set) { + const dependencies = new Set(); + function traverse({ filename, children }: NodeJS.Module) { + if (ignored.has(filename) || dependencies.has(filename) || filename.includes('/node_modules/')) return; + dependencies.add(filename); + children.forEach(traverse); + } + traverse(require.cache[filename]); + return dependencies; +} + +export function unwrapExports(module: any) { + return module?.default || module; +} + +const logger = new Logger('watch'); + +export function coerce(val: any) { + // resolve error when stack is undefined, e.g. axios error with status code 401 + const { message, stack } = val instanceof Error && val.stack ? val : new Error(val as any); + const lines = stack.split('\n'); + const index = lines.findIndex((line) => line.endsWith(message)); + return lines.slice(index).join('\n'); +} + +class Watcher { + private root: string; + private watcher: FSWatcher; + private externals: Set; + private accepted: Set; + private declined: Set; + private stashed = new Set(); + + constructor(private config: any) { + this.externals = new Set(Object.keys(require.cache)); + bus.on('app/ready', () => this.start()); + } + + start() { + this.root = resolve(process.cwd()); + this.watcher = watch(this.root, { + ...this.config, + ignored: ['**/node_modules/**', '**/.git/**', '**/logs/**', '**/.cache/**'], + }); + logger.info(`Start watching changes in ${this.root}`); + + // files independent from any plugins will trigger a full reload + const triggerLocalReload = debounce(() => this.triggerLocalReload(), 1000); + + this.watcher.on('change', (path) => { + logger.debug('change detected:', relative(this.root, path)); + + if (this.externals.has(path)) { + logger.warn('Require full reload'); + } else if (require.cache[path]) { + this.stashed.add(path); + triggerLocalReload(); + } + }); + } + + stop() { + return this.watcher.close(); + } + + private analyzeChanges() { + /** files pending classification */ + const pending: string[] = []; + + this.accepted = new Set(this.stashed); + this.declined = new Set(this.externals); + + this.stashed.forEach((filename) => { + const { children } = require.cache[filename]; + for (const { filename } of children) { + if (this.accepted.has(filename) || this.declined.has(filename) || filename.includes('/node_modules/')) continue; + pending.push(filename); + } + }); + + while (pending.length) { + let index = 0; + let hasUpdate = false; + while (index < pending.length) { + const filename = pending[index]; + const { children } = require.cache[filename]; + let isDeclined = true; + let isAccepted = false; + for (const { filename } of children) { + if (this.declined.has(filename) || filename.includes('/node_modules/')) continue; + if (this.accepted.has(filename)) { + isAccepted = true; + break; + } else { + isDeclined = false; + if (!pending.includes(filename)) { + hasUpdate = true; + pending.push(filename); + } + } + } + if (isAccepted || isDeclined) { + hasUpdate = true; + pending.splice(index, 1); + if (isAccepted) { + this.accepted.add(filename); + } else { + this.declined.add(filename); + } + } else { + index++; + } + } + // infinite loop + if (!hasUpdate) break; + } + + for (const filename of pending) { + this.declined.add(filename); + } + } + + private triggerLocalReload() { + const start = Date.now(); + this.analyzeChanges(); + console.log(this.accepted); + + /** plugins pending classification */ + const pending = new Map(); + + /** plugins that should be reloaded */ + const reloads = new Map(); + + // we assume that plugin entry files are "atomic" + // that is, reloading them will not cause any other reloads + for (const filename in require.cache) { + const module = require.cache[filename]; + const plugin = unwrapExports(module.exports); + const runtime = runtimes[filename]; + if (!runtime || this.declined.has(filename)) continue; + pending.set(filename, runtime); + if (!runtimes[filename]?.sideEffect && !plugin?.['sideEffect']) this.declined.add(filename); + } + + for (const [filename, runtime] of pending) { + // check if it is a dependent of the changed file + this.declined.delete(filename); + const dependencies = [...loadDependencies(filename, this.declined)]; + + // we only detect reloads at plugin level + // a plugin will be reloaded if any of its dependencies are accepted + if (!dependencies.some((dep) => this.accepted.has(dep))) continue; + dependencies.forEach((dep) => this.accepted.add(dep)); + + // prepare for reload + let isMarked = false; + const visited = new Set(); + const queued = [runtime]; + while (queued.length) { + const runtime = queued.shift(); + if (visited.has(runtime)) continue; + visited.add(runtime); + if (reloads.has(runtime)) { + isMarked = true; + break; + } + } + if (!isMarked) reloads.set(runtime, filename); + } + + const backup: Record = {}; + for (const filename of this.accepted) { + backup[filename] = require.cache[filename]; + delete require.cache[filename]; + } + + function rollback() { + for (const filename in backup) { + require.cache[filename] = backup[filename]; + } + } + + logger.debug('Will reload the following file(s): %o', this.accepted.keys()); + const attempts = {}; + try { + for (const [, filename] of reloads) { + attempts[filename] = unwrapExports(require(filename)); + } + } catch (err) { + logger.warn(err); + return rollback(); + } + + try { + for (const [runtime, filename] of reloads) { + const path = relative(this.root, filename); + + try { + runtime.dispose(); + } catch (err) { + logger.warn(`failed to dispose plugin at %c\n${coerce(err)}`, path); + } + + try { + runtime.load(attempts[filename]); + logger.info('reload plugin at %c', path); + } catch (err) { + logger.warn(`failed to reload plugin at %c\n${coerce(err)}`, path); + throw err; + } + } + } catch { + // rollback require.cache and plugin states + rollback(); + logger.warn('Rolling back changes'); + return; + } + + // reset stashed files + this.stashed = new Set(); + logger.success('Reload done in %d ms', Date.now() - start); + } +} + +export default new Watcher({}); diff --git a/packages/migrate/script.ts b/packages/migrate/script.ts index e0eed6d17..e3920f0ac 100644 --- a/packages/migrate/script.ts +++ b/packages/migrate/script.ts @@ -1,3 +1,9 @@ -import './scripts/hustoj'; -import './scripts/vijos'; -import './scripts/syzoj'; +import { apply as hustoj } from './scripts/hustoj'; +import { apply as syzoj } from './scripts/syzoj'; +import { apply as vijos } from './scripts/vijos'; + +export function apply(ctx) { + hustoj(ctx); + vijos(ctx); + syzoj(ctx); +} diff --git a/packages/migrate/scripts/hustoj.ts b/packages/migrate/scripts/hustoj.ts index f4e1f3060..cab44a455 100644 --- a/packages/migrate/scripts/hustoj.ts +++ b/packages/migrate/scripts/hustoj.ts @@ -6,7 +6,7 @@ import mysql from 'mysql'; import TurndownService from 'turndown'; import { STATUS } from '@hydrooj/utils/lib/status'; import { noop, Time } from '@hydrooj/utils/lib/utils'; -import { addScript, Schema } from 'hydrooj'; +import { Schema } from 'hydrooj'; import { NotFoundError } from 'hydrooj/src/error'; import { postJudge } from 'hydrooj/src/handler/judge'; import { RecordDoc } from 'hydrooj/src/interface'; @@ -311,8 +311,9 @@ export async function run({ return true; } -addScript('migrateHustoj', 'migrate from hustoj') - .args(Schema.object({ +export const apply = (ctx) => ctx.addScript( + 'migrateHustoj', 'migrate from hustoj', + Schema.object({ host: Schema.string().required(), port: Schema.number().required(), name: Schema.string().required(), @@ -321,4 +322,6 @@ addScript('migrateHustoj', 'migrate from hustoj') domainId: Schema.string().required(), contestType: Schema.string().required(), dataDir: Schema.string().required(), - })).action(run); + }), + run, +); diff --git a/packages/migrate/scripts/syzoj.ts b/packages/migrate/scripts/syzoj.ts index 9a3cf0e38..034239cf6 100644 --- a/packages/migrate/scripts/syzoj.ts +++ b/packages/migrate/scripts/syzoj.ts @@ -5,7 +5,7 @@ import mariadb from 'mariadb'; import { ObjectID } from 'mongodb'; import { STATUS } from '@hydrooj/utils/lib/status'; import { noop, Time } from '@hydrooj/utils/lib/utils'; -import { addScript, Schema } from 'hydrooj'; +import { Schema } from 'hydrooj'; import { NotFoundError } from 'hydrooj/src/error'; import { postJudge } from 'hydrooj/src/handler/judge'; import { DiscussionDoc, DiscussionReplyDoc, RecordDoc } from 'hydrooj/src/interface'; @@ -227,7 +227,7 @@ export async function run({ hint: pdoc.limit_and_hint, }); for (const match of content.matchAll(fileReg)) { - const [,origialPath, pid, filename] = match; + const [, origialPath, pid, filename] = match; if (!problemAdditionalFile[`P${pdoc.id}`]) problemAdditionalFile[`P${pdoc.id}`] = [{ fromPid: pid, filename }]; else problemAdditionalFile[`P${pdoc.id}`].push({ fromPid: pid, filename }); content = content.replace(origialPath, `file://${filename}`); @@ -508,8 +508,9 @@ export async function run({ return true; } -addScript('migrateSyzoj', 'migrate from syzoj') - .args(Schema.object({ +export const apply = (ctx) => ctx.addScript( + 'migrateSyzoj', 'migrate from syzoj', + Schema.object({ host: Schema.string().required(), port: Schema.number().required(), name: Schema.string().required(), @@ -517,4 +518,6 @@ addScript('migrateSyzoj', 'migrate from syzoj') password: Schema.string().required(), domainId: Schema.string().default('system'), dataDir: Schema.string().default('/opt/syzoj/web/uploads'), - })).action(run); + }), + run, +); diff --git a/packages/migrate/scripts/vijos.ts b/packages/migrate/scripts/vijos.ts index 2d324cfa2..98cb98cf9 100644 --- a/packages/migrate/scripts/vijos.ts +++ b/packages/migrate/scripts/vijos.ts @@ -2,7 +2,7 @@ /* eslint-disable no-await-in-loop */ import mongodb, { Cursor, Db } from 'mongodb'; import { - addScript, DiscussionTailReplyDoc, MessageDoc, + DiscussionTailReplyDoc, MessageDoc, RecordDoc, Schema, TestCase, TrainingNode, } from 'hydrooj'; @@ -507,11 +507,14 @@ export async function run({ return true; } -addScript('migrateVijos', 'migrate from vijos') - .args(Schema.object({ +export const apply = (ctx) => ctx.addScript( + 'migrateVijos', 'migrate from vijos', + Schema.object({ host: Schema.string().required(), port: Schema.number().required(), name: Schema.string().required(), username: Schema.string().required(), password: Schema.string().required(), - })).action(run); + }), + run, +); diff --git a/packages/sonic/script.ts b/packages/sonic/script.ts index dcd46b55f..3013b6207 100644 --- a/packages/sonic/script.ts +++ b/packages/sonic/script.ts @@ -1,4 +1,4 @@ -import { addScript, Schema } from 'hydrooj'; +import { Schema } from 'hydrooj'; import DomainModel from 'hydrooj/src/model/domain'; import { iterateAllProblem, iterateAllProblemInDomain } from 'hydrooj/src/pipelineUtils'; import sonic from './service'; @@ -28,8 +28,10 @@ async function run({ domainId }, report) { return true; } -addScript('ensureSonicSearch', 'Sonic problem search re-index') - .args(Schema.object({ +export const apply = (ctx) => ctx.addScript( + 'ensureSonicSearch', 'Sonic problem search re-index', + Schema.object({ domainId: Schema.string(), - })) - .action(run); + }), + run, +);