core: add provideModule and injectUI API

packages/fps-importer/handler.ts

@@ -4,10 +4,9 @@ import decodeHTML from 'decode-html';
 import fs from 'fs-extra';
 import { filter } from 'lodash';
 import xml2js from 'xml2js';
-import type { ContentNode, ProblemConfigFile } from 'hydrooj';
+import type { ContentNode, PluginContext, ProblemConfigFile } from 'hydrooj';
 import { FileTooLargeError, ValidationError } from 'hydrooj/src/error';
 import { buildContent } from 'hydrooj/src/lib/content';
-import { ProblemAdd } from 'hydrooj/src/lib/ui';
 import { PERM } from 'hydrooj/src/model/builtin';
 import problem from 'hydrooj/src/model/problem';
 import solution from 'hydrooj/src/model/solution';
@@ -130,7 +129,7 @@ class FpsProblemImportHandler extends Handler {
 }
 
 export const sideEffect = true;
-export async function apply(ctx) {
+export async function apply(ctx: PluginContext) {
     ctx.Route('problem_import_fps', '/problem/import/fps', FpsProblemImportHandler, PERM.PERM_CREATE_PROBLEM);
-    ProblemAdd('problem_import_fps', {}, 'copy', 'From FPS File');
+    ctx.inject('ProblemAdd', 'problem_import_fps', { icon: 'copy', text: 'From FPS File' });
 }

packages/hydrooj/src/handler/import.ts

@@ -5,7 +5,7 @@ import AdmZip from 'adm-zip';
 import fs from 'fs-extra';
 import yaml from 'js-yaml';
 import { ValidationError } from '../error';
-import { ProblemAdd } from '../lib/ui';
+import { PluginContext } from '../loader';
 import { PERM, PRIV } from '../model/builtin';
 import problem, { ProblemDoc } from '../model/problem';
 import { Handler } from '../service/server';
@@ -93,7 +93,7 @@ class ProblemImportHydroHandler extends Handler {
     }
 }
 
-export async function apply(ctx) {
+export async function apply(ctx: PluginContext) {
-    ProblemAdd('problem_import_hydro', {}, 'copy', 'Import From Hydro');
     ctx.Route('problem_import_hydro', '/problem/import/hydro', ProblemImportHydroHandler, PERM.PERM_CREATE_PROBLEM);
+    ctx.inject('ProblemAdd', 'problem_import_hydro', { icon: 'copy', text: 'Import From Hydro' });
 }

packages/hydrooj/src/handler/user.ts

@@ -6,7 +6,7 @@ import {
     SystemError, UserAlreadyExistError, UserFacingError,
     UserNotFoundError, ValidationError, VerifyPasswordError,
 } from '../error';
-import { OAuthUserResponse, Udoc, User } from '../interface';
+import { Udoc, User } from '../interface';
 import avatar from '../lib/avatar';
 import { sendMail } from '../lib/mail';
 import { isEmail, isPassword } from '../lib/validator';
@@ -359,7 +359,7 @@ class OauthHandler extends Handler {
 
     @param('type', Types.String)
     async get(domainId: string, type: string) {
-        await global.Hydro.lib[`oauth_${type}`]?.get?.call(this);
+        await global.Hydro.module.oauth[type]?.get.call(this);
     }
 }
 
@@ -367,8 +367,8 @@ class OauthCallbackHandler extends Handler {
     noCheckPermView = true;
 
     async get(args: any) {
-        if (!global.Hydro.lib[`oauth_${args.type}`]) throw new UserFacingError('Oauth type');
+        if (!global.Hydro.module.oauth[args.type]) throw new UserFacingError('Oauth type');
-        const r = await global.Hydro.lib[`oauth_${args.type}`].callback.call(this, args) as OAuthUserResponse;
+        const r = await global.Hydro.module.oauth[args.type].callback.call(this, args);
         const uid = await oauth.get(r._id);
         if (uid) {
             await user.setById(uid, { loginat: new Date(), loginip: this.request.ip });

packages/hydrooj/src/init.ts

@@ -19,6 +19,12 @@ if (!global.Hydro) {
         script: {},
         // @ts-ignore
         lib: {},
+        module: new Proxy({} as any, {
+            get(self, key) {
+                if (!self[key]) self[key] = {};
+                return self[key];
+            },
+        }),
         // @ts-ignore
         ui: {
             template: {},

packages/hydrooj/src/interface.ts

@@ -3,6 +3,7 @@ import type { Dictionary, NumericDictionary } from 'lodash';
 import type { ItemBucketMetadata } from 'minio';
 import type { Cursor, ObjectID } from 'mongodb';
 import type { ProblemDoc } from './model/problem';
+import type { Handler } from './service/server';
 
 type document = typeof import('./model/document');
 
@@ -686,10 +687,11 @@ export interface ProblemSearchOptions {
 
 export type ProblemSearch = (domainId: string, q: string, options?: ProblemSearchOptions) => Promise<ProblemSearchResponse>;
 
+export { Context as PluginContext } from './service/module';
+
 export interface Lib extends Record<string, any> {
     difficulty: typeof import('./lib/difficulty'),
     buildContent: typeof import('./lib/content').buildContent,
-    'hash.hydro': typeof import('./lib/hash.hydro'),
     i18n: typeof import('./lib/i18n'),
     mail: typeof import('./lib/mail'),
     paginate: typeof import('./lib/paginate'),
@@ -705,27 +707,36 @@ export interface Lib extends Record<string, any> {
 
 export type UIInjectableFields = 'ProblemAdd' | 'Nav' | 'UserDropdown';
 export interface UI {
-    template: Dictionary<string>,
+    template: Record<string, string>,
     nodes: Record<UIInjectableFields, any[]>,
     getNodes: typeof import('./lib/ui').getNodes,
-    Nav: typeof import('./lib/ui').Nav,
-    ProblemAdd: typeof import('./lib/ui').ProblemAdd,
     inject: typeof import('./lib/ui').inject,
 }
 
+export interface ModuleInterfaces {
+    oauth: {
+        text: string;
+        icon?: string;
+        get: (this: Handler) => Promise<void>;
+        callback: (this: Handler, args: Record<string, any>) => Promise<OAuthUserResponse>;
+    };
+    hash: (password: string, salt: string, user: User) => boolean | string;
+}
+
 export interface HydroGlobal {
-    version: Record<string, string>,
+    version: Record<string, string>;
-    model: Model,
+    model: Model;
     /** @deprecated */
-    handler: Record<string, Function>,
+    handler: Record<string, Function>;
-    script: Record<string, Script>,
+    script: Record<string, Script>;
-    service: Service,
+    service: Service;
-    lib: Lib,
+    lib: Lib;
-    ui: UI,
+    module: { [K in keyof ModuleInterfaces]: Record<string, ModuleInterfaces[K]> };
-    error: typeof import('./error'),
+    ui: UI;
-    Logger: typeof import('./logger').Logger,
+    error: typeof import('./error');
-    logger: typeof import('./logger').logger,
+    Logger: typeof import('./logger').Logger;
-    locales: Record<string, Record<string, string>>,
+    logger: typeof import('./logger').logger;
+    locales: Record<string, Record<string, string>>;
 }
 
 declare global {

packages/hydrooj/src/lib/hash.hydro.ts

@@ -1,9 +1,8 @@
 import crypto from 'crypto';
 
 function hash(password: string, salt: string) {
-    return crypto.pbkdf2Sync(password, salt, 100000, 64, 'sha256').toString('hex').substr(0, 64);
+    return crypto.pbkdf2Sync(password, salt, 100000, 64, 'sha256').toString('hex').substring(0, 64);
 }
 
 export = hash;
-
+global.Hydro.module.hash.hydro = hash;
-global.Hydro.lib['hash.hydro'] = hash;

packages/hydrooj/src/lib/ui.ts

@@ -31,7 +31,9 @@ export const nodes = new Proxy({}, {
     },
 });
 export function inject(node: UIInjectableFields, name: string, args: Record<string, any> = {}, ...permPrivChecker: PermPrivChecker) {
-    nodes[node].push({ name, args: args || {}, checker: buildChecker(...permPrivChecker) });
+    const obj = { name, args: args || {}, checker: buildChecker(...permPrivChecker) };
+    nodes[node].push(obj);
+    return () => { nodes[node] = nodes[node].filter((i) => i !== obj); };
 }
 export function getNodes(name: UIInjectableFields) {
     return nodes[name];
@@ -65,5 +67,4 @@ inject('ProblemAdd', 'problem_create', { icon: 'add', text: 'Create Problem' });
 global.Hydro.ui.inject = inject;
 global.Hydro.ui.nodes = nodes as any;
 global.Hydro.ui.getNodes = getNodes;
-global.Hydro.ui.Nav = Nav;
+Object.assign(global.Hydro.ui, { ProblemAdd, Nav });
-global.Hydro.ui.ProblemAdd = ProblemAdd;

packages/hydrooj/src/model/user.ts

@@ -132,7 +132,7 @@ export class User {
     }
 
     checkPassword(password: string) {
-        const h = global.Hydro.lib[`hash.${this.hashType}`];
+        const h = global.Hydro.module.hash[this.hashType];
         if (!h) throw new Error('Unknown hash method');
         const result = h(password, this._salt, this);
         if (result !== true && result !== this._hash) {

packages/hydrooj/src/service/module.ts

@@ -1,4 +1,6 @@
 import Schema from 'schemastery';
+import { inject } from '../lib/ui';
+import { ModuleInterfaces } from '../loader';
 import * as bus from './bus';
 
 export const runtimes: Record<string, Runtime> = Object.create(null);
@@ -9,6 +11,12 @@ function addScript<K>(name: string, description: string, validate: Schema<K>, ru
     return () => delete global.Hydro.script[name];
 }
 
+function provideModule<T extends keyof ModuleInterfaces>(type: T, id: string, module: ModuleInterfaces[T]) {
+    if (global.Hydro.module[type][id]) throw new Error(`duplicate script ${type}/${id} registered.`);
+    global.Hydro.module[type as any][id] = module;
+    return () => delete global.Hydro.module[type][id];
+}
+
 export interface Context {
     Route: typeof import('./server').Route;
     Connection: typeof import('./server').Connection;
@@ -16,6 +24,8 @@ export interface Context {
     once: typeof bus['once'];
     off: typeof bus['off'];
     addScript: typeof addScript;
+    provideModule: typeof provideModule;
+    inject: typeof inject;
 }
 
 export class Runtime {
@@ -46,6 +56,8 @@ export class Runtime {
             setInterval: T(setInterval, clearInterval),
             setTimeout: T(setTimeout, clearTimeout),
             setImmediate: T(setImmediate, clearImmediate),
+            provideModule: T(provideModule),
+            inject: T(inject),
         } as Context);
         if (module.sideEffect) this.sideEffect = true;
         this.loaded = true;

packages/hydrooj/src/service/server.ts

@@ -7,7 +7,6 @@ import Body from 'koa-body';
 import Compress from 'koa-compress';
 import proxy from 'koa-proxies';
 import cache from 'koa-static-cache';
-import { filter } from 'lodash';
 import WebSocket from 'ws';
 import { parseMemoryMB } from '@hydrooj/utils/lib/utils';
 import {
@@ -176,11 +175,11 @@ export class Handler extends HandlerCommon {
     async init() {
         if (!argv.options.benchmark) await this.limitRate('global', 5, 88);
         if (!this.noCheckPermView && !this.user.hasPriv(PRIV.PRIV_VIEW_ALL_DOMAIN)) this.checkPerm(PERM.PERM_VIEW);
-        this.loginMethods = filter(Object.keys(global.Hydro.lib), (str) => str.startsWith('oauth_'))
+        this.loginMethods = Object.keys(global.Hydro.module.oauth)
             .map((key) => ({
-                id: key.split('_')[1],
+                id: key,
-                icon: global.Hydro.lib[key].icon,
+                icon: global.Hydro.module.oauth[key].icon,
-                text: global.Hydro.lib[key].text,
+                text: global.Hydro.module.oauth[key].text,
             }));
     }
 

packages/hydrooj/src/service/watcher.ts

@@ -52,7 +52,8 @@ class Watcher {
         if (process.env.WATCH_ROOT) roots.push(process.env.WATCH_ROOT);
         this.watcher = watch(roots, {
             ...this.config,
-            ignored: ['**/node_modules/**', '**/.git/**', '**/logs/**', '**/.cache/**'],
+            ignored: ['node_modules', '.git', 'logs', '.cache']
+                .map((i) => `**/${i}/**`).concat('**/tsconfig.tsbuildinfo'),
         });
         logger.info(`Start watching changes in ${this.root}`);
 

packages/import-qduoj/handler.ts

@@ -4,10 +4,9 @@ import path from 'path';
 import AdmZip from 'adm-zip';
 import fs from 'fs-extra';
 import yaml from 'js-yaml';
-import type { ContentNode, ProblemConfigFile } from 'hydrooj';
+import type { ContentNode, PluginContext, ProblemConfigFile } from 'hydrooj';
 import { ValidationError } from 'hydrooj/src/error';
 import { buildContent } from 'hydrooj/src/lib/content';
-import { ProblemAdd } from 'hydrooj/src/lib/ui';
 import { PERM } from 'hydrooj/src/model/builtin';
 import problem from 'hydrooj/src/model/problem';
 import { Handler } from 'hydrooj/src/service/server';
@@ -126,7 +125,7 @@ class ImportQduojHandler extends Handler {
     }
 }
 
-export async function apply(ctx) {
+export async function apply(ctx: PluginContext) {
     ctx.Route('problem_import_qduoj', '/problem/import/qduoj', ImportQduojHandler, PERM.PERM_CREATE_PROBLEM);
-    ProblemAdd('problem_import_qduoj', {}, 'copy', 'From QDUOJ Export');
+    ctx.inject('ProblemAdd', 'problem_import_qduoj', { icon: 'copy', text: 'From QDUOJ Export' });
 }

packages/login-with-github/lib.ts

@@ -1,6 +1,5 @@
-import 'hydrooj';
-
 import superagent from 'superagent';
+import { PluginContext } from 'hydrooj';
 import { ForbiddenError } from 'hydrooj/src/error';
 
 declare module 'hydrooj' {
@@ -9,9 +8,6 @@ declare module 'hydrooj' {
         'login-with-github.secret': string;
         'login-with-github.endpoint': string;
     }
-    interface Lib {
-        oauth_github: typeof import('./lib');
-    }
 }
 
 async function get() {
@@ -75,8 +71,10 @@ async function callback({ state, code }) {
     return ret;
 }
 
-global.Hydro.lib.oauth_github = {
+export function apply(ctx:PluginContext) {
-    text: 'Login with Github',
+    ctx.provideModule('oauth', 'github', {
-    callback,
+        text: 'Login with Github',
-    get,
+        callback,
-};
+        get,
+    });
+}

packages/login-with-google/lib.ts

@@ -1,15 +1,11 @@
-import 'hydrooj';
-
 import * as superagent from 'superagent';
+import { PluginContext } from 'hydrooj';
 
 declare module 'hydrooj' {
     interface SystemKeys {
         'login-with-google.id': string,
         'login-with-google.secret': string,
     }
-    interface Lib {
-        oauth_google: typeof import('./lib'),
-    }
 }
 
 async function get() {
@@ -81,8 +77,10 @@ async function callback({
     };
 }
 
-global.Hydro.lib.oauth_google = {
+export function apply(ctx: PluginContext) {
-    text: 'Login with Google',
+    ctx.provideModule('oauth', 'google', {
-    callback,
+        text: 'Login with Google',
-    get,
+        callback,
-};
+        get,
+    });
+}

packages/migrate/index.ts

@@ -0,0 +1,31 @@
+import { md5, sha1 } from '@hydrooj/utils/lib/utils';
+import { PluginContext } from 'hydrooj';
+import { apply as hustoj } from './scripts/hustoj';
+import { apply as syzoj } from './scripts/syzoj';
+import { apply as vijos } from './scripts/vijos';
+
+const RE_MD5 = /^[\da-f]{32}$/;
+
+export function apply(ctx: PluginContext) {
+    hustoj(ctx);
+    vijos(ctx);
+    syzoj(ctx);
+    ctx.provideModule('hash', 'hust', ($password, $saved) => {
+        $password = md5($password);
+        if (RE_MD5.test($saved)) return $password === $saved;
+        const $svd = Buffer.from($saved, 'base64').toString('hex');
+        const $salt = Buffer.from($svd.substr(40), 'hex').toString();
+        const $hash = Buffer.concat([
+            Buffer.from(sha1($password + $salt), 'hex'),
+            Buffer.from($salt),
+        ]).toString('base64');
+        if ($hash.trim() === $saved.trim()) return true;
+        return false;
+    });
+    ctx.provideModule('hash', 'vj2', (password, salt, { uname }) => {
+        const pmd5 = md5(password);
+        const mixedSha1 = sha1(md5(uname.toLowerCase() + pmd5) + salt + sha1(pmd5 + salt));
+        return `${Buffer.from(uname).toString('base64')}|${mixedSha1}`;
+    });
+    ctx.provideModule('hash', 'syzoj', (password: string) => md5(`${password}syzoj2_xxx`));
+}

packages/migrate/lib.ts

@@ -1,25 +0,0 @@
-import { md5, sha1 } from '@hydrooj/utils/lib/utils';
-
-const RE_MD5 = /^[\da-f]{32}$/;
-
-global.Hydro.lib['hash.hust'] = ($password: string, $saved: string) => {
-    $password = md5($password);
-    if (RE_MD5.test($saved)) return $password === $saved;
-    const $svd = Buffer.from($saved, 'base64').toString('hex');
-    const $salt = Buffer.from($svd.substr(40), 'hex').toString();
-    const $hash = Buffer.concat([
-        Buffer.from(sha1($password + $salt), 'hex'),
-        Buffer.from($salt),
-    ]).toString('base64');
-    if ($hash.trim() === $saved.trim()) return true;
-    return false;
-};
-
-global.Hydro.lib['hash.vj2'] = (password: string, salt: string, udoc: { uname: string; unameLower: string }) => {
-    const { uname, unameLower } = udoc;
-    const pmd5 = md5(password);
-    const mixedSha1 = sha1(md5(unameLower + pmd5) + salt + sha1(pmd5 + salt));
-    return `${Buffer.from(uname).toString('base64')}|${mixedSha1}`;
-};
-
-global.Hydro.lib['hash.syzoj'] = (password: string) => md5(`${password}syzoj2_xxx`);

packages/migrate/script.ts

@@ -1,9 +0,0 @@
-import { apply as hustoj } from './scripts/hustoj';
-import { apply as syzoj } from './scripts/syzoj';
-import { apply as vijos } from './scripts/vijos';
-
-export function apply(ctx) {
-    hustoj(ctx);
-    vijos(ctx);
-    syzoj(ctx);
-}

packages/sonic/index.ts

@@ -0,0 +1,94 @@
+import { Schema } from 'hydrooj';
+import { Logger } from 'hydrooj/src/logger';
+import DomainModel from 'hydrooj/src/model/domain';
+import * as system from 'hydrooj/src/model/system';
+import { iterateAllProblem, iterateAllProblemInDomain } from 'hydrooj/src/pipelineUtils';
+import sonic from './service';
+
+const logger = new Logger('sonic');
+
+global.Hydro.lib.problemSearch = async (domainId, query, opts) => {
+    const limit = opts?.limit || system.get('pagination.problem');
+    const ids = await sonic.query('problem', `${domainId}@title`, query, { limit });
+    if (limit - ids.length > 0) ids.push(...await sonic.query('problem', `${domainId}@content`, query, { limit: limit - ids.length }));
+    return {
+        countRelation: ids.length >= limit ? 'gte' : 'eq',
+        total: ids.length,
+        hits: ids,
+    };
+};
+
+async function run({ domainId }, report) {
+    if (domainId) await sonic.flushb('problem', domainId);
+    else await sonic.flushc('problem');
+    let i = 0;
+    const cb = async (pdoc) => {
+        i++;
+        if (!(i % 1000)) report({ message: `${i} problems indexed` });
+        const union = await DomainModel.searchUnion({ union: pdoc.domainId, problem: true });
+        const tasks = [];
+        for (const did of [pdoc.domainId, ...union.map((j) => j._id)]) {
+            tasks.push(
+                pdoc.title && sonic.push(
+                    'problem', `${did}@title`, `${pdoc.domainId}/${pdoc.docId}`,
+                    `${pdoc.pid || ''} ${pdoc.title} ${pdoc.tag.join(' ')}`,
+                ),
+                pdoc.content.toString() && sonic.push('problem', `${did}@content`, `${pdoc.domainId}/${pdoc.docId}`, pdoc.content.toString()),
+            );
+        }
+        await Promise.all(tasks).catch((e) => console.log(`${pdoc.domainId}/${pdoc.docId}`, e));
+    };
+    if (domainId) await iterateAllProblemInDomain(domainId, ['title', 'content'], cb);
+    else await iterateAllProblem(['title', 'content', 'tag'], cb);
+    return true;
+}
+
+export function apply(ctx) {
+    ctx.on('problem/add', async (doc, docId) => {
+        const union = await DomainModel.searchUnion({ union: doc.domainId, problem: true });
+        const tasks = [];
+        for (const domainId of [doc.domainId, ...union.map((i) => i._id)]) {
+            tasks.push(
+                sonic.push('problem', `${domainId}@title`, `${doc.domainId}/${docId}`, `${doc.pid || ''} ${doc.title} ${doc.tag?.join(' ')}`),
+                sonic.push('problem', `${domainId}@content`, `${doc.domainId}/${docId}`, doc.content.toString()),
+            );
+        }
+        Promise.all(tasks).catch((e) => logger.error(e));
+    });
+
+    ctx.on('problem/edit', async (pdoc) => {
+        const union = await DomainModel.searchUnion({ union: pdoc.domainId, problem: true });
+        const tasks = [];
+        const id = `${pdoc.domainId}/${pdoc.docId}`;
+        for (const domainId of [pdoc.domainId, ...union.map((i) => i._id)]) {
+            tasks.push(
+                sonic.flusho('problem', `${domainId}@title`, id)
+                    .then(() => sonic.push('problem', `${domainId}@title`, id, `${pdoc.pid || ''} ${pdoc.title} ${pdoc.tag?.join(' ')}`)),
+                sonic.flusho('problem', `${domainId}@content`, id)
+                    .then(() => sonic.push('problem', `${domainId}@content`, id, pdoc.content.toString())),
+            );
+        }
+        Promise.all(tasks).catch((e) => logger.error(e));
+    });
+
+    ctx.on('problem/del', async (domainId, docId) => {
+        const union = await DomainModel.searchUnion({ union: domainId, problem: true });
+        const tasks = [];
+        const id = `${domainId}/${docId}`;
+        for (const domain of [domainId, ...union.map((i) => i._id)]) {
+            tasks.push(
+                sonic.flusho('problem', `${domain}@title`, id),
+                sonic.flusho('problem', `${domain}@content`, id),
+            );
+        }
+        await Promise.all(tasks);
+    });
+
+    ctx.addScript(
+        'ensureSonicSearch', 'Sonic problem search re-index',
+        Schema.object({
+            domainId: Schema.string(),
+        }),
+        run,
+    );
+}