core: config: refactor (allow schemastery)

README.md

@@ -132,6 +132,10 @@ Hydro 用户群：1085853538
 - core: 移除旧版本 bus 调用
 - migrate: 修复 hustoj 自动运行
 - core: 强制要求使用 inject 声明插件依赖
+- core: 移除 global.Hydro.service.db
+- core: 移除 global.Hydro.service.service
+- core: 移除 global.Hydro.service.storage
+- core: Settings: 支持使用 Schemastery
 
 ## Hydro 4.19.0 / UI 4.57.0
 

packages/hydrooj/src/entry/cli.ts

@@ -6,7 +6,9 @@ import cac from 'cac';
 import fs from 'fs-extra';
 import { ObjectId } from 'mongodb';
 import { Context } from '../context';
-import db from '../service/db';
+import { load as loadOptions } from '../options';
+import { MongoService } from '../service/db';
+import { ConfigService } from '../settings';
 import {
     addon, builtinModel, lib, model, script, service, setting,
 } from './common';
@@ -109,10 +111,15 @@ export async function load(ctx: Context) {
     require('../service/bus').apply(ctx);
     const pending = global.addons;
     const fail = [];
-    await db.start();
-    await require('../settings').loadConfig();
+    await ctx.plugin(MongoService, loadOptions());
+    await ctx.plugin(ConfigService);
     await require('../model/system').runConfig();
-    await require('../service/storage').loadStorageService();
+    ctx = await new Promise((resolve) => {
+        ctx.inject(['loader'], (c) => {
+            resolve(c);
+        });
+    });
+    await ctx.loader.reloadPlugin(require.resolve('../service/storage'), 'file', 'hydrooj/service/storage');
     require('../lib/index');
     await Promise.all([
         lib(pending, fail, ctx),
@@ -131,7 +138,7 @@ export async function load(ctx: Context) {
     await addon(pending, fail, ctx);
     const scriptDir = path.resolve(__dirname, '..', 'script');
     for (const h of await fs.readdir(scriptDir)) {
-        ctx.loader.reloadPlugin(path.resolve(scriptDir, h), {}, `hydrooj/script/${h.split('.')[0]}`);
+        ctx.loader.reloadPlugin(path.resolve(scriptDir, h), '', `hydrooj/script/${h.split('.')[0]}`);
     }
     await script(pending, fail, ctx);
     await cli();

packages/hydrooj/src/entry/common.ts

@@ -42,7 +42,7 @@ const getLoader = (type: LoadTask, filename: string, dontLoad = false) => async
             try {
                 if (dontLoad) throw new Error(`deprecated ${name} in %s will not be load: ${i}`);
                 const m = unwrapExports(require(p));
-                if (m.apply) ctx.loader.reloadPlugin(p, {});
+                if (m.apply) ctx.loader.reloadPlugin(p, '');
                 else logger.info(`${name} init: %s`, i);
             } catch (e) {
                 fail.push(i);
@@ -69,7 +69,7 @@ export async function builtinModel(ctx: Context) {
     const models = await fs.readdir(modelDir);
     for (const t of models.filter((i) => i.endsWith('.ts'))) {
         const q = path.resolve(modelDir, t);
-        if ('apply' in require(q)) ctx.loader.reloadPlugin(q, {}, `hydrooj/model/${t.split('.')[0]}`);
+        if ('apply' in require(q)) ctx.loader.reloadPlugin(q, '', `hydrooj/model/${t.split('.')[0]}`);
     }
 }
 

packages/hydrooj/src/entry/worker.ts

@@ -7,7 +7,8 @@ import fs from 'fs-extra';
 import { Context } from '../context';
 import { Logger } from '../logger';
 import { load } from '../options';
-import db from '../service/db';
+import { MongoService } from '../service/db';
+import { ConfigService } from '../settings';
 import {
     addon, builtinModel, handler, lib, locale, model,
     script, service, setting, template,
@@ -22,8 +23,8 @@ export async function apply(ctx: Context) {
     require('../utils');
     require('../error');
     require('../service/bus').apply(ctx);
-    const config = load();
-    if (!process.env.CI && !config) {
+    const url = await MongoService.getUrl();
+    if (!url) {
         logger.info('Starting setup');
         await require('./setup').load(ctx);
     }
@@ -33,12 +34,16 @@ export async function apply(ctx: Context) {
         locale(pending, fail),
         template(pending, fail),
     ]);
-    await db.start();
-    await require('../settings').loadConfig();
+    await ctx.plugin(MongoService, load());
+    await ctx.plugin(ConfigService);
     const modelSystem = require('../model/system');
     await modelSystem.runConfig();
-    const storage = require('../service/storage');
-    await storage.loadStorageService();
+    ctx = await new Promise((resolve) => {
+        ctx.inject(['loader'], (c) => {
+            resolve(c);
+        });
+    });
+    await ctx.loader.reloadPlugin(require.resolve('../service/storage'), 'file', 'hydrooj/service/storage');
     await ctx.plugin(require('../service/hmr').default, { watch: argv.options.watch });
     await require('../service/worker').apply(ctx);
     await require('../service/server').apply(ctx);
@@ -61,14 +66,14 @@ export async function apply(ctx: Context) {
     const handlerDir = path.resolve(__dirname, '..', 'handler');
     const handlers = await fs.readdir(handlerDir);
     for (const h of handlers.filter((i) => i.endsWith('.ts'))) {
-        ctx.loader.reloadPlugin(path.resolve(handlerDir, h), {}, `hydrooj/handler/${h.split('.')[0]}`);
+        ctx.loader.reloadPlugin(path.resolve(handlerDir, h), '', `hydrooj/handler/${h.split('.')[0]}`);
     }
     ctx.plugin(require('../service/migration').default);
     await handler(pending, fail, ctx);
     await addon(pending, fail, ctx);
     const scriptDir = path.resolve(__dirname, '..', 'script');
     for (const h of await fs.readdir(scriptDir)) {
-        ctx.loader.reloadPlugin(path.resolve(scriptDir, h), {}, `hydrooj/script/${h.split('.')[0]}`);
+        ctx.loader.reloadPlugin(path.resolve(scriptDir, h), '', `hydrooj/script/${h.split('.')[0]}`);
     }
     await script(pending, fail, ctx);
     await ctx.parallel('app/started');

packages/hydrooj/src/handler/manage.ts

@@ -17,7 +17,6 @@ import { check } from '../service/check';
 import {
     ConnectionHandler, Handler, param, requireSudo, Types,
 } from '../service/server';
-import { configSource, saveConfig, SystemSettings } from '../settings';
 import * as judge from './judge';
 
 const logger = new Logger('manage');
@@ -184,27 +183,26 @@ class SystemConfigHandler extends SystemHandler {
     @requireSudo
     async get() {
         this.response.template = 'manage_config.html';
-        let value = configSource;
+        let value = this.ctx.config.configSource;
         try {
-            value = yaml.dump(Schema.intersect(SystemSettings)(yaml.load(configSource)));
+            value = yaml.dump(Schema.intersect(this.ctx.config.settings)(yaml.load(value)));
         } catch (e) { }
         this.response.body = {
-            schema: Schema.intersect(SystemSettings).toJSON(),
+            schema: Schema.intersect(this.ctx.config.settings).toJSON(),
             value,
         };
     }
 
     @requireSudo
     @param('value', Types.String)
-    async post(domainId: string, value: string) {
+    async post({ }, value: string) {
         let config;
         try {
             config = yaml.load(value);
-            Schema.intersect(SystemSettings)(config);
         } catch (e) {
             throw new ValidationError('value');
         }
-        await saveConfig(config);
+        await this.ctx.config.saveConfig(config);
     }
 }
 

packages/hydrooj/src/handler/misc.ts

@@ -15,7 +15,6 @@ import {
     Handler, param, post, requireSudo, Types,
 } from '../service/server';
 import { encodeRFC5987ValueChars } from '../service/storage';
-import { builtinConfig } from '../settings';
 import { md5, sortFiles } from '../utils';
 
 class SwitchLanguageHandler extends Handler {
@@ -110,7 +109,7 @@ export class StorageHandler extends Handler {
     @param('expire', Types.UnsignedInt)
     @param('secret', Types.String)
     async get(domainId: string, target: string, filename = '', expire: number, secret: string) {
-        const expected = md5(`${target}/${expire}/${builtinConfig.file.secret}`);
+        const expected = md5(`${target}/${expire}/${this.ctx.config.get('file.secret')}`);
         if (expire < Date.now()) throw new AccessDeniedError();
         if (secret !== expected) throw new AccessDeniedError();
         this.response.body = await storage.get(target);

packages/hydrooj/src/interface.ts

@@ -599,9 +599,6 @@ export interface Model {
 export interface HydroService {
     /** @deprecated */
     bus: Context,
-    db: typeof import('./service/db').default,
-    server: typeof import('./service/server'),
-    storage: typeof import('./service/storage').default,
 }
 
 export interface GeoIP {

packages/hydrooj/src/loader.ts

@@ -14,11 +14,14 @@ import './ui';
 import * as I18n from './lib/i18n';
 
 import { Logger } from './logger';
-import { Context, Service, ScopeStatus } from './context';
+import {
+    Context, Service, ScopeStatus, EffectScope,
+} from './context';
 // eslint-disable-next-line import/no-duplicates
 import { sleep, unwrapExports } from './utils';
 import { PRIV } from './model/builtin';
 import { getAddons } from './options';
+import Schema from 'schemastery';
 
 const argv = cac().parse();
 const logger = new Logger('loader');
@@ -43,29 +46,20 @@ if (process.env.NIX_PROFILES) {
     }
 }
 
-export function resolveConfig(plugin: any, config: any) {
-    if (config === false) return;
-    if (config === true) config = undefined;
-    config ??= {};
-    const schema = plugin['Config'] || plugin['schema'];
-    if (schema && plugin['schema'] !== false) config = schema(config);
-    return config;
-}
-
 export class Loader extends Service {
-    public state: Record<string, any> = Object.create(null);
+    public state: Record<string, EffectScope> = Object.create(null);
     public config: {};
     public suspend = false;
     public cache: Record<string, string> = Object.create(null);
     // public warnings: Record<string, string> = Object.create(null);
 
+    static inject = ['config', 'timer', 'i18n', 'logger'];
+
     constructor(ctx: Context) {
         super(ctx, 'loader');
-    }
 
-    [Service.setup]() {
-        this.ctx.on('app/started', () => {
-            this.ctx.setInterval(async () => {
+        ctx.on('app/started', () => {
+            ctx.setInterval(async () => {
                 const pending = Object.entries(this.state).filter((v) => v[1].status === ScopeStatus.PENDING);
                 if (pending.length) {
                     logger.warn('Plugins are still pending: %s', pending.map((v) => v[0]).join(', '));
@@ -103,18 +97,27 @@ export class Loader extends Service {
         }
     }
 
-    async reloadPlugin(key: string, config: any, asName = '') {
+    resolveConfig(plugin: any, configScope: string) {
+        const schema = plugin['Config'] || plugin['schema'];
+        if (!schema) return {};
+        const schemaRequest = configScope ? Schema.object({
+            [configScope]: schema,
+        }) : schema;
+        const res = this.ctx.config.requestConfig(schemaRequest);
+        return configScope ? res[configScope] : res;
+    }
+
+    async reloadPlugin(key: string, configScope: string, asName = '') {
+        const plugin = await this.resolvePlugin(key);
+        if (!plugin) return;
+        const config = this.resolveConfig(plugin, configScope);
+        if (asName) plugin.name = asName;
         let fork = this.state[key];
         if (fork) {
             logger.info('reload plugin %c', key.split('node_modules').pop());
             fork.update(config);
         } else {
             logger.info('apply plugin %c', key.split('node_modules').pop());
-            const plugin = await this.resolvePlugin(key);
-            if (!plugin) return;
-            resolveConfig(plugin, config);
-            if (asName) plugin.name = asName;
-            // fork = parent.plugin(plugin, this.interpolate(config));
             fork = this.ctx.plugin(plugin, config);
             if (!fork) return;
             this.state[key] = fork;
@@ -129,7 +132,8 @@ export class Loader extends Service {
             try {
                 this.cache[name] ||= require.resolve(name, { paths: HYDROPATH });
             } catch (e) {
-                logger.error(err.message);
+                logger.error('Failed to resolve plugin %s', name);
+                logger.error(err);
                 return;
             }
         }
@@ -138,14 +142,12 @@ export class Loader extends Service {
 }
 
 app.plugin(I18n);
+app.plugin(Loader);
 
 async function preload() {
     global.app = await new Promise((resolve) => {
-        app.inject(['timer', 'i18n', 'logger'], (c) => {
-            c.plugin(Loader);
-            c.inject(['loader'], (ctx) => {
-                resolve(ctx);
-            });
+        app.inject(['timer', 'i18n', 'logger', '$api'], (c) => {
+            resolve(c);
         });
     });
     for (const a of [path.resolve(__dirname, '..'), ...getAddons()]) {

packages/hydrooj/src/model/setting.ts

@@ -4,6 +4,7 @@ import * as cordis from '@cordisjs/core';
 import yaml from 'js-yaml';
 import { Dictionary } from 'lodash';
 import moment from 'moment-timezone';
+import Schema from 'schemastery';
 import { LangConfig, parseLang } from '@hydrooj/common';
 import { retry } from '@hydrooj/utils';
 import { Context, Service } from '../context';
@@ -70,7 +71,55 @@ export const Setting = (
     };
 };
 
-export const PreferenceSetting = (...settings: _Setting[]) => {
+declare global {
+    namespace Schemastery {
+        interface Meta<T> { // eslint-disable-line @typescript-eslint/no-unused-vars
+            family: string;
+        }
+    }
+}
+
+function schemaToSettings(schema: Schema<any>) {
+    const result: _Setting[] = [];
+    const processNode = (key: string, s: Schema<number> | Schema<string> | Schema<boolean>, defaultFamily = 'setting_basic') => {
+        if (s.dict) throw new Error('Dict is not supported here');
+        let flag = (s.meta?.hidden ? FLAG_HIDDEN : 0)
+            | (s.meta?.disabled ? FLAG_DISABLED : 0);
+        const type = s.type === 'number' ? 'number'
+            : s.type === 'boolean' ? 'checkbox'
+                : s.meta?.role === 'textarea' ? 'textarea' : 'text';
+        if (s.meta?.role === 'password') flag |= FLAG_SECRET;
+        const options = {};
+        for (const item of s.list || []) {
+            if (item.type !== 'const') throw new Error('List item must be a constant');
+            options[item.value] = item.meta?.description || item.value;
+        }
+        return {
+            family: s.meta?.family || defaultFamily,
+            key,
+            value: s.meta?.default,
+            name: key,
+            desc: s.meta?.description,
+            flag,
+            subType: '',
+            type: s.list ? 'select' : type,
+            range: s.list ? options : null,
+        } as _Setting;
+    };
+    if (!schema.dict) return [];
+    for (const key in schema.dict) {
+        const value = schema.dict[key];
+        if (value.dict) {
+            for (const subkey in value.dict) {
+                result.push(processNode(`${key}.${subkey}`, value.dict[subkey], value.meta?.family));
+            }
+        } else result.push(processNode(key, value));
+    }
+    return result;
+}
+
+export const PreferenceSetting = (...settings: _Setting[] | Schema<any>[]) => {
+    settings = settings.flatMap((s) => (s instanceof Schema ? schemaToSettings(s) : s) as _Setting[]);
     for (const setting of settings) {
         if (PREFERENCE_SETTINGS.find((s) => s.key === setting.key)) logger.warn(`Duplicate setting key: ${setting.key}`);
         PREFERENCE_SETTINGS.push(setting);
@@ -89,7 +138,8 @@ export const PreferenceSetting = (...settings: _Setting[]) => {
         }
     };
 };
-export const AccountSetting = (...settings: _Setting[]) => {
+export const AccountSetting = (...settings: _Setting[] | Schema<any>[]) => {
+    settings = settings.flatMap((s) => (s instanceof Schema ? schemaToSettings(s) : s) as _Setting[]);
     for (const setting of settings) {
         if (ACCOUNT_SETTINGS.find((s) => s.key === setting.key)) logger.warn(`Duplicate setting key: ${setting.key}`);
         ACCOUNT_SETTINGS.push(setting);
@@ -108,7 +158,8 @@ export const AccountSetting = (...settings: _Setting[]) => {
         }
     };
 };
-export const DomainUserSetting = (...settings: _Setting[]) => {
+export const DomainUserSetting = (...settings: _Setting[] | Schema<any>[]) => {
+    settings = settings.flatMap((s) => (s instanceof Schema ? schemaToSettings(s) : s) as _Setting[]);
     for (const setting of settings) {
         if (DOMAIN_USER_SETTINGS.find((s) => s.key === setting.key)) logger.warn(`Duplicate setting key: ${setting.key}`);
         DOMAIN_USER_SETTINGS.push(setting);
@@ -123,7 +174,8 @@ export const DomainUserSetting = (...settings: _Setting[]) => {
         }
     };
 };
-export const DomainSetting = (...settings: _Setting[]) => {
+export const DomainSetting = (...settings: _Setting[] | Schema<any>[]) => {
+    settings = settings.flatMap((s) => (s instanceof Schema ? schemaToSettings(s) : s) as _Setting[]);
     for (const setting of settings) {
         if (DOMAIN_SETTINGS.find((s) => s.key === setting.key)) logger.warn(`Duplicate setting key: ${setting.key}`);
         DOMAIN_SETTINGS.push(setting);
@@ -138,7 +190,8 @@ export const DomainSetting = (...settings: _Setting[]) => {
         }
     };
 };
-export const SystemSetting = (...settings: _Setting[]) => {
+export const SystemSetting = (...settings: _Setting[] | Schema<any>[]) => {
+    settings = settings.flatMap((s) => (s instanceof Schema ? schemaToSettings(s) : s) as _Setting[]);
     for (const setting of settings) {
         if (SYSTEM_SETTINGS.find((s) => s.key === setting.key)) logger.warn(`Duplicate setting key: ${setting.key}`);
         SYSTEM_SETTINGS.push(setting);
@@ -324,6 +377,7 @@ export class SettingService extends Service {
     }
 }
 
+export const inject = ['db'];
 export async function apply(ctx: Context) {
     ctx.provide('setting', undefined, true);
     ctx.plugin(SettingService);
@@ -331,7 +385,7 @@ export async function apply(ctx: Context) {
     const system = global.Hydro.model.system;
     for (const setting of SYSTEM_SETTINGS) {
         if (!setting.value) continue;
-        const current = await global.Hydro.service.db.collection('system').findOne({ _id: setting.key });
+        const current = await ctx.db.collection('system').findOne({ _id: setting.key });
         if (!current || current.value == null || current.value === '') {
             await retry(system.set, setting.key, setting.value);
         }
@@ -355,6 +409,8 @@ export async function apply(ctx: Context) {
 
 global.Hydro.model.setting = {
     apply,
+    inject,
+
     SettingService,
     Setting,
     PreferenceSetting,

packages/hydrooj/src/service/db.ts

@@ -4,6 +4,7 @@ import {
 } from 'mongodb';
 import mongoUri from 'mongodb-uri';
 import { Time } from '@hydrooj/utils';
+import { Context, Service } from '../context';
 import { ValidationError } from '../error';
 import { Logger } from '../logger';
 import { load } from '../options';
@@ -25,12 +26,28 @@ interface MongoConfig {
     collectionMap?: Record<string, string>,
 }
 
-class MongoService {
+declare module '../context' {
+    interface Context {
+        db: MongoService;
+    }
+}
+
+export class MongoService extends Service {
     public client: MongoClient;
     public db: Db;
-    private opts: MongoConfig;
 
-    static buildUrl(opts: MongoConfig) {
+    constructor(ctx: Context, private config: MongoConfig = {}) {
+        super(ctx, 'db');
+    }
+
+    static async getUrl() {
+        if (process.env.CI) {
+            const { MongoMemoryServer } = require('mongodb-memory-server');
+            const mongod = await MongoMemoryServer.create();
+            return mongod.getUri();
+        }
+        const opts = load();
+        if (!opts) return null;
         let mongourl = `${opts.protocol || 'mongodb'}://`;
         if (opts.username) mongourl += `${opts.username}:${encodeURIComponent(opts.password)}@`;
         mongourl += `${opts.host}:${opts.port}/${opts.name}`;
@@ -38,25 +55,18 @@ class MongoService {
         return mongourl;
     }
 
-    async start() {
-        const opts = load() || {};
-        let mongourl = MongoService.buildUrl(opts);
+    async [Service.setup]() {
+        const mongourl = await MongoService.getUrl();
         const url = mongoUri.parse(mongourl);
-        if (process.env.CI) {
-            const { MongoMemoryServer } = require('mongodb-memory-server');
-            const mongod = await MongoMemoryServer.create();
-            mongourl = mongod.getUri();
-        }
-        this.opts = opts;
         this.client = await MongoClient.connect(mongourl);
         this.db = this.client.db(url.database || 'hydro');
         await bus.parallel('database/connect', this.db);
-        setInterval(() => this.fixExpireAfter(), Time.hour);
+        this.ctx.setInterval(() => this.fixExpireAfter(), Time.hour);
     }
 
     public collection<K extends keyof Collections>(c: K) {
-        let coll = this.opts.prefix ? `${this.opts.prefix}.${c}` : c;
-        if (this.opts.collectionMap?.[coll]) coll = this.opts.collectionMap[coll];
+        let coll = this.config.prefix ? `${this.config.prefix}.${c}` : c;
+        if (this.config.collectionMap?.[coll]) coll = this.config.collectionMap[coll];
         return this.db.collection<Collections[K]>(coll);
     }
 
@@ -173,7 +183,11 @@ class MongoService {
     }
 }
 
-const service = new MongoService();
-global.Hydro.service.db = service;
-export default service;
-export const collection = service.collection.bind(service);
+/** @deprecated use ctx.db instead */
+const db = new Proxy({} as MongoService, {
+    get(target, prop) {
+        return app.get('db')?.[prop];
+    },
+});
+
+export default db;

packages/hydrooj/src/service/server.ts

@@ -1,7 +1,6 @@
 import { resolve } from 'path';
 import cac from 'cac';
 import fs from 'fs-extra';
-import proxy from 'koa-proxies';
 import cache from 'koa-static-cache';
 import { type FindCursor, ObjectId } from 'mongodb';
 import {
@@ -18,7 +17,6 @@ import { PERM, PRIV } from '../model/builtin';
 import * as opcount from '../model/opcount';
 import * as OplogModel from '../model/oplog';
 import * as system from '../model/system';
-import { builtinConfig } from '../settings';
 import db from './db';
 import baseLayer from './layers/base';
 import domainLayer from './layers/domain';
@@ -93,26 +91,6 @@ export async function apply(ctx: Context) {
     });
     if (process.env.HYDRO_CLI) return;
     ctx.inject(['server'], ({ server, on }) => {
-        let endpoint = builtinConfig.file.endPoint;
-        if (builtinConfig.file.type === 's3' && !builtinConfig.file.pathStyle) {
-            try {
-                const parsed = new URL(builtinConfig.file.endPoint);
-                parsed.hostname = `${builtinConfig.file.bucket}.${parsed.hostname}`;
-                endpoint = parsed.toString();
-            } catch (e) {
-                logger.warn('Failed to parse file endpoint');
-            }
-        }
-        const proxyMiddleware = proxy('/fs', {
-            target: endpoint,
-            changeOrigin: true,
-            rewrite: (p) => p.replace('/fs', ''),
-        });
-        server.addCaptureRoute('/fs/', async (c, next) => {
-            if (c.request.search.toLowerCase().includes('x-amz-credential')) return await proxyMiddleware(c, next);
-            c.request.path = c.path = c.path.split('/fs')[1];
-            return await next();
-        });
         server.addHandlerLayer('init', async (c, next) => {
             const init = Date.now();
             try {

packages/hydrooj/src/service/storage.ts

@@ -12,9 +12,12 @@ import {
     copyFile, createReadStream, createWriteStream, ensureDir,
     existsSync, remove, stat, writeFile,
 } from 'fs-extra';
+import proxy from 'koa-proxies';
 import { lookup } from 'mime-types';
+import { nanoid } from 'nanoid';
+import Schema from 'schemastery';
+import { Context } from '../context';
 import { Logger } from '../logger';
-import { builtinConfig } from '../settings';
 import { MaybeArray } from '../typeutils';
 import { md5, streamToBuffer } from '../utils';
 
@@ -57,6 +60,43 @@ const convertPath = (p: string) => {
     return p;
 };
 
+const defaultPath = process.env.CI ? '/tmp/file'
+    : process.env.DEFAULT_STORE_PATH || '/data/file/hydro';
+const FileSetting = Schema.intersect([
+    Schema.object({
+        type: Schema.union([
+            Schema.const('file').description('local file provider'),
+            Schema.const('s3').description('s3 provider'),
+        ] as const).description('provider type'),
+        endPointForUser: Schema.string().default('/fs/'),
+        endPointForJudge: Schema.string().default('/fs/'),
+    }).description('setting_file'),
+    Schema.union([
+        Schema.object({
+            type: Schema.const('file').required(),
+            path: Schema.string().default(defaultPath).description('Storage path'),
+            secret: Schema.string().description('Download file sign secret').default(nanoid()),
+        }),
+        Schema.object({
+            type: Schema.const('s3').required(),
+            endPoint: Schema.string(),
+            accessKey: Schema.string().description('access key'),
+            secretKey: Schema.string().description('secret key').role('secret'),
+            bucket: Schema.string().default('hydro'),
+            region: Schema.string().default('us-east-1'),
+            pathStyle: Schema.boolean().default(true),
+        }),
+    ] as const),
+] as const).default({
+    type: 'file',
+    path: defaultPath,
+    endPointForUser: '/fs/',
+    endPointForJudge: '/fs/',
+    secret: nanoid(),
+});
+
+export const Config = FileSetting;
+
 class RemoteStorageService {
     public client: S3Client;
     public error = '';
@@ -67,9 +107,12 @@ class RemoteStorageService {
         judge: null,
     };
 
+    constructor(private config: ReturnType<typeof FileSetting>) {
+    }
+
     async start() {
         try {
-            logger.info('Starting storage service with endpoint:', builtinConfig.file.endPoint);
+            logger.info('Starting storage service with endpoint:', this.config.endPoint);
             const {
                 endPoint,
                 accessKey,
@@ -79,7 +122,7 @@ class RemoteStorageService {
                 pathStyle,
                 endPointForUser,
                 endPointForJudge,
-            } = builtinConfig.file;
+            } = this.config;
             this.bucket = bucket;
             const base = {
                 region,
@@ -261,13 +304,16 @@ class LocalStorageService {
     opts: null;
     private replaceWithAlternativeUrlFor: Record<'user' | 'judge', (originalUrl: string) => string>;
 
+    constructor(private config: ReturnType<typeof FileSetting>) {
+    }
+
     async start() {
-        logger.debug('Loading local storage service with path:', builtinConfig.file.path);
-        await ensureDir(builtinConfig.file.path);
-        this.dir = builtinConfig.file.path;
+        logger.debug('Loading local storage service with path:', this.config.path);
+        await ensureDir(this.config.path);
+        this.dir = this.config.path;
         this.replaceWithAlternativeUrlFor = {
-            user: parseAlternativeEndpointUrl(builtinConfig.file.endPointForUser),
-            judge: parseAlternativeEndpointUrl(builtinConfig.file.endPointForJudge),
+            user: parseAlternativeEndpointUrl(this.config.endPointForUser),
+            judge: parseAlternativeEndpointUrl(this.config.endPointForJudge),
         };
     }
 
@@ -314,7 +360,7 @@ class LocalStorageService {
         if (filename) url.searchParams.set('filename', filename);
         const expire = (Date.now() + (noExpire ? 7 * 24 * 3600 : 600) * 1000).toString();
         url.searchParams.set('expire', expire);
-        url.searchParams.set('secret', md5(`${target}/${expire}/${builtinConfig.file.secret}`));
+        url.searchParams.set('secret', md5(`${target}/${expire}/${this.config.secret}`));
         if (useAlternativeEndpointFor) return this.replaceWithAlternativeUrlFor[useAlternativeEndpointFor](url.toString());
         return `/${url.toString().split('localhost/')[1]}`;
     }
@@ -333,16 +379,47 @@ class LocalStorageService {
     }
 }
 
-let service; // eslint-disable-line import/no-mutable-exports
+let service;
 
-export async function loadStorageService() {
-    service = builtinConfig.file.type === 's3' ? new RemoteStorageService() : new LocalStorageService();
-    global.Hydro.service.storage = service;
+export async function apply(ctx: Context, config: ReturnType<typeof FileSetting>) {
+    if (config.type === 's3') {
+        service = new RemoteStorageService(config);
+    } else {
+        service = new LocalStorageService(config);
+    }
     await service.start();
+    ctx.inject(['server'], ({ server }) => {
+        let endpoint = config.endPoint;
+        if (config.type === 's3' && !config.pathStyle) {
+            try {
+                const parsed = new URL(config.endPoint);
+                parsed.hostname = `${config.bucket}.${parsed.hostname}`;
+                endpoint = parsed.toString();
+            } catch (e) {
+                logger.warn('Failed to parse file endpoint');
+            }
+        }
+        const proxyMiddleware = proxy('/fs', {
+            target: endpoint,
+            changeOrigin: true,
+            rewrite: (p) => p.replace('/fs', ''),
+        });
+        server.addCaptureRoute('/fs/', async (c, next) => {
+            if (c.request.search.toLowerCase().includes('x-amz-credential')) {
+                c.nolog = true;
+                return await proxyMiddleware(c, next);
+            }
+            c.request.path = c.path = c.path.split('/fs')[1];
+            return await next();
+        });
+    });
+    ctx.set('storage', service);
 }
 
-export default new Proxy({}, {
+/** @deprecated use ctx.storage instead */
+const serviceProxy = new Proxy({}, {
     get(self, key) {
         return service[key];
     },
 }) as RemoteStorageService | LocalStorageService;
+export default serviceProxy;

packages/hydrooj/src/settings.ts

@@ -1,110 +1,120 @@
 import yaml from 'js-yaml';
-import { nanoid } from 'nanoid';
 import Schema from 'schemastery';
+import { Context, Service } from './context';
 import { Logger } from './logger';
-import { Flatten } from './typeutils';
 
-const defaultPath = process.env.CI ? '/tmp/file'
-    : process.env.DEFAULT_STORE_PATH || '/data/file/hydro';
-const FileSetting = Schema.intersect([
-    Schema.object({
-        type: Schema.union([
-            Schema.const('file').description('local file provider'),
-            Schema.const('s3').description('s3 provider'),
-        ] as const).description('provider type'),
-        endPointForUser: Schema.string().default('/fs/'),
-        endPointForJudge: Schema.string().default('/fs/'),
-    }).description('setting_file'),
-    Schema.union([
-        Schema.object({
-            type: Schema.const('file').required(),
-            path: Schema.string().default(defaultPath).description('Storage path'),
-            secret: Schema.string().description('Download file sign secret').default(nanoid()),
-        }),
-        Schema.object({
-            type: Schema.const('s3').required(),
-            endPoint: Schema.string(),
-            accessKey: Schema.string().description('access key'),
-            secretKey: Schema.string().description('secret key').role('secret'),
-            bucket: Schema.string().default('hydro'),
-            region: Schema.string().default('us-east-1'),
-            pathStyle: Schema.boolean().default(true),
-        }),
-    ] as const),
-] as const).default({
-    type: 'file',
-    path: defaultPath,
-    endPointForUser: '/fs/',
-    endPointForJudge: '/fs/',
-    secret: nanoid(),
-});
-
-const builtinSettings = Schema.object({
-    file: FileSetting,
-});
-export const SystemSettings: Schema[] = [builtinSettings];
-export let configSource = ''; // eslint-disable-line import/no-mutable-exports
-export let systemConfig: any = {}; // eslint-disable-line import/no-mutable-exports
 const logger = new Logger('settings');
-const update = [];
 
-export async function loadConfig() {
-    const config = await global.Hydro.service.db.collection('system').findOne({ _id: 'config' });
-    try {
-        configSource = config?.value || '{}';
-        systemConfig = yaml.load(configSource);
-        logger.info('Successfully loaded config');
-        for (const u of update) u();
-    } catch (e) {
-        logger.error('Failed to load config', e.message);
+declare module './context' {
+    interface Context {
+        config: ConfigService;
     }
 }
-export async function saveConfig(config: any) {
-    Schema.intersect(SystemSettings)(config);
-    const value = yaml.dump(config);
-    await global.Hydro.service.db.collection('system').updateOne({ _id: 'config' }, { $set: { value } }, { upsert: true });
-    loadConfig();
-}
-export async function setConfig(key: string, value: any) {
-    const path = key.split('.');
-    if (path.filter((i) => ['__proto__', 'prototype'].includes(i)).length) throw new Error('Invalid key');
-    const t = path.pop();
-    let cursor = systemConfig;
-    for (const p of path) {
-        cursor[p] ||= {};
-        cursor = cursor[p];
-    }
-    cursor[t] = value;
-    await saveConfig(systemConfig);
-}
 
-export function requestConfig<T, S>(s: Schema<T, S>): {
-    config: ReturnType<Schema<T, S>>,
-    setConfig: (key: keyof Flatten<ReturnType<Schema<T, S>>> & string, value: any) => Promise<void>,
-} {
-    SystemSettings.push(s);
-    let curValue = s(systemConfig);
-    update.push(() => {
+export class ConfigService extends Service {
+    static inject = ['db'];
+
+    settings: Schema[] = [];
+    systemConfig: any = {};
+    configSource: string = '';
+
+    constructor(ctx: Context) {
+        super(ctx, 'config');
+    }
+
+    async [Service.setup]() {
+        return await this.loadConfig();
+    }
+
+    async loadConfig() {
+        const config = await this.ctx.db.collection('system').findOne({ _id: 'config' });
         try {
-            curValue = s(systemConfig);
+            this.configSource = config?.value || '{}';
+            this.systemConfig = yaml.load(this.configSource);
+            this.ctx.emit('system/setting', { config: this.configSource });
+            logger.info('Successfully loaded config');
         } catch (e) {
-            logger.warn('Cannot read config: ', e.message);
-            curValue = null;
+            logger.error('Failed to load config', e.message);
         }
-    });
-    return {
-        config: new Proxy(curValue as any, {
-            get(self, key: string) {
-                return curValue?.[key];
-            },
-            set(self) {
-                throw new Error(`Not allowed to set setting ${self.p.join('.')}`);
-            },
-        }),
-        setConfig,
-    };
-}
+    }
 
-const builtin = requestConfig(builtinSettings);
-export const builtinConfig = builtin.config;
-export const setBuiltinConfig = builtin.setConfig;
+    applyDelta(source: any, key: string, value: any) {
+        const path = key.split('.');
+        if (path.filter((i) => ['__proto__', 'prototype'].includes(i)).length) return false;
+        const t = path.pop();
+        const root = JSON.parse(JSON.stringify(source));
+        let cursor = root;
+        for (const p of path) {
+            cursor[p] ||= {};
+            cursor = cursor[p];
+        }
+        cursor[t] = value;
+        return root;
+    }
+
+    isPatchValid(key: string, value: any) {
+        const root = this.applyDelta(this.systemConfig, key, value);
+        try {
+            Schema.intersect(this.settings)(root);
+        } catch (e) {
+            return false;
+        }
+        return true;
+    }
+
+    async saveConfig(config: any) {
+        Schema.intersect(this.settings)(config);
+        const value = yaml.dump(config);
+        await this.ctx.db.collection('system').updateOne({ _id: 'config' }, { $set: { value } }, { upsert: true });
+        await this.loadConfig();
+    }
+
+    async setConfig(key: string, value: any) {
+        const newConfig = this.applyDelta(this.systemConfig, key, value);
+        await this.saveConfig(newConfig);
+    }
+
+    get(path: string) {
+        let root = this.systemConfig;
+        for (const p of path.split('.')) {
+            root = root[p];
+        }
+        return root;
+    }
+
+    requestConfig<T, S>(s: Schema<T, S>): ReturnType<Schema<T, S>> {
+        this.ctx.effect(() => {
+            this.settings.push(s);
+            return () => {
+                this.settings = this.settings.filter((v) => v !== s);
+            };
+        });
+        let curValue = s(this.systemConfig);
+        this.ctx.on('system/setting', () => {
+            try {
+                curValue = s(this.systemConfig);
+            } catch (e) {
+                logger.warn('Cannot read config: ', e.message);
+                curValue = null;
+            }
+        });
+        const that = this;
+        const getAccess = (path: (string | symbol)[]) => {
+            let currentValue = curValue;
+            for (const p of path) {
+                currentValue = currentValue[p];
+            }
+            if (typeof currentValue !== 'object') return curValue;
+            return new Proxy(currentValue, {
+                get(self, key: string) {
+                    return getAccess(path.concat(key));
+                },
+                set(self, p: string | symbol, newValue: any) {
+                    that.setConfig(path.concat(p).join(','), newValue);
+                    return true;
+                },
+            });
+        };
+        return getAccess([]);
+    }
+}

packages/hydrooj/src/upgrade.ts

@@ -28,7 +28,6 @@ import {
 } from './pipelineUtils';
 import db from './service/db';
 import { MigrationScript } from './service/migration';
-import { setBuiltinConfig } from './settings';
 import welcome from './welcome';
 
 const logger = new Logger('upgrade');
@@ -423,7 +422,7 @@ export const coreScripts: MigrationScript[] = [
             'file.pathStyle', 'file.endPointForUser', 'file.endPointForJudge',
         ] as any[]) as any;
         if ((endPoint && accessKey) || process.env.MINIO_ACCESS_KEY) {
-            await setBuiltinConfig('file', {
+            await app.get('config').setConfig('file', {
                 type: 's3',
                 endPoint: process.env.MINIO_ACCESS_KEY ? 'http://127.0.0.1:9000/' : endPoint,
                 accessKey: process.env.MINIO_ACCESS_KEY || accessKey,
@@ -702,4 +701,5 @@ export const coreScripts: MigrationScript[] = [
             } catch { }
         });
     },
+    // Start Hydro v5
 ];

packages/hydrooj/src/utils.ts

@@ -1,3 +1,5 @@
+import { isNullable } from 'cosmokit';
+
 export function buildProjection<T extends string | number = string>(fields: readonly T[]): Record<T, true> {
     const o: Record<T, true> = Object.create(null);
     for (const k of fields) o[k] = true;
@@ -17,9 +19,13 @@ export function ArgMethod(target: any, funcName: string, obj: any) {
 }
 
 export function unwrapExports(module: any) {
-    return (!module || typeof module !== 'object') ? module
-        : 'apply' in module && typeof module.apply === 'function' ? module
-            : 'default' in module ? module.default : module;
+    if (isNullable(module)) return module;
+    if (typeof module !== 'object') return module;
+    if ('apply' in module && typeof module.apply === 'function') return module;
+    // https://github.com/evanw/esbuild/issues/2623
+    // https://esbuild.github.io/content-types/#default-interop
+    if (!module.__esModule) return module;
+    return module.default ?? module;
 }
 
 export * from '@hydrooj/utils/lib/utils';

packages/ui-default/index.ts

@@ -1,7 +1,7 @@
 /* eslint-disable global-require */
 import {
   ContestModel, Context, Handler, ObjectId, param, PERM, PRIV, ProblemModel, Schema,
-  SettingModel, SystemModel, SystemSettings, Types, UserModel,
+  SettingModel, SystemModel, Types, UserModel,
 } from 'hydrooj';
 import convert from 'schemastery-jsonschema';
 import markdown from './backendlib/markdown';
@@ -74,7 +74,7 @@ class MarkdownHandler extends Handler {
 
 class SystemConfigSchemaHandler extends Handler {
   async get() {
-    const schema = convert(Schema.intersect(SystemSettings) as any, true);
+    const schema = convert(Schema.intersect(this.ctx.config.settings) as any, true);
     this.response.body = schema;
   }
 }