diff --git a/README.md b/README.md index ff15f3d72..7ab39a30b 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,10 @@ Hydro 用户群:1085853538 - core: 移除旧版本 bus 调用 - migrate: 修复 hustoj 自动运行 - core: 强制要求使用 inject 声明插件依赖 +- core: 移除 global.Hydro.service.db +- core: 移除 global.Hydro.service.service +- core: 移除 global.Hydro.service.storage +- core: Settings: 支持使用 Schemastery ## Hydro 4.19.0 / UI 4.57.0 diff --git a/packages/hydrooj/src/entry/cli.ts b/packages/hydrooj/src/entry/cli.ts index 70433bee5..aafdf4c38 100644 --- a/packages/hydrooj/src/entry/cli.ts +++ b/packages/hydrooj/src/entry/cli.ts @@ -6,7 +6,9 @@ import cac from 'cac'; import fs from 'fs-extra'; import { ObjectId } from 'mongodb'; import { Context } from '../context'; -import db from '../service/db'; +import { load as loadOptions } from '../options'; +import { MongoService } from '../service/db'; +import { ConfigService } from '../settings'; import { addon, builtinModel, lib, model, script, service, setting, } from './common'; @@ -109,10 +111,15 @@ export async function load(ctx: Context) { require('../service/bus').apply(ctx); const pending = global.addons; const fail = []; - await db.start(); - await require('../settings').loadConfig(); + await ctx.plugin(MongoService, loadOptions()); + await ctx.plugin(ConfigService); await require('../model/system').runConfig(); - await require('../service/storage').loadStorageService(); + ctx = await new Promise((resolve) => { + ctx.inject(['loader'], (c) => { + resolve(c); + }); + }); + await ctx.loader.reloadPlugin(require.resolve('../service/storage'), 'file', 'hydrooj/service/storage'); require('../lib/index'); await Promise.all([ lib(pending, fail, ctx), @@ -131,7 +138,7 @@ export async function load(ctx: Context) { await addon(pending, fail, ctx); const scriptDir = path.resolve(__dirname, '..', 'script'); for (const h of await fs.readdir(scriptDir)) { - ctx.loader.reloadPlugin(path.resolve(scriptDir, h), {}, `hydrooj/script/${h.split('.')[0]}`); + ctx.loader.reloadPlugin(path.resolve(scriptDir, h), '', `hydrooj/script/${h.split('.')[0]}`); } await script(pending, fail, ctx); await cli(); diff --git a/packages/hydrooj/src/entry/common.ts b/packages/hydrooj/src/entry/common.ts index 76d37ed91..2148deb37 100644 --- a/packages/hydrooj/src/entry/common.ts +++ b/packages/hydrooj/src/entry/common.ts @@ -42,7 +42,7 @@ const getLoader = (type: LoadTask, filename: string, dontLoad = false) => async try { if (dontLoad) throw new Error(`deprecated ${name} in %s will not be load: ${i}`); const m = unwrapExports(require(p)); - if (m.apply) ctx.loader.reloadPlugin(p, {}); + if (m.apply) ctx.loader.reloadPlugin(p, ''); else logger.info(`${name} init: %s`, i); } catch (e) { fail.push(i); @@ -69,7 +69,7 @@ export async function builtinModel(ctx: Context) { const models = await fs.readdir(modelDir); for (const t of models.filter((i) => i.endsWith('.ts'))) { const q = path.resolve(modelDir, t); - if ('apply' in require(q)) ctx.loader.reloadPlugin(q, {}, `hydrooj/model/${t.split('.')[0]}`); + if ('apply' in require(q)) ctx.loader.reloadPlugin(q, '', `hydrooj/model/${t.split('.')[0]}`); } } diff --git a/packages/hydrooj/src/entry/worker.ts b/packages/hydrooj/src/entry/worker.ts index bcda013c5..ffe3accf4 100644 --- a/packages/hydrooj/src/entry/worker.ts +++ b/packages/hydrooj/src/entry/worker.ts @@ -7,7 +7,8 @@ import fs from 'fs-extra'; import { Context } from '../context'; import { Logger } from '../logger'; import { load } from '../options'; -import db from '../service/db'; +import { MongoService } from '../service/db'; +import { ConfigService } from '../settings'; import { addon, builtinModel, handler, lib, locale, model, script, service, setting, template, @@ -22,8 +23,8 @@ export async function apply(ctx: Context) { require('../utils'); require('../error'); require('../service/bus').apply(ctx); - const config = load(); - if (!process.env.CI && !config) { + const url = await MongoService.getUrl(); + if (!url) { logger.info('Starting setup'); await require('./setup').load(ctx); } @@ -33,12 +34,16 @@ export async function apply(ctx: Context) { locale(pending, fail), template(pending, fail), ]); - await db.start(); - await require('../settings').loadConfig(); + await ctx.plugin(MongoService, load()); + await ctx.plugin(ConfigService); const modelSystem = require('../model/system'); await modelSystem.runConfig(); - const storage = require('../service/storage'); - await storage.loadStorageService(); + ctx = await new Promise((resolve) => { + ctx.inject(['loader'], (c) => { + resolve(c); + }); + }); + await ctx.loader.reloadPlugin(require.resolve('../service/storage'), 'file', 'hydrooj/service/storage'); await ctx.plugin(require('../service/hmr').default, { watch: argv.options.watch }); await require('../service/worker').apply(ctx); await require('../service/server').apply(ctx); @@ -61,14 +66,14 @@ export async function apply(ctx: Context) { const handlerDir = path.resolve(__dirname, '..', 'handler'); const handlers = await fs.readdir(handlerDir); for (const h of handlers.filter((i) => i.endsWith('.ts'))) { - ctx.loader.reloadPlugin(path.resolve(handlerDir, h), {}, `hydrooj/handler/${h.split('.')[0]}`); + ctx.loader.reloadPlugin(path.resolve(handlerDir, h), '', `hydrooj/handler/${h.split('.')[0]}`); } ctx.plugin(require('../service/migration').default); await handler(pending, fail, ctx); await addon(pending, fail, ctx); const scriptDir = path.resolve(__dirname, '..', 'script'); for (const h of await fs.readdir(scriptDir)) { - ctx.loader.reloadPlugin(path.resolve(scriptDir, h), {}, `hydrooj/script/${h.split('.')[0]}`); + ctx.loader.reloadPlugin(path.resolve(scriptDir, h), '', `hydrooj/script/${h.split('.')[0]}`); } await script(pending, fail, ctx); await ctx.parallel('app/started'); diff --git a/packages/hydrooj/src/handler/manage.ts b/packages/hydrooj/src/handler/manage.ts index 6097f1e4e..00a52dbfa 100644 --- a/packages/hydrooj/src/handler/manage.ts +++ b/packages/hydrooj/src/handler/manage.ts @@ -17,7 +17,6 @@ import { check } from '../service/check'; import { ConnectionHandler, Handler, param, requireSudo, Types, } from '../service/server'; -import { configSource, saveConfig, SystemSettings } from '../settings'; import * as judge from './judge'; const logger = new Logger('manage'); @@ -184,27 +183,26 @@ class SystemConfigHandler extends SystemHandler { @requireSudo async get() { this.response.template = 'manage_config.html'; - let value = configSource; + let value = this.ctx.config.configSource; try { - value = yaml.dump(Schema.intersect(SystemSettings)(yaml.load(configSource))); + value = yaml.dump(Schema.intersect(this.ctx.config.settings)(yaml.load(value))); } catch (e) { } this.response.body = { - schema: Schema.intersect(SystemSettings).toJSON(), + schema: Schema.intersect(this.ctx.config.settings).toJSON(), value, }; } @requireSudo @param('value', Types.String) - async post(domainId: string, value: string) { + async post({ }, value: string) { let config; try { config = yaml.load(value); - Schema.intersect(SystemSettings)(config); } catch (e) { throw new ValidationError('value'); } - await saveConfig(config); + await this.ctx.config.saveConfig(config); } } diff --git a/packages/hydrooj/src/handler/misc.ts b/packages/hydrooj/src/handler/misc.ts index 95ce302ef..48ce4df6f 100644 --- a/packages/hydrooj/src/handler/misc.ts +++ b/packages/hydrooj/src/handler/misc.ts @@ -15,7 +15,6 @@ import { Handler, param, post, requireSudo, Types, } from '../service/server'; import { encodeRFC5987ValueChars } from '../service/storage'; -import { builtinConfig } from '../settings'; import { md5, sortFiles } from '../utils'; class SwitchLanguageHandler extends Handler { @@ -110,7 +109,7 @@ export class StorageHandler extends Handler { @param('expire', Types.UnsignedInt) @param('secret', Types.String) async get(domainId: string, target: string, filename = '', expire: number, secret: string) { - const expected = md5(`${target}/${expire}/${builtinConfig.file.secret}`); + const expected = md5(`${target}/${expire}/${this.ctx.config.get('file.secret')}`); if (expire < Date.now()) throw new AccessDeniedError(); if (secret !== expected) throw new AccessDeniedError(); this.response.body = await storage.get(target); diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 5242488cf..8bf9e477c 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -599,9 +599,6 @@ export interface Model { export interface HydroService { /** @deprecated */ bus: Context, - db: typeof import('./service/db').default, - server: typeof import('./service/server'), - storage: typeof import('./service/storage').default, } export interface GeoIP { diff --git a/packages/hydrooj/src/loader.ts b/packages/hydrooj/src/loader.ts index 09cad37b6..c0ce4412c 100644 --- a/packages/hydrooj/src/loader.ts +++ b/packages/hydrooj/src/loader.ts @@ -14,11 +14,14 @@ import './ui'; import * as I18n from './lib/i18n'; import { Logger } from './logger'; -import { Context, Service, ScopeStatus } from './context'; +import { + Context, Service, ScopeStatus, EffectScope, +} from './context'; // eslint-disable-next-line import/no-duplicates import { sleep, unwrapExports } from './utils'; import { PRIV } from './model/builtin'; import { getAddons } from './options'; +import Schema from 'schemastery'; const argv = cac().parse(); const logger = new Logger('loader'); @@ -43,29 +46,20 @@ if (process.env.NIX_PROFILES) { } } -export function resolveConfig(plugin: any, config: any) { - if (config === false) return; - if (config === true) config = undefined; - config ??= {}; - const schema = plugin['Config'] || plugin['schema']; - if (schema && plugin['schema'] !== false) config = schema(config); - return config; -} - export class Loader extends Service { - public state: Record = Object.create(null); + public state: Record = Object.create(null); public config: {}; public suspend = false; public cache: Record = Object.create(null); // public warnings: Record = Object.create(null); + static inject = ['config', 'timer', 'i18n', 'logger']; + constructor(ctx: Context) { super(ctx, 'loader'); - } - [Service.setup]() { - this.ctx.on('app/started', () => { - this.ctx.setInterval(async () => { + ctx.on('app/started', () => { + ctx.setInterval(async () => { const pending = Object.entries(this.state).filter((v) => v[1].status === ScopeStatus.PENDING); if (pending.length) { logger.warn('Plugins are still pending: %s', pending.map((v) => v[0]).join(', ')); @@ -103,18 +97,27 @@ export class Loader extends Service { } } - async reloadPlugin(key: string, config: any, asName = '') { + resolveConfig(plugin: any, configScope: string) { + const schema = plugin['Config'] || plugin['schema']; + if (!schema) return {}; + const schemaRequest = configScope ? Schema.object({ + [configScope]: schema, + }) : schema; + const res = this.ctx.config.requestConfig(schemaRequest); + return configScope ? res[configScope] : res; + } + + async reloadPlugin(key: string, configScope: string, asName = '') { + const plugin = await this.resolvePlugin(key); + if (!plugin) return; + const config = this.resolveConfig(plugin, configScope); + if (asName) plugin.name = asName; let fork = this.state[key]; if (fork) { logger.info('reload plugin %c', key.split('node_modules').pop()); fork.update(config); } else { logger.info('apply plugin %c', key.split('node_modules').pop()); - const plugin = await this.resolvePlugin(key); - if (!plugin) return; - resolveConfig(plugin, config); - if (asName) plugin.name = asName; - // fork = parent.plugin(plugin, this.interpolate(config)); fork = this.ctx.plugin(plugin, config); if (!fork) return; this.state[key] = fork; @@ -129,7 +132,8 @@ export class Loader extends Service { try { this.cache[name] ||= require.resolve(name, { paths: HYDROPATH }); } catch (e) { - logger.error(err.message); + logger.error('Failed to resolve plugin %s', name); + logger.error(err); return; } } @@ -138,14 +142,12 @@ export class Loader extends Service { } app.plugin(I18n); +app.plugin(Loader); async function preload() { global.app = await new Promise((resolve) => { - app.inject(['timer', 'i18n', 'logger'], (c) => { - c.plugin(Loader); - c.inject(['loader'], (ctx) => { - resolve(ctx); - }); + app.inject(['timer', 'i18n', 'logger', '$api'], (c) => { + resolve(c); }); }); for (const a of [path.resolve(__dirname, '..'), ...getAddons()]) { diff --git a/packages/hydrooj/src/model/setting.ts b/packages/hydrooj/src/model/setting.ts index a60ee380f..5c8f0d3ea 100644 --- a/packages/hydrooj/src/model/setting.ts +++ b/packages/hydrooj/src/model/setting.ts @@ -4,6 +4,7 @@ import * as cordis from '@cordisjs/core'; import yaml from 'js-yaml'; import { Dictionary } from 'lodash'; import moment from 'moment-timezone'; +import Schema from 'schemastery'; import { LangConfig, parseLang } from '@hydrooj/common'; import { retry } from '@hydrooj/utils'; import { Context, Service } from '../context'; @@ -70,7 +71,55 @@ export const Setting = ( }; }; -export const PreferenceSetting = (...settings: _Setting[]) => { +declare global { + namespace Schemastery { + interface Meta { // eslint-disable-line @typescript-eslint/no-unused-vars + family: string; + } + } +} + +function schemaToSettings(schema: Schema) { + const result: _Setting[] = []; + const processNode = (key: string, s: Schema | Schema | Schema, defaultFamily = 'setting_basic') => { + if (s.dict) throw new Error('Dict is not supported here'); + let flag = (s.meta?.hidden ? FLAG_HIDDEN : 0) + | (s.meta?.disabled ? FLAG_DISABLED : 0); + const type = s.type === 'number' ? 'number' + : s.type === 'boolean' ? 'checkbox' + : s.meta?.role === 'textarea' ? 'textarea' : 'text'; + if (s.meta?.role === 'password') flag |= FLAG_SECRET; + const options = {}; + for (const item of s.list || []) { + if (item.type !== 'const') throw new Error('List item must be a constant'); + options[item.value] = item.meta?.description || item.value; + } + return { + family: s.meta?.family || defaultFamily, + key, + value: s.meta?.default, + name: key, + desc: s.meta?.description, + flag, + subType: '', + type: s.list ? 'select' : type, + range: s.list ? options : null, + } as _Setting; + }; + if (!schema.dict) return []; + for (const key in schema.dict) { + const value = schema.dict[key]; + if (value.dict) { + for (const subkey in value.dict) { + result.push(processNode(`${key}.${subkey}`, value.dict[subkey], value.meta?.family)); + } + } else result.push(processNode(key, value)); + } + return result; +} + +export const PreferenceSetting = (...settings: _Setting[] | Schema[]) => { + settings = settings.flatMap((s) => (s instanceof Schema ? schemaToSettings(s) : s) as _Setting[]); for (const setting of settings) { if (PREFERENCE_SETTINGS.find((s) => s.key === setting.key)) logger.warn(`Duplicate setting key: ${setting.key}`); PREFERENCE_SETTINGS.push(setting); @@ -89,7 +138,8 @@ export const PreferenceSetting = (...settings: _Setting[]) => { } }; }; -export const AccountSetting = (...settings: _Setting[]) => { +export const AccountSetting = (...settings: _Setting[] | Schema[]) => { + settings = settings.flatMap((s) => (s instanceof Schema ? schemaToSettings(s) : s) as _Setting[]); for (const setting of settings) { if (ACCOUNT_SETTINGS.find((s) => s.key === setting.key)) logger.warn(`Duplicate setting key: ${setting.key}`); ACCOUNT_SETTINGS.push(setting); @@ -108,7 +158,8 @@ export const AccountSetting = (...settings: _Setting[]) => { } }; }; -export const DomainUserSetting = (...settings: _Setting[]) => { +export const DomainUserSetting = (...settings: _Setting[] | Schema[]) => { + settings = settings.flatMap((s) => (s instanceof Schema ? schemaToSettings(s) : s) as _Setting[]); for (const setting of settings) { if (DOMAIN_USER_SETTINGS.find((s) => s.key === setting.key)) logger.warn(`Duplicate setting key: ${setting.key}`); DOMAIN_USER_SETTINGS.push(setting); @@ -123,7 +174,8 @@ export const DomainUserSetting = (...settings: _Setting[]) => { } }; }; -export const DomainSetting = (...settings: _Setting[]) => { +export const DomainSetting = (...settings: _Setting[] | Schema[]) => { + settings = settings.flatMap((s) => (s instanceof Schema ? schemaToSettings(s) : s) as _Setting[]); for (const setting of settings) { if (DOMAIN_SETTINGS.find((s) => s.key === setting.key)) logger.warn(`Duplicate setting key: ${setting.key}`); DOMAIN_SETTINGS.push(setting); @@ -138,7 +190,8 @@ export const DomainSetting = (...settings: _Setting[]) => { } }; }; -export const SystemSetting = (...settings: _Setting[]) => { +export const SystemSetting = (...settings: _Setting[] | Schema[]) => { + settings = settings.flatMap((s) => (s instanceof Schema ? schemaToSettings(s) : s) as _Setting[]); for (const setting of settings) { if (SYSTEM_SETTINGS.find((s) => s.key === setting.key)) logger.warn(`Duplicate setting key: ${setting.key}`); SYSTEM_SETTINGS.push(setting); @@ -324,6 +377,7 @@ export class SettingService extends Service { } } +export const inject = ['db']; export async function apply(ctx: Context) { ctx.provide('setting', undefined, true); ctx.plugin(SettingService); @@ -331,7 +385,7 @@ export async function apply(ctx: Context) { const system = global.Hydro.model.system; for (const setting of SYSTEM_SETTINGS) { if (!setting.value) continue; - const current = await global.Hydro.service.db.collection('system').findOne({ _id: setting.key }); + const current = await ctx.db.collection('system').findOne({ _id: setting.key }); if (!current || current.value == null || current.value === '') { await retry(system.set, setting.key, setting.value); } @@ -355,6 +409,8 @@ export async function apply(ctx: Context) { global.Hydro.model.setting = { apply, + inject, + SettingService, Setting, PreferenceSetting, diff --git a/packages/hydrooj/src/service/db.ts b/packages/hydrooj/src/service/db.ts index 701b7375e..51a2ea12f 100644 --- a/packages/hydrooj/src/service/db.ts +++ b/packages/hydrooj/src/service/db.ts @@ -4,6 +4,7 @@ import { } from 'mongodb'; import mongoUri from 'mongodb-uri'; import { Time } from '@hydrooj/utils'; +import { Context, Service } from '../context'; import { ValidationError } from '../error'; import { Logger } from '../logger'; import { load } from '../options'; @@ -25,12 +26,28 @@ interface MongoConfig { collectionMap?: Record, } -class MongoService { +declare module '../context' { + interface Context { + db: MongoService; + } +} + +export class MongoService extends Service { public client: MongoClient; public db: Db; - private opts: MongoConfig; - static buildUrl(opts: MongoConfig) { + constructor(ctx: Context, private config: MongoConfig = {}) { + super(ctx, 'db'); + } + + static async getUrl() { + if (process.env.CI) { + const { MongoMemoryServer } = require('mongodb-memory-server'); + const mongod = await MongoMemoryServer.create(); + return mongod.getUri(); + } + const opts = load(); + if (!opts) return null; let mongourl = `${opts.protocol || 'mongodb'}://`; if (opts.username) mongourl += `${opts.username}:${encodeURIComponent(opts.password)}@`; mongourl += `${opts.host}:${opts.port}/${opts.name}`; @@ -38,25 +55,18 @@ class MongoService { return mongourl; } - async start() { - const opts = load() || {}; - let mongourl = MongoService.buildUrl(opts); + async [Service.setup]() { + const mongourl = await MongoService.getUrl(); const url = mongoUri.parse(mongourl); - if (process.env.CI) { - const { MongoMemoryServer } = require('mongodb-memory-server'); - const mongod = await MongoMemoryServer.create(); - mongourl = mongod.getUri(); - } - this.opts = opts; this.client = await MongoClient.connect(mongourl); this.db = this.client.db(url.database || 'hydro'); await bus.parallel('database/connect', this.db); - setInterval(() => this.fixExpireAfter(), Time.hour); + this.ctx.setInterval(() => this.fixExpireAfter(), Time.hour); } public collection(c: K) { - let coll = this.opts.prefix ? `${this.opts.prefix}.${c}` : c; - if (this.opts.collectionMap?.[coll]) coll = this.opts.collectionMap[coll]; + let coll = this.config.prefix ? `${this.config.prefix}.${c}` : c; + if (this.config.collectionMap?.[coll]) coll = this.config.collectionMap[coll]; return this.db.collection(coll); } @@ -173,7 +183,11 @@ class MongoService { } } -const service = new MongoService(); -global.Hydro.service.db = service; -export default service; -export const collection = service.collection.bind(service); +/** @deprecated use ctx.db instead */ +const db = new Proxy({} as MongoService, { + get(target, prop) { + return app.get('db')?.[prop]; + }, +}); + +export default db; diff --git a/packages/hydrooj/src/service/server.ts b/packages/hydrooj/src/service/server.ts index 0dbd17307..7d261f31b 100644 --- a/packages/hydrooj/src/service/server.ts +++ b/packages/hydrooj/src/service/server.ts @@ -1,7 +1,6 @@ import { resolve } from 'path'; import cac from 'cac'; import fs from 'fs-extra'; -import proxy from 'koa-proxies'; import cache from 'koa-static-cache'; import { type FindCursor, ObjectId } from 'mongodb'; import { @@ -18,7 +17,6 @@ import { PERM, PRIV } from '../model/builtin'; import * as opcount from '../model/opcount'; import * as OplogModel from '../model/oplog'; import * as system from '../model/system'; -import { builtinConfig } from '../settings'; import db from './db'; import baseLayer from './layers/base'; import domainLayer from './layers/domain'; @@ -93,26 +91,6 @@ export async function apply(ctx: Context) { }); if (process.env.HYDRO_CLI) return; ctx.inject(['server'], ({ server, on }) => { - let endpoint = builtinConfig.file.endPoint; - if (builtinConfig.file.type === 's3' && !builtinConfig.file.pathStyle) { - try { - const parsed = new URL(builtinConfig.file.endPoint); - parsed.hostname = `${builtinConfig.file.bucket}.${parsed.hostname}`; - endpoint = parsed.toString(); - } catch (e) { - logger.warn('Failed to parse file endpoint'); - } - } - const proxyMiddleware = proxy('/fs', { - target: endpoint, - changeOrigin: true, - rewrite: (p) => p.replace('/fs', ''), - }); - server.addCaptureRoute('/fs/', async (c, next) => { - if (c.request.search.toLowerCase().includes('x-amz-credential')) return await proxyMiddleware(c, next); - c.request.path = c.path = c.path.split('/fs')[1]; - return await next(); - }); server.addHandlerLayer('init', async (c, next) => { const init = Date.now(); try { diff --git a/packages/hydrooj/src/service/storage.ts b/packages/hydrooj/src/service/storage.ts index 0d9ed5ae6..0581ca428 100644 --- a/packages/hydrooj/src/service/storage.ts +++ b/packages/hydrooj/src/service/storage.ts @@ -12,9 +12,12 @@ import { copyFile, createReadStream, createWriteStream, ensureDir, existsSync, remove, stat, writeFile, } from 'fs-extra'; +import proxy from 'koa-proxies'; import { lookup } from 'mime-types'; +import { nanoid } from 'nanoid'; +import Schema from 'schemastery'; +import { Context } from '../context'; import { Logger } from '../logger'; -import { builtinConfig } from '../settings'; import { MaybeArray } from '../typeutils'; import { md5, streamToBuffer } from '../utils'; @@ -57,6 +60,43 @@ const convertPath = (p: string) => { return p; }; +const defaultPath = process.env.CI ? '/tmp/file' + : process.env.DEFAULT_STORE_PATH || '/data/file/hydro'; +const FileSetting = Schema.intersect([ + Schema.object({ + type: Schema.union([ + Schema.const('file').description('local file provider'), + Schema.const('s3').description('s3 provider'), + ] as const).description('provider type'), + endPointForUser: Schema.string().default('/fs/'), + endPointForJudge: Schema.string().default('/fs/'), + }).description('setting_file'), + Schema.union([ + Schema.object({ + type: Schema.const('file').required(), + path: Schema.string().default(defaultPath).description('Storage path'), + secret: Schema.string().description('Download file sign secret').default(nanoid()), + }), + Schema.object({ + type: Schema.const('s3').required(), + endPoint: Schema.string(), + accessKey: Schema.string().description('access key'), + secretKey: Schema.string().description('secret key').role('secret'), + bucket: Schema.string().default('hydro'), + region: Schema.string().default('us-east-1'), + pathStyle: Schema.boolean().default(true), + }), + ] as const), +] as const).default({ + type: 'file', + path: defaultPath, + endPointForUser: '/fs/', + endPointForJudge: '/fs/', + secret: nanoid(), +}); + +export const Config = FileSetting; + class RemoteStorageService { public client: S3Client; public error = ''; @@ -67,9 +107,12 @@ class RemoteStorageService { judge: null, }; + constructor(private config: ReturnType) { + } + async start() { try { - logger.info('Starting storage service with endpoint:', builtinConfig.file.endPoint); + logger.info('Starting storage service with endpoint:', this.config.endPoint); const { endPoint, accessKey, @@ -79,7 +122,7 @@ class RemoteStorageService { pathStyle, endPointForUser, endPointForJudge, - } = builtinConfig.file; + } = this.config; this.bucket = bucket; const base = { region, @@ -261,13 +304,16 @@ class LocalStorageService { opts: null; private replaceWithAlternativeUrlFor: Record<'user' | 'judge', (originalUrl: string) => string>; + constructor(private config: ReturnType) { + } + async start() { - logger.debug('Loading local storage service with path:', builtinConfig.file.path); - await ensureDir(builtinConfig.file.path); - this.dir = builtinConfig.file.path; + logger.debug('Loading local storage service with path:', this.config.path); + await ensureDir(this.config.path); + this.dir = this.config.path; this.replaceWithAlternativeUrlFor = { - user: parseAlternativeEndpointUrl(builtinConfig.file.endPointForUser), - judge: parseAlternativeEndpointUrl(builtinConfig.file.endPointForJudge), + user: parseAlternativeEndpointUrl(this.config.endPointForUser), + judge: parseAlternativeEndpointUrl(this.config.endPointForJudge), }; } @@ -314,7 +360,7 @@ class LocalStorageService { if (filename) url.searchParams.set('filename', filename); const expire = (Date.now() + (noExpire ? 7 * 24 * 3600 : 600) * 1000).toString(); url.searchParams.set('expire', expire); - url.searchParams.set('secret', md5(`${target}/${expire}/${builtinConfig.file.secret}`)); + url.searchParams.set('secret', md5(`${target}/${expire}/${this.config.secret}`)); if (useAlternativeEndpointFor) return this.replaceWithAlternativeUrlFor[useAlternativeEndpointFor](url.toString()); return `/${url.toString().split('localhost/')[1]}`; } @@ -333,16 +379,47 @@ class LocalStorageService { } } -let service; // eslint-disable-line import/no-mutable-exports +let service; -export async function loadStorageService() { - service = builtinConfig.file.type === 's3' ? new RemoteStorageService() : new LocalStorageService(); - global.Hydro.service.storage = service; +export async function apply(ctx: Context, config: ReturnType) { + if (config.type === 's3') { + service = new RemoteStorageService(config); + } else { + service = new LocalStorageService(config); + } await service.start(); + ctx.inject(['server'], ({ server }) => { + let endpoint = config.endPoint; + if (config.type === 's3' && !config.pathStyle) { + try { + const parsed = new URL(config.endPoint); + parsed.hostname = `${config.bucket}.${parsed.hostname}`; + endpoint = parsed.toString(); + } catch (e) { + logger.warn('Failed to parse file endpoint'); + } + } + const proxyMiddleware = proxy('/fs', { + target: endpoint, + changeOrigin: true, + rewrite: (p) => p.replace('/fs', ''), + }); + server.addCaptureRoute('/fs/', async (c, next) => { + if (c.request.search.toLowerCase().includes('x-amz-credential')) { + c.nolog = true; + return await proxyMiddleware(c, next); + } + c.request.path = c.path = c.path.split('/fs')[1]; + return await next(); + }); + }); + ctx.set('storage', service); } -export default new Proxy({}, { +/** @deprecated use ctx.storage instead */ +const serviceProxy = new Proxy({}, { get(self, key) { return service[key]; }, }) as RemoteStorageService | LocalStorageService; +export default serviceProxy; diff --git a/packages/hydrooj/src/settings.ts b/packages/hydrooj/src/settings.ts index f333e1b51..d55f67b53 100644 --- a/packages/hydrooj/src/settings.ts +++ b/packages/hydrooj/src/settings.ts @@ -1,110 +1,120 @@ import yaml from 'js-yaml'; -import { nanoid } from 'nanoid'; import Schema from 'schemastery'; +import { Context, Service } from './context'; import { Logger } from './logger'; -import { Flatten } from './typeutils'; -const defaultPath = process.env.CI ? '/tmp/file' - : process.env.DEFAULT_STORE_PATH || '/data/file/hydro'; -const FileSetting = Schema.intersect([ - Schema.object({ - type: Schema.union([ - Schema.const('file').description('local file provider'), - Schema.const('s3').description('s3 provider'), - ] as const).description('provider type'), - endPointForUser: Schema.string().default('/fs/'), - endPointForJudge: Schema.string().default('/fs/'), - }).description('setting_file'), - Schema.union([ - Schema.object({ - type: Schema.const('file').required(), - path: Schema.string().default(defaultPath).description('Storage path'), - secret: Schema.string().description('Download file sign secret').default(nanoid()), - }), - Schema.object({ - type: Schema.const('s3').required(), - endPoint: Schema.string(), - accessKey: Schema.string().description('access key'), - secretKey: Schema.string().description('secret key').role('secret'), - bucket: Schema.string().default('hydro'), - region: Schema.string().default('us-east-1'), - pathStyle: Schema.boolean().default(true), - }), - ] as const), -] as const).default({ - type: 'file', - path: defaultPath, - endPointForUser: '/fs/', - endPointForJudge: '/fs/', - secret: nanoid(), -}); - -const builtinSettings = Schema.object({ - file: FileSetting, -}); -export const SystemSettings: Schema[] = [builtinSettings]; -export let configSource = ''; // eslint-disable-line import/no-mutable-exports -export let systemConfig: any = {}; // eslint-disable-line import/no-mutable-exports const logger = new Logger('settings'); -const update = []; -export async function loadConfig() { - const config = await global.Hydro.service.db.collection('system').findOne({ _id: 'config' }); - try { - configSource = config?.value || '{}'; - systemConfig = yaml.load(configSource); - logger.info('Successfully loaded config'); - for (const u of update) u(); - } catch (e) { - logger.error('Failed to load config', e.message); +declare module './context' { + interface Context { + config: ConfigService; } } -export async function saveConfig(config: any) { - Schema.intersect(SystemSettings)(config); - const value = yaml.dump(config); - await global.Hydro.service.db.collection('system').updateOne({ _id: 'config' }, { $set: { value } }, { upsert: true }); - loadConfig(); -} -export async function setConfig(key: string, value: any) { - const path = key.split('.'); - if (path.filter((i) => ['__proto__', 'prototype'].includes(i)).length) throw new Error('Invalid key'); - const t = path.pop(); - let cursor = systemConfig; - for (const p of path) { - cursor[p] ||= {}; - cursor = cursor[p]; - } - cursor[t] = value; - await saveConfig(systemConfig); -} -export function requestConfig(s: Schema): { - config: ReturnType>, - setConfig: (key: keyof Flatten>> & string, value: any) => Promise, -} { - SystemSettings.push(s); - let curValue = s(systemConfig); - update.push(() => { +export class ConfigService extends Service { + static inject = ['db']; + + settings: Schema[] = []; + systemConfig: any = {}; + configSource: string = ''; + + constructor(ctx: Context) { + super(ctx, 'config'); + } + + async [Service.setup]() { + return await this.loadConfig(); + } + + async loadConfig() { + const config = await this.ctx.db.collection('system').findOne({ _id: 'config' }); try { - curValue = s(systemConfig); + this.configSource = config?.value || '{}'; + this.systemConfig = yaml.load(this.configSource); + this.ctx.emit('system/setting', { config: this.configSource }); + logger.info('Successfully loaded config'); } catch (e) { - logger.warn('Cannot read config: ', e.message); - curValue = null; + logger.error('Failed to load config', e.message); } - }); - return { - config: new Proxy(curValue as any, { - get(self, key: string) { - return curValue?.[key]; - }, - set(self) { - throw new Error(`Not allowed to set setting ${self.p.join('.')}`); - }, - }), - setConfig, - }; -} + } -const builtin = requestConfig(builtinSettings); -export const builtinConfig = builtin.config; -export const setBuiltinConfig = builtin.setConfig; + applyDelta(source: any, key: string, value: any) { + const path = key.split('.'); + if (path.filter((i) => ['__proto__', 'prototype'].includes(i)).length) return false; + const t = path.pop(); + const root = JSON.parse(JSON.stringify(source)); + let cursor = root; + for (const p of path) { + cursor[p] ||= {}; + cursor = cursor[p]; + } + cursor[t] = value; + return root; + } + + isPatchValid(key: string, value: any) { + const root = this.applyDelta(this.systemConfig, key, value); + try { + Schema.intersect(this.settings)(root); + } catch (e) { + return false; + } + return true; + } + + async saveConfig(config: any) { + Schema.intersect(this.settings)(config); + const value = yaml.dump(config); + await this.ctx.db.collection('system').updateOne({ _id: 'config' }, { $set: { value } }, { upsert: true }); + await this.loadConfig(); + } + + async setConfig(key: string, value: any) { + const newConfig = this.applyDelta(this.systemConfig, key, value); + await this.saveConfig(newConfig); + } + + get(path: string) { + let root = this.systemConfig; + for (const p of path.split('.')) { + root = root[p]; + } + return root; + } + + requestConfig(s: Schema): ReturnType> { + this.ctx.effect(() => { + this.settings.push(s); + return () => { + this.settings = this.settings.filter((v) => v !== s); + }; + }); + let curValue = s(this.systemConfig); + this.ctx.on('system/setting', () => { + try { + curValue = s(this.systemConfig); + } catch (e) { + logger.warn('Cannot read config: ', e.message); + curValue = null; + } + }); + const that = this; + const getAccess = (path: (string | symbol)[]) => { + let currentValue = curValue; + for (const p of path) { + currentValue = currentValue[p]; + } + if (typeof currentValue !== 'object') return curValue; + return new Proxy(currentValue, { + get(self, key: string) { + return getAccess(path.concat(key)); + }, + set(self, p: string | symbol, newValue: any) { + that.setConfig(path.concat(p).join(','), newValue); + return true; + }, + }); + }; + return getAccess([]); + } +} diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index 1bcb1f90d..5a4dca4c5 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -28,7 +28,6 @@ import { } from './pipelineUtils'; import db from './service/db'; import { MigrationScript } from './service/migration'; -import { setBuiltinConfig } from './settings'; import welcome from './welcome'; const logger = new Logger('upgrade'); @@ -423,7 +422,7 @@ export const coreScripts: MigrationScript[] = [ 'file.pathStyle', 'file.endPointForUser', 'file.endPointForJudge', ] as any[]) as any; if ((endPoint && accessKey) || process.env.MINIO_ACCESS_KEY) { - await setBuiltinConfig('file', { + await app.get('config').setConfig('file', { type: 's3', endPoint: process.env.MINIO_ACCESS_KEY ? 'http://127.0.0.1:9000/' : endPoint, accessKey: process.env.MINIO_ACCESS_KEY || accessKey, @@ -702,4 +701,5 @@ export const coreScripts: MigrationScript[] = [ } catch { } }); }, + // Start Hydro v5 ]; diff --git a/packages/hydrooj/src/utils.ts b/packages/hydrooj/src/utils.ts index 2f03138b6..c58c9badd 100644 --- a/packages/hydrooj/src/utils.ts +++ b/packages/hydrooj/src/utils.ts @@ -1,3 +1,5 @@ +import { isNullable } from 'cosmokit'; + export function buildProjection(fields: readonly T[]): Record { const o: Record = Object.create(null); for (const k of fields) o[k] = true; @@ -17,9 +19,13 @@ export function ArgMethod(target: any, funcName: string, obj: any) { } export function unwrapExports(module: any) { - return (!module || typeof module !== 'object') ? module - : 'apply' in module && typeof module.apply === 'function' ? module - : 'default' in module ? module.default : module; + if (isNullable(module)) return module; + if (typeof module !== 'object') return module; + if ('apply' in module && typeof module.apply === 'function') return module; + // https://github.com/evanw/esbuild/issues/2623 + // https://esbuild.github.io/content-types/#default-interop + if (!module.__esModule) return module; + return module.default ?? module; } export * from '@hydrooj/utils/lib/utils'; diff --git a/packages/ui-default/index.ts b/packages/ui-default/index.ts index 68663331a..94f5dbb74 100644 --- a/packages/ui-default/index.ts +++ b/packages/ui-default/index.ts @@ -1,7 +1,7 @@ /* eslint-disable global-require */ import { ContestModel, Context, Handler, ObjectId, param, PERM, PRIV, ProblemModel, Schema, - SettingModel, SystemModel, SystemSettings, Types, UserModel, + SettingModel, SystemModel, Types, UserModel, } from 'hydrooj'; import convert from 'schemastery-jsonschema'; import markdown from './backendlib/markdown'; @@ -74,7 +74,7 @@ class MarkdownHandler extends Handler { class SystemConfigSchemaHandler extends Handler { async get() { - const schema = convert(Schema.intersect(SystemSettings) as any, true); + const schema = convert(Schema.intersect(this.ctx.config.settings) as any, true); this.response.body = schema; } }