Weishen/Hydro
2
0 Fork 0
mirror of https://gitcode.com/gh_mirrors/hy/Hydro.git synced 2026-10-20 01:38:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

add support for config migration

Browse Source
This commit is contained in:
undefined committed 2025-03-16 01:13:08 +08:00
1 parent e0628c6197
commit f5eab3b87b
14 files changed
+206 -151
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/hydrooj/src/entry/cli.ts
+2 -2
View File
@@ -119,7 +119,7 @@ export async function load(ctx: Context) {
resolve(c);
});
});
await ctx.loader.reloadPlugin(require.resolve('../service/storage'), 'file', 'hydrooj/service/storage');
await ctx.loader.reloadPlugin(require.resolve('../service/storage'), 'file');
require('../lib/index');
await Promise.all([
lib(pending, fail, ctx),
@@ -138,7 +138,7 @@ export async function load(ctx: Context) {
await addon(pending, fail, ctx);
const scriptDir = path.resolve(__dirname, '..', 'script');
for (const h of await fs.readdir(scriptDir)) {
ctx.loader.reloadPlugin(path.resolve(scriptDir, h), '', `hydrooj/script/${h.split('.')[0]}`);
ctx.loader.reloadPlugin(path.resolve(scriptDir, h), '');
}
await script(pending, fail, ctx);
await cli();
packages/hydrooj/src/entry/common.ts
+13 -17
View File
@@ -22,35 +22,31 @@ function locateFile(basePath: string, filenames: string[]) {
return null;
}
type LoadTask = 'handler' | 'model' | 'addon' | 'lib' | 'script' | 'service';
const getLoader = (type: LoadTask, filename: string, dontLoad = false) => async function loader(pending: string[], fail: string[], ctx: Context) {
for (const i of pending) {
type LoadTask = 'model' | 'addon' | 'service';
const getLoader = (type: LoadTask, filename: string) => async function loader(pending: Record<string, string>, fail: string[], ctx: Context) {
for (const [name, i] of Object.entries(pending)) {
const p = locateFile(i, [`${filename}.ts`, `${filename}.js`]);
if (p && !fail.includes(i)) {
const name = type.replace(/^(.)/, (t) => t.toUpperCase());
const loadType = type.replace(/^(.)/, (t) => t.toUpperCase());
try {
if (dontLoad) throw new Error(`deprecated ${name} in %s will not be load: ${i}`);
const m = unwrapExports(require(p));
if (m.apply) ctx.loader.reloadPlugin(p, '');
else logger.info(`${name} init: %s`, i);
if (m.apply) ctx.loader.reloadPlugin(p, name);
else logger.info(`${loadType} init: %s`, i);
} catch (e) {
fail.push(i);
app.injectUI(
'Notification', `${name} load fail: {0}`,
'Notification', `${loadType} load fail: {0}`,
{ args: [i], type: 'warn' }, PRIV.PRIV_VIEW_SYSTEM_NOTIFICATION,
);
logger.info(`${name} load fail: %s`, i);
logger.info(`${loadType} load fail: %s`, i);
logger.error(e);
}
}
}
};
export const handler = getLoader('handler', 'handler', true);
export const addon = getLoader('addon', 'index');
export const model = getLoader('model', 'model');
export const lib = getLoader('lib', 'lib', true);
export const script = getLoader('script', 'script', true);
export const service = getLoader('service', 'service');
export async function builtinModel(ctx: Context) {
@@ -58,12 +54,12 @@ export async function builtinModel(ctx: Context) {
const models = await fs.readdir(modelDir);
for (const t of models.filter((i) => i.endsWith('.ts'))) {
const q = path.resolve(modelDir, t);
if ('apply' in require(q)) ctx.loader.reloadPlugin(q, '', `hydrooj/model/${t.split('.')[0]}`);
if ('apply' in require(q)) ctx.loader.reloadPlugin(q, '');
}
}
export async function locale(pending: string[], fail: string[]) {
for (const i of pending) {
export async function locale(pending: Record<string, string>, fail: string[]) {
for (const i of Object.values(pending)) {
const p = locateFile(i, ['locale', 'locales']);
if (p && (await fs.stat(p)).isDirectory() && !fail.includes(i)) {
try {
@@ -85,14 +81,14 @@ export async function locale(pending: string[], fail: string[]) {
}
}
export async function setting(pending: string[], fail: string[], modelSetting: typeof import('../model/setting')) {
export async function setting(pending: Record<string, string>, fail: string[], modelSetting: typeof import('../model/setting')) {
const map = {
system: modelSetting.SystemSetting,
account: modelSetting.AccountSetting,
preference: modelSetting.PreferenceSetting,
domain: modelSetting.DomainSetting,
};
for (const i of pending) {
for (const i of Object.values(pending)) {
let p = path.resolve(i, 'setting.yaml');
const t = i.split(path.sep);
// TODO: change this name setting to package name
packages/hydrooj/src/entry/worker.ts
+7 -12
View File
@@ -10,8 +10,7 @@ import { load } from '../options';
import { MongoService } from '../service/db';
import { ConfigService } from '../settings';
import {
addon, builtinModel, handler, lib, locale, model,
script, service, setting,
addon, builtinModel, locale, model, service,
} from './common';
const argv = cac().parse();
@@ -42,9 +41,9 @@ export async function apply(ctx: Context) {
});
await ctx.plugin(require('../service/hmr').default, { watch: argv.options.watch });
await Promise.all([
ctx.loader.reloadPlugin(require.resolve('../service/storage'), 'file', 'hydrooj/service/storage'),
ctx.loader.reloadPlugin(require.resolve('../service/worker'), 'worker', 'hydrooj/service/worker'),
ctx.loader.reloadPlugin(require.resolve('../service/server'), 'server', 'hydrooj/service/server'),
ctx.loader.reloadPlugin(require.resolve('../service/storage'), 'file'),
ctx.loader.reloadPlugin(require.resolve('../service/worker'), 'worker'),
ctx.loader.reloadPlugin(require.resolve('../service/server'), 'server'),
]);
ctx = await new Promise((resolve) => {
ctx.inject(['server'], (c) => {
@@ -53,9 +52,7 @@ export async function apply(ctx: Context) {
});
await require('../service/api').apply(ctx);
require('../lib/index');
await lib(pending, fail, ctx);
await setting(pending, fail, require('../model/setting'));
ctx.plugin(require('../service/monitor'));
ctx.plugin(require('../service/check'));
await service(pending, fail, ctx);
@@ -65,16 +62,14 @@ export async function apply(ctx: Context) {
const handlerDir = path.resolve(__dirname, '..', 'handler');
const handlers = await fs.readdir(handlerDir);
for (const h of handlers.filter((i) => i.endsWith('.ts'))) {
ctx.loader.reloadPlugin(path.resolve(handlerDir, h), '', `hydrooj/handler/${h.split('.')[0]}`);
ctx.loader.reloadPlugin(path.resolve(handlerDir, h), '');
}
ctx.plugin(require('../service/migration').default);
await handler(pending, fail, ctx);
await addon(pending, fail, ctx);
const scriptDir = path.resolve(__dirname, '..', 'script');
for (const h of await fs.readdir(scriptDir)) {
ctx.loader.reloadPlugin(path.resolve(scriptDir, h), '', `hydrooj/script/${h.split('.')[0]}`);
ctx.loader.reloadPlugin(path.resolve(scriptDir, h), '');
}
await script(pending, fail, ctx);
await ctx.parallel('app/started');
if (process.env.NODE_APP_INSTANCE === '0') {
await new Promise((resolve, reject) => {
@@ -90,7 +85,7 @@ export async function apply(ctx: Context) {
});
});
}
for (const f of global.addons) {
for (const f of Object.values(global.addons)) {
const dir = path.join(f, 'public');
// eslint-disable-next-line no-await-in-loop
if (await fs.pathExists(dir)) await fs.copy(dir, path.join(os.homedir(), '.hydro/static'));
packages/hydrooj/src/init.ts
+1 -1
View File
@@ -23,7 +23,7 @@ if (!global.Hydro) {
error: {},
locales: {},
};
global.addons = [];
global.addons = {};
}
global.app = new (require('./context').Context)();
process.on('exit', () => {
packages/hydrooj/src/interface.ts
+1 -1
View File
@@ -655,5 +655,5 @@ declare global {
var bus: Context; // eslint-disable-line
var app: Context; // eslint-disable-line
var Hydro: HydroGlobal; // eslint-disable-line
var addons: string[]; // eslint-disable-line
var addons: Record<string, string>; // eslint-disable-line
}
packages/hydrooj/src/loader.ts
+11 -9
View File
@@ -97,27 +97,29 @@ export class Loader extends Service {
}
}
resolveConfig(plugin: any, configScope: string) {
async resolveConfig(plugin: any, configScope: string) {
const schema = plugin['Config'] || plugin['schema'];
if (!schema) return {};
const schemaRequest = configScope ? Schema.object({
[configScope]: schema,
}) : schema;
await this.ctx.config._tryMigrateConfig(schemaRequest);
const res = this.ctx.config.requestConfig(schemaRequest);
return configScope ? res[configScope] : res;
}
async reloadPlugin(key: string, configScope: string, asName = '') {
const plugin = await this.resolvePlugin(key);
async reloadPlugin(key: string, configScope: string) {
const plugin = this.resolvePlugin(key);
if (!plugin) return;
const config = this.resolveConfig(plugin, configScope);
if (asName) plugin.name = asName;
const config = await this.resolveConfig(plugin, configScope);
let fork = this.state[key];
logger.info(
`%s plugin %c${configScope ? ' with scope %c' : ''}`,
fork ? 'reload' : 'apply', key.split('node_modules').pop(), configScope,
);
if (fork) {
logger.info('reload plugin %c', key.split('node_modules').pop());
fork.update(config);
} else {
logger.info('apply plugin %c', key.split('node_modules').pop());
fork = this.ctx.plugin(plugin, config);
if (!fork) return;
this.state[key] = fork;
@@ -125,7 +127,7 @@ export class Loader extends Service {
return fork;
}
async resolvePlugin(name: string) {
resolvePlugin(name: string) {
try {
this.cache[name] ||= require.resolve(name);
} catch (err) {
@@ -159,7 +161,7 @@ async function preload() {
const name = payload.name.startsWith('@hydrooj/') ? payload.name.split('@hydrooj/')[1] : payload.name;
global.Hydro.version[name] = payload.version;
const modulePath = path.dirname(packagejson);
global.addons.push(modulePath);
global.addons[name] = modulePath;
} catch (e) {
logger.error(`Addon not found: ${a}`);
logger.error(e);
packages/hydrooj/src/service/monitor.ts
+1 -1
View File
@@ -39,7 +39,7 @@ export async function feedback(): Promise<[string, StatusUpdate]> {
problemCount,
discussionCount,
recordCount,
addons: global.addons,
addons: Object.values(global.addons),
memory: inf.memory,
osinfo: inf.osinfo,
cpu: inf.cpu,
packages/hydrooj/src/service/server.ts
+1 -1
View File
@@ -106,7 +106,7 @@ export async function apply(ctx: Context) {
}
});
for (const addon of [...global.addons].reverse()) {
for (const addon of [...Object.values(global.addons)].reverse()) {
const dir = resolve(addon, 'public');
if (!fs.existsSync(dir)) continue;
server.addServerLayer(`${addon}_public`, cache(dir, {
packages/hydrooj/src/settings.ts
+58 -3
View File
@@ -13,24 +13,38 @@ declare module './context' {
export class ConfigService extends Service {
static inject = ['db'];
static name = 'config';
static blacklist = ['__proto__', 'prototype', 'constructor'];
settings: Schema[] = [];
private systemConfig: any = {};
configSource: string = '';
private applied: any = {};
private initialValues = {};
private _lastMigrate = Promise.resolve();
constructor(ctx: Context) {
super(ctx, 'config');
}
async [Service.setup]() {
const payload = await this.ctx.db.collection('system').find({}).toArray();
this.initialValues = Object.fromEntries(payload.map((v) => [v._id, v.value]));
return await this.loadConfig();
}
_applySchema() {
this.applied = this.settings.length ? Schema.intersect(this.settings)(this.systemConfig) : this.systemConfig;
}
async loadConfig() {
const config = await this.ctx.db.collection('system').findOne({ _id: 'config' });
const config = await this.ctx.db.collection('system').findOne({ _id: 'config' }, {
readPreference: 'primary', readConcern: 'majority',
});
try {
this.configSource = config?.value || '{}';
this.systemConfig = yaml.load(this.configSource);
this._applySchema();
this.ctx.emit('system/setting', { config: this.configSource });
logger.info('Successfully loaded config');
} catch (e) {
@@ -40,7 +54,7 @@ export class ConfigService extends Service {
applyDelta(source: any, key: string, value: any) {
const path = key.split('.');
if (path.some((i) => ['__proto__', 'prototype', 'constructor'].includes(i))) return false;
if (path.some((i) => ConfigService.blacklist.includes(i))) return false;
const t = path.pop();
const root = JSON.parse(JSON.stringify(source));
let cursor = root;
@@ -69,6 +83,45 @@ export class ConfigService extends Service {
await this.loadConfig();
}
async _actualMigrate(schema: Schema<any>) {
const processNode = async (path: string[], node: Schema<any, any>) => {
for (const item of node.list || []) await processNode(path, item); // eslint-disable-line no-await-in-loop
for (const key in node.dict || {}) await processNode(path.concat(key), node.dict[key]); // eslint-disable-line no-await-in-loop
if (['string', 'number', 'boolean'].includes(node.type)) {
const value = this.initialValues[path.join('.')];
const migrated = this.initialValues[`${path.join('.')}__migrated`];
if (migrated || !Object.hasOwn(this.initialValues, path.join('.'))) return;
let parsed;
try {
if (node.type === 'string') parsed = value;
if (node.type === 'number') parsed = +value;
if (node.type === 'boolean') parsed = !!value && !['off', '0', 'false'].includes(value);
} catch (e) { }
if (parsed === undefined) return;
await this.ctx.db.collection('system').updateOne({ _id: `${path.join('.')}__migrated` }, { $set: { value: true } }, { upsert: true });
this.ctx.logger.info('Migrating %s: %o', path.join('.'), parsed);
await this.saveConfig(this.applyDelta(this.systemConfig, path.join('.'), parsed));
}
};
await processNode([], schema);
}
async _tryMigrateConfig(schema: Schema<any>) {
this._lastMigrate = this._lastMigrate.then(() => this._actualMigrate(schema));
await this._lastMigrate;
}
get(key: string) {
const parts = key.split('.');
if (parts.some((p) => ConfigService.blacklist.includes(p.toString()))) throw new Error('Invalid path');
let currentValue = this.applied;
for (const p of parts) {
if (!currentValue) return undefined;
currentValue = currentValue[p];
}
return currentValue;
}
async setConfig(key: string, value: any) {
const newConfig = this.applyDelta(this.systemConfig, key, value);
await this.saveConfig(newConfig);
@@ -78,8 +131,10 @@ export class ConfigService extends Service {
if (dynamic) {
this.ctx.effect(() => {
this.settings.push(s);
this._applySchema();
return () => {
this.settings = this.settings.filter((v) => v !== s);
this._applySchema();
};
});
}
@@ -96,7 +151,7 @@ export class ConfigService extends Service {
}
const that = this;
const getAccess = (path: (string | symbol)[]) => {
if (path.some((p) => ['__proto__', 'prototype', 'constructor'].includes(String(p)))) throw new Error('Invalid path');
if (path.some((p) => ConfigService.blacklist.includes(p.toString()))) throw new Error('Invalid path');
let currentValue = curValue;
for (const p of path) {
currentValue = currentValue[p];