Weishen/Hydro
2
0 Fork 0
mirror of https://gitcode.com/gh_mirrors/hy/Hydro.git synced 2026-10-20 01:38:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

add support for config migration

Browse Source
This commit is contained in:
undefined committed 2025-03-16 01:13:08 +08:00
1 parent e0628c6197
commit f5eab3b87b
14 files changed
+206 -151
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/elastic/index.ts
+101 -95
View File
@@ -1,11 +1,9 @@
import { Client } from '@elastic/elasticsearch'; import { Client } from '@elastic/elasticsearch';
import { import {
_, Context, iterateAllProblem, iterateAllProblemInDomain, _, Context, iterateAllProblem, iterateAllProblemInDomain,
ProblemDoc, ProblemModel, Schema, SystemModel, ProblemDoc, ProblemModel, ProblemSearch, Schema, Service, SystemModel,
} from 'hydrooj'; } from 'hydrooj';
const client = new Client({ node: SystemModel.get('elastic-search.url') || 'http://127.0.0.1:9200' });
const indexOmit = ['_id', 'docType', 'data', 'additional_file', 'config', 'stats', 'assign']; const indexOmit = ['_id', 'docType', 'data', 'additional_file', 'config', 'stats', 'assign'];
const processDocument = (doc: Partial<ProblemDoc>) => { const processDocument = (doc: Partial<ProblemDoc>) => {
doc.content &&= doc.content.replace(/[[\]【】()（）]/g, ' '); doc.content &&= doc.content.replace(/[[\]【】()（）]/g, ' ');
@@ -19,102 +17,110 @@ const processDocument = (doc: Partial<ProblemDoc>) => {
return _.omit(doc, indexOmit); return _.omit(doc, indexOmit);
}; };
async function problemSearch(domainId, q, opts) { export default class ElasticSearchService extends Service {
const allowedSize = SystemModel.get('elasic-search.indexSize') || 10000; static Config = Schema.object({
const size = opts?.limit || SystemModel.get('pagination.problem'); url: Schema.string().role('url').default('http://127.0.0.1:9200'),
const from = Math.min(allowedSize - size, opts?.skip || 0); indexSize: Schema.number().role('indexSize').default(10000).min(0).step(1),
const res = await client.search({
index: 'problem',
size,
from,
query: {
simple_query_string: {
query: q.replace(/([a-z0-9]{2,})/gi, (i) => `${i}~5`),
fields: ['tag^5', 'pid^4', 'title^3', 'content'],
},
},
post_filter: {
bool: {
minimum_should_match: 1,
should: [{ match: { domainId } }],
},
},
}); });
let hits = res.hits.hits.map((i) => i._id);
if (!opts.skip) { client: Client;
let pdoc = await ProblemModel.get(domainId, +q || q, ProblemModel.PROJECTION_LIST);
if (pdoc) { constructor(ctx: Context, private config: ReturnType<typeof ElasticSearchService.Config>) {
hits = hits.filter((i) => i !== `${pdoc.domainId}/${pdoc.docId}`); console.log('elastic-search', config);
hits.unshift(`${pdoc.domainId}/${pdoc.docId}`); super(ctx, 'elastic-search');
} else if (/^P\d+$/.test(q)) { this.client = new Client({ node: config.url });
pdoc = await ProblemModel.get(domainId, +q.substring(1), ProblemModel.PROJECTION_LIST); }
if (pdoc) hits.unshift(`${pdoc.domainId}/${pdoc.docId}`);
async problemSearch(domainId: string, q: string, opts): ReturnType<ProblemSearch> {
const size = opts?.limit || SystemModel.get('pagination.problem');
const from = Math.min(this.config.indexSize - size, opts?.skip || 0);
const res = await this.client.search({
index: 'problem',
size,
from,
query: {
simple_query_string: {
query: q.replace(/([a-z0-9]{2,})/gi, (i) => `${i}~5`),
fields: ['tag^5', 'pid^4', 'title^3', 'content'],
},
},
post_filter: {
bool: {
minimum_should_match: 1,
should: [{ match: { domainId } }],
},
},
});
let hits = res.hits.hits.map((i) => i._id);
if (!opts.skip) {
let pdoc = await ProblemModel.get(domainId, +q || q, ProblemModel.PROJECTION_LIST);
if (pdoc) {
hits = hits.filter((i) => i !== `${pdoc.domainId}/${pdoc.docId}`);
hits.unshift(`${pdoc.domainId}/${pdoc.docId}`);
} else if (/^P\d+$/.test(q)) {
pdoc = await ProblemModel.get(domainId, +q.substring(1), ProblemModel.PROJECTION_LIST);
if (pdoc) hits.unshift(`${pdoc.domainId}/${pdoc.docId}`);
}
} }
return {
countRelation: typeof res.hits.total === 'number' ? 'eq' : res.hits.total.relation,
total: typeof res.hits.total === 'number' ? res.hits.total : res.hits.total.value,
hits: Array.from(new Set(hits)),
};
} }
return {
countRelation: typeof res.hits.total === 'number' ? 'eq' : res.hits.total.relation,
total: typeof res.hits.total === 'number' ? res.hits.total : res.hits.total.value,
hits: Array.from(new Set(hits)),
};
}
async function run({ domainId }, report) { async run({ domainId }, report) {
try { try {
if (domainId) await client.deleteByQuery({ index: 'problem', query: { match: { domainId } } }); if (domainId) await this.client.deleteByQuery({ index: 'problem', query: { match: { domainId } } });
else await client.deleteByQuery({ index: 'problem', query: { match_all: {} } }); else await this.client.deleteByQuery({ index: 'problem', query: { match_all: {} } });
} catch (e) { } catch (e) {
if (!e.message.includes('index_not_found_exception')) throw e; if (!e.message.includes('index_not_found_exception')) throw e;
}
let i = 0;
const cb = async (pdoc) => {
i++;
if (!(i % 1000)) report({ message: `${i} problems indexed` });
await this.client.index({
index: 'problem',
id: `${pdoc.domainId}/${pdoc.docId}`,
document: processDocument(pdoc),
});
};
if (domainId) await iterateAllProblemInDomain(domainId, ProblemModel.PROJECTION_PUBLIC, cb);
else await iterateAllProblem(ProblemModel.PROJECTION_PUBLIC, cb);
await this.client.indices.refresh({ index: 'problem' });
return true;
} }
let i = 0;
const cb = async (pdoc) => { async [Service.setup]() {
i++; await this.client.nodes.stats();
if (!(i % 1000)) report({ message: `${i} problems indexed` }); this.ctx.on('problem/add', async (doc, docId) => {
await client.index({ await this.client.index({
index: 'problem', index: 'problem',
id: `${pdoc.domainId}/${pdoc.docId}`, id: `${doc.domainId}/${docId}`,
document: processDocument(pdoc), document: processDocument(doc),
});
}); });
}; this.ctx.on('problem/edit', async (pdoc) => {
if (domainId) await iterateAllProblemInDomain(domainId, ProblemModel.PROJECTION_PUBLIC, cb); await this.client.index({
else await iterateAllProblem(ProblemModel.PROJECTION_PUBLIC, cb); index: 'problem',
await client.indices.refresh({ index: 'problem' }); id: `${pdoc.domainId}/${pdoc.docId}`,
return true; document: processDocument(pdoc),
});
});
this.ctx.on('problem/del', async (domainId, docId) => {
await this.client.delete({
index: 'problem',
id: `${domainId}/${docId}`,
});
});
this.ctx.addScript(
'ensureElasticSearch', 'Elastic problem search re-index',
Schema.object({
domainId: Schema.string(),
}),
this.run,
);
this.ctx.provideModule('problemSearch', 'elastic', this.problemSearch);
}
} }
export const apply = (ctx: Context) => {
ctx.setting.SystemSetting(Schema.object({
'elastic-search': Schema.object({
url: Schema.string().role('url').default('http://127.0.0.1:9200'),
indexSize: Schema.number().role('indexSize').default(10000).min(0).step(1),
}),
}));
ctx.on('problem/add', async (doc, docId) => {
await client.index({
index: 'problem',
id: `${doc.domainId}/${docId}`,
document: processDocument(doc),
});
});
ctx.on('problem/edit', async (pdoc) => {
await client.index({
index: 'problem',
id: `${pdoc.domainId}/${pdoc.docId}`,
document: processDocument(pdoc),
});
});
ctx.on('problem/del', async (domainId, docId) => {
await client.delete({
index: 'problem',
id: `${domainId}/${docId}`,
});
});
ctx.addScript(
'ensureElasticSearch', 'Elastic problem search re-index',
Schema.object({
domainId: Schema.string(),
}),
run,
);
ctx.provideModule('problemSearch', 'elastic', problemSearch);
};
packages/fps-importer/index.ts
+5 -6
View File
@@ -3,7 +3,7 @@ import decodeHTML from 'decode-html';
import xml2js from 'xml2js'; import xml2js from 'xml2js';
import { import {
_, AdmZip, BadRequestError, buildContent, Context, FileTooLargeError, fs, Handler, _, AdmZip, BadRequestError, buildContent, Context, FileTooLargeError, fs, Handler,
PERM, ProblemConfigFile, ProblemModel, ProblemType, SettingModel, SolutionModel, SystemModel, ValidationError, yaml, PERM, ProblemConfigFile, ProblemModel, ProblemType, Schema, SolutionModel, SystemModel, ValidationError, yaml,
} from 'hydrooj'; } from 'hydrooj';
const knownRemoteMapping = { const knownRemoteMapping = {
@@ -115,6 +115,10 @@ class FpsProblemImportHandler extends Handler {
} }
} }
export const Config = Schema.object({
limit: Schema.number().role('limit').default(64 * 1024 * 1024).min(1).description('Maximum file size for FPS problemset import'),
}).description('FPS Importer');
export async function apply(ctx: Context) { export async function apply(ctx: Context) {
ctx.Route('problem_import_fps', '/problem/import/fps', FpsProblemImportHandler, PERM.PERM_CREATE_PROBLEM); ctx.Route('problem_import_fps', '/problem/import/fps', FpsProblemImportHandler, PERM.PERM_CREATE_PROBLEM);
ctx.injectUI('ProblemAdd', 'problem_import_fps', { icon: 'copy', text: 'From FPS File' }); ctx.injectUI('ProblemAdd', 'problem_import_fps', { icon: 'copy', text: 'From FPS File' });
@@ -133,9 +137,4 @@ Importing a large problem set on a machine with insufficient memory may cause a
'problem.import.fps.hint3': 'If you really need it, this limit can be changed in the system settings. \ 'problem.import.fps.hint3': 'If you really need it, this limit can be changed in the system settings. \
We strongly recommend that you use the zip format to store or exchange problemsets.', We strongly recommend that you use the zip format to store or exchange problemsets.',
}); });
ctx.inject(['setting'], (c) => {
c.setting.SystemSetting(
SettingModel.Setting('setting_limits', 'import-fps.limit', 64 * 1024 * 1024, 'text', 'Maximum file size for FPS problemset import'),
);
});
} }
packages/hydrooj/src/entry/cli.ts
+2 -2
View File
@@ -119,7 +119,7 @@ export async function load(ctx: Context) {
resolve(c); resolve(c);
}); });
}); });
await ctx.loader.reloadPlugin(require.resolve('../service/storage'), 'file', 'hydrooj/service/storage'); await ctx.loader.reloadPlugin(require.resolve('../service/storage'), 'file');
require('../lib/index'); require('../lib/index');
await Promise.all([ await Promise.all([
lib(pending, fail, ctx), lib(pending, fail, ctx),
@@ -138,7 +138,7 @@ export async function load(ctx: Context) {
await addon(pending, fail, ctx); await addon(pending, fail, ctx);
const scriptDir = path.resolve(__dirname, '..', 'script'); const scriptDir = path.resolve(__dirname, '..', 'script');
for (const h of await fs.readdir(scriptDir)) { for (const h of await fs.readdir(scriptDir)) {
ctx.loader.reloadPlugin(path.resolve(scriptDir, h), '', `hydrooj/script/${h.split('.')[0]}`); ctx.loader.reloadPlugin(path.resolve(scriptDir, h), '');
} }
await script(pending, fail, ctx); await script(pending, fail, ctx);
await cli(); await cli();
packages/hydrooj/src/entry/common.ts
+13 -17
View File
@@ -22,35 +22,31 @@ function locateFile(basePath: string, filenames: string[]) {
return null; return null;
} }
type LoadTask = 'handler' | 'model' | 'addon' | 'lib' | 'script' | 'service'; type LoadTask = 'model' | 'addon' | 'service';
const getLoader = (type: LoadTask, filename: string, dontLoad = false) => async function loader(pending: string[], fail: string[], ctx: Context) { const getLoader = (type: LoadTask, filename: string) => async function loader(pending: Record<string, string>, fail: string[], ctx: Context) {
for (const i of pending) { for (const [name, i] of Object.entries(pending)) {
const p = locateFile(i, [`${filename}.ts`, `${filename}.js`]); const p = locateFile(i, [`${filename}.ts`, `${filename}.js`]);
if (p && !fail.includes(i)) { if (p && !fail.includes(i)) {
const name = type.replace(/^(.)/, (t) => t.toUpperCase()); const loadType = type.replace(/^(.)/, (t) => t.toUpperCase());
try { try {
if (dontLoad) throw new Error(`deprecated ${name} in %s will not be load: ${i}`);
const m = unwrapExports(require(p)); const m = unwrapExports(require(p));
if (m.apply) ctx.loader.reloadPlugin(p, ''); if (m.apply) ctx.loader.reloadPlugin(p, name);
else logger.info(`${name} init: %s`, i); else logger.info(`${loadType} init: %s`, i);
} catch (e) { } catch (e) {
fail.push(i); fail.push(i);
app.injectUI( app.injectUI(
'Notification', `${name} load fail: {0}`, 'Notification', `${loadType} load fail: {0}`,
{ args: [i], type: 'warn' }, PRIV.PRIV_VIEW_SYSTEM_NOTIFICATION, { args: [i], type: 'warn' }, PRIV.PRIV_VIEW_SYSTEM_NOTIFICATION,
); );
logger.info(`${name} load fail: %s`, i); logger.info(`${loadType} load fail: %s`, i);
logger.error(e); logger.error(e);
} }
} }
} }
}; };
export const handler = getLoader('handler', 'handler', true);
export const addon = getLoader('addon', 'index'); export const addon = getLoader('addon', 'index');
export const model = getLoader('model', 'model'); export const model = getLoader('model', 'model');
export const lib = getLoader('lib', 'lib', true);
export const script = getLoader('script', 'script', true);
export const service = getLoader('service', 'service'); export const service = getLoader('service', 'service');
export async function builtinModel(ctx: Context) { export async function builtinModel(ctx: Context) {
@@ -58,12 +54,12 @@ export async function builtinModel(ctx: Context) {
const models = await fs.readdir(modelDir); const models = await fs.readdir(modelDir);
for (const t of models.filter((i) => i.endsWith('.ts'))) { for (const t of models.filter((i) => i.endsWith('.ts'))) {
const q = path.resolve(modelDir, t); const q = path.resolve(modelDir, t);
if ('apply' in require(q)) ctx.loader.reloadPlugin(q, '', `hydrooj/model/${t.split('.')[0]}`); if ('apply' in require(q)) ctx.loader.reloadPlugin(q, '');
} }
} }
export async function locale(pending: string[], fail: string[]) { export async function locale(pending: Record<string, string>, fail: string[]) {
for (const i of pending) { for (const i of Object.values(pending)) {
const p = locateFile(i, ['locale', 'locales']); const p = locateFile(i, ['locale', 'locales']);
if (p && (await fs.stat(p)).isDirectory() && !fail.includes(i)) { if (p && (await fs.stat(p)).isDirectory() && !fail.includes(i)) {
try { try {
@@ -85,14 +81,14 @@ export async function locale(pending: string[], fail: string[]) {
} }
} }
export async function setting(pending: string[], fail: string[], modelSetting: typeof import('../model/setting')) { export async function setting(pending: Record<string, string>, fail: string[], modelSetting: typeof import('../model/setting')) {
const map = { const map = {
system: modelSetting.SystemSetting, system: modelSetting.SystemSetting,
account: modelSetting.AccountSetting, account: modelSetting.AccountSetting,
preference: modelSetting.PreferenceSetting, preference: modelSetting.PreferenceSetting,
domain: modelSetting.DomainSetting, domain: modelSetting.DomainSetting,
}; };
for (const i of pending) { for (const i of Object.values(pending)) {
let p = path.resolve(i, 'setting.yaml'); let p = path.resolve(i, 'setting.yaml');
const t = i.split(path.sep); const t = i.split(path.sep);
// TODO: change this name setting to package name // TODO: change this name setting to package name
packages/hydrooj/src/entry/worker.ts
+7 -12
View File
@@ -10,8 +10,7 @@ import { load } from '../options';
import { MongoService } from '../service/db'; import { MongoService } from '../service/db';
import { ConfigService } from '../settings'; import { ConfigService } from '../settings';
import { import {
addon, builtinModel, handler, lib, locale, model, addon, builtinModel, locale, model, service,
script, service, setting,
} from './common'; } from './common';
const argv = cac().parse(); const argv = cac().parse();
@@ -42,9 +41,9 @@ export async function apply(ctx: Context) {
}); });
await ctx.plugin(require('../service/hmr').default, { watch: argv.options.watch }); await ctx.plugin(require('../service/hmr').default, { watch: argv.options.watch });
await Promise.all([ await Promise.all([
ctx.loader.reloadPlugin(require.resolve('../service/storage'), 'file', 'hydrooj/service/storage'), ctx.loader.reloadPlugin(require.resolve('../service/storage'), 'file'),
ctx.loader.reloadPlugin(require.resolve('../service/worker'), 'worker', 'hydrooj/service/worker'), ctx.loader.reloadPlugin(require.resolve('../service/worker'), 'worker'),
ctx.loader.reloadPlugin(require.resolve('../service/server'), 'server', 'hydrooj/service/server'), ctx.loader.reloadPlugin(require.resolve('../service/server'), 'server'),
]); ]);
ctx = await new Promise((resolve) => { ctx = await new Promise((resolve) => {
ctx.inject(['server'], (c) => { ctx.inject(['server'], (c) => {
@@ -53,9 +52,7 @@ export async function apply(ctx: Context) {
}); });
await require('../service/api').apply(ctx); await require('../service/api').apply(ctx);
require('../lib/index'); require('../lib/index');
await lib(pending, fail, ctx);
await setting(pending, fail, require('../model/setting'));
ctx.plugin(require('../service/monitor')); ctx.plugin(require('../service/monitor'));
ctx.plugin(require('../service/check')); ctx.plugin(require('../service/check'));
await service(pending, fail, ctx); await service(pending, fail, ctx);
@@ -65,16 +62,14 @@ export async function apply(ctx: Context) {
const handlerDir = path.resolve(__dirname, '..', 'handler'); const handlerDir = path.resolve(__dirname, '..', 'handler');
const handlers = await fs.readdir(handlerDir); const handlers = await fs.readdir(handlerDir);
for (const h of handlers.filter((i) => i.endsWith('.ts'))) { for (const h of handlers.filter((i) => i.endsWith('.ts'))) {
ctx.loader.reloadPlugin(path.resolve(handlerDir, h), '', `hydrooj/handler/${h.split('.')[0]}`); ctx.loader.reloadPlugin(path.resolve(handlerDir, h), '');
} }
ctx.plugin(require('../service/migration').default); ctx.plugin(require('../service/migration').default);
await handler(pending, fail, ctx);
await addon(pending, fail, ctx); await addon(pending, fail, ctx);
const scriptDir = path.resolve(__dirname, '..', 'script'); const scriptDir = path.resolve(__dirname, '..', 'script');
for (const h of await fs.readdir(scriptDir)) { for (const h of await fs.readdir(scriptDir)) {
ctx.loader.reloadPlugin(path.resolve(scriptDir, h), '', `hydrooj/script/${h.split('.')[0]}`); ctx.loader.reloadPlugin(path.resolve(scriptDir, h), '');
} }
await script(pending, fail, ctx);
await ctx.parallel('app/started'); await ctx.parallel('app/started');
if (process.env.NODE_APP_INSTANCE === '0') { if (process.env.NODE_APP_INSTANCE === '0') {
await new Promise((resolve, reject) => { await new Promise((resolve, reject) => {
@@ -90,7 +85,7 @@ export async function apply(ctx: Context) {
}); });
}); });
} }
for (const f of global.addons) { for (const f of Object.values(global.addons)) {
const dir = path.join(f, 'public'); const dir = path.join(f, 'public');
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
if (await fs.pathExists(dir)) await fs.copy(dir, path.join(os.homedir(), '.hydro/static')); if (await fs.pathExists(dir)) await fs.copy(dir, path.join(os.homedir(), '.hydro/static'));
packages/hydrooj/src/init.ts
+1 -1
View File
@@ -23,7 +23,7 @@ if (!global.Hydro) {
error: {}, error: {},
locales: {}, locales: {},
}; };
global.addons = []; global.addons = {};
} }
global.app = new (require('./context').Context)(); global.app = new (require('./context').Context)();
process.on('exit', () => { process.on('exit', () => {
packages/hydrooj/src/interface.ts
+1 -1
View File
@@ -655,5 +655,5 @@ declare global {
var bus: Context; // eslint-disable-line var bus: Context; // eslint-disable-line
var app: Context; // eslint-disable-line var app: Context; // eslint-disable-line
var Hydro: HydroGlobal; // eslint-disable-line var Hydro: HydroGlobal; // eslint-disable-line
var addons: string[]; // eslint-disable-line var addons: Record<string, string>; // eslint-disable-line
} }
packages/hydrooj/src/loader.ts
+11 -9
View File
@@ -97,27 +97,29 @@ export class Loader extends Service {
} }
} }
resolveConfig(plugin: any, configScope: string) { async resolveConfig(plugin: any, configScope: string) {
const schema = plugin['Config'] || plugin['schema']; const schema = plugin['Config'] || plugin['schema'];
if (!schema) return {}; if (!schema) return {};
const schemaRequest = configScope ? Schema.object({ const schemaRequest = configScope ? Schema.object({
[configScope]: schema, [configScope]: schema,
}) : schema; }) : schema;
await this.ctx.config._tryMigrateConfig(schemaRequest);
const res = this.ctx.config.requestConfig(schemaRequest); const res = this.ctx.config.requestConfig(schemaRequest);
return configScope ? res[configScope] : res; return configScope ? res[configScope] : res;
} }
async reloadPlugin(key: string, configScope: string, asName = '') { async reloadPlugin(key: string, configScope: string) {
const plugin = await this.resolvePlugin(key); const plugin = this.resolvePlugin(key);
if (!plugin) return; if (!plugin) return;
const config = this.resolveConfig(plugin, configScope); const config = await this.resolveConfig(plugin, configScope);
if (asName) plugin.name = asName;
let fork = this.state[key]; let fork = this.state[key];
logger.info(
`%s plugin %c${configScope ? ' with scope %c' : ''}`,
fork ? 'reload' : 'apply', key.split('node_modules').pop(), configScope,
);
if (fork) { if (fork) {
logger.info('reload plugin %c', key.split('node_modules').pop());
fork.update(config); fork.update(config);
} else { } else {
logger.info('apply plugin %c', key.split('node_modules').pop());
fork = this.ctx.plugin(plugin, config); fork = this.ctx.plugin(plugin, config);
if (!fork) return; if (!fork) return;
this.state[key] = fork; this.state[key] = fork;
@@ -125,7 +127,7 @@ export class Loader extends Service {
return fork; return fork;
} }
async resolvePlugin(name: string) { resolvePlugin(name: string) {
try { try {
this.cache[name] ||= require.resolve(name); this.cache[name] ||= require.resolve(name);
} catch (err) { } catch (err) {
@@ -159,7 +161,7 @@ async function preload() {
const name = payload.name.startsWith('@hydrooj/') ? payload.name.split('@hydrooj/')[1] : payload.name; const name = payload.name.startsWith('@hydrooj/') ? payload.name.split('@hydrooj/')[1] : payload.name;
global.Hydro.version[name] = payload.version; global.Hydro.version[name] = payload.version;
const modulePath = path.dirname(packagejson); const modulePath = path.dirname(packagejson);
global.addons.push(modulePath); global.addons[name] = modulePath;
} catch (e) { } catch (e) {
logger.error(`Addon not found: ${a}`); logger.error(`Addon not found: ${a}`);
logger.error(e); logger.error(e);
packages/hydrooj/src/service/monitor.ts
+1 -1
View File
@@ -39,7 +39,7 @@ export async function feedback(): Promise<[string, StatusUpdate]> {
problemCount, problemCount,
discussionCount, discussionCount,
recordCount, recordCount,
addons: global.addons, addons: Object.values(global.addons),
memory: inf.memory, memory: inf.memory,
osinfo: inf.osinfo, osinfo: inf.osinfo,
cpu: inf.cpu, cpu: inf.cpu,
packages/hydrooj/src/service/server.ts
+1 -1
View File
@@ -106,7 +106,7 @@ export async function apply(ctx: Context) {
} }
}); });
for (const addon of [...global.addons].reverse()) { for (const addon of [...Object.values(global.addons)].reverse()) {
const dir = resolve(addon, 'public'); const dir = resolve(addon, 'public');
if (!fs.existsSync(dir)) continue; if (!fs.existsSync(dir)) continue;
server.addServerLayer(`${addon}_public`, cache(dir, { server.addServerLayer(`${addon}_public`, cache(dir, {
packages/hydrooj/src/settings.ts
+58 -3
View File
@@ -13,24 +13,38 @@ declare module './context' {
export class ConfigService extends Service { export class ConfigService extends Service {
static inject = ['db']; static inject = ['db'];
static name = 'config';
static blacklist = ['__proto__', 'prototype', 'constructor'];
settings: Schema[] = []; settings: Schema[] = [];
private systemConfig: any = {}; private systemConfig: any = {};
configSource: string = ''; configSource: string = '';
private applied: any = {};
private initialValues = {};
private _lastMigrate = Promise.resolve();
constructor(ctx: Context) { constructor(ctx: Context) {
super(ctx, 'config'); super(ctx, 'config');
} }
async [Service.setup]() { async [Service.setup]() {
const payload = await this.ctx.db.collection('system').find({}).toArray();
this.initialValues = Object.fromEntries(payload.map((v) => [v._id, v.value]));
return await this.loadConfig(); return await this.loadConfig();
} }
_applySchema() {
this.applied = this.settings.length ? Schema.intersect(this.settings)(this.systemConfig) : this.systemConfig;
}
async loadConfig() { async loadConfig() {
const config = await this.ctx.db.collection('system').findOne({ _id: 'config' }); const config = await this.ctx.db.collection('system').findOne({ _id: 'config' }, {
readPreference: 'primary', readConcern: 'majority',
});
try { try {
this.configSource = config?.value || '{}'; this.configSource = config?.value || '{}';
this.systemConfig = yaml.load(this.configSource); this.systemConfig = yaml.load(this.configSource);
this._applySchema();
this.ctx.emit('system/setting', { config: this.configSource }); this.ctx.emit('system/setting', { config: this.configSource });
logger.info('Successfully loaded config'); logger.info('Successfully loaded config');
} catch (e) { } catch (e) {
@@ -40,7 +54,7 @@ export class ConfigService extends Service {
applyDelta(source: any, key: string, value: any) { applyDelta(source: any, key: string, value: any) {
const path = key.split('.'); const path = key.split('.');
if (path.some((i) => ['__proto__', 'prototype', 'constructor'].includes(i))) return false; if (path.some((i) => ConfigService.blacklist.includes(i))) return false;
const t = path.pop(); const t = path.pop();
const root = JSON.parse(JSON.stringify(source)); const root = JSON.parse(JSON.stringify(source));
let cursor = root; let cursor = root;
@@ -69,6 +83,45 @@ export class ConfigService extends Service {
await this.loadConfig(); await this.loadConfig();
} }
async _actualMigrate(schema: Schema<any>) {
const processNode = async (path: string[], node: Schema<any, any>) => {
for (const item of node.list || []) await processNode(path, item); // eslint-disable-line no-await-in-loop
for (const key in node.dict || {}) await processNode(path.concat(key), node.dict[key]); // eslint-disable-line no-await-in-loop
if (['string', 'number', 'boolean'].includes(node.type)) {
const value = this.initialValues[path.join('.')];
const migrated = this.initialValues[`${path.join('.')}__migrated`];
if (migrated || !Object.hasOwn(this.initialValues, path.join('.'))) return;
let parsed;
try {
if (node.type === 'string') parsed = value;
if (node.type === 'number') parsed = +value;
if (node.type === 'boolean') parsed = !!value && !['off', '0', 'false'].includes(value);
} catch (e) { }
if (parsed === undefined) return;
await this.ctx.db.collection('system').updateOne({ _id: `${path.join('.')}__migrated` }, { $set: { value: true } }, { upsert: true });
this.ctx.logger.info('Migrating %s: %o', path.join('.'), parsed);
await this.saveConfig(this.applyDelta(this.systemConfig, path.join('.'), parsed));
}
};
await processNode([], schema);
}
async _tryMigrateConfig(schema: Schema<any>) {
this._lastMigrate = this._lastMigrate.then(() => this._actualMigrate(schema));
await this._lastMigrate;
}
get(key: string) {
const parts = key.split('.');
if (parts.some((p) => ConfigService.blacklist.includes(p.toString()))) throw new Error('Invalid path');
let currentValue = this.applied;
for (const p of parts) {
if (!currentValue) return undefined;
currentValue = currentValue[p];
}
return currentValue;
}
async setConfig(key: string, value: any) { async setConfig(key: string, value: any) {
const newConfig = this.applyDelta(this.systemConfig, key, value); const newConfig = this.applyDelta(this.systemConfig, key, value);
await this.saveConfig(newConfig); await this.saveConfig(newConfig);
@@ -78,8 +131,10 @@ export class ConfigService extends Service {
if (dynamic) { if (dynamic) {
this.ctx.effect(() => { this.ctx.effect(() => {
this.settings.push(s); this.settings.push(s);
this._applySchema();
return () => { return () => {
this.settings = this.settings.filter((v) => v !== s); this.settings = this.settings.filter((v) => v !== s);
this._applySchema();
}; };
}); });
} }
@@ -96,7 +151,7 @@ export class ConfigService extends Service {
} }
const that = this; const that = this;
const getAccess = (path: (string | symbol)[]) => { const getAccess = (path: (string | symbol)[]) => {
if (path.some((p) => ['__proto__', 'prototype', 'constructor'].includes(String(p)))) throw new Error('Invalid path'); if (path.some((p) => ConfigService.blacklist.includes(p.toString()))) throw new Error('Invalid path');
let currentValue = curValue; let currentValue = curValue;
for (const p of path) { for (const p of path) {
currentValue = currentValue[p]; currentValue = currentValue[p];
packages/ui-default/backendlib/builder.ts
+1 -1
View File
@@ -91,7 +91,7 @@ export async function buildUI() {
const entryPoints: string[] = []; const entryPoints: string[] = [];
const lazyModules: string[] = []; const lazyModules: string[] = [];
const newFiles = ['entry.js']; const newFiles = ['entry.js'];
for (const addon of global.addons) { for (const addon of Object.values(global.addons)) {
let publicPath = resolve(addon, 'frontend'); let publicPath = resolve(addon, 'frontend');
if (!fs.existsSync(publicPath)) publicPath = resolve(addon, 'public'); if (!fs.existsSync(publicPath)) publicPath = resolve(addon, 'public');
if (!fs.existsSync(publicPath)) continue; if (!fs.existsSync(publicPath)) continue;
packages/ui-default/backendlib/template.ts
+1 -1
View File
@@ -252,7 +252,7 @@ export class TemplateService extends Service {
} }
async [Service.setup]() { async [Service.setup]() {
const pending = global.addons; const pending = Object.values(global.addons);
const logger = this.ctx.logger('template'); const logger = this.ctx.logger('template');
for (const i of pending) { for (const i of pending) {
const p = locateFile(i, ['template', 'templates']); const p = locateFile(i, ['template', 'templates']);
packages/vjudge/src/index.ts
+3 -1
View File
@@ -189,6 +189,8 @@ class VJudgeService extends Service {
private providers: Record<string, any> = {}; private providers: Record<string, any> = {};
private pool: Record<string, AccountService> = {}; private pool: Record<string, AccountService> = {};
async [Service.setup]() { async [Service.setup]() {
if (process.env.NODE_APP_INSTANCE !== '0') return;
if (process.env.HYDRO_CLI) return;
this.accounts = await coll.find().toArray(); this.accounts = await coll.find().toArray();
this.ctx.setInterval(this.sync.bind(this), Time.week); this.ctx.setInterval(this.sync.bind(this), Time.week);
} }
@@ -258,9 +260,9 @@ export * from './interface';
export const name = 'vjudge'; export const name = 'vjudge';
export async function apply(ctx: Context) { export async function apply(ctx: Context) {
ctx.plugin(VJudgeService);
if (process.env.NODE_APP_INSTANCE !== '0') return; if (process.env.NODE_APP_INSTANCE !== '0') return;
if (process.env.HYDRO_CLI) return; if (process.env.HYDRO_CLI) return;
ctx.plugin(VJudgeService);
ctx.inject(['migration'], async (c) => { ctx.inject(['migration'], async (c) => {
c.migration.registerChannel('vjudge', [ c.migration.registerChannel('vjudge', [
async function init() { }, // eslint-disable-line async function init() { }, // eslint-disable-line