diff --git a/package.json b/package.json index 6d1077036..dcb946c74 100644 --- a/package.json +++ b/package.json @@ -19,26 +19,26 @@ "devDependencies": { "@shelf/jest-mongodb": "^1.2.3", "@types/cross-spawn": "^6.0.2", - "@types/jest": "^26.0.13", - "@types/node": "^14.6.4", - "@types/semver": "^7.3.3", + "@types/jest": "^26.0.14", + "@types/node": "^14.11.2", + "@types/semver": "^7.3.4", "@typescript-eslint/eslint-plugin": "^3.9.1", "@typescript-eslint/parser": "^3.9.1", "cross-spawn": "^7.0.3", - "eslint": "^7.8.1", + "eslint": "^7.9.0", "eslint-config-airbnb-base": "^14.2.0", "eslint-import-resolver-typescript": "^2.3.0", "eslint-plugin-import": "^2.22.0", "globby": "^11.0.1", "jest": "^26.4.2", "latest-version": "^5.1.0", - "ora": "^5.0.0", + "ora": "^5.1.0", "semver": "^7.3.2", - "ts-jest": "^26.3.0", - "typescript": "^4.0.2" + "ts-jest": "^26.4.0", + "typescript": "^4.0.3" }, "dependencies": { "fs-extra": "^9.0.1", - "yargs": "^15.4.1" + "yargs": "^16.0.3" } -} \ No newline at end of file +} diff --git a/packages/hydrojudge/package.json b/packages/hydrojudge/package.json index 603ce1993..462bec25e 100644 --- a/packages/hydrojudge/package.json +++ b/packages/hydrojudge/package.json @@ -7,14 +7,14 @@ "linux" ], "dependencies": { - "axios": "^0.19.0", + "axios": "^0.20.0", "bson": "^4.1.0", "fs-extra": "^9.0.1", "js-yaml": "^3.14.0", "lodash": "^4.17.20", "p-queue": "^6.6.1", "shell-quote": "^1.7.2", - "systeminformation": "^4.27.3" + "systeminformation": "^4.27.5" }, "peerDependencies": { "hydrooj": "^2.13.3" @@ -30,4 +30,4 @@ "build": "tsc", "lint": "eslint . --ext ts --fix" } -} \ No newline at end of file +} diff --git a/packages/hydrooj/TODO b/packages/hydrooj/TODO index 51a23ef05..28a6cf6fe 100644 --- a/packages/hydrooj/TODO +++ b/packages/hydrooj/TODO @@ -11,4 +11,5 @@ message.markAsRead 解离mail模块 存储系统抽象化 extend problem settings -+oplog \ No newline at end of file ++oplog +将系统设置存于本地而非数据库 \ No newline at end of file diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index fe83348f3..a6b696efc 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "2.13.17", + "version": "2.13.18", "bin": "bin/hydrooj.js", "main": "dist/loader.js", "typings": "dist/loader.d.ts", @@ -24,15 +24,15 @@ "lodash": "^4.17.20", "lru-cache": "^6.0.0", "moment-timezone": "^0.5.31", - "mongodb": "^3.6.1", + "mongodb": "^3.6.2", "nodemailer": "^6.4.10", "ps-list": "^7.2.0", - "serialize-javascript": "^4.0.0", + "serialize-javascript": "^5.0.1", "sockjs": "^0.3.20", "superagent": "^6.1.0", "supports-color": "^7.2.0", - "systeminformation": "^4.27.3", - "yargs": "^15.4.1" + "systeminformation": "^4.27.5", + "yargs": "^16.0.3" }, "devDependencies": { "@types/adm-zip": "^0.4.33", @@ -49,7 +49,7 @@ "@types/sockjs": "^0.3.32", "@types/superagent": "^4.1.8", "@types/supports-color": "^5.3.0", - "@types/yargs": "^15.0.5" + "@types/yargs": "^15.0.7" }, "scripts": { "build": "tsc", diff --git a/packages/hydrooj/src/check.ts b/packages/hydrooj/src/check.ts index 67ee9a8a8..86684c2b1 100644 --- a/packages/hydrooj/src/check.ts +++ b/packages/hydrooj/src/check.ts @@ -34,11 +34,11 @@ const checks: Dictionary = { } }, async checkMail(ctx, log, warn) { - const from = await system.get('smtp.from'); + const from = system.get('smtp.from'); if (!from) warn('SMTP account was not provided, email verification disabled.'); }, async checkIpHeader(ctx, log, warn) { - const header = await system.get('server.xff'); + const header = system.get('server.xff'); if (header && !ctx.request.ip) warn('IP header seems incorrect.\nCheck dashboard>settings>server.'); }, }; diff --git a/packages/hydrooj/src/entry/master.ts b/packages/hydrooj/src/entry/master.ts index 0c4252632..0241928f2 100644 --- a/packages/hydrooj/src/entry/master.ts +++ b/packages/hydrooj/src/entry/master.ts @@ -49,7 +49,10 @@ export async function load(call: Entry) { fs.removeSync(lockfile); }); await new Promise((resolve) => { - bus.once('database/connect', resolve); + bus.once('database/connect', () => { + bus.once('database/config', resolve); + require('../model/system'); + }); require('../service/db'); }); require('../service/monitor'); diff --git a/packages/hydrooj/src/entry/worker.ts b/packages/hydrooj/src/entry/worker.ts index 5e8c3ca04..7b28431d5 100644 --- a/packages/hydrooj/src/entry/worker.ts +++ b/packages/hydrooj/src/entry/worker.ts @@ -20,7 +20,10 @@ export async function load() { uistatic(pending, fail), ]); await new Promise((resolve) => { - bus.once('database/connect', resolve); + bus.once('database/connect', () => { + bus.once('database/config', resolve); + require('../model/system'); + }); require('../service/db'); }); for (const i of builtinLib) require(`../lib/${i}`); diff --git a/packages/hydrooj/src/handler/domain.ts b/packages/hydrooj/src/handler/domain.ts index a48c5bd87..52fe3cd45 100644 --- a/packages/hydrooj/src/handler/domain.ts +++ b/packages/hydrooj/src/handler/domain.ts @@ -201,7 +201,7 @@ class DomainJoinApplicationsHandler extends ManageHandler { if (!this.response.body.joinSettings) { delete this.response.body.expirations[domain.JOIN_EXPIRATION_KEEP_CURRENT]; } - this.response.body.url_prefix = await system.get('server.url'); + this.response.body.url_prefix = system.get('server.url'); this.response.body.path = [ ['Hydro', 'homepage'], ['domain', null], diff --git a/packages/hydrooj/src/handler/home.ts b/packages/hydrooj/src/handler/home.ts index 66db8f03a..90b0bec20 100644 --- a/packages/hydrooj/src/handler/home.ts +++ b/packages/hydrooj/src/handler/home.ts @@ -159,13 +159,13 @@ class HomeSecurityHandler extends Handler { if (udoc) throw new UserAlreadyExistError(email); const [code] = await token.add( token.TYPE_CHANGEMAIL, - await system.get('changemail_token_expire_seconds'), + system.get('changemail_token_expire_seconds'), { uid: this.user._id, email }, ); const m = await this.renderHTML('user_changemail_mail.html', { path: `home/changeMail/${code}`, uname: this.user.uname, - url_prefix: await system.get('server.url'), + url_prefix: system.get('server.url'), }); await mail.sendMail(email, 'Change Email', 'user_changemail_mail', m); this.response.template = 'user_changemail_mail_sent.html'; diff --git a/packages/hydrooj/src/handler/manage.ts b/packages/hydrooj/src/handler/manage.ts index a8fc04f80..08b807732 100644 --- a/packages/hydrooj/src/handler/manage.ts +++ b/packages/hydrooj/src/handler/manage.ts @@ -78,7 +78,7 @@ class SystemDashboardHandler extends SystemHandler { } async postRestart() { - const count = await system.get('server.worker'); + const count = system.get('server.worker'); process.send({ event: 'restart', count }); this.back(); } @@ -141,7 +141,7 @@ class SystemSettingHandler extends SystemHandler { for (const s of this.response.body.settings) { // FIXME no-await-in-loop // eslint-disable-next-line no-await-in-loop - this.response.body.current[s.key] = await system.get(s.key); + this.response.body.current[s.key] = system.get(s.key); } } diff --git a/packages/hydrooj/src/handler/misc.ts b/packages/hydrooj/src/handler/misc.ts index 71df6a2d7..296a69090 100644 --- a/packages/hydrooj/src/handler/misc.ts +++ b/packages/hydrooj/src/handler/misc.ts @@ -26,7 +26,7 @@ class FileDownloadHandler extends Handler { class FileUploadHandler extends Handler { async getQuota() { - let quota = await system.get('user.quota'); + let quota = system.get('user.quota'); if (this.user.hasPriv(PRIV.PRIV_UNLIMITED_QUOTA)) { quota = 2 ** 63 - 1; } diff --git a/packages/hydrooj/src/handler/user.ts b/packages/hydrooj/src/handler/user.ts index ae259925a..46198dba9 100644 --- a/packages/hydrooj/src/handler/user.ts +++ b/packages/hydrooj/src/handler/user.ts @@ -62,13 +62,13 @@ class UserRegisterHandler extends Handler { this.limitRate('send_mail', 3600, 30); const t = await token.add( token.TYPE_REGISTRATION, - await system.get('session.unsaved_expire_seconds'), + system.get('session.unsaved_expire_seconds'), { mail }, ); - if (await system.get('smtp.user')) { + if (system.get('smtp.user')) { const m = await this.renderHTML('user_register_mail.html', { path: `register/${t[0]}`, - url_prefix: await system.get('server.url'), + url_prefix: system.get('server.url'), }); await sendMail(mail, 'Sign Up', 'user_register_mail', m); this.response.template = 'user_register_mail_sent.html'; @@ -107,18 +107,18 @@ class UserRegisterWithCodeHandler extends Handler { class UserLostPassHandler extends Handler { async get() { - if (!await system.get('smtp.user')) throw new SystemError('Cannot send mail'); + if (!system.get('smtp.user')) throw new SystemError('Cannot send mail'); this.response.template = 'user_lostpass.html'; } @param('mail', Types.String, isEmail) async post(domainId: string, mail: string) { - if (!await system.get('smtp.user')) throw new SystemError('Cannot send mail'); + if (!system.get('smtp.user')) throw new SystemError('Cannot send mail'); const udoc = await user.getByEmail('system', mail); if (!udoc) throw new UserNotFoundError(mail); const [tid] = await token.add( token.TYPE_LOSTPASS, - await system.get('session.unsaved_expire_seconds'), + system.get('session.unsaved_expire_seconds'), { uid: udoc._id }, ); const m = await this.renderHTML('user_lostpass_mail', { url: `lostpass/${tid}`, uname: udoc.uname }); diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 5685372ac..002a36ef8 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -18,6 +18,7 @@ export interface SystemKeys { 'smtp.port': number, 'smtp.secure': boolean, 'user': number, + 'server.name': string, 'server.url': string, 'server.xff': string, 'server.worker': number, diff --git a/packages/hydrooj/src/model/system.ts b/packages/hydrooj/src/model/system.ts index c2329626b..e46a2ee19 100644 --- a/packages/hydrooj/src/model/system.ts +++ b/packages/hydrooj/src/model/system.ts @@ -1,72 +1,45 @@ -import Lru from 'lru-cache'; import { NumberKeys } from '../typeutils'; import { SystemKeys } from '../interface'; import * as db from '../service/db'; +import * as bus from '../service/bus'; const coll = db.collection('system'); +const cache: Record = {}; -const cache = new Lru({ - maxAge: 5000, -}); - -export async function get(_id: K): Promise { - const res = cache.get(_id); - if (res !== undefined) return res; - const doc = await coll.findOne({ _id }); - if (doc) { - cache.set(_id, doc.value); - return doc.value; - } - return null; +export function get(key: K): SystemKeys[K] { + return cache[key]; } -export async function getMany< +export function getMany< A extends keyof SystemKeys, B extends keyof SystemKeys, - >(keys: [A, B]): Promise<[SystemKeys[A], SystemKeys[B]]> -export async function getMany< + >(keys: [A, B]): [SystemKeys[A], SystemKeys[B]] +export function getMany< A extends keyof SystemKeys, B extends keyof SystemKeys, C extends keyof SystemKeys, - >(keys: [A, B, C]): Promise<[SystemKeys[A], SystemKeys[B], SystemKeys[C]]> -export async function getMany< + >(keys: [A, B, C]): [SystemKeys[A], SystemKeys[B], SystemKeys[C]] +export function getMany< A extends keyof SystemKeys, B extends keyof SystemKeys, C extends keyof SystemKeys, D extends keyof SystemKeys, - >(keys: [A, B, C, D]): Promise<[SystemKeys[A], SystemKeys[B], SystemKeys[C], SystemKeys[D]]> -export async function getMany< + >(keys: [A, B, C, D]): [SystemKeys[A], SystemKeys[B], SystemKeys[C], SystemKeys[D]] +export function getMany< A extends keyof SystemKeys, B extends keyof SystemKeys, C extends keyof SystemKeys, D extends keyof SystemKeys, E extends keyof SystemKeys, - >(keys: [A, B, C, D, E]): Promise<[SystemKeys[A], SystemKeys[B], SystemKeys[C], SystemKeys[D], SystemKeys[E]]> -export async function getMany< + >(keys: [A, B, C, D, E]): [SystemKeys[A], SystemKeys[B], SystemKeys[C], SystemKeys[D], SystemKeys[E]] +export function getMany< A extends keyof SystemKeys, B extends keyof SystemKeys, C extends keyof SystemKeys, D extends keyof SystemKeys, E extends keyof SystemKeys, F extends keyof SystemKeys, - >(keys: [A, B, C, D, E, F]): Promise<[SystemKeys[A], SystemKeys[B], SystemKeys[C], SystemKeys[D], SystemKeys[E], SystemKeys[F]]> -export async function getMany(keys: (keyof SystemKeys)[]): Promise -export async function getMany(keys: string[]): Promise { - const r = []; - let success = true; - for (const key of keys) { - const res = cache.get(key); - if (res !== undefined) r.push(res); - else success = false; - } - if (success) return r; - const docs = await coll.find({ _id: { $in: keys } }).toArray(); - const dict = {}; - const res = []; - for (const doc of docs) { - dict[doc._id] = doc.value; - } - for (const key of keys) { - res.push(dict[key] || null); - } - return res; + >(keys: [A, B, C, D, E, F]): [SystemKeys[A], SystemKeys[B], SystemKeys[C], SystemKeys[D], SystemKeys[E], SystemKeys[F]] +export function getMany(keys: (keyof SystemKeys)[]): any[] +export function getMany(keys: string[]): any[] { + return keys.map((key) => cache[key]); } export async function set(_id: K, value: SystemKeys[K]) { - cache.set(_id, value); const res = await coll.findOneAndUpdate( { _id }, { $set: { value } }, { upsert: true, returnOriginal: false }, ); + cache[_id] = res.value.value; return res.value.value; } @@ -78,10 +51,16 @@ export async function inc>(_id: K) { { $inc: { value: 1 } }, { upsert: true, returnOriginal: false }, ); - cache.set(_id, res.value.value); + cache[_id] = res.value.value; return res.value.value; } +(async () => { + const config = await coll.find({}).toArray(); + for (const i of config) cache[i._id] = i.value; + bus.emit('database/config'); +})(); + global.Hydro.model.system = { get, getMany, diff --git a/packages/hydrooj/src/service/bus.ts b/packages/hydrooj/src/service/bus.ts index c3f2748eb..eb9c05d38 100644 --- a/packages/hydrooj/src/service/bus.ts +++ b/packages/hydrooj/src/service/bus.ts @@ -31,6 +31,7 @@ export interface EventMap { 'app/exit': () => VoidReturn 'database/connect': (db: Db) => void + 'database/config': () => void 'monitor/update': (type: 'server' | 'judger', $set: any) => VoidReturn diff --git a/packages/hydrooj/src/service/server.ts b/packages/hydrooj/src/service/server.ts index a61d7e41c..544721ec0 100644 --- a/packages/hydrooj/src/service/server.ts +++ b/packages/hydrooj/src/service/server.ts @@ -169,7 +169,7 @@ function _descriptor(v: ParamOption) { : item.source === 'get' ? this.request.query : item.source === 'route' - ? this.request.params + ? { ...this.request.params, domainId: this.domainId } : this.request.body; const value = src[item.name]; if (!item.isOptional || value) { @@ -263,7 +263,7 @@ export function requireCsrfToken(target: any, funcName: string, obj: any) { } export async function prepare() { - app.keys = await system.get('session.keys') as unknown as string[]; + app.keys = system.get('session.keys') as unknown as string[]; if (argv.public) { app.use(cache(argv.public, { maxAge: 0, @@ -286,6 +286,8 @@ export class Handler { args: any; + domainId: string; + ctx: Koa.Context; request: { @@ -398,7 +400,7 @@ export class Handler { } renderTitle(str: string) { - return `${this.translate(str)} - Hydro`; + return `${this.translate(str)} - ${system.get('server.name')}`; } checkPerm(...args: bigint[]) { @@ -419,9 +421,9 @@ export class Handler { else args[key] = kwargs[key].toString(); } try { - if (this.args.domainId !== 'system' || args.domainId) { + if (this.domainId !== 'system' || args.domainId) { name += '_with_domainId'; - args.domainId = args.domainId || this.args.domainId; + args.domainId = args.domainId || this.domainId; } const { anchor, query } = args; if (query) res = router.url(name, args, { query }); @@ -464,7 +466,7 @@ export class Handler { } async init({ domainId }) { - const xff = await system.get('server.xff') as string; + const xff = system.get('server.xff') as string; if (xff) this.request.ip = this.request.headers[xff.toLowerCase()]; [this.domain] = await Promise.all([ domain.get(domainId), @@ -560,8 +562,8 @@ export class Handler { async saveCookie() { const expireSeconds = this.session.save - ? await system.get('session.expire_seconds') as number - : await system.get('session.unsaved_expire_seconds') as number; + ? system.get('session.expire_seconds') as number + : system.get('session.unsaved_expire_seconds') as number; if (this.session._id) { await token.update( this.session._id, @@ -587,7 +589,7 @@ export class Handler { ); } const cookie: SetOption = { - secure: !!await system.get('session.secure'), + secure: !!system.get('session.secure'), httpOnly: false, }; if (this.session.save) { @@ -694,18 +696,20 @@ export class ConnectionHandler { conn: sockjs.Connection; request: { - params: any - headers: any - ip: string + params: any; + headers: any; + ip: string; } - session: any + domainId: string; - args: any + session: any; - user: any + args: any; - domain: DomainDoc + user: User; + + domain: DomainDoc; constructor(conn: sockjs.Connection) { this.conn = conn; @@ -738,7 +742,7 @@ export class ConnectionHandler { } renderTitle(str: string) { - return `${this.translate(str)} - Hydro`; + return `${this.translate(str)} - ${system.get('server.name')}`; } checkPerm(...args: bigint[]) { @@ -832,6 +836,7 @@ export function Connection( try { const args = { domainId: 'system', ...h.request.params }; h.args = args; + h.domainId = args.domainId; const cookie = await new Promise((r) => { conn.once('data', r); }); @@ -860,8 +865,8 @@ export function Connection( } // TODO use postInit? -export async function start() { - const port = await system.get('server.port'); +export function start() { + const port = system.get('server.port'); if (argv.debug) { app.use(async (ctx: Context, next: Function) => { const startTime = new Date().getTime(); @@ -880,7 +885,7 @@ export async function start() { }); } app.use(async (ctx, next) => { - const xff = await system.get('server.xff'); + const xff = system.get('server.xff'); const ip = xff ? ctx.request.headers[xff] : ctx.request.ip; try { await opcount.inc('global', ip, 10, 100); diff --git a/packages/hydrooj/test/model.problem.spec.ts b/packages/hydrooj/test/model.problem.spec.ts index 1ffbea8f5..3c9e0d319 100644 --- a/packages/hydrooj/test/model.problem.spec.ts +++ b/packages/hydrooj/test/model.problem.spec.ts @@ -17,6 +17,7 @@ describe('Model.Problem', () => { test('add_get', async () => { const pid = await problem.add(DOMAIN_ID, PNAME, TITLE, CONTENT, UID); + expect(pid).toBeTruthy(); let pdoc = await problem.get(DOMAIN_ID, PNAME); expect(pdoc.domainId).toStrictEqual(DOMAIN_ID); expect(pdoc.title).toStrictEqual(TITLE); @@ -24,13 +25,8 @@ describe('Model.Problem', () => { expect(pdoc.owner).toStrictEqual(UID); expect(pdoc.docId).toStrictEqual(pid); expect(pdoc.pid).toStrictEqual(PNAME); - pdoc = await problem.get(DOMAIN_ID, pid); - expect(pdoc.domainId).toStrictEqual(DOMAIN_ID); - expect(pdoc.title).toStrictEqual(TITLE); - expect(pdoc.content).toStrictEqual(CONTENT); - expect(pdoc.owner).toStrictEqual(UID); - expect(pdoc.docId).toStrictEqual(pid); - expect(pdoc.pid).toStrictEqual(PNAME); + const pdoc1 = await problem.get(DOMAIN_ID, pid); + expect(pdoc1).toStrictEqual(pdoc); const pdocs = await problem.getMulti(DOMAIN_ID, {}).toArray(); expect(pdocs.length).toStrictEqual(1); expect(pdocs[0].docId).toStrictEqual(pid); diff --git a/packages/login-with-qq/service.ts b/packages/login-with-qq/service.ts index 741b7fd85..c9b16eff2 100644 --- a/packages/login-with-qq/service.ts +++ b/packages/login-with-qq/service.ts @@ -12,7 +12,7 @@ declare module 'hydrooj' { export async function postInit() { const { system, token } = global.Hydro.model; - const [url, accessToken] = await system.getMany(['login-with-qq.url', 'login-with-qq.token']); + const [url, accessToken] = system.getMany(['login-with-qq.url', 'login-with-qq.token']); if (!url) return; const headers: Record = {}; if (accessToken) headers.Authorization = `Bearer ${accessToken}`; diff --git a/packages/migrate-vijos/package.json b/packages/migrate-vijos/package.json index f6237bba7..7b9d41a11 100644 --- a/packages/migrate-vijos/package.json +++ b/packages/migrate-vijos/package.json @@ -18,6 +18,6 @@ }, "dependencies": { "js-yaml": "^3.14.0", - "mongodb": "^3.6.1" + "mongodb": "^3.6.2" } -} \ No newline at end of file +}