From 70872fe9314d508301e4da4b90381c793b7a9010 Mon Sep 17 00:00:00 2001 From: undefined Date: Wed, 3 Jul 2024 18:09:48 +0800 Subject: [PATCH] core: add migration service --- packages/hydrooj/package.json | 14 +++--- packages/hydrooj/src/entry/worker.ts | 33 ++++++-------- packages/hydrooj/src/interface.ts | 2 - packages/hydrooj/src/model/user.ts | 2 +- packages/hydrooj/src/service/migration.ts | 52 +++++++++++++++++++++++ packages/hydrooj/src/upgrade.ts | 46 +++++++++++--------- 6 files changed, 99 insertions(+), 50 deletions(-) create mode 100644 packages/hydrooj/src/service/migration.ts diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index f750db6bb..80ea2e7aa 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -12,10 +12,10 @@ }, "preferUnplugged": true, "dependencies": { - "@aws-sdk/client-s3": "3.600.0", - "@aws-sdk/lib-storage": "3.600.0", - "@aws-sdk/s3-presigned-post": "3.600.0", - "@aws-sdk/s3-request-presigner": "3.600.0", + "@aws-sdk/client-s3": "3.609.0", + "@aws-sdk/lib-storage": "3.609.0", + "@aws-sdk/s3-presigned-post": "3.609.0", + "@aws-sdk/s3-request-presigner": "3.609.0", "@graphql-tools/schema": "^10.0.4", "@hydrooj/framework": "workspace:^", "@hydrooj/utils": "workspace:^", @@ -34,7 +34,7 @@ "koa-proxies": "^0.12.4", "koa-static-cache": "^5.1.4", "lodash": "^4.17.21", - "lru-cache": "^10.2.2", + "lru-cache": "^10.3.0", "mime-types": "^2.1.35", "moment-timezone": "^0.5.45", "mongodb": "^5.9.2", @@ -43,7 +43,7 @@ "nodemailer": "^6.9.14", "notp": "^2.0.3", "p-queue": "^8.0.1", - "path-to-regexp": "^6.2.2", + "path-to-regexp": "^7.0.0", "sanitize-filename": "^1.6.3", "saslprep": "^1.0.3", "schemastery": "^3.14.6", @@ -59,7 +59,7 @@ "@types/fs-extra": "^11.0.4", "@types/js-yaml": "^4.0.9", "@types/koa-static-cache": "^5.1.4", - "@types/lodash": "^4.17.5", + "@types/lodash": "^4.17.6", "@types/mime-types": "^2.1.4", "@types/mongodb-uri": "^0.9.4", "@types/nodemailer": "^6.4.15", diff --git a/packages/hydrooj/src/entry/worker.ts b/packages/hydrooj/src/entry/worker.ts index 88ce696ca..51be0b8ab 100644 --- a/packages/hydrooj/src/entry/worker.ts +++ b/packages/hydrooj/src/entry/worker.ts @@ -65,10 +65,10 @@ export async function apply(ctx: Context) { for (const h of handlers.filter((i) => i.endsWith('.ts'))) { ctx.loader.reloadPlugin(ctx, path.resolve(handlerDir, h), {}, `hydrooj/handler/${h.split('.')[0]}`); } + ctx.plugin(require('../service/migration').default); await handler(pending, fail, ctx); await addon(pending, fail, ctx); await ctx.lifecycle.flush(); - for (const i in global.Hydro.handler) await global.Hydro.handler[i](); const scriptDir = path.resolve(__dirname, '..', 'script'); for (const h of await fs.readdir(scriptDir)) { ctx.loader.reloadPlugin(ctx, path.resolve(scriptDir, h), {}, `hydrooj/script/${h.split('.')[0]}`); @@ -78,27 +78,18 @@ export async function apply(ctx: Context) { await ctx.lifecycle.flush(); await ctx.parallel('app/started'); if (process.env.NODE_APP_INSTANCE === '0') { - const { default: scripts, init } = require('../upgrade'); - let dbVer = (await modelSystem.get('db.ver')) ?? 0; - const isFresh = !dbVer; - const expected = scripts.length; - if (isFresh) { - await init(); - await modelSystem.set('db.ver', expected); - } else { - while (dbVer < expected) { - const func = scripts[dbVer]; - if (typeof func !== 'function') { - dbVer++; - continue; + await new Promise((resolve, reject) => { + ctx.inject(['migration'], async (c) => { + c.migration.registerChannel('hydrooj', require('../upgrade').coreScripts); + try { + await c.migration.doUpgrade(); + resolve(null); + } catch (e) { + logger.error('Upgrade failed: %O', e); + reject(e); } - logger.info('Upgrading database: from %d to %d', dbVer, expected); - const result = await func(); - if (!result) break; - dbVer++; - await modelSystem.set('db.ver', dbVer); - } - } + }); + }); } for (const f of global.addons) { const dir = path.join(f, 'public'); diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 34a959fbe..7f6e20211 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -808,8 +808,6 @@ export interface ModuleInterfaces { export interface HydroGlobal { version: Record; model: Model; - /** @deprecated */ - handler: Record; script: Record; service: HydroService; lib: Lib; diff --git a/packages/hydrooj/src/model/user.ts b/packages/hydrooj/src/model/user.ts index eb2bda954..8a4389bd1 100644 --- a/packages/hydrooj/src/model/user.ts +++ b/packages/hydrooj/src/model/user.ts @@ -179,7 +179,7 @@ declare module '@hydrooj/framework' { interface UserModel extends User { } } -function handleMailLower(mail: string) { +export function handleMailLower(mail: string) { const [n, d] = mail.trim().toLowerCase().split('@'); const [name] = n.split('+'); return `${name.replace(/\./g, '')}@${d === 'googlemail.com' ? 'gmail.com' : d}`; diff --git a/packages/hydrooj/src/service/migration.ts b/packages/hydrooj/src/service/migration.ts new file mode 100644 index 000000000..176968ab5 --- /dev/null +++ b/packages/hydrooj/src/service/migration.ts @@ -0,0 +1,52 @@ +/* eslint-disable no-await-in-loop */ +import { Logger } from '@hydrooj/utils'; +import { Context, Service } from '../context'; +import * as system from '../model/system'; + +export type MigrationScript = null | (() => Promise); +const logger = new Logger('migration'); + +declare module '../context' { + interface Context { + migration: MigrationService; + } +} + +export default class MigrationService extends Service { + private channels: Record = {}; + + constructor(ctx: Context) { + super(ctx, 'migration', true); + } + + async registerChannel(name: string, s: MigrationScript[]) { + this.channels[name] = s; + } + + async doUpgrade() { + for (const channel in this.channels) { + const name = `db.ver${channel === 'hydrooj' ? '' : `-${channel}`}`; + let dbVer = system.get(name) ?? 0; + const scripts = this.channels[channel]; + const isFresh = !dbVer; + const expected = scripts.length; + if (isFresh) { + await scripts[0]?.(); + await system.set(name, expected); + continue; + } + while (dbVer < expected) { + const func = scripts[dbVer]; + if (typeof func !== 'function') { + dbVer++; + continue; + } + logger.info('Upgrading database [%s]: from %d to %d', channel, dbVer, expected); + const result = await func(); + if (!result) break; + dbVer++; + await system.set(name, dbVer); + } + } + } +} diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index 7ed111307..c88551eb9 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -21,37 +21,35 @@ import StorageModel from './model/storage'; import * as system from './model/system'; import TaskModel from './model/task'; import * as training from './model/training'; -import user from './model/user'; +import user, { handleMailLower } from './model/user'; import { iterateAllContest, iterateAllDomain, iterateAllProblem, iterateAllUser, } from './pipelineUtils'; import db from './service/db'; +import { MigrationScript } from './service/migration'; import { setBuiltinConfig } from './settings'; import welcome from './welcome'; const logger = new Logger('upgrade'); -type UpgradeScript = void | (() => Promise); const unsupportedUpgrade = async function _26_27() { throw new Error('This upgrade was no longer supported in hydrooj@4. \ Please use hydrooj@3 to perform these upgrades before upgrading to v4'); }; -export async function init() { - if (!await user.getById('system', 0)) { - await user.create('Guest@hydro.local', 'Guest', String.random(32), 0, '127.0.0.1', PRIV.PRIV_REGISTER_USER); - } - if (!await user.getById('system', 1)) { - await user.create('Hydro@hydro.local', 'Hydro', String.random(32), 1, '127.0.0.1', PRIV.PRIV_USER_PROFILE); - } - const ddoc = await domain.get('system'); - if (!ddoc) await domain.add('system', 1, 'Hydro', 'Welcome to Hydro!'); - await welcome(); - return true; -} - -const scripts: UpgradeScript[] = [ +export const coreScripts: MigrationScript[] = [ // Mark as used - null, + async function init() { + if (!await user.getById('system', 0)) { + await user.create('Guest@hydro.local', 'Guest', String.random(32), 0, '127.0.0.1', PRIV.PRIV_REGISTER_USER); + } + if (!await user.getById('system', 1)) { + await user.create('Hydro@hydro.local', 'Hydro', String.random(32), 1, '127.0.0.1', PRIV.PRIV_USER_PROFILE); + } + const ddoc = await domain.get('system'); + if (!ddoc) await domain.add('system', 1, 'Hydro', 'Welcome to Hydro!'); + await welcome(); + return true; + }, // Init ...new Array(26).fill(unsupportedUpgrade), null, @@ -675,6 +673,16 @@ const scripts: UpgradeScript[] = [ if (bulk.batches.length) await bulk.execute(); return true; }, + async function _89_90() { + return await iterateAllUser(async (udoc) => { + const wanted = handleMailLower(udoc.mail); + if (wanted !== udoc.mailLower) { + if (await user.getByEmail('system', wanted)) { + console.warn('Email conflict: %s', wanted); + return; + } + await user.coll.updateOne({ _id: udoc._id }, { $set: { mailLower: wanted } }); + } + }); + }, ]; - -export default scripts;