core: add migration service

packages/hydrooj/package.json

@@ -12,10 +12,10 @@
     },
     "preferUnplugged": true,
     "dependencies": {
-        "@aws-sdk/client-s3": "3.600.0",
+        "@aws-sdk/client-s3": "3.609.0",
-        "@aws-sdk/lib-storage": "3.600.0",
+        "@aws-sdk/lib-storage": "3.609.0",
-        "@aws-sdk/s3-presigned-post": "3.600.0",
+        "@aws-sdk/s3-presigned-post": "3.609.0",
-        "@aws-sdk/s3-request-presigner": "3.600.0",
+        "@aws-sdk/s3-request-presigner": "3.609.0",
         "@graphql-tools/schema": "^10.0.4",
         "@hydrooj/framework": "workspace:^",
         "@hydrooj/utils": "workspace:^",
@@ -34,7 +34,7 @@
         "koa-proxies": "^0.12.4",
         "koa-static-cache": "^5.1.4",
         "lodash": "^4.17.21",
-        "lru-cache": "^10.2.2",
+        "lru-cache": "^10.3.0",
         "mime-types": "^2.1.35",
         "moment-timezone": "^0.5.45",
         "mongodb": "^5.9.2",
@@ -43,7 +43,7 @@
         "nodemailer": "^6.9.14",
         "notp": "^2.0.3",
         "p-queue": "^8.0.1",
-        "path-to-regexp": "^6.2.2",
+        "path-to-regexp": "^7.0.0",
         "sanitize-filename": "^1.6.3",
         "saslprep": "^1.0.3",
         "schemastery": "^3.14.6",
@@ -59,7 +59,7 @@
         "@types/fs-extra": "^11.0.4",
         "@types/js-yaml": "^4.0.9",
         "@types/koa-static-cache": "^5.1.4",
-        "@types/lodash": "^4.17.5",
+        "@types/lodash": "^4.17.6",
         "@types/mime-types": "^2.1.4",
         "@types/mongodb-uri": "^0.9.4",
         "@types/nodemailer": "^6.4.15",

packages/hydrooj/src/entry/worker.ts

@@ -65,10 +65,10 @@ export async function apply(ctx: Context) {
     for (const h of handlers.filter((i) => i.endsWith('.ts'))) {
         ctx.loader.reloadPlugin(ctx, path.resolve(handlerDir, h), {}, `hydrooj/handler/${h.split('.')[0]}`);
     }
+    ctx.plugin(require('../service/migration').default);
     await handler(pending, fail, ctx);
     await addon(pending, fail, ctx);
     await ctx.lifecycle.flush();
-    for (const i in global.Hydro.handler) await global.Hydro.handler[i]();
     const scriptDir = path.resolve(__dirname, '..', 'script');
     for (const h of await fs.readdir(scriptDir)) {
         ctx.loader.reloadPlugin(ctx, path.resolve(scriptDir, h), {}, `hydrooj/script/${h.split('.')[0]}`);
@@ -78,27 +78,18 @@ export async function apply(ctx: Context) {
     await ctx.lifecycle.flush();
     await ctx.parallel('app/started');
     if (process.env.NODE_APP_INSTANCE === '0') {
-        const { default: scripts, init } = require('../upgrade');
+        await new Promise((resolve, reject) => {
-        let dbVer = (await modelSystem.get('db.ver')) ?? 0;
+            ctx.inject(['migration'], async (c) => {
-        const isFresh = !dbVer;
+                c.migration.registerChannel('hydrooj', require('../upgrade').coreScripts);
-        const expected = scripts.length;
+                try {
-        if (isFresh) {
+                    await c.migration.doUpgrade();
-            await init();
+                    resolve(null);
-            await modelSystem.set('db.ver', expected);
+                } catch (e) {
-        } else {
+                    logger.error('Upgrade failed: %O', e);
-            while (dbVer < expected) {
+                    reject(e);
-                const func = scripts[dbVer];
-                if (typeof func !== 'function') {
-                    dbVer++;
-                    continue;
-                }
-                logger.info('Upgrading database: from %d to %d', dbVer, expected);
-                const result = await func();
-                if (!result) break;
-                dbVer++;
-                await modelSystem.set('db.ver', dbVer);
-            }
                 }
+            });
+        });
     }
     for (const f of global.addons) {
         const dir = path.join(f, 'public');

packages/hydrooj/src/interface.ts

@@ -808,8 +808,6 @@ export interface ModuleInterfaces {
 export interface HydroGlobal {
     version: Record<string, string>;
     model: Model;
-    /** @deprecated */
-    handler: Record<string, Function>;
     script: Record<string, Script>;
     service: HydroService;
     lib: Lib;

packages/hydrooj/src/model/user.ts

@@ -179,7 +179,7 @@ declare module '@hydrooj/framework' {
     interface UserModel extends User { }
 }
 
-function handleMailLower(mail: string) {
+export function handleMailLower(mail: string) {
     const [n, d] = mail.trim().toLowerCase().split('@');
     const [name] = n.split('+');
     return `${name.replace(/\./g, '')}@${d === 'googlemail.com' ? 'gmail.com' : d}`;

packages/hydrooj/src/service/migration.ts

@@ -0,0 +1,52 @@
+/* eslint-disable no-await-in-loop */
+import { Logger } from '@hydrooj/utils';
+import { Context, Service } from '../context';
+import * as system from '../model/system';
+
+export type MigrationScript = null | (() => Promise<boolean | void>);
+const logger = new Logger('migration');
+
+declare module '../context' {
+    interface Context {
+        migration: MigrationService;
+    }
+}
+
+export default class MigrationService extends Service {
+    private channels: Record<string, MigrationScript[]> = {};
+
+    constructor(ctx: Context) {
+        super(ctx, 'migration', true);
+    }
+
+    async registerChannel(name: string, s: MigrationScript[]) {
+        this.channels[name] = s;
+    }
+
+    async doUpgrade() {
+        for (const channel in this.channels) {
+            const name = `db.ver${channel === 'hydrooj' ? '' : `-${channel}`}`;
+            let dbVer = system.get(name) ?? 0;
+            const scripts = this.channels[channel];
+            const isFresh = !dbVer;
+            const expected = scripts.length;
+            if (isFresh) {
+                await scripts[0]?.();
+                await system.set(name, expected);
+                continue;
+            }
+            while (dbVer < expected) {
+                const func = scripts[dbVer];
+                if (typeof func !== 'function') {
+                    dbVer++;
+                    continue;
+                }
+                logger.info('Upgrading database [%s]: from %d to %d', channel, dbVer, expected);
+                const result = await func();
+                if (!result) break;
+                dbVer++;
+                await system.set(name, dbVer);
+            }
+        }
+    }
+}

packages/hydrooj/src/upgrade.ts

@@ -21,22 +21,24 @@ import StorageModel from './model/storage';
 import * as system from './model/system';
 import TaskModel from './model/task';
 import * as training from './model/training';
-import user from './model/user';
+import user, { handleMailLower } from './model/user';
 import {
     iterateAllContest, iterateAllDomain, iterateAllProblem, iterateAllUser,
 } from './pipelineUtils';
 import db from './service/db';
+import { MigrationScript } from './service/migration';
 import { setBuiltinConfig } from './settings';
 import welcome from './welcome';
 
 const logger = new Logger('upgrade');
-type UpgradeScript = void | (() => Promise<boolean | void>);
 const unsupportedUpgrade = async function _26_27() {
     throw new Error('This upgrade was no longer supported in hydrooj@4. \
 Please use hydrooj@3 to perform these upgrades before upgrading to v4');
 };
 
-export async function init() {
+export const coreScripts: MigrationScript[] = [
+    // Mark as used
+    async function init() {
         if (!await user.getById('system', 0)) {
             await user.create('Guest@hydro.local', 'Guest', String.random(32), 0, '127.0.0.1', PRIV.PRIV_REGISTER_USER);
         }
@@ -47,11 +49,7 @@ export async function init() {
         if (!ddoc) await domain.add('system', 1, 'Hydro', 'Welcome to Hydro!');
         await welcome();
         return true;
-}
+    },
-
-const scripts: UpgradeScript[] = [
-    // Mark as used
-    null,
     // Init
     ...new Array(26).fill(unsupportedUpgrade),
     null,
@@ -675,6 +673,16 @@ const scripts: UpgradeScript[] = [
         if (bulk.batches.length) await bulk.execute();
         return true;
     },
+    async function _89_90() {
+        return await iterateAllUser(async (udoc) => {
+            const wanted = handleMailLower(udoc.mail);
+            if (wanted !== udoc.mailLower) {
+                if (await user.getByEmail('system', wanted)) {
+                    console.warn('Email conflict: %s', wanted);
+                    return;
+                }
+                await user.coll.updateOne({ _id: udoc._id }, { $set: { mailLower: wanted } });
+            }
+        });
+    },
 ];
-
-export default scripts;