Weishen/Hydro
2
0 Fork 0
mirror of https://gitcode.com/gh_mirrors/hy/Hydro.git synced 2026-10-19 01:38:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

core: add migration service

Browse Source
This commit is contained in:
undefined committed 2024-07-03 18:09:48 +08:00
1 parent 9e4643c1bf
commit 70872fe931
6 files changed
+99 -50
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/hydrooj/package.json
+7 -7
View File
@@ -12,10 +12,10 @@
},
"preferUnplugged": true,
"dependencies": {
"@aws-sdk/client-s3": "3.600.0",
"@aws-sdk/lib-storage": "3.600.0",
"@aws-sdk/s3-presigned-post": "3.600.0",
"@aws-sdk/s3-request-presigner": "3.600.0",
"@aws-sdk/client-s3": "3.609.0",
"@aws-sdk/lib-storage": "3.609.0",
"@aws-sdk/s3-presigned-post": "3.609.0",
"@aws-sdk/s3-request-presigner": "3.609.0",
"@graphql-tools/schema": "^10.0.4",
"@hydrooj/framework": "workspace:^",
"@hydrooj/utils": "workspace:^",
@@ -34,7 +34,7 @@
"koa-proxies": "^0.12.4",
"koa-static-cache": "^5.1.4",
"lodash": "^4.17.21",
"lru-cache": "^10.2.2",
"lru-cache": "^10.3.0",
"mime-types": "^2.1.35",
"moment-timezone": "^0.5.45",
"mongodb": "^5.9.2",
@@ -43,7 +43,7 @@
"nodemailer": "^6.9.14",
"notp": "^2.0.3",
"p-queue": "^8.0.1",
"path-to-regexp": "^6.2.2",
"path-to-regexp": "^7.0.0",
"sanitize-filename": "^1.6.3",
"saslprep": "^1.0.3",
"schemastery": "^3.14.6",
@@ -59,7 +59,7 @@
"@types/fs-extra": "^11.0.4",
"@types/js-yaml": "^4.0.9",
"@types/koa-static-cache": "^5.1.4",
"@types/lodash": "^4.17.5",
"@types/lodash": "^4.17.6",
"@types/mime-types": "^2.1.4",
"@types/mongodb-uri": "^0.9.4",
"@types/nodemailer": "^6.4.15",
packages/hydrooj/src/entry/worker.ts
+12 -21
View File
@@ -65,10 +65,10 @@ export async function apply(ctx: Context) {
for (const h of handlers.filter((i) => i.endsWith('.ts'))) {
ctx.loader.reloadPlugin(ctx, path.resolve(handlerDir, h), {}, `hydrooj/handler/${h.split('.')[0]}`);
}
ctx.plugin(require('../service/migration').default);
await handler(pending, fail, ctx);
await addon(pending, fail, ctx);
await ctx.lifecycle.flush();
for (const i in global.Hydro.handler) await global.Hydro.handler[i]();
const scriptDir = path.resolve(__dirname, '..', 'script');
for (const h of await fs.readdir(scriptDir)) {
ctx.loader.reloadPlugin(ctx, path.resolve(scriptDir, h), {}, `hydrooj/script/${h.split('.')[0]}`);
@@ -78,27 +78,18 @@ export async function apply(ctx: Context) {
await ctx.lifecycle.flush();
await ctx.parallel('app/started');
if (process.env.NODE_APP_INSTANCE === '0') {
const { default: scripts, init } = require('../upgrade');
let dbVer = (await modelSystem.get('db.ver')) ?? 0;
const isFresh = !dbVer;
const expected = scripts.length;
if (isFresh) {
await init();
await modelSystem.set('db.ver', expected);
} else {
while (dbVer < expected) {
const func = scripts[dbVer];
if (typeof func !== 'function') {
dbVer++;
continue;
await new Promise((resolve, reject) => {
ctx.inject(['migration'], async (c) => {
c.migration.registerChannel('hydrooj', require('../upgrade').coreScripts);
try {
await c.migration.doUpgrade();
resolve(null);
} catch (e) {
logger.error('Upgrade failed: %O', e);
reject(e);
}
logger.info('Upgrading database: from %d to %d', dbVer, expected);
const result = await func();
if (!result) break;
dbVer++;
await modelSystem.set('db.ver', dbVer);
}
}
});
});
}
for (const f of global.addons) {
const dir = path.join(f, 'public');
packages/hydrooj/src/interface.ts
-2
View File
@@ -808,8 +808,6 @@ export interface ModuleInterfaces {
export interface HydroGlobal {
version: Record<string, string>;
model: Model;
/** @deprecated */
handler: Record<string, Function>;
script: Record<string, Script>;
service: HydroService;
lib: Lib;
packages/hydrooj/src/model/user.ts
+1 -1
View File
@@ -179,7 +179,7 @@ declare module '@hydrooj/framework' {
interface UserModel extends User { }
}
function handleMailLower(mail: string) {
export function handleMailLower(mail: string) {
const [n, d] = mail.trim().toLowerCase().split('@');
const [name] = n.split('+');
return `${name.replace(/\./g, '')}@${d === 'googlemail.com' ? 'gmail.com' : d}`;
packages/hydrooj/src/service/migration.ts
+52
View File
@@ -0,0 +1,52 @@
/* eslint-disable no-await-in-loop */
import { Logger } from '@hydrooj/utils';
import { Context, Service } from '../context';
import * as system from '../model/system';
export type MigrationScript = null | (() => Promise<boolean | void>);
const logger = new Logger('migration');
declare module '../context' {
interface Context {
migration: MigrationService;
}
}
export default class MigrationService extends Service {
private channels: Record<string, MigrationScript[]> = {};
constructor(ctx: Context) {
super(ctx, 'migration', true);
}
async registerChannel(name: string, s: MigrationScript[]) {
this.channels[name] = s;
}
async doUpgrade() {
for (const channel in this.channels) {
const name = `db.ver${channel === 'hydrooj' ? '' : `-${channel}`}`;
let dbVer = system.get(name) ?? 0;
const scripts = this.channels[channel];
const isFresh = !dbVer;
const expected = scripts.length;
if (isFresh) {
await scripts[0]?.();
await system.set(name, expected);
continue;
}
while (dbVer < expected) {
const func = scripts[dbVer];
if (typeof func !== 'function') {
dbVer++;
continue;
}
logger.info('Upgrading database [%s]: from %d to %d', channel, dbVer, expected);
const result = await func();
if (!result) break;
dbVer++;
await system.set(name, dbVer);
}
}
}
}
packages/hydrooj/src/upgrade.ts
+27 -19
View File
@@ -21,37 +21,35 @@ import StorageModel from './model/storage';
import * as system from './model/system';
import TaskModel from './model/task';
import * as training from './model/training';
import user from './model/user';
import user, { handleMailLower } from './model/user';
import {
iterateAllContest, iterateAllDomain, iterateAllProblem, iterateAllUser,
} from './pipelineUtils';
import db from './service/db';
import { MigrationScript } from './service/migration';
import { setBuiltinConfig } from './settings';
import welcome from './welcome';
const logger = new Logger('upgrade');
type UpgradeScript = void | (() => Promise<boolean | void>);
const unsupportedUpgrade = async function _26_27() {
throw new Error('This upgrade was no longer supported in hydrooj@4. \
Please use hydrooj@3 to perform these upgrades before upgrading to v4');
};
export async function init() {
if (!await user.getById('system', 0)) {
await user.create('Guest@hydro.local', 'Guest', String.random(32), 0, '127.0.0.1', PRIV.PRIV_REGISTER_USER);
}
if (!await user.getById('system', 1)) {
await user.create('Hydro@hydro.local', 'Hydro', String.random(32), 1, '127.0.0.1', PRIV.PRIV_USER_PROFILE);
}
const ddoc = await domain.get('system');
if (!ddoc) await domain.add('system', 1, 'Hydro', 'Welcome to Hydro!');
await welcome();
return true;
}
const scripts: UpgradeScript[] = [
export const coreScripts: MigrationScript[] = [
// Mark as used
null,
async function init() {
if (!await user.getById('system', 0)) {
await user.create('Guest@hydro.local', 'Guest', String.random(32), 0, '127.0.0.1', PRIV.PRIV_REGISTER_USER);
}
if (!await user.getById('system', 1)) {
await user.create('Hydro@hydro.local', 'Hydro', String.random(32), 1, '127.0.0.1', PRIV.PRIV_USER_PROFILE);
}
const ddoc = await domain.get('system');
if (!ddoc) await domain.add('system', 1, 'Hydro', 'Welcome to Hydro!');
await welcome();
return true;
},
// Init
...new Array(26).fill(unsupportedUpgrade),
null,
@@ -675,6 +673,16 @@ const scripts: UpgradeScript[] = [
if (bulk.batches.length) await bulk.execute();
return true;
},
async function _89_90() {
return await iterateAllUser(async (udoc) => {
const wanted = handleMailLower(udoc.mail);
if (wanted !== udoc.mailLower) {
if (await user.getByEmail('system', wanted)) {
console.warn('Email conflict: %s', wanted);
return;
}
await user.coll.updateOne({ _id: udoc._id }, { $set: { mailLower: wanted } });
}
});
},
];
export default scripts;