Weishen/Hydro
2
0 Fork 0
mirror of https://gitcode.com/gh_mirrors/hy/Hydro.git synced 2026-10-20 01:38:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

core: upgrade: use seperate pipeline for fresh install

Browse Source
This commit is contained in:
undefined committed 2023-01-08 20:48:25 +08:00
1 parent 9c40c8da47
commit 06e0223793
2 files changed
+26 -71
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/hydrooj/src/entry/worker.ts
+7 -2
View File
@@ -71,13 +71,17 @@ export async function apply(ctx: Context) {
await ctx.lifecycle.flush(); await ctx.lifecycle.flush();
await ctx.parallel('app/started'); await ctx.parallel('app/started');
if (process.env.NODE_APP_INSTANCE === '0') { if (process.env.NODE_APP_INSTANCE === '0') {
const scripts = require('../upgrade').default; const { default: scripts, init } = require('../upgrade');
let dbVer = (await modelSystem.get('db.ver')) ?? 0; let dbVer = (await modelSystem.get('db.ver')) ?? 0;
const isFresh = !dbVer; const isFresh = !dbVer;
const expected = scripts.length; const expected = scripts.length;
if (isFresh) {
await init();
await modelSystem.set('db.ver', expected);
} else {
while (dbVer < expected) { while (dbVer < expected) {
const func = scripts[dbVer]; const func = scripts[dbVer];
if (typeof func !== 'function' || (isFresh && func.toString().includes('_FRESH_INSTALL_IGNORE'))) { if (typeof func !== 'function') {
dbVer++; dbVer++;
continue; continue;
} }
@@ -88,6 +92,7 @@ export async function apply(ctx: Context) {
await modelSystem.set('db.ver', dbVer); await modelSystem.set('db.ver', dbVer);
} }
} }
}
logger.success('Server started'); logger.success('Server started');
process.send?.('ready'); process.send?.('ready');
await ctx.parallel('app/ready'); await ctx.parallel('app/ready');
packages/hydrooj/src/upgrade.ts
+19 -69
View File
@@ -5,6 +5,7 @@ import yaml from 'js-yaml';
import { pick } from 'lodash'; import { pick } from 'lodash';
import moment from 'moment-timezone'; import moment from 'moment-timezone';
import { ObjectID } from 'mongodb'; import { ObjectID } from 'mongodb';
import { sleep } from '@hydrooj/utils';
import { buildContent } from './lib/content'; import { buildContent } from './lib/content';
import { Logger } from './logger'; import { Logger } from './logger';
import { PERM, PRIV, STATUS } from './model/builtin'; import { PERM, PRIV, STATUS } from './model/builtin';
@@ -30,25 +31,29 @@ import welcome from './welcome';
const logger = new Logger('upgrade'); const logger = new Logger('upgrade');
type UpgradeScript = void | (() => Promise<boolean | void>); type UpgradeScript = void | (() => Promise<boolean | void>);
const unsupportedUpgrade = async function _26_27() { const unsupportedUpgrade = async function _26_27() {
const _FRESH_INSTALL_IGNORE = 1;
throw new Error('This upgrade was no longer supported in hydrooj@4. \ throw new Error('This upgrade was no longer supported in hydrooj@4. \
Please use hydrooj@3 to perform these upgrades before upgrading to v4'); Please use hydrooj@3 to perform these upgrades before upgrading to v4');
}; };
export async function init() {
if (!await user.getById('system', 0)) {
await user.create('Guest@hydro.local', 'Guest', String.random(32), 0, '127.0.0.1', PRIV.PRIV_REGISTER_USER);
}
if (!await user.getById('system', 1)) {
await user.create('Hydro@hydro.local', 'Hydro', String.random(32), 1, '127.0.0.1', PRIV.PRIV_USER_PROFILE);
}
const ddoc = await domain.get('system');
if (!ddoc) await domain.add('system', 1, 'Hydro', 'Welcome to Hydro!');
await welcome();
return true;
}
const scripts: UpgradeScript[] = [ const scripts: UpgradeScript[] = [
// Mark as used // Mark as used
null, null,
// Init // Init
async function _1_2() { ...new Array(26).fill(unsupportedUpgrade),
const ddoc = await domain.get('system');
if (!ddoc) await domain.add('system', 1, 'Hydro', 'Welcome to Hydro!');
await welcome();
// TODO discussion node?
return true;
},
...new Array(25).fill(unsupportedUpgrade),
async function _27_28() { async function _27_28() {
const _FRESH_INSTALL_IGNORE = 1;
const cursor = document.coll.find({ docType: document.TYPE_DISCUSSION }); const cursor = document.coll.find({ docType: document.TYPE_DISCUSSION });
for await (const data of cursor) { for await (const data of cursor) {
const t = Math.exp(-0.15 * (((new Date().getTime() / 1000) - data._id.generationTime) / 3600)); const t = Math.exp(-0.15 * (((new Date().getTime() / 1000) - data._id.generationTime) / 3600));
@@ -59,7 +64,6 @@ const scripts: UpgradeScript[] = [
return true; return true;
}, },
async function _28_29() { async function _28_29() {
const _FRESH_INSTALL_IGNORE = 1;
return await iterateAllProblem(['content', 'html'], async (pdoc) => { return await iterateAllProblem(['content', 'html'], async (pdoc) => {
try { try {
const parsed = JSON.parse(pdoc.content); const parsed = JSON.parse(pdoc.content);
@@ -77,12 +81,10 @@ const scripts: UpgradeScript[] = [
}); });
}, },
async function _29_30() { async function _29_30() {
const _FRESH_INSTALL_IGNORE = 1;
return await iterateAllDomain((ddoc) => RecordModel.coll.updateMany({ domainId: ddoc._id }, { $set: { pdomain: ddoc._id } })); return await iterateAllDomain((ddoc) => RecordModel.coll.updateMany({ domainId: ddoc._id }, { $set: { pdomain: ddoc._id } }));
}, },
// Add send_message priv to user // Add send_message priv to user
async function _30_31() { async function _30_31() {
const _FRESH_INSTALL_IGNORE = 1;
return await iterateAllUser((udoc) => user.setPriv(udoc._id, udoc.priv | PRIV.PRIV_SEND_MESSAGE)); return await iterateAllUser((udoc) => user.setPriv(udoc._id, udoc.priv | PRIV.PRIV_SEND_MESSAGE));
}, },
null, null,
@@ -97,23 +99,19 @@ const scripts: UpgradeScript[] = [
return true; return true;
}, },
async function _33_34() { async function _33_34() {
const _FRESH_INSTALL_IGNORE = 1;
return await iterateAllProblem(['content'], async (pdoc) => { return await iterateAllProblem(['content'], async (pdoc) => {
if (typeof pdoc.content !== 'string') return; if (typeof pdoc.content !== 'string') return;
await problem.edit(pdoc.domainId, pdoc.docId, { content: pdoc.content.replace(/%file%:\/\//g, 'file://') }); await problem.edit(pdoc.domainId, pdoc.docId, { content: pdoc.content.replace(/%file%:\/\//g, 'file://') });
}); });
}, },
async function _34_35() { async function _34_35() {
const _FRESH_INSTALL_IGNORE = 1;
return await iterateAllDomain((ddoc) => domain.edit(ddoc._id, { lower: ddoc._id.toLowerCase() })); return await iterateAllDomain((ddoc) => domain.edit(ddoc._id, { lower: ddoc._id.toLowerCase() }));
}, },
async function _35_36() { async function _35_36() {
const _FRESH_INSTALL_IGNORE = 1;
await RecordModel.coll.updateMany({}, { $unset: { effective: '' } }); await RecordModel.coll.updateMany({}, { $unset: { effective: '' } });
return true; return true;
}, },
async function _36_37() { async function _36_37() {
const _FRESH_INSTALL_IGNORE = 1;
for await (const doc of document.collStatus.find()) { for await (const doc of document.collStatus.find()) {
await document.collStatus.deleteMany({ await document.collStatus.deleteMany({
...pick(doc, ['docId', 'domainId', 'uid', 'docType']), ...pick(doc, ['docId', 'domainId', 'uid', 'docType']),
@@ -123,7 +121,6 @@ const scripts: UpgradeScript[] = [
return true; return true;
}, },
async function _37_38() { async function _37_38() {
const _FRESH_INSTALL_IGNORE = 1;
return await iterateAllProblem(['docId', 'domainId', 'config'], async (pdoc) => { return await iterateAllProblem(['docId', 'domainId', 'config'], async (pdoc) => {
logger.info('%s/%s', pdoc.domainId, pdoc.docId); logger.info('%s/%s', pdoc.domainId, pdoc.docId);
if (typeof pdoc.config !== 'string') return; if (typeof pdoc.config !== 'string') return;
@@ -135,14 +132,12 @@ const scripts: UpgradeScript[] = [
}); });
}, },
async function _38_39() { async function _38_39() {
const _FRESH_INSTALL_IGNORE = 1;
return await iterateAllDomain(async (ddoc) => { return await iterateAllDomain(async (ddoc) => {
ddoc.roles.root = '-1'; ddoc.roles.root = '-1';
await domain.setRoles(ddoc._id, ddoc.roles); await domain.setRoles(ddoc._id, ddoc.roles);
}); });
}, },
async function _39_40() { async function _39_40() {
const _FRESH_INSTALL_IGNORE = 1;
return await iterateAllDomain(async ({ _id }) => { return await iterateAllDomain(async ({ _id }) => {
const ddocs = await discussion.getMulti(_id, { parentType: document.TYPE_PROBLEM }).toArray(); const ddocs = await discussion.getMulti(_id, { parentType: document.TYPE_PROBLEM }).toArray();
for (const ddoc of ddocs) { for (const ddoc of ddocs) {
@@ -152,7 +147,6 @@ const scripts: UpgradeScript[] = [
}); });
}, },
async function _40_41() { async function _40_41() {
const _FRESH_INSTALL_IGNORE = 1;
// Ignore drop index failure // Ignore drop index failure
await db.collection('storage').dropIndex('path_1').catch(() => { }); await db.collection('storage').dropIndex('path_1').catch(() => { });
await db.collection('storage').updateMany( await db.collection('storage').updateMany(
@@ -162,7 +156,6 @@ const scripts: UpgradeScript[] = [
return true; return true;
}, },
async function _41_42() { async function _41_42() {
const _FRESH_INSTALL_IGNORE = 1;
return await iterateAllDomain(async (ddoc) => { return await iterateAllDomain(async (ddoc) => {
const cursor = discussion.getMulti(ddoc._id, { parentType: document.TYPE_CONTEST }); const cursor = discussion.getMulti(ddoc._id, { parentType: document.TYPE_CONTEST });
while (await cursor.hasNext()) { while (await cursor.hasNext()) {
@@ -172,29 +165,8 @@ const scripts: UpgradeScript[] = [
} }
}); });
}, },
async function _42_43() { null,
const _FRESH_INSTALL_IGNORE = 1;
const processer = (i) => {
i.status = i.accept ? STATUS.STATUS_ACCEPTED : STATUS.STATUS_WRONG_ANSWER;
return i;
};
return await iterateAllDomain(async (ddoc) => {
const tdocs = await contest.getMulti(ddoc._id, { rule: 'acm' }).toArray();
for (const tdoc of tdocs) {
const tsdocs = await contest.getMultiStatus(ddoc._id, { docId: tdoc.docId }).toArray();
for (const tsdoc of tsdocs) {
const $set: any = {};
if (tsdoc.journal?.length) $set.journal = tsdoc.journal.map(processer);
if (tsdoc.detail?.length) $set.detail = tsdoc.detail.map(processer);
if (Object.keys($set).length) {
await contest.setStatus(ddoc._id, tdoc.docId, tsdoc.uid, $set);
}
}
}
});
},
async function _43_44() { async function _43_44() {
const _FRESH_INSTALL_IGNORE = 1;
const processer = (i) => { const processer = (i) => {
i.status = i.accept ? STATUS.STATUS_ACCEPTED : STATUS.STATUS_WRONG_ANSWER; i.status = i.accept ? STATUS.STATUS_ACCEPTED : STATUS.STATUS_WRONG_ANSWER;
return i; return i;
@@ -215,11 +187,9 @@ const scripts: UpgradeScript[] = [
}); });
}, },
async function _44_45() { async function _44_45() {
const _FRESH_INSTALL_IGNORE = 1;
return await iterateAllUser((udoc) => user.setById(udoc._id, { ip: [udoc.regip] }, { regip: '' })); return await iterateAllUser((udoc) => user.setById(udoc._id, { ip: [udoc.regip] }, { regip: '' }));
}, },
async function _45_46() { async function _45_46() {
const _FRESH_INSTALL_IGNORE = 1;
return await iterateAllDomain(async (ddoc) => { return await iterateAllDomain(async (ddoc) => {
const ddocs = await discussion.getMulti(ddoc._id, { const ddocs = await discussion.getMulti(ddoc._id, {
docType: document.TYPE_DISCUSSION, docType: document.TYPE_DISCUSSION,
@@ -235,7 +205,6 @@ const scripts: UpgradeScript[] = [
}, },
null, null,
async function _47_48() { async function _47_48() {
const _FRESH_INSTALL_IGNORE = 1;
await document.coll.updateMany({ docType: document.TYPE_HOMEWORK }, { $set: { docType: document.TYPE_CONTEST } }); await document.coll.updateMany({ docType: document.TYPE_HOMEWORK }, { $set: { docType: document.TYPE_CONTEST } });
await document.collStatus.updateMany({ docType: document.TYPE_HOMEWORK }, { $set: { docType: document.TYPE_CONTEST } }); await document.collStatus.updateMany({ docType: document.TYPE_HOMEWORK }, { $set: { docType: document.TYPE_CONTEST } });
await RecordModel.coll.deleteMany({ 'contest.tid': { $ne: null }, hidden: true }); await RecordModel.coll.deleteMany({ 'contest.tid': { $ne: null }, hidden: true });
@@ -246,23 +215,19 @@ const scripts: UpgradeScript[] = [
return true; return true;
}, },
async function _48_49() { async function _48_49() {
const _FRESH_INSTALL_IGNORE = 1;
await RecordModel.coll.updateMany({ input: { $exists: true } }, { $set: { contest: new ObjectID('000000000000000000000000') } }); await RecordModel.coll.updateMany({ input: { $exists: true } }, { $set: { contest: new ObjectID('000000000000000000000000') } });
return true; return true;
}, },
async function _49_50() { async function _49_50() {
const _FRESH_INSTALL_IGNORE = 1;
await db.collection('user').updateMany({}, { $unset: { ratingHistory: '' } }); await db.collection('user').updateMany({}, { $unset: { ratingHistory: '' } });
await db.collection('domain').updateMany({}, { $unset: { pidCounter: '' } }); await db.collection('domain').updateMany({}, { $unset: { pidCounter: '' } });
return true; return true;
}, },
async function _50_51() { async function _50_51() {
const _FRESH_INSTALL_IGNORE = 1;
await db.collection('domain.user').updateMany({}, { $unset: { ratingHistory: '' } }); await db.collection('domain.user').updateMany({}, { $unset: { ratingHistory: '' } });
return true; return true;
}, },
async function _51_52() { async function _51_52() {
const _FRESH_INSTALL_IGNORE = 1;
const mapping: Record<string, number> = {}; const mapping: Record<string, number> = {};
const isStringPid = (i: string) => i.toString().includes(':'); const isStringPid = (i: string) => i.toString().includes(':');
async function getProblem(domainId: string, target: string) { async function getProblem(domainId: string, target: string) {
@@ -317,7 +282,6 @@ const scripts: UpgradeScript[] = [
return true; return true;
}, },
async function _52_53() { async function _52_53() {
const _FRESH_INSTALL_IGNORE = 1;
const cursor = db.collection('document').find({ docType: document.TYPE_CONTEST }); const cursor = db.collection('document').find({ docType: document.TYPE_CONTEST });
for await (const tdoc of cursor) { for await (const tdoc of cursor) {
const pdocs = await problem.getMulti(tdoc.domainId, { docId: { $in: tdoc.pids } }).toArray(); const pdocs = await problem.getMulti(tdoc.domainId, { docId: { $in: tdoc.pids } }).toArray();
@@ -332,7 +296,6 @@ const scripts: UpgradeScript[] = [
return true; return true;
}, },
async function _53_54() { async function _53_54() {
const _FRESH_INSTALL_IGNORE = 1;
let ddocs = await db.collection('document').find({ docType: 21, parentType: 10 }) let ddocs = await db.collection('document').find({ docType: 21, parentType: 10 })
.project({ _id: 1, parentId: 1 }).toArray(); .project({ _id: 1, parentId: 1 }).toArray();
ddocs = ddocs.filter((i) => Number.isSafeInteger(+i.parentId)); ddocs = ddocs.filter((i) => Number.isSafeInteger(+i.parentId));
@@ -346,7 +309,6 @@ const scripts: UpgradeScript[] = [
return true; return true;
}, },
async function _54_55() { async function _54_55() {
const _FRESH_INSTALL_IGNORE = 1;
const bulk = db.collection('document').initializeUnorderedBulkOp(); const bulk = db.collection('document').initializeUnorderedBulkOp();
function sortable(source: string) { function sortable(source: string) {
return source.replace(/(\d+)/g, (str) => (str.length >= 6 ? str : ('0'.repeat(6 - str.length) + str))); return source.replace(/(\d+)/g, (str) => (str.length >= 6 ? str : ('0'.repeat(6 - str.length) + str)));
@@ -358,18 +320,15 @@ const scripts: UpgradeScript[] = [
return true; return true;
}, },
async function _55_56() { async function _55_56() {
const _FRESH_INSTALL_IGNORE = 1;
await db.collection('document').updateMany({ docType: document.TYPE_PROBLEM }, { $unset: { difficulty: '' } }); await db.collection('document').updateMany({ docType: document.TYPE_PROBLEM }, { $unset: { difficulty: '' } });
return true; return true;
}, },
async function _56_57() { async function _56_57() {
const _FRESH_INSTALL_IGNORE = 1;
await db.collection('oplog').deleteMany({ type: 'user.login' }); await db.collection('oplog').deleteMany({ type: 'user.login' });
return true; return true;
}, },
null, null,
async function _58_59() { async function _58_59() {
const _FRESH_INSTALL_IGNORE = 1;
const tasks = await db.collection('task').find({ type: 'schedule', subType: 'contest.problemHide' }).toArray(); const tasks = await db.collection('task').find({ type: 'schedule', subType: 'contest.problemHide' }).toArray();
for (const task of tasks) { for (const task of tasks) {
await ScheduleModel.add({ ...task, subType: 'contest', operation: ['unhide'] }); await ScheduleModel.add({ ...task, subType: 'contest', operation: ['unhide'] });
@@ -396,7 +355,6 @@ const scripts: UpgradeScript[] = [
return true; return true;
}, },
async function _61_62() { async function _61_62() {
const _FRESH_INSTALL_IGNORE = 1;
const priv = +system.get('default.priv'); const priv = +system.get('default.priv');
if (priv & PRIV.PRIV_REGISTER_USER) { if (priv & PRIV.PRIV_REGISTER_USER) {
const udocs = await user.getMulti({ priv }).project({ _id: 1 }).toArray(); const udocs = await user.getMulti({ priv }).project({ _id: 1 }).toArray();
@@ -408,7 +366,6 @@ const scripts: UpgradeScript[] = [
return true; return true;
}, },
async function _62_63() { async function _62_63() {
const _FRESH_INSTALL_IGNORE = 1;
const uids = new Set<number>(); const uids = new Set<number>();
await iterateAllDomain(async (ddoc) => { await iterateAllDomain(async (ddoc) => {
const pdocs = await problem.getMulti(ddoc._id, { config: /type: objective/ }) const pdocs = await problem.getMulti(ddoc._id, { config: /type: objective/ })
@@ -515,15 +472,13 @@ const scripts: UpgradeScript[] = [
endPointForUser, endPointForUser,
endPointForJudge, endPointForJudge,
}); });
setTimeout(() => { await sleep(1000);
logger.info('Upgrade done. please restart the server.'); logger.info('Upgrade done. please restart the server.');
process.exit(0); process.exit(0);
}, 1000);
} }
return true; return true;
}, },
async function _67_68() { async function _67_68() {
const _FRESH_INSTALL_IGNORE = 1;
const rdocs = RecordModel.coll.find({ code: /^@@hydro_submission_file@@/ }); const rdocs = RecordModel.coll.find({ code: /^@@hydro_submission_file@@/ });
for await (const rdoc of rdocs) { for await (const rdoc of rdocs) {
await RecordModel.update(rdoc.domainId, rdoc._id, { await RecordModel.update(rdoc.domainId, rdoc._id, {
@@ -534,7 +489,6 @@ const scripts: UpgradeScript[] = [
return true; return true;
}, },
async function _68_69() { async function _68_69() {
const _FRESH_INSTALL_IGNORE = 1;
await db.collection('cache' as any).deleteMany({}); await db.collection('cache' as any).deleteMany({});
return true; return true;
}, },
@@ -544,18 +498,14 @@ const scripts: UpgradeScript[] = [
await TaskModel.coll.deleteMany({}); await TaskModel.coll.deleteMany({});
return true; return true;
}, },
async function _70_71() { null,
async function _71_72() {
return await iterateAllContest(async (tdoc) => { return await iterateAllContest(async (tdoc) => {
if (['acm', 'homework'].includes(tdoc.rule)) { if (['acm', 'homework'].includes(tdoc.rule)) {
await contest.recalcStatus(tdoc.domainId, tdoc.docId); await contest.recalcStatus(tdoc.domainId, tdoc.docId);
} }
}); });
}, },
async function _71_72() {
return await iterateAllContest(async (tdoc) => {
if (tdoc.rule === 'homework') await contest.recalcStatus(tdoc.domainId, tdoc.docId);
});
},
async function _72_73() { async function _72_73() {
await db.collection('oplog').updateMany({}, { $unset: { 'args.verifyPassword': '' } }); await db.collection('oplog').updateMany({}, { $unset: { 'args.verifyPassword': '' } });
return true; return true;