From fc3321688fda5bd8be0a33108351873ee3d69e36 Mon Sep 17 00:00:00 2001 From: undefined Date: Sat, 19 Dec 2020 01:33:23 +0800 Subject: [PATCH] core: db upd --- packages/hydrooj/src/entry/worker.ts | 1 + packages/hydrooj/src/model/system.ts | 8 +- packages/hydrooj/src/service/storage.ts | 3 +- packages/hydrooj/src/upgrade.ts | 117 ++++++++++++++++-------- 4 files changed, 89 insertions(+), 40 deletions(-) diff --git a/packages/hydrooj/src/entry/worker.ts b/packages/hydrooj/src/entry/worker.ts index 71d51ab93..f6d9dcc54 100644 --- a/packages/hydrooj/src/entry/worker.ts +++ b/packages/hydrooj/src/entry/worker.ts @@ -25,6 +25,7 @@ export async function load() { const opts = options(); await db.start(opts); const modelSystem = require('../model/system'); + await modelSystem.runConfig(); const [endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge] = modelSystem.getMany([ 'file.endPoint', 'file.accessKey', 'file.secretKey', 'file.bucket', 'file.region', 'file.endPointForUser', 'file.endPointForJudge', diff --git a/packages/hydrooj/src/model/system.ts b/packages/hydrooj/src/model/system.ts index d37a99cfd..6872a6ed4 100644 --- a/packages/hydrooj/src/model/system.ts +++ b/packages/hydrooj/src/model/system.ts @@ -7,7 +7,9 @@ import * as bus from '../service/bus'; const coll = db.collection('system'); const cache: Record = {}; -export function get(key: K): SystemKeys[K] { +export function get(key: K): SystemKeys[K] +export function get(key: string): any +export function get(key: string): any { return cache[key]; } @@ -34,7 +36,9 @@ export function getMany(keys: string[]): any[] { return keys.map((key) => cache[key]); } -export async function set(_id: K, value: SystemKeys[K]) { +export async function set(_id: K, value: SystemKeys[K]): Promise +export async function set(_id: string, value: K): Promise +export async function set(_id: string, value: any) { const res = await coll.findOneAndUpdate( { _id }, { $set: { value } }, diff --git a/packages/hydrooj/src/service/storage.ts b/packages/hydrooj/src/service/storage.ts index d1dc206c1..edbc2a115 100644 --- a/packages/hydrooj/src/service/storage.ts +++ b/packages/hydrooj/src/service/storage.ts @@ -90,7 +90,8 @@ class StorageService { return await this.client.putObject(this.opts.bucket, target, file, meta); } - async get(target: string) { + async get(target: string, path?: string) { + if (path) return await this.client.fGetObject(this.opts.bucket, target, path); return await this.client.getObject(this.opts.bucket, target); } diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index cdf7e0f24..bbf51bbae 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -1,14 +1,24 @@ /* eslint-disable no-await-in-loop */ import { ObjectID, Collection } from 'mongodb'; import { terminal } from 'terminal-kit'; +import AdmZip from 'adm-zip'; import { Ufdoc } from './interface'; import { Logger } from './logger'; +import { streamToBuffer } from './utils'; import gridfs from './service/gridfs'; import storage from './service/storage'; import db from './service/db'; import * as problem from './model/problem'; +import * as user from './model/user'; import * as domain from './model/domain'; import * as document from './model/document'; +import * as system from './model/system'; + +declare module 'terminal-kit/Terminal' { + interface ProgressBarOptions { + y?: number; + } +} const logger = new Logger('upgrade'); type UpgradeScript = () => Promise; @@ -49,7 +59,6 @@ const scripts: UpgradeScript[] = [ logger.error('Cannot upgrade. Please change storage config.'); return false; } - const coll: Collection = db.collection('file'); const collFile = db.collection('fs.files'); const collChunk = db.collection('fs.chunks'); @@ -60,46 +69,80 @@ const scripts: UpgradeScript[] = [ collChunk.deleteMany({ files_id: _id }), ]); } - async function get(_id: ObjectID, reject?: Function) { - const stream = gridfs.openDownloadStream(_id); - stream.on('error', (err) => { - console.error(err); - if (reject) reject(err); - }); - return stream; - } - terminal.clear('Upgrade'); - logger.info(''); + terminal.clear('Changing storage engine. This may take a long time.'); // Problem file and User file - // @ts-ignore - const progress = terminal.progressBar({ title: 'Changing storage engine. This may take a long time.', y: 1 }); - const ddocs = await domain.getMulti().toArray(); - const totalProgress = terminal.progressBar({ - // @ts-ignore - items: ddocs.length, title: 'Domain', y: 3, eta: true, - }); - for (let i = 0; i < ddocs.length; i++) { - const ddoc = ddocs[i]; - totalProgress.startItem(ddoc._id); - const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['docId', 'data', 'title']).toArray(); - // @ts-ignore - const domainProgress = terminal.progressBar({ - // @ts-ignore - items: pdocs.length, title: 'Problems', y: 4, eta: true, + try { + let savedProgress = system.get('upgrade.file.progress.domain'); + if (savedProgress) savedProgress = JSON.parse(savedProgress); + else savedProgress = { pdocs: [] }; + const ddocs = await domain.getMulti().toArray(); + const totalProgress = terminal.progressBar({ + items: ddocs.length, title: 'Domain', y: 3, eta: true, }); - for (let j = 0; j < pdocs.length; j++) { - const pdoc = pdocs[j]; - domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`); - // TODO copy file - domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`); + for (let i = 0; i < ddocs.length; i++) { + const ddoc = ddocs[i]; + totalProgress.startItem(ddoc._id); + const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray(); + const domainProgress = terminal.progressBar({ + items: pdocs.length, title: 'Problems', y: 4, eta: true, + }); + for (let j = 0; j < pdocs.length; j++) { + const pdoc = pdocs[j]; + domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`); + if (!savedProgress.pdocs.includes(`${pdoc.domainId}/${pdoc.docId}`) && pdoc.data instanceof ObjectID) { + try { + const file = await streamToBuffer(gridfs.openDownloadStream(pdoc.data)); + const zip = new AdmZip(file); + const entries = zip.getEntries(); + await Promise.all(entries.map( + (entry) => storage.put(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/${entry.name}`, entry.getData()), + )); + } catch (e) { + if (e.toString().includes('FileNotFound')) { + logger.error('\n\n\n\n\nProblem data not found %s/%s', pdoc.domainId, pdoc.docId); + } else throw e; + } + savedProgress.pdocs.push(`${pdoc.domainId}/${pdoc.docId}`); + } + system.set('upgrade.file.progress.domain', JSON.stringify(savedProgress)); + domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`); + } + domainProgress.stop(); + totalProgress.itemDone(ddoc._id); } - domainProgress.stop(); - totalProgress.itemDone(ddoc._id); + totalProgress.stop(); + const udocs = await user.getMulti().project({ _id: 1, uname: 1 }).toArray(); + const userfileProgress = terminal.progressBar({ + items: udocs.length, title: 'Users', y: 3, + }); + for (const udoc of udocs) { + userfileProgress.startItem(`${udoc._id}: ${udoc.uname}`); + const ufdocs = await coll.find({ owner: udoc._id }).toArray(); + const currentUser = terminal.progressBar({ + items: ufdocs.length, title: `${udoc._id}: ${udoc.uname}`, y: 4, eta: true, + }); + const namelist = []; + for (const ufdoc of ufdocs) { + currentUser.startItem(`${ufdoc._id}: ${ufdoc.filename || ''}`); + ufdoc.filename = ufdoc.filename || ufdoc._id.toHexString(); + const file = await streamToBuffer(gridfs.openDownloadStream(ufdoc._id)); + if (!namelist.includes(ufdoc.filename)) { + await storage.put(`user/${udoc._id}/${ufdoc.filename}`, file); + } else { + await storage.put(`user/${udoc._id}/${ufdoc._id}`, file); + } + namelist.push(ufdoc.filename); + currentUser.itemDone(`${ufdoc._id}: ${ufdoc.filename || ''}`); + } + currentUser.stop(); + userfileProgress.itemDone(`${udoc._id}: ${udoc.uname}`); + } + userfileProgress.stop(); + return true; + } catch (e) { + logger.error('\n\n\n\n\n%o', e); + return false; } - totalProgress.stop(); - // TODO userfile - progress.stop(); - return false; }, ];