diff --git a/packages/hydrojudge/src/service.ts b/packages/hydrojudge/src/service.ts index 4bc73349b..9c3f4fffc 100644 --- a/packages/hydrojudge/src/service.ts +++ b/packages/hydrojudge/src/service.ts @@ -66,7 +66,7 @@ async function postInit() { const filePath = path.join(homedir(), '.cache', 'hydro', 'judge', domainId, pid); await fs.ensureDir(filePath); if (!files.length) throw new SystemError('Problem data not found.'); - let etags: Record; + let etags: Record = {}; try { etags = JSON.parse(fs.readFileSync(path.join(filePath, 'etags')).toString()); } catch (e) { /* ignore */ } diff --git a/packages/hydrojudge/src/utils.ts b/packages/hydrojudge/src/utils.ts index f75a03f3d..8981f6877 100644 --- a/packages/hydrojudge/src/utils.ts +++ b/packages/hydrojudge/src/utils.ts @@ -109,6 +109,14 @@ export function restrictFile(p: string) { export function ensureFile(folder: string) { return (file: string, message: string) => { + // Historical issue + if (file.includes('/')) { + const f = path.join(folder, restrictFile(file.split('/')[1])); + if (fs.existsSync(f)) { + const stat = fs.statSync(f); + if (stat.isFile()) return f; + } + } const f = path.join(folder, restrictFile(file)); if (!fs.existsSync(f)) throw new FormatError(message, [file]); const stat = fs.statSync(f); diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index c29b26624..fc2f31601 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -27,6 +27,7 @@ "moment-timezone": "^0.5.32", "mongodb": "^3.6.3", "nodemailer": "^6.4.17", + "p-queue": "^6.6.2", "serialize-javascript": "^5.0.1", "sockjs": "^0.3.20", "superagent": "^6.1.0", @@ -58,4 +59,4 @@ "build": "tsc", "build:watch": "tsc --watch" } -} \ No newline at end of file +} diff --git a/packages/hydrooj/src/handler/problem.ts b/packages/hydrooj/src/handler/problem.ts index f2c753325..c56f5aa52 100644 --- a/packages/hydrooj/src/handler/problem.ts +++ b/packages/hydrooj/src/handler/problem.ts @@ -453,12 +453,13 @@ export class ProblemFilesHandler extends ProblemDetailHandler { @param('testdata', Types.Boolean) @param('additional_file', Types.Boolean) async get(domainId: string, getTestdata = true, getAdditionalFile = true) { + const canReadData = this.user._id === this.pdoc.owner || this.user.hasPerm(PERM.PERM_READ_PROBLEM_DATA); const [testdata, additional_file] = await Promise.all([ - getTestdata - ? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/testdata`, true) + getTestdata && canReadData + ? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/testdata/`, true) : [], getAdditionalFile - ? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file`, true) + ? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/`, true) : [], ]); this.response.body.testdata = testdata; @@ -671,7 +672,7 @@ export class ProblemImportHandler extends Handler { this.response.template = 'problem_import.html'; } - async post(domainId: string) { + async post({ domainId }) { if (!this.request.files.file) throw new ValidationError('file'); const stat = await fs.stat(this.request.files.file.path); if (stat.size > 128 * 1024 * 1024) throw new BadRequestError('File too large'); diff --git a/packages/hydrooj/src/model/builtin.ts b/packages/hydrooj/src/model/builtin.ts index 958bf3d69..14e21ad1c 100644 --- a/packages/hydrooj/src/model/builtin.ts +++ b/packages/hydrooj/src/model/builtin.ts @@ -18,11 +18,17 @@ export const PERM = { // Problem and Record PERM_CREATE_PROBLEM: 1n << 4n, PERM_EDIT_PROBLEM: 1n << 5n, + /** + * @deprecated + */ PERM_EDIT_PROBLEM_SELF: 1n << 6n, PERM_VIEW_PROBLEM: 1n << 7n, PERM_VIEW_PROBLEM_HIDDEN: 1n << 8n, PERM_SUBMIT_PROBLEM: 1n << 9n, PERM_READ_PROBLEM_DATA: 1n << 10n, + /** + * @deprecated + */ PERM_READ_PROBLEM_DATA_SELF: 1n << 11n, PERM_READ_RECORD_CODE: 1n << 12n, PERM_REJUDGE_PROBLEM: 1n << 13n, diff --git a/packages/hydrooj/src/service/server.ts b/packages/hydrooj/src/service/server.ts index 10eae3cef..6c2c9a374 100644 --- a/packages/hydrooj/src/service/server.ts +++ b/packages/hydrooj/src/service/server.ts @@ -520,7 +520,7 @@ export class Handler { try { await this.render(error instanceof UserFacingError ? 'error.html' : 'bsod.html', { error }); } catch (e) { - console.error(e); + logger.error(e); // this.response.body.error = {}; } } @@ -615,8 +615,8 @@ export class Handler { async onerror(error: HydroError) { if (!error.msg) error.msg = () => error.message; if (!(error instanceof NotFoundError)) { - console.error(error.msg(), error.params); - console.error(error.stack); + logger.error(error.msg(), error.params); + logger.error(error.stack); } this.response.status = error instanceof UserFacingError ? error.code : 500; this.response.template = error instanceof UserFacingError ? 'error.html' : 'bsod.html'; @@ -807,7 +807,7 @@ export class ConnectionHandler { async message(message: any) { } // eslint-disable-line onerror(err: HydroError) { - if (!(err instanceof NotFoundError)) console.error(err); + if (!(err instanceof NotFoundError)) logger.error(err); this.send({ error: { name: err.name, @@ -895,13 +895,7 @@ export function start() { if (ctx.response.headers.nolog) return; ctx._remoteAddress = ctx.request.ip; const status = ctx.response.status; - const color = status >= 500 ? 31 // red - : status >= 400 ? 33 // yellow - : status >= 300 ? 36 // cyan - : status >= 200 ? 32 // green - : 0; // no color - logger.debug(`\ -\x1b[0m${ctx.request.method} ${ctx.request.path} \x1b[${color}m${ctx.response.status}\x1b[0m ${endTime - startTime}ms ${ctx.response.length}\x1b[0m`); + logger.debug(`${ctx.request.method} ${ctx.request.path} ${ctx.response.status} ${endTime - startTime}ms ${ctx.response.length}`); }); } app.use(async (ctx, next) => { diff --git a/packages/hydrooj/src/ui.ts b/packages/hydrooj/src/ui.ts index beb36643a..2774af5dd 100644 --- a/packages/hydrooj/src/ui.ts +++ b/packages/hydrooj/src/ui.ts @@ -51,7 +51,11 @@ if (cluster.isMaster) { bus.on('message/log', (message) => { LogBox.appendLog(message); }); + const history = ['']; + let current = 0; ShellInput.on('submit', async (input) => { + history.push(input); + current = history.length; ShellInput.input.setContent(' '.repeat(terminal.width), false, false); ShellInput.input.setContent('', false, false); if (input[0] === '@') { @@ -61,6 +65,17 @@ if (cluster.isMaster) { } } else bus.parallel('message/run', input); }); + ShellInput.on('key', (key) => { + if (key === 'UP') { + current--; + ShellInput.input.setContent(' '.repeat(terminal.width), false, false); + ShellInput.input.setContent(history[current] || '', false, false); + } else if (key === 'DOWN') { + current++; + ShellInput.input.setContent(' '.repeat(terminal.width), false, false); + ShellInput.input.setContent(history[current] || '', false, false); + } + }); terminal.on('key', (key, matches, data) => { if (key === 'CTRL_C') terminate(); }); diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index 1ab0bf4e6..e52d32023 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -1,7 +1,9 @@ /* eslint-disable no-await-in-loop */ import { ObjectID, Collection } from 'mongodb'; +// @ts-ignore import { terminal } from 'terminal-kit'; import AdmZip from 'adm-zip'; +import Queue from 'p-queue'; import { Ufdoc } from './interface'; import { Logger } from './logger'; import { streamToBuffer } from './utils'; @@ -60,83 +62,82 @@ const scripts: UpgradeScript[] = [ return false; } const coll: Collection = db.collection('file'); - const collFile = db.collection('fs.files'); - const collChunk = db.collection('fs.chunks'); - function del(_id: ObjectID) { - return Promise.all([ - coll.deleteOne({ _id }), - collFile.deleteOne({ _id }), - collChunk.deleteMany({ files_id: _id }), - ]); - } logger.info('Changing storage engine. This may take a long time.'); // Problem file and User file - try { - let savedProgress = system.get('upgrade.file.progress.domain'); - if (savedProgress) savedProgress = JSON.parse(savedProgress); - else savedProgress = { pdocs: [] }; - const ddocs = await domain.getMulti().project({ _id: 1 }).toArray(); - logger.info('Total found %d domains.', ddocs.length); - for (let i = 0; i < ddocs.length; i++) { - const ddoc = ddocs[i]; - logger.info('Domain %s (%d/%d)', ddoc._id, i + 1, ddocs.length); - const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray(); - const domainProgress = terminal.progressBar({ items: pdocs.length, title: 'Problems', inline: true }); - for (let j = 0; j < pdocs.length; j++) { - const pdoc = pdocs[j]; - domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`); - if (!savedProgress.pdocs.includes(`${pdoc.domainId}/${pdoc.docId}`) && pdoc.data instanceof ObjectID) { - try { - const file = await streamToBuffer(gridfs.openDownloadStream(pdoc.data)); - const zip = new AdmZip(file); - const entries = zip.getEntries(); - await Promise.all(entries.map( - (entry) => storage.put(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/${entry.name}`, entry.getData()), - )); - } catch (e) { - if (e.toString().includes('FileNotFound')) { - logger.error('Problem data not found %s/%s', pdoc.domainId, pdoc.docId); - } else throw e; + let savedProgress = system.get('upgrade.file.progress.domain'); + if (savedProgress) savedProgress = JSON.parse(savedProgress); + else savedProgress = { pdocs: [] }; + const ddocs = await domain.getMulti().project({ _id: 1 }).toArray(); + logger.info('Total found %d domains.', ddocs.length); + for (let i = 0; i < ddocs.length; i++) { + const ddoc = ddocs[i]; + logger.info('Domain %s (%d/%d)', ddoc._id, i + 1, ddocs.length); + const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray(); + const domainProgress = terminal.progressBar({ items: pdocs.length, title: 'Problems', inline: true }); + for (let j = 0; j < pdocs.length; j++) { + const pdoc = pdocs[j]; + domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`); + if (!savedProgress.pdocs.includes(`${pdoc.domainId}/${pdoc.docId}`) && pdoc.data instanceof ObjectID) { + try { + const [file, current] = await Promise.all([ + streamToBuffer(gridfs.openDownloadStream(pdoc.data)), + storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`, true), + ]); + const zip = new AdmZip(file); + const entries = zip.getEntries(); + if (entries.map((entry) => entry.entryName).sort().join('?') !== current.map((f) => f.name).sort().join('?')) { + await storage.del(current.map((entry) => entry.prefix + entry.name)); + const queue = new Queue({ concurrency: 5 }); + const tasks = []; + for (const entry of entries) { + tasks.push(queue.add(() => storage.put( + `problem/${pdoc.domainId}/${pdoc.docId}/testdata/${entry.entryName}`, + entry.getData(), + ))); + } + await Promise.all(tasks); } - savedProgress.pdocs.push(`${pdoc.domainId}/${pdoc.docId}`); + } catch (e) { + if (e.toString().includes('FileNotFound')) { + logger.error('Problem data not found %s/%s', pdoc.domainId, pdoc.docId); + } else throw e; } - system.set('upgrade.file.progress.domain', JSON.stringify(savedProgress)); - domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`); + savedProgress.pdocs.push(`${pdoc.domainId}/${pdoc.docId}`); } - domainProgress.stop(); + system.set('upgrade.file.progress.domain', JSON.stringify(savedProgress)); + domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`); } - const udocs = await user.getMulti().project({ _id: 1, uname: 1 }).toArray(); - const userfileProgress = terminal.progressBar({ - items: udocs.length, title: 'Users', y: 3, - }); - for (const udoc of udocs) { - userfileProgress.startItem(`${udoc._id}: ${udoc.uname}`); - const ufdocs = await coll.find({ owner: udoc._id }).toArray(); - const currentUser = terminal.progressBar({ - items: ufdocs.length, title: `${udoc._id}: ${udoc.uname}`, y: 4, eta: true, - }); - const namelist = []; - for (const ufdoc of ufdocs) { - currentUser.startItem(`${ufdoc._id}: ${ufdoc.filename || ''}`); - ufdoc.filename = ufdoc.filename || ufdoc._id.toHexString(); - const file = await streamToBuffer(gridfs.openDownloadStream(ufdoc._id)); - if (!namelist.includes(ufdoc.filename)) { - await storage.put(`user/${udoc._id}/${ufdoc.filename}`, file); - } else { - await storage.put(`user/${udoc._id}/${ufdoc._id}`, file); - } - namelist.push(ufdoc.filename); - currentUser.itemDone(`${ufdoc._id}: ${ufdoc.filename || ''}`); - } - currentUser.stop(); - userfileProgress.itemDone(`${udoc._id}: ${udoc.uname}`); - } - userfileProgress.stop(); - return true; - } catch (e) { - logger.error(e); - return false; + domainProgress.stop(); } + const udocs = await user.getMulti().project({ _id: 1, uname: 1 }).toArray(); + const userfileProgress = terminal.progressBar({ + items: udocs.length, title: 'Users', y: 3, + }); + for (const udoc of udocs) { + userfileProgress.startItem(`${udoc._id}: ${udoc.uname}`); + const ufdocs = await coll.find({ owner: udoc._id }).toArray(); + const currentUser = terminal.progressBar({ + items: ufdocs.length, title: `${udoc._id}: ${udoc.uname}`, y: 4, eta: true, + }); + const namelist = []; + for (const ufdoc of ufdocs) { + currentUser.startItem(`${ufdoc._id}: ${ufdoc.filename || ''}`); + ufdoc.filename = ufdoc.filename || ufdoc._id.toHexString(); + const file = await streamToBuffer(gridfs.openDownloadStream(ufdoc._id)); + if (!namelist.includes(ufdoc.filename)) { + await storage.put(`user/${udoc._id}/${ufdoc.filename}`, file); + } else { + await storage.put(`user/${udoc._id}/${ufdoc._id}`, file); + } + namelist.push(ufdoc.filename); + currentUser.itemDone(`${ufdoc._id}: ${ufdoc.filename || ''}`); + } + currentUser.stop(); + userfileProgress.itemDone(`${udoc._id}: ${udoc.uname}`); + } + userfileProgress.stop(); + logger.success('Files copied successfully. You can now remove collection `file` `fs.files` `fs.chunks` in the database.'); + return true; }, ]; diff --git a/packages/import-qduoj/handler.ts b/packages/import-qduoj/handler.ts index 8b9ba1a19..87cb8d04a 100644 --- a/packages/import-qduoj/handler.ts +++ b/packages/import-qduoj/handler.ts @@ -4,25 +4,20 @@ import os from 'os'; import fs from 'fs-extra'; import AdmZip from 'adm-zip'; import yaml from 'js-yaml'; -import { ObjectID } from 'mongodb'; import { ContentNode, LocalProblemConfig } from 'hydrooj'; -import { - Route, Handler, param, Types, -} from 'hydrooj/dist/service/server'; -import { BadRequestError } from 'hydrooj/dist/error'; -import { streamToBuffer } from 'hydrooj/dist/utils'; +import { Route, Handler } from 'hydrooj/dist/service/server'; +import * as storage from 'hydrooj/dist/service/storage'; +import { BadRequestError, ValidationError } from 'hydrooj/dist/error'; import { ProblemAdd } from 'hydrooj/dist/lib/ui'; -import * as file from 'hydrooj/dist/model/file'; import * as problem from 'hydrooj/dist/model/problem'; import { PERM } from 'hydrooj/dist/model/builtin'; -const processing = {}; fs.ensureDirSync('/tmp/hydro/import-qduoj'); class ImportQduojHandler extends Handler { - async ImportFromFile(domainId: string, id: string, zipfile: Buffer) { + async ImportFromFile(domainId: string, zipfile: string) { const zip = new AdmZip(zipfile); - const tmp = path.resolve(os.tmpdir(), 'hydro', 'import-qduoj', id); + const tmp = path.resolve(os.tmpdir(), 'hydro', 'import-qduoj', String.random(32)); await new Promise((resolve, reject) => { zip.extractAllToAsync(tmp, true, (err) => (err ? resolve() : reject(err))); }); @@ -79,15 +74,20 @@ class ImportQduojHandler extends Handler { }); } const pid = await problem.add(domainId, pdoc.display_id, pdoc.title, content, this.user._id, pdoc.tags); - const testdata = new AdmZip(); const config: LocalProblemConfig = { time: `${pdoc.time_limit}ms`, memory: `${pdoc.memory_limit}m`, subtasks: [], }; for (const tc of pdoc.test_case_score) { - testdata.addLocalFile(path.join(tmp, 'testcase', tc.input_name)); - testdata.addLocalFile(path.join(tmp, 'testcase', tc.output_name)); + await storage.put( + `problem/${domainId}/${pid}/testdata/${tc.input_name}`, + path.join(tmp, 'testcase', tc.input_name), + ); + await storage.put( + `problem/${domainId}/${pid}/testdata/${tc.output_name}`, + path.join(tmp, 'testcase', tc.output_name), + ); config.subtasks.push({ score: tc.score, cases: [{ @@ -96,43 +96,27 @@ class ImportQduojHandler extends Handler { }], }); } - testdata.addFile('config.yaml', Buffer.from(yaml.safeDump(config))); - const f = path.resolve(os.tmpdir(), 'hydro', `${Math.random()}.zip`); - await new Promise((resolve, reject) => { - testdata.writeZip(f, (err) => (err ? resolve() : reject(err))); - }); - try { - await problem.setTestdata(domainId, pid, f); - await problem.edit(domainId, pid, { html: true }); - } finally { - await fs.unlink(f); - } + await storage.put( + `problem/${domainId}/${pid}/testdata/config.yaml`, + Buffer.from(yaml.safeDump(config)), + ); + await problem.edit(domainId, pid, { html: true }); } } finally { await fs.remove(tmp); } } - @param('ufid', Types.ObjectID, true) - async get(domainId: string, ufid?: ObjectID) { - if (ufid) { - if (processing[ufid.toHexString()]) { - this.response.body = 'Processing'; - return; - } - const stat = await file.getMeta(ufid); - if (stat.size > 128 * 1024 * 1024) throw new BadRequestError('File too large'); - const stream = await file.get(ufid); - processing[ufid.toHexString()] = true; - try { - const buf = await streamToBuffer(stream); - await this.ImportFromFile(domainId, ufid.toHexString(), buf); - await file.del(ufid); - } catch (e) { - processing[ufid.toHexString()] = false; - throw e; - } - } else this.response.template = 'problem_import_qduoj.html'; + async get() { + this.response.template = 'problem_import.html'; + } + + async post({ domainId }) { + if (!this.request.files.file) throw new ValidationError('file'); + const stat = await fs.stat(this.request.files.file.path); + if (stat.size > 128 * 1024 * 1024) throw new BadRequestError('File too large'); + await this.ImportFromFile(domainId, this.request.files.file.path); + this.response.redirect = this.url('problem_main'); } }