diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index 01f821f4f..08bdf9709 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -33,6 +33,7 @@ "superagent": "^6.1.0", "supports-color": "^8.0.0", "systeminformation": "^4.31.1", + "terminal-kit": "^1.44.0", "wtfnode": "^0.8.4", "yargs": "^16.2.0" }, @@ -52,6 +53,7 @@ "@types/sockjs": "^0.3.32", "@types/superagent": "^4.1.8", "@types/supports-color": "^7.2.0", + "@types/terminal-kit": "^1.28.2", "@types/wtfnode": "^0.7.0", "@types/yargs": "^15.0.11" }, diff --git a/packages/hydrooj/src/entry/cli.ts b/packages/hydrooj/src/entry/cli.ts index c5b79a816..85b26eba8 100644 --- a/packages/hydrooj/src/entry/cli.ts +++ b/packages/hydrooj/src/entry/cli.ts @@ -8,6 +8,7 @@ import { import options from '../options'; import * as bus from '../service/bus'; import db from '../service/db'; +import storage from '../service/storage'; const COMMENTS = /((\/\/.*$)|(\/\*[\s\S]*?\*\/))/mg; const ARR = /=>.*$/mg; @@ -56,6 +57,15 @@ export async function load() { require('../options'); const opts = options(); await db.start(opts); + const modelSystem = require('../model/system'); + const [endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge] = modelSystem.getMany([ + 'file.endPoint', 'file.accessKey', 'file.secretKey', 'file.bucket', 'file.region', + 'file.endPointForUser', 'file.endPointForJudge', + ]); + const sopts = { + endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge, + }; + await storage.start(sopts); for (const i of builtinLib) require(`../lib/${i}`); await lib(pending, fail); require('../service/gridfs'); diff --git a/packages/hydrooj/src/entry/master.ts b/packages/hydrooj/src/entry/master.ts index 40f9e2357..77bddff8b 100644 --- a/packages/hydrooj/src/entry/master.ts +++ b/packages/hydrooj/src/entry/master.ts @@ -11,6 +11,7 @@ import { Logger } from '../logger'; import options from '../options'; import * as bus from '../service/bus'; import db from '../service/db'; +import storage from '../service/storage'; const logger = new Logger('entry/master'); const tmpdir = path.resolve(os.tmpdir(), 'hydro'); @@ -52,16 +53,25 @@ export async function load(call: Entry) { }); const opts = options(); await db.start(opts); - await require('../model/system').runConfig(); + const modelSystem = require('../model/system'); + await modelSystem.runConfig(); + const [endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge] = modelSystem.getMany([ + 'file.endPoint', 'file.accessKey', 'file.secretKey', 'file.bucket', 'file.region', + 'file.endPointForUser', 'file.endPointForJudge', + ]); + const sopts = { + endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge, + }; + await storage.start(sopts); require('../service/monitor'); for (const i of builtinModel) require(`../model/${i}`); - const modelSystem = require('../model/system'); const scripts = require('../upgrade'); let dbVer = (await modelSystem.get('db.ver')) ?? 0; const expected = scripts.length; while (dbVer < expected) { logger.info('Upgrading database: from %d to %d', dbVer, expected); - await scripts[dbVer](); + const result = await scripts[dbVer](); + if (!result) break; dbVer++; await modelSystem.set('db.ver', dbVer); } diff --git a/packages/hydrooj/src/entry/worker.ts b/packages/hydrooj/src/entry/worker.ts index 29f2611bd..71d51ab93 100644 --- a/packages/hydrooj/src/entry/worker.ts +++ b/packages/hydrooj/src/entry/worker.ts @@ -7,6 +7,7 @@ import { import options from '../options'; import * as bus from '../service/bus'; import db from '../service/db'; +import storage from '../service/storage'; export async function load() { const pending = global.addons; @@ -23,10 +24,15 @@ export async function load() { ]); const opts = options(); await db.start(opts); - await new Promise((resolve) => { - bus.once('database/config', resolve); - require('../model/system'); - }); + const modelSystem = require('../model/system'); + const [endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge] = modelSystem.getMany([ + 'file.endPoint', 'file.accessKey', 'file.secretKey', 'file.bucket', 'file.region', + 'file.endPointForUser', 'file.endPointForJudge', + ]); + const sopts = { + endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge, + }; + await storage.start(sopts); for (const i of builtinLib) require(`../lib/${i}`); await lib(pending, fail); require('../service/gridfs'); diff --git a/packages/hydrooj/src/handler/home.ts b/packages/hydrooj/src/handler/home.ts index 3e13aeb28..5f1dac755 100644 --- a/packages/hydrooj/src/handler/home.ts +++ b/packages/hydrooj/src/handler/home.ts @@ -7,6 +7,7 @@ import * as bus from '../service/bus'; import { Route, Connection, Handler, ConnectionHandler, param, Types, } from '../service/server'; +import storage from '../service/storage'; import * as misc from '../lib/misc'; import { md5 } from '../lib/crypto'; import * as mail from '../lib/mail'; @@ -15,7 +16,6 @@ import * as message from '../model/message'; import * as document from '../model/document'; import * as system from '../model/system'; import * as user from '../model/user'; -import * as file from '../model/file'; import * as setting from '../model/setting'; import * as domain from '../model/domain'; import * as discussion from '../model/discussion'; @@ -383,18 +383,18 @@ class HomeFileHandler extends Handler { ['Hydro', 'homepage'], ['domain_file', null], ]; - const ufdocs = await file.getMulti({ owner: this.user._id }).toArray(); + const ufdocs = await storage.list(`user/${this.user._id}/`); this.response.template = 'home_file.html'; this.response.body = { ufdocs, path }; } - @param('ufid', Types.ObjectID) - async postDelete(domainId: string, ufid: ObjectID) { - const ufdoc = await file.getMeta(ufid); - if (ufdoc.owner !== this.user._id) this.checkPriv(PRIV.PRIV_DELETE_FILE); + @param('ufid', Types.String) + async postDelete(domainId: string, target: string) { + if (!target.startsWith(`user/${this.user._id}`)) this.checkPriv(PRIV.PRIV_DELETE_FILE); else this.checkPriv(PRIV.PRIV_DELETE_FILE_SELF); - const result = await file.del(ufdoc._id); - if (result) await user.inc(this.user._id, 'usage', -ufdoc.size); + const { size } = await storage.getMeta(target); + await storage.del(target); + await user.inc(this.user._id, 'usage', -size); this.back(); } } diff --git a/packages/hydrooj/src/handler/import.syzoj.ts b/packages/hydrooj/src/handler/import.syzoj.ts index 39aa36ea0..96ab96e33 100644 --- a/packages/hydrooj/src/handler/import.syzoj.ts +++ b/packages/hydrooj/src/handler/import.syzoj.ts @@ -15,6 +15,7 @@ import { PERM } from '../model/builtin'; import { Route, Handler, Types, param, post, } from '../service/server'; +import storage from '../service/storage'; import { isPid } from '../lib/validator'; import download from '../lib/download'; @@ -89,14 +90,13 @@ class ProblemImportSYZOJHandler extends Handler { const urls = {}; if (r.body.error) return; for (const t of r.body.downloadInfo) urls[t.filename] = t.downloadUrl; - const zip = new AdmZip(); for (const f of result.body.testData) { const p = new PassThrough(); superagent.get(urls[f.filename]).pipe(p); // eslint-disable-next-line no-await-in-loop - zip.addFile(f.filename, await streamToBuffer(p)); + await storage.put(`problem/${domainId}/${docId}/testdata/${f.filename}`, p); } - await problem.setTestdata(domainId, docId, zip.toBuffer()); + // TODO additional_file if (judge) { await problem.edit(domainId, docId, { config: { @@ -182,7 +182,12 @@ class ProblemImportSYZOJHandler extends Handler { w.on('error', reject); r.pipe(w); }); - await problem.setTestdata(domainId, docId, file); + const zip = new AdmZip(file); + const entries = zip.getEntries(); + for (const entry of entries) { + // eslint-disable-next-line no-await-in-loop + await storage.put(`problem/${domainId}/${docId}/testdata/${entry.entryName}`, entry.getData()); + } await problem.edit(domainId, docId, { config: { time: `${p.time_limit}ms`, diff --git a/packages/hydrooj/src/handler/misc.ts b/packages/hydrooj/src/handler/misc.ts index ea78a5a96..7e628d221 100644 --- a/packages/hydrooj/src/handler/misc.ts +++ b/packages/hydrooj/src/handler/misc.ts @@ -1,29 +1,17 @@ /* eslint-disable camelcase */ import fs from 'fs'; -import { ObjectID } from 'mongodb'; +import { BadRequestError } from '../error'; import { PRIV } from '../model/builtin'; import * as system from '../model/system'; -import * as file from '../model/file'; import * as user from '../model/user'; import db from '../service/db'; import { Route, Handler, Types, param, } from '../service/server'; -import { BadRequestError } from '../error'; +import storage from '../service/storage'; const coll = db.collection('status'); -class FileDownloadHandler extends Handler { - @param('fileId', Types.ObjectID) - @param('secret', Types.String) - @param('name', Types.String, true) - async get(domainId: string, fileId: ObjectID, secret: string, name?: string) { - if (name) name = Buffer.from(name.replace(/~/g, '/'), 'base64').toString(); - const doc = await file.getWithSecret(fileId, secret); - this.response.attachment(name || fileId.toHexString(), doc); - } -} - class FileUploadHandler extends Handler { async getQuota() { let quota = system.get('user.quota'); @@ -44,15 +32,14 @@ class FileUploadHandler extends Handler { if (!this.request.files.file) throw new BadRequestError(); const quota = await this.getQuota(); const lfdoc = await fs.promises.stat(this.request.files.file.path); - let ufid: ObjectID; + const ufid = `user/${this.user._id}/${title}`; const udoc = await user.inc(this.user._id, 'usage', lfdoc.size); try { - ufid = await file.add(this.request.files.file.path, title, this.user._id); + await storage.put(ufid, this.request.files.file.path); } catch (e) { await user.inc(this.user._id, 'usage', -lfdoc.size); throw e; } - const fdoc = await file.getMeta(ufid); this.response.template = 'fs_upload.html'; if (redirect) { this.response.redirect = redirect.includes('?') @@ -60,7 +47,7 @@ class FileUploadHandler extends Handler { : `${redirect}?ufid=${ufid}`; } this.response.body = { - fdoc, ufid, usage: udoc.usage, quota, + ufid, usage: udoc.usage, quota, }; } } @@ -138,8 +125,6 @@ class SwitchLanguageHandler extends Handler { } export async function apply() { - Route('file_download', '/fs/:fileId/:secret', FileDownloadHandler); - Route('file_download_with_name', '/fs/:fileId/:name/:secret', FileDownloadHandler); Route('file_upload', '/fs/upload', FileUploadHandler, PRIV.PRIV_CREATE_FILE); Route('status', '/status', StatusHandler); Route('status_update', '/status/update', StatusUpdateHandler); diff --git a/packages/hydrooj/src/handler/problem.ts b/packages/hydrooj/src/handler/problem.ts index 7e3103447..f2c753325 100644 --- a/packages/hydrooj/src/handler/problem.ts +++ b/packages/hydrooj/src/handler/problem.ts @@ -1,11 +1,11 @@ import { isSafeInteger, flatten, pick } from 'lodash'; import yaml from 'js-yaml'; +import fs from 'fs-extra'; import { FilterQuery, ObjectID } from 'mongodb'; import AdmZip from 'adm-zip'; import { - NoProblemError, ProblemDataNotFoundError, BadRequestError, + NoProblemError, BadRequestError, PermissionError, SolutionNotFoundError, ProblemNotFoundError, ValidationError, - PermissionError, } from '../error'; import { streamToBuffer } from '../utils'; import { @@ -14,13 +14,13 @@ import { import paginate from '../lib/paginate'; import { isTitle, isContent, isPid } from '../lib/validator'; import { ProblemAdd } from '../lib/ui'; -import * as file from '../model/file'; import * as problem from '../model/problem'; import * as record from '../model/record'; import * as domain from '../model/domain'; import * as user from '../model/user'; import * as solution from '../model/solution'; import { PERM, PRIV, CONSTANT } from '../model/builtin'; +import storage from '../service/storage'; import * as bus from '../service/bus'; import { Route, Connection, Handler, ConnectionHandler, Types, param, post, route, @@ -239,13 +239,15 @@ export class ProblemExportHandler extends ProblemDetailHandler { const hasPerm = (this.user._id === this.pdoc.owner && this.user.hasPerm(PERM.PERM_READ_PROBLEM_DATA_SELF)) || this.user.hasPerm(PERM.PERM_READ_PROBLEM_DATA_SELF); const pdoc = pick(this.pdoc, ['pid', 'acMsg', 'content', 'config', 'title', 'html', 'tag', 'category']); - let zip: AdmZip; + const zip = new AdmZip(); if (hasPerm) { - if (this.pdoc.data instanceof ObjectID) { - const buf = await streamToBuffer(await file.get(this.pdoc.data)); - zip = new AdmZip(buf); - } else zip = new AdmZip(); - } else zip = new AdmZip(); + const files = await storage.list(`problem/${this.domainId}/${this.pdoc.docId}/`, true); + for (const file of files) { + // eslint-disable-next-line no-await-in-loop + const buf = await streamToBuffer(storage.get(`${file.prefix}${file.name}`)); + zip.addFile(file.name, buf); + } + } zip.addFile('problem.json', Buffer.from(JSON.stringify(pdoc))); this.response.attachment(`${this.pdoc.title}.zip`, zip.toBuffer()); } @@ -447,59 +449,67 @@ export class ProblemEditHandler extends ProblemManageHandler { } } -export class ProblemDataUploadHandler extends ProblemManageHandler { - async get() { - if (this.pdoc.data instanceof ObjectID) { - const f = await file.getMeta(this.pdoc.data); - this.response.body.md5 = f.md5; - } else if (this.pdoc.data) { - this.response.body.from = this.pdoc.data; - } - this.response.template = 'problem_upload.html'; +export class ProblemFilesHandler extends ProblemDetailHandler { + @param('testdata', Types.Boolean) + @param('additional_file', Types.Boolean) + async get(domainId: string, getTestdata = true, getAdditionalFile = true) { + const [testdata, additional_file] = await Promise.all([ + getTestdata + ? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/testdata`, true) + : [], + getAdditionalFile + ? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file`, true) + : [], + ]); + this.response.body.testdata = testdata; + this.response.body.additional_file = additional_file; + this.response.template = 'problem_files.html'; } - async post({ domainId }) { - if (!this.request.files.file) throw new ValidationError('file'); - this.pdoc = await problem.setTestdata( - domainId, this.pdoc.docId, this.request.files.file.path, - ); - if (this.pdoc.data instanceof ObjectID) { - const f = await file.getMeta(this.pdoc.data); - this.response.body.md5 = f.md5; - } - this.response.template = 'problem_upload.html'; - } -} - -export class ProblemDataDownloadHandler extends ProblemDetailHandler { - async get({ pid }) { - if (!this.user.hasPriv(PRIV.PRIV_JUDGE)) { + @post('files', Types.Array) + @post('type', Types.Range(['testdata', 'additional_file']), true) + async postGetLinks(domainId: string, files: string[], type = 'testdata') { + const isJudge = this.user.hasPriv(PRIV.PRIV_JUDGE); + if (type === 'testdata' && !isJudge) { if (this.user._id !== this.pdoc.owner) { this.checkPerm(PERM.PERM_READ_PROBLEM_DATA); } else this.checkPerm(PERM.PERM_READ_PROBLEM_DATA_SELF); } - if (this.pdoc.data instanceof ObjectID) { - this.response.redirect = await file.url(this.pdoc.data, `${this.pdoc.title}.zip`); - } else if (this.pdoc.data) { - if (!this.pdoc.data.host) { - this.response.redirect = this.url('problem_data', { - domainId: this.pdoc.data.domainId, - pid: this.pdoc.data.pid, - }); - } else { - const [scheme, raw] = this.pdoc.data.host.split('//'); - const args = JSON.parse(Buffer.from(raw, 'base64').toString()); - if (scheme === 'hydro') { - const [secure, host, port, domainId, id] = args; - this.response.redirect = `http${secure ? 's' : ''}://${host}:${port}/d/${domainId}/p/${id}/data`; - } else if (scheme === 'syzoj') { - const [secure, host, port, id] = args; - this.response.redirect = `http${secure ? 's' : ''}://${host}:${port}/problem/${id}/testdata/download`; - } else { - throw new ProblemDataNotFoundError(pid); - } + const links = {}; + for (const file of files) { + // eslint-disable-next-line no-await-in-loop + links[file] = await storage.signDownloadLink( + `problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${file}`, + file, false, isJudge ? 'judge' : 'user', + ); + } + this.response.body.links = links; + } + + @post('filename', Types.String) + @post('type', Types.String, true) + async postUploadFile(domainId: string, filename: string, type = 'testdata') { + if (!this.request.files.file) throw new ValidationError('file'); + if (this.pdoc.owner !== this.user._id) this.checkPerm(PERM.PERM_EDIT_PROBLEM); + else this.checkPerm(PERM.PERM_EDIT_PROBLEM_SELF); + if (filename === 'testdata.zip') { + const zip = new AdmZip(this.request.files.file.path); + const entries = zip.getEntries(); + for (const entry of entries) { + // eslint-disable-next-line no-await-in-loop + await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${entry.name}`, entry.getData()); } - } else throw new ProblemDataNotFoundError(pid); + } else await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${filename}`, this.request.files.file.path); + this.back(); + } + + @post('files', Types.Array) + @post('type', Types.Range(['testdata', 'additional_file']), true) + async postDeleteFiles(domainId: string, files: string[], type = 'testdata') { + if (this.pdoc.owner !== this.user._id) this.checkPerm(PERM.PERM_EDIT_PROBLEM); + else this.checkPerm(PERM.PERM_EDIT_PROBLEM_SELF); + await storage.del(files.map((file) => `problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${file}`)); + this.back(); } } @@ -611,23 +621,18 @@ export class ProblemSolutionHandler extends ProblemDetailHandler { export class ProblemSolutionRawHandler extends ProblemDetailHandler { @param('psid', Types.ObjectID) - async get(domainId: string, psid: ObjectID) { + @route('psrid', Types.ObjectID, true) + async get(domainId: string, psid: ObjectID, psrid?: ObjectID) { this.checkPerm(PERM.PERM_VIEW_PROBLEM_SOLUTION); - const psdoc = await solution.get(domainId, psid); + if (psrid) { + const [psdoc, psrdoc] = await solution.getReply(domainId, psid, psrid); + if ((!psdoc) || psdoc.pid !== this.pdoc.docId) throw new SolutionNotFoundError(psid, psrid); + this.response.body = psrdoc.content; + } else { + const psdoc = await solution.get(domainId, psid); + this.response.body = psdoc.content; + } this.response.type = 'text/markdown'; - this.response.body = psdoc.content; - } -} - -export class ProblemSolutionReplyRawHandler extends ProblemDetailHandler { - @param('psid', Types.ObjectID) - @param('psrid', Types.ObjectID) - async get(domainId: string, psid: ObjectID, psrid: ObjectID) { - this.checkPerm(PERM.PERM_VIEW_PROBLEM_SOLUTION); - const [psdoc, psrdoc] = await solution.getReply(domainId, psid, psrid); - if ((!psdoc) || psdoc.pid !== this.pdoc.docId) throw new SolutionNotFoundError(psid, psrid); - this.response.type = 'text/markdown'; - this.response.body = psrdoc.content; } } @@ -662,21 +667,26 @@ export class ProblemCreateHandler extends Handler { } export class ProblemImportHandler extends Handler { - @param('ufid', Types.ObjectID, true) - async get(domainId: string, ufid?: ObjectID) { - if (ufid) { - const stat = await file.getMeta(ufid); - if (stat.size > 128 * 1024 * 1024) throw new BadRequestError('File too large'); - const stream = await file.get(ufid); - const buf = await streamToBuffer(stream); - const zip = new AdmZip(buf); - const pdoc = JSON.parse(zip.getEntry('problem.json').getData().toString()); - const pid = await problem.add(domainId, pdoc.pid, pdoc.title, pdoc.content, this.user._id, pdoc.tags, pdoc.category); - await problem.setTestdata(domainId, pid, buf); - await problem.edit(domainId, pid, { html: pdoc.html }); - this.response.redirect = this.url('problem_detail', { pid }); - await file.del(ufid); - } else this.response.template = 'problem_import.html'; + async get() { + this.response.template = 'problem_import.html'; + } + + async post(domainId: string) { + if (!this.request.files.file) throw new ValidationError('file'); + const stat = await fs.stat(this.request.files.file.path); + if (stat.size > 128 * 1024 * 1024) throw new BadRequestError('File too large'); + const zip = new AdmZip(this.request.files.file.path); + const pdoc = JSON.parse(zip.getEntry('problem.json').getData().toString()); + const pid = await problem.add(domainId, pdoc.pid, pdoc.title, pdoc.content, this.user._id, pdoc.tags, pdoc.category); + const entries = zip.getEntries(); + for (const entry of entries) { + if (entry.name.startsWith('testdata/') || entry.name.startsWith('additional_file')) { + // eslint-disable-next-line no-await-in-loop + await storage.put(`problem/${domainId}/${pid}/${entry.name}`, entry.getData()); + } + } + await problem.edit(domainId, pid, { html: pdoc.html }); + this.response.redirect = this.url('problem_detail', { pid }); } } @@ -692,11 +702,10 @@ export async function apply() { Route('problem_settings', '/p/:pid/settings', ProblemSettingsHandler); Route('problem_statistics', '/p/:pid/statistics', ProblemStatisticsHandler); Route('problem_edit', '/p/:pid/edit', ProblemEditHandler); - Route('problem_upload', '/p/:pid/upload', ProblemDataUploadHandler); - Route('problem_data', '/p/:pid/data', ProblemDataDownloadHandler); + Route('problem_files', '/p/:pid/files', ProblemFilesHandler); Route('problem_solution', '/p/:pid/solution', ProblemSolutionHandler); Route('problem_solution_raw', '/p/:pid/solution/:psid/raw', ProblemSolutionRawHandler); - Route('problem_solution_reply_raw', '/p/:pid/solution/:psid/:psrid/raw', ProblemSolutionReplyRawHandler); + Route('problem_solution_reply_raw', '/p/:pid/solution/:psid/:psrid/raw', ProblemSolutionRawHandler); Route('problem_create', '/problem/create', ProblemCreateHandler, PERM.PERM_CREATE_PROBLEM); Route('problem_import', '/problem/import', ProblemImportHandler, PERM.PERM_CREATE_PROBLEM); Connection('problem_pretest_conn', '/conn/pretest', ProblemPretestConnectionHandler); diff --git a/packages/hydrooj/src/handler/record.ts b/packages/hydrooj/src/handler/record.ts index 573e00bf0..47e01377c 100644 --- a/packages/hydrooj/src/handler/record.ts +++ b/packages/hydrooj/src/handler/record.ts @@ -2,7 +2,6 @@ import { FilterQuery, ObjectID } from 'mongodb'; import { PermissionError, RecordNotFoundError } from '../error'; import { PERM, CONSTANT, STATUS } from '../model/builtin'; import * as problem from '../model/problem'; -import * as file from '../model/file'; import * as record from '../model/record'; import * as contest from '../model/contest'; import * as user from '../model/user'; @@ -12,7 +11,6 @@ import { Route, Handler, Connection, ConnectionHandler, Types, param, } from '../service/server'; import { Rdoc } from '../interface'; -import { streamToBuffer } from '../utils'; const RecordHandler = contest.ContestHandlerMixin(Handler); @@ -132,35 +130,6 @@ class RecordDetailHandler extends RecordHandler { } } -class RecordHackHandler extends Handler { - @param('rid', Types.ObjectID) - @param('ufid', Types.ObjectID) - async get(domainId: string, rid: ObjectID, ufid?: ObjectID) { - if (ufid) { - const stream = await file.get(ufid); - const buf = await streamToBuffer(stream); - const input = buf.toString(); - await this.post(domainId, rid, input); - } else this.response.template = 'record_hack.html'; - } - - @param('rid', Types.ObjectID) - @param('input', Types.String) - async post(domainId: string, rid: ObjectID, input: string) { - const rdoc = await record.get(domainId, rid); - if (!rdoc) throw new RecordNotFoundError(domainId, rid); - const newRid = await record.add( - domainId, rdoc.pid, this.user._id, - rdoc.lang, rdoc.code, true, - { hack: input }, - ); - bus.boardcast('record/change', rdoc); - this.response.body = { rid: newRid }; - this.response.redirect = this.url('record_detail', { rid: newRid }); - // TODO handle hack(modify testdata, rejudge problem) - } -} - /** * @deprecated * use RecordDetailHandler.postRejudge instead. @@ -261,7 +230,6 @@ class RecordDetailConnectionHandler extends contest.ContestHandlerMixin(Connecti export async function apply() { Route('record_main', '/record', RecordListHandler); Route('record_detail', '/record/:rid', RecordDetailHandler); - Route('record_hack', '/record/:rid/hack', RecordHackHandler); Route('record_rejudge', '/record/:rid/rejudge', RecordRejudgeHandler); Connection('record_conn', '/record-conn', RecordMainConnectionHandler); Connection('record_detail_conn', '/record-detail-conn', RecordDetailConnectionHandler); diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 00d7d0761..440a7079e 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -12,6 +12,13 @@ export interface System { } export interface SystemKeys { + 'file.endPoint': string, + 'file.accessKey': string, + 'file.secretKey': string, + 'file.bucket': string, + 'file.region': string, + 'file.endPointForUser': string, + 'file.endPointForJudge': string, 'smtp.user': string, 'smtp.from': string, 'smtp.pass': string, @@ -515,7 +522,6 @@ export interface Model { discussion: typeof import('./model/discussion'), document: typeof import('./model/document'), domain: typeof import('./model/domain'), - file: typeof import('./model/file'), message: typeof import('./model/message'), opcount: typeof import('./model/opcount'), problem: typeof import('./model/problem'), diff --git a/packages/hydrooj/src/loader.ts b/packages/hydrooj/src/loader.ts index a436c9cb0..2468f2a4b 100644 --- a/packages/hydrooj/src/loader.ts +++ b/packages/hydrooj/src/loader.ts @@ -10,7 +10,7 @@ import wtfnode from 'wtfnode'; import fs from 'fs-extra'; import { argv } from 'yargs'; import AdmZip from 'adm-zip'; -import { } from './utils'; +import './utils'; export * from './interface'; diff --git a/packages/hydrooj/src/model/file.ts b/packages/hydrooj/src/model/file.ts deleted file mode 100644 index 5dd20a55b..000000000 --- a/packages/hydrooj/src/model/file.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { ObjectID, Collection } from 'mongodb'; -import * as fs from 'fs'; -import { Dictionary } from 'lodash'; -import { ForbiddenError, NotFoundError } from '../error'; -import { Ufdoc } from '../interface'; -import db from '../service/db'; -import * as bus from '../service/bus'; -import gridfs from '../service/gridfs'; -import hash from '../lib/hash.hydro'; -import { bufferToStream } from '../utils'; - -const coll: Collection = db.collection('file'); -const collFile = db.collection('fs.files'); -const collChunk = db.collection('fs.chunks'); - -function _timestamp() { - return Math.floor(Number(new Date()) / 1000000); -} - -export async function add( - f: NodeJS.ReadableStream | Buffer | string, filename: string, - owner = 1, meta = {}, -) { - let stream: NodeJS.ReadableStream; - if (typeof f === 'string') stream = fs.createReadStream(f); - else if (f instanceof Buffer) stream = bufferToStream(f); - else stream = f; - const w = gridfs.openUploadStream(filename); - await coll.insertOne({ - ...meta, - _id: w.id, - secret: String.random(32), - owner, - filename, - }); - await new Promise((resolve, reject) => { - w.on('error', reject); - w.on('finish', resolve); - stream.pipe(w); - }); - const c = await gridfs.find({ _id: w.id }).toArray(); - await coll.updateOne({ _id: w.id }, { $set: { md5: c[0].md5, size: c[0].length } }); - return w.id; -} - -export function del(_id: ObjectID) { - return Promise.all([ - coll.deleteOne({ _id }), - collFile.deleteOne({ _id }), - collChunk.deleteMany({ files_id: _id }), - ]); -} - -export async function getWithSecret(_id: ObjectID, secret: string, reject?: Function) { - const file = await coll.findOne({ _id }); - if (!file) throw new NotFoundError(_id); - const timestamp = _timestamp(); - if (hash(file.secret, timestamp.toString()) !== secret) { - if (hash(file.secret, (timestamp - 1).toString()) !== secret) { - throw new ForbiddenError(); - } - } - const stream = gridfs.openDownloadStream(_id); - stream.on('error', (err) => { - console.error(err); - if (reject) reject(err); - }); - return stream; -} - -export async function get(_id: ObjectID, reject?: Function) { - const file = await coll.findOne({ _id }); - if (!file) throw new NotFoundError(_id); - const stream = gridfs.openDownloadStream(_id); - stream.on('error', (err) => { - console.error(err); - if (reject) reject(err); - }); - return stream; -} - -export function getMeta(_id: ObjectID): Promise { - return coll.findOne({ _id }); -} - -export async function getMetaDict(ufids: ObjectID[]): Promise> { - const r = {}; - ufids = Array.from(new Set(ufids)); - const ufdocs = await coll.find({ _id: { $in: ufids } }).toArray(); - for (const ufdoc of ufdocs) { - r[ufdoc._id.toHexString()] = ufdoc; - } - return r; -} - -export function getMulti(query: any) { - return coll.find(query); -} - -async function _url(_id: ObjectID, name?: string) { - const file = await coll.findOne({ _id }); - const secret = hash(file.secret, _timestamp().toString()); - if (name) return `/fs/${_id}/${Buffer.from(name).toString('base64').replace(/\//g, '~')}/${secret}`; - return `/fs/${_id}/${secret}`; -} - -function __url(file: Ufdoc, name?: string) { - const secret = hash(file.secret, _timestamp().toString()); - if (name) return `/fs/${file._id}/${Buffer.from(name).toString('base64').replace(/\//g, '~')}/${secret}`; - return `/fs/${file._id}/${secret}`; -} - -export function url(_id: ObjectID, name?: string): Promise; -export function url(file: Ufdoc, name: string): string; -export function url(arg0: ObjectID | Ufdoc, name?: string) { - if (arg0 instanceof ObjectID) return _url(arg0, name); - return __url(arg0, name); -} - -function ensureIndexes() { - return Promise.all([ - collChunk.createIndex({ files_id: 1, n: 1 }, { unique: true }), - collFile.createIndex({ md5: 1 }), - ]); -} - -bus.once('app/started', ensureIndexes); -global.Hydro.model.file = { - add, get, del, getMeta, getWithSecret, getMulti, getMetaDict, url, -}; diff --git a/packages/hydrooj/src/model/problem.ts b/packages/hydrooj/src/model/problem.ts index ab97f53c4..9d3c8088a 100644 --- a/packages/hydrooj/src/model/problem.ts +++ b/packages/hydrooj/src/model/problem.ts @@ -1,15 +1,12 @@ import { ObjectID, FilterQuery } from 'mongodb'; import { Dictionary } from 'lodash'; -import { safeLoad } from 'js-yaml'; import { STATUS } from './builtin'; -import * as file from './file'; import * as document from './document'; import * as domain from './domain'; import { ProblemStatusDoc, ProblemDataSource, Pdict } from '../interface'; import { Content } from '../loader'; import { NumberKeys, Projection } from '../typeutils'; import { ProblemNotFoundError } from '../error'; -import * as testdataConfig from '../lib/testdataConfig'; export interface Pdoc { _id: ObjectID @@ -188,14 +185,6 @@ export function setStar(domainId: string, pid: number, uid: number, star: boolea return document.setStatus(domainId, document.TYPE_PROBLEM, pid, uid, { star }); } -export async function setTestdata(domainId: string, _id: number, f: string | Buffer) { - const pdoc = await get(domainId, _id); - const config = await testdataConfig.readConfig(f); - const id = await file.add(f, 'data.zip', 1); - if (pdoc.data instanceof ObjectID) file.del(pdoc.data); - return await edit(domainId, _id, { data: id, config: safeLoad(config) as any }); -} - global.Hydro.model.problem = { Pdoc, @@ -218,6 +207,5 @@ global.Hydro.model.problem = { getListStatus, getMultiStatus, setStar, - setTestdata, updateStatus, }; diff --git a/packages/hydrooj/src/model/setting.ts b/packages/hydrooj/src/model/setting.ts index f32916bc4..376962f85 100644 --- a/packages/hydrooj/src/model/setting.ts +++ b/packages/hydrooj/src/model/setting.ts @@ -120,6 +120,13 @@ DomainUserSetting( ); SystemSetting( + Setting('setting_file', 'file.endPoint', null, null, 'text', 'Storage engine endPoint'), + Setting('setting_file', 'file.accessKey', null, null, 'text', 'Storage engine accessKey'), + Setting('setting_file', 'file.secretKey', null, null, 'password', 'Storage engine secret', null, FLAG_SECRET), + Setting('setting_file', 'file.bucket', null, 'hydro', 'text', 'Storage engine bucket'), + Setting('setting_file', 'file.region', null, 'us-east-1', 'text', 'Storage engine region'), + Setting('setting_file', 'file.endPointForUser', null, null, 'text', 'EndPoint for user'), + Setting('setting_file', 'file.endPointForJudge', null, null, 'text', 'EndPoint for judge'), Setting('setting_smtp', 'smtp.user', null, null, 'text', 'SMTP Username'), Setting('setting_smtp', 'smtp.pass', null, null, 'password', 'SMTP Password', null, FLAG_SECRET), Setting('setting_smtp', 'smtp.host', null, null, 'text', 'SMTP Server Host'), diff --git a/packages/hydrooj/src/model/system.ts b/packages/hydrooj/src/model/system.ts index 6a3c95ef6..d37a99cfd 100644 --- a/packages/hydrooj/src/model/system.ts +++ b/packages/hydrooj/src/model/system.ts @@ -60,7 +60,7 @@ export async function runConfig() { for (const setting of SYSTEM_SETTINGS) { if (setting.value) cache[setting.key] = setting.value; } - const config = await coll.find({}).toArray(); + const config = await coll.find().toArray(); for (const i of config) cache[i._id] = i.value; bus.emit('database/config'); } diff --git a/packages/hydrooj/src/service/storage.ts b/packages/hydrooj/src/service/storage.ts index eea40e3fa..d1dc206c1 100644 --- a/packages/hydrooj/src/service/storage.ts +++ b/packages/hydrooj/src/service/storage.ts @@ -1,41 +1,88 @@ import { Readable } from 'stream'; -import Minio, { BucketItem, ItemBucketMetadata } from 'minio'; +import { URL } from 'url'; +import { Client, BucketItem, ItemBucketMetadata } from 'minio'; interface StorageOptions { - host?: string, - port?: number, - useSSL?: boolean, - accessKey?: string, - secretKey?: string, - bucket?: string, - region?: string, + endPoint: string; + accessKey: string; + secretKey: string; + bucket: string; + region?: string; + endPointForUser?: string; + endPointForJudge?: string; } -const defaultConfig: StorageOptions = { - host: 'play.min.io', - port: 9000, - useSSL: true, - accessKey: 'Q3AM3UQ867SPQQA43P2F', - secretKey: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG', - bucket: 'hydro', - region: 'us-east-1', -}; +interface MinioEndpointConfig { + endPoint: string; + port: number; + useSSL: boolean; +} + +function parseMainEndpointUrl(endpoint: string): MinioEndpointConfig { + const url = new URL(endpoint); + const result: Partial = {}; + if (url.pathname !== '/') throw new Error('Main MinIO endpoint URL of a sub-directory is not supported.'); + if (url.username || url.password || url.hash || url.search) { + throw new Error('Authorization, search parameters and hash are not supported for main MinIO endpoint URL.'); + } + if (url.protocol === 'http:') result.useSSL = false; + else if (url.protocol === 'https:') result.useSSL = true; + else { + throw new Error( + `Invalid protocol "${url.protocol}" for main MinIO endpoint URL. Only HTTP and HTTPS are supported.`, + ); + } + result.endPoint = url.hostname; + result.port = url.port ? Number(url.port) : result.useSSL ? 443 : 80; + return result as MinioEndpointConfig; +} +function parseAlternativeEndpointUrl(endpoint: string): (originalUrl: string) => string { + if (!endpoint) return (originalUrl) => originalUrl; + const url = new URL(endpoint); + if (url.hash || url.search) throw new Error('Search parameters and hash are not supported for alternative MinIO endpoint URL.'); + if (!url.pathname.endsWith('/')) throw new Error("Alternative MinIO endpoint URL's pathname must ends with '/'."); + return (originalUrl) => { + const parsedOriginUrl = new URL(originalUrl); + return new URL(parsedOriginUrl.pathname.slice(1) + parsedOriginUrl.search + parsedOriginUrl.hash, url).toString(); + }; +} +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent +function encodeRFC5987ValueChars(str: string) { + return ( + encodeURIComponent(str) + // Note that although RFC3986 reserves "!", RFC5987 does not, + // so we do not need to escape it + .replace(/['()]/g, escape) // i.e., %27 %28 %29 + .replace(/\*/g, '%2A') + // The following are not required for percent-encoding per RFC5987, + // so we can allow for a little better readability over the wire: |`^ + .replace(/%(?:7C|60|5E)/g, unescape) + ); +} class StorageService { - public client: Minio.Client; + public client: Client; + public error = ''; private opts: StorageOptions; + private replaceWithAlternativeUrlFor: Record<'user' | 'judge', (originalUrl: string) => string>; async start(opts: StorageOptions) { - this.opts = { ...defaultConfig, ...opts }; - this.client = new Minio.Client({ - endPoint: this.opts.host, - port: this.opts.port, - useSSL: this.opts.useSSL, - accessKey: this.opts.accessKey, - secretKey: this.opts.secretKey, - }); - const exists = await this.client.bucketExists(this.opts.bucket); - if (!exists) await this.client.makeBucket(this.opts.bucket, this.opts.region); + try { + this.opts = opts; + this.client = new Client({ + ...parseMainEndpointUrl(this.opts.endPoint), + accessKey: this.opts.accessKey, + secretKey: this.opts.secretKey, + }); + const exists = await this.client.bucketExists(this.opts.bucket); + if (!exists) await this.client.makeBucket(this.opts.bucket, this.opts.region); + this.replaceWithAlternativeUrlFor = { + user: parseAlternativeEndpointUrl(this.opts.endPointForUser), + judge: parseAlternativeEndpointUrl(this.opts.endPointForJudge), + }; + } catch (e) { + this.error = e.toString(); + } } async put(target: string, file: string | Buffer | Readable, meta: ItemBucketMetadata = {}) { @@ -56,11 +103,33 @@ class StorageService { const stream = this.client.listObjects(this.opts.bucket, target, recursive); return await new Promise((resolve, reject) => { const results: BucketItem[] = []; - stream.on('data', (result) => results.push(result)); + stream.on('data', (result) => results.push({ + ...result, + prefix: target, + name: result.name.split(target)[1], + })); stream.on('end', () => resolve(results)); stream.on('error', reject); }); } + + async getMeta(target: string) { + const result = await this.client.statObject(this.opts.bucket, target); + return { ...result.metaData, ...result }; + } + + async signDownloadLink(target: string, filename?: string, noExpire = false, useAlternativeEndpointFor?: 'user' | 'judge'): Promise { + const url = await this.client.presignedGetObject( + this.opts.bucket, + target, + noExpire ? 24 * 60 * 60 * 7 : 30 * 60, + filename + ? { 'response-content-disposition': `attachment; filename="${encodeRFC5987ValueChars(filename)}"` } + : {}, + ); + if (useAlternativeEndpointFor) return this.replaceWithAlternativeUrlFor[useAlternativeEndpointFor](url); + return url; + } } const service = new StorageService(); diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index e7fe694a5..cdf7e0f24 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -1,9 +1,17 @@ /* eslint-disable no-await-in-loop */ +import { ObjectID, Collection } from 'mongodb'; +import { terminal } from 'terminal-kit'; +import { Ufdoc } from './interface'; +import { Logger } from './logger'; +import gridfs from './service/gridfs'; +import storage from './service/storage'; +import db from './service/db'; +import * as problem from './model/problem'; import * as domain from './model/domain'; import * as document from './model/document'; -import db from './service/db'; -type UpgradeScript = () => Promise; +const logger = new Logger('upgrade'); +type UpgradeScript = () => Promise; const scripts: UpgradeScript[] = [ // Init @@ -11,6 +19,7 @@ const scripts: UpgradeScript[] = [ const ddoc = await domain.get('system'); if (!ddoc) await domain.add('system', 0, 'Hydro', 'Hydro System'); // TODO discussion node? + return true; }, // Add history column to ddoc,drdoc,psdoc async function _1_2() { @@ -29,9 +38,68 @@ const scripts: UpgradeScript[] = [ }); if (bulk.length) await bulk.execute(); } + return true; }, async function _2_3() { await db.collection('document').updateMany({ pid: /^\d+$/i }, { $unset: { pid: '' } }); + return true; + }, + async function _3_4() { + if (storage.error) { + logger.error('Cannot upgrade. Please change storage config.'); + return false; + } + + const coll: Collection = db.collection('file'); + const collFile = db.collection('fs.files'); + const collChunk = db.collection('fs.chunks'); + function del(_id: ObjectID) { + return Promise.all([ + coll.deleteOne({ _id }), + collFile.deleteOne({ _id }), + collChunk.deleteMany({ files_id: _id }), + ]); + } + async function get(_id: ObjectID, reject?: Function) { + const stream = gridfs.openDownloadStream(_id); + stream.on('error', (err) => { + console.error(err); + if (reject) reject(err); + }); + return stream; + } + terminal.clear('Upgrade'); + logger.info(''); + // Problem file and User file + // @ts-ignore + const progress = terminal.progressBar({ title: 'Changing storage engine. This may take a long time.', y: 1 }); + const ddocs = await domain.getMulti().toArray(); + const totalProgress = terminal.progressBar({ + // @ts-ignore + items: ddocs.length, title: 'Domain', y: 3, eta: true, + }); + for (let i = 0; i < ddocs.length; i++) { + const ddoc = ddocs[i]; + totalProgress.startItem(ddoc._id); + const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['docId', 'data', 'title']).toArray(); + // @ts-ignore + const domainProgress = terminal.progressBar({ + // @ts-ignore + items: pdocs.length, title: 'Problems', y: 4, eta: true, + }); + for (let j = 0; j < pdocs.length; j++) { + const pdoc = pdocs[j]; + domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`); + // TODO copy file + domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`); + } + domainProgress.stop(); + totalProgress.itemDone(ddoc._id); + } + totalProgress.stop(); + // TODO userfile + progress.stop(); + return false; }, ];