core: testdata(zip-> multi files)

packages/hydrooj/package.json

@@ -33,6 +33,7 @@
         "superagent": "^6.1.0",
         "supports-color": "^8.0.0",
         "systeminformation": "^4.31.1",
+        "terminal-kit": "^1.44.0",
         "wtfnode": "^0.8.4",
         "yargs": "^16.2.0"
     },
@@ -52,6 +53,7 @@
         "@types/sockjs": "^0.3.32",
         "@types/superagent": "^4.1.8",
         "@types/supports-color": "^7.2.0",
+        "@types/terminal-kit": "^1.28.2",
         "@types/wtfnode": "^0.7.0",
         "@types/yargs": "^15.0.11"
     },

packages/hydrooj/src/entry/cli.ts

@@ -8,6 +8,7 @@ import {
 import options from '../options';
 import * as bus from '../service/bus';
 import db from '../service/db';
+import storage from '../service/storage';
 
 const COMMENTS = /((\/\/.*$)|(\/\*[\s\S]*?\*\/))/mg;
 const ARR = /=>.*$/mg;
@@ -56,6 +57,15 @@ export async function load() {
     require('../options');
     const opts = options();
     await db.start(opts);
+    const modelSystem = require('../model/system');
+    const [endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge] = modelSystem.getMany([
+        'file.endPoint', 'file.accessKey', 'file.secretKey', 'file.bucket', 'file.region',
+        'file.endPointForUser', 'file.endPointForJudge',
+    ]);
+    const sopts = {
+        endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge,
+    };
+    await storage.start(sopts);
     for (const i of builtinLib) require(`../lib/${i}`);
     await lib(pending, fail);
     require('../service/gridfs');

packages/hydrooj/src/entry/master.ts

@@ -11,6 +11,7 @@ import { Logger } from '../logger';
 import options from '../options';
 import * as bus from '../service/bus';
 import db from '../service/db';
+import storage from '../service/storage';
 
 const logger = new Logger('entry/master');
 const tmpdir = path.resolve(os.tmpdir(), 'hydro');
@@ -52,16 +53,25 @@ export async function load(call: Entry) {
     });
     const opts = options();
     await db.start(opts);
-    await require('../model/system').runConfig();
+    const modelSystem = require('../model/system');
+    await modelSystem.runConfig();
+    const [endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge] = modelSystem.getMany([
+        'file.endPoint', 'file.accessKey', 'file.secretKey', 'file.bucket', 'file.region',
+        'file.endPointForUser', 'file.endPointForJudge',
+    ]);
+    const sopts = {
+        endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge,
+    };
+    await storage.start(sopts);
     require('../service/monitor');
     for (const i of builtinModel) require(`../model/${i}`);
-    const modelSystem = require('../model/system');
     const scripts = require('../upgrade');
     let dbVer = (await modelSystem.get('db.ver')) ?? 0;
     const expected = scripts.length;
     while (dbVer < expected) {
         logger.info('Upgrading database: from %d to %d', dbVer, expected);
-        await scripts[dbVer]();
+        const result = await scripts[dbVer]();
+        if (!result) break;
         dbVer++;
         await modelSystem.set('db.ver', dbVer);
     }

packages/hydrooj/src/entry/worker.ts

@@ -7,6 +7,7 @@ import {
 import options from '../options';
 import * as bus from '../service/bus';
 import db from '../service/db';
+import storage from '../service/storage';
 
 export async function load() {
     const pending = global.addons;
@@ -23,10 +24,15 @@ export async function load() {
     ]);
     const opts = options();
     await db.start(opts);
-    await new Promise((resolve) => {
-        bus.once('database/config', resolve);
-        require('../model/system');
-    });
+    const modelSystem = require('../model/system');
+    const [endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge] = modelSystem.getMany([
+        'file.endPoint', 'file.accessKey', 'file.secretKey', 'file.bucket', 'file.region',
+        'file.endPointForUser', 'file.endPointForJudge',
+    ]);
+    const sopts = {
+        endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge,
+    };
+    await storage.start(sopts);
     for (const i of builtinLib) require(`../lib/${i}`);
     await lib(pending, fail);
     require('../service/gridfs');

packages/hydrooj/src/handler/home.ts

@@ -7,6 +7,7 @@ import * as bus from '../service/bus';
 import {
     Route, Connection, Handler, ConnectionHandler, param, Types,
 } from '../service/server';
+import storage from '../service/storage';
 import * as misc from '../lib/misc';
 import { md5 } from '../lib/crypto';
 import * as mail from '../lib/mail';
@@ -15,7 +16,6 @@ import * as message from '../model/message';
 import * as document from '../model/document';
 import * as system from '../model/system';
 import * as user from '../model/user';
-import * as file from '../model/file';
 import * as setting from '../model/setting';
 import * as domain from '../model/domain';
 import * as discussion from '../model/discussion';
@@ -383,18 +383,18 @@ class HomeFileHandler extends Handler {
             ['Hydro', 'homepage'],
             ['domain_file', null],
         ];
-        const ufdocs = await file.getMulti({ owner: this.user._id }).toArray();
+        const ufdocs = await storage.list(`user/${this.user._id}/`);
         this.response.template = 'home_file.html';
         this.response.body = { ufdocs, path };
     }
 
-    @param('ufid', Types.ObjectID)
-    async postDelete(domainId: string, ufid: ObjectID) {
-        const ufdoc = await file.getMeta(ufid);
-        if (ufdoc.owner !== this.user._id) this.checkPriv(PRIV.PRIV_DELETE_FILE);
+    @param('ufid', Types.String)
+    async postDelete(domainId: string, target: string) {
+        if (!target.startsWith(`user/${this.user._id}`)) this.checkPriv(PRIV.PRIV_DELETE_FILE);
         else this.checkPriv(PRIV.PRIV_DELETE_FILE_SELF);
-        const result = await file.del(ufdoc._id);
-        if (result) await user.inc(this.user._id, 'usage', -ufdoc.size);
+        const { size } = await storage.getMeta(target);
+        await storage.del(target);
+        await user.inc(this.user._id, 'usage', -size);
         this.back();
     }
 }

packages/hydrooj/src/handler/import.syzoj.ts

@@ -15,6 +15,7 @@ import { PERM } from '../model/builtin';
 import {
     Route, Handler, Types, param, post,
 } from '../service/server';
+import storage from '../service/storage';
 import { isPid } from '../lib/validator';
 import download from '../lib/download';
 
@@ -89,14 +90,13 @@ class ProblemImportSYZOJHandler extends Handler {
                 const urls = {};
                 if (r.body.error) return;
                 for (const t of r.body.downloadInfo) urls[t.filename] = t.downloadUrl;
-                const zip = new AdmZip();
                 for (const f of result.body.testData) {
                     const p = new PassThrough();
                     superagent.get(urls[f.filename]).pipe(p);
                     // eslint-disable-next-line no-await-in-loop
-                    zip.addFile(f.filename, await streamToBuffer(p));
+                    await storage.put(`problem/${domainId}/${docId}/testdata/${f.filename}`, p);
                 }
-                await problem.setTestdata(domainId, docId, zip.toBuffer());
+                // TODO additional_file
                 if (judge) {
                     await problem.edit(domainId, docId, {
                         config: {
@@ -182,7 +182,12 @@ class ProblemImportSYZOJHandler extends Handler {
                 w.on('error', reject);
                 r.pipe(w);
             });
-            await problem.setTestdata(domainId, docId, file);
+            const zip = new AdmZip(file);
+            const entries = zip.getEntries();
+            for (const entry of entries) {
+                // eslint-disable-next-line no-await-in-loop
+                await storage.put(`problem/${domainId}/${docId}/testdata/${entry.entryName}`, entry.getData());
+            }
             await problem.edit(domainId, docId, {
                 config: {
                     time: `${p.time_limit}ms`,

packages/hydrooj/src/handler/misc.ts

@@ -1,29 +1,17 @@
 /* eslint-disable camelcase */
 import fs from 'fs';
-import { ObjectID } from 'mongodb';
+import { BadRequestError } from '../error';
 import { PRIV } from '../model/builtin';
 import * as system from '../model/system';
-import * as file from '../model/file';
 import * as user from '../model/user';
 import db from '../service/db';
 import {
     Route, Handler, Types, param,
 } from '../service/server';
-import { BadRequestError } from '../error';
+import storage from '../service/storage';
 
 const coll = db.collection('status');
 
-class FileDownloadHandler extends Handler {
-    @param('fileId', Types.ObjectID)
-    @param('secret', Types.String)
-    @param('name', Types.String, true)
-    async get(domainId: string, fileId: ObjectID, secret: string, name?: string) {
-        if (name) name = Buffer.from(name.replace(/~/g, '/'), 'base64').toString();
-        const doc = await file.getWithSecret(fileId, secret);
-        this.response.attachment(name || fileId.toHexString(), doc);
-    }
-}
-
 class FileUploadHandler extends Handler {
     async getQuota() {
         let quota = system.get('user.quota');
@@ -44,15 +32,14 @@ class FileUploadHandler extends Handler {
         if (!this.request.files.file) throw new BadRequestError();
         const quota = await this.getQuota();
         const lfdoc = await fs.promises.stat(this.request.files.file.path);
-        let ufid: ObjectID;
+        const ufid = `user/${this.user._id}/${title}`;
         const udoc = await user.inc(this.user._id, 'usage', lfdoc.size);
         try {
-            ufid = await file.add(this.request.files.file.path, title, this.user._id);
+            await storage.put(ufid, this.request.files.file.path);
         } catch (e) {
             await user.inc(this.user._id, 'usage', -lfdoc.size);
             throw e;
         }
-        const fdoc = await file.getMeta(ufid);
         this.response.template = 'fs_upload.html';
         if (redirect) {
             this.response.redirect = redirect.includes('?')
@@ -60,7 +47,7 @@ class FileUploadHandler extends Handler {
                 : `${redirect}?ufid=${ufid}`;
         }
         this.response.body = {
-            fdoc, ufid, usage: udoc.usage, quota,
+            ufid, usage: udoc.usage, quota,
         };
     }
 }
@@ -138,8 +125,6 @@ class SwitchLanguageHandler extends Handler {
 }
 
 export async function apply() {
-    Route('file_download', '/fs/:fileId/:secret', FileDownloadHandler);
-    Route('file_download_with_name', '/fs/:fileId/:name/:secret', FileDownloadHandler);
     Route('file_upload', '/fs/upload', FileUploadHandler, PRIV.PRIV_CREATE_FILE);
     Route('status', '/status', StatusHandler);
     Route('status_update', '/status/update', StatusUpdateHandler);

packages/hydrooj/src/handler/problem.ts

@@ -1,11 +1,11 @@
 import { isSafeInteger, flatten, pick } from 'lodash';
 import yaml from 'js-yaml';
+import fs from 'fs-extra';
 import { FilterQuery, ObjectID } from 'mongodb';
 import AdmZip from 'adm-zip';
 import {
-    NoProblemError, ProblemDataNotFoundError, BadRequestError,
+    NoProblemError, BadRequestError, PermissionError,
     SolutionNotFoundError, ProblemNotFoundError, ValidationError,
-    PermissionError,
 } from '../error';
 import { streamToBuffer } from '../utils';
 import {
@@ -14,13 +14,13 @@ import {
 import paginate from '../lib/paginate';
 import { isTitle, isContent, isPid } from '../lib/validator';
 import { ProblemAdd } from '../lib/ui';
-import * as file from '../model/file';
 import * as problem from '../model/problem';
 import * as record from '../model/record';
 import * as domain from '../model/domain';
 import * as user from '../model/user';
 import * as solution from '../model/solution';
 import { PERM, PRIV, CONSTANT } from '../model/builtin';
+import storage from '../service/storage';
 import * as bus from '../service/bus';
 import {
     Route, Connection, Handler, ConnectionHandler, Types, param, post, route,
@@ -239,13 +239,15 @@ export class ProblemExportHandler extends ProblemDetailHandler {
         const hasPerm = (this.user._id === this.pdoc.owner && this.user.hasPerm(PERM.PERM_READ_PROBLEM_DATA_SELF))
             || this.user.hasPerm(PERM.PERM_READ_PROBLEM_DATA_SELF);
         const pdoc = pick(this.pdoc, ['pid', 'acMsg', 'content', 'config', 'title', 'html', 'tag', 'category']);
-        let zip: AdmZip;
+        const zip = new AdmZip();
         if (hasPerm) {
-            if (this.pdoc.data instanceof ObjectID) {
-                const buf = await streamToBuffer(await file.get(this.pdoc.data));
-                zip = new AdmZip(buf);
-            } else zip = new AdmZip();
-        } else zip = new AdmZip();
+            const files = await storage.list(`problem/${this.domainId}/${this.pdoc.docId}/`, true);
+            for (const file of files) {
+                // eslint-disable-next-line no-await-in-loop
+                const buf = await streamToBuffer(storage.get(`${file.prefix}${file.name}`));
+                zip.addFile(file.name, buf);
+            }
+        }
         zip.addFile('problem.json', Buffer.from(JSON.stringify(pdoc)));
         this.response.attachment(`${this.pdoc.title}.zip`, zip.toBuffer());
     }
@@ -447,59 +449,67 @@ export class ProblemEditHandler extends ProblemManageHandler {
     }
 }
 
-export class ProblemDataUploadHandler extends ProblemManageHandler {
-    async get() {
-        if (this.pdoc.data instanceof ObjectID) {
-            const f = await file.getMeta(this.pdoc.data);
-            this.response.body.md5 = f.md5;
-        } else if (this.pdoc.data) {
-            this.response.body.from = this.pdoc.data;
-        }
-        this.response.template = 'problem_upload.html';
+export class ProblemFilesHandler extends ProblemDetailHandler {
+    @param('testdata', Types.Boolean)
+    @param('additional_file', Types.Boolean)
+    async get(domainId: string, getTestdata = true, getAdditionalFile = true) {
+        const [testdata, additional_file] = await Promise.all([
+            getTestdata
+                ? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/testdata`, true)
+                : [],
+            getAdditionalFile
+                ? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file`, true)
+                : [],
+        ]);
+        this.response.body.testdata = testdata;
+        this.response.body.additional_file = additional_file;
+        this.response.template = 'problem_files.html';
     }
 
-    async post({ domainId }) {
-        if (!this.request.files.file) throw new ValidationError('file');
-        this.pdoc = await problem.setTestdata(
-            domainId, this.pdoc.docId, this.request.files.file.path,
-        );
-        if (this.pdoc.data instanceof ObjectID) {
-            const f = await file.getMeta(this.pdoc.data);
-            this.response.body.md5 = f.md5;
-        }
-        this.response.template = 'problem_upload.html';
-    }
-}
-
-export class ProblemDataDownloadHandler extends ProblemDetailHandler {
-    async get({ pid }) {
-        if (!this.user.hasPriv(PRIV.PRIV_JUDGE)) {
+    @post('files', Types.Array)
+    @post('type', Types.Range(['testdata', 'additional_file']), true)
+    async postGetLinks(domainId: string, files: string[], type = 'testdata') {
+        const isJudge = this.user.hasPriv(PRIV.PRIV_JUDGE);
+        if (type === 'testdata' && !isJudge) {
             if (this.user._id !== this.pdoc.owner) {
                 this.checkPerm(PERM.PERM_READ_PROBLEM_DATA);
             } else this.checkPerm(PERM.PERM_READ_PROBLEM_DATA_SELF);
         }
-        if (this.pdoc.data instanceof ObjectID) {
-            this.response.redirect = await file.url(this.pdoc.data, `${this.pdoc.title}.zip`);
-        } else if (this.pdoc.data) {
-            if (!this.pdoc.data.host) {
-                this.response.redirect = this.url('problem_data', {
-                    domainId: this.pdoc.data.domainId,
-                    pid: this.pdoc.data.pid,
-                });
-            } else {
-                const [scheme, raw] = this.pdoc.data.host.split('//');
-                const args = JSON.parse(Buffer.from(raw, 'base64').toString());
-                if (scheme === 'hydro') {
-                    const [secure, host, port, domainId, id] = args;
-                    this.response.redirect = `http${secure ? 's' : ''}://${host}:${port}/d/${domainId}/p/${id}/data`;
-                } else if (scheme === 'syzoj') {
-                    const [secure, host, port, id] = args;
-                    this.response.redirect = `http${secure ? 's' : ''}://${host}:${port}/problem/${id}/testdata/download`;
-                } else {
-                    throw new ProblemDataNotFoundError(pid);
-                }
+        const links = {};
+        for (const file of files) {
+            // eslint-disable-next-line no-await-in-loop
+            links[file] = await storage.signDownloadLink(
+                `problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${file}`,
+                file, false, isJudge ? 'judge' : 'user',
+            );
+        }
+        this.response.body.links = links;
+    }
+
+    @post('filename', Types.String)
+    @post('type', Types.String, true)
+    async postUploadFile(domainId: string, filename: string, type = 'testdata') {
+        if (!this.request.files.file) throw new ValidationError('file');
+        if (this.pdoc.owner !== this.user._id) this.checkPerm(PERM.PERM_EDIT_PROBLEM);
+        else this.checkPerm(PERM.PERM_EDIT_PROBLEM_SELF);
+        if (filename === 'testdata.zip') {
+            const zip = new AdmZip(this.request.files.file.path);
+            const entries = zip.getEntries();
+            for (const entry of entries) {
+                // eslint-disable-next-line no-await-in-loop
+                await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${entry.name}`, entry.getData());
             }
-        } else throw new ProblemDataNotFoundError(pid);
+        } else await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${filename}`, this.request.files.file.path);
+        this.back();
+    }
+
+    @post('files', Types.Array)
+    @post('type', Types.Range(['testdata', 'additional_file']), true)
+    async postDeleteFiles(domainId: string, files: string[], type = 'testdata') {
+        if (this.pdoc.owner !== this.user._id) this.checkPerm(PERM.PERM_EDIT_PROBLEM);
+        else this.checkPerm(PERM.PERM_EDIT_PROBLEM_SELF);
+        await storage.del(files.map((file) => `problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${file}`));
+        this.back();
     }
 }
 
@@ -611,23 +621,18 @@ export class ProblemSolutionHandler extends ProblemDetailHandler {
 
 export class ProblemSolutionRawHandler extends ProblemDetailHandler {
     @param('psid', Types.ObjectID)
-    async get(domainId: string, psid: ObjectID) {
+    @route('psrid', Types.ObjectID, true)
+    async get(domainId: string, psid: ObjectID, psrid?: ObjectID) {
         this.checkPerm(PERM.PERM_VIEW_PROBLEM_SOLUTION);
-        const psdoc = await solution.get(domainId, psid);
+        if (psrid) {
+            const [psdoc, psrdoc] = await solution.getReply(domainId, psid, psrid);
+            if ((!psdoc) || psdoc.pid !== this.pdoc.docId) throw new SolutionNotFoundError(psid, psrid);
+            this.response.body = psrdoc.content;
+        } else {
+            const psdoc = await solution.get(domainId, psid);
+            this.response.body = psdoc.content;
+        }
         this.response.type = 'text/markdown';
-        this.response.body = psdoc.content;
-    }
-}
-
-export class ProblemSolutionReplyRawHandler extends ProblemDetailHandler {
-    @param('psid', Types.ObjectID)
-    @param('psrid', Types.ObjectID)
-    async get(domainId: string, psid: ObjectID, psrid: ObjectID) {
-        this.checkPerm(PERM.PERM_VIEW_PROBLEM_SOLUTION);
-        const [psdoc, psrdoc] = await solution.getReply(domainId, psid, psrid);
-        if ((!psdoc) || psdoc.pid !== this.pdoc.docId) throw new SolutionNotFoundError(psid, psrid);
-        this.response.type = 'text/markdown';
-        this.response.body = psrdoc.content;
     }
 }
 
@@ -662,21 +667,26 @@ export class ProblemCreateHandler extends Handler {
 }
 
 export class ProblemImportHandler extends Handler {
-    @param('ufid', Types.ObjectID, true)
-    async get(domainId: string, ufid?: ObjectID) {
-        if (ufid) {
-            const stat = await file.getMeta(ufid);
-            if (stat.size > 128 * 1024 * 1024) throw new BadRequestError('File too large');
-            const stream = await file.get(ufid);
-            const buf = await streamToBuffer(stream);
-            const zip = new AdmZip(buf);
-            const pdoc = JSON.parse(zip.getEntry('problem.json').getData().toString());
-            const pid = await problem.add(domainId, pdoc.pid, pdoc.title, pdoc.content, this.user._id, pdoc.tags, pdoc.category);
-            await problem.setTestdata(domainId, pid, buf);
-            await problem.edit(domainId, pid, { html: pdoc.html });
-            this.response.redirect = this.url('problem_detail', { pid });
-            await file.del(ufid);
-        } else this.response.template = 'problem_import.html';
+    async get() {
+        this.response.template = 'problem_import.html';
+    }
+
+    async post(domainId: string) {
+        if (!this.request.files.file) throw new ValidationError('file');
+        const stat = await fs.stat(this.request.files.file.path);
+        if (stat.size > 128 * 1024 * 1024) throw new BadRequestError('File too large');
+        const zip = new AdmZip(this.request.files.file.path);
+        const pdoc = JSON.parse(zip.getEntry('problem.json').getData().toString());
+        const pid = await problem.add(domainId, pdoc.pid, pdoc.title, pdoc.content, this.user._id, pdoc.tags, pdoc.category);
+        const entries = zip.getEntries();
+        for (const entry of entries) {
+            if (entry.name.startsWith('testdata/') || entry.name.startsWith('additional_file')) {
+                // eslint-disable-next-line no-await-in-loop
+                await storage.put(`problem/${domainId}/${pid}/${entry.name}`, entry.getData());
+            }
+        }
+        await problem.edit(domainId, pid, { html: pdoc.html });
+        this.response.redirect = this.url('problem_detail', { pid });
     }
 }
 
@@ -692,11 +702,10 @@ export async function apply() {
     Route('problem_settings', '/p/:pid/settings', ProblemSettingsHandler);
     Route('problem_statistics', '/p/:pid/statistics', ProblemStatisticsHandler);
     Route('problem_edit', '/p/:pid/edit', ProblemEditHandler);
-    Route('problem_upload', '/p/:pid/upload', ProblemDataUploadHandler);
-    Route('problem_data', '/p/:pid/data', ProblemDataDownloadHandler);
+    Route('problem_files', '/p/:pid/files', ProblemFilesHandler);
     Route('problem_solution', '/p/:pid/solution', ProblemSolutionHandler);
     Route('problem_solution_raw', '/p/:pid/solution/:psid/raw', ProblemSolutionRawHandler);
-    Route('problem_solution_reply_raw', '/p/:pid/solution/:psid/:psrid/raw', ProblemSolutionReplyRawHandler);
+    Route('problem_solution_reply_raw', '/p/:pid/solution/:psid/:psrid/raw', ProblemSolutionRawHandler);
     Route('problem_create', '/problem/create', ProblemCreateHandler, PERM.PERM_CREATE_PROBLEM);
     Route('problem_import', '/problem/import', ProblemImportHandler, PERM.PERM_CREATE_PROBLEM);
     Connection('problem_pretest_conn', '/conn/pretest', ProblemPretestConnectionHandler);

packages/hydrooj/src/handler/record.ts

@@ -2,7 +2,6 @@ import { FilterQuery, ObjectID } from 'mongodb';
 import { PermissionError, RecordNotFoundError } from '../error';
 import { PERM, CONSTANT, STATUS } from '../model/builtin';
 import * as problem from '../model/problem';
-import * as file from '../model/file';
 import * as record from '../model/record';
 import * as contest from '../model/contest';
 import * as user from '../model/user';
@@ -12,7 +11,6 @@ import {
     Route, Handler, Connection, ConnectionHandler, Types, param,
 } from '../service/server';
 import { Rdoc } from '../interface';
-import { streamToBuffer } from '../utils';
 
 const RecordHandler = contest.ContestHandlerMixin(Handler);
 
@@ -132,35 +130,6 @@ class RecordDetailHandler extends RecordHandler {
     }
 }
 
-class RecordHackHandler extends Handler {
-    @param('rid', Types.ObjectID)
-    @param('ufid', Types.ObjectID)
-    async get(domainId: string, rid: ObjectID, ufid?: ObjectID) {
-        if (ufid) {
-            const stream = await file.get(ufid);
-            const buf = await streamToBuffer(stream);
-            const input = buf.toString();
-            await this.post(domainId, rid, input);
-        } else this.response.template = 'record_hack.html';
-    }
-
-    @param('rid', Types.ObjectID)
-    @param('input', Types.String)
-    async post(domainId: string, rid: ObjectID, input: string) {
-        const rdoc = await record.get(domainId, rid);
-        if (!rdoc) throw new RecordNotFoundError(domainId, rid);
-        const newRid = await record.add(
-            domainId, rdoc.pid, this.user._id,
-            rdoc.lang, rdoc.code, true,
-            { hack: input },
-        );
-        bus.boardcast('record/change', rdoc);
-        this.response.body = { rid: newRid };
-        this.response.redirect = this.url('record_detail', { rid: newRid });
-        // TODO handle hack(modify testdata, rejudge problem)
-    }
-}
-
 /**
  * @deprecated
  * use RecordDetailHandler.postRejudge instead.
@@ -261,7 +230,6 @@ class RecordDetailConnectionHandler extends contest.ContestHandlerMixin(Connecti
 export async function apply() {
     Route('record_main', '/record', RecordListHandler);
     Route('record_detail', '/record/:rid', RecordDetailHandler);
-    Route('record_hack', '/record/:rid/hack', RecordHackHandler);
     Route('record_rejudge', '/record/:rid/rejudge', RecordRejudgeHandler);
     Connection('record_conn', '/record-conn', RecordMainConnectionHandler);
     Connection('record_detail_conn', '/record-detail-conn', RecordDetailConnectionHandler);

packages/hydrooj/src/interface.ts

@@ -12,6 +12,13 @@ export interface System {
 }
 
 export interface SystemKeys {
+    'file.endPoint': string,
+    'file.accessKey': string,
+    'file.secretKey': string,
+    'file.bucket': string,
+    'file.region': string,
+    'file.endPointForUser': string,
+    'file.endPointForJudge': string,
     'smtp.user': string,
     'smtp.from': string,
     'smtp.pass': string,
@@ -515,7 +522,6 @@ export interface Model {
     discussion: typeof import('./model/discussion'),
     document: typeof import('./model/document'),
     domain: typeof import('./model/domain'),
-    file: typeof import('./model/file'),
     message: typeof import('./model/message'),
     opcount: typeof import('./model/opcount'),
     problem: typeof import('./model/problem'),

packages/hydrooj/src/loader.ts

@@ -10,7 +10,7 @@ import wtfnode from 'wtfnode';
 import fs from 'fs-extra';
 import { argv } from 'yargs';
 import AdmZip from 'adm-zip';
-import { } from './utils';
+import './utils';
 
 export * from './interface';
 

packages/hydrooj/src/model/file.ts

@@ -1,130 +0,0 @@
-import { ObjectID, Collection } from 'mongodb';
-import * as fs from 'fs';
-import { Dictionary } from 'lodash';
-import { ForbiddenError, NotFoundError } from '../error';
-import { Ufdoc } from '../interface';
-import db from '../service/db';
-import * as bus from '../service/bus';
-import gridfs from '../service/gridfs';
-import hash from '../lib/hash.hydro';
-import { bufferToStream } from '../utils';
-
-const coll: Collection<Ufdoc> = db.collection('file');
-const collFile = db.collection('fs.files');
-const collChunk = db.collection('fs.chunks');
-
-function _timestamp() {
-    return Math.floor(Number(new Date()) / 1000000);
-}
-
-export async function add(
-    f: NodeJS.ReadableStream | Buffer | string, filename: string,
-    owner = 1, meta = {},
-) {
-    let stream: NodeJS.ReadableStream;
-    if (typeof f === 'string') stream = fs.createReadStream(f);
-    else if (f instanceof Buffer) stream = bufferToStream(f);
-    else stream = f;
-    const w = gridfs.openUploadStream(filename);
-    await coll.insertOne({
-        ...meta,
-        _id: w.id,
-        secret: String.random(32),
-        owner,
-        filename,
-    });
-    await new Promise((resolve, reject) => {
-        w.on('error', reject);
-        w.on('finish', resolve);
-        stream.pipe(w);
-    });
-    const c = await gridfs.find({ _id: w.id }).toArray();
-    await coll.updateOne({ _id: w.id }, { $set: { md5: c[0].md5, size: c[0].length } });
-    return w.id;
-}
-
-export function del(_id: ObjectID) {
-    return Promise.all([
-        coll.deleteOne({ _id }),
-        collFile.deleteOne({ _id }),
-        collChunk.deleteMany({ files_id: _id }),
-    ]);
-}
-
-export async function getWithSecret(_id: ObjectID, secret: string, reject?: Function) {
-    const file = await coll.findOne({ _id });
-    if (!file) throw new NotFoundError(_id);
-    const timestamp = _timestamp();
-    if (hash(file.secret, timestamp.toString()) !== secret) {
-        if (hash(file.secret, (timestamp - 1).toString()) !== secret) {
-            throw new ForbiddenError();
-        }
-    }
-    const stream = gridfs.openDownloadStream(_id);
-    stream.on('error', (err) => {
-        console.error(err);
-        if (reject) reject(err);
-    });
-    return stream;
-}
-
-export async function get(_id: ObjectID, reject?: Function) {
-    const file = await coll.findOne({ _id });
-    if (!file) throw new NotFoundError(_id);
-    const stream = gridfs.openDownloadStream(_id);
-    stream.on('error', (err) => {
-        console.error(err);
-        if (reject) reject(err);
-    });
-    return stream;
-}
-
-export function getMeta(_id: ObjectID): Promise<Ufdoc> {
-    return coll.findOne({ _id });
-}
-
-export async function getMetaDict(ufids: ObjectID[]): Promise<Dictionary<Ufdoc>> {
-    const r = {};
-    ufids = Array.from(new Set(ufids));
-    const ufdocs = await coll.find({ _id: { $in: ufids } }).toArray();
-    for (const ufdoc of ufdocs) {
-        r[ufdoc._id.toHexString()] = ufdoc;
-    }
-    return r;
-}
-
-export function getMulti(query: any) {
-    return coll.find(query);
-}
-
-async function _url(_id: ObjectID, name?: string) {
-    const file = await coll.findOne({ _id });
-    const secret = hash(file.secret, _timestamp().toString());
-    if (name) return `/fs/${_id}/${Buffer.from(name).toString('base64').replace(/\//g, '~')}/${secret}`;
-    return `/fs/${_id}/${secret}`;
-}
-
-function __url(file: Ufdoc, name?: string) {
-    const secret = hash(file.secret, _timestamp().toString());
-    if (name) return `/fs/${file._id}/${Buffer.from(name).toString('base64').replace(/\//g, '~')}/${secret}`;
-    return `/fs/${file._id}/${secret}`;
-}
-
-export function url(_id: ObjectID, name?: string): Promise<string>;
-export function url(file: Ufdoc, name: string): string;
-export function url(arg0: ObjectID | Ufdoc, name?: string) {
-    if (arg0 instanceof ObjectID) return _url(arg0, name);
-    return __url(arg0, name);
-}
-
-function ensureIndexes() {
-    return Promise.all([
-        collChunk.createIndex({ files_id: 1, n: 1 }, { unique: true }),
-        collFile.createIndex({ md5: 1 }),
-    ]);
-}
-
-bus.once('app/started', ensureIndexes);
-global.Hydro.model.file = {
-    add, get, del, getMeta, getWithSecret, getMulti, getMetaDict, url,
-};

packages/hydrooj/src/model/problem.ts

@@ -1,15 +1,12 @@
 import { ObjectID, FilterQuery } from 'mongodb';
 import { Dictionary } from 'lodash';
-import { safeLoad } from 'js-yaml';
 import { STATUS } from './builtin';
-import * as file from './file';
 import * as document from './document';
 import * as domain from './domain';
 import { ProblemStatusDoc, ProblemDataSource, Pdict } from '../interface';
 import { Content } from '../loader';
 import { NumberKeys, Projection } from '../typeutils';
 import { ProblemNotFoundError } from '../error';
-import * as testdataConfig from '../lib/testdataConfig';
 
 export interface Pdoc {
     _id: ObjectID
@@ -188,14 +185,6 @@ export function setStar(domainId: string, pid: number, uid: number, star: boolea
     return document.setStatus(domainId, document.TYPE_PROBLEM, pid, uid, { star });
 }
 
-export async function setTestdata(domainId: string, _id: number, f: string | Buffer) {
-    const pdoc = await get(domainId, _id);
-    const config = await testdataConfig.readConfig(f);
-    const id = await file.add(f, 'data.zip', 1);
-    if (pdoc.data instanceof ObjectID) file.del(pdoc.data);
-    return await edit(domainId, _id, { data: id, config: safeLoad(config) as any });
-}
-
 global.Hydro.model.problem = {
     Pdoc,
 
@@ -218,6 +207,5 @@ global.Hydro.model.problem = {
     getListStatus,
     getMultiStatus,
     setStar,
-    setTestdata,
     updateStatus,
 };

packages/hydrooj/src/model/setting.ts

@@ -120,6 +120,13 @@ DomainUserSetting(
 );
 
 SystemSetting(
+    Setting('setting_file', 'file.endPoint', null, null, 'text', 'Storage engine endPoint'),
+    Setting('setting_file', 'file.accessKey', null, null, 'text', 'Storage engine accessKey'),
+    Setting('setting_file', 'file.secretKey', null, null, 'password', 'Storage engine secret', null, FLAG_SECRET),
+    Setting('setting_file', 'file.bucket', null, 'hydro', 'text', 'Storage engine bucket'),
+    Setting('setting_file', 'file.region', null, 'us-east-1', 'text', 'Storage engine region'),
+    Setting('setting_file', 'file.endPointForUser', null, null, 'text', 'EndPoint for user'),
+    Setting('setting_file', 'file.endPointForJudge', null, null, 'text', 'EndPoint for judge'),
     Setting('setting_smtp', 'smtp.user', null, null, 'text', 'SMTP Username'),
     Setting('setting_smtp', 'smtp.pass', null, null, 'password', 'SMTP Password', null, FLAG_SECRET),
     Setting('setting_smtp', 'smtp.host', null, null, 'text', 'SMTP Server Host'),

packages/hydrooj/src/model/system.ts

@@ -60,7 +60,7 @@ export async function runConfig() {
     for (const setting of SYSTEM_SETTINGS) {
         if (setting.value) cache[setting.key] = setting.value;
     }
-    const config = await coll.find({}).toArray();
+    const config = await coll.find().toArray();
     for (const i of config) cache[i._id] = i.value;
     bus.emit('database/config');
 }

packages/hydrooj/src/service/storage.ts

@@ -1,41 +1,88 @@
 import { Readable } from 'stream';
-import Minio, { BucketItem, ItemBucketMetadata } from 'minio';
+import { URL } from 'url';
+import { Client, BucketItem, ItemBucketMetadata } from 'minio';
 
 interface StorageOptions {
-    host?: string,
-    port?: number,
-    useSSL?: boolean,
-    accessKey?: string,
-    secretKey?: string,
-    bucket?: string,
-    region?: string,
+    endPoint: string;
+    accessKey: string;
+    secretKey: string;
+    bucket: string;
+    region?: string;
+    endPointForUser?: string;
+    endPointForJudge?: string;
 }
 
-const defaultConfig: StorageOptions = {
-    host: 'play.min.io',
-    port: 9000,
-    useSSL: true,
-    accessKey: 'Q3AM3UQ867SPQQA43P2F',
-    secretKey: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG',
-    bucket: 'hydro',
-    region: 'us-east-1',
-};
+interface MinioEndpointConfig {
+    endPoint: string;
+    port: number;
+    useSSL: boolean;
+}
+
+function parseMainEndpointUrl(endpoint: string): MinioEndpointConfig {
+    const url = new URL(endpoint);
+    const result: Partial<MinioEndpointConfig> = {};
+    if (url.pathname !== '/') throw new Error('Main MinIO endpoint URL of a sub-directory is not supported.');
+    if (url.username || url.password || url.hash || url.search) {
+        throw new Error('Authorization, search parameters and hash are not supported for main MinIO endpoint URL.');
+    }
+    if (url.protocol === 'http:') result.useSSL = false;
+    else if (url.protocol === 'https:') result.useSSL = true;
+    else {
+        throw new Error(
+            `Invalid protocol "${url.protocol}" for main MinIO endpoint URL. Only HTTP and HTTPS are supported.`,
+        );
+    }
+    result.endPoint = url.hostname;
+    result.port = url.port ? Number(url.port) : result.useSSL ? 443 : 80;
+    return result as MinioEndpointConfig;
+}
+function parseAlternativeEndpointUrl(endpoint: string): (originalUrl: string) => string {
+    if (!endpoint) return (originalUrl) => originalUrl;
+    const url = new URL(endpoint);
+    if (url.hash || url.search) throw new Error('Search parameters and hash are not supported for alternative MinIO endpoint URL.');
+    if (!url.pathname.endsWith('/')) throw new Error("Alternative MinIO endpoint URL's pathname must ends with '/'.");
+    return (originalUrl) => {
+        const parsedOriginUrl = new URL(originalUrl);
+        return new URL(parsedOriginUrl.pathname.slice(1) + parsedOriginUrl.search + parsedOriginUrl.hash, url).toString();
+    };
+}
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent
+function encodeRFC5987ValueChars(str: string) {
+    return (
+        encodeURIComponent(str)
+            // Note that although RFC3986 reserves "!", RFC5987 does not,
+            // so we do not need to escape it
+            .replace(/['()]/g, escape) // i.e., %27 %28 %29
+            .replace(/\*/g, '%2A')
+            // The following are not required for percent-encoding per RFC5987,
+            // so we can allow for a little better readability over the wire: |`^
+            .replace(/%(?:7C|60|5E)/g, unescape)
+    );
+}
 
 class StorageService {
-    public client: Minio.Client;
+    public client: Client;
+    public error = '';
     private opts: StorageOptions;
+    private replaceWithAlternativeUrlFor: Record<'user' | 'judge', (originalUrl: string) => string>;
 
     async start(opts: StorageOptions) {
-        this.opts = { ...defaultConfig, ...opts };
-        this.client = new Minio.Client({
-            endPoint: this.opts.host,
-            port: this.opts.port,
-            useSSL: this.opts.useSSL,
-            accessKey: this.opts.accessKey,
-            secretKey: this.opts.secretKey,
-        });
-        const exists = await this.client.bucketExists(this.opts.bucket);
-        if (!exists) await this.client.makeBucket(this.opts.bucket, this.opts.region);
+        try {
+            this.opts = opts;
+            this.client = new Client({
+                ...parseMainEndpointUrl(this.opts.endPoint),
+                accessKey: this.opts.accessKey,
+                secretKey: this.opts.secretKey,
+            });
+            const exists = await this.client.bucketExists(this.opts.bucket);
+            if (!exists) await this.client.makeBucket(this.opts.bucket, this.opts.region);
+            this.replaceWithAlternativeUrlFor = {
+                user: parseAlternativeEndpointUrl(this.opts.endPointForUser),
+                judge: parseAlternativeEndpointUrl(this.opts.endPointForJudge),
+            };
+        } catch (e) {
+            this.error = e.toString();
+        }
     }
 
     async put(target: string, file: string | Buffer | Readable, meta: ItemBucketMetadata = {}) {
@@ -56,11 +103,33 @@ class StorageService {
         const stream = this.client.listObjects(this.opts.bucket, target, recursive);
         return await new Promise<BucketItem[]>((resolve, reject) => {
             const results: BucketItem[] = [];
-            stream.on('data', (result) => results.push(result));
+            stream.on('data', (result) => results.push({
+                ...result,
+                prefix: target,
+                name: result.name.split(target)[1],
+            }));
             stream.on('end', () => resolve(results));
             stream.on('error', reject);
         });
     }
+
+    async getMeta(target: string) {
+        const result = await this.client.statObject(this.opts.bucket, target);
+        return { ...result.metaData, ...result };
+    }
+
+    async signDownloadLink(target: string, filename?: string, noExpire = false, useAlternativeEndpointFor?: 'user' | 'judge'): Promise<string> {
+        const url = await this.client.presignedGetObject(
+            this.opts.bucket,
+            target,
+            noExpire ? 24 * 60 * 60 * 7 : 30 * 60,
+            filename
+                ? { 'response-content-disposition': `attachment; filename="${encodeRFC5987ValueChars(filename)}"` }
+                : {},
+        );
+        if (useAlternativeEndpointFor) return this.replaceWithAlternativeUrlFor[useAlternativeEndpointFor](url);
+        return url;
+    }
 }
 
 const service = new StorageService();

packages/hydrooj/src/upgrade.ts

@@ -1,9 +1,17 @@
 /* eslint-disable no-await-in-loop */
+import { ObjectID, Collection } from 'mongodb';
+import { terminal } from 'terminal-kit';
+import { Ufdoc } from './interface';
+import { Logger } from './logger';
+import gridfs from './service/gridfs';
+import storage from './service/storage';
+import db from './service/db';
+import * as problem from './model/problem';
 import * as domain from './model/domain';
 import * as document from './model/document';
-import db from './service/db';
 
-type UpgradeScript = () => Promise<void>;
+const logger = new Logger('upgrade');
+type UpgradeScript = () => Promise<boolean | void>;
 
 const scripts: UpgradeScript[] = [
     // Init
@@ -11,6 +19,7 @@ const scripts: UpgradeScript[] = [
         const ddoc = await domain.get('system');
         if (!ddoc) await domain.add('system', 0, 'Hydro', 'Hydro System');
         // TODO discussion node?
+        return true;
     },
     // Add history column to ddoc,drdoc,psdoc
     async function _1_2() {
@@ -29,9 +38,68 @@ const scripts: UpgradeScript[] = [
             });
             if (bulk.length) await bulk.execute();
         }
+        return true;
     },
     async function _2_3() {
         await db.collection('document').updateMany({ pid: /^\d+$/i }, { $unset: { pid: '' } });
+        return true;
+    },
+    async function _3_4() {
+        if (storage.error) {
+            logger.error('Cannot upgrade. Please change storage config.');
+            return false;
+        }
+
+        const coll: Collection<Ufdoc> = db.collection('file');
+        const collFile = db.collection('fs.files');
+        const collChunk = db.collection('fs.chunks');
+        function del(_id: ObjectID) {
+            return Promise.all([
+                coll.deleteOne({ _id }),
+                collFile.deleteOne({ _id }),
+                collChunk.deleteMany({ files_id: _id }),
+            ]);
+        }
+        async function get(_id: ObjectID, reject?: Function) {
+            const stream = gridfs.openDownloadStream(_id);
+            stream.on('error', (err) => {
+                console.error(err);
+                if (reject) reject(err);
+            });
+            return stream;
+        }
+        terminal.clear('Upgrade');
+        logger.info('');
+        // Problem file and User file
+        // @ts-ignore
+        const progress = terminal.progressBar({ title: 'Changing storage engine. This may take a long time.', y: 1 });
+        const ddocs = await domain.getMulti().toArray();
+        const totalProgress = terminal.progressBar({
+            // @ts-ignore
+            items: ddocs.length, title: 'Domain', y: 3, eta: true,
+        });
+        for (let i = 0; i < ddocs.length; i++) {
+            const ddoc = ddocs[i];
+            totalProgress.startItem(ddoc._id);
+            const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['docId', 'data', 'title']).toArray();
+            // @ts-ignore
+            const domainProgress = terminal.progressBar({
+                // @ts-ignore
+                items: pdocs.length, title: 'Problems', y: 4, eta: true,
+            });
+            for (let j = 0; j < pdocs.length; j++) {
+                const pdoc = pdocs[j];
+                domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`);
+                // TODO copy file
+                domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`);
+            }
+            domainProgress.stop();
+            totalProgress.itemDone(ddoc._id);
+        }
+        totalProgress.stop();
+        // TODO userfile
+        progress.stop();
+        return false;
     },
 ];