core: handle image download cache

packages/hydrooj/locales/zh.yaml

@@ -321,7 +321,9 @@ Extension Score Penalty: 延期递交扣分规则
 Failed to join the domain. You are already a member.: 加入域失败，您已是该域的成员。
 Feedback: 反馈
 Field {0} or {1} validation failed.: 字段 {0} 或 {1} 验证失败。
+Field {0} or {1} validation failed. ({2}): 字段 {0} 或 {1} 验证失败。 ({2})
 Field {0} validation failed.: 字段 {0} 验证失败。
+Field {0} validation failed. ({2}): 字段 {0} 验证失败。 ({2})
 Filter: 过滤
 Footer: 页脚
 Forgot Password and/or Username: 忘记密码和/或用户名

packages/hydrooj/package.json

@@ -28,9 +28,11 @@
         "koa-static-cache": "^5.1.4",
         "lodash": "^4.17.21",
         "lru-cache": "^6.0.0",
+        "mime-types": "^2.1.32",
         "minio": "7.0.17",
         "moment-timezone": "^0.5.33",
         "mongodb": "^3.6.11",
+        "nanoid": "^3.1.25",
         "nodemailer": "^6.6.3",
         "p-queue": "^7.1.0",
         "reflect-metadata": "^0.1.13",
@@ -50,6 +52,7 @@
         "@types/koa-static-cache": "^5.1.1",
         "@types/lodash": "^4.14.172",
         "@types/lru-cache": "^5.1.1",
+        "@types/mime-types": "^2.1.1",
         "@types/minio": "^7.0.9",
         "@types/mongodb": "^3.6.20",
         "@types/nodemailer": "^6.4.4",

packages/hydrooj/src/entry/common.ts

@@ -30,8 +30,7 @@ export const builtinHandler = [
 ];
 
 export const builtinScript = [
-    'rating', 'problemStat', 'blacklist', 'deleteUser', 'updateFilelist',
-    'storageUsage',
+    'rating', 'problemStat', 'blacklist', 'deleteUser', 'storageUsage',
 ];
 
 function getFiles(folder: string, base = ''): string[] {

packages/hydrooj/src/error.ts

@@ -56,21 +56,27 @@ export const InvalidTokenError = Err('InvalidTokenError', ForbiddenError);
 export const BlacklistedError = Err('BlacklistedError', ForbiddenError, 'Address or user {0} is blacklisted.');
 export const VerifyPasswordError = Err('VerifyPasswordError', ForbiddenError, "Passwords don't match.");
 export const OpcountExceededError = Err('OpcountExceededError', ForbiddenError, 'Too frequent operations of {0} (limit: {2} operations in {1} seconds).');
-export const PermissionError = Err('PermissionError', ForbiddenError, function () {
+export const PermissionError = Err('PermissionError', ForbiddenError, function (this: HydroError) {
     if (typeof this.params[0] === 'bigint') {
         this.params[0] = require('./model/builtin').PERMS.find(({ key }) => key === this.params[0])?.desc || this.params[0];
     }
     return "You don't have the required permission ({0}) in this domain.";
 });
-export const PrivilegeError = Err('PrivilegeError', ForbiddenError, function () {
+export const PrivilegeError = Err('PrivilegeError', ForbiddenError, function (this: HydroError) {
     if (this.params.includes(global.Hydro.model.builtin.PRIV.PRIV_USER_PROFILE)) {
         return "You're not logged in.";
     }
     return "You don't have the required privilege.";
 });
-export const ValidationError = Err('ValidationError', ForbiddenError, function () {
-    if (this.params.length === 1) return 'Field {0} validation failed.';
-    return 'Field {0} or {1} validation failed.';
+export const ValidationError = Err('ValidationError', ForbiddenError, function (this: HydroError) {
+    if (this.params.length === 3) {
+        return this.params[1]
+            ? 'Field {0} or {1} validation failed. ({2})'
+            : 'Field {0} validation failed. ({2})';
+    }
+    return this.params[1]
+        ? 'Field {0} or {1} validation failed.'
+        : 'Field {0} validation failed.';
 });
 export const ContestNotAttendedError = Err('ContestNotAttendedError', ForbiddenError, "You haven't attended this contest yet.");
 export const RequireProError = Err('RequireProError', ForbiddenError, 'RequireProError');

packages/hydrooj/src/handler/contest.ts

@@ -2,9 +2,12 @@ import moment from 'moment-timezone';
 import { ObjectID } from 'mongodb';
 import AdmZip from 'adm-zip';
 import { Time } from '@hydrooj/utils/lib/utils';
+import { lookup } from 'mime-types';
 import {
     ContestNotLiveError, ValidationError, ProblemNotFoundError,
-    ContestNotAttendedError, PermissionError, BadRequestError, ContestNotFoundError, InvalidTokenError, RecordNotFoundError,
+    ContestNotAttendedError, PermissionError, BadRequestError,
+    ContestNotFoundError, InvalidTokenError, RecordNotFoundError,
+    NotFoundError,
 } from '../error';
 import { ProblemDoc, Tdoc, User } from '../interface';
 import paginate from '../lib/paginate';
@@ -315,11 +318,22 @@ export class ContestProblemFileDownloadHandler extends ContestProblemHandler {
     async get(domainId: string, filename: string, noDisposition = false) {
         // @ts-ignore
         if (typeof this.pdoc.docId === 'string') this.pdoc.docId = this.pdoc.docId.split(':')[1];
+        const target = `problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/${filename}`;
+        const file = await storage.getMeta(target);
+        if (!file) throw new NotFoundError(filename);
+        this.response.etag = file.etag;
+        const type = lookup(filename).toString();
+        const shouldProxy = ['image', 'video', 'audio', 'pdf', 'vnd'].filter((i) => type.includes(i)).length;
+        if (shouldProxy && file.size! < 32 * 1024 * 1024) {
+            this.response.body = await storage.get(target);
+            this.response.type = file['Content-Type'] || type;
+            if (!noDisposition) this.response.addHeader('Content-Disposition', `attachment; filename=${filename}`);
+        } else {
             this.response.redirect = await storage.signDownloadLink(
-            `problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/${filename}`,
-            noDisposition ? undefined : filename, false, 'user',
+                target, noDisposition ? undefined : filename, false, 'user',
             );
         }
+    }
 }
 
 export class ContestDetailProblemSubmitHandler extends ContestProblemHandler {

packages/hydrooj/src/handler/homework.ts

@@ -3,9 +3,10 @@ import moment from 'moment-timezone';
 import { ObjectID } from 'mongodb';
 import AdmZip from 'adm-zip';
 import { Time } from '@hydrooj/utils/lib/utils';
+import { lookup } from 'mime-types';
 import {
     ValidationError, HomeworkNotLiveError, ProblemNotFoundError,
-    HomeworkNotAttendedError, BadRequestError,
+    HomeworkNotAttendedError, BadRequestError, NotFoundError,
 } from '../error';
 import {
     PenaltyRules, Tdoc, ProblemDoc, User,
@@ -181,11 +182,22 @@ export class HomeworkProblemFileDownloadHandler extends HomeworkDetailProblemHan
     async get(domainId: string, filename: string, noDisposition = false) {
         // @ts-ignore
         if (typeof this.pdoc.docId === 'string') this.pdoc.docId = this.pdoc.docId.split(':')[1];
+        const target = `problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/${filename}`;
+        const file = await storage.getMeta(target);
+        if (!file) throw new NotFoundError(filename);
+        this.response.etag = file.etag;
+        const type = lookup(filename).toString();
+        const shouldProxy = ['image', 'video', 'audio', 'pdf', 'vnd'].filter((i) => type.includes(i)).length;
+        if (shouldProxy && file.size! < 32 * 1024 * 1024) {
+            this.response.body = await storage.get(target);
+            this.response.type = file['Content-Type'] || type;
+            if (!noDisposition) this.response.addHeader('Content-Disposition', `attachment; filename=${filename}`);
+        } else {
             this.response.redirect = await storage.signDownloadLink(
-            `problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/${filename}`,
-            noDisposition ? undefined : filename, false, 'user',
+                target, noDisposition ? undefined : filename, false, 'user',
             );
         }
+    }
 }
 
 class HomeworkDetailProblemSubmitHandler extends HomeworkDetailProblemHandler {

packages/hydrooj/src/handler/misc.ts

@@ -1,6 +1,11 @@
 /* eslint-disable camelcase */
 import { pick } from 'lodash';
-import { BadRequestError, ForbiddenError, ValidationError } from '../error';
+import { lookup } from 'mime-types';
+import {
+    BadRequestError, ForbiddenError, NotFoundError,
+    ValidationError,
+} from '../error';
+import { sortFiles } from '../utils';
 import { PRIV } from '../model/builtin';
 import storage from '../model/storage';
 import * as system from '../model/system';
@@ -8,7 +13,6 @@ import user from '../model/user';
 import {
     Route, Handler, Types, param, post,
 } from '../service/server';
-import { sortFiles } from '../utils';
 
 class CheckInHandler extends Handler {
     async prepare() {
@@ -78,7 +82,7 @@ export class FilesHandler extends Handler {
             throw new ForbiddenError('File size limit exceeded.');
         }
         if (!filename) filename = this.request.files.file.name || String.random(16);
-        if (filename.includes('/') || filename.includes('..')) throw new ValidationError('filename', 'Bad filename');
+        if (filename.includes('/') || filename.includes('..')) throw new ValidationError('filename', null, 'Bad filename');
         if (this.user._files.filter((i) => i.name === filename).length) throw new BadRequestError('file exists');
         await storage.put(`user/${this.user._id}/${filename}`, this.request.files.file.path);
         const meta = await storage.getMeta(`user/${this.user._id}/${filename}`);
@@ -104,10 +108,23 @@ export class FSDownloadHandler extends Handler {
     @param('filename', Types.Name)
     @param('noDisposition', Types.Boolean)
     async get(domainId: string, uid: number, filename: string, noDisposition = false) {
+        this.response.addHeader('Cache-Control', 'public');
+        const target = `user/${uid}/${filename}`;
+        const file = await storage.getMeta(target);
+        if (!file) throw new NotFoundError(filename);
+        this.response.etag = file.etag;
+        const type = lookup(filename).toString();
+        const shouldProxy = ['image', 'video', 'audio', 'pdf', 'vnd'].filter((i) => type.includes(i)).length;
+        if (shouldProxy && file.size! < 32 * 1024 * 1024) {
+            this.response.body = await storage.get(target);
+            this.response.type = file['Content-Type'] || type;
+            if (!noDisposition) this.response.addHeader('Content-Disposition', `attachment; filename=${filename}`);
+        } else {
             this.response.redirect = await storage.signDownloadLink(
-            `user/${uid}/${filename}`, noDisposition ? undefined : filename, false, 'user',
+                target, noDisposition ? undefined : filename, false, 'user',
             );
         }
+    }
 }
 
 export async function apply() {

packages/hydrooj/src/handler/problem.ts

@@ -2,10 +2,11 @@ import { isSafeInteger, flatten } from 'lodash';
 import { FilterQuery, ObjectID } from 'mongodb';
 import AdmZip from 'adm-zip';
 import { sortFiles } from '@hydrooj/utils/lib/utils';
+import { lookup } from 'mime-types';
 import {
     NoProblemError, PermissionError, ValidationError,
     SolutionNotFoundError, ProblemNotFoundError, BadRequestError,
-    ForbiddenError,
+    ForbiddenError, NotFoundError,
 } from '../error';
 import {
     ProblemDoc, User, ProblemStatusDoc,
@@ -373,7 +374,7 @@ export class ProblemFilesHandler extends ProblemDetailHandler {
             throw new ForbiddenError('File size limit exceeded.');
         }
         if (!filename) filename = this.request.files.file.name || String.random(16);
-        if (filename.includes('/') || filename.includes('..')) throw new ValidationError('filename', 'Bad filename');
+        if (filename.includes('/') || filename.includes('..')) throw new ValidationError('filename', null, 'Bad filename');
         if (!this.user.own(this.pdoc, PERM.PERM_EDIT_PROBLEM_SELF)) this.checkPerm(PERM.PERM_EDIT_PROBLEM);
         if (filename.endsWith('.zip')) {
             const zip = new AdmZip(this.request.files.file.path);
@@ -414,11 +415,22 @@ export class ProblemFileDownloadHandler extends ProblemDetailHandler {
         if (type === 'testdata' && !this.user.own(this.pdoc)) {
             if (!this.user.hasPriv(PRIV.PRIV_READ_PROBLEM_DATA)) this.checkPerm(PERM.PERM_READ_PROBLEM_DATA);
         }
+        const target = `problem/${domainId}/${this.pdoc.docId}/${type}/${filename}`;
+        const file = await storage.getMeta(target);
+        if (!file) throw new NotFoundError(filename);
+        this.response.etag = file.etag;
+        const type = lookup(filename).toString();
+        const shouldProxy = ['image', 'video', 'audio', 'pdf', 'vnd'].filter((i) => type.includes(i)).length;
+        if (shouldProxy && file.size! < 32 * 1024 * 1024) {
+            this.response.body = await storage.get(target);
+            this.response.type = file['Content-Type'] || type;
+            if (!noDisposition) this.response.addHeader('Content-Disposition', `attachment; filename=${filename}`);
+        } else {
             this.response.redirect = await storage.signDownloadLink(
-            `problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${filename}`,
-            noDisposition ? undefined : filename, false, 'user',
+                target, noDisposition ? undefined : filename, false, 'user',
             );
         }
+    }
 }
 
 export class ProblemSolutionHandler extends ProblemDetailHandler {

packages/hydrooj/src/handler/training.ts

@@ -50,7 +50,7 @@ async function _parseDagJson(domainId: string, _dag: string): Promise<Tdoc['dag'
             parsed.push(newNode);
         }
     } catch (e) {
-        throw new ValidationError('dag', [e.message]);
+        throw new ValidationError('dag', null, e.message);
     }
     return parsed;
 }

packages/hydrooj/src/model/storage.ts

@@ -1,6 +1,9 @@
 import { escapeRegExp } from 'lodash';
 import { ItemBucketMetadata } from 'minio';
+import { nanoid } from 'nanoid';
+import { lookup } from 'mime-types';
 import moment from 'moment';
+import { extname } from 'path';
 import type { Readable } from 'stream';
 import TaskModel from './task';
 import storage from '../service/storage';
@@ -11,25 +14,15 @@ export class StorageModel {
     static coll = db.collection('storage');
 
     static async put(path: string, file: string | Buffer | Readable, meta: ItemBucketMetadata = {}) {
-        const [current, place] = await Promise.all([
-            StorageModel.coll.findOne({ path }),
-            StorageModel.coll.findOne({ _id: path }),
-        ]);
-        if (current) {
-            await storage.put(current._id, file, meta);
-            const { metaData, size, etag } = await storage.getMeta(current._id);
-            await StorageModel.coll.updateOne({ path }, {
-                $set: {
-                    meta: metaData, size, etag, lastModified: new Date(), autoDelete: null,
-                },
-            });
-            return path;
-        }
-        const target = place ? path + Math.random().toString(16) : path;
-        await storage.put(target, file, meta);
-        const { metaData, size, etag } = await storage.getMeta(target);
+        await StorageModel.del([path]);
+        meta['Content-Type'] = (path.endsWith('.ans') || path.endsWith('.out'))
+            ? 'text/plain'
+            : lookup(path) || 'application/octet-stream';
+        const _id = nanoid() + extname(path);
+        await storage.put(_id, file, meta);
+        const { metaData, size, etag } = await storage.getMeta(_id);
         await StorageModel.coll.insertOne({
-            _id: target, meta: metaData, path, size, etag, lastModified: new Date(),
+            _id, meta: metaData, path, size, etag, lastModified: new Date(),
         });
         return path;
     }
@@ -71,8 +64,9 @@ export class StorageModel {
     }
 
     static async signDownloadLink(target: string, filename?: string, noExpire = false, useAlternativeEndpointFor?: 'user' | 'judge') {
-        await StorageModel.coll.updateOne({ path: target }, { $set: { lastUsage: new Date() } });
-        return await storage.signDownloadLink(target, filename, noExpire, useAlternativeEndpointFor);
+        const res = await StorageModel.coll.findOneAndUpdate({ path: target }, { $set: { lastUsage: new Date() } });
+        if (!res.value) return '';
+        return await storage.signDownloadLink(res.value._id, filename, noExpire, useAlternativeEndpointFor);
     }
 }
 

packages/hydrooj/src/script/storageUsage.ts

@@ -8,7 +8,7 @@ export async function run(_: {}, report: Function) {
     const cursor = storage.coll.find({ path: { $regex: /^problem\//i } }).sort({ path: 1 });
     const count = await cursor.count();
     report({ message: `Total ${count} files` });
-    let current: string;
+    let current = '';
     let memory = 0;
     let start = new Date().getTime();
     // eslint-disable-next-line no-constant-condition

packages/hydrooj/src/script/updateFilelist.ts

@@ -1,30 +0,0 @@
-import storage from '../model/storage';
-import { iterateAllProblem, iterateAllProblemInDomain } from '../pipelineUtils';
-
-export const description = 'Sync problem filelist from s3 service';
-
-export async function run({ domainId }, report: Function) {
-    const cb = async (pdoc) => {
-        report({ message: `${pdoc.domainId}/${pdoc.docId}` });
-        const [data, additional_file] = await Promise.all([
-            storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`),
-            storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/additional_file/`),
-        ]) as any;
-        for (let i = 0; i < data.length; i++) {
-            data[i]._id = data[i].name;
-        }
-        for (let i = 0; i < additional_file.length; i++) {
-            additional_file[i]._id = additional_file[i].name;
-        }
-        return { data, additional_file };
-    };
-    if (domainId) await iterateAllProblemInDomain(domainId, [], cb);
-    else await iterateAllProblem([], cb);
-    return true;
-}
-
-export const validate = {
-    domainId: 'string?',
-};
-
-global.Hydro.script.updateFilelist = { run, description, validate };

packages/hydrooj/src/service/server.ts

@@ -409,7 +409,9 @@ export class Handler extends HandlerCommon {
         template?: string,
         redirect?: string,
         disposition?: string,
+        etag?: string,
         attachment: (name: string, stream?: any) => void,
+        addHeader: (name: string, value: string) => void,
     };
 
     csrfToken: string;
@@ -452,6 +454,7 @@ export class Handler extends HandlerCommon {
                     ctx.body = streamOrBuffer.pipe(new PassThrough());
                 }
             },
+            addHeader: (name: string, value: string) => this.ctx.set(name, value),
             disposition: null,
         };
         this.UiContext = cloneDeep(UiContextBase);
@@ -592,18 +595,23 @@ export class Handler extends HandlerCommon {
     async putResponse() {
         if (this.response.disposition) this.ctx.set('Content-Disposition', this.response.disposition);
         if (!this.response.body) return;
-        if (this.response.redirect && !this.request.json) {
+        if (this.response.etag) {
+            this.ctx.set('ETag', this.response.etag);
+            this.ctx.set('Cache-Control', 'public');
+        }
+        if (this.response.etag && this.request.headers['if-none-match'] === this.response.etag) {
+            this.ctx.response.status = 304;
+        } else if (this.response.redirect && !this.request.json) {
             this.ctx.response.type = 'application/octet-stream';
             this.ctx.response.status = 302;
             this.ctx.redirect(this.response.redirect);
         } else {
-            this.ctx.response.body = this.response.body;
+            this.ctx.body = this.response.body;
             this.ctx.response.status = this.response.status || 200;
-            this.ctx.response.type = this.request.json
+            this.ctx.response.type = this.response.type
+                || (this.request.json
                     ? 'application/json'
-                : this.response.type
-                    ? this.response.type
-                    : this.ctx.response.type;
+                    : this.ctx.response.type);
         }
     }
 
@@ -731,7 +739,7 @@ const Checker = (permPrivChecker) => {
             perm = item;
         }
     }
-    return function check() {
+    return function check(this: Handler) {
         checker();
         if (perm) this.checkPerm(perm); // lgtm [js/trivial-conditional]
         if (priv) this.checkPriv(priv);

packages/hydrooj/src/service/storage.ts

@@ -195,11 +195,10 @@ class StorageService {
         try {
             const headers: Record<string, string> = {};
             if (filename) headers['response-content-disposition'] = `attachment; filename="${encodeRFC5987ValueChars(filename)}"`;
-            if (/\.(jpe?g|png|gif)$/i.test(target)) headers['cache-control'] = 'max-age=604800';
             const url = await this.client.presignedGetObject(
                 this.opts.bucket,
                 target,
-                noExpire ? 24 * 60 * 60 * 7 : 30 * 60,
+                noExpire ? 24 * 60 * 60 * 7 : 10 * 60,
                 headers,
             );
             if (useAlternativeEndpointFor) return this.replaceWithAlternativeUrlFor[useAlternativeEndpointFor](url);

packages/import-qduoj/handler.ts

@@ -122,7 +122,7 @@ class ImportQduojHandler extends Handler {
     async post({ domainId }) {
         if (!this.request.files.file) throw new ValidationError('file');
         const stat = await fs.stat(this.request.files.file.path);
-        if (stat.size > 128 * 1024 * 1024) throw new BadRequestError('File too large');
+        if (stat.size > 128 * 1024 * 1024) throw new ValidationError('file', 'File too large');
         await this.ImportFromFile(domainId, this.request.files.file.path);
         this.response.redirect = this.url('problem_main');
     }

packages/ui-default/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hydrooj/ui-default",
-  "version": "4.17.5",
+  "version": "4.17.6",
   "author": "undefined <i@undefined.moe>",
   "license": "AGPL-3.0",
   "main": "hydro.js",

packages/ui-default/utils/request.js

@@ -9,6 +9,9 @@ const request = {
       $
         .ajax({
           dataType: 'json',
+          headers: {
+            Accept: 'application/json',
+          },
           ...options,
         })
         .fail((jqXHR, textStatus, errorThrown) => {