Merge branch 'master' into contest-access-lock

packages/hydrooj/src/handler/contest.ts

@@ -3,7 +3,9 @@ import { statSync } from 'fs-extra';
 import { pick } from 'lodash';
 import moment from 'moment-timezone';
 import { ObjectID } from 'mongodb';
-import { Counter, sortFiles, Time } from '@hydrooj/utils/lib/utils';
+import {
+    Counter, sortFiles, streamToBuffer, Time,
+} from '@hydrooj/utils/lib/utils';
 import {
     BadRequestError, ContestNotEndedError, ContestNotFoundError, ContestNotLiveError,
     ContestScoreboardHiddenError, FileLimitExceededError, FileUploadError, ForbiddenError,
@@ -506,9 +508,18 @@ export class ContestCodeHandler extends Handler {
         const rdocs = await record.getMulti(domainId, {
             _id: { $in: Array.from(Object.keys(rnames)).map((id) => new ObjectID(id)) },
         }).toArray();
-        for (const rdoc of rdocs) {
-            zip.addFile(`${rnames[rdoc._id.toHexString()]}.${rdoc.lang}`, Buffer.from(rdoc.code));
-        }
+        await Promise.all(rdocs.map(async (rdoc) => {
+            if (rdoc.files?.code) {
+                const [id, filename] = rdoc.files?.code?.split('#') || [];
+                if (!id) return;
+                zip.addFile(
+                    `${rnames[rdoc._id.toHexString()]}.${filename || 'txt'}`,
+                    await streamToBuffer(storage.get(id)),
+                );
+            } else if (rdoc.code) {
+                zip.addFile(`${rnames[rdoc._id.toHexString()]}.${rdoc.lang}`, Buffer.from(rdoc.code));
+            }
+        }));
         this.binary(zip.toBuffer(), `${tdoc.title}.zip`);
     }
 }

packages/hydrooj/src/handler/problem.ts

@@ -444,6 +444,7 @@ export class ProblemDetailHandler extends ContestDetailBaseHandler {
             contest: { $ne: new ObjectID('0'.repeat(24)) },
             'files.hack': { $exists: false },
         }).project({ _id: 1, contest: 1 }).toArray();
+        if (!this.pdoc.config || typeof this.pdoc.config === 'string') throw new ProblemConfigError();
         const priority = await record.submissionPriority(this.user._id, -10000 - rdocs.length * 5 - 50);
         await record.reset(domainId, rdocs.map((rdoc) => rdoc._id), true);
         await Promise.all([
@@ -527,13 +528,13 @@ export class ProblemSubmitHandler extends ProblemDetailHandler {
             if (!file || file.size === 0) throw new ValidationError('code');
             const sizeLimit = config.type === 'submit_answer' ? 128 * 1024 * 1024 : 65535;
             if (file.size > sizeLimit) throw new ValidationError('file');
-            if (file.size > 65535 || await isBinaryFile(file.filepath, file.size)) {
+            if (config.type !== 'submit_answer' || (file.size < 65535 && !await isBinaryFile(file.filepath, file.size))) {
+                // TODO auto detect & convert encoding
+                code = await readFile(file.filepath, 'utf-8');
+            } else {
                 const id = nanoid();
                 await storage.put(`submission/${this.user._id}/${id}`, file.filepath, this.user._id);
                 files.code = `${this.user._id}/${id}#${file.originalFilename}`;
-            } else {
-                // TODO auto detect & convert encoding
-                code = await readFile(file.filepath, 'utf-8');
             }
         }
         const rid = await record.add(

packages/hydrooj/src/handler/record.ts

@@ -1,7 +1,8 @@
 import { omit, pick, throttle } from 'lodash';
 import { FilterQuery, ObjectID } from 'mongodb';
 import {
-    ContestNotAttendedError, ContestNotFoundError, HackRejudgeFailedError, PermissionError,
+    ContestNotAttendedError, ContestNotFoundError, HackRejudgeFailedError,
+    PermissionError, ProblemConfigError,
     ProblemNotFoundError, RecordNotFoundError, UserNotFoundError,
 } from '../error';
 import { RecordDoc, Tdoc } from '../interface';
@@ -191,6 +192,8 @@ class RecordDetailHandler extends ContestDetailBaseHandler {
 
     @param('rid', Types.ObjectID)
     async postRejudge(domainId: string, rid: ObjectID) {
+        const pdoc = await problem.get(domainId, this.rdoc.pid);
+        if (!pdoc?.config || typeof pdoc.config === 'string') throw new ProblemConfigError();
         const priority = await record.submissionPriority(this.user._id, -20);
         const isContest = this.rdoc.contest && this.rdoc.contest.toString() !== '000000000000000000000000';
         await record.reset(domainId, rid, true);

packages/hydrooj/src/handler/user.ts

@@ -129,7 +129,7 @@ class UserLoginHandler extends Handler {
         if (udoc._tfa && !verifyToken(udoc._tfa, tfa)) throw new InvalidTokenError('2FA');
         udoc.checkPassword(password);
         await user.setById(udoc._id, { loginat: new Date(), loginip: this.request.ip });
-        if (!udoc.hasPriv(PRIV.PRIV_USER_PROFILE)) throw new BlacklistedError(uname);
+        if (!udoc.hasPriv(PRIV.PRIV_USER_PROFILE)) throw new BlacklistedError(uname, udoc.banReason);
         this.session.viewLang = '';
         this.session.uid = udoc._id;
         this.session.sudo = null;
@@ -471,5 +471,5 @@ export async function apply(ctx) {
     ctx.Route('user_lostpass', '/lostpass', UserLostPassHandler);
     ctx.Route('user_lostpass_with_code', '/lostpass/:code', UserLostPassWithCodeHandler);
     ctx.Route('user_delete', '/user/delete', UserDeleteHandler, PRIV.PRIV_USER_PROFILE);
-    ctx.Route('user_detail', '/user/:uid', UserDetailHandler);
+    ctx.Route('user_detail', '/user/:uid(-?\\d+)', UserDetailHandler);
 }

packages/hydrooj/src/model/record.ts

@@ -232,7 +232,7 @@ export default class RecordModel {
 
 export function apply(ctx: Context) {
     // Mark problem as deleted
-    ctx.on('problem/delete', (domainId, docId) => RecordModel.coll.updateMany({ domainId, pid: docId }, { $set: { pid: -1 } }));
+    ctx.on('problem/delete', (domainId, docId) => RecordModel.coll.deleteMany({ domainId, pid: docId }));
     ctx.on('domain/delete', (domainId) => RecordModel.coll.deleteMany({ domainId }));
     ctx.on('ready', () => db.ensureIndexes(
         RecordModel.coll,

packages/hydrooj/src/model/setting.ts

@@ -152,6 +152,7 @@ AccountSetting(
         'Choose the background image in your profile page.'),
     Setting('setting_storage', 'unreadMsg', 0, 'number', 'Unread Message Count', null, FLAG_DISABLED | FLAG_HIDDEN),
     Setting('setting_storage', 'badge', '', 'text', 'badge info', null, FLAG_DISABLED | FLAG_HIDDEN),
+    Setting('setting_storage', 'banReason', '', 'text', 'ban reason', null, FLAG_DISABLED | FLAG_HIDDEN),
 );
 
 DomainSetting(

packages/hydrooj/src/model/user.ts

@@ -272,8 +272,7 @@ class UserModel {
         if (_id < -999) return null;
         const udoc = await coll.findOne({ _id });
         if (!udoc) throw new UserNotFoundError(_id);
-        udoc[field] = udoc[field] + n || n;
-        await coll.updateOne({ _id }, { $set: { [field]: udoc[field] } });
+        await coll.updateOne({ _id }, { $inc: { [field]: n } });
         deleteUserCache(udoc);
         return udoc;
     }
@@ -401,9 +400,9 @@ class UserModel {
     }
 
     @ArgMethod
-    static ban(uid: number) {
+    static ban(uid: number, reason = '') {
         return Promise.all([
-            UserModel.setPriv(uid, PRIV.PRIV_NONE),
+            UserModel.setById(uid, { priv: PRIV.PRIV_NONE, banReason: reason }),
             token.delByUid(uid),
         ]);
     }

packages/hydrooj/src/service/server.ts

@@ -1,5 +1,6 @@
 import http from 'http';
-import { resolve } from 'path';
+import { tmpdir } from 'os';
+import { join, resolve } from 'path';
 import cac from 'cac';
 import fs from 'fs-extra';
 import Koa from 'koa';
@@ -484,14 +485,22 @@ export async function apply(pluginContext: Context) {
 ${ctx.response.status} ${endTime - startTime}ms ${ctx.response.length}`);
         });
     }
+    const uploadDir = join(tmpdir(), 'hydro', 'upload', process.env.NODE_APP_INSTANCE || '0');
+    fs.ensureDirSync(uploadDir);
+    logger.debug('Using upload dir: %s', uploadDir);
     app.use(Body({
         multipart: true,
         jsonLimit: '8mb',
         formLimit: '8mb',
         formidable: {
+            uploadDir,
             maxFileSize: parseMemoryMB(system.get('server.upload') || '256m') * 1024 * 1024,
+            keepExtensions: true,
         },
     }));
+    pluginContext.on('app/exit', () => {
+        fs.emptyDirSync(uploadDir);
+    });
     const layers = [baseLayer, rendererLayer(router, logger), responseLayer(logger), userLayer];
     app.use(async (ctx, next) => await next().catch(console.error)).use(domainLayer);
     app.use(router.routes()).use(router.allowedMethods());

packages/hydrooj/src/service/storage.ts

@@ -60,6 +60,14 @@ export function encodeRFC5987ValueChars(str: string) {
     );
 }
 
+const convertPath = (p: string) => {
+    p = p.trim();
+    if (p.includes('..') || p.includes('//') || p.endsWith('/.') || p === '.' || p.includes('/./')) {
+        throw new Error('Invalid path');
+    }
+    return p;
+};
+
 class RemoteStorageService {
     public client: S3Client;
     public error = '';
@@ -113,7 +121,7 @@ class RemoteStorageService {
     }
 
     async put(target: string, file: string | Buffer | Readable, meta: Record<string, string> = {}) {
-        if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
+        target = convertPath(target);
         if (typeof file === 'string') file = createReadStream(file);
         const params: PutObjectCommandInput = {
             Bucket: this.opts.bucket,
@@ -138,7 +146,7 @@ class RemoteStorageService {
     }
 
     async get(target: string, path?: string) {
-        if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
+        target = convertPath(target);
         const res = await this.client.send(new GetObjectCommand({
             Bucket: this.opts.bucket,
             Key: target,
@@ -163,9 +171,8 @@ class RemoteStorageService {
     }
 
     async del(target: string | string[]) {
-        if (typeof target === 'string') {
-            if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
-        } else if (target.find((t) => t.includes('..') || t.includes('//'))) throw new Error('Invalid path');
+        if (typeof target === 'string') target = convertPath(target);
+        else target = target.map(convertPath);
         if (typeof target === 'string') {
             return await this.client.send(new DeleteObjectCommand({
                 Bucket: this.opts.bucket,
@@ -186,7 +193,7 @@ class RemoteStorageService {
     }
 
     async getMeta(target: string) {
-        if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
+        target = convertPath(target);
         const res = await this.client.send(new HeadObjectCommand({
             Bucket: this.opts.bucket,
             Key: target,
@@ -200,7 +207,7 @@ class RemoteStorageService {
     }
 
     async signDownloadLink(target: string, filename?: string, noExpire = false, useAlternativeEndpointFor?: 'user' | 'judge'): Promise<string> {
-        if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
+        target = convertPath(target);
         const url = await getSignedUrl(this.client, new GetObjectCommand({
             Bucket: this.opts.bucket,
             Key: target,
@@ -234,8 +241,8 @@ class RemoteStorageService {
     }
 
     async copy(src: string, target: string) {
-        if (src.includes('..') || src.includes('//')) throw new Error('Invalid path');
-        if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
+        src = convertPath(src);
+        target = convertPath(target);
         return await this.client.send(new CopyObjectCommand({
             Bucket: this.opts.bucket,
             Key: target,
@@ -262,8 +269,7 @@ class LocalStorageService {
     }
 
     async put(target: string, file: string | Buffer | Readable) {
-        if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
-        target = resolve(this.dir, target);
+        target = resolve(this.dir, convertPath(target));
         await ensureDir(dirname(target));
         if (typeof file === 'string') await copyFile(file, target);
         else if (file instanceof Buffer) await writeFile(target, file);
@@ -271,22 +277,19 @@ class LocalStorageService {
     }
 
     async get(target: string, path?: string) {
-        if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
-        target = resolve(this.dir, target);
+        target = resolve(this.dir, convertPath(target));
         if (!existsSync(target)) throw new Error('File not found');
         if (path) await copyFile(target, path);
         return createReadStream(target);
     }
 
     async del(target: MaybeArray<string>) {
-        const targets = typeof target === 'string' ? [target] : target;
-        if (targets.find((i) => i.includes('..') || i.includes('//'))) throw new Error('Invalid path');
+        const targets = (typeof target === 'string' ? [target] : target).map(convertPath);
         await Promise.all(targets.map((i) => remove(resolve(this.dir, i))));
     }
 
     async getMeta(target: string) {
-        if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
-        target = resolve(this.dir, target);
+        target = resolve(this.dir, convertPath(target));
         const file = await stat(target);
         return {
             size: file.size,
@@ -302,7 +305,7 @@ class LocalStorageService {
     }
 
     async signDownloadLink(target: string, filename = '', noExpire = false, useAlternativeEndpointFor?: 'user' | 'judge'): Promise<string> {
-        if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
+        target = convertPath(target);
         const url = new URL('https://localhost/storage');
         url.searchParams.set('target', target);
         if (filename) url.searchParams.set('filename', filename);
@@ -322,10 +325,8 @@ class LocalStorageService {
     }
 
     async copy(src: string, target: string) {
-        if (src.includes('..') || src.includes('//')) throw new Error('Invalid path');
-        if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
-        src = resolve(this.dir, src);
-        target = resolve(this.dir, target);
+        src = resolve(this.dir, convertPath(src));
+        target = resolve(this.dir, convertPath(target));
         await copyFile(src, target);
         return { etag: target, lastModified: new Date() };
     }

packages/hydrooj/src/ui.ts

@@ -2,8 +2,14 @@
 import { Logger } from './logger';
 import * as bus from './service/bus';
 
+let terminating = false;
 async function terminate() {
+    if (terminating) process.exit(1);
     let hasError = false;
+    terminating = true;
+    setTimeout(() => {
+        new Logger('exit').info('Cleaning up temporary files... (Press Ctrl-C again to force exit)');
+    }, 1000);
     try {
         await bus.parallel('app/exit');
     } catch (e) {
@@ -12,6 +18,7 @@ async function terminate() {
     process.exit(hasError ? 1 : 0);
 }
 process.on('SIGINT', terminate);
+process.on('SIGTERM', terminate);
 
 const shell = new Logger('shell');
 async function executeCommand(input: string) {