judge: support generate testdata (#695)

Co-authored-by: panda <panda_dtdyy@outlook.com>

packages/hydrooj/src/handler/judge.ts

@@ -1,20 +1,27 @@
 import assert from 'assert';
+import fs from 'fs-extra';
 import yaml from 'js-yaml';
 import { omit } from 'lodash';
 import { ObjectId } from 'mongodb';
+import sanitize from 'sanitize-filename';
+import {
+    FileLimitExceededError, ForbiddenError, ProblemIsReferencedError, ValidationError,
+} from '../error';
 import {
     JudgeResultBody, ProblemConfigFile, RecordDoc, Task, TestCase,
 } from '../interface';
 import { Logger } from '../logger';
 import * as builtin from '../model/builtin';
-import { STATUS } from '../model/builtin';
+import { PERM, STATUS } from '../model/builtin';
 import * as contest from '../model/contest';
 import domain from '../model/domain';
 import problem from '../model/problem';
 import record from '../model/record';
 import * as setting from '../model/setting';
 import storage from '../model/storage';
+import * as system from '../model/system';
 import task from '../model/task';
+import user from '../model/user';
 import * as bus from '../service/bus';
 import { updateJudge } from '../service/monitor';
 import {
@@ -189,11 +196,46 @@ export class SubmissionDataDownloadHandler extends Handler {
     }
 }
 
+export async function processJudgeFileCallback(rid: ObjectId, filename: string, filePath: string) {
+    const rdoc = await record.get(rid);
+    const [pdoc, udoc] = await Promise.all([
+        problem.get(rdoc.domainId, rdoc.pid),
+        user.getById(rdoc.domainId, rdoc.uid),
+    ]);
+    if (!udoc.own(pdoc, PERM.PERM_EDIT_PROBLEM_SELF) && !udoc.hasPerm(PERM.PERM_EDIT_PROBLEM)) throw new ForbiddenError();
+    if (pdoc.reference) throw new ProblemIsReferencedError('edit files');
+    const stat = await fs.stat(filePath);
+    if ((pdoc.data?.length || 0)
+        + (pdoc.additional_file?.length || 0)
+        >= system.get('limit.problem_files_max')) {
+        throw new FileLimitExceededError('count');
+    }
+    const size = Math.sum(
+        (pdoc.data || []).map((i) => i.size),
+        (pdoc.additional_file || []).map((i) => i.size),
+        stat.size,
+    );
+    if (size >= system.get('limit.problem_files_max_size')) {
+        throw new FileLimitExceededError('size');
+    }
+    await problem.addTestdata(pdoc.domainId, pdoc.docId, sanitize(filename), fs.createReadStream(filePath), udoc._id);
+}
+
+export class JudgeFileUpdateHandler extends Handler {
+    @post('rid', Types.ObjectId)
+    @post('name', Types.Filename)
+    async post(domainId: string, rid: ObjectId, filename: string) {
+        if (!this.request.files.file) throw new ValidationError('file');
+        await processJudgeFileCallback(rid, filename, this.request.files.file.filepath);
+        this.response.body = { ok: 1 };
+    }
+}
+
 class JudgeConnectionHandler extends ConnectionHandler {
     category = '#judge';
     processing: Task[] = [];
     closed = false;
-    query: any = { type: 'judge' };
+    query: any = { type: { $in: ['judge', 'generate'] } };
     rdocs: Record<string, RecordDoc> = {};
     ip: string;
     concurrency = 1;
@@ -282,6 +324,7 @@ class JudgeConnectionHandler extends ConnectionHandler {
 
 export async function apply(ctx) {
     ctx.Route('judge_files_download', '/judge/files', JudgeFilesDownloadHandler, builtin.PRIV.PRIV_JUDGE);
+    ctx.Route('judge_files_upload', '/judge/upload', JudgeFileUpdateHandler, builtin.PRIV.PRIV_JUDGE);
     ctx.Route('judge_submission_download', '/judge/code', SubmissionDataDownloadHandler, builtin.PRIV.PRIV_JUDGE);
     ctx.Connection('judge_conn', '/judge/conn', JudgeConnectionHandler, builtin.PRIV.PRIV_JUDGE);
 }

packages/hydrooj/src/handler/problem.ts

@@ -519,7 +519,7 @@ export class ProblemSubmitHandler extends ProblemDetailHandler {
         }
         if (pretest) {
             if (setting.langs[lang]?.pretest) lang = setting.langs[lang].pretest as string;
-            if (!['default', 'fileio', 'remote_judge'].includes(this.response.body.pdoc.config?.type)) {
+            if (!['default', 'remote_judge'].includes(this.response.body.pdoc.config?.type)) {
                 throw new ProblemNotAllowPretestError('type');
             }
         }
@@ -543,7 +543,7 @@ export class ProblemSubmitHandler extends ProblemDetailHandler {
         }
         const rid = await record.add(
             domainId, this.pdoc.docId, this.user._id, lang, code, true,
-            pretest ? { input, type: 'pretest' } : { contest: tid, files, type: tid ? 'contest' : 'judge' },
+            pretest ? { input, type: 'pretest' } : { contest: tid, files, type: 'judge' },
         );
         const rdoc = await record.get(domainId, rid);
         if (!pretest) {
@@ -698,6 +698,12 @@ export class ProblemFilesHandler extends ProblemDetailHandler {
         } else this.response.template = 'problem_files.html';
     }
 
+    async post() {
+        if (this.args.operation === 'get_links') return;
+        if (this.pdoc.reference) throw new ProblemIsReferencedError('edit files');
+        if (!this.user.own(this.pdoc, PERM.PERM_EDIT_PROBLEM_SELF)) this.checkPerm(PERM.PERM_EDIT_PROBLEM);
+    }
+
     @post('files', Types.Set)
     @post('type', Types.Range(['testdata', 'additional_file']), true)
     async postGetLinks(domainId: string, files: Set<string>, type = 'testdata') {
@@ -730,10 +736,8 @@ export class ProblemFilesHandler extends ProblemDetailHandler {
     @post('filename', Types.Filename, true)
     @post('type', Types.Range(['testdata', 'additional_file']), true)
     async postUploadFile(domainId: string, filename: string, type = 'testdata') {
-        if (this.pdoc.reference) throw new ProblemIsReferencedError('edit files');
         if (!this.request.files.file) throw new ValidationError('file');
         filename ||= this.request.files.file.originalFilename || String.random(16);
-        if (!this.user.own(this.pdoc, PERM.PERM_EDIT_PROBLEM_SELF)) this.checkPerm(PERM.PERM_EDIT_PROBLEM);
         const files = [];
         if (filename.endsWith('.zip') && type === 'testdata') {
             let zip: AdmZip;
@@ -744,7 +748,7 @@ export class ProblemFilesHandler extends ProblemDetailHandler {
             }
             const entries = zip.getEntries();
             for (const entry of entries) {
-                if (!entry.name) continue;
+                if (!entry.name || entry.isDirectory) continue;
                 files.push({
                     type,
                     name: sanitize(entry.name),
@@ -788,9 +792,7 @@ export class ProblemFilesHandler extends ProblemDetailHandler {
     @post('newNames', Types.ArrayOf(Types.Filename))
     @post('type', Types.Range(['testdata', 'additional_file']), true)
     async postRenameFiles(domainId: string, files: string[], newNames: string[], type = 'testdata') {
-        if (this.pdoc.reference) throw new ProblemIsReferencedError('rename files');
         if (files.length !== newNames.length) throw new ValidationError('files', 'newNames');
-        if (!this.user.own(this.pdoc, PERM.PERM_EDIT_PROBLEM_SELF)) this.checkPerm(PERM.PERM_EDIT_PROBLEM);
         await Promise.all(files.map(async (file, index) => {
             const newName = newNames[index];
             if (type === 'testdata') await problem.renameTestdata(domainId, this.pdoc.docId, file, newName, this.user._id);
@@ -802,12 +804,21 @@ export class ProblemFilesHandler extends ProblemDetailHandler {
     @post('files', Types.ArrayOf(Types.Filename))
     @post('type', Types.Range(['testdata', 'additional_file']), true)
     async postDeleteFiles(domainId: string, files: string[], type = 'testdata') {
-        if (this.pdoc.reference) throw new ProblemIsReferencedError('delete files');
-        if (!this.user.own(this.pdoc, PERM.PERM_EDIT_PROBLEM_SELF)) this.checkPerm(PERM.PERM_EDIT_PROBLEM);
         if (type === 'testdata') await problem.delTestdata(domainId, this.pdoc.docId, files, this.user._id);
         else await problem.delAdditionalFile(domainId, this.pdoc.docId, files, this.user._id);
         this.back();
     }
+
+    @post('std', Types.Filename)
+    @post('gen', Types.Filename)
+    async postGenerateTestdata(domainId: string, std: string, gen: string) {
+        if (!this.pdoc.data.find((i) => i.name === std)) throw new BadRequestError();
+        if (!this.pdoc.data.find((i) => i.name === gen)) throw new BadRequestError();
+        const rid = await record.add(domainId, this.pdoc.docId, this.user._id, '_', `${gen}\n${std}`, true, {
+            type: 'generate',
+        });
+        this.response.redirect = this.url('record_detail', { rid });
+    }
 }
 
 export class ProblemFileDownloadHandler extends ProblemDetailHandler {

packages/hydrooj/src/interface.ts

@@ -342,11 +342,13 @@ export interface JudgeMeta {
     problemOwner: number;
     hackRejudge?: string;
     rejudge?: boolean;
+    // FIXME stricter types
+    type?: string;
 }
 
 export interface JudgeRequest extends Omit<RecordDoc, '_id' | 'testCases'> {
     priority: number;
-    type: 'judge';
+    type: 'judge' | 'generate';
     rid: ObjectId;
     config: ProblemConfigFile;
     meta: JudgeMeta;

packages/hydrooj/src/lib/testdataConfig.ts

@@ -33,10 +33,7 @@ export async function parseConfig(config: string | ProblemConfigFile = {}) {
     if (result.timeMax < result.timeMin) result.timeMax = result.timeMin = 1000;
     if (cfg.langs) result.langs = cfg.langs;
     if (cfg.redirect) result.redirect = cfg.redirect.split('/', 2) as any;
-    if (cfg.filename && result.type === 'default') {
-        result.type = 'fileio';
-        result.subType = cfg.filename;
-    }
+    if (cfg.filename && result.type === 'default') result.subType = cfg.filename;
     return result;
 }
 

packages/hydrooj/src/model/record.ts

@@ -26,6 +26,9 @@ export default class RecordModel {
         'files',
     ];
 
+    static RECORD_PRETEST = new ObjectId('000000000000000000000000');
+    static RECORD_GENERATE = new ObjectId('000000000000000000000001');
+
     static async submissionPriority(uid: number, base: number = 0) {
         const timeRecent = await RecordModel.coll
             .find({ _id: { $gte: Time.getObjectID(moment().add(-30, 'minutes')) }, uid, rejudged: { $ne: true } })
@@ -82,8 +85,9 @@ export default class RecordModel {
         }
         meta = { ...meta, problemOwner: pdoc.owner };
         if (typeof pdoc.config === 'string') throw new Error(pdoc.config);
-        const type = (pdoc.config.type === 'remote_judge' && rdoc.contest?.toHexString() !== '0'.repeat(24)) ? 'remotejudge' : 'judge';
-        config.type = pdoc.config.type === 'fileio' ? 'default' : pdoc.config.type as any;
+        let type = 'judge';
+        if (pdoc.config.type === 'remote_judge' && rdoc.contest?.toHexString() !== '0'.repeat(24)) type = 'remotejudge';
+        else if (meta?.type === 'generate') type = 'generate';
         return await task.addMany(rids.map((rid) => ({
             ...(pdoc.config as any),
             priority,
@@ -104,7 +108,7 @@ export default class RecordModel {
             contest?: ObjectId,
             input?: string,
             files?: Record<string, string>,
-            type: 'judge' | 'rejudge' | 'contest' | 'pretest' | 'hack',
+            type: 'judge' | 'rejudge' | 'pretest' | 'hack' | 'generate',
         } = { type: 'judge' },
     ) {
         const data: RecordDoc = {
@@ -125,6 +129,7 @@ export default class RecordModel {
             judgeAt: null,
             rejudged: false,
         };
+        let isContest = !!args.contest;
         if (args.contest) data.contest = args.contest;
         if (args.files) data.files = args.files;
         if (args.type === 'rejudge') {
@@ -132,12 +137,16 @@ export default class RecordModel {
             data.rejudged = true;
         } else if (args.type === 'pretest') {
             data.input = args.input || '';
-            data.contest = new ObjectId('000000000000000000000000');
+            isContest = false;
+            data.contest = RecordModel.RECORD_PRETEST;
         }
         const res = await RecordModel.coll.insertOne(data);
         if (addTask) {
-            const priority = await RecordModel.submissionPriority(uid, args.type === 'pretest' ? -20 : (args.type === 'contest' ? 50 : 0));
-            await RecordModel.judge(domainId, res.insertedId, priority, args.type === 'contest' ? { detail: false } : {}, { rejudge: data.rejudged });
+            const priority = await RecordModel.submissionPriority(uid, args.type === 'pretest' ? -20 : (isContest ? 50 : 0));
+            await RecordModel.judge(domainId, res.insertedId, priority, isContest ? { detail: false } : {}, {
+                type: args.type,
+                rejudge: data.rejudged,
+            });
         }
         return res.insertedId;
     }

packages/hydrooj/src/model/schedule.ts

@@ -6,6 +6,7 @@ import { Schedule } from '../interface';
 import { Logger } from '../logger';
 import * as bus from '../service/bus';
 import db from '../service/db';
+import RecordModel from './record';
 
 const logger = new Logger('model/schedule');
 const coll = db.collection('schedule');
@@ -136,7 +137,7 @@ export async function apply(ctx: Context) {
     ctx.worker = Worker;
 
     Worker.addHandler('task.daily', async () => {
-        await global.Hydro.model.record.coll.deleteMany({ contest: new ObjectId('000000000000000000000000') });
+        await RecordModel.coll.deleteMany({ contest: { $in: [RecordModel.RECORD_PRETEST, RecordModel.RECORD_GENERATE] } });
         await global.Hydro.script.rp?.run({}, new Logger('task/rp').debug);
         await global.Hydro.script.problemStat?.run({}, new Logger('task/problem').debug);
         if (global.Hydro.model.system.get('server.checkUpdate') && !(new Date().getDay() % 3)) {

packages/hydrooj/src/script/problemStat.ts

@@ -1,8 +1,8 @@
 /* eslint-disable no-await-in-loop */
-import { ObjectId } from 'mongodb';
 import Schema from 'schemastery';
 import { STATUS } from '../model/builtin';
 import * as document from '../model/document';
+import RecordModel from '../model/record';
 import db from '../service/db';
 
 const sumStatus = (status) => ({ $sum: { $cond: [{ $eq: ['$status', status] }, 1, 0] } });
@@ -10,7 +10,7 @@ const sumStatus = (status) => ({ $sum: { $cond: [{ $eq: ['$status', status] }, 1
 export async function udoc(report) {
     report({ message: 'Udoc' });
     const pipeline = [
-        { $match: { contest: { $ne: new ObjectId('000000000000000000000000') } } },
+        { $match: { contest: { $nin: [RecordModel.RECORD_PRETEST, RecordModel.RECORD_GENERATE] } } },
         {
             $group: {
                 _id: { domainId: '$domainId', pid: '$pid', uid: '$uid' },
@@ -49,7 +49,7 @@ export async function udoc(report) {
 export async function psdoc(report) {
     report({ message: 'Psdoc' });
     const pipeline = [
-        { $match: { contest: { $ne: new ObjectId('000000000000000000000000') } } },
+        { $match: { contest: { $nin: [RecordModel.RECORD_PRETEST, RecordModel.RECORD_GENERATE] } } },
         {
             $group: {
                 _id: { domainId: '$domainId', pid: '$pid', uid: '$uid' },
@@ -68,7 +68,7 @@ export async function pdoc(report) {
     const pipeline = [
         {
             $match: {
-                contest: { $ne: new ObjectId('000000000000000000000000') },
+                contest: { $nin: [RecordModel.RECORD_PRETEST, RecordModel.RECORD_GENERATE] },
                 status: { $ne: STATUS.STATUS_CANCELED },
             },
         },