judge: merge processTestdata

packages/hydrojudge/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@hydrooj/hydrojudge",
     "bin": "bin/hydrojudge.js",
-    "version": "2.20.1",
+    "version": "2.20.2",
     "main": "package.json",
     "author": "undefined <i@undefined.moe>",
     "repository": "https://github.com/hydro-dev/Hydro.git",

packages/hydrojudge/setting.yaml

@@ -37,11 +37,6 @@ memoryMax:
   type: text
   name: memoryMax
   default: 512m
-retry_delay_sec:
-  type: number
-  name: retry_delay_sec
-  desc: Retry delay sec
-  default: 15
 cache_dir:
   type: text
   name: cache_dir

packages/hydrojudge/src/cases.ts

@@ -1,3 +1,4 @@
+/* eslint-disable no-await-in-loop */
 import path from 'path';
 import fs from 'fs-extra';
 import yaml from 'js-yaml';
@@ -35,6 +36,32 @@ async function collectFiles(folder: string) {
     return files;
 }
 
+export async function processTestdata(folder: string) {
+    let files = await fs.readdir(folder);
+    if (files.length <= 2) {
+        if (files.length === 2) files.splice(files.indexOf('version'), 1);
+        if (fs.statSync(path.resolve(folder, files[0])).isDirectory()) {
+            folder = path.resolve(folder, files[0]);
+            files = await fs.readdir(folder);
+        }
+    }
+    const ini = files.filter((i) => i.toLowerCase() === 'config.ini')[0];
+    if (!ini) return;
+    const t = fs.readFileSync(path.resolve(folder, ini), 'utf8');
+    await fs.writeFile(path.resolve(folder, 'config.ini'), t.toLowerCase());
+    for (const i of files) {
+        if (i.toLowerCase() === 'input') await fs.rename(`${folder}/${i}`, `${folder}/input`);
+        if (i.toLowerCase() === 'output') await fs.rename(`${folder}/${i}`, `${folder}/output`);
+    }
+    await Promise.all(['input', 'output'].flatMap(async (f) => {
+        const dir = path.resolve(folder, f);
+        const sf = await fs.readdir(dir);
+        return sf
+            .filter((i) => i !== i.toLowerCase())
+            .map((i) => fs.rename(`${dir}/${i}`, `${dir}/${i.toLowerCase()}`));
+    }));
+}
+
 export default async function readCases(folder: string, cfg: Record<string, any> = {}, args) {
     const iniConfig = path.resolve(folder, 'config.ini');
     const yamlConfig = path.resolve(folder, 'config.yaml');

packages/hydrojudge/src/check.ts

@@ -16,6 +16,9 @@ export async function check(config: CheckConfig): Promise<[number, number, strin
 }
 
 export async function compileChecker(getLang: Function, checkerType: string, checker: string, copyIn: any): Promise<Execute> {
+    if (['default', 'strict'].includes(checkerType)) {
+        return { execute: '', copyIn: {}, clean: () => Promise.resolve(null) };
+    }
     if (!checkers[checkerType]) throw new SystemError('Unknown checker type {0}.', [checkerType]);
     if (checkerType === 'testlib') copyIn['testlib.h'] = { src: testlibSrc };
     const s = checker.replace('@', '.').split('.');

packages/hydrojudge/src/config.ts

@@ -1,4 +1,3 @@
-/* eslint-disable prefer-const */
 import os from 'os';
 import path from 'path';
 import cac from 'cac';
@@ -9,19 +8,19 @@ const argv = cac().parse();
 // eslint-disable-next-line @typescript-eslint/naming-convention
 let CONFIG_FILE = path.resolve(os.homedir(), '.config', 'hydro', 'judge.yaml');
 
-let config = {
+const config = {
     cache_dir: path.resolve(os.homedir(), '.cache', 'hydro', 'judge'),
     tmp_dir: path.resolve(os.tmpdir(), 'hydro', 'judge'),
     tmpfs_size: '256m',
     stdio_size: '32m',
     memoryMax: '512m',
     strict_memory: false,
-    retry_delay_sec: 15,
     sandbox_host: 'http://localhost:5050',
     testcases_max: 100,
     total_time_limit: 60,
     processLimit: 32,
     parallelism: 2,
+    singleTaskParallelism: 2,
     rerun: 0,
     rate: 1,
     config: null,
@@ -29,10 +28,11 @@ let config = {
     env: `\
 PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 HOME=/w`,
+    hosts: {},
     secret: String.random(32),
 };
 
-export function getConfig(key: string) {
+export function getConfig(key: keyof typeof config) {
     return global.Hydro ? global.Hydro.model.system.get(`hydrojudge.${key}`) : config[key];
 }
 
@@ -61,5 +61,5 @@ if (!global.Hydro) {
         config.sandbox_host = path.resolve(process.env.EXECUTION_HOST || argv.options.sandbox);
     }
     const configFile = fs.readFileSync(CONFIG_FILE).toString();
-    config = { ...config, ...yaml.load(configFile) as any };
+    Object.assign(config, yaml.load(configFile) as any);
 }

packages/hydrojudge/src/daemon.ts

@@ -18,7 +18,7 @@ import './utils';
 import { getConfig } from './config';
 import * as Session from './hosts/index';
 import log from './log';
-import { Queue, sleep } from './utils';
+import { Queue } from './utils';
 
 declare global {
     namespace NodeJS {
@@ -28,8 +28,8 @@ declare global {
         }
     }
 }
-if (!global.onDestroy) global.onDestroy = [];
-if (!global.hosts) global.hosts = [];
+global.onDestroy ||= [];
+global.hosts ||= [];
 let exit = false;
 
 const terminate = async () => {
@@ -60,28 +60,16 @@ async function worker(queue: Queue<any>) {
 
 async function daemon() {
     const _hosts = getConfig('hosts');
-    const delay = getConfig('retry_delay_sec');
     const hosts = {};
     const queue = new Queue<any>();
+    worker(queue).catch((e) => log.error(e));
     for (const i in _hosts) {
         _hosts[i].host = _hosts[i].host || i;
-        hosts[i] = new Session[_hosts[i].type || 'vj4'](_hosts[i]);
+        hosts[i] = new Session[_hosts[i].type || 'hydro'](_hosts[i]);
         await hosts[i].init();
     }
+    for (const i in hosts) hosts[i].consume(queue);
     global.hosts = hosts;
-    worker(queue);
-    for (const i in hosts) {
-        while ('Orz twd2') {
-            try {
-                await hosts[i].consume(queue);
-                break;
-            } catch (e) {
-                log.error(e, e.stack);
-                log.info(`在 ${delay} 秒后重试`);
-                await sleep(delay * 1000);
-            }
-        }
-    }
 }
 
 if (require.main === module) daemon();

packages/hydrojudge/src/error.ts

@@ -5,7 +5,7 @@ export class CompileError extends Error {
     stdout: string;
     stderr: string;
     status: string;
-    type: string;
+    type = 'CompileError';
 
     constructor(obj: string | CompileErrorInfo) {
         super('Compile Error');
@@ -17,48 +17,29 @@ export class CompileError extends Error {
             this.stderr = obj.stderr || '';
             this.status = obj.status ? STATUS_TEXTS[obj.status] || '' : '';
         }
-        this.type = 'CompileError';
     }
 }
 
 export class FormatError extends Error {
-    type: string;
-    params: any[];
+    type = 'FormatError';
 
-    constructor(message: string, params = []) {
+    constructor(message: string, public params = []) {
         super(message);
-        this.type = 'FormatError';
-        this.params = params;
     }
 }
 
 export class RuntimeError extends Error {
-    type: string;
-    detail: any;
+    type = 'RuntimeError';
 
-    constructor(detail: string, message: string) {
+    constructor(public detail: string, message: string) {
         super(message);
-        this.type = 'RuntimeError';
-        this.detail = detail;
     }
 }
 
 export class SystemError extends Error {
-    type: string;
-    params: any[];
+    type = 'SystemError';
 
-    constructor(message: string, params = []) {
+    constructor(message: string, public params = []) {
         super(message);
-        this.type = 'SystemError';
-        this.params = params;
-    }
-}
-
-export class TooFrequentError extends Error {
-    type: string;
-
-    constructor(message: string) {
-        super(message);
-        this.type = 'TooFrequentError';
     }
 }

packages/hydrojudge/src/hosts/hydro.ts

@@ -10,7 +10,7 @@ import WebSocket from 'ws';
 import { LangConfig } from '@hydrooj/utils/lib/lang';
 import { STATUS } from '@hydrooj/utils/lib/status';
 import type { JudgeResultBody } from 'hydrooj';
-import readCases from '../cases';
+import readCases, { processTestdata } from '../cases';
 import { getConfig } from '../config';
 import { CompileError, FormatError, SystemError } from '../error';
 import judge from '../judge';
@@ -232,7 +232,7 @@ export default class Hydro {
             queue.start();
             await Promise.all(tasks);
             fs.writeFileSync(path.join(filePath, 'etags'), JSON.stringify(version));
-            await this.processData(filePath).catch(noop);
+            await processTestdata(filePath);
         }
         fs.writeFileSync(path.join(filePath, 'lastUsage'), new Date().getTime().toString());
         return filePath;
@@ -332,34 +332,6 @@ export default class Hydro {
         }
     }
 
-    async processData(folder: string) { // eslint-disable-line class-methods-use-this
-        let files = await fs.readdir(folder);
-        let ini = false;
-        if (files.length <= 2) {
-            if (files.length === 2) files.splice(files.indexOf('version'), 1);
-            const s = fs.statSync(path.resolve(folder, files[0]));
-            if (s.isDirectory()) folder = path.resolve(folder, files[0]);
-        }
-        files = await fs.readdir(folder);
-        for (const i of files) {
-            if (i.toLowerCase() === 'config.ini') {
-                ini = true;
-                await fs.rename(`${folder}/${i}`, `${folder}/config.ini`);
-                break;
-            }
-        }
-        if (ini) {
-            for (const i of files) {
-                if (i.toLowerCase() === 'input') await fs.rename(`${folder}/${i}`, `${folder}/input`);
-                else if (i.toLowerCase() === 'output') await fs.rename(`${folder}/${i}`, `${folder}/output`);
-            }
-            files = await fs.readdir(`${folder}/input`);
-            for (const i of files) await fs.rename(`${folder}/input/${i}`, `${folder}/input/${i.toLowerCase()}`);
-            files = await fs.readdir(`${folder}/output`);
-            for (const i of files) await fs.rename(`${folder}/output/${i}`, `${folder}/output/${i.toLowerCase()}`);
-        }
-    }
-
     async retry(queue: Queue<any>) {
         this.consume(queue).catch(() => {
             setTimeout(() => this.retry(queue), 30000);

packages/hydrojudge/src/judge/default.ts

@@ -177,21 +177,17 @@ export const judge = async (ctx: Context, startPromise = Promise.resolve()) => {
             ),
             ctx.next,
         ),
-        (() => {
-            if (['default', 'strict'].includes(ctx.config.checker_type || 'default')) {
-                return { execute: '', copyIn: {}, clean: () => Promise.resolve(null) };
-            }
-            const copyIn = { user_code: ctx.code };
-            for (const file of ctx.config.judge_extra_files) {
-                copyIn[parseFilename(file)] = { src: file };
-            }
-            return compileChecker(
-                ctx.getLang,
-                ctx.config.checker_type,
-                ctx.config.checker,
-                copyIn,
-            );
-        })(),
+        compileChecker(
+            ctx.getLang,
+            ctx.config.checker_type,
+            ctx.config.checker,
+            {
+                user_code: ctx.code,
+                ...Object.fromEntries(
+                    (ctx.config.judge_extra_files || []).map((i) => [parseFilename(i), { src: i }]),
+                ),
+            },
+        ),
     ]);
     ctx.clean.push(ctx.execute.clean, ctx.checker.clean);
     await startPromise;
@@ -202,7 +198,7 @@ export const judge = async (ctx: Context, startPromise = Promise.resolve()) => {
     ctx.total_memory_usage_kb = 0;
     ctx.total_time_usage_ms = 0;
     ctx.rerun = getConfig('rerun') || 0;
-    ctx.queue = new Queue({ concurrency: getConfig('parallelism') });
+    ctx.queue = new Queue({ concurrency: getConfig('singleTaskParallelism') });
     ctx.failed = {};
     for (const sid in ctx.config.subtasks) tasks.push(judgeSubtask(ctx.config.subtasks[sid], sid)(ctx));
     const scores = await Promise.all(tasks);

packages/hydrojudge/src/judge/interface.ts

@@ -52,8 +52,8 @@ export interface Config {
     detail?: boolean;
     filename?: string;
 
-    judge_extra_files?: string[];
-    user_extra_files?: string[];
+    judge_extra_files: string[];
+    user_extra_files: string[];
     template?: Record<string, [string, string]>;
 
     checker?: string;

packages/hydrojudge/src/judge/submit_answer.ts

@@ -122,9 +122,6 @@ export const judge = async (ctx: Context, startPromise = Promise.resolve()) => {
     if (!ctx.config.subtasks.length) throw new FormatError('Problem data not found.');
     startPromise.then(() => ctx.next({ status: STATUS.STATUS_COMPILING }));
     ctx.checker = await (() => {
-        if (['default', 'strict'].includes(ctx.config.checker_type || 'default')) {
-            return { execute: '', copyIn: {}, clean: () => Promise.resolve(null) };
-        }
         const copyIn = { user_code: ctx.code };
         for (const file of ctx.config.judge_extra_files) {
             copyIn[parseFilename(file)] = { src: file };

packages/hydrojudge/src/log.ts

@@ -1,29 +1,9 @@
 import { format, inspect, InspectOptions } from 'util';
 import cac from 'cac';
+import { Time } from '@hydrooj/utils/lib/utils';
 
 const argv = cac().parse();
 
-namespace Time {
-    export const second = 1000;
-    export const minute = second * 60;
-    export const hour = minute * 60;
-    export const day = hour * 24;
-    export const week = day * 7;
-    export function formatTimeShort(ms: number) {
-        const abs = Math.abs(ms);
-        if (abs >= day - hour / 2) {
-            return `${Math.round(ms / day)}d`;
-        } if (abs >= hour - minute / 2) {
-            return `${Math.round(ms / hour)}h`;
-        } if (abs >= minute - second / 2) {
-            return `${Math.round(ms / minute)}m`;
-        } if (abs >= second) {
-            return `${Math.round(ms / second)}s`;
-        }
-        return `${ms}ms`;
-    }
-}
-
 const colors = [
     20, 21, 26, 27, 32, 33, 38, 39, 40, 41, 42, 43, 44, 45, 56, 57, 62,
     63, 68, 69, 74, 75, 76, 77, 78, 79, 80, 81, 92, 93, 98, 99, 112, 113,

packages/hydrojudge/src/sandbox.ts

@@ -1,5 +1,6 @@
 import cac from 'cac';
 import fs from 'fs-extra';
+import PQueue from 'p-queue';
 import { ParseEntry } from 'shell-quote';
 import { STATUS } from '@hydrooj/utils/lib/status';
 import { getConfig } from './config';
@@ -191,3 +192,9 @@ export async function run(execute: string, params?: Parameter): Promise<SandboxA
     }
     return await adaptResult(result, params);
 }
+
+const queue = new PQueue({ concurrency: getConfig('parallelism') });
+
+export function runQueued(execute: string, params?: Parameter, priority = 0) {
+    return queue.add(() => run(execute, params), { priority });
+}

packages/hydrojudge/src/service.ts

@@ -10,7 +10,7 @@ import { STATUS } from '@hydrooj/utils/lib/status';
 import type { JudgeResultBody } from 'hydrooj';
 import { Logger } from 'hydrooj/src/logger';
 import * as monitor from 'hydrooj/src/service/monitor';
-import readCases from './cases';
+import readCases, { processTestdata } from './cases';
 import { getConfig } from './config';
 import { CompileError, FormatError, SystemError } from './error';
 import { CopyInFile } from './sandbox/interface';
@@ -23,7 +23,6 @@ declare module 'hydrooj/src/interface' {
         'hydrojudge.cache_dir': string;
         'hydrojudge.tmp_dir': string;
         'hydrojudge.tmpfs_size': string;
-        'hydrojudge.retry_delay_sec': number;
         'hydrojudge.sandbox_host': string;
         'hydrojudge.memoryMax': string;
         'hydrojudge.testcases_max': number;
@@ -54,28 +53,6 @@ async function postInit() {
         monitor.updateJudge({ mid, ...inf });
     }, 1200000);
 
-    async function processData(folder: string) {
-        let files = await fs.readdir(folder);
-        let ini = false;
-        for (const i of files) {
-            if (i.toLowerCase() === 'config.ini') {
-                ini = true;
-                await fs.rename(`${folder}/${i}`, `${folder}/config.ini`);
-                break;
-            }
-        }
-        if (ini) {
-            for (const i of files) {
-                if (i.toLowerCase() === 'input') await fs.rename(`${folder}/${i}`, `${folder}/input`);
-                else if (i.toLowerCase() === 'output') await fs.rename(`${folder}/${i}`, `${folder}/output`);
-            }
-            files = await fs.readdir(`${folder}/input`);
-            for (const i of files) await fs.rename(`${folder}/input/${i}`, `${folder}/input/${i.toLowerCase()}`);
-            files = await fs.readdir(`${folder}/output`);
-            for (const i of files) await fs.rename(`${folder}/output/${i}`, `${folder}/output/${i.toLowerCase()}`);
-        }
-    }
-
     async function fetchCodeFile(name: string) {
         const target = path.join('/tmp/hydro/judge', name.replace(/\//g, '_'));
         await storage.get(`submission/${name}`, target);
@@ -104,7 +81,7 @@ async function postInit() {
         }
         fs.writeFileSync(path.join(filePath, 'etags'), JSON.stringify(version));
         fs.writeFileSync(path.join(filePath, 'lastUsage'), new Date().getTime().toString());
-        await processData(filePath).catch(noop);
+        await processTestdata(filePath);
         return filePath;
     }
 

packages/hydrojudge/src/sysinfo.ts

@@ -1,3 +1,4 @@
+// TODO this should be moved into web
 import path from 'path';
 import fs from 'fs-extra';
 import { noop } from 'lodash';
@@ -30,6 +31,8 @@ int main(){
                 type: 'default',
                 time: '3s',
                 memory: '256m',
+                user_extra_files: [],
+                judge_extra_files: [],
             },
             stat: {},
             clean: [],