From b74f6b64f8b04bd1717e3e1d166c32b88cc07f0f Mon Sep 17 00:00:00 2001 From: undefined Date: Sun, 15 May 2022 23:12:18 +0800 Subject: [PATCH] judge: merge processTestdata --- packages/hydrojudge/package.json | 2 +- packages/hydrojudge/setting.yaml | 5 --- packages/hydrojudge/src/cases.ts | 27 +++++++++++++++ packages/hydrojudge/src/check.ts | 3 ++ packages/hydrojudge/src/config.ts | 10 +++--- packages/hydrojudge/src/daemon.ts | 24 ++++---------- packages/hydrojudge/src/error.ts | 33 ++++--------------- packages/hydrojudge/src/hosts/hydro.ts | 32 ++---------------- packages/hydrojudge/src/judge/default.ts | 28 +++++++--------- packages/hydrojudge/src/judge/interface.ts | 4 +-- .../hydrojudge/src/judge/submit_answer.ts | 3 -- packages/hydrojudge/src/log.ts | 22 +------------ packages/hydrojudge/src/sandbox.ts | 7 ++++ packages/hydrojudge/src/service.ts | 27 ++------------- packages/hydrojudge/src/sysinfo.ts | 3 ++ 15 files changed, 78 insertions(+), 152 deletions(-) diff --git a/packages/hydrojudge/package.json b/packages/hydrojudge/package.json index 2db6ae3de..b57564e63 100644 --- a/packages/hydrojudge/package.json +++ b/packages/hydrojudge/package.json @@ -1,7 +1,7 @@ { "name": "@hydrooj/hydrojudge", "bin": "bin/hydrojudge.js", - "version": "2.20.1", + "version": "2.20.2", "main": "package.json", "author": "undefined ", "repository": "https://github.com/hydro-dev/Hydro.git", diff --git a/packages/hydrojudge/setting.yaml b/packages/hydrojudge/setting.yaml index d9533b83a..9fa0f23f7 100644 --- a/packages/hydrojudge/setting.yaml +++ b/packages/hydrojudge/setting.yaml @@ -37,11 +37,6 @@ memoryMax: type: text name: memoryMax default: 512m -retry_delay_sec: - type: number - name: retry_delay_sec - desc: Retry delay sec - default: 15 cache_dir: type: text name: cache_dir diff --git a/packages/hydrojudge/src/cases.ts b/packages/hydrojudge/src/cases.ts index 2eacf86c9..baf38cd5d 100644 --- a/packages/hydrojudge/src/cases.ts +++ b/packages/hydrojudge/src/cases.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-await-in-loop */ import path from 'path'; import fs from 'fs-extra'; import yaml from 'js-yaml'; @@ -35,6 +36,32 @@ async function collectFiles(folder: string) { return files; } +export async function processTestdata(folder: string) { + let files = await fs.readdir(folder); + if (files.length <= 2) { + if (files.length === 2) files.splice(files.indexOf('version'), 1); + if (fs.statSync(path.resolve(folder, files[0])).isDirectory()) { + folder = path.resolve(folder, files[0]); + files = await fs.readdir(folder); + } + } + const ini = files.filter((i) => i.toLowerCase() === 'config.ini')[0]; + if (!ini) return; + const t = fs.readFileSync(path.resolve(folder, ini), 'utf8'); + await fs.writeFile(path.resolve(folder, 'config.ini'), t.toLowerCase()); + for (const i of files) { + if (i.toLowerCase() === 'input') await fs.rename(`${folder}/${i}`, `${folder}/input`); + if (i.toLowerCase() === 'output') await fs.rename(`${folder}/${i}`, `${folder}/output`); + } + await Promise.all(['input', 'output'].flatMap(async (f) => { + const dir = path.resolve(folder, f); + const sf = await fs.readdir(dir); + return sf + .filter((i) => i !== i.toLowerCase()) + .map((i) => fs.rename(`${dir}/${i}`, `${dir}/${i.toLowerCase()}`)); + })); +} + export default async function readCases(folder: string, cfg: Record = {}, args) { const iniConfig = path.resolve(folder, 'config.ini'); const yamlConfig = path.resolve(folder, 'config.yaml'); diff --git a/packages/hydrojudge/src/check.ts b/packages/hydrojudge/src/check.ts index 8155c878c..1223c6dab 100644 --- a/packages/hydrojudge/src/check.ts +++ b/packages/hydrojudge/src/check.ts @@ -16,6 +16,9 @@ export async function check(config: CheckConfig): Promise<[number, number, strin } export async function compileChecker(getLang: Function, checkerType: string, checker: string, copyIn: any): Promise { + if (['default', 'strict'].includes(checkerType)) { + return { execute: '', copyIn: {}, clean: () => Promise.resolve(null) }; + } if (!checkers[checkerType]) throw new SystemError('Unknown checker type {0}.', [checkerType]); if (checkerType === 'testlib') copyIn['testlib.h'] = { src: testlibSrc }; const s = checker.replace('@', '.').split('.'); diff --git a/packages/hydrojudge/src/config.ts b/packages/hydrojudge/src/config.ts index 51cdbab56..b20e597ea 100644 --- a/packages/hydrojudge/src/config.ts +++ b/packages/hydrojudge/src/config.ts @@ -1,4 +1,3 @@ -/* eslint-disable prefer-const */ import os from 'os'; import path from 'path'; import cac from 'cac'; @@ -9,19 +8,19 @@ const argv = cac().parse(); // eslint-disable-next-line @typescript-eslint/naming-convention let CONFIG_FILE = path.resolve(os.homedir(), '.config', 'hydro', 'judge.yaml'); -let config = { +const config = { cache_dir: path.resolve(os.homedir(), '.cache', 'hydro', 'judge'), tmp_dir: path.resolve(os.tmpdir(), 'hydro', 'judge'), tmpfs_size: '256m', stdio_size: '32m', memoryMax: '512m', strict_memory: false, - retry_delay_sec: 15, sandbox_host: 'http://localhost:5050', testcases_max: 100, total_time_limit: 60, processLimit: 32, parallelism: 2, + singleTaskParallelism: 2, rerun: 0, rate: 1, config: null, @@ -29,10 +28,11 @@ let config = { env: `\ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/w`, + hosts: {}, secret: String.random(32), }; -export function getConfig(key: string) { +export function getConfig(key: keyof typeof config) { return global.Hydro ? global.Hydro.model.system.get(`hydrojudge.${key}`) : config[key]; } @@ -61,5 +61,5 @@ if (!global.Hydro) { config.sandbox_host = path.resolve(process.env.EXECUTION_HOST || argv.options.sandbox); } const configFile = fs.readFileSync(CONFIG_FILE).toString(); - config = { ...config, ...yaml.load(configFile) as any }; + Object.assign(config, yaml.load(configFile) as any); } diff --git a/packages/hydrojudge/src/daemon.ts b/packages/hydrojudge/src/daemon.ts index 858f12fec..02cfe91ad 100644 --- a/packages/hydrojudge/src/daemon.ts +++ b/packages/hydrojudge/src/daemon.ts @@ -18,7 +18,7 @@ import './utils'; import { getConfig } from './config'; import * as Session from './hosts/index'; import log from './log'; -import { Queue, sleep } from './utils'; +import { Queue } from './utils'; declare global { namespace NodeJS { @@ -28,8 +28,8 @@ declare global { } } } -if (!global.onDestroy) global.onDestroy = []; -if (!global.hosts) global.hosts = []; +global.onDestroy ||= []; +global.hosts ||= []; let exit = false; const terminate = async () => { @@ -60,28 +60,16 @@ async function worker(queue: Queue) { async function daemon() { const _hosts = getConfig('hosts'); - const delay = getConfig('retry_delay_sec'); const hosts = {}; const queue = new Queue(); + worker(queue).catch((e) => log.error(e)); for (const i in _hosts) { _hosts[i].host = _hosts[i].host || i; - hosts[i] = new Session[_hosts[i].type || 'vj4'](_hosts[i]); + hosts[i] = new Session[_hosts[i].type || 'hydro'](_hosts[i]); await hosts[i].init(); } + for (const i in hosts) hosts[i].consume(queue); global.hosts = hosts; - worker(queue); - for (const i in hosts) { - while ('Orz twd2') { - try { - await hosts[i].consume(queue); - break; - } catch (e) { - log.error(e, e.stack); - log.info(`在 ${delay} 秒后重试`); - await sleep(delay * 1000); - } - } - } } if (require.main === module) daemon(); diff --git a/packages/hydrojudge/src/error.ts b/packages/hydrojudge/src/error.ts index 29419e5ab..ff79b9a0b 100644 --- a/packages/hydrojudge/src/error.ts +++ b/packages/hydrojudge/src/error.ts @@ -5,7 +5,7 @@ export class CompileError extends Error { stdout: string; stderr: string; status: string; - type: string; + type = 'CompileError'; constructor(obj: string | CompileErrorInfo) { super('Compile Error'); @@ -17,48 +17,29 @@ export class CompileError extends Error { this.stderr = obj.stderr || ''; this.status = obj.status ? STATUS_TEXTS[obj.status] || '' : ''; } - this.type = 'CompileError'; } } export class FormatError extends Error { - type: string; - params: any[]; + type = 'FormatError'; - constructor(message: string, params = []) { + constructor(message: string, public params = []) { super(message); - this.type = 'FormatError'; - this.params = params; } } export class RuntimeError extends Error { - type: string; - detail: any; + type = 'RuntimeError'; - constructor(detail: string, message: string) { + constructor(public detail: string, message: string) { super(message); - this.type = 'RuntimeError'; - this.detail = detail; } } export class SystemError extends Error { - type: string; - params: any[]; + type = 'SystemError'; - constructor(message: string, params = []) { + constructor(message: string, public params = []) { super(message); - this.type = 'SystemError'; - this.params = params; - } -} - -export class TooFrequentError extends Error { - type: string; - - constructor(message: string) { - super(message); - this.type = 'TooFrequentError'; } } diff --git a/packages/hydrojudge/src/hosts/hydro.ts b/packages/hydrojudge/src/hosts/hydro.ts index 072e000c1..6824b5af0 100644 --- a/packages/hydrojudge/src/hosts/hydro.ts +++ b/packages/hydrojudge/src/hosts/hydro.ts @@ -10,7 +10,7 @@ import WebSocket from 'ws'; import { LangConfig } from '@hydrooj/utils/lib/lang'; import { STATUS } from '@hydrooj/utils/lib/status'; import type { JudgeResultBody } from 'hydrooj'; -import readCases from '../cases'; +import readCases, { processTestdata } from '../cases'; import { getConfig } from '../config'; import { CompileError, FormatError, SystemError } from '../error'; import judge from '../judge'; @@ -232,7 +232,7 @@ export default class Hydro { queue.start(); await Promise.all(tasks); fs.writeFileSync(path.join(filePath, 'etags'), JSON.stringify(version)); - await this.processData(filePath).catch(noop); + await processTestdata(filePath); } fs.writeFileSync(path.join(filePath, 'lastUsage'), new Date().getTime().toString()); return filePath; @@ -332,34 +332,6 @@ export default class Hydro { } } - async processData(folder: string) { // eslint-disable-line class-methods-use-this - let files = await fs.readdir(folder); - let ini = false; - if (files.length <= 2) { - if (files.length === 2) files.splice(files.indexOf('version'), 1); - const s = fs.statSync(path.resolve(folder, files[0])); - if (s.isDirectory()) folder = path.resolve(folder, files[0]); - } - files = await fs.readdir(folder); - for (const i of files) { - if (i.toLowerCase() === 'config.ini') { - ini = true; - await fs.rename(`${folder}/${i}`, `${folder}/config.ini`); - break; - } - } - if (ini) { - for (const i of files) { - if (i.toLowerCase() === 'input') await fs.rename(`${folder}/${i}`, `${folder}/input`); - else if (i.toLowerCase() === 'output') await fs.rename(`${folder}/${i}`, `${folder}/output`); - } - files = await fs.readdir(`${folder}/input`); - for (const i of files) await fs.rename(`${folder}/input/${i}`, `${folder}/input/${i.toLowerCase()}`); - files = await fs.readdir(`${folder}/output`); - for (const i of files) await fs.rename(`${folder}/output/${i}`, `${folder}/output/${i.toLowerCase()}`); - } - } - async retry(queue: Queue) { this.consume(queue).catch(() => { setTimeout(() => this.retry(queue), 30000); diff --git a/packages/hydrojudge/src/judge/default.ts b/packages/hydrojudge/src/judge/default.ts index a1efc69e4..e9a9aab0a 100644 --- a/packages/hydrojudge/src/judge/default.ts +++ b/packages/hydrojudge/src/judge/default.ts @@ -177,21 +177,17 @@ export const judge = async (ctx: Context, startPromise = Promise.resolve()) => { ), ctx.next, ), - (() => { - if (['default', 'strict'].includes(ctx.config.checker_type || 'default')) { - return { execute: '', copyIn: {}, clean: () => Promise.resolve(null) }; - } - const copyIn = { user_code: ctx.code }; - for (const file of ctx.config.judge_extra_files) { - copyIn[parseFilename(file)] = { src: file }; - } - return compileChecker( - ctx.getLang, - ctx.config.checker_type, - ctx.config.checker, - copyIn, - ); - })(), + compileChecker( + ctx.getLang, + ctx.config.checker_type, + ctx.config.checker, + { + user_code: ctx.code, + ...Object.fromEntries( + (ctx.config.judge_extra_files || []).map((i) => [parseFilename(i), { src: i }]), + ), + }, + ), ]); ctx.clean.push(ctx.execute.clean, ctx.checker.clean); await startPromise; @@ -202,7 +198,7 @@ export const judge = async (ctx: Context, startPromise = Promise.resolve()) => { ctx.total_memory_usage_kb = 0; ctx.total_time_usage_ms = 0; ctx.rerun = getConfig('rerun') || 0; - ctx.queue = new Queue({ concurrency: getConfig('parallelism') }); + ctx.queue = new Queue({ concurrency: getConfig('singleTaskParallelism') }); ctx.failed = {}; for (const sid in ctx.config.subtasks) tasks.push(judgeSubtask(ctx.config.subtasks[sid], sid)(ctx)); const scores = await Promise.all(tasks); diff --git a/packages/hydrojudge/src/judge/interface.ts b/packages/hydrojudge/src/judge/interface.ts index bfad1f22d..1ebb1f78b 100644 --- a/packages/hydrojudge/src/judge/interface.ts +++ b/packages/hydrojudge/src/judge/interface.ts @@ -52,8 +52,8 @@ export interface Config { detail?: boolean; filename?: string; - judge_extra_files?: string[]; - user_extra_files?: string[]; + judge_extra_files: string[]; + user_extra_files: string[]; template?: Record; checker?: string; diff --git a/packages/hydrojudge/src/judge/submit_answer.ts b/packages/hydrojudge/src/judge/submit_answer.ts index 00c3fa9d8..699f58cdd 100644 --- a/packages/hydrojudge/src/judge/submit_answer.ts +++ b/packages/hydrojudge/src/judge/submit_answer.ts @@ -122,9 +122,6 @@ export const judge = async (ctx: Context, startPromise = Promise.resolve()) => { if (!ctx.config.subtasks.length) throw new FormatError('Problem data not found.'); startPromise.then(() => ctx.next({ status: STATUS.STATUS_COMPILING })); ctx.checker = await (() => { - if (['default', 'strict'].includes(ctx.config.checker_type || 'default')) { - return { execute: '', copyIn: {}, clean: () => Promise.resolve(null) }; - } const copyIn = { user_code: ctx.code }; for (const file of ctx.config.judge_extra_files) { copyIn[parseFilename(file)] = { src: file }; diff --git a/packages/hydrojudge/src/log.ts b/packages/hydrojudge/src/log.ts index 611e5da29..b67edf409 100644 --- a/packages/hydrojudge/src/log.ts +++ b/packages/hydrojudge/src/log.ts @@ -1,29 +1,9 @@ import { format, inspect, InspectOptions } from 'util'; import cac from 'cac'; +import { Time } from '@hydrooj/utils/lib/utils'; const argv = cac().parse(); -namespace Time { - export const second = 1000; - export const minute = second * 60; - export const hour = minute * 60; - export const day = hour * 24; - export const week = day * 7; - export function formatTimeShort(ms: number) { - const abs = Math.abs(ms); - if (abs >= day - hour / 2) { - return `${Math.round(ms / day)}d`; - } if (abs >= hour - minute / 2) { - return `${Math.round(ms / hour)}h`; - } if (abs >= minute - second / 2) { - return `${Math.round(ms / minute)}m`; - } if (abs >= second) { - return `${Math.round(ms / second)}s`; - } - return `${ms}ms`; - } -} - const colors = [ 20, 21, 26, 27, 32, 33, 38, 39, 40, 41, 42, 43, 44, 45, 56, 57, 62, 63, 68, 69, 74, 75, 76, 77, 78, 79, 80, 81, 92, 93, 98, 99, 112, 113, diff --git a/packages/hydrojudge/src/sandbox.ts b/packages/hydrojudge/src/sandbox.ts index 86d7f1f3f..feb6a5a0c 100644 --- a/packages/hydrojudge/src/sandbox.ts +++ b/packages/hydrojudge/src/sandbox.ts @@ -1,5 +1,6 @@ import cac from 'cac'; import fs from 'fs-extra'; +import PQueue from 'p-queue'; import { ParseEntry } from 'shell-quote'; import { STATUS } from '@hydrooj/utils/lib/status'; import { getConfig } from './config'; @@ -191,3 +192,9 @@ export async function run(execute: string, params?: Parameter): Promise run(execute, params), { priority }); +} diff --git a/packages/hydrojudge/src/service.ts b/packages/hydrojudge/src/service.ts index bec7390e5..a8ac4f3c0 100644 --- a/packages/hydrojudge/src/service.ts +++ b/packages/hydrojudge/src/service.ts @@ -10,7 +10,7 @@ import { STATUS } from '@hydrooj/utils/lib/status'; import type { JudgeResultBody } from 'hydrooj'; import { Logger } from 'hydrooj/src/logger'; import * as monitor from 'hydrooj/src/service/monitor'; -import readCases from './cases'; +import readCases, { processTestdata } from './cases'; import { getConfig } from './config'; import { CompileError, FormatError, SystemError } from './error'; import { CopyInFile } from './sandbox/interface'; @@ -23,7 +23,6 @@ declare module 'hydrooj/src/interface' { 'hydrojudge.cache_dir': string; 'hydrojudge.tmp_dir': string; 'hydrojudge.tmpfs_size': string; - 'hydrojudge.retry_delay_sec': number; 'hydrojudge.sandbox_host': string; 'hydrojudge.memoryMax': string; 'hydrojudge.testcases_max': number; @@ -54,28 +53,6 @@ async function postInit() { monitor.updateJudge({ mid, ...inf }); }, 1200000); - async function processData(folder: string) { - let files = await fs.readdir(folder); - let ini = false; - for (const i of files) { - if (i.toLowerCase() === 'config.ini') { - ini = true; - await fs.rename(`${folder}/${i}`, `${folder}/config.ini`); - break; - } - } - if (ini) { - for (const i of files) { - if (i.toLowerCase() === 'input') await fs.rename(`${folder}/${i}`, `${folder}/input`); - else if (i.toLowerCase() === 'output') await fs.rename(`${folder}/${i}`, `${folder}/output`); - } - files = await fs.readdir(`${folder}/input`); - for (const i of files) await fs.rename(`${folder}/input/${i}`, `${folder}/input/${i.toLowerCase()}`); - files = await fs.readdir(`${folder}/output`); - for (const i of files) await fs.rename(`${folder}/output/${i}`, `${folder}/output/${i.toLowerCase()}`); - } - } - async function fetchCodeFile(name: string) { const target = path.join('/tmp/hydro/judge', name.replace(/\//g, '_')); await storage.get(`submission/${name}`, target); @@ -104,7 +81,7 @@ async function postInit() { } fs.writeFileSync(path.join(filePath, 'etags'), JSON.stringify(version)); fs.writeFileSync(path.join(filePath, 'lastUsage'), new Date().getTime().toString()); - await processData(filePath).catch(noop); + await processTestdata(filePath); return filePath; } diff --git a/packages/hydrojudge/src/sysinfo.ts b/packages/hydrojudge/src/sysinfo.ts index 1c4650a57..ac2c0f63d 100644 --- a/packages/hydrojudge/src/sysinfo.ts +++ b/packages/hydrojudge/src/sysinfo.ts @@ -1,3 +1,4 @@ +// TODO this should be moved into web import path from 'path'; import fs from 'fs-extra'; import { noop } from 'lodash'; @@ -30,6 +31,8 @@ int main(){ type: 'default', time: '3s', memory: '256m', + user_extra_files: [], + judge_extra_files: [], }, stat: {}, clean: [],