From b11da3eb2c2497b4de2314622ff02fb91fa0cb01 Mon Sep 17 00:00:00 2001 From: undefined Date: Mon, 18 Mar 2024 03:37:55 +0800 Subject: [PATCH] judge: bring back vj4 support --- packages/hydrojudge/langs.yaml | 28 +++ packages/hydrojudge/src/daemon.ts | 5 +- packages/hydrojudge/src/hosts/vj4.ts | 311 +++++++++++++++++++++++++++ packages/utils/lib/common.ts | 1 + 4 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 packages/hydrojudge/langs.yaml create mode 100644 packages/hydrojudge/src/hosts/vj4.ts diff --git a/packages/hydrojudge/langs.yaml b/packages/hydrojudge/langs.yaml new file mode 100644 index 000000000..f8cde0ae9 --- /dev/null +++ b/packages/hydrojudge/langs.yaml @@ -0,0 +1,28 @@ +c: + compile: /usr/bin/gcc -O2 -Wall -std=c99 -o foo foo.c -lm +cc: + compile: /usr/bin/g++ -O2 -Wall -std=c++11 -o foo foo.cc -lm +pas: + compile: /usr/bin/fpc -O2 -ofoo foo.pas +java: + compile: /usr/bin/javac -d /w -encoding UTF-8 Main.java + code_file: Main.java + execute: /usr/bin/java -Dfile.encoding=UTF-8 Main +py3: + compile: /usr/bin/python3 -c "import py_compile; py_compile.compile('/w/foo.py', '/w/foo.pyc', doraise=True)" + execute: /usr/bin/python3 foo.pyc +php: + execute: /usr/bin/php foo.php +rs: + compile: /usr/bin/rustc -O -o /w/foo /w/foo.rs +hs: + compile: /usr/bin/ghc -O -outputdir /tmp -o /w/foo /w/foo.hs +js: + execute: /usr/bin/jsc foo.js +go: + compile: /usr/bin/go build -o /w/foo /w/foo.go +rb: + execute: /usr/bin/ruby foo.rb +cs: + compile: /usr/bin/mcs csc -optimize+ -out:/w/foo.exe /w/foo.cs + execute: /usr/bin/mono foo.exe \ No newline at end of file diff --git a/packages/hydrojudge/src/daemon.ts b/packages/hydrojudge/src/daemon.ts index 0433324e2..e4e071c23 100644 --- a/packages/hydrojudge/src/daemon.ts +++ b/packages/hydrojudge/src/daemon.ts @@ -19,10 +19,11 @@ import PQueue from 'p-queue'; import { fs } from '@hydrooj/utils'; import { getConfig } from './config'; import HydroHost from './hosts/hydro'; +import Vj4Host from './hosts/vj4'; import log from './log'; import { versionCheck } from './sandbox'; -const hosts: Record = {}; +const hosts: Record = {}; let exit = false; const terminate = async () => { @@ -53,7 +54,7 @@ async function daemon() { queue.on('error', (e) => log.error(e)); for (const i in _hosts) { _hosts[i].host ||= i; - hosts[i] = new HydroHost(_hosts[i]); + hosts[i] = _hosts[i].type === 'vj4' ? new Vj4Host(_hosts[i]) : new HydroHost(_hosts[i]); await hosts[i].init(); } for (const i in hosts) hosts[i].consume(queue); diff --git a/packages/hydrojudge/src/hosts/vj4.ts b/packages/hydrojudge/src/hosts/vj4.ts new file mode 100644 index 000000000..0eefaad1c --- /dev/null +++ b/packages/hydrojudge/src/hosts/vj4.ts @@ -0,0 +1,311 @@ +/* eslint-disable no-await-in-loop */ +import path from 'path'; +import AdmZip from 'adm-zip'; +import fs from 'fs-extra'; +import { noop } from 'lodash'; +import PQueue from 'p-queue'; +import superagent from 'superagent'; +import WebSocket from 'ws'; +import { parseLang } from '@hydrooj/utils/lib/lang'; +import { STATUS } from '@hydrooj/utils/lib/status'; +import type { JudgeResultBody } from 'hydrooj'; +import { getConfig } from '../config'; +import { FormatError, SystemError } from '../error'; +import { Session } from '../interface'; +import log from '../log'; +import { JudgeTask } from '../task'; +import { pipeRequest } from '../utils'; + +function removeNixPath(text: string) { + return text.replace(/\/nix\/store\/[a-z0-9]{32}-/g, '/nix/'); +} + +const langs = parseLang(fs.readFileSync(path.resolve(__dirname, '../../langs.yaml'), 'utf-8')); + +export default class VJ4 implements Session { + config: any; + progress = 0; + ws: WebSocket; + + async fetchFile(): Promise { + throw new Error('not implemented'); + } + + async postFile() { + throw new Error('not implemented'); + } + + getLang(name: string, doThrow = true) { + if (langs[name]) return langs[name]; + if (name === 'cpp' && langs.cc) return langs.cc; + if (doThrow) throw new SystemError('Unsupported language {0}', [name]); + return null; + } + + send(tag: string | number, key: 'next' | 'end', data: any) { + if (data.case && typeof data.case.message === 'string') data.case.message = removeNixPath(data.case.message); + if (data.message) { + if (typeof data.message !== 'string') data.message = data.message.message.format(data.message.params); + data.judge_text = removeNixPath(data.message); + delete data.message; + } + if (typeof data.compilerText === 'string') { + data.compiler_text = removeNixPath(data.compilerText); + delete data.compilerText; + } + if (data.status === STATUS.STATUS_FORMAT_ERROR) { + data.status = STATUS.STATUS_SYSTEM_ERROR; + } + if (data.case) { + data.case.time_ms = data.case.time || 0; + data.case.memory_kb = data.case.memory || 0; + data.case.judge_text = data.case.message || ''; + delete data.case.time; + delete data.case.memory; + delete data.case.message; + if (data.case?.judge_text && typeof data.case?.judge_text !== 'string') { + data.case.judge_text = data.case.judge_text.message.format(data.case.judge_text.params); + } + } + if (data.time) data.time_ms = data.time; + if (data.memory) data.memory_kb = data.memory; + if (key === 'end') { + data.time_ms ||= 0; + data.memory_kb ||= 0; + data.score ||= 0; + } + if (data.progress) this.progress = data.progress; + if (data.addProgress) { + this.progress += data.addProgress; + data.progress = this.progress; + } + this.ws.send(JSON.stringify({ ...data, tag, key })); + } + + getNext(t: JudgeTask) { + return (data: Partial) => { + log.debug('Next: %o', data); + this.send((t.request as any).tag, 'next', data); + }; + } + + getEnd(t: JudgeTask) { + return (data: Partial) => { + log.info('End: %o', data); + this.send((t.request as any).tag, 'end', data); + }; + } + + get(url: string) { + url = new URL(url, this.config.server_url).toString(); + const p = superagent.get(url); + if (this.config.cookie) return p.set('Cookie', this.config.cookie); + return p; + } + + post(url: string, data?: any) { + url = new URL(url, this.config.server_url).toString(); + let t = superagent.post(url).type('form'); + if (this.config.cookie) t = t.set('Cookie', this.config.cookie); + return data ? t.send(data) : t; + } + + constructor(config) { + this.config = config; + this.config.detail ??= true; + this.config.cookie ||= ''; + this.config.last_update_at ||= 0; + if (!this.config.server_url.startsWith('http')) this.config.server_url = `http://${this.config.server_url}`; + if (!this.config.server_url.endsWith('/')) this.config.server_url = `${this.config.server_url}/`; + } + + async init() { + await this.setCookie(this.config.cookie || ''); + await this.ensureLogin(); + setInterval(() => { this.get('judge/noop'); }, 30000000); + } + + async problemDataVersion(domainId: string, pid: string) { + let location: string; + await this.ensureLogin(); + const res = await this.get(`d/${domainId}/p/${pid}/data`).redirects(0).ok((r) => r.status === 302 || r.status === 404); + if (res.status === 404) throw new FormatError(`没有找到测试数据 ${domainId}/${pid}`); + if (res.status === 302) { + location = res.headers.location; + if (!location.includes('/fs/')) throw new Error(); + } + return res.headers.location; + } + + async problemData(domainId: string, pid: string, savePath: string, retry = 3, next?) { + log.info(`Getting problem data: ${this.config.host}/${domainId}/${pid}`); + await this.ensureLogin(); + if (next) next({ judge_text: '正在同步测试数据，请稍后' }); + const tmpFilePath = path.resolve(getConfig('cache_dir'), `download_${this.config.host}_${domainId}_${pid}`); + try { + await pipeRequest( + this.get(`${this.config.server_url}d/${domainId}/p/${pid}/data`), + fs.createWriteStream(tmpFilePath), + ); + fs.ensureDirSync(path.dirname(savePath)); + const zip = new AdmZip(tmpFilePath); + const entries = zip.getEntries(); + if (entries.length > 512) throw new FormatError('Too many files'); + await new Promise((resolve, reject) => { + zip.extractAllToAsync(savePath, true, (e) => { + if (e) reject(e); + else resolve(null); + }); + }); + await fs.unlink(tmpFilePath); + await this.processData(savePath).catch(noop); + } catch (e) { + if (retry) await this.problemData(domainId, pid, savePath, retry - 1); + else throw e; + } + return savePath; + } + + async recordPretestData(rid: string, savePath: string) { + log.info(`Getting pretest data: ${this.config.host}/${rid}`); + const tmpFilePath = path.resolve(getConfig('cache_dir'), `download_${this.config.host}_${rid}`); + await this.ensureLogin(); + const w = fs.createWriteStream(tmpFilePath); + await pipeRequest(this.get(`records/${rid}/data`), w); + fs.ensureDirSync(path.dirname(savePath)); + const zip = new AdmZip(tmpFilePath); + const entries = zip.getEntries(); + if (entries.length > 512) throw new FormatError('Too many files'); + await new Promise((resolve, reject) => { + zip.extractAllToAsync(savePath, true, (e) => { + if (e) reject(e); + else resolve(null); + }); + }); + await fs.unlink(tmpFilePath); + await this.processData(savePath); + return savePath; + } + + async consume(queue: PQueue) { + log.info('正在连接 %sjudge/consume-conn', this.config.server_url); + const res = await this.get('judge/consume-conn/info'); + this.ws = new WebSocket(`${this.config.server_url.replace(/^http/i, 'ws')}judge/consume-conn/websocket?t=${res.body.entropy}`, { + headers: { cookie: this.config.cookie }, + }); + this.ws.on('message', (data) => { + const request = JSON.parse(data.toString()); + if (!request.event) { + request.domainId = request.domain_id; + request.source = request.type === 1 ? `__record__/${request.rid}` : `${request.domain_id}/${request.pid}`; + request.meta = {}; + request.config = { + type: 'default', + }; + queue.add(() => new JudgeTask(this, request).handle().catch(console.error)); + } + }); + this.ws.on('close', (data, reason) => { + log.warn(`[${this.config.host}] Websocket 断开:`, data, reason); + setTimeout(() => { + this.retry(queue); + }, 30000); + }); + this.ws.on('error', (e) => { + log.error(`[${this.config.host}] Websocket 错误:`, e); + setTimeout(() => { + this.retry(queue); + }, 30000); + }); + await new Promise((resolve) => { + this.ws.once('open', () => { resolve(null); }); + }); + log.info(`[${this.config.host}] 已连接`); + } + + async setCookie(cookie: string) { + this.config.cookie = cookie; + } + + async login() { + log.info('[%s] Updating session', this.config.host); + const res = await this.post('/login', { + uname: this.config.uname, password: this.config.password, rememberme: 'on', + }).redirects(0).ok((i) => i.status === 302); + await this.setCookie(res.headers['set-cookie']); + } + + async ensureLogin() { + try { + await this.get('/judge/noop'); + } catch (e) { + await this.login(); + } + } + + async processData(folder: string) { // eslint-disable-line class-methods-use-this + let files = await fs.readdir(folder); + if (files.length <= 2) { + if (files.length === 2) files.splice(files.indexOf('version'), 1); + const s = fs.statSync(path.resolve(folder, files[0])); + if (s.isDirectory()) folder = path.resolve(folder, files[0]); + } + files = await fs.readdir(folder); + const iniFile = files.find((i) => i.toLowerCase() === 'config.ini'); + if (iniFile) { + const file = await fs.readFile(`${folder}/${iniFile}`, 'utf-8'); + await fs.writeFile(`${folder}/config.ini`, file.toLowerCase()); + for (const i of files) { + if (i.toLowerCase() === 'input') await fs.rename(`${folder}/${i}`, `${folder}/input`); + else if (i.toLowerCase() === 'output') await fs.rename(`${folder}/${i}`, `${folder}/output`); + } + files = await fs.readdir(`${folder}/input`); + for (const i of files) { + if (fs.statSync(`${folder}/input/${i}`).size > 64 * 1024 * 1024) { + await fs.rename(`${folder}/input/${i}`, `${folder}/${i.toLowerCase()}`); + } else { + const buffer = await fs.readFile(`${folder}/input/${i}`); + const data = buffer.toString().replace(/\r/g, ''); + await fs.unlink(`${folder}/input/${i}`); + await fs.writeFile(`${folder}/${i.toLowerCase()}`, data); + } + } + files = await fs.readdir(`${folder}/output`); + for (const i of files) await fs.rename(`${folder}/output/${i.toLowerCase()}`, `${folder}/${i}`); + } + } + + async cacheOpen(source: string, files: string[], next) { + const [domainId, pid] = source.split('/'); + const domainDir = path.join(getConfig('cache_dir'), this.config.host, domainId); + const filePath = path.join(domainDir, pid); + if (domainId === '__record__') return await this.recordPretestData(pid, filePath); + const version = await this.problemDataVersion(domainId, pid); + if (fs.existsSync(filePath)) { + let ver; + try { + ver = fs.readFileSync(path.join(filePath, 'version')).toString(); + } catch (e) { /* ignore */ } + if (version === ver) { + fs.writeFileSync(path.join(filePath, 'lastUsage'), new Date().getTime().toString()); + return filePath; + } + fs.removeSync(filePath); + } + fs.ensureDirSync(domainDir); + await this.problemData(domainId, pid, filePath, 3, next); + fs.writeFileSync(path.join(filePath, 'version'), version); + fs.writeFileSync(path.join(filePath, 'lastUsage'), new Date().getTime().toString()); + return filePath; + } + + async retry(queue: PQueue) { + this.consume(queue).catch(() => { + setTimeout(() => { + this.retry(queue); + }, 30000); + }); + } + + dispose() { } +} diff --git a/packages/utils/lib/common.ts b/packages/utils/lib/common.ts index b4915c3e7..a254ba670 100644 --- a/packages/utils/lib/common.ts +++ b/packages/utils/lib/common.ts @@ -328,6 +328,7 @@ export function readSubtasksFromFiles(files: string[], config) { for (const func of rule.output) { if (config.noOutputFile) c.output = '/dev/null'; else c.output = func(data); + if (c.output === file) continue; if (c.output === '/dev/null' || files.includes(c.output)) { match = true; if (!subtask[sid]) {