diff --git a/packages/hydrojudge/src/cases.ts b/packages/hydrojudge/src/cases.ts index afae6a5a0..91534d8ac 100644 --- a/packages/hydrojudge/src/cases.ts +++ b/packages/hydrojudge/src/cases.ts @@ -1,7 +1,9 @@ import fs from 'fs-extra'; import path from 'path'; import yaml from 'js-yaml'; -import { Dictionary, sum } from 'lodash'; +import { sum } from 'lodash'; +import readYamlCases, { convertIniConfig } from '@hydrooj/utils/lib/cases'; +import { changeErrorType } from '@hydrooj/utils/lib/utils'; import { FormatError, SystemError } from './error'; import { parseTimeMS, parseMemoryMB, ensureFile } from './utils'; import { getConfig } from './config'; @@ -193,102 +195,6 @@ async function readAutoCases(folder, { next }, cfg) { return config; } -export async function readYamlCases(folder: string, cfg: Dictionary = {}, args) { - const config: any = { - checker_type: 'default', - count: 0, - subtasks: [], - judge_extra_files: [], - user_extra_files: [], - }; - const next = args.next; - const checkFile = ensureFile(folder); - config.checker_type = cfg.checker_type || 'default'; - if (cfg.checker) config.checker = checkFile(cfg.checker, 'Cannot find checker {0}.'); - if (cfg.interactor) config.interactor = checkFile(cfg.interactor, 'Cannot find interactor {0}.'); - if (cfg.judge_extra_files) { - if (typeof cfg.judge_extra_files === 'string') { - config.judge_extra_files = [checkFile(cfg.judge_extra_files, 'Cannot find judge extra file {0}.')]; - } else if (cfg.judge_extra_files instanceof Array) { - for (const file of cfg.judge_extra_files) { - config.judge_extra_files.push(checkFile(file, 'Cannot find judge extra file {0}.')); - } - } else throw new FormatError('Invalid judge_extra_files config.'); - } - if (cfg.user_extra_files) { - if (typeof cfg.user_extra_files === 'string') { - config.user_extra_files = [checkFile(cfg.user_extra_files, 'Cannot find user extra file {0}.')]; - } else if (cfg.user_extra_files instanceof Array) { - for (const file of cfg.user_extra_files) { - config.user_extra_files.push(checkFile(file, 'Cannot find user extra file {0}.')); - } - } else throw new FormatError('Invalid user_extra_files config.'); - } - if (cfg.outputs) { - config.type = 'submit_answer'; - } else if (cfg.cases) { - config.subtasks = [{ - score: parseInt(cfg.score, 10) || Math.floor(100 / cfg.cases.length), - time_limit_ms: parseTimeMS(cfg.time || '1s'), - memory_limit_mb: parseMemoryMB(cfg.memory || '512m'), - cases: [], - type: 'sum', - }]; - for (const c of cfg.cases) { - config.count++; - config.subtasks[0].cases.push({ - input: c.input ? checkFile(c.input, 'Cannot find input file {0}.') : '/dev/null', - output: c.output ? checkFile(c.output, 'Cannot find output file {0}.') : '/dev/null', - id: config.count, - }); - } - } else if (cfg.subtasks) { - for (const subtask of cfg.subtasks) { - const cases = []; - for (const c of subtask.cases) { - config.count++; - cases.push({ - input: c.input ? checkFile(c.input, 'Cannot find input file {0}.') : '/dev/null', - output: c.output ? checkFile(c.output, 'Cannot find output file {0}.') : '/dev/null', - id: config.count, - }); - } - config.subtasks.push({ - score: parseInt(subtask.score, 10), - if: subtask.if || [], - cases, - time_limit_ms: parseTimeMS(subtask.time || cfg.time || '1s'), - memory_limit_mb: parseMemoryMB(subtask.memory || cfg.memory || '512m'), - }); - } - } else { - const c = await readAutoCases(folder, { next }, cfg); - config.subtasks = c.subtasks; - config.count = c.count; - } - if (config.type === 'submit_answer' && !cfg.outputs) throw new FormatError('outputs config not found'); - return Object.assign(cfg, config); -} - -function convertIniConfig(ini: string) { - const f = ini.split('\n'); - const count = parseInt(f[0], 10); - const res = { subtasks: [] }; - for (let i = 1; i <= count; i++) { - if (!f[i] || !f[i].trim()) throw new FormatError('Testcada count incorrect.'); - const [input, output, time, score, memory] = f[i].split('|'); - const cur = { - cases: [{ input: `input/${input.toLowerCase()}`, output: `output/${output.toLowerCase()}` }], - score: parseInt(score, 10), - time: `${time}s`, - memory: '512m', - }; - if (!Number.isNaN(parseInt(memory, 10))) cur.memory = `${Math.floor(parseInt(memory, 10) / 1024)}m`; - res.subtasks.push(cur); - } - return res; -} - function isValidConfig(config) { if (config.count > (getConfig('testcases_max') || 100)) { throw new FormatError('Too many testcases. Cancelled.'); @@ -309,9 +215,23 @@ export default async function readCases(folder: string, cfg: Record } else if (fs.existsSync(ymlConfig)) { config = { ...yaml.load(fs.readFileSync(ymlConfig).toString()) as object, ...cfg }; } else if (fs.existsSync(iniConfig)) { - config = { ...convertIniConfig(fs.readFileSync(iniConfig).toString()), ...cfg }; + try { + config = { ...convertIniConfig(fs.readFileSync(iniConfig).toString()), ...cfg }; + } catch (e) { + throw changeErrorType(e, FormatError); + } } else config = cfg; - const result = await readYamlCases(folder, config, args); + let result; + try { + result = await readYamlCases(config, ensureFile(folder)); + } catch (e) { + throw changeErrorType(e, FormatError); + } + if (!(result.outputs || result.subtasks.length)) { + const c = await readAutoCases(folder, args, cfg); + config.subtasks = c.subtasks; + config.count = c.count; + } isValidConfig(result); return result; } diff --git a/packages/hydrooj/src/entry/common.ts b/packages/hydrooj/src/entry/common.ts index 2b12037d8..779ff1434 100644 --- a/packages/hydrooj/src/entry/common.ts +++ b/packages/hydrooj/src/entry/common.ts @@ -12,8 +12,7 @@ const logger = new Logger('common', true); export const builtinLib = [ 'jwt', 'download', 'i18n', 'mail', 'useragent', 'crypto', 'misc', 'paginate', 'hash.hydro', 'rank', - 'validator', 'ui', 'testdata.convert.ini', 'testdataConfig', 'difficulty', - 'content', + 'validator', 'ui', 'testdataConfig', 'difficulty', 'content', ]; export const builtinModel = [ diff --git a/packages/hydrooj/src/entry/master.ts b/packages/hydrooj/src/entry/master.ts index 6081bf151..407ddae53 100644 --- a/packages/hydrooj/src/entry/master.ts +++ b/packages/hydrooj/src/entry/master.ts @@ -65,7 +65,7 @@ export async function load(call: Entry) { endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge, }; const storage = require('../service/storage'); - storage.start(sopts); + await storage.start(sopts); require('../service/monitor'); for (const i of builtinModel) require(`../model/${i}`); const scripts = require('../upgrade').default; diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 95bfad782..713a90040 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -546,7 +546,6 @@ export interface Lib extends Record { rank: typeof import('./lib/rank'), rating: typeof import('./lib/rating'), testdataConfig: typeof import('./lib/testdataConfig'), - 'testdata.convert.ini': typeof import('./lib/testdata.convert.ini'), useragent: typeof import('./lib/useragent'), validator: typeof import('./lib/validator'), template?: any, diff --git a/packages/hydrooj/src/lib/testdata.convert.ini.ts b/packages/hydrooj/src/lib/testdata.convert.ini.ts deleted file mode 100644 index 976d19928..000000000 --- a/packages/hydrooj/src/lib/testdata.convert.ini.ts +++ /dev/null @@ -1,22 +0,0 @@ -import yaml from 'js-yaml'; - -function convert(ini: string) { - const f = ini.split('\n'); - const count = parseInt(f[0], 10); - const res = { subtasks: [] }; - for (let i = 1; i <= count; i++) { - const [input, output, time, score, memory] = f[i].split('|'); - const cur = { - cases: [{ input: `input/${input}`, output: `output/${output}` }], - score: parseInt(score, 10), - time: `${time}s`, - memory: '512m', - }; - if (!Number.isNaN(parseInt(memory, 10))) cur.memory = `${Math.floor(parseInt(memory, 10) / 1024)}m`; - res.subtasks.push(cur); - } - return yaml.dump(res); -} - -global.Hydro.lib['testdata.convert.ini'] = convert; -export = convert; diff --git a/packages/hydrooj/src/lib/testdataConfig.ts b/packages/hydrooj/src/lib/testdataConfig.ts index b858c9d75..226d96b99 100644 --- a/packages/hydrooj/src/lib/testdataConfig.ts +++ b/packages/hydrooj/src/lib/testdataConfig.ts @@ -1,23 +1,7 @@ -import Zip from 'adm-zip'; +import readYamlCases from '@hydrooj/utils/lib/cases'; import { load } from 'js-yaml'; import type { ProblemConfig } from '../interface'; -export async function readConfig(file: string | Buffer) { - const data = new Zip(file); - const entries = data.getEntries(); - for (const entry of entries) { - if (entry.name.toLowerCase() === 'config.yaml') { - return entry.getData().toString(); - } - if (entry.name.toLowerCase() === 'config.ini') { - const ini = entry.getData().toString(); - const conv = global.Hydro.lib['testdata.convert.ini']; - return conv(ini); - } - } - return ''; -} - interface ParseResult { count: number, memoryMax: number, @@ -37,7 +21,7 @@ export async function parseConfig(config: string | ProblemConfig) { let cfg: ProblemConfig = {}; if (typeof config === 'string') { // TODO should validate here? - cfg = load(config) as ProblemConfig; + cfg = readYamlCases(load(config) as Record) as ProblemConfig; } else cfg = config; if (cfg.cases) { for (const c of cfg.cases) { @@ -57,4 +41,4 @@ export async function parseConfig(config: string | ProblemConfig) { } } -global.Hydro.lib.testdataConfig = { readConfig, parseConfig }; +global.Hydro.lib.testdataConfig = { parseConfig }; diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index bd3ddc2b1..a49235f5f 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -4,21 +4,21 @@ import { FilterQuery, ObjectID } from 'mongodb'; import AdmZip from 'adm-zip'; import Queue from 'p-queue'; import yaml from 'js-yaml'; +import { convertIniConfig } from '@hydrooj/utils/lib/cases'; import { Progress } from './ui'; +import { Rdoc } from './interface'; import { Logger } from './logger'; import { streamToBuffer } from './utils'; import { iterateAllDomain, iterateAllProblem } from './pipelineUtils'; import gridfs from './service/gridfs'; import storage from './service/storage'; import db from './service/db'; +import difficultyAlgorithm from './lib/difficulty'; import problem from './model/problem'; import domain from './model/domain'; import * as document from './model/document'; import * as system from './model/system'; -import difficultyAlgorithm from './lib/difficulty'; import { STATUS } from './model/builtin'; -import RecordModel from './model/record'; -import { Rdoc } from './interface'; const logger = new Logger('upgrade'); type UpgradeScript = void | (() => Promise); @@ -262,6 +262,33 @@ const scripts: UpgradeScript[] = [ } return true; }, + async function _20_21() { + await iterateAllProblem([], async (pdoc) => { + let config: string; + try { + const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.yaml`); + config = (await streamToBuffer(file)).toString('utf-8'); + logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.yaml`); + } catch (e) { + try { + const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.yml`); + config = (await streamToBuffer(file)).toString('utf-8'); + logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.yml`); + } catch (err) { + try { + const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.ini`); + config = yaml.dump(convertIniConfig((await streamToBuffer(file)).toString('utf-8'))); + logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.ini`); + } catch (error) { + logger.warn('Config for %s/%s(%s) not found', pdoc.domainId, pdoc.docId, pdoc.pid || pdoc.docId); + // no config found + } + } + } + if (config) await problem.edit(pdoc.domainId, pdoc.docId, { config }); + }); + return true; + }, ]; export default scripts; diff --git a/packages/utils/lib/cases.ts b/packages/utils/lib/cases.ts new file mode 100644 index 000000000..b13089d4c --- /dev/null +++ b/packages/utils/lib/cases.ts @@ -0,0 +1,92 @@ +import { parseTimeMS, parseMemoryMB } from './utils'; + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +export default async function readYamlCases(cfg: Record = {}, checkFile = (s: string, errMsg: string) => s) { + const config: any = { + checker_type: 'default', + count: 0, + subtasks: [], + judge_extra_files: [], + user_extra_files: [], + }; + config.checker_type = cfg.checker_type || 'default'; + if (cfg.checker) config.checker = checkFile(cfg.checker, 'Cannot find checker {0}.'); + if (cfg.interactor) config.interactor = checkFile(cfg.interactor, 'Cannot find interactor {0}.'); + if (cfg.judge_extra_files) { + if (typeof cfg.judge_extra_files === 'string') { + config.judge_extra_files = [checkFile(cfg.judge_extra_files, 'Cannot find judge extra file {0}.')]; + } else if (cfg.judge_extra_files instanceof Array) { + for (const file of cfg.judge_extra_files) { + config.judge_extra_files.push(checkFile(file, 'Cannot find judge extra file {0}.')); + } + } else throw new Error('Invalid judge_extra_files config.'); + } + if (cfg.user_extra_files) { + if (typeof cfg.user_extra_files === 'string') { + config.user_extra_files = [checkFile(cfg.user_extra_files, 'Cannot find user extra file {0}.')]; + } else if (cfg.user_extra_files instanceof Array) { + for (const file of cfg.user_extra_files) { + config.user_extra_files.push(checkFile(file, 'Cannot find user extra file {0}.')); + } + } else throw new Error('Invalid user_extra_files config.'); + } + if (cfg.outputs) { + config.type = 'submit_answer'; + } else if (cfg.cases) { + config.subtasks = [{ + score: parseInt(cfg.score, 10) || Math.floor(100 / cfg.cases.length), + time_limit_ms: parseTimeMS(cfg.time || '1s'), + memory_limit_mb: parseMemoryMB(cfg.memory || '512m'), + cases: [], + type: 'sum', + }]; + for (const c of cfg.cases) { + config.count++; + config.subtasks[0].cases.push({ + input: c.input ? checkFile(c.input, 'Cannot find input file {0}.') : '/dev/null', + output: c.output ? checkFile(c.output, 'Cannot find output file {0}.') : '/dev/null', + id: config.count, + }); + } + } else if (cfg.subtasks) { + for (const subtask of cfg.subtasks) { + const cases = []; + for (const c of subtask.cases) { + config.count++; + cases.push({ + input: c.input ? checkFile(c.input, 'Cannot find input file {0}.') : '/dev/null', + output: c.output ? checkFile(c.output, 'Cannot find output file {0}.') : '/dev/null', + id: config.count, + }); + } + config.subtasks.push({ + score: parseInt(subtask.score, 10), + if: subtask.if || [], + cases, + time_limit_ms: parseTimeMS(subtask.time || cfg.time || '1s'), + memory_limit_mb: parseMemoryMB(subtask.memory || cfg.memory || '512m'), + }); + } + } + if (config.type === 'submit_answer' && !cfg.outputs) throw new Error('outputs config not found'); + return Object.assign(cfg, config); +} + +export function convertIniConfig(ini: string) { + const f = ini.split('\n'); + const count = parseInt(f[0], 10); + const res = { subtasks: [] }; + for (let i = 1; i <= count; i++) { + if (!f[i] || !f[i].trim()) throw new Error('Testcada count incorrect.'); + const [input, output, time, score, memory] = f[i].split('|'); + const cur = { + cases: [{ input: `input/${input.toLowerCase()}`, output: `output/${output.toLowerCase()}` }], + score: parseInt(score, 10), + time: `${time}s`, + memory: '512m', + }; + if (!Number.isNaN(parseInt(memory, 10))) cur.memory = `${Math.floor(parseInt(memory, 10) / 1024)}m`; + res.subtasks.push(cur); + } + return res; +} diff --git a/packages/utils/lib/utils.ts b/packages/utils/lib/utils.ts index 195762943..64f8a9b4a 100644 --- a/packages/utils/lib/utils.ts +++ b/packages/utils/lib/utils.ts @@ -297,6 +297,12 @@ export function formatSeconds(_seconds = '0') { ); } +export function changeErrorType(err: Error, Err: any) { + const e = new Err(err.message); + e.stack = err.stack; + return e; +} + export async function retry(func: Function, ...args: any[]): Promise; export async function retry(times: number, func: Function, ...args: any[]): Promise; // eslint-disable-next-line consistent-return