core: dump config.yaml to database (for #65)

packages/hydrojudge/src/cases.ts

@@ -1,7 +1,9 @@
 import fs from 'fs-extra';
 import path from 'path';
 import yaml from 'js-yaml';
-import { Dictionary, sum } from 'lodash';
+import { sum } from 'lodash';
+import readYamlCases, { convertIniConfig } from '@hydrooj/utils/lib/cases';
+import { changeErrorType } from '@hydrooj/utils/lib/utils';
 import { FormatError, SystemError } from './error';
 import { parseTimeMS, parseMemoryMB, ensureFile } from './utils';
 import { getConfig } from './config';
@@ -193,102 +195,6 @@ async function readAutoCases(folder, { next }, cfg) {
     return config;
 }
 
-export async function readYamlCases(folder: string, cfg: Dictionary<any> = {}, args) {
-    const config: any = {
-        checker_type: 'default',
-        count: 0,
-        subtasks: [],
-        judge_extra_files: [],
-        user_extra_files: [],
-    };
-    const next = args.next;
-    const checkFile = ensureFile(folder);
-    config.checker_type = cfg.checker_type || 'default';
-    if (cfg.checker) config.checker = checkFile(cfg.checker, 'Cannot find checker {0}.');
-    if (cfg.interactor) config.interactor = checkFile(cfg.interactor, 'Cannot find interactor {0}.');
-    if (cfg.judge_extra_files) {
-        if (typeof cfg.judge_extra_files === 'string') {
-            config.judge_extra_files = [checkFile(cfg.judge_extra_files, 'Cannot find judge extra file {0}.')];
-        } else if (cfg.judge_extra_files instanceof Array) {
-            for (const file of cfg.judge_extra_files) {
-                config.judge_extra_files.push(checkFile(file, 'Cannot find judge extra file {0}.'));
-            }
-        } else throw new FormatError('Invalid judge_extra_files config.');
-    }
-    if (cfg.user_extra_files) {
-        if (typeof cfg.user_extra_files === 'string') {
-            config.user_extra_files = [checkFile(cfg.user_extra_files, 'Cannot find user extra file {0}.')];
-        } else if (cfg.user_extra_files instanceof Array) {
-            for (const file of cfg.user_extra_files) {
-                config.user_extra_files.push(checkFile(file, 'Cannot find user extra file {0}.'));
-            }
-        } else throw new FormatError('Invalid user_extra_files config.');
-    }
-    if (cfg.outputs) {
-        config.type = 'submit_answer';
-    } else if (cfg.cases) {
-        config.subtasks = [{
-            score: parseInt(cfg.score, 10) || Math.floor(100 / cfg.cases.length),
-            time_limit_ms: parseTimeMS(cfg.time || '1s'),
-            memory_limit_mb: parseMemoryMB(cfg.memory || '512m'),
-            cases: [],
-            type: 'sum',
-        }];
-        for (const c of cfg.cases) {
-            config.count++;
-            config.subtasks[0].cases.push({
-                input: c.input ? checkFile(c.input, 'Cannot find input file {0}.') : '/dev/null',
-                output: c.output ? checkFile(c.output, 'Cannot find output file {0}.') : '/dev/null',
-                id: config.count,
-            });
-        }
-    } else if (cfg.subtasks) {
-        for (const subtask of cfg.subtasks) {
-            const cases = [];
-            for (const c of subtask.cases) {
-                config.count++;
-                cases.push({
-                    input: c.input ? checkFile(c.input, 'Cannot find input file {0}.') : '/dev/null',
-                    output: c.output ? checkFile(c.output, 'Cannot find output file {0}.') : '/dev/null',
-                    id: config.count,
-                });
-            }
-            config.subtasks.push({
-                score: parseInt(subtask.score, 10),
-                if: subtask.if || [],
-                cases,
-                time_limit_ms: parseTimeMS(subtask.time || cfg.time || '1s'),
-                memory_limit_mb: parseMemoryMB(subtask.memory || cfg.memory || '512m'),
-            });
-        }
-    } else {
-        const c = await readAutoCases(folder, { next }, cfg);
-        config.subtasks = c.subtasks;
-        config.count = c.count;
-    }
-    if (config.type === 'submit_answer' && !cfg.outputs) throw new FormatError('outputs config not found');
-    return Object.assign(cfg, config);
-}
-
-function convertIniConfig(ini: string) {
-    const f = ini.split('\n');
-    const count = parseInt(f[0], 10);
-    const res = { subtasks: [] };
-    for (let i = 1; i <= count; i++) {
-        if (!f[i] || !f[i].trim()) throw new FormatError('Testcada count incorrect.');
-        const [input, output, time, score, memory] = f[i].split('|');
-        const cur = {
-            cases: [{ input: `input/${input.toLowerCase()}`, output: `output/${output.toLowerCase()}` }],
-            score: parseInt(score, 10),
-            time: `${time}s`,
-            memory: '512m',
-        };
-        if (!Number.isNaN(parseInt(memory, 10))) cur.memory = `${Math.floor(parseInt(memory, 10) / 1024)}m`;
-        res.subtasks.push(cur);
-    }
-    return res;
-}
-
 function isValidConfig(config) {
     if (config.count > (getConfig('testcases_max') || 100)) {
         throw new FormatError('Too many testcases. Cancelled.');
@@ -309,9 +215,23 @@ export default async function readCases(folder: string, cfg: Record<string, any>
     } else if (fs.existsSync(ymlConfig)) {
         config = { ...yaml.load(fs.readFileSync(ymlConfig).toString()) as object, ...cfg };
     } else if (fs.existsSync(iniConfig)) {
-        config = { ...convertIniConfig(fs.readFileSync(iniConfig).toString()), ...cfg };
+        try {
+            config = { ...convertIniConfig(fs.readFileSync(iniConfig).toString()), ...cfg };
+        } catch (e) {
+            throw changeErrorType(e, FormatError);
+        }
     } else config = cfg;
-    const result = await readYamlCases(folder, config, args);
+    let result;
+    try {
+        result = await readYamlCases(config, ensureFile(folder));
+    } catch (e) {
+        throw changeErrorType(e, FormatError);
+    }
+    if (!(result.outputs || result.subtasks.length)) {
+        const c = await readAutoCases(folder, args, cfg);
+        config.subtasks = c.subtasks;
+        config.count = c.count;
+    }
     isValidConfig(result);
     return result;
 }

packages/hydrooj/src/entry/common.ts

@@ -12,8 +12,7 @@ const logger = new Logger('common', true);
 export const builtinLib = [
     'jwt', 'download', 'i18n', 'mail', 'useragent',
     'crypto', 'misc', 'paginate', 'hash.hydro', 'rank',
-    'validator', 'ui', 'testdata.convert.ini', 'testdataConfig', 'difficulty',
-    'content',
+    'validator', 'ui', 'testdataConfig', 'difficulty', 'content',
 ];
 
 export const builtinModel = [

packages/hydrooj/src/entry/master.ts

@@ -65,7 +65,7 @@ export async function load(call: Entry) {
         endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge,
     };
     const storage = require('../service/storage');
-    storage.start(sopts);
+    await storage.start(sopts);
     require('../service/monitor');
     for (const i of builtinModel) require(`../model/${i}`);
     const scripts = require('../upgrade').default;

packages/hydrooj/src/interface.ts

@@ -546,7 +546,6 @@ export interface Lib extends Record<string, any> {
     rank: typeof import('./lib/rank'),
     rating: typeof import('./lib/rating'),
     testdataConfig: typeof import('./lib/testdataConfig'),
-    'testdata.convert.ini': typeof import('./lib/testdata.convert.ini'),
     useragent: typeof import('./lib/useragent'),
     validator: typeof import('./lib/validator'),
     template?: any,

packages/hydrooj/src/lib/testdata.convert.ini.ts

@@ -1,22 +0,0 @@
-import yaml from 'js-yaml';
-
-function convert(ini: string) {
-    const f = ini.split('\n');
-    const count = parseInt(f[0], 10);
-    const res = { subtasks: [] };
-    for (let i = 1; i <= count; i++) {
-        const [input, output, time, score, memory] = f[i].split('|');
-        const cur = {
-            cases: [{ input: `input/${input}`, output: `output/${output}` }],
-            score: parseInt(score, 10),
-            time: `${time}s`,
-            memory: '512m',
-        };
-        if (!Number.isNaN(parseInt(memory, 10))) cur.memory = `${Math.floor(parseInt(memory, 10) / 1024)}m`;
-        res.subtasks.push(cur);
-    }
-    return yaml.dump(res);
-}
-
-global.Hydro.lib['testdata.convert.ini'] = convert;
-export = convert;

packages/hydrooj/src/lib/testdataConfig.ts

@@ -1,23 +1,7 @@
-import Zip from 'adm-zip';
+import readYamlCases from '@hydrooj/utils/lib/cases';
 import { load } from 'js-yaml';
 import type { ProblemConfig } from '../interface';
 
-export async function readConfig(file: string | Buffer) {
-    const data = new Zip(file);
-    const entries = data.getEntries();
-    for (const entry of entries) {
-        if (entry.name.toLowerCase() === 'config.yaml') {
-            return entry.getData().toString();
-        }
-        if (entry.name.toLowerCase() === 'config.ini') {
-            const ini = entry.getData().toString();
-            const conv = global.Hydro.lib['testdata.convert.ini'];
-            return conv(ini);
-        }
-    }
-    return '';
-}
-
 interface ParseResult {
     count: number,
     memoryMax: number,
@@ -37,7 +21,7 @@ export async function parseConfig(config: string | ProblemConfig) {
     let cfg: ProblemConfig = {};
     if (typeof config === 'string') {
         // TODO should validate here?
-        cfg = load(config) as ProblemConfig;
+        cfg = readYamlCases(load(config) as Record<string, any>) as ProblemConfig;
     } else cfg = config;
     if (cfg.cases) {
         for (const c of cfg.cases) {
@@ -57,4 +41,4 @@ export async function parseConfig(config: string | ProblemConfig) {
     }
 }
 
-global.Hydro.lib.testdataConfig = { readConfig, parseConfig };
+global.Hydro.lib.testdataConfig = { parseConfig };

packages/hydrooj/src/upgrade.ts

@@ -4,21 +4,21 @@ import { FilterQuery, ObjectID } from 'mongodb';
 import AdmZip from 'adm-zip';
 import Queue from 'p-queue';
 import yaml from 'js-yaml';
+import { convertIniConfig } from '@hydrooj/utils/lib/cases';
 import { Progress } from './ui';
+import { Rdoc } from './interface';
 import { Logger } from './logger';
 import { streamToBuffer } from './utils';
 import { iterateAllDomain, iterateAllProblem } from './pipelineUtils';
 import gridfs from './service/gridfs';
 import storage from './service/storage';
 import db from './service/db';
+import difficultyAlgorithm from './lib/difficulty';
 import problem from './model/problem';
 import domain from './model/domain';
 import * as document from './model/document';
 import * as system from './model/system';
-import difficultyAlgorithm from './lib/difficulty';
 import { STATUS } from './model/builtin';
-import RecordModel from './model/record';
-import { Rdoc } from './interface';
 
 const logger = new Logger('upgrade');
 type UpgradeScript = void | (() => Promise<boolean | void>);
@@ -262,6 +262,33 @@ const scripts: UpgradeScript[] = [
         }
         return true;
     },
+    async function _20_21() {
+        await iterateAllProblem([], async (pdoc) => {
+            let config: string;
+            try {
+                const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.yaml`);
+                config = (await streamToBuffer(file)).toString('utf-8');
+                logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.yaml`);
+            } catch (e) {
+                try {
+                    const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.yml`);
+                    config = (await streamToBuffer(file)).toString('utf-8');
+                    logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.yml`);
+                } catch (err) {
+                    try {
+                        const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.ini`);
+                        config = yaml.dump(convertIniConfig((await streamToBuffer(file)).toString('utf-8')));
+                        logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.ini`);
+                    } catch (error) {
+                        logger.warn('Config for %s/%s(%s) not found', pdoc.domainId, pdoc.docId, pdoc.pid || pdoc.docId);
+                        // no config found
+                    }
+                }
+            }
+            if (config) await problem.edit(pdoc.domainId, pdoc.docId, { config });
+        });
+        return true;
+    },
 ];
 
 export default scripts;

packages/utils/lib/cases.ts

@@ -0,0 +1,92 @@
+import { parseTimeMS, parseMemoryMB } from './utils';
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+export default async function readYamlCases(cfg: Record<string, any> = {}, checkFile = (s: string, errMsg: string) => s) {
+    const config: any = {
+        checker_type: 'default',
+        count: 0,
+        subtasks: [],
+        judge_extra_files: [],
+        user_extra_files: [],
+    };
+    config.checker_type = cfg.checker_type || 'default';
+    if (cfg.checker) config.checker = checkFile(cfg.checker, 'Cannot find checker {0}.');
+    if (cfg.interactor) config.interactor = checkFile(cfg.interactor, 'Cannot find interactor {0}.');
+    if (cfg.judge_extra_files) {
+        if (typeof cfg.judge_extra_files === 'string') {
+            config.judge_extra_files = [checkFile(cfg.judge_extra_files, 'Cannot find judge extra file {0}.')];
+        } else if (cfg.judge_extra_files instanceof Array) {
+            for (const file of cfg.judge_extra_files) {
+                config.judge_extra_files.push(checkFile(file, 'Cannot find judge extra file {0}.'));
+            }
+        } else throw new Error('Invalid judge_extra_files config.');
+    }
+    if (cfg.user_extra_files) {
+        if (typeof cfg.user_extra_files === 'string') {
+            config.user_extra_files = [checkFile(cfg.user_extra_files, 'Cannot find user extra file {0}.')];
+        } else if (cfg.user_extra_files instanceof Array) {
+            for (const file of cfg.user_extra_files) {
+                config.user_extra_files.push(checkFile(file, 'Cannot find user extra file {0}.'));
+            }
+        } else throw new Error('Invalid user_extra_files config.');
+    }
+    if (cfg.outputs) {
+        config.type = 'submit_answer';
+    } else if (cfg.cases) {
+        config.subtasks = [{
+            score: parseInt(cfg.score, 10) || Math.floor(100 / cfg.cases.length),
+            time_limit_ms: parseTimeMS(cfg.time || '1s'),
+            memory_limit_mb: parseMemoryMB(cfg.memory || '512m'),
+            cases: [],
+            type: 'sum',
+        }];
+        for (const c of cfg.cases) {
+            config.count++;
+            config.subtasks[0].cases.push({
+                input: c.input ? checkFile(c.input, 'Cannot find input file {0}.') : '/dev/null',
+                output: c.output ? checkFile(c.output, 'Cannot find output file {0}.') : '/dev/null',
+                id: config.count,
+            });
+        }
+    } else if (cfg.subtasks) {
+        for (const subtask of cfg.subtasks) {
+            const cases = [];
+            for (const c of subtask.cases) {
+                config.count++;
+                cases.push({
+                    input: c.input ? checkFile(c.input, 'Cannot find input file {0}.') : '/dev/null',
+                    output: c.output ? checkFile(c.output, 'Cannot find output file {0}.') : '/dev/null',
+                    id: config.count,
+                });
+            }
+            config.subtasks.push({
+                score: parseInt(subtask.score, 10),
+                if: subtask.if || [],
+                cases,
+                time_limit_ms: parseTimeMS(subtask.time || cfg.time || '1s'),
+                memory_limit_mb: parseMemoryMB(subtask.memory || cfg.memory || '512m'),
+            });
+        }
+    }
+    if (config.type === 'submit_answer' && !cfg.outputs) throw new Error('outputs config not found');
+    return Object.assign(cfg, config);
+}
+
+export function convertIniConfig(ini: string) {
+    const f = ini.split('\n');
+    const count = parseInt(f[0], 10);
+    const res = { subtasks: [] };
+    for (let i = 1; i <= count; i++) {
+        if (!f[i] || !f[i].trim()) throw new Error('Testcada count incorrect.');
+        const [input, output, time, score, memory] = f[i].split('|');
+        const cur = {
+            cases: [{ input: `input/${input.toLowerCase()}`, output: `output/${output.toLowerCase()}` }],
+            score: parseInt(score, 10),
+            time: `${time}s`,
+            memory: '512m',
+        };
+        if (!Number.isNaN(parseInt(memory, 10))) cur.memory = `${Math.floor(parseInt(memory, 10) / 1024)}m`;
+        res.subtasks.push(cur);
+    }
+    return res;
+}

packages/utils/lib/utils.ts

@@ -297,6 +297,12 @@ export function formatSeconds(_seconds = '0') {
     );
 }
 
+export function changeErrorType(err: Error, Err: any) {
+    const e = new Err(err.message);
+    e.stack = err.stack;
+    return e;
+}
+
 export async function retry(func: Function, ...args: any[]): Promise<any>;
 export async function retry(times: number, func: Function, ...args: any[]): Promise<any>;
 // eslint-disable-next-line consistent-return