core: dump config.yaml to database (for #65)

packages/hydrooj/src/entry/common.ts

@@ -12,8 +12,7 @@ const logger = new Logger('common', true);
 export const builtinLib = [
     'jwt', 'download', 'i18n', 'mail', 'useragent',
     'crypto', 'misc', 'paginate', 'hash.hydro', 'rank',
-    'validator', 'ui', 'testdata.convert.ini', 'testdataConfig', 'difficulty',
-    'content',
+    'validator', 'ui', 'testdataConfig', 'difficulty', 'content',
 ];
 
 export const builtinModel = [

packages/hydrooj/src/entry/master.ts

@@ -65,7 +65,7 @@ export async function load(call: Entry) {
         endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge,
     };
     const storage = require('../service/storage');
-    storage.start(sopts);
+    await storage.start(sopts);
     require('../service/monitor');
     for (const i of builtinModel) require(`../model/${i}`);
     const scripts = require('../upgrade').default;

packages/hydrooj/src/interface.ts

@@ -546,7 +546,6 @@ export interface Lib extends Record<string, any> {
     rank: typeof import('./lib/rank'),
     rating: typeof import('./lib/rating'),
     testdataConfig: typeof import('./lib/testdataConfig'),
-    'testdata.convert.ini': typeof import('./lib/testdata.convert.ini'),
     useragent: typeof import('./lib/useragent'),
     validator: typeof import('./lib/validator'),
     template?: any,

packages/hydrooj/src/lib/testdata.convert.ini.ts

@@ -1,22 +0,0 @@
-import yaml from 'js-yaml';
-
-function convert(ini: string) {
-    const f = ini.split('\n');
-    const count = parseInt(f[0], 10);
-    const res = { subtasks: [] };
-    for (let i = 1; i <= count; i++) {
-        const [input, output, time, score, memory] = f[i].split('|');
-        const cur = {
-            cases: [{ input: `input/${input}`, output: `output/${output}` }],
-            score: parseInt(score, 10),
-            time: `${time}s`,
-            memory: '512m',
-        };
-        if (!Number.isNaN(parseInt(memory, 10))) cur.memory = `${Math.floor(parseInt(memory, 10) / 1024)}m`;
-        res.subtasks.push(cur);
-    }
-    return yaml.dump(res);
-}
-
-global.Hydro.lib['testdata.convert.ini'] = convert;
-export = convert;

packages/hydrooj/src/lib/testdataConfig.ts

@@ -1,23 +1,7 @@
-import Zip from 'adm-zip';
+import readYamlCases from '@hydrooj/utils/lib/cases';
 import { load } from 'js-yaml';
 import type { ProblemConfig } from '../interface';
 
-export async function readConfig(file: string | Buffer) {
-    const data = new Zip(file);
-    const entries = data.getEntries();
-    for (const entry of entries) {
-        if (entry.name.toLowerCase() === 'config.yaml') {
-            return entry.getData().toString();
-        }
-        if (entry.name.toLowerCase() === 'config.ini') {
-            const ini = entry.getData().toString();
-            const conv = global.Hydro.lib['testdata.convert.ini'];
-            return conv(ini);
-        }
-    }
-    return '';
-}
-
 interface ParseResult {
     count: number,
     memoryMax: number,
@@ -37,7 +21,7 @@ export async function parseConfig(config: string | ProblemConfig) {
     let cfg: ProblemConfig = {};
     if (typeof config === 'string') {
         // TODO should validate here?
-        cfg = load(config) as ProblemConfig;
+        cfg = readYamlCases(load(config) as Record<string, any>) as ProblemConfig;
     } else cfg = config;
     if (cfg.cases) {
         for (const c of cfg.cases) {
@@ -57,4 +41,4 @@ export async function parseConfig(config: string | ProblemConfig) {
     }
 }
 
-global.Hydro.lib.testdataConfig = { readConfig, parseConfig };
+global.Hydro.lib.testdataConfig = { parseConfig };

packages/hydrooj/src/upgrade.ts

@@ -4,21 +4,21 @@ import { FilterQuery, ObjectID } from 'mongodb';
 import AdmZip from 'adm-zip';
 import Queue from 'p-queue';
 import yaml from 'js-yaml';
+import { convertIniConfig } from '@hydrooj/utils/lib/cases';
 import { Progress } from './ui';
+import { Rdoc } from './interface';
 import { Logger } from './logger';
 import { streamToBuffer } from './utils';
 import { iterateAllDomain, iterateAllProblem } from './pipelineUtils';
 import gridfs from './service/gridfs';
 import storage from './service/storage';
 import db from './service/db';
+import difficultyAlgorithm from './lib/difficulty';
 import problem from './model/problem';
 import domain from './model/domain';
 import * as document from './model/document';
 import * as system from './model/system';
-import difficultyAlgorithm from './lib/difficulty';
 import { STATUS } from './model/builtin';
-import RecordModel from './model/record';
-import { Rdoc } from './interface';
 
 const logger = new Logger('upgrade');
 type UpgradeScript = void | (() => Promise<boolean | void>);
@@ -262,6 +262,33 @@ const scripts: UpgradeScript[] = [
         }
         return true;
     },
+    async function _20_21() {
+        await iterateAllProblem([], async (pdoc) => {
+            let config: string;
+            try {
+                const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.yaml`);
+                config = (await streamToBuffer(file)).toString('utf-8');
+                logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.yaml`);
+            } catch (e) {
+                try {
+                    const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.yml`);
+                    config = (await streamToBuffer(file)).toString('utf-8');
+                    logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.yml`);
+                } catch (err) {
+                    try {
+                        const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.ini`);
+                        config = yaml.dump(convertIniConfig((await streamToBuffer(file)).toString('utf-8')));
+                        logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.ini`);
+                    } catch (error) {
+                        logger.warn('Config for %s/%s(%s) not found', pdoc.domainId, pdoc.docId, pdoc.pid || pdoc.docId);
+                        // no config found
+                    }
+                }
+            }
+            if (config) await problem.edit(pdoc.domainId, pdoc.docId, { config });
+        });
+        return true;
+    },
 ];
 
 export default scripts;