core: drop array-style problem content (#882)

packages/fps-importer/index.ts

@@ -2,7 +2,7 @@
 import decodeHTML from 'decode-html';
 import xml2js from 'xml2js';
 import {
-    _, AdmZip, BadRequestError, buildContent, ContentNode, Context, FileTooLargeError, fs, Handler,
+    _, AdmZip, BadRequestError, buildContent, Context, FileTooLargeError, fs, Handler,
     PERM, ProblemConfigFile, ProblemModel, ProblemType, SettingModel, SolutionModel, SystemModel, ValidationError, yaml,
 } from 'hydrooj';
 
@@ -19,48 +19,14 @@ class FpsProblemImportHandler extends Handler {
         if (!result?.fps) throw new BadRequestError('Selected file is not a valid FPS problemset.');
         for (const p of result.fps.item) {
             const markdown = [p.description?.[0], p.input?.[0], p.output?.[0], p.hint?.[0]].some((i) => i?.includes('[md]'));
-            const content: ContentNode[] = [];
+            const content = buildContent({
-            const stripBBCode = (input: string) => input.replace(/\[md]/g, '').replace(/\[\/md]/g, '');
+                description: p.description?.[0],
-
+                input: p.input?.[0],
-            if (p.description?.[0]) {
+                output: p.output?.[0],
-                content.push({
+                samples: p.sample_input?.map((input, i) => [input, p.sample_output[i]]),
-                    type: 'Text',
+                hint: p.hint?.[0],
-                    subType: markdown ? 'markdown' : 'html',
+                source: p.source?.join(' '),
-                    sectionTitle: this.translate('Description'),
+            }, 'html').replace(/<math xm<x>lns=/g, '<math xmlns=').replace(/\[\/?md]/g, '');
-                    text: stripBBCode(p.description[0]),
-                });
-            }
-            if (p.input?.[0]) {
-                content.push({
-                    type: 'Text',
-                    subType: markdown ? 'markdown' : 'html',
-                    sectionTitle: this.translate('Input Format'),
-                    text: stripBBCode(p.input[0]),
-                });
-            }
-            if (p.output?.[0]) {
-                content.push({
-                    type: 'Text',
-                    subType: markdown ? 'markdown' : 'html',
-                    sectionTitle: this.translate('Output Format'),
-                    text: stripBBCode(p.output[0]),
-                });
-            }
-            if (p.sample_input?.length) {
-                content.push(...p.sample_input.map((input, i) => ({
-                    type: 'Sample',
-                    sectionTitle: this.translate('Sample'),
-                    payload: [input, p.sample_output[i]],
-                })));
-            }
-            if (p.hint?.[0]) {
-                content.push({
-                    type: 'Text',
-                    subType: markdown ? 'markdown' : 'html',
-                    sectionTitle: this.translate('Hint'),
-                    text: stripBBCode(p.hint[0]),
-                });
-            }
             const config: ProblemConfigFile = {
                 time: p.time_limit[0]._ + p.time_limit[0].$.unit,
                 memory: p.memory_limit[0]._ + p.memory_limit[0].$.unit,
@@ -72,17 +38,17 @@ class FpsProblemImportHandler extends Handler {
             }
             const title = decodeHTML(p.title.join(' '));
             const tags = _.filter(p.source, (i: string) => i.trim()).flatMap((i) => i.split(' ')).filter((i) => i);
-            const pid = await ProblemModel.add(domainId, null, title, buildContent(content, 'html'), this.user._id, tags);
+            const pid = await ProblemModel.add(domainId, null, title, content, this.user._id, tags);
             const tasks: Promise<any>[] = [ProblemModel.addTestdata(domainId, pid, 'config.yaml', Buffer.from(yaml.dump(config)))];
             if (!markdown) tasks.push(ProblemModel.edit(domainId, pid, { html: true }));
-            const addTestdata = (node: any, id: string, ext: string) => {
+            const addTestdata = (node: any, index: string, ext: string) => {
-                if (!node && typeof node !== 'string') return; // Ignore file not exist
+                if (!node || !['object', 'string'].includes(typeof node)) return; // Ignore file not exist
-                let c = node;
+                const id = node.$?.name || `${index}`;
-                if (node.$?.name) {
+                // `filename` attribute introduced by winterant/OnlineJudge
-                    id = node.$.name;
+                // PLEASE, respect the spec
-                    c = node._ || '';
+                const filename = node.$?.filename || `${id}.${ext}`;
-                }
+                const c = node._ || node;
-                tasks.push(ProblemModel.addTestdata(domainId, pid, `${id}.${ext}`, Buffer.from(c)));
+                tasks.push(ProblemModel.addTestdata(domainId, pid, filename, Buffer.from(c)));
             };
             if (p.test_output) {
                 for (let i = 0; i < p.test_input.length; i++) {
@@ -90,6 +56,7 @@ class FpsProblemImportHandler extends Handler {
                     addTestdata(p.test_output[i], `${i + 1}`, 'out');
                 }
             } else if (p.test_input) {
+                // Some version of hustoj exports only test_input section
                 for (let i = 0; i < p.test_input.length / 2; i++) {
                     addTestdata(p.test_input[2 * i], `${i + 1}`, 'in');
                     addTestdata(p.test_input[2 * i + 1], `${i + 1}`, 'out');

packages/hydrooj/src/interface.ts

@@ -221,26 +221,7 @@ export interface ProblemConfig {
     hackable?: boolean;
 }
 
-export interface PlainContentNode {
+export type Content = string | Record<string, string>;
-    type: 'Plain',
-    subType: 'html' | 'markdown',
-    text: string,
-}
-export interface TextContentNode {
-    type: 'Text',
-    subType: 'html' | 'markdown',
-    sectionTitle: string,
-    text: string,
-}
-export interface SampleContentNode {
-    type: 'Sample',
-    text: string,
-    sectionTitle: string,
-    payload: [string, string],
-}
-// TODO drop contentNode support
-export type ContentNode = PlainContentNode | TextContentNode | SampleContentNode;
-export type Content = string | ContentNode[] | Record<string, ContentNode[]>;
 
 export interface Document {
     _id: ObjectId;

packages/hydrooj/src/lib/content.ts

@@ -1,17 +1,18 @@
-import { ContentNode } from '../interface';
-
 export interface ProblemSource {
     description?: string,
     input?: string,
     output?: string,
     samples?: [string, string][],
+    samplesRaw?: string,
     hint?: string,
     source?: string,
 }
 
-export function buildContent(source: ProblemSource | ContentNode[], type: 'markdown' | 'html' = 'markdown', translate?: Function) {
+export function buildContent(source: ProblemSource, type: 'markdown' | 'html' = 'markdown', translate?: Function) {
     const _ = translate || ((s: string) => s);
     let cnt = 0;
+    // Keep it for backward compatibility, but don't add to typings
+    // required by upgrade 90_91
     if (source instanceof Array) {
         return type === 'html'
             ? source.map((node) => [
@@ -54,6 +55,7 @@ export function buildContent(source: ProblemSource | ContentNode[], type: 'markd
                 sample[1],
                 '</code></pre>',
             ].join('')),
+            ...source.samplesRaw ? [source.samplesRaw] : [],
             ...source.hint ? [`<h2>${_('Hint')}</h2>`, source.hint] : [],
             ...source.source ? [`<h2>${_('Source')}</h2>`, source.source] : [],
         ].join('\n')
@@ -69,6 +71,7 @@ export function buildContent(source: ProblemSource | ContentNode[], type: 'markd
                 sample[1],
                 '```',
             ].join('\n')),
+            ...source.samplesRaw ? [source.samplesRaw] : [],
             ...source.hint ? [`## ${_('Hint')}`, source.hint] : [],
             ...source.source ? [`## ${_('Source')}`, source.source] : [],
         ].join('\n');

packages/hydrooj/src/model/problem.ts

@@ -45,12 +45,7 @@ function findOverrideContent(dir: string, base: string) {
     if (!files.length) return null;
     for (const file of files) {
         const lang = file.slice(8, -3);
-        let content: string | any[] = fs.readFileSync(path.join(dir, file), 'utf8');
+        languages[lang] = fs.readFileSync(path.join(dir, file), 'utf8');
-        try {
-            content = JSON.parse(content);
-            if (!(content instanceof Array)) content = JSON.stringify(content);
-        } catch (e) { }
-        languages[lang] = content;
     }
     return JSON.stringify(languages);
 }
@@ -548,12 +543,12 @@ export class ProblemModel {
                 const docId = overridePid
                     ? (await ProblemModel.edit(domainId, overridePid, {
                         title: pdoc.title.trim(),
-                        content: overrideContent || pdoc.content || 'No content',
+                        content: overrideContent || pdoc.content.toString() || 'No content',
                         tag,
                         difficulty: pdoc.difficulty,
                     })).docId
                     : await ProblemModel.add(
-                        domainId, pid, pdoc.title.trim(), overrideContent || pdoc.content || 'No content',
+                        domainId, pid, pdoc.title.trim(), overrideContent || pdoc.content.toString() || 'No content',
                         operator || pdoc.owner, tag, { hidden: pdoc.hidden, difficulty: pdoc.difficulty },
                     );
                 // TODO delete unused file when updating pdoc

packages/hydrooj/src/upgrade.ts

@@ -1,3 +1,4 @@
+/* eslint-disable consistent-return */
 /* eslint-disable @typescript-eslint/no-unused-vars */
 /* eslint-disable no-await-in-loop */
 /* eslint-disable @typescript-eslint/naming-convention */
@@ -51,25 +52,7 @@ export const coreScripts: MigrationScript[] = [
         return true;
     },
     // Init
-    ...new Array(26).fill(unsupportedUpgrade),
+    ...new Array(28).fill(unsupportedUpgrade),
-    null,
-    async function _28_29() {
-        return await iterateAllProblem(['content', 'html'], async (pdoc) => {
-            try {
-                const parsed = JSON.parse(pdoc.content);
-                if (parsed instanceof Array) {
-                    await problem.edit(pdoc.domainId, pdoc.docId, { content: buildContent(parsed, pdoc.html ? 'html' : 'markdown') });
-                    return;
-                }
-                const res = {};
-                for (const key in parsed) {
-                    if (typeof parsed[key] === 'string') res[key] = parsed[key];
-                    else res[key] = buildContent(parsed[key]);
-                }
-                await problem.edit(pdoc.domainId, pdoc.docId, { content: JSON.stringify(res) });
-            } catch { }
-        });
-    },
     async function _29_30() {
         return await iterateAllDomain((ddoc) => RecordModel.coll.updateMany({ domainId: ddoc._id }, { $set: { pdomain: ddoc._id } }));
     },
@@ -685,4 +668,38 @@ export const coreScripts: MigrationScript[] = [
             }
         });
     },
+    async function _90_91() {
+        await document.collStatus.updateMany({ docType: document.TYPE_PROBLEM }, { $unset: { nSubmit: '', nAccept: '' } });
+        const psdocs = await document.coll.find({ docType: document.TYPE_PROBLEM_SOLUTION, vote: { $ne: 0 } })
+            .project({ docId: 1, domainId: 1 }).toArray();
+        for (const psdoc of psdocs) {
+            const filter = { docType: document.TYPE_PROBLEM_SOLUTION, domainId: psdoc.domainId, docId: psdoc.docId };
+            const [upvote, downvote] = await Promise.all([
+                document.collStatus.countDocuments({ ...filter, vote: 1 }),
+                document.collStatus.countDocuments({ ...filter, vote: -1 }),
+            ]);
+            if (upvote - downvote !== psdoc.vote) {
+                await document.set(psdoc.domainId, document.TYPE_PROBLEM_SOLUTION, psdoc.docId, { vote: upvote - downvote });
+            }
+        }
+        await iterateAllProblem(['domainId', 'docId', 'tag'], async (pdoc) => {
+            if (pdoc.tag?.some((i) => typeof i !== 'string')) {
+                return { tag: pdoc.tag.filter((i) => i).map((i) => i.toString()) };
+            }
+        });
+        return await iterateAllProblem(['domainId', 'docId', 'content', 'html'], async (pdoc) => {
+            try {
+                const parsed = JSON.parse(pdoc.content);
+                if (parsed instanceof Array) {
+                    return { content: buildContent(parsed as any, pdoc.html ? 'html' : 'markdown') };
+                }
+                const res = {};
+                for (const key in parsed) {
+                    if (typeof parsed[key] === 'string') res[key] = parsed[key];
+                    else res[key] = buildContent(parsed[key]);
+                }
+                return { content: JSON.stringify(res) };
+            } catch { }
+        });
+    },
 ];

packages/import-qduoj/index.ts

@@ -2,13 +2,42 @@
 import os from 'os';
 import path from 'path';
 import {
-    AdmZip, buildContent, ContentNode, Context, fs, Handler, PERM,
+    AdmZip, buildContent, Context, FileTooLargeError, fs, Handler, PERM,
-    ProblemConfigFile, ProblemModel, ValidationError, yaml,
+    ProblemConfigFile, ProblemModel, Schema, ValidationError, yaml,
 } from 'hydrooj';
 
 const tmpdir = path.join(os.tmpdir(), 'hydro', 'import-qduoj');
 fs.ensureDirSync(tmpdir);
 
+const StringValue = Schema.object({
+    value: Schema.string(),
+});
+const ProblemSchema = Schema.object({
+    tags: Schema.array(Schema.string()),
+    title: Schema.string().required(),
+    description: StringValue,
+    input_description: StringValue,
+    output_description: StringValue,
+    samples: Schema.array(Schema.object({
+        input: Schema.string(),
+        output: Schema.string(),
+    })),
+    hint: StringValue,
+    source: StringValue,
+    display_id: Schema.string(),
+    time_limit: Schema.union([Schema.number(), Schema.string()]).required(),
+    memory_limit: Schema.union([Schema.number(), Schema.string()]).required(),
+    spj: Schema.object({
+        language: Schema.string().required(),
+        code: Schema.string().required(),
+    }),
+    test_case_score: Schema.array(Schema.object({
+        input_name: Schema.string().required(),
+        output_name: Schema.string().required(),
+        score: Schema.number().required(),
+    })),
+});
+
 class ImportQduojHandler extends Handler {
     async fromFile(domainId: string, zipfile: string) {
         let zip: AdmZip;
@@ -27,55 +56,15 @@ class ImportQduojHandler extends Handler {
             for (const { name: folder } of folders.filter((i) => i.isDirectory())) {
                 if (!fs.existsSync(path.join(tmp, folder, 'problem.json'))) continue;
                 const buf = await fs.readFile(path.join(tmp, folder, 'problem.json'));
-                const pdoc = JSON.parse(buf.toString());
+                const pdoc = ProblemSchema(JSON.parse(buf.toString()));
-                const content: ContentNode[] = [];
+                const content = buildContent({
-                if (pdoc.description?.value) {
+                    description: pdoc.description?.value,
-                    content.push({
+                    input: pdoc.input_description?.value,
-                        type: 'Text',
+                    output: pdoc.output_description?.value,
-                        subType: 'html',
+                    samples: pdoc.samples.map((sample) => [sample.input, sample.output]),
-                        sectionTitle: this.translate('Description'),
+                    hint: pdoc.hint?.value,
-                        text: pdoc.description.value,
+                    source: pdoc.source?.value,
-                    });
+                }, 'html');
-                }
-                if (pdoc.input_description?.value) {
-                    content.push({
-                        type: 'Text',
-                        subType: 'html',
-                        sectionTitle: this.translate('Input Format'),
-                        text: pdoc.input_description.value,
-                    });
-                }
-                if (pdoc.output_description?.value) {
-                    content.push({
-                        type: 'Text',
-                        subType: 'html',
-                        sectionTitle: this.translate('Output Format'),
-                        text: pdoc.output_description.value,
-                    });
-                }
-                if (pdoc.samples?.length) {
-                    content.push(...pdoc.samples.map((sample) => ({
-                        type: 'Sample',
-                        sectionTitle: this.translate('Sample'),
-                        payload: [sample.input, sample.output],
-                    })));
-                }
-                if (pdoc.hint?.value) {
-                    content.push({
-                        type: 'Text',
-                        subType: 'html',
-                        sectionTitle: this.translate('Hint'),
-                        text: pdoc.hint.value,
-                    });
-                }
-                if (pdoc.source?.value) {
-                    content.push({
-                        type: 'Text',
-                        subType: 'html',
-                        sectionTitle: this.translate('Source'),
-                        text: pdoc.source.value,
-                    });
-                }
                 if (+pdoc.display_id) pdoc.display_id = `P${pdoc.display_id}`;
                 const isValidPid = async (id: string) => {
                     if (!(/^[A-Za-z]+[0-9A-Za-z]*$/.test(id))) return false;
@@ -84,7 +73,7 @@ class ImportQduojHandler extends Handler {
                 };
                 if (!await isValidPid(pdoc.display_id)) pdoc.display_id = null;
                 const pid = await ProblemModel.add(
-                    domainId, pdoc.display_id, pdoc.title, buildContent(content, 'html'),
+                    domainId, pdoc.display_id, pdoc.title, content,
                     this.user._id, pdoc.tags || [],
                 );
                 const config: ProblemConfigFile = {
@@ -141,7 +130,7 @@ class ImportQduojHandler extends Handler {
     async post({ domainId }) {
         if (!this.request.files.file) throw new ValidationError('file');
         const stat = await fs.stat(this.request.files.file.filepath);
-        if (stat.size > 128 * 1024 * 1024) throw new ValidationError('file', 'File too large');
+        if (stat.size > 256 * 1024 * 1024) throw new FileTooLargeError('256m');
         await this.fromFile(domainId, this.request.files.file.filepath);
         this.response.redirect = this.url('problem_main');
     }

packages/migrate/scripts/hustoj.ts

@@ -168,6 +168,7 @@ export async function run({
             }
             if (!pidMap[pdoc.problem_id]) {
                 const files = {};
+                const markdown = [pdoc.description?.[0], pdoc.input?.[0], pdoc.output?.[0], pdoc.hint?.[0]].some((i) => i?.includes('[md]'));
                 let content = buildContent({
                     description: pdoc.description,
                     input: pdoc.input,
@@ -175,7 +176,7 @@ export async function run({
                     samples: [[pdoc.sample_input.trim(), pdoc.sample_output.trim()]],
                     hint: pdoc.hint,
                     source: pdoc.source,
-                }, 'html').replace(/<math xm<x>lns=/g, '<math xmlns=');
+                }, 'html').replace(/<math xm<x>lns=/g, '<math xmlns=').replace(/\[\/?md]/g, '');
                 const uploadFiles = content.matchAll(/(?:src|href)="\/upload\/([^"]+\/([^"]+))"/g);
                 for (const file of uploadFiles) {
                     try {
@@ -191,6 +192,7 @@ export async function run({
                     1, pdoc.source?.trim().length ? pdoc.source.split(' ').map((i) => i.trim()).filter((i) => i) : [],
                     { hidden: pdoc.defunct === 'Y' },
                 );
+                if (!markdown) await ProblemModel.edit(domainId, pid, { html: true });
                 pidMap[pdoc.problem_id] = pid;
                 await Promise.all(Object.keys(files).map((filename) => ProblemModel.addAdditionalFile(domainId, pid, filename, files[filename])));
                 if (Object.keys(files).length) report({ message: `move ${Object.keys(files).length} file for problem ${pid}` });

packages/ui-default/backendlib/template.ts

@@ -1,8 +1,7 @@
 import * as status from '@hydrooj/utils/lib/status';
 import { findFileSync } from '@hydrooj/utils/lib/utils';
 import {
-  avatar, buildContent, Context,
+  avatar, Context, fs, PERM, PRIV, STATUS, yaml,
-  fs, PERM, PRIV, STATUS, yaml,
 } from 'hydrooj';
 import jsesc from 'jsesc';
 import nunjucks from 'nunjucks';
@@ -86,14 +85,13 @@ class Nunjucks extends nunjucks.Environment {
       } catch {
         s = content;
       }
-      if (typeof s === 'object' && !(s instanceof Array)) {
+      if (typeof s === 'object') {
         const langs = Object.keys(s);
         const f = langs.filter((i) => i.startsWith(language));
         if (s[language]) s = s[language];
         else if (f.length) s = s[f[0]];
         else s = s[langs[0]];
       }
-      if (s instanceof Array) s = buildContent(s, html ? 'html' : 'markdown', (str) => str.translate(language));
       return ensureTag(html ? xss.process(s) : markdown.render(s));
     });
     this.addFilter('contentLang', (content) => {
@@ -103,9 +101,7 @@ class Nunjucks extends nunjucks.Environment {
       } catch {
         s = content;
       }
-      if (typeof s === 'object' && !(s instanceof Array)) {
+      if (typeof s === 'object') return Object.keys(s);
-        return Object.keys(s);
-      }
       return [];
     });
     this.addFilter('log', (self) => {

packages/vjudge/src/providers/codeforces.ts

@@ -218,23 +218,13 @@ export default class CodeforcesProvider extends BasicFetcher implements IBasicPr
             files,
             tag,
             difficulty: getDifficulty(tag),
-            content: buildContent([
+            content: buildContent({
-                {
+                description: description.replace(/tex-span/g, 'katex'),
-                    type: 'Text', sectionTitle: 'Description', subType: 'markdown', text: description.replace(/tex-span/g, 'katex'),
+                input: input?.replace(/tex-span/g, 'katex'),
-                },
+                output: output?.replace(/tex-span/g, 'katex'),
-                ...input ? [{
+                samplesRaw: inputs.map((ipt, idx) => [ipt, outputs[idx]]).join('\n'),
-                    type: 'Text', sectionTitle: 'Input', subType: 'markdown', text: input.replace(/tex-span/g, 'katex'),
+                hint: note?.replace(/tex-span/g, 'katex'),
-                }] : [],
+            }),
-                ...output ? [{
-                    type: 'Text', sectionTitle: 'Output', subType: 'markdown', text: output.replace(/tex-span/g, 'katex'),
-                }] : [],
-                ...inputs.length ? [{
-                    type: 'Plain', text: [...inputs, ...outputs].join('\n'),
-                }] : [] as any,
-                ...note ? [{
-                    type: 'Text', sectionTitle: 'Note', subType: 'markdown', text: note.replace(/tex-span/g, 'katex'),
-                }] : [],
-            ]),
         };
     }