diff --git a/packages/fps-importer/index.ts b/packages/fps-importer/index.ts index 3c5f8914c..ec5f65732 100644 --- a/packages/fps-importer/index.ts +++ b/packages/fps-importer/index.ts @@ -2,7 +2,7 @@ import decodeHTML from 'decode-html'; import xml2js from 'xml2js'; import { - _, AdmZip, BadRequestError, buildContent, ContentNode, Context, FileTooLargeError, fs, Handler, + _, AdmZip, BadRequestError, buildContent, Context, FileTooLargeError, fs, Handler, PERM, ProblemConfigFile, ProblemModel, ProblemType, SettingModel, SolutionModel, SystemModel, ValidationError, yaml, } from 'hydrooj'; @@ -19,48 +19,14 @@ class FpsProblemImportHandler extends Handler { if (!result?.fps) throw new BadRequestError('Selected file is not a valid FPS problemset.'); for (const p of result.fps.item) { const markdown = [p.description?.[0], p.input?.[0], p.output?.[0], p.hint?.[0]].some((i) => i?.includes('[md]')); - const content: ContentNode[] = []; - const stripBBCode = (input: string) => input.replace(/\[md]/g, '').replace(/\[\/md]/g, ''); - - if (p.description?.[0]) { - content.push({ - type: 'Text', - subType: markdown ? 'markdown' : 'html', - sectionTitle: this.translate('Description'), - text: stripBBCode(p.description[0]), - }); - } - if (p.input?.[0]) { - content.push({ - type: 'Text', - subType: markdown ? 'markdown' : 'html', - sectionTitle: this.translate('Input Format'), - text: stripBBCode(p.input[0]), - }); - } - if (p.output?.[0]) { - content.push({ - type: 'Text', - subType: markdown ? 'markdown' : 'html', - sectionTitle: this.translate('Output Format'), - text: stripBBCode(p.output[0]), - }); - } - if (p.sample_input?.length) { - content.push(...p.sample_input.map((input, i) => ({ - type: 'Sample', - sectionTitle: this.translate('Sample'), - payload: [input, p.sample_output[i]], - }))); - } - if (p.hint?.[0]) { - content.push({ - type: 'Text', - subType: markdown ? 'markdown' : 'html', - sectionTitle: this.translate('Hint'), - text: stripBBCode(p.hint[0]), - }); - } + const content = buildContent({ + description: p.description?.[0], + input: p.input?.[0], + output: p.output?.[0], + samples: p.sample_input?.map((input, i) => [input, p.sample_output[i]]), + hint: p.hint?.[0], + source: p.source?.join(' '), + }, 'html').replace(/lns=/g, ' i.trim()).flatMap((i) => i.split(' ')).filter((i) => i); - const pid = await ProblemModel.add(domainId, null, title, buildContent(content, 'html'), this.user._id, tags); + const pid = await ProblemModel.add(domainId, null, title, content, this.user._id, tags); const tasks: Promise[] = [ProblemModel.addTestdata(domainId, pid, 'config.yaml', Buffer.from(yaml.dump(config)))]; if (!markdown) tasks.push(ProblemModel.edit(domainId, pid, { html: true })); - const addTestdata = (node: any, id: string, ext: string) => { - if (!node && typeof node !== 'string') return; // Ignore file not exist - let c = node; - if (node.$?.name) { - id = node.$.name; - c = node._ || ''; - } - tasks.push(ProblemModel.addTestdata(domainId, pid, `${id}.${ext}`, Buffer.from(c))); + const addTestdata = (node: any, index: string, ext: string) => { + if (!node || !['object', 'string'].includes(typeof node)) return; // Ignore file not exist + const id = node.$?.name || `${index}`; + // `filename` attribute introduced by winterant/OnlineJudge + // PLEASE, respect the spec + const filename = node.$?.filename || `${id}.${ext}`; + const c = node._ || node; + tasks.push(ProblemModel.addTestdata(domainId, pid, filename, Buffer.from(c))); }; if (p.test_output) { for (let i = 0; i < p.test_input.length; i++) { @@ -90,6 +56,7 @@ class FpsProblemImportHandler extends Handler { addTestdata(p.test_output[i], `${i + 1}`, 'out'); } } else if (p.test_input) { + // Some version of hustoj exports only test_input section for (let i = 0; i < p.test_input.length / 2; i++) { addTestdata(p.test_input[2 * i], `${i + 1}`, 'in'); addTestdata(p.test_input[2 * i + 1], `${i + 1}`, 'out'); diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 60c04d3a5..9ffda1dda 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -221,26 +221,7 @@ export interface ProblemConfig { hackable?: boolean; } -export interface PlainContentNode { - type: 'Plain', - subType: 'html' | 'markdown', - text: string, -} -export interface TextContentNode { - type: 'Text', - subType: 'html' | 'markdown', - sectionTitle: string, - text: string, -} -export interface SampleContentNode { - type: 'Sample', - text: string, - sectionTitle: string, - payload: [string, string], -} -// TODO drop contentNode support -export type ContentNode = PlainContentNode | TextContentNode | SampleContentNode; -export type Content = string | ContentNode[] | Record; +export type Content = string | Record; export interface Document { _id: ObjectId; diff --git a/packages/hydrooj/src/lib/content.ts b/packages/hydrooj/src/lib/content.ts index 89932442e..058dd7b61 100644 --- a/packages/hydrooj/src/lib/content.ts +++ b/packages/hydrooj/src/lib/content.ts @@ -1,17 +1,18 @@ -import { ContentNode } from '../interface'; - export interface ProblemSource { description?: string, input?: string, output?: string, samples?: [string, string][], + samplesRaw?: string, hint?: string, source?: string, } -export function buildContent(source: ProblemSource | ContentNode[], type: 'markdown' | 'html' = 'markdown', translate?: Function) { +export function buildContent(source: ProblemSource, type: 'markdown' | 'html' = 'markdown', translate?: Function) { const _ = translate || ((s: string) => s); let cnt = 0; + // Keep it for backward compatibility, but don't add to typings + // required by upgrade 90_91 if (source instanceof Array) { return type === 'html' ? source.map((node) => [ @@ -54,6 +55,7 @@ export function buildContent(source: ProblemSource | ContentNode[], type: 'markd sample[1], '', ].join('')), + ...source.samplesRaw ? [source.samplesRaw] : [], ...source.hint ? [`

${_('Hint')}

`, source.hint] : [], ...source.source ? [`

${_('Source')}

`, source.source] : [], ].join('\n') @@ -69,6 +71,7 @@ export function buildContent(source: ProblemSource | ContentNode[], type: 'markd sample[1], '```', ].join('\n')), + ...source.samplesRaw ? [source.samplesRaw] : [], ...source.hint ? [`## ${_('Hint')}`, source.hint] : [], ...source.source ? [`## ${_('Source')}`, source.source] : [], ].join('\n'); diff --git a/packages/hydrooj/src/model/problem.ts b/packages/hydrooj/src/model/problem.ts index dd5aeca36..7c5420dd1 100644 --- a/packages/hydrooj/src/model/problem.ts +++ b/packages/hydrooj/src/model/problem.ts @@ -45,12 +45,7 @@ function findOverrideContent(dir: string, base: string) { if (!files.length) return null; for (const file of files) { const lang = file.slice(8, -3); - let content: string | any[] = fs.readFileSync(path.join(dir, file), 'utf8'); - try { - content = JSON.parse(content); - if (!(content instanceof Array)) content = JSON.stringify(content); - } catch (e) { } - languages[lang] = content; + languages[lang] = fs.readFileSync(path.join(dir, file), 'utf8'); } return JSON.stringify(languages); } @@ -548,12 +543,12 @@ export class ProblemModel { const docId = overridePid ? (await ProblemModel.edit(domainId, overridePid, { title: pdoc.title.trim(), - content: overrideContent || pdoc.content || 'No content', + content: overrideContent || pdoc.content.toString() || 'No content', tag, difficulty: pdoc.difficulty, })).docId : await ProblemModel.add( - domainId, pid, pdoc.title.trim(), overrideContent || pdoc.content || 'No content', + domainId, pid, pdoc.title.trim(), overrideContent || pdoc.content.toString() || 'No content', operator || pdoc.owner, tag, { hidden: pdoc.hidden, difficulty: pdoc.difficulty }, ); // TODO delete unused file when updating pdoc diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index 303dfef41..1bcb1f90d 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -1,3 +1,4 @@ +/* eslint-disable consistent-return */ /* eslint-disable @typescript-eslint/no-unused-vars */ /* eslint-disable no-await-in-loop */ /* eslint-disable @typescript-eslint/naming-convention */ @@ -51,25 +52,7 @@ export const coreScripts: MigrationScript[] = [ return true; }, // Init - ...new Array(26).fill(unsupportedUpgrade), - null, - async function _28_29() { - return await iterateAllProblem(['content', 'html'], async (pdoc) => { - try { - const parsed = JSON.parse(pdoc.content); - if (parsed instanceof Array) { - await problem.edit(pdoc.domainId, pdoc.docId, { content: buildContent(parsed, pdoc.html ? 'html' : 'markdown') }); - return; - } - const res = {}; - for (const key in parsed) { - if (typeof parsed[key] === 'string') res[key] = parsed[key]; - else res[key] = buildContent(parsed[key]); - } - await problem.edit(pdoc.domainId, pdoc.docId, { content: JSON.stringify(res) }); - } catch { } - }); - }, + ...new Array(28).fill(unsupportedUpgrade), async function _29_30() { return await iterateAllDomain((ddoc) => RecordModel.coll.updateMany({ domainId: ddoc._id }, { $set: { pdomain: ddoc._id } })); }, @@ -685,4 +668,38 @@ export const coreScripts: MigrationScript[] = [ } }); }, + async function _90_91() { + await document.collStatus.updateMany({ docType: document.TYPE_PROBLEM }, { $unset: { nSubmit: '', nAccept: '' } }); + const psdocs = await document.coll.find({ docType: document.TYPE_PROBLEM_SOLUTION, vote: { $ne: 0 } }) + .project({ docId: 1, domainId: 1 }).toArray(); + for (const psdoc of psdocs) { + const filter = { docType: document.TYPE_PROBLEM_SOLUTION, domainId: psdoc.domainId, docId: psdoc.docId }; + const [upvote, downvote] = await Promise.all([ + document.collStatus.countDocuments({ ...filter, vote: 1 }), + document.collStatus.countDocuments({ ...filter, vote: -1 }), + ]); + if (upvote - downvote !== psdoc.vote) { + await document.set(psdoc.domainId, document.TYPE_PROBLEM_SOLUTION, psdoc.docId, { vote: upvote - downvote }); + } + } + await iterateAllProblem(['domainId', 'docId', 'tag'], async (pdoc) => { + if (pdoc.tag?.some((i) => typeof i !== 'string')) { + return { tag: pdoc.tag.filter((i) => i).map((i) => i.toString()) }; + } + }); + return await iterateAllProblem(['domainId', 'docId', 'content', 'html'], async (pdoc) => { + try { + const parsed = JSON.parse(pdoc.content); + if (parsed instanceof Array) { + return { content: buildContent(parsed as any, pdoc.html ? 'html' : 'markdown') }; + } + const res = {}; + for (const key in parsed) { + if (typeof parsed[key] === 'string') res[key] = parsed[key]; + else res[key] = buildContent(parsed[key]); + } + return { content: JSON.stringify(res) }; + } catch { } + }); + }, ]; diff --git a/packages/import-qduoj/index.ts b/packages/import-qduoj/index.ts index e27b1f19c..970457ac1 100644 --- a/packages/import-qduoj/index.ts +++ b/packages/import-qduoj/index.ts @@ -2,13 +2,42 @@ import os from 'os'; import path from 'path'; import { - AdmZip, buildContent, ContentNode, Context, fs, Handler, PERM, - ProblemConfigFile, ProblemModel, ValidationError, yaml, + AdmZip, buildContent, Context, FileTooLargeError, fs, Handler, PERM, + ProblemConfigFile, ProblemModel, Schema, ValidationError, yaml, } from 'hydrooj'; const tmpdir = path.join(os.tmpdir(), 'hydro', 'import-qduoj'); fs.ensureDirSync(tmpdir); +const StringValue = Schema.object({ + value: Schema.string(), +}); +const ProblemSchema = Schema.object({ + tags: Schema.array(Schema.string()), + title: Schema.string().required(), + description: StringValue, + input_description: StringValue, + output_description: StringValue, + samples: Schema.array(Schema.object({ + input: Schema.string(), + output: Schema.string(), + })), + hint: StringValue, + source: StringValue, + display_id: Schema.string(), + time_limit: Schema.union([Schema.number(), Schema.string()]).required(), + memory_limit: Schema.union([Schema.number(), Schema.string()]).required(), + spj: Schema.object({ + language: Schema.string().required(), + code: Schema.string().required(), + }), + test_case_score: Schema.array(Schema.object({ + input_name: Schema.string().required(), + output_name: Schema.string().required(), + score: Schema.number().required(), + })), +}); + class ImportQduojHandler extends Handler { async fromFile(domainId: string, zipfile: string) { let zip: AdmZip; @@ -27,55 +56,15 @@ class ImportQduojHandler extends Handler { for (const { name: folder } of folders.filter((i) => i.isDirectory())) { if (!fs.existsSync(path.join(tmp, folder, 'problem.json'))) continue; const buf = await fs.readFile(path.join(tmp, folder, 'problem.json')); - const pdoc = JSON.parse(buf.toString()); - const content: ContentNode[] = []; - if (pdoc.description?.value) { - content.push({ - type: 'Text', - subType: 'html', - sectionTitle: this.translate('Description'), - text: pdoc.description.value, - }); - } - if (pdoc.input_description?.value) { - content.push({ - type: 'Text', - subType: 'html', - sectionTitle: this.translate('Input Format'), - text: pdoc.input_description.value, - }); - } - if (pdoc.output_description?.value) { - content.push({ - type: 'Text', - subType: 'html', - sectionTitle: this.translate('Output Format'), - text: pdoc.output_description.value, - }); - } - if (pdoc.samples?.length) { - content.push(...pdoc.samples.map((sample) => ({ - type: 'Sample', - sectionTitle: this.translate('Sample'), - payload: [sample.input, sample.output], - }))); - } - if (pdoc.hint?.value) { - content.push({ - type: 'Text', - subType: 'html', - sectionTitle: this.translate('Hint'), - text: pdoc.hint.value, - }); - } - if (pdoc.source?.value) { - content.push({ - type: 'Text', - subType: 'html', - sectionTitle: this.translate('Source'), - text: pdoc.source.value, - }); - } + const pdoc = ProblemSchema(JSON.parse(buf.toString())); + const content = buildContent({ + description: pdoc.description?.value, + input: pdoc.input_description?.value, + output: pdoc.output_description?.value, + samples: pdoc.samples.map((sample) => [sample.input, sample.output]), + hint: pdoc.hint?.value, + source: pdoc.source?.value, + }, 'html'); if (+pdoc.display_id) pdoc.display_id = `P${pdoc.display_id}`; const isValidPid = async (id: string) => { if (!(/^[A-Za-z]+[0-9A-Za-z]*$/.test(id))) return false; @@ -84,7 +73,7 @@ class ImportQduojHandler extends Handler { }; if (!await isValidPid(pdoc.display_id)) pdoc.display_id = null; const pid = await ProblemModel.add( - domainId, pdoc.display_id, pdoc.title, buildContent(content, 'html'), + domainId, pdoc.display_id, pdoc.title, content, this.user._id, pdoc.tags || [], ); const config: ProblemConfigFile = { @@ -141,7 +130,7 @@ class ImportQduojHandler extends Handler { async post({ domainId }) { if (!this.request.files.file) throw new ValidationError('file'); const stat = await fs.stat(this.request.files.file.filepath); - if (stat.size > 128 * 1024 * 1024) throw new ValidationError('file', 'File too large'); + if (stat.size > 256 * 1024 * 1024) throw new FileTooLargeError('256m'); await this.fromFile(domainId, this.request.files.file.filepath); this.response.redirect = this.url('problem_main'); } diff --git a/packages/migrate/scripts/hustoj.ts b/packages/migrate/scripts/hustoj.ts index 2b3550274..a2f0716a2 100644 --- a/packages/migrate/scripts/hustoj.ts +++ b/packages/migrate/scripts/hustoj.ts @@ -168,6 +168,7 @@ export async function run({ } if (!pidMap[pdoc.problem_id]) { const files = {}; + const markdown = [pdoc.description?.[0], pdoc.input?.[0], pdoc.output?.[0], pdoc.hint?.[0]].some((i) => i?.includes('[md]')); let content = buildContent({ description: pdoc.description, input: pdoc.input, @@ -175,7 +176,7 @@ export async function run({ samples: [[pdoc.sample_input.trim(), pdoc.sample_output.trim()]], hint: pdoc.hint, source: pdoc.source, - }, 'html').replace(/lns=/g, 'lns=/g, ' i.trim()).filter((i) => i) : [], { hidden: pdoc.defunct === 'Y' }, ); + if (!markdown) await ProblemModel.edit(domainId, pid, { html: true }); pidMap[pdoc.problem_id] = pid; await Promise.all(Object.keys(files).map((filename) => ProblemModel.addAdditionalFile(domainId, pid, filename, files[filename]))); if (Object.keys(files).length) report({ message: `move ${Object.keys(files).length} file for problem ${pid}` }); diff --git a/packages/ui-default/backendlib/template.ts b/packages/ui-default/backendlib/template.ts index aea026729..7758a60dd 100644 --- a/packages/ui-default/backendlib/template.ts +++ b/packages/ui-default/backendlib/template.ts @@ -1,8 +1,7 @@ import * as status from '@hydrooj/utils/lib/status'; import { findFileSync } from '@hydrooj/utils/lib/utils'; import { - avatar, buildContent, Context, - fs, PERM, PRIV, STATUS, yaml, + avatar, Context, fs, PERM, PRIV, STATUS, yaml, } from 'hydrooj'; import jsesc from 'jsesc'; import nunjucks from 'nunjucks'; @@ -86,14 +85,13 @@ class Nunjucks extends nunjucks.Environment { } catch { s = content; } - if (typeof s === 'object' && !(s instanceof Array)) { + if (typeof s === 'object') { const langs = Object.keys(s); const f = langs.filter((i) => i.startsWith(language)); if (s[language]) s = s[language]; else if (f.length) s = s[f[0]]; else s = s[langs[0]]; } - if (s instanceof Array) s = buildContent(s, html ? 'html' : 'markdown', (str) => str.translate(language)); return ensureTag(html ? xss.process(s) : markdown.render(s)); }); this.addFilter('contentLang', (content) => { @@ -103,9 +101,7 @@ class Nunjucks extends nunjucks.Environment { } catch { s = content; } - if (typeof s === 'object' && !(s instanceof Array)) { - return Object.keys(s); - } + if (typeof s === 'object') return Object.keys(s); return []; }); this.addFilter('log', (self) => { diff --git a/packages/vjudge/src/providers/codeforces.ts b/packages/vjudge/src/providers/codeforces.ts index 941da8db0..0e48785c8 100644 --- a/packages/vjudge/src/providers/codeforces.ts +++ b/packages/vjudge/src/providers/codeforces.ts @@ -218,23 +218,13 @@ export default class CodeforcesProvider extends BasicFetcher implements IBasicPr files, tag, difficulty: getDifficulty(tag), - content: buildContent([ - { - type: 'Text', sectionTitle: 'Description', subType: 'markdown', text: description.replace(/tex-span/g, 'katex'), - }, - ...input ? [{ - type: 'Text', sectionTitle: 'Input', subType: 'markdown', text: input.replace(/tex-span/g, 'katex'), - }] : [], - ...output ? [{ - type: 'Text', sectionTitle: 'Output', subType: 'markdown', text: output.replace(/tex-span/g, 'katex'), - }] : [], - ...inputs.length ? [{ - type: 'Plain', text: [...inputs, ...outputs].join('\n'), - }] : [] as any, - ...note ? [{ - type: 'Text', sectionTitle: 'Note', subType: 'markdown', text: note.replace(/tex-span/g, 'katex'), - }] : [], - ]), + content: buildContent({ + description: description.replace(/tex-span/g, 'katex'), + input: input?.replace(/tex-span/g, 'katex'), + output: output?.replace(/tex-span/g, 'katex'), + samplesRaw: inputs.map((ipt, idx) => [ipt, outputs[idx]]).join('\n'), + hint: note?.replace(/tex-span/g, 'katex'), + }), }; }