From e1a1e35f940c862cec57dfcd6af815d5b1b8c395 Mon Sep 17 00:00:00 2001 From: undefined Date: Wed, 2 Mar 2022 17:54:05 +0800 Subject: [PATCH] core&ui: drop Import.SYZOJ support --- packages/hydrooj/locales/ko.yaml | 1 - packages/hydrooj/locales/zh.yaml | 4 - packages/hydrooj/package.json | 14 +- packages/hydrooj/src/handler/import.ts | 327 +----------------- packages/hydrooj/src/lib/ui.ts | 1 - .../components/problem/create.page.js | 2 +- packages/ui-default/locales/ko.yaml | 1 - packages/ui-default/locales/zh.yaml | 1 - packages/ui-default/package.json | 2 +- 9 files changed, 21 insertions(+), 332 deletions(-) diff --git a/packages/hydrooj/locales/ko.yaml b/packages/hydrooj/locales/ko.yaml index dc10205a6..714bed81b 100644 --- a/packages/hydrooj/locales/ko.yaml +++ b/packages/hydrooj/locales/ko.yaml @@ -342,7 +342,6 @@ If enabled, source code will be emailed to you after the submission is accepted. If left blank, the built-in template of the corresponding language will be used.: 공백으로 둔다면, 기본 언어로 적용합니다. Images: 사진 Import from: '다음에서 가져오기: ' -Import problem from syzoj.: syzoj에서 문제 가져오기 Import Problem: 문제 가져오기 Import: 가져오기 In 1 day: 하루 안에 diff --git a/packages/hydrooj/locales/zh.yaml b/packages/hydrooj/locales/zh.yaml index b3d8712ca..3abcbedb4 100644 --- a/packages/hydrooj/locales/zh.yaml +++ b/packages/hydrooj/locales/zh.yaml @@ -377,10 +377,7 @@ If enabled, source code will be emailed to you after the submission is accepted. If left blank, the built-in template of the corresponding language will be used.: 若留空，则将使用对应语言的内置代码模板。 Images: 图片 Import From Hydro: 从 Hydro 导入 -Import From SYZOJ: 从 SYZOJ 导入 -Import From SYZOJ.: 从 SYZOJ 导入 Import from: 导入自 -Import problem from syzoj.: 从运行 syzoj 系统的 OJ 导入题目。 Import Problem: 导入题目 Import User: 导入用户 Import: 导入 @@ -585,7 +582,6 @@ problem_create: 创建题目 problem_detail: 题目详情 problem_edit: 编辑题目 problem_files: 题目文件 -problem_import_syzoj: 从 SYZOJ 导入题目 problem_import: 导入题目 problem_main: 题库 problem_solution: 题解 diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index 61304b0de..c15327254 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "3.5.7", + "version": "3.6.0", "bin": "bin/hydrooj.js", "main": "src/loader", "module": "src/loader", @@ -12,7 +12,7 @@ }, "preferUnplugged": true, "dependencies": { - "@graphql-tools/schema": "^8.3.1", + "@graphql-tools/schema": "^8.3.2", "@hydrooj/utils": "workspace:*", "adm-zip": "0.5.5", "ajv": "^8.10.0", @@ -23,14 +23,14 @@ "emoji-regex": "^10.0.0", "emojis-list": "2.1.0", "esbuild": "0.14.3", - "fs-extra": "^10.0.0", + "fs-extra": "^10.0.1", "graphql": "^16.3.0", "graphql-scalars": "1.14.1", "js-yaml": "^4.1.0", "koa": "^2.13.4", "koa-body": "^4.2.0", "koa-compress": "^5.1.0", - "koa-proxies": "^0.12.1", + "koa-proxies": "^0.12.2", "koa-router": "^10.1.1", "koa-static-cache": "^5.1.4", "lodash": "^4.17.21", @@ -39,7 +39,7 @@ "minio": "7.0.25", "moment-timezone": "^0.5.34", "mongodb": "^3.7.3", - "nanoid": "^3.3.0", + "nanoid": "^3.3.1", "nodemailer": "^6.7.2", "notp": "^2.0.3", "p-queue": "^7.2.0", @@ -61,7 +61,7 @@ "@types/koa-compress": "^4.0.3", "@types/koa-router": "^7.4.4", "@types/koa-static-cache": "^5.1.1", - "@types/lodash": "^4.14.178", + "@types/lodash": "^4.14.179", "@types/lru-cache": "^5.1.1", "@types/mime-types": "^2.1.1", "@types/minio": "^7.0.12", @@ -74,7 +74,7 @@ "@types/source-map-support": "^0.5.4", "@types/superagent": "^4.1.15", "moment": "^2.29.1", - "pm2": "^5.1.2" + "pm2": "^5.2.0" }, "peerDependencies": { "pm2": "*" diff --git a/packages/hydrooj/src/handler/import.ts b/packages/hydrooj/src/handler/import.ts index 0b0136592..b726fc93f 100644 --- a/packages/hydrooj/src/handler/import.ts +++ b/packages/hydrooj/src/handler/import.ts @@ -1,326 +1,25 @@ /* eslint-disable no-await-in-loop */ -import assert from 'assert'; import os from 'os'; import path from 'path'; -import { PassThrough } from 'stream'; import AdmZip from 'adm-zip'; import fs from 'fs-extra'; import yaml from 'js-yaml'; -import { filter } from 'lodash'; -import superagent from 'superagent'; -import { noop } from '@hydrooj/utils/lib/utils'; -import { RemoteOnlineJudgeError, ValidationError } from '../error'; -import type { ContentNode, ProblemConfigFile, SubtaskConfig } from '../interface'; -import { buildContent } from '../lib/content'; -import download from '../lib/download'; +import { ValidationError } from '../error'; import { ProblemAdd } from '../lib/ui'; -import { isPid, parsePid } from '../lib/validator'; -import { Logger } from '../logger'; import { PERM, PRIV } from '../model/builtin'; import problem, { ProblemDoc } from '../model/problem'; -import TaskModel from '../model/task'; -import { - Handler, post, Route, Types, -} from '../service/server'; +import { Handler, Route } from '../service/server'; -const RE_SYZOJ = /(https?):\/\/([^/]+)\/(problem|p)\/([0-9]+)\/?/i; -const logger = new Logger('import.syzoj'); - -const ScoreTypeMap = { - GroupMin: 'min', - Sum: 'sum', - GroupMul: 'max', -}; -const LanguageMap = { - cpp: 'cc', -}; - -async function syzojSync(info) { - const { - protocol, host, domainId, docId, - } = info; - const body = JSON.parse(info.body); - const pid = body.meta.id; - const judge = body.judgeInfo; - const rename = {}; - if (judge) { - const config: ProblemConfigFile = { - time: `${judge.timeLimit}ms`, - memory: `${judge.memoryLimit}m`, - }; - if (judge.extraSourceFiles) { - const files = []; - for (const key in judge.extraSourceFiles) { - for (const file in judge.extraSourceFiles[key]) { - files.push(file); - } - } - config.user_extra_files = files; - } - if (judge.checker?.type === 'custom') { - config.checker_type = judge.checker.interface; - if (LanguageMap[judge.checker.language]) { - rename[judge.checker.filename] = `chk.${LanguageMap[judge.checker.language]}`; - config.checker = `chk.${LanguageMap[judge.checker.language]}`; - } else config.checker = judge.checker.filename; - } - if (judge.subtasks?.length) { - config.subtasks = []; - for (const subtask of judge.subtasks) { - const current: SubtaskConfig = { - score: subtask.points, - type: ScoreTypeMap[subtask.scoringType], - cases: subtask.testcases.map((i) => ({ input: i.inputFile, output: i.outputFile })), - }; - if (subtask.dependencies) current.if = subtask.dependencies; - config.subtasks.push(current); - } - } - await problem.addTestdata(domainId, docId, 'config.yaml', Buffer.from(yaml.dump(config))); - } - const r = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/downloadProblemFiles`) - .send({ - problemId: pid, - type: 'TestData', - filenameList: body.testData.map((node) => node.filename), - }); - if (r.body.error) throw new Error(r.body.error.message || r.body.error); - for (const f of r.body.downloadInfo) { - const p = new PassThrough(); - superagent.get(f.downloadUrl).pipe(p); - // eslint-disable-next-line no-await-in-loop - await problem.addTestdata(domainId, docId, rename[f.filename] || f.filename, p); - } - const a = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/downloadProblemFiles`) - .send({ - problemId: pid, - type: 'AdditionalFile', - filenameList: body.additionalFiles.map((node) => node.filename), - }); - if (a.body.error) throw new Error(a.body.error.message || a.body.error); - for (const f of a.body.downloadInfo) { - const p = new PassThrough(); - superagent.get(f.downloadUrl).pipe(p); - // eslint-disable-next-line no-await-in-loop - await problem.addAdditionalFile(domainId, docId, f.filename, p); - } -} -TaskModel.Worker.addHandler('import.syzoj', syzojSync); - -class ProblemImportSYZOJHandler extends Handler { - async get() { - this.response.template = 'problem_import_syzoj.html'; - } - - async v2(domainId: string, target: string, hidden = false, url: string) { - const res = await superagent.get(`${url}export`); - assert(res.status === 200, new RemoteOnlineJudgeError('Cannot connect to target server')); - assert(res.body.success, new RemoteOnlineJudgeError((res.body.error || {}).message)); - const p = res.body.obj; - const content: ContentNode[] = []; - if (p.description) { - content.push({ - type: 'Text', - subType: 'markdown', - sectionTitle: this.translate('Problem Description'), - text: p.description, - }); - } - if (p.input_format) { - content.push({ - type: 'Text', - subType: 'markdown', - sectionTitle: this.translate('Input Format'), - text: p.input_format, - }); - } - if (p.output_format) { - content.push({ - type: 'Text', - subType: 'markdown', - sectionTitle: this.translate('Output Format'), - text: p.output_format, - }); - } - if (p.example) { - content.push({ - type: 'Text', - subType: 'markdown', - sectionTitle: this.translate('Sample'), - text: p.example, - }); - } - if (p.hint) { - content.push({ - type: 'Text', - subType: 'markdown', - sectionTitle: this.translate('Hint'), - text: p.hint, - }); - } - if (p.limit_and_hint) { - content.push({ - type: 'Text', - subType: 'markdown', - sectionTitle: this.translate('Limit And Hint'), - text: p.limit_and_hint, - }); - } - const c = buildContent(content, 'markdown'); - const docId = await problem.add( - domainId, target, p.title, c, this.user._id, p.tags || [], hidden, - ); - const r = download(`${url}testdata/download`); - const file = path.resolve(os.tmpdir(), 'hydro', `import_${domainId}_${docId}.zip`); - const w = fs.createWriteStream(file); - try { - await new Promise((resolve, reject) => { - w.on('finish', resolve); - w.on('error', reject); - r.pipe(w); - }); - const zip = new AdmZip(file); - const entries = zip.getEntries(); - for (const entry of entries) { - // eslint-disable-next-line no-await-in-loop - await problem.addTestdata(domainId, docId, entry.entryName, entry.getData()); - } - const filename = p.file_io_input_name ? p.file_io_input_name.split('.')[0] : null; - const config = { - time: `${p.time_limit}ms`, - memory: `${p.memory_limit}m`, - filename, - type: p.type === 'traditional' ? 'default' : p.type, - }; - await problem.addTestdata(domainId, docId, 'config.yaml', Buffer.from(yaml.dump(config))); - } finally { - fs.unlinkSync(file); - } - if (p.have_additional_file) { - const r1 = download(`${url}download/additional_file`); - const file1 = path.resolve(os.tmpdir(), 'hydro', `import_${domainId}_${docId}_a.zip`); - const w1 = fs.createWriteStream(file1); - try { - await new Promise((resolve, reject) => { - w1.on('finish', resolve); - w1.on('error', reject); - r1.pipe(w1); - }); - const zip = new AdmZip(file1); - const entries = zip.getEntries(); - for (const entry of entries) { - await problem.addAdditionalFile(domainId, docId, entry.entryName.replace('/', '_'), entry.getData()); - } - } finally { - fs.unlinkSync(file1); - } - } - return docId; - } - - async v3( - domainId: string, target: string, hidden: boolean, - protocol: string, host: string, pid: string | number, - wait = false, - ) { - let tagsOfLocale = this.user.viewLang || this.session.viewLang; - if (tagsOfLocale === 'en') tagsOfLocale = 'en_US'; - else tagsOfLocale = 'zh_CN'; - const result = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/getProblem`) - .send({ - displayId: +pid, - localizedContentsOfAllLocales: true, - tagsOfLocale, - samples: true, - judgeInfo: true, - testData: true, - additionalFiles: true, - }); - const content = {}; - for (const c of result.body.localizedContentsOfAllLocales) { - const sections = c.contentSections; - for (const section of sections) { - section.subType = 'markdown'; - if (section.type === 'Sample') { - section.payload = [ - result.body.samples[section.sampleId].inputData, - result.body.samples[section.sampleId].outputData, - ]; - delete section.sampleId; - } - } - let locale = c.locale; - if (locale === 'en_US') locale = 'en'; - else if (locale === 'zh_CN') locale = 'zh'; - content[locale] = sections; - } - const tags = result.body.tagsOfLocale.map((node) => node.name); - const title = [ - ...filter( - result.body.localizedContentsOfAllLocales, - (node) => node.locale === (this.user.viewLang || this.session.viewLang), - ), - ...result.body.localizedContentsOfAllLocales, - ][0].title; - const docId = await problem.add( - domainId, target, title, JSON.stringify(content), this.user._id, tags || [], hidden, - ); - const payload = { - protocol, host, pid, domainId, docId, body: JSON.stringify(result.body), - }; - if (wait) await syzojSync(payload); - else await TaskModel.add({ ...payload, type: 'schedule', subType: 'import.syzoj' }); - return docId; - } - - @post('url', Types.Content, true) - @post('pid', Types.Name, true, isPid, parsePid) - @post('hidden', Types.Boolean) - @post('prefix', Types.Name, true) - @post('start', Types.UnsignedInt, true) - @post('end', Types.UnsignedInt, true) - async post( - domainId: string, url: string, targetPid: string, hidden = false, - prefix: string, start: number, end: number, - ) { - if (/^(.+)\/(\d+)\.\.(\d+)$/.test(url)) { - const res = /^(.+)\/(\d+)\.\.(\d+)$/.exec(url)!; - prefix = res[1]; - start = +res[2]; - end = +res[3]; - if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start > end) { - throw new ValidationError('end'); - } - } - if (prefix) { - let version = 2; - if (!prefix.endsWith('/')) prefix += '/'; - if (prefix.endsWith('/p/')) version = 3; - else prefix = `${prefix.split('/problem/')[0]}/problem/`; - const base = `${prefix}${start}/`; - assert(base.match(RE_SYZOJ), new ValidationError('prefix')); - const [, protocol, host] = RE_SYZOJ.exec(base); - (async () => { - for (let i = start; i <= end; i++) { - // eslint-disable-next-line no-await-in-loop - if (version === 3) await this.v3(domainId, undefined, hidden, protocol, host, i, true).catch(noop); - // eslint-disable-next-line no-await-in-loop - else await this.v2(domainId, undefined, hidden, `${prefix + i}/`).catch(noop); - logger.info('%s %d-%d-%d', prefix, start, i, end); - } - })().catch(logger.error); - this.response.redirect = this.url('problem_main'); - } else { - assert(url.match(RE_SYZOJ), new ValidationError('url')); - if (!url.endsWith('/')) url += '/'; - const [, protocol, host, n, pid] = RE_SYZOJ.exec(url); - const docId = n === 'p' - ? await this.v3(domainId, targetPid, hidden, protocol, host, pid, false) - : await this.v2(domainId, targetPid, hidden, url); - this.response.body = { pid: targetPid || docId }; - this.response.redirect = this.url('problem_detail', { pid: targetPid || docId }); - } +function findOverrideContent(dir: string) { + let files = fs.readdirSync(dir); + if (files.includes('problem.md')) return fs.readFileSync(path.join(dir, 'problem.md'), 'utf8'); + const languages = {}; + files = files.filter((i) => /^problem_[a-zA-Z_]+\.md$/.test(i)); + for (const file of files) { + const lang = file.slice(8, -3); + languages[lang] = fs.readFileSync(path.join(dir, file), 'utf8'); } + return JSON.stringify(languages); } class ProblemImportHydroHandler extends Handler { @@ -348,8 +47,7 @@ class ProblemImportHydroHandler extends Handler { const pdoc: ProblemDoc = yaml.load(content) as any; const current = await problem.get(domainId, pdoc.pid); const pid = current ? undefined : pdoc.pid; - const overrideContent = fs.existsSync(path.join(tmpdir, i, 'problem.md')) - ? fs.readFileSync(path.join(tmpdir, i, 'problem.md'), 'utf8') : ''; + const overrideContent = findOverrideContent(path.join(tmpdir, i)); const docId = await problem.add( domainId, pid, pdoc.title, overrideContent || pdoc.content, keepUser ? pdoc.owner : this.user._id, pdoc.tag, pdoc.hidden, @@ -388,7 +86,6 @@ class ProblemImportHydroHandler extends Handler { export async function apply() { ProblemAdd('problem_import_hydro', {}, 'copy', 'Import From Hydro'); - Route('problem_import_syzoj', '/problem/import/syzoj', ProblemImportSYZOJHandler, PERM.PERM_CREATE_PROBLEM); Route('problem_import_hydro', '/problem/import/hydro', ProblemImportHydroHandler, PERM.PERM_CREATE_PROBLEM); } diff --git a/packages/hydrooj/src/lib/ui.ts b/packages/hydrooj/src/lib/ui.ts index 329bb1cf0..67d2cc776 100644 --- a/packages/hydrooj/src/lib/ui.ts +++ b/packages/hydrooj/src/lib/ui.ts @@ -77,7 +77,6 @@ Nav('ranking', {}, 'ranking', PERM.PERM_VIEW_RANKING); Nav('domain_dashboard', {}, 'domain', PERM.PERM_EDIT_DOMAIN); Nav('manage_dashboard', {}, 'manage', PRIV.PRIV_EDIT_SYSTEM); ProblemAdd('problem_create'); -ProblemAdd('problem_import_syzoj', {}, 'copy', 'Import From SYZOJ'); global.Hydro.ui.Nav = Nav; global.Hydro.ui.ProblemAdd = ProblemAdd; diff --git a/packages/ui-default/components/problem/create.page.js b/packages/ui-default/components/problem/create.page.js index 3363ec4ce..fd431b863 100644 --- a/packages/ui-default/components/problem/create.page.js +++ b/packages/ui-default/components/problem/create.page.js @@ -3,7 +3,7 @@ import { InfoDialog } from 'vj/components/dialog'; import tpl from 'vj/utils/tpl'; import i18n from 'vj/utils/i18n'; -export default new NamedPage(['problem_create', 'problem_edit', 'problem_import_syzoj'], () => { +export default new NamedPage(['problem_create', 'problem_edit'], () => { $('input[name="pid"]').on('blur', () => { if (/^[0-9]+$/.test($('input[name="pid"]').val())) { new InfoDialog({ diff --git a/packages/ui-default/locales/ko.yaml b/packages/ui-default/locales/ko.yaml index cf7074db3..f352f7d81 100644 --- a/packages/ui-default/locales/ko.yaml +++ b/packages/ui-default/locales/ko.yaml @@ -347,7 +347,6 @@ If enabled, source code will be emailed to you after the submission is accepted. If left blank, the built-in template of the corresponding language will be used.: 공백으로 둔다면, 기본 언어로 적용합니다. Images: 사진 Import from: '다음에서 가져오기: ' -Import problem from syzoj.: syzoj에서 문제 가져오기 Import Problem: 문제 가져오기 Import: 가져오기 In 1 day: 하루 안에 diff --git a/packages/ui-default/locales/zh.yaml b/packages/ui-default/locales/zh.yaml index 88e16daae..c4d15aeea 100644 --- a/packages/ui-default/locales/zh.yaml +++ b/packages/ui-default/locales/zh.yaml @@ -381,7 +381,6 @@ If left blank, the built-in template of the corresponding language will be used. Images With Size: 指定大小的图片 Images: 图片 Import from: 导入自 -Import problem from syzoj.: 从运行 syzoj 系统的 OJ 导入题目。 Import Problem: 导入题目 Import: 导入 In 1 day: 一天后 diff --git a/packages/ui-default/package.json b/packages/ui-default/package.json index 2b4ee808f..4bfe9f1fa 100644 --- a/packages/ui-default/package.json +++ b/packages/ui-default/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/ui-default", - "version": "4.32.12", + "version": "4.32.13", "author": "undefined ", "license": "AGPL-3.0", "main": "hydro.js",