core&ui: drop Import.SYZOJ support

packages/hydrooj/locales/ko.yaml

@@ -342,7 +342,6 @@ If enabled, source code will be emailed to you after the submission is accepted.
 If left blank, the built-in template of the corresponding language will be used.: 공백으로 둔다면, 기본 언어로 적용합니다.
 Images: 사진
 Import from: '다음에서 가져오기: '
-Import problem from syzoj.: syzoj에서 문제 가져오기
 Import Problem: 문제 가져오기
 Import: 가져오기
 In 1 day: 하루 안에

packages/hydrooj/locales/zh.yaml

@@ -377,10 +377,7 @@ If enabled, source code will be emailed to you after the submission is accepted.
 If left blank, the built-in template of the corresponding language will be used.: 若留空，则将使用对应语言的内置代码模板。
 Images: 图片
 Import From Hydro: 从 Hydro 导入
-Import From SYZOJ: 从 SYZOJ 导入
-Import From SYZOJ.: 从 SYZOJ 导入
 Import from: 导入自
-Import problem from syzoj.: 从运行 syzoj 系统的 OJ 导入题目。
 Import Problem: 导入题目
 Import User: 导入用户
 Import: 导入
@@ -585,7 +582,6 @@ problem_create: 创建题目
 problem_detail: 题目详情
 problem_edit: 编辑题目
 problem_files: 题目文件
-problem_import_syzoj: 从 SYZOJ 导入题目
 problem_import: 导入题目
 problem_main: 题库
 problem_solution: 题解

packages/hydrooj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "hydrooj",
-    "version": "3.5.7",
+    "version": "3.6.0",
     "bin": "bin/hydrooj.js",
     "main": "src/loader",
     "module": "src/loader",
@@ -12,7 +12,7 @@
     },
     "preferUnplugged": true,
     "dependencies": {
-        "@graphql-tools/schema": "^8.3.1",
+        "@graphql-tools/schema": "^8.3.2",
         "@hydrooj/utils": "workspace:*",
         "adm-zip": "0.5.5",
         "ajv": "^8.10.0",
@@ -23,14 +23,14 @@
         "emoji-regex": "^10.0.0",
         "emojis-list": "2.1.0",
         "esbuild": "0.14.3",
-        "fs-extra": "^10.0.0",
+        "fs-extra": "^10.0.1",
         "graphql": "^16.3.0",
         "graphql-scalars": "1.14.1",
         "js-yaml": "^4.1.0",
         "koa": "^2.13.4",
         "koa-body": "^4.2.0",
         "koa-compress": "^5.1.0",
-        "koa-proxies": "^0.12.1",
+        "koa-proxies": "^0.12.2",
         "koa-router": "^10.1.1",
         "koa-static-cache": "^5.1.4",
         "lodash": "^4.17.21",
@@ -39,7 +39,7 @@
         "minio": "7.0.25",
         "moment-timezone": "^0.5.34",
         "mongodb": "^3.7.3",
-        "nanoid": "^3.3.0",
+        "nanoid": "^3.3.1",
         "nodemailer": "^6.7.2",
         "notp": "^2.0.3",
         "p-queue": "^7.2.0",
@@ -61,7 +61,7 @@
         "@types/koa-compress": "^4.0.3",
         "@types/koa-router": "^7.4.4",
         "@types/koa-static-cache": "^5.1.1",
-        "@types/lodash": "^4.14.178",
+        "@types/lodash": "^4.14.179",
         "@types/lru-cache": "^5.1.1",
         "@types/mime-types": "^2.1.1",
         "@types/minio": "^7.0.12",
@@ -74,7 +74,7 @@
         "@types/source-map-support": "^0.5.4",
         "@types/superagent": "^4.1.15",
         "moment": "^2.29.1",
-        "pm2": "^5.1.2"
+        "pm2": "^5.2.0"
     },
     "peerDependencies": {
         "pm2": "*"

packages/hydrooj/src/handler/import.ts

@@ -1,326 +1,25 @@
 /* eslint-disable no-await-in-loop */
-import assert from 'assert';
 import os from 'os';
 import path from 'path';
-import { PassThrough } from 'stream';
 import AdmZip from 'adm-zip';
 import fs from 'fs-extra';
 import yaml from 'js-yaml';
-import { filter } from 'lodash';
-import superagent from 'superagent';
-import { noop } from '@hydrooj/utils/lib/utils';
-import { RemoteOnlineJudgeError, ValidationError } from '../error';
-import type { ContentNode, ProblemConfigFile, SubtaskConfig } from '../interface';
-import { buildContent } from '../lib/content';
-import download from '../lib/download';
+import { ValidationError } from '../error';
 import { ProblemAdd } from '../lib/ui';
-import { isPid, parsePid } from '../lib/validator';
-import { Logger } from '../logger';
 import { PERM, PRIV } from '../model/builtin';
 import problem, { ProblemDoc } from '../model/problem';
-import TaskModel from '../model/task';
-import {
-    Handler, post, Route, Types,
-} from '../service/server';
+import { Handler, Route } from '../service/server';
 
-const RE_SYZOJ = /(https?):\/\/([^/]+)\/(problem|p)\/([0-9]+)\/?/i;
-const logger = new Logger('import.syzoj');
-
-const ScoreTypeMap = {
-    GroupMin: 'min',
-    Sum: 'sum',
-    GroupMul: 'max',
-};
-const LanguageMap = {
-    cpp: 'cc',
-};
-
-async function syzojSync(info) {
-    const {
-        protocol, host, domainId, docId,
-    } = info;
-    const body = JSON.parse(info.body);
-    const pid = body.meta.id;
-    const judge = body.judgeInfo;
-    const rename = {};
-    if (judge) {
-        const config: ProblemConfigFile = {
-            time: `${judge.timeLimit}ms`,
-            memory: `${judge.memoryLimit}m`,
-        };
-        if (judge.extraSourceFiles) {
-            const files = [];
-            for (const key in judge.extraSourceFiles) {
-                for (const file in judge.extraSourceFiles[key]) {
-                    files.push(file);
-                }
-            }
-            config.user_extra_files = files;
-        }
-        if (judge.checker?.type === 'custom') {
-            config.checker_type = judge.checker.interface;
-            if (LanguageMap[judge.checker.language]) {
-                rename[judge.checker.filename] = `chk.${LanguageMap[judge.checker.language]}`;
-                config.checker = `chk.${LanguageMap[judge.checker.language]}`;
-            } else config.checker = judge.checker.filename;
-        }
-        if (judge.subtasks?.length) {
-            config.subtasks = [];
-            for (const subtask of judge.subtasks) {
-                const current: SubtaskConfig = {
-                    score: subtask.points,
-                    type: ScoreTypeMap[subtask.scoringType],
-                    cases: subtask.testcases.map((i) => ({ input: i.inputFile, output: i.outputFile })),
-                };
-                if (subtask.dependencies) current.if = subtask.dependencies;
-                config.subtasks.push(current);
-            }
-        }
-        await problem.addTestdata(domainId, docId, 'config.yaml', Buffer.from(yaml.dump(config)));
-    }
-    const r = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/downloadProblemFiles`)
-        .send({
-            problemId: pid,
-            type: 'TestData',
-            filenameList: body.testData.map((node) => node.filename),
-        });
-    if (r.body.error) throw new Error(r.body.error.message || r.body.error);
-    for (const f of r.body.downloadInfo) {
-        const p = new PassThrough();
-        superagent.get(f.downloadUrl).pipe(p);
-        // eslint-disable-next-line no-await-in-loop
-        await problem.addTestdata(domainId, docId, rename[f.filename] || f.filename, p);
-    }
-    const a = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/downloadProblemFiles`)
-        .send({
-            problemId: pid,
-            type: 'AdditionalFile',
-            filenameList: body.additionalFiles.map((node) => node.filename),
-        });
-    if (a.body.error) throw new Error(a.body.error.message || a.body.error);
-    for (const f of a.body.downloadInfo) {
-        const p = new PassThrough();
-        superagent.get(f.downloadUrl).pipe(p);
-        // eslint-disable-next-line no-await-in-loop
-        await problem.addAdditionalFile(domainId, docId, f.filename, p);
-    }
-}
-TaskModel.Worker.addHandler('import.syzoj', syzojSync);
-
-class ProblemImportSYZOJHandler extends Handler {
-    async get() {
-        this.response.template = 'problem_import_syzoj.html';
-    }
-
-    async v2(domainId: string, target: string, hidden = false, url: string) {
-        const res = await superagent.get(`${url}export`);
-        assert(res.status === 200, new RemoteOnlineJudgeError('Cannot connect to target server'));
-        assert(res.body.success, new RemoteOnlineJudgeError((res.body.error || {}).message));
-        const p = res.body.obj;
-        const content: ContentNode[] = [];
-        if (p.description) {
-            content.push({
-                type: 'Text',
-                subType: 'markdown',
-                sectionTitle: this.translate('Problem Description'),
-                text: p.description,
-            });
-        }
-        if (p.input_format) {
-            content.push({
-                type: 'Text',
-                subType: 'markdown',
-                sectionTitle: this.translate('Input Format'),
-                text: p.input_format,
-            });
-        }
-        if (p.output_format) {
-            content.push({
-                type: 'Text',
-                subType: 'markdown',
-                sectionTitle: this.translate('Output Format'),
-                text: p.output_format,
-            });
-        }
-        if (p.example) {
-            content.push({
-                type: 'Text',
-                subType: 'markdown',
-                sectionTitle: this.translate('Sample'),
-                text: p.example,
-            });
-        }
-        if (p.hint) {
-            content.push({
-                type: 'Text',
-                subType: 'markdown',
-                sectionTitle: this.translate('Hint'),
-                text: p.hint,
-            });
-        }
-        if (p.limit_and_hint) {
-            content.push({
-                type: 'Text',
-                subType: 'markdown',
-                sectionTitle: this.translate('Limit And Hint'),
-                text: p.limit_and_hint,
-            });
-        }
-        const c = buildContent(content, 'markdown');
-        const docId = await problem.add(
-            domainId, target, p.title, c, this.user._id, p.tags || [], hidden,
-        );
-        const r = download(`${url}testdata/download`);
-        const file = path.resolve(os.tmpdir(), 'hydro', `import_${domainId}_${docId}.zip`);
-        const w = fs.createWriteStream(file);
-        try {
-            await new Promise((resolve, reject) => {
-                w.on('finish', resolve);
-                w.on('error', reject);
-                r.pipe(w);
-            });
-            const zip = new AdmZip(file);
-            const entries = zip.getEntries();
-            for (const entry of entries) {
-                // eslint-disable-next-line no-await-in-loop
-                await problem.addTestdata(domainId, docId, entry.entryName, entry.getData());
-            }
-            const filename = p.file_io_input_name ? p.file_io_input_name.split('.')[0] : null;
-            const config = {
-                time: `${p.time_limit}ms`,
-                memory: `${p.memory_limit}m`,
-                filename,
-                type: p.type === 'traditional' ? 'default' : p.type,
-            };
-            await problem.addTestdata(domainId, docId, 'config.yaml', Buffer.from(yaml.dump(config)));
-        } finally {
-            fs.unlinkSync(file);
-        }
-        if (p.have_additional_file) {
-            const r1 = download(`${url}download/additional_file`);
-            const file1 = path.resolve(os.tmpdir(), 'hydro', `import_${domainId}_${docId}_a.zip`);
-            const w1 = fs.createWriteStream(file1);
-            try {
-                await new Promise((resolve, reject) => {
-                    w1.on('finish', resolve);
-                    w1.on('error', reject);
-                    r1.pipe(w1);
-                });
-                const zip = new AdmZip(file1);
-                const entries = zip.getEntries();
-                for (const entry of entries) {
-                    await problem.addAdditionalFile(domainId, docId, entry.entryName.replace('/', '_'), entry.getData());
-                }
-            } finally {
-                fs.unlinkSync(file1);
-            }
-        }
-        return docId;
-    }
-
-    async v3(
-        domainId: string, target: string, hidden: boolean,
-        protocol: string, host: string, pid: string | number,
-        wait = false,
-    ) {
-        let tagsOfLocale = this.user.viewLang || this.session.viewLang;
-        if (tagsOfLocale === 'en') tagsOfLocale = 'en_US';
-        else tagsOfLocale = 'zh_CN';
-        const result = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/getProblem`)
-            .send({
-                displayId: +pid,
-                localizedContentsOfAllLocales: true,
-                tagsOfLocale,
-                samples: true,
-                judgeInfo: true,
-                testData: true,
-                additionalFiles: true,
-            });
-        const content = {};
-        for (const c of result.body.localizedContentsOfAllLocales) {
-            const sections = c.contentSections;
-            for (const section of sections) {
-                section.subType = 'markdown';
-                if (section.type === 'Sample') {
-                    section.payload = [
-                        result.body.samples[section.sampleId].inputData,
-                        result.body.samples[section.sampleId].outputData,
-                    ];
-                    delete section.sampleId;
-                }
-            }
-            let locale = c.locale;
-            if (locale === 'en_US') locale = 'en';
-            else if (locale === 'zh_CN') locale = 'zh';
-            content[locale] = sections;
-        }
-        const tags = result.body.tagsOfLocale.map((node) => node.name);
-        const title = [
-            ...filter(
-                result.body.localizedContentsOfAllLocales,
-                (node) => node.locale === (this.user.viewLang || this.session.viewLang),
-            ),
-            ...result.body.localizedContentsOfAllLocales,
-        ][0].title;
-        const docId = await problem.add(
-            domainId, target, title, JSON.stringify(content), this.user._id, tags || [], hidden,
-        );
-        const payload = {
-            protocol, host, pid, domainId, docId, body: JSON.stringify(result.body),
-        };
-        if (wait) await syzojSync(payload);
-        else await TaskModel.add({ ...payload, type: 'schedule', subType: 'import.syzoj' });
-        return docId;
-    }
-
-    @post('url', Types.Content, true)
-    @post('pid', Types.Name, true, isPid, parsePid)
-    @post('hidden', Types.Boolean)
-    @post('prefix', Types.Name, true)
-    @post('start', Types.UnsignedInt, true)
-    @post('end', Types.UnsignedInt, true)
-    async post(
-        domainId: string, url: string, targetPid: string, hidden = false,
-        prefix: string, start: number, end: number,
-    ) {
-        if (/^(.+)\/(\d+)\.\.(\d+)$/.test(url)) {
-            const res = /^(.+)\/(\d+)\.\.(\d+)$/.exec(url)!;
-            prefix = res[1];
-            start = +res[2];
-            end = +res[3];
-            if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start > end) {
-                throw new ValidationError('end');
-            }
-        }
-        if (prefix) {
-            let version = 2;
-            if (!prefix.endsWith('/')) prefix += '/';
-            if (prefix.endsWith('/p/')) version = 3;
-            else prefix = `${prefix.split('/problem/')[0]}/problem/`;
-            const base = `${prefix}${start}/`;
-            assert(base.match(RE_SYZOJ), new ValidationError('prefix'));
-            const [, protocol, host] = RE_SYZOJ.exec(base);
-            (async () => {
-                for (let i = start; i <= end; i++) {
-                    // eslint-disable-next-line no-await-in-loop
-                    if (version === 3) await this.v3(domainId, undefined, hidden, protocol, host, i, true).catch(noop);
-                    // eslint-disable-next-line no-await-in-loop
-                    else await this.v2(domainId, undefined, hidden, `${prefix + i}/`).catch(noop);
-                    logger.info('%s %d-%d-%d', prefix, start, i, end);
-                }
-            })().catch(logger.error);
-            this.response.redirect = this.url('problem_main');
-        } else {
-            assert(url.match(RE_SYZOJ), new ValidationError('url'));
-            if (!url.endsWith('/')) url += '/';
-            const [, protocol, host, n, pid] = RE_SYZOJ.exec(url);
-            const docId = n === 'p'
-                ? await this.v3(domainId, targetPid, hidden, protocol, host, pid, false)
-                : await this.v2(domainId, targetPid, hidden, url);
-            this.response.body = { pid: targetPid || docId };
-            this.response.redirect = this.url('problem_detail', { pid: targetPid || docId });
-        }
+function findOverrideContent(dir: string) {
+    let files = fs.readdirSync(dir);
+    if (files.includes('problem.md')) return fs.readFileSync(path.join(dir, 'problem.md'), 'utf8');
+    const languages = {};
+    files = files.filter((i) => /^problem_[a-zA-Z_]+\.md$/.test(i));
+    for (const file of files) {
+        const lang = file.slice(8, -3);
+        languages[lang] = fs.readFileSync(path.join(dir, file), 'utf8');
     }
+    return JSON.stringify(languages);
 }
 
 class ProblemImportHydroHandler extends Handler {
@@ -348,8 +47,7 @@ class ProblemImportHydroHandler extends Handler {
                 const pdoc: ProblemDoc = yaml.load(content) as any;
                 const current = await problem.get(domainId, pdoc.pid);
                 const pid = current ? undefined : pdoc.pid;
-                const overrideContent = fs.existsSync(path.join(tmpdir, i, 'problem.md'))
-                    ? fs.readFileSync(path.join(tmpdir, i, 'problem.md'), 'utf8') : '';
+                const overrideContent = findOverrideContent(path.join(tmpdir, i));
                 const docId = await problem.add(
                     domainId, pid, pdoc.title, overrideContent || pdoc.content,
                     keepUser ? pdoc.owner : this.user._id, pdoc.tag, pdoc.hidden,
@@ -388,7 +86,6 @@ class ProblemImportHydroHandler extends Handler {
 
 export async function apply() {
     ProblemAdd('problem_import_hydro', {}, 'copy', 'Import From Hydro');
-    Route('problem_import_syzoj', '/problem/import/syzoj', ProblemImportSYZOJHandler, PERM.PERM_CREATE_PROBLEM);
     Route('problem_import_hydro', '/problem/import/hydro', ProblemImportHydroHandler, PERM.PERM_CREATE_PROBLEM);
 }
 

packages/hydrooj/src/lib/ui.ts

@@ -77,7 +77,6 @@ Nav('ranking', {}, 'ranking', PERM.PERM_VIEW_RANKING);
 Nav('domain_dashboard', {}, 'domain', PERM.PERM_EDIT_DOMAIN);
 Nav('manage_dashboard', {}, 'manage', PRIV.PRIV_EDIT_SYSTEM);
 ProblemAdd('problem_create');
-ProblemAdd('problem_import_syzoj', {}, 'copy', 'Import From SYZOJ');
 
 global.Hydro.ui.Nav = Nav;
 global.Hydro.ui.ProblemAdd = ProblemAdd;

packages/ui-default/components/problem/create.page.js

@@ -3,7 +3,7 @@ import { InfoDialog } from 'vj/components/dialog';
 import tpl from 'vj/utils/tpl';
 import i18n from 'vj/utils/i18n';
 
-export default new NamedPage(['problem_create', 'problem_edit', 'problem_import_syzoj'], () => {
+export default new NamedPage(['problem_create', 'problem_edit'], () => {
   $('input[name="pid"]').on('blur', () => {
     if (/^[0-9]+$/.test($('input[name="pid"]').val())) {
       new InfoDialog({

packages/ui-default/locales/ko.yaml

@@ -347,7 +347,6 @@ If enabled, source code will be emailed to you after the submission is accepted.
 If left blank, the built-in template of the corresponding language will be used.: 공백으로 둔다면, 기본 언어로 적용합니다.
 Images: 사진
 Import from: '다음에서 가져오기: '
-Import problem from syzoj.: syzoj에서 문제 가져오기
 Import Problem: 문제 가져오기
 Import: 가져오기
 In 1 day: 하루 안에

packages/ui-default/locales/zh.yaml

@@ -381,7 +381,6 @@ If left blank, the built-in template of the corresponding language will be used.
 Images With Size: 指定大小的图片
 Images: 图片
 Import from: 导入自
-Import problem from syzoj.: 从运行 syzoj 系统的 OJ 导入题目。
 Import Problem: 导入题目
 Import: 导入
 In 1 day: 一天后

packages/ui-default/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hydrooj/ui-default",
-  "version": "4.32.12",
+  "version": "4.32.13",
   "author": "undefined <i@undefined.moe>",
   "license": "AGPL-3.0",
   "main": "hydro.js",