core: support ContentNode[] format

packages/hydrooj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "hydrooj",
-    "version": "2.13.61",
+    "version": "2.14.0",
     "bin": "bin/hydrooj.js",
     "main": "dist/loader.js",
     "typings": "dist/loader.d.ts",

packages/hydrooj/src/entry/common.ts

@@ -12,6 +12,7 @@ export const builtinLib = [
     'jwt', 'download', 'i18n', 'mail', 'useragent',
     'crypto', 'misc', 'paginate', 'hash.hydro', 'rank',
     'validator', 'ui', 'sysinfo', 'testdata.convert.ini', 'testdataConfig',
+    'content',
 ];
 
 export const builtinModel = [

packages/hydrooj/src/handler/import.syzoj.ts

@@ -3,16 +3,22 @@ import path from 'path';
 import os from 'os';
 import assert from 'assert';
 import superagent from 'superagent';
+import { filter } from 'lodash';
+import AdmZip from 'adm-zip';
 import { ValidationError, RemoteOnlineJudgeError } from '../error';
+import { streamToBuffer } from '../utils';
+import { Logger } from '../logger';
+import type { ContentNode } from '../interface';
 import * as problem from '../model/problem';
 import { PERM } from '../model/builtin';
 import {
-    Route, Handler, Types, param,
+    Route, Handler, Types, param, post,
 } from '../service/server';
 import { isPid } from '../lib/validator';
 import download from '../lib/download';
 
-const RE_SYZOJ = /https?:\/\/([a-zA-Z0-9.]+)\/problem\/([0-9]+)\/?/i;
+const RE_SYZOJ = /(https?):\/\/([a-zA-Z0-9.:]+)\/(problem|p)\/([0-9]+)\/?/i;
+const logger = new Logger('import.syzoj');
 
 class ProblemImportSYZOJHandler extends Handler {
     async get() {
@@ -26,57 +32,142 @@ class ProblemImportSYZOJHandler extends Handler {
         };
     }
 
-    @param('url', Types.String)
+    @post('url', Types.String, true)
     @param('pid', Types.String, true, isPid)
     @param('hidden', Types.Boolean)
     async post(domainId: string, url: string, targetPid: string, hidden = false) {
         assert(url.match(RE_SYZOJ), new ValidationError('url'));
         if (!url.endsWith('/')) url += '/';
-        const [, host, pid] = RE_SYZOJ.exec(url);
+        const [, protocol, host, n, pid] = RE_SYZOJ.exec(url);
+        if (n === 'p') {
+            // SYZOJ-NG
+            const result = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/getProblem`)
+                .send({
+                    displayId: +pid,
+                    localizedContentsOfAllLocales: true,
+                    tagsOfLocale: this.user.viewLang || this.session.viewLang,
+                    samples: true,
+                    testData: true,
+                    additionalFiles: true,
+                });
+            const content = {};
+            for (const c of result.body.localizedContentsOfAllLocales) {
+                const sections = c.contentSections;
+                for (const section of sections) {
+                    section.subType = 'markdown';
+                    if (section.type === 'Sample') {
+                        section.payload = [
+                            result.body.samples[section.sampleId].inputData,
+                            result.body.samples[section.sampleId].outputData,
+                        ];
+                        delete section.sampleId;
+                    }
+                }
+                content[c.locale] = sections;
+            }
+            const tags = result.body.tagsOfLocale.map((node) => node.name);
+            const title = [
+                ...filter(
+                    result.body.localizedContentsOfAllLocales,
+                    (node) => node.locale === (this.user.viewLang || this.session.viewLang),
+                ),
+                ...result.body.localizedContentsOfAllLocales,
+            ][0].title;
+            const docId = await problem.add(
+                domainId, targetPid || `${host}_${pid}`, title, content, this.user._id,
+                tags || [], [], null, hidden,
+            );
+            (async () => {
+                const judge = result.body.judgeInfo;
+                const r = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/downloadProblemFiles`)
+                    .send({
+                        problemId: +pid,
+                        type: 'TestData',
+                        filenameList: result.body.testData.map((node) => node.filename),
+                    });
+                const urls = {};
+                for (const t of r.body.downloadInfo) urls[t.filename] = t.downloadUrl;
+                const zip = new AdmZip();
+                for (const f of result.body.testData) {
+                    // eslint-disable-next-line no-await-in-loop
+                    const c = await superagent.get(urls[f.filename]);
+                    // eslint-disable-next-line no-await-in-loop
+                    zip.addFile(f.filename, await streamToBuffer(c));
+                }
+                await problem.setTestdata(domainId, docId, zip.toBuffer());
+                await problem.edit(domainId, docId, {
+                    config: {
+                        time: `${judge.timeLimit}ms`,
+                        memory: `${judge.memoryLimit}m`,
+                        // TODO other config
+                    },
+                });
+            })().catch(logger.error);
+            this.response.body = { pid: pid || docId };
+            this.response.redirect = this.url('problem_settings', { pid: pid || docId });
+        } else {
             const res = await superagent.get(`${url}export`);
             assert(res.status === 200, new RemoteOnlineJudgeError('Cannot connect to target server'));
             assert(res.body.success, new RemoteOnlineJudgeError((res.body.error || {}).message));
             const p = res.body.obj;
-        const content = [];
+            const content: ContentNode[] = [];
             if (p.description) {
-            content.push(
+                content.push({
-                `## ${this.translate('Description')}`,
+                    type: 'Text',
-                p.description,
+                    subType: 'markdown',
-            );
+                    sectionTitle: this.translate('Description'),
+                    text: p.description,
+                });
             }
             if (p.input_format) {
-            content.push(
+                content.push({
-                `## ${this.translate('Input Format')}`,
+                    type: 'Text',
-                p.input_format,
+                    subType: 'markdown',
-            );
+                    sectionTitle: this.translate('Input Format'),
+                    text: p.input_format,
+                });
             }
             if (p.output_format) {
-            content.push(
+                content.push({
-                `## ${this.translate('Output Format')}`,
+                    type: 'Text',
-                p.output_format,
+                    subType: 'markdown',
-            );
+                    sectionTitle: this.translate('Output Format'),
+                    text: p.output_format,
+                });
+            }
+            if (p.example) {
+                content.push({
+                    type: 'Plain',
+                    subType: 'markdown',
+                    text: p.example,
+                });
             }
-        if (p.example) content.push(p.example);
             if (p.hint) {
-            content.push(
+                content.push({
-                `## ${this.translate('Hint')}`,
+                    type: 'Text',
-                p.hint,
+                    subType: 'markdown',
-            );
+                    sectionTitle: this.translate('Hint'),
+                    text: p.hint,
+                });
             }
             if (p.limit_and_hint) {
-            content.push(
+                content.push({
-                `## ${this.translate('Limit And Hint')}`,
+                    type: 'Text',
-                p.limit_and_hint,
+                    subType: 'markdown',
-            );
+                    sectionTitle: this.translate('Limit And Hint'),
+                    text: p.output_format,
+                });
             }
             if (p.have_additional_file) {
-            content.push(
+                content.push({
-                `## ${this.translate('Additional File')}`,
+                    type: 'Text',
-                `${url}download/additional_file`,
+                    subType: 'markdown',
-            );
+                    sectionTitle: this.translate('Additional File'),
+                    text: `${url}download/additional_file`,
+                });
             }
             const docId = await problem.add(
-            domainId, targetPid || `${host}_${pid}`, p.title, content.join('  \n'), this.user._id,
+                domainId, targetPid || `${host}_${pid}`, p.title, content, this.user._id,
                 p.tags || [], [], null, hidden,
             );
             const r = download(`${url}testdata/download`);
@@ -100,6 +191,7 @@ class ProblemImportSYZOJHandler extends Handler {
             this.response.redirect = this.url('problem_settings', { pid: pid || docId });
         }
     }
+}
 
 export async function apply() {
     Route('problem_import_syzoj', '/problem/import/syzoj', ProblemImportSYZOJHandler, PERM.PERM_CREATE_PROBLEM);

packages/hydrooj/src/handler/problem.ts

@@ -437,6 +437,9 @@ export class ProblemEditHandler extends ProblemManageHandler {
     @param('content', Types.String, isContent)
     @post('pid', Types.String, isPid, true)
     async post(domainId: string, title: string, content: string, newPid: string = '') {
+        try {
+            content = JSON.parse(content);
+        } catch { /* Ignore */ }
         const $update: Partial<Pdoc> = { title, content, pid: newPid };
         let pdoc = await problem.get(domainId, this.request.params.pid);
         pdoc = await problem.edit(domainId, pdoc.docId, $update);
@@ -646,6 +649,9 @@ export class ProblemCreateHandler extends Handler {
     @param('content', Types.String, isContent)
     @param('hidden', Types.Boolean)
     async post(domainId: string, title: string, pid: string, content: string, hidden = false) {
+        try {
+            content = JSON.parse(content);
+        } catch { /* Ignore */ }
         const docId = await problem.add(
             domainId, pid, title, content,
             this.user._id, [], [], null, hidden,

packages/hydrooj/src/interface.ts

@@ -156,6 +156,26 @@ export interface RemoteProblemConfig {
 
 export type ProblemConfig = LocalProblemConfig | RemoteProblemConfig;
 
+export interface PlainContentNode {
+    type: 'Plain',
+    subType: 'html' | 'markdown',
+    text: string,
+}
+export interface TextContentNode {
+    type: 'Text',
+    subType: 'html' | 'markdown',
+    sectionTitle: string,
+    text: string,
+}
+export interface SampleContentNode {
+    type: 'Sample',
+    text: string,
+    sectionTitle: string,
+    payload: [string, string],
+}
+export type ContentNode = PlainContentNode | TextContentNode | SampleContentNode;
+export type Content = string | ContentNode[] | Record<string, ContentNode[]>
+
 declare module './model/problem' {
     interface Pdoc {
         domainId: string,
@@ -164,7 +184,7 @@ declare module './model/problem' {
         pid: string,
         owner: number,
         title: string,
-        content: string,
+        content: Content,
         nSubmit: number,
         nAccept: number,
         tag: string[],
@@ -525,6 +545,7 @@ interface GeoIP {
 
 export interface Lib {
     download: typeof import('./lib/download'),
+    buildContent: typeof import('./lib/content').buildContent,
     'hash.hydro': typeof import('./lib/hash.hydro'),
     i18n: typeof import('./lib/i18n'),
     jwt: typeof import('./lib/jwt'),

packages/hydrooj/src/lib/content.ts

@@ -0,0 +1,74 @@
+import { ContentNode } from '../interface';
+
+export interface ProblemSource {
+    description?: string,
+    input?: string,
+    output?: string,
+    samples?: [string, string][],
+    hint?: string,
+    source?: string,
+}
+
+export function buildContent(source: ProblemSource | ContentNode[], type: 'markdown' | 'html' = 'markdown', translate?: Function) {
+    const _ = translate || ((s: string) => s);
+    if (source instanceof Array) {
+        return type === 'html'
+            ? source.map((node) => [
+                node.type !== 'Plain' ? `<h2>${node.sectionTitle}</h2>` : '',
+                node.type === 'Sample'
+                    ? `<h2>${_('Sample Input')}<h2><pre>${node.payload[0]}</pre><h2>${_('Sample Output')}</h2><pre>${node.payload[1]}</pre>`
+                    : '',
+                node.text,
+            ].join('\n')).join('\n')
+            : source.map((node) => [
+                node.type !== 'Plain' ? `## ${node.sectionTitle}` : '',
+                ...node.type === 'Sample'
+                    ? [
+                        `## ${_('Sample Input')}`,
+                        '```',
+                        node.payload[0],
+                        '```',
+                        `## ${_('Sample Output')}`,
+                        '```',
+                        node.payload[1],
+                        '```',
+                    ]
+                    : [],
+                node.text,
+            ].join('\n')).join('\n');
+    }
+    return type === 'html'
+        ? [
+            ...source.description ? [`<h2>${_('Description')}</h2>`, source.description] : [],
+            ...source.input ? [`<h2>${_('Input Format')}</h2>`, source.input] : [],
+            ...source.output ? [`<h2>'${_('Output Format')}</h2>`, source.output] : [],
+            ...(source.samples).map((sample, i) => [
+                `<h2>${_('Sample Input')} ${i + 1}</h2><pre>`,
+                sample[0],
+                `</pre><h2>${_('Sample Output')} ${i + 1}</h2><pre>`,
+                sample[1],
+                '</pre>',
+            ].join('\n')),
+            ...source.hint ? [`<h2>${_('Hint')}</h2>`, source.hint] : [],
+            ...source.source ? [`<h2>${_('Source')}</h2>`, source.source] : [],
+        ].join('\n')
+        : [
+            ...source.description ? [`## ${_('Description')}`, source.description] : [],
+            ...source.input ? [`## ${_('Input Format')}`, source.input] : [],
+            ...source.output ? [`## '${_('Output Format')}`, source.output] : [],
+            ...(source.samples).map((sample, i) => [
+                `## ${_('Sample Input')} ${i + 1}`,
+                '```',
+                sample[0],
+                '```',
+                `## ${_('Sample Output')} ${i + 1}`,
+                '```',
+                sample[1],
+                '```',
+            ].join('\n')),
+            ...source.hint ? [`## ${_('Hint')}`, source.hint] : [],
+            ...source.source ? [`## ${_('Source')}`, source.source] : [],
+        ].join('\n');
+}
+
+global.Hydro.lib.buildContent = buildContent;

packages/hydrooj/src/model/document.ts

@@ -10,6 +10,7 @@ import { buildProjection } from '../utils';
 import { NumberKeys, ArrayKeys, Projection } from '../typeutils';
 import * as db from '../service/db';
 import * as bus from '../service/bus';
+import { Content } from '../loader';
 
 type DocID = ObjectID | string | number;
 
@@ -46,19 +47,19 @@ export interface DocStatusType {
 }
 
 export async function add<T extends keyof DocType, K extends DocType[T]['docId']>(
-    domainId: string, content: string, owner: number,
+    domainId: string, content: Content, owner: number,
     docType: T, docId: K,
     parentType?: DocType[T]['parentType'], parentId?: DocType[T]['parentId'],
     args?: Partial<DocType[T]>,
 ): Promise<K>
 export async function add<T extends keyof DocType>(
-    domainId: string, content: string, owner: number,
+    domainId: string, content: Content, owner: number,
     docType: T, docId: null,
     parentType?: DocType[T]['parentType'], parentId?: DocType[T]['parentId'],
     args?: Partial<DocType[T]>,
 ): Promise<ObjectID>
 export async function add(
-    domainId: string, content: string, owner: number,
+    domainId: string, content: Content, owner: number,
     docType: number, docId: DocID = null,
     parentType: number | null = null, parentId: DocID = null,
     args: any = {},

packages/hydrooj/src/model/problem.ts

@@ -6,6 +6,7 @@ import * as file from './file';
 import * as document from './document';
 import * as domain from './domain';
 import { ProblemStatusDoc, ProblemDataSource, Pdict } from '../interface';
+import { Content } from '../loader';
 import { NumberKeys, Projection } from '../typeutils';
 import { ProblemNotFoundError } from '../error';
 import * as testdataConfig from '../lib/testdataConfig';
@@ -78,7 +79,7 @@ export const PROJECTION_PUBLIC: Pdoc.Field[] = [
 ];
 
 export async function add(
-    domainId: string, pid: string = null, title: string, content: string, owner: number,
+    domainId: string, pid: string = null, title: string, content: Content, owner: number,
     tag: string[] = [], category: string[] = [], data: ProblemDataSource = null, hidden = false,
 ) {
     const pidCounter = await domain.inc(domainId, 'pidCounter', 1);

packages/hydrooj/src/service/minio.ts.i

@@ -0,0 +1,319 @@
+import { URL } from 'url';
+import { Readable } from 'stream';
+import { v4 as UUID } from 'uuid';
+import { Client as MinioClient } from 'minio';
+
+import { FileEntity } from './file.entity';
+
+import { FileUploadInfoDto, SignedFileUploadRequestDto } from './dto';
+
+// 10 minutes upload expire time
+const FILE_UPLOAD_EXPIRE_TIME = 10 * 60;
+// 20 minutes download expire time
+const FILE_DOWNLOAD_EXPIRE_TIME = 20 * 60 * 60;
+
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent
+function encodeRFC5987ValueChars(str: string) {
+    return (
+        encodeURIComponent(str)
+            // Note that although RFC3986 reserves "!", RFC5987 does not,
+            // so we do not need to escape it
+            .replace(/['()]/g, escape) // i.e., %27 %28 %29
+            .replace(/\*/g, '%2A')
+            // The following are not required for percent-encoding per RFC5987,
+            // so we can allow for a little better readability over the wire: |`^
+            .replace(/%(?:7C|60|5E)/g, unescape)
+    );
+}
+
+interface MinioEndpointConfig {
+    endPoint: string;
+    port: number;
+    useSSL: boolean;
+}
+
+function parseMainEndpointUrl(endpoint: string): MinioEndpointConfig {
+    const url = new URL(endpoint);
+    const result: Partial<MinioEndpointConfig> = {};
+    if (url.pathname !== '/') throw new Error('Main MinIO endpoint URL of a sub-directory is not supported.');
+    if (url.username || url.password || url.hash || url.search) {
+        throw new Error('Authorization, search parameters and hash are not supported for main MinIO endpoint URL.');
+    }
+    if (url.protocol === 'http:') result.useSSL = false;
+    else if (url.protocol === 'https:') result.useSSL = true;
+    else {
+        throw new Error(
+            `Invalid protocol "${url.protocol}" for main MinIO endpoint URL. Only HTTP and HTTPS are supported.`,
+        );
+    }
+    result.endPoint = url.hostname;
+    result.port = url.port ? Number(url.port) : result.useSSL ? 443 : 80;
+    return result as MinioEndpointConfig;
+}
+
+function parseAlternativeEndpointUrl(endpoint: string): (originalUrl: string) => string {
+    if (!endpoint) return (originalUrl) => originalUrl;
+    const url = new URL(endpoint);
+    if (url.hash || url.search) {
+        throw new Error('Search parameters and hash are not supported for alternative MinIO endpoint URL.');
+    }
+    if (!url.pathname.endsWith('/')) throw new Error("Alternative MinIO endpoint URL's pathname must ends with '/'.");
+    return (originalUrl) => {
+        const parsedOriginUrl = new URL(originalUrl);
+        return new URL(parsedOriginUrl.pathname.slice(1) + parsedOriginUrl.search + parsedOriginUrl.hash, url).toString();
+    };
+}
+
+export enum AlternativeUrlFor {
+    User,
+    Judge
+}
+
+const minioClient = new MinioClient({
+    ...parseMainEndpointUrl(config.endpoint),
+    accessKey:
+})
+
+export class FileService {
+    private readonly minioClient: MinioClient;
+
+    private readonly bucket: string;
+
+    private readonly replaceWithAlternativeUrlFor: Record<AlternativeUrlFor, (originalUrl: string) => string>;
+
+    constructor(
+        private readonly connection: Connection,
+        private readonly fileRepository: Repository<FileEntity>,
+        private readonly configService: ConfigService,
+    ) {
+        const config = this.configService.config.services.minio;
+
+        this.minioClient = new MinioClient({
+            ...parseMainEndpointUrl(config.endpoint),
+            accessKey: config.accessKey,
+            secretKey: config.secretKey,
+        });
+        this.bucket = config.bucket;
+        this.replaceWithAlternativeUrlFor = {
+            [AlternativeUrlFor.User]: parseAlternativeEndpointUrl(config.endpointForUser),
+            [AlternativeUrlFor.Judge]: parseAlternativeEndpointUrl(config.endpointForJudge),
+        };
+    }
+
+    fileExistsInMinio(uuid: string): Promise<boolean> {
+        return new Promise((resolve) =>
+            this.minioClient
+                .statObject(this.bucket, uuid)
+                .then(() => resolve(true))
+                .catch(() => resolve(false)));
+    }
+
+    async uploadFile(uuid: string, streamOrBufferOrFile: string | Buffer | Readable, retryCount = 10): Promise<void> {
+        for (let i = 0; i < retryCount; i++) {
+            try {
+                /* eslint-disable no-await-in-loop */
+                if (typeof streamOrBufferOrFile === 'string') { await this.minioClient.fPutObject(this.bucket, uuid, streamOrBufferOrFile, {}); } else await this.minioClient.putObject(this.bucket, uuid, streamOrBufferOrFile, {});
+                /* eslint-enable no-await-in-loop */
+            } catch (e) {
+                if (i === retryCount - 1) throw e;
+                else {
+                    // eslint-disable-next-line no-await-in-loop
+                    await new Promise((resolve) => setTimeout(resolve, 1000));
+                    continue;
+                }
+            }
+        }
+    }
+
+    async onModuleInit(): Promise<void> {
+        let bucketExists: boolean;
+        try {
+            bucketExists = await this.minioClient.bucketExists(this.bucket);
+        } catch (e) {
+            throw new Error(
+                `Error initializing the MinIO client. Please check your configuration file and MinIO server. ${e}`,
+            );
+        }
+
+        if (!bucketExists) {
+            throw new Error(
+                `MinIO bucket ${this.bucket} doesn't exist. Please check your configuration file and MinIO server.`,
+            );
+        }
+    }
+
+    /**
+     * Process the client's request about uploading a file.
+     *
+     * * If the user has not uploaded the file, return a signed upload info object.
+     * * If the user has uploaded the file, check the file existance, file size and limits.
+     *
+     * @note
+     * The `checkLimit` function may return different result for each call with the same upload info.
+     *
+     * e.g. When a user uploaded other files before upload this file, the quota is enough before uploading but
+     *      not enough after uploading.
+     *
+     * If the file is checked to be exceeding the limit after uploaded, it will be deleted.
+     *
+     * @return A `SignedFileUploadRequestDto` to return to the client if the file is not uploaded.
+     * @return `string` error message if client says the file is uploaded but we do not accept it according to some errors.
+     * @return A `FileEntity` if the file is uploaded successfully and saved to the database.
+     */
+    async processUploadRequest<LimitCheckErrorType extends string>(
+        uploadInfo: FileUploadInfoDto,
+        checkLimit: (size: number) => Promise<LimitCheckErrorType> | LimitCheckErrorType,
+        transactionalEntityManager: EntityManager,
+    ): Promise<FileEntity | SignedFileUploadRequestDto | LimitCheckErrorType | 'FILE_UUID_EXISTS' | 'FILE_NOT_UPLOADED'> {
+        const limitCheckError = await checkLimit(uploadInfo.size);
+        if (limitCheckError) {
+            this.deleteUnfinishedUploadedFile(uploadInfo.uuid);
+            return limitCheckError;
+        }
+
+        if (uploadInfo.uuid) {
+            // The client says the file is uploaded
+
+            if ((await transactionalEntityManager.count(FileEntity, { uuid: uploadInfo.uuid })) !== 0) return 'FILE_UUID_EXISTS';
+
+            // Check file existance
+            try {
+                await this.minioClient.statObject(this.bucket, uploadInfo.uuid);
+            } catch (e) {
+                if (e.message === 'The specified key does not exist.') {
+                    return 'FILE_NOT_UPLOADED';
+                }
+                throw e;
+            }
+
+            // Save to the database
+            const file = new FileEntity();
+            file.uuid = uploadInfo.uuid;
+            file.size = uploadInfo.size;
+            file.uploadTime = new Date();
+
+            await transactionalEntityManager.save(FileEntity, file);
+
+            return file;
+        }
+        // The client says it want to upload a file for this request
+
+        return await this.signUploadRequest(uploadInfo.size, uploadInfo.size);
+    }
+
+    /**
+     * Sign a upload request for given size. The alternative MinIO endpoint for user will be used in the POST URL.
+     */
+    private async signUploadRequest(minSize?: number, maxSize?: number): Promise<SignedFileUploadRequestDto> {
+        const uuid = UUID();
+        const policy = this.minioClient.newPostPolicy();
+        policy.setBucket(this.bucket);
+        policy.setKey(uuid);
+        policy.setExpires(new Date(Date.now() + FILE_UPLOAD_EXPIRE_TIME * 1000));
+        if (minSize != null || maxSize != null) {
+            policy.setContentLengthRange(minSize || 0, maxSize || 0);
+        }
+        const policyResult = await this.minioClient.presignedPostPolicy(policy);
+
+        return {
+            uuid,
+            method: 'POST',
+            url: this.replaceWithAlternativeUrlFor[AlternativeUrlFor.User](policyResult.postURL),
+            extraFormData: policyResult.formData,
+            fileFieldName: 'file',
+        };
+    }
+
+    /**
+     * @return A function to run after transaction, to delete the file(s) actually.
+     */
+    async deleteFile(uuid: string | string[], transactionalEntityManager: EntityManager): Promise<() => void> {
+        if (typeof uuid === 'string') {
+            await transactionalEntityManager.delete(FileEntity, { uuid });
+            return () =>
+                this.minioClient.removeObject(this.bucket, uuid).catch((e) => {
+                    logger.error(`Failed to delete file ${uuid}: ${e}`);
+                });
+        }
+        if (uuid.length > 0) {
+            await transactionalEntityManager.delete(FileEntity, { uuid: In(uuid) });
+            return () =>
+                this.minioClient.removeObjects(this.bucket, uuid).catch((e) => {
+                    logger.error(`Failed to delete file [${uuid}]: ${e}`);
+                });
+        }
+        return () => {
+            /* do nothing */
+        };
+    }
+
+    /**
+     * Delete a user-uploaded file before calling finishUpload()
+     */
+    deleteUnfinishedUploadedFile(uuid: string): void {
+        this.minioClient.removeObject(this.bucket, uuid).catch((e) => {
+            if (e.message === 'The specified key does not exist.') return;
+            logger.error(`Failed to delete unfinished uploaded file ${uuid}: ${e}`);
+        });
+    }
+
+    async getFileSizes(uuids: string[], transcationalEntityManager: EntityManager): Promise<number[]> {
+        if (uuids.length === 0) return [];
+        const uniqueUuids = Array.from(new Set(uuids));
+        const files = await transcationalEntityManager.find(FileEntity, {
+            uuid: In(uniqueUuids),
+        });
+        const map = Object.fromEntries(files.map((file) => [file.uuid, file]));
+        return uuids.map((uuid) => map[uuid].size);
+    }
+
+    async signDownloadLink({
+        uuid,
+        downloadFilename,
+        noExpire,
+        useAlternativeEndpointFor,
+    }: {
+        uuid: string;
+        downloadFilename?: string;
+        noExpire?: boolean;
+        useAlternativeEndpointFor?: AlternativeUrlFor;
+    }): Promise<string> {
+        const url = await this.minioClient.presignedGetObject(
+            this.bucket,
+            uuid,
+            // The maximum expire time is 7 days
+            noExpire ? 24 * 60 * 60 * 7 : FILE_DOWNLOAD_EXPIRE_TIME,
+            !downloadFilename
+                ? {}
+                : {
+                    'response-content-disposition': `attachment; filename="${encodeRFC5987ValueChars(downloadFilename)}"`,
+                },
+        );
+
+        if (useAlternativeEndpointFor != null) return this.replaceWithAlternativeUrlFor[useAlternativeEndpointFor](url);
+        return url;
+    }
+
+    async runMaintainceTasks(): Promise<void> {
+        // Delete unused files
+        // TODO: Use listObjectsV2 instead, which returns at most 1000 objects in a time
+        const stream = this.minioClient.listObjects(this.bucket);
+        const deleteList: string[] = [];
+        await new Promise((resolve, reject) => {
+            const promises: Promise<void>[] = [];
+            stream.on('data', (object) => {
+                promises.push(
+                    (async () => {
+                        const uuid = object.name;
+                        if (!(await this.fileRepository.count({ uuid }))) {
+                            deleteList.push(uuid);
+                        }
+                    })(),
+                );
+            });
+            stream.on('end', () => Promise.all(promises).then(resolve).catch(reject));
+            stream.on('error', reject);
+        });
+        await this.minioClient.removeObjects(this.bucket, deleteList);
+    }
+}

packages/hydrooj/src/utils.ts

@@ -4,6 +4,7 @@ import cluster from 'cluster';
 import path from 'path';
 import serialize from 'serialize-javascript';
 import Lru from 'lru-cache';
+import type { ContentNode } from './interface';
 
 declare global {
     interface StringConstructor {
@@ -224,48 +225,3 @@ function deepen(modifyString: (source: string) => string) {
 export const camelCase = deepen((source) => source.replace(/[_-][a-z]/g, (str) => str.slice(1).toUpperCase()));
 export const paramCase = deepen((source) => source.replace(/_/g, '-').replace(/(?<!^)[A-Z]/g, (str) => `-${str.toLowerCase()}`));
 export const snakeCase = deepen((source) => source.replace(/-/g, '_').replace(/(?<!^)[A-Z]/g, (str) => `_${str.toLowerCase()}`));
-
-interface ProblemSource {
-    description?: string,
-    input?: string,
-    output?: string,
-    samples?: [string, string][],
-    hint?: string,
-    source?: string,
-}
-
-export function buildContent(source: ProblemSource, type: 'markdown' | 'html' = 'markdown', translate?: Function) {
-    const _ = translate || ((s: string) => s);
-    return type === 'html'
-        ? [
-            ...source.description ? [`<h2>${_('Description')}</h2>`, source.description] : [],
-            ...source.input ? [`<h2>${_('Input Format')}</h2>`, source.input] : [],
-            ...source.output ? [`<h2>'${_('Output Format')}</h2>`, source.output] : [],
-            ...(source.samples).map((sample, i) => [
-                `<h2>${_('Sample Input')} ${i + 1}</h2><pre>`,
-                sample[0],
-                `</pre><h2>${_('Sample Output')} ${i + 1}</h2><pre>`,
-                sample[1],
-                '</pre>',
-            ].join('\n')),
-            ...source.hint ? [`<h2>${_('Hint')}</h2>`, source.hint] : [],
-            ...source.source ? [`<h2>${_('Source')}</h2>`, source.source] : [],
-        ].join('\n')
-        : [
-            ...source.description ? [`## ${_('Description')}`, source.description] : [],
-            ...source.input ? [`## ${_('Input Format')}`, source.input] : [],
-            ...source.output ? [`## '${_('Output Format')}`, source.output] : [],
-            ...(source.samples).map((sample, i) => [
-                `## ${_('Sample Input')} ${i + 1}`,
-                '```',
-                sample[0],
-                '```',
-                `## ${_('Sample Output')} ${i + 1}`,
-                '```',
-                sample[1],
-                '```',
-            ].join('\n')),
-            ...source.hint ? [`## ${_('Hint')}`, source.hint] : [],
-            ...source.source ? [`## ${_('Source')}`, source.source] : [],
-        ].join('\n');
-}

packages/import-qduoj/handler.ts

@@ -5,12 +5,12 @@ import fs from 'fs-extra';
 import AdmZip from 'adm-zip';
 import yaml from 'js-yaml';
 import { ObjectID } from 'mongodb';
-import { LocalProblemConfig } from 'hydrooj';
+import { ContentNode, LocalProblemConfig } from 'hydrooj';
 import {
     Route, Handler, param, Types,
 } from 'hydrooj/dist/service/server';
 import { BadRequestError } from 'hydrooj/dist/error';
-import { buildContent, streamToBuffer } from 'hydrooj/dist/utils';
+import { streamToBuffer } from 'hydrooj/dist/utils';
 import { ProblemAdd } from 'hydrooj/dist/lib/ui';
 import * as file from 'hydrooj/dist/model/file';
 import * as problem from 'hydrooj/dist/model/problem';
@@ -31,14 +31,53 @@ class ImportQduojHandler extends Handler {
             for (const folder of folders) {
                 const buf = await fs.readFile(path.join(tmp, folder, 'problem.json'));
                 const pdoc = JSON.parse(buf.toString());
-                const content = buildContent({
+                const content: ContentNode[] = [];
-                    description: pdoc.description?.value,
+                if (pdoc.description?.value) {
-                    input: pdoc.input_description?.value,
+                    content.push({
-                    output: pdoc.output_description?.value,
+                        type: 'Text',
-                    samples: pdoc.samples.map((i) => [i.input, i.output]),
+                        subType: 'html',
-                    hint: pdoc.hint?.value,
+                        sectionTitle: this.translate('Description'),
-                    source: pdoc.source?.value,
+                        text: pdoc.description.value,
-                }, 'html');
+                    });
+                }
+                if (pdoc.input_description?.value) {
+                    content.push({
+                        type: 'Text',
+                        subType: 'html',
+                        sectionTitle: this.translate('Input Format'),
+                        text: pdoc.input_description.value,
+                    });
+                }
+                if (pdoc.output_description?.value) {
+                    content.push({
+                        type: 'Text',
+                        subType: 'html',
+                        sectionTitle: this.translate('Output Format'),
+                        text: pdoc.output_description.value,
+                    });
+                }
+                if (pdoc.samples?.length) {
+                    content.push(...pdoc.samples.map((sample) => ({
+                        type: 'Sample',
+                        payload: [sample.input, sample.output],
+                    })));
+                }
+                if (pdoc.hint?.value) {
+                    content.push({
+                        type: 'Text',
+                        subType: 'html',
+                        sectionTitle: this.translate('Hint'),
+                        text: pdoc.hint.value,
+                    });
+                }
+                if (pdoc.source?.value) {
+                    content.push({
+                        type: 'Text',
+                        subType: 'html',
+                        sectionTitle: this.translate('Source'),
+                        text: pdoc.source.value,
+                    });
+                }
                 const pid = await problem.add(domainId, pdoc.display_id, pdoc.title, content, this.user._id, pdoc.tags);
                 const testdata = new AdmZip();
                 const config: LocalProblemConfig = {

packages/import-qduoj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hydrooj/import-qduoj",
-    "version": "1.0.0",
+    "version": "1.1.0",
     "description": "Import QDUOJ problem export",
     "main": "package.json",
     "repository": "https://github.com/hydro-dev/Hydro.git",
@@ -15,7 +15,7 @@
         "bson": "^4.1.0"
     },
     "peerDependencies": {
-        "hydrooj": "^2.13.0"
+        "hydrooj": "^2.14.0"
     },
     "scripts": {
         "build": "tsc",