diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index e09ad5bbd..f0b500954 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "2.13.61", + "version": "2.14.0", "bin": "bin/hydrooj.js", "main": "dist/loader.js", "typings": "dist/loader.d.ts", diff --git a/packages/hydrooj/src/entry/common.ts b/packages/hydrooj/src/entry/common.ts index 2b5e9ca4c..e0e98cef3 100644 --- a/packages/hydrooj/src/entry/common.ts +++ b/packages/hydrooj/src/entry/common.ts @@ -12,6 +12,7 @@ export const builtinLib = [ 'jwt', 'download', 'i18n', 'mail', 'useragent', 'crypto', 'misc', 'paginate', 'hash.hydro', 'rank', 'validator', 'ui', 'sysinfo', 'testdata.convert.ini', 'testdataConfig', + 'content', ]; export const builtinModel = [ diff --git a/packages/hydrooj/src/handler/import.syzoj.ts b/packages/hydrooj/src/handler/import.syzoj.ts index c8a23cc08..11276f0d3 100644 --- a/packages/hydrooj/src/handler/import.syzoj.ts +++ b/packages/hydrooj/src/handler/import.syzoj.ts @@ -3,16 +3,22 @@ import path from 'path'; import os from 'os'; import assert from 'assert'; import superagent from 'superagent'; +import { filter } from 'lodash'; +import AdmZip from 'adm-zip'; import { ValidationError, RemoteOnlineJudgeError } from '../error'; +import { streamToBuffer } from '../utils'; +import { Logger } from '../logger'; +import type { ContentNode } from '../interface'; import * as problem from '../model/problem'; import { PERM } from '../model/builtin'; import { - Route, Handler, Types, param, + Route, Handler, Types, param, post, } from '../service/server'; import { isPid } from '../lib/validator'; import download from '../lib/download'; -const RE_SYZOJ = /https?:\/\/([a-zA-Z0-9.]+)\/problem\/([0-9]+)\/?/i; +const RE_SYZOJ = /(https?):\/\/([a-zA-Z0-9.:]+)\/(problem|p)\/([0-9]+)\/?/i; +const logger = new Logger('import.syzoj'); class ProblemImportSYZOJHandler extends Handler { async get() { @@ -26,78 +32,164 @@ class ProblemImportSYZOJHandler extends Handler { }; } - @param('url', Types.String) + @post('url', Types.String, true) @param('pid', Types.String, true, isPid) @param('hidden', Types.Boolean) async post(domainId: string, url: string, targetPid: string, hidden = false) { assert(url.match(RE_SYZOJ), new ValidationError('url')); if (!url.endsWith('/')) url += '/'; - const [, host, pid] = RE_SYZOJ.exec(url); - const res = await superagent.get(`${url}export`); - assert(res.status === 200, new RemoteOnlineJudgeError('Cannot connect to target server')); - assert(res.body.success, new RemoteOnlineJudgeError((res.body.error || {}).message)); - const p = res.body.obj; - const content = []; - if (p.description) { - content.push( - `## ${this.translate('Description')}`, - p.description, + const [, protocol, host, n, pid] = RE_SYZOJ.exec(url); + if (n === 'p') { + // SYZOJ-NG + const result = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/getProblem`) + .send({ + displayId: +pid, + localizedContentsOfAllLocales: true, + tagsOfLocale: this.user.viewLang || this.session.viewLang, + samples: true, + testData: true, + additionalFiles: true, + }); + const content = {}; + for (const c of result.body.localizedContentsOfAllLocales) { + const sections = c.contentSections; + for (const section of sections) { + section.subType = 'markdown'; + if (section.type === 'Sample') { + section.payload = [ + result.body.samples[section.sampleId].inputData, + result.body.samples[section.sampleId].outputData, + ]; + delete section.sampleId; + } + } + content[c.locale] = sections; + } + const tags = result.body.tagsOfLocale.map((node) => node.name); + const title = [ + ...filter( + result.body.localizedContentsOfAllLocales, + (node) => node.locale === (this.user.viewLang || this.session.viewLang), + ), + ...result.body.localizedContentsOfAllLocales, + ][0].title; + const docId = await problem.add( + domainId, targetPid || `${host}_${pid}`, title, content, this.user._id, + tags || [], [], null, hidden, ); - } - if (p.input_format) { - content.push( - `## ${this.translate('Input Format')}`, - p.input_format, + (async () => { + const judge = result.body.judgeInfo; + const r = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/downloadProblemFiles`) + .send({ + problemId: +pid, + type: 'TestData', + filenameList: result.body.testData.map((node) => node.filename), + }); + const urls = {}; + for (const t of r.body.downloadInfo) urls[t.filename] = t.downloadUrl; + const zip = new AdmZip(); + for (const f of result.body.testData) { + // eslint-disable-next-line no-await-in-loop + const c = await superagent.get(urls[f.filename]); + // eslint-disable-next-line no-await-in-loop + zip.addFile(f.filename, await streamToBuffer(c)); + } + await problem.setTestdata(domainId, docId, zip.toBuffer()); + await problem.edit(domainId, docId, { + config: { + time: `${judge.timeLimit}ms`, + memory: `${judge.memoryLimit}m`, + // TODO other config + }, + }); + })().catch(logger.error); + this.response.body = { pid: pid || docId }; + this.response.redirect = this.url('problem_settings', { pid: pid || docId }); + } else { + const res = await superagent.get(`${url}export`); + assert(res.status === 200, new RemoteOnlineJudgeError('Cannot connect to target server')); + assert(res.body.success, new RemoteOnlineJudgeError((res.body.error || {}).message)); + const p = res.body.obj; + const content: ContentNode[] = []; + if (p.description) { + content.push({ + type: 'Text', + subType: 'markdown', + sectionTitle: this.translate('Description'), + text: p.description, + }); + } + if (p.input_format) { + content.push({ + type: 'Text', + subType: 'markdown', + sectionTitle: this.translate('Input Format'), + text: p.input_format, + }); + } + if (p.output_format) { + content.push({ + type: 'Text', + subType: 'markdown', + sectionTitle: this.translate('Output Format'), + text: p.output_format, + }); + } + if (p.example) { + content.push({ + type: 'Plain', + subType: 'markdown', + text: p.example, + }); + } + if (p.hint) { + content.push({ + type: 'Text', + subType: 'markdown', + sectionTitle: this.translate('Hint'), + text: p.hint, + }); + } + if (p.limit_and_hint) { + content.push({ + type: 'Text', + subType: 'markdown', + sectionTitle: this.translate('Limit And Hint'), + text: p.output_format, + }); + } + if (p.have_additional_file) { + content.push({ + type: 'Text', + subType: 'markdown', + sectionTitle: this.translate('Additional File'), + text: `${url}download/additional_file`, + }); + } + const docId = await problem.add( + domainId, targetPid || `${host}_${pid}`, p.title, content, this.user._id, + p.tags || [], [], null, hidden, ); + const r = download(`${url}testdata/download`); + const file = path.resolve(os.tmpdir(), 'hydro', `import_${domainId}_${docId}.zip`); + const w = fs.createWriteStream(file); + await new Promise((resolve, reject) => { + w.on('finish', resolve); + w.on('error', reject); + r.pipe(w); + }); + await problem.setTestdata(domainId, docId, file); + await problem.edit(domainId, docId, { + config: { + time: `${p.time_limit}ms`, + memory: `${p.memory_limit}m`, + filename: p.file_io_input_name.split('.')[0], + type: p.type === 'traditional' ? 'default' : p.type, + }, + }); + this.response.body = { pid: pid || docId }; + this.response.redirect = this.url('problem_settings', { pid: pid || docId }); } - if (p.output_format) { - content.push( - `## ${this.translate('Output Format')}`, - p.output_format, - ); - } - if (p.example) content.push(p.example); - if (p.hint) { - content.push( - `## ${this.translate('Hint')}`, - p.hint, - ); - } - if (p.limit_and_hint) { - content.push( - `## ${this.translate('Limit And Hint')}`, - p.limit_and_hint, - ); - } - if (p.have_additional_file) { - content.push( - `## ${this.translate('Additional File')}`, - `${url}download/additional_file`, - ); - } - const docId = await problem.add( - domainId, targetPid || `${host}_${pid}`, p.title, content.join(' \n'), this.user._id, - p.tags || [], [], null, hidden, - ); - const r = download(`${url}testdata/download`); - const file = path.resolve(os.tmpdir(), 'hydro', `import_${domainId}_${docId}.zip`); - const w = fs.createWriteStream(file); - await new Promise((resolve, reject) => { - w.on('finish', resolve); - w.on('error', reject); - r.pipe(w); - }); - await problem.setTestdata(domainId, docId, file); - await problem.edit(domainId, docId, { - config: { - time: `${p.time_limit}ms`, - memory: `${p.memory_limit}m`, - filename: p.file_io_input_name.split('.')[0], - type: p.type === 'traditional' ? 'default' : p.type, - }, - }); - this.response.body = { pid: pid || docId }; - this.response.redirect = this.url('problem_settings', { pid: pid || docId }); } } diff --git a/packages/hydrooj/src/handler/problem.ts b/packages/hydrooj/src/handler/problem.ts index fe8ade99b..fe460d945 100644 --- a/packages/hydrooj/src/handler/problem.ts +++ b/packages/hydrooj/src/handler/problem.ts @@ -437,6 +437,9 @@ export class ProblemEditHandler extends ProblemManageHandler { @param('content', Types.String, isContent) @post('pid', Types.String, isPid, true) async post(domainId: string, title: string, content: string, newPid: string = '') { + try { + content = JSON.parse(content); + } catch { /* Ignore */ } const $update: Partial = { title, content, pid: newPid }; let pdoc = await problem.get(domainId, this.request.params.pid); pdoc = await problem.edit(domainId, pdoc.docId, $update); @@ -646,6 +649,9 @@ export class ProblemCreateHandler extends Handler { @param('content', Types.String, isContent) @param('hidden', Types.Boolean) async post(domainId: string, title: string, pid: string, content: string, hidden = false) { + try { + content = JSON.parse(content); + } catch { /* Ignore */ } const docId = await problem.add( domainId, pid, title, content, this.user._id, [], [], null, hidden, diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index ab40be334..78d6334f9 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -156,6 +156,26 @@ export interface RemoteProblemConfig { export type ProblemConfig = LocalProblemConfig | RemoteProblemConfig; +export interface PlainContentNode { + type: 'Plain', + subType: 'html' | 'markdown', + text: string, +} +export interface TextContentNode { + type: 'Text', + subType: 'html' | 'markdown', + sectionTitle: string, + text: string, +} +export interface SampleContentNode { + type: 'Sample', + text: string, + sectionTitle: string, + payload: [string, string], +} +export type ContentNode = PlainContentNode | TextContentNode | SampleContentNode; +export type Content = string | ContentNode[] | Record + declare module './model/problem' { interface Pdoc { domainId: string, @@ -164,7 +184,7 @@ declare module './model/problem' { pid: string, owner: number, title: string, - content: string, + content: Content, nSubmit: number, nAccept: number, tag: string[], @@ -525,6 +545,7 @@ interface GeoIP { export interface Lib { download: typeof import('./lib/download'), + buildContent: typeof import('./lib/content').buildContent, 'hash.hydro': typeof import('./lib/hash.hydro'), i18n: typeof import('./lib/i18n'), jwt: typeof import('./lib/jwt'), diff --git a/packages/hydrooj/src/lib/content.ts b/packages/hydrooj/src/lib/content.ts new file mode 100644 index 000000000..6b712c7c2 --- /dev/null +++ b/packages/hydrooj/src/lib/content.ts @@ -0,0 +1,74 @@ +import { ContentNode } from '../interface'; + +export interface ProblemSource { + description?: string, + input?: string, + output?: string, + samples?: [string, string][], + hint?: string, + source?: string, +} + +export function buildContent(source: ProblemSource | ContentNode[], type: 'markdown' | 'html' = 'markdown', translate?: Function) { + const _ = translate || ((s: string) => s); + if (source instanceof Array) { + return type === 'html' + ? source.map((node) => [ + node.type !== 'Plain' ? `

${node.sectionTitle}

` : '', + node.type === 'Sample' + ? `

${_('Sample Input')}

${node.payload[0]}

${_('Sample Output')}

${node.payload[1]}
` + : '', + node.text, + ].join('\n')).join('\n') + : source.map((node) => [ + node.type !== 'Plain' ? `## ${node.sectionTitle}` : '', + ...node.type === 'Sample' + ? [ + `## ${_('Sample Input')}`, + '```', + node.payload[0], + '```', + `## ${_('Sample Output')}`, + '```', + node.payload[1], + '```', + ] + : [], + node.text, + ].join('\n')).join('\n'); + } + return type === 'html' + ? [ + ...source.description ? [`

${_('Description')}

`, source.description] : [], + ...source.input ? [`

${_('Input Format')}

`, source.input] : [], + ...source.output ? [`

'${_('Output Format')}

`, source.output] : [], + ...(source.samples).map((sample, i) => [ + `

${_('Sample Input')} ${i + 1}

`,
+                sample[0],
+                `

${_('Sample Output')} ${i + 1}

`,
+                sample[1],
+                '
', + ].join('\n')), + ...source.hint ? [`

${_('Hint')}

`, source.hint] : [], + ...source.source ? [`

${_('Source')}

`, source.source] : [], + ].join('\n') + : [ + ...source.description ? [`## ${_('Description')}`, source.description] : [], + ...source.input ? [`## ${_('Input Format')}`, source.input] : [], + ...source.output ? [`## '${_('Output Format')}`, source.output] : [], + ...(source.samples).map((sample, i) => [ + `## ${_('Sample Input')} ${i + 1}`, + '```', + sample[0], + '```', + `## ${_('Sample Output')} ${i + 1}`, + '```', + sample[1], + '```', + ].join('\n')), + ...source.hint ? [`## ${_('Hint')}`, source.hint] : [], + ...source.source ? [`## ${_('Source')}`, source.source] : [], + ].join('\n'); +} + +global.Hydro.lib.buildContent = buildContent; diff --git a/packages/hydrooj/src/model/document.ts b/packages/hydrooj/src/model/document.ts index 00ae1bfed..e0eacf48b 100644 --- a/packages/hydrooj/src/model/document.ts +++ b/packages/hydrooj/src/model/document.ts @@ -10,6 +10,7 @@ import { buildProjection } from '../utils'; import { NumberKeys, ArrayKeys, Projection } from '../typeutils'; import * as db from '../service/db'; import * as bus from '../service/bus'; +import { Content } from '../loader'; type DocID = ObjectID | string | number; @@ -46,19 +47,19 @@ export interface DocStatusType { } export async function add( - domainId: string, content: string, owner: number, + domainId: string, content: Content, owner: number, docType: T, docId: K, parentType?: DocType[T]['parentType'], parentId?: DocType[T]['parentId'], args?: Partial, ): Promise export async function add( - domainId: string, content: string, owner: number, + domainId: string, content: Content, owner: number, docType: T, docId: null, parentType?: DocType[T]['parentType'], parentId?: DocType[T]['parentId'], args?: Partial, ): Promise export async function add( - domainId: string, content: string, owner: number, + domainId: string, content: Content, owner: number, docType: number, docId: DocID = null, parentType: number | null = null, parentId: DocID = null, args: any = {}, diff --git a/packages/hydrooj/src/model/problem.ts b/packages/hydrooj/src/model/problem.ts index 5a239c5fe..ab97f53c4 100644 --- a/packages/hydrooj/src/model/problem.ts +++ b/packages/hydrooj/src/model/problem.ts @@ -6,6 +6,7 @@ import * as file from './file'; import * as document from './document'; import * as domain from './domain'; import { ProblemStatusDoc, ProblemDataSource, Pdict } from '../interface'; +import { Content } from '../loader'; import { NumberKeys, Projection } from '../typeutils'; import { ProblemNotFoundError } from '../error'; import * as testdataConfig from '../lib/testdataConfig'; @@ -78,7 +79,7 @@ export const PROJECTION_PUBLIC: Pdoc.Field[] = [ ]; export async function add( - domainId: string, pid: string = null, title: string, content: string, owner: number, + domainId: string, pid: string = null, title: string, content: Content, owner: number, tag: string[] = [], category: string[] = [], data: ProblemDataSource = null, hidden = false, ) { const pidCounter = await domain.inc(domainId, 'pidCounter', 1); diff --git a/packages/hydrooj/src/service/minio.ts.i b/packages/hydrooj/src/service/minio.ts.i new file mode 100644 index 000000000..053cc492a --- /dev/null +++ b/packages/hydrooj/src/service/minio.ts.i @@ -0,0 +1,319 @@ +import { URL } from 'url'; +import { Readable } from 'stream'; +import { v4 as UUID } from 'uuid'; +import { Client as MinioClient } from 'minio'; + +import { FileEntity } from './file.entity'; + +import { FileUploadInfoDto, SignedFileUploadRequestDto } from './dto'; + +// 10 minutes upload expire time +const FILE_UPLOAD_EXPIRE_TIME = 10 * 60; +// 20 minutes download expire time +const FILE_DOWNLOAD_EXPIRE_TIME = 20 * 60 * 60; + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent +function encodeRFC5987ValueChars(str: string) { + return ( + encodeURIComponent(str) + // Note that although RFC3986 reserves "!", RFC5987 does not, + // so we do not need to escape it + .replace(/['()]/g, escape) // i.e., %27 %28 %29 + .replace(/\*/g, '%2A') + // The following are not required for percent-encoding per RFC5987, + // so we can allow for a little better readability over the wire: |`^ + .replace(/%(?:7C|60|5E)/g, unescape) + ); +} + +interface MinioEndpointConfig { + endPoint: string; + port: number; + useSSL: boolean; +} + +function parseMainEndpointUrl(endpoint: string): MinioEndpointConfig { + const url = new URL(endpoint); + const result: Partial = {}; + if (url.pathname !== '/') throw new Error('Main MinIO endpoint URL of a sub-directory is not supported.'); + if (url.username || url.password || url.hash || url.search) { + throw new Error('Authorization, search parameters and hash are not supported for main MinIO endpoint URL.'); + } + if (url.protocol === 'http:') result.useSSL = false; + else if (url.protocol === 'https:') result.useSSL = true; + else { + throw new Error( + `Invalid protocol "${url.protocol}" for main MinIO endpoint URL. Only HTTP and HTTPS are supported.`, + ); + } + result.endPoint = url.hostname; + result.port = url.port ? Number(url.port) : result.useSSL ? 443 : 80; + return result as MinioEndpointConfig; +} + +function parseAlternativeEndpointUrl(endpoint: string): (originalUrl: string) => string { + if (!endpoint) return (originalUrl) => originalUrl; + const url = new URL(endpoint); + if (url.hash || url.search) { + throw new Error('Search parameters and hash are not supported for alternative MinIO endpoint URL.'); + } + if (!url.pathname.endsWith('/')) throw new Error("Alternative MinIO endpoint URL's pathname must ends with '/'."); + return (originalUrl) => { + const parsedOriginUrl = new URL(originalUrl); + return new URL(parsedOriginUrl.pathname.slice(1) + parsedOriginUrl.search + parsedOriginUrl.hash, url).toString(); + }; +} + +export enum AlternativeUrlFor { + User, + Judge +} + +const minioClient = new MinioClient({ + ...parseMainEndpointUrl(config.endpoint), + accessKey: +}) + +export class FileService { + private readonly minioClient: MinioClient; + + private readonly bucket: string; + + private readonly replaceWithAlternativeUrlFor: Record string>; + + constructor( + private readonly connection: Connection, + private readonly fileRepository: Repository, + private readonly configService: ConfigService, + ) { + const config = this.configService.config.services.minio; + + this.minioClient = new MinioClient({ + ...parseMainEndpointUrl(config.endpoint), + accessKey: config.accessKey, + secretKey: config.secretKey, + }); + this.bucket = config.bucket; + this.replaceWithAlternativeUrlFor = { + [AlternativeUrlFor.User]: parseAlternativeEndpointUrl(config.endpointForUser), + [AlternativeUrlFor.Judge]: parseAlternativeEndpointUrl(config.endpointForJudge), + }; + } + + fileExistsInMinio(uuid: string): Promise { + return new Promise((resolve) => + this.minioClient + .statObject(this.bucket, uuid) + .then(() => resolve(true)) + .catch(() => resolve(false))); + } + + async uploadFile(uuid: string, streamOrBufferOrFile: string | Buffer | Readable, retryCount = 10): Promise { + for (let i = 0; i < retryCount; i++) { + try { + /* eslint-disable no-await-in-loop */ + if (typeof streamOrBufferOrFile === 'string') { await this.minioClient.fPutObject(this.bucket, uuid, streamOrBufferOrFile, {}); } else await this.minioClient.putObject(this.bucket, uuid, streamOrBufferOrFile, {}); + /* eslint-enable no-await-in-loop */ + } catch (e) { + if (i === retryCount - 1) throw e; + else { + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => setTimeout(resolve, 1000)); + continue; + } + } + } + } + + async onModuleInit(): Promise { + let bucketExists: boolean; + try { + bucketExists = await this.minioClient.bucketExists(this.bucket); + } catch (e) { + throw new Error( + `Error initializing the MinIO client. Please check your configuration file and MinIO server. ${e}`, + ); + } + + if (!bucketExists) { + throw new Error( + `MinIO bucket ${this.bucket} doesn't exist. Please check your configuration file and MinIO server.`, + ); + } + } + + /** + * Process the client's request about uploading a file. + * + * * If the user has not uploaded the file, return a signed upload info object. + * * If the user has uploaded the file, check the file existance, file size and limits. + * + * @note + * The `checkLimit` function may return different result for each call with the same upload info. + * + * e.g. When a user uploaded other files before upload this file, the quota is enough before uploading but + * not enough after uploading. + * + * If the file is checked to be exceeding the limit after uploaded, it will be deleted. + * + * @return A `SignedFileUploadRequestDto` to return to the client if the file is not uploaded. + * @return `string` error message if client says the file is uploaded but we do not accept it according to some errors. + * @return A `FileEntity` if the file is uploaded successfully and saved to the database. + */ + async processUploadRequest( + uploadInfo: FileUploadInfoDto, + checkLimit: (size: number) => Promise | LimitCheckErrorType, + transactionalEntityManager: EntityManager, + ): Promise { + const limitCheckError = await checkLimit(uploadInfo.size); + if (limitCheckError) { + this.deleteUnfinishedUploadedFile(uploadInfo.uuid); + return limitCheckError; + } + + if (uploadInfo.uuid) { + // The client says the file is uploaded + + if ((await transactionalEntityManager.count(FileEntity, { uuid: uploadInfo.uuid })) !== 0) return 'FILE_UUID_EXISTS'; + + // Check file existance + try { + await this.minioClient.statObject(this.bucket, uploadInfo.uuid); + } catch (e) { + if (e.message === 'The specified key does not exist.') { + return 'FILE_NOT_UPLOADED'; + } + throw e; + } + + // Save to the database + const file = new FileEntity(); + file.uuid = uploadInfo.uuid; + file.size = uploadInfo.size; + file.uploadTime = new Date(); + + await transactionalEntityManager.save(FileEntity, file); + + return file; + } + // The client says it want to upload a file for this request + + return await this.signUploadRequest(uploadInfo.size, uploadInfo.size); + } + + /** + * Sign a upload request for given size. The alternative MinIO endpoint for user will be used in the POST URL. + */ + private async signUploadRequest(minSize?: number, maxSize?: number): Promise { + const uuid = UUID(); + const policy = this.minioClient.newPostPolicy(); + policy.setBucket(this.bucket); + policy.setKey(uuid); + policy.setExpires(new Date(Date.now() + FILE_UPLOAD_EXPIRE_TIME * 1000)); + if (minSize != null || maxSize != null) { + policy.setContentLengthRange(minSize || 0, maxSize || 0); + } + const policyResult = await this.minioClient.presignedPostPolicy(policy); + + return { + uuid, + method: 'POST', + url: this.replaceWithAlternativeUrlFor[AlternativeUrlFor.User](policyResult.postURL), + extraFormData: policyResult.formData, + fileFieldName: 'file', + }; + } + + /** + * @return A function to run after transaction, to delete the file(s) actually. + */ + async deleteFile(uuid: string | string[], transactionalEntityManager: EntityManager): Promise<() => void> { + if (typeof uuid === 'string') { + await transactionalEntityManager.delete(FileEntity, { uuid }); + return () => + this.minioClient.removeObject(this.bucket, uuid).catch((e) => { + logger.error(`Failed to delete file ${uuid}: ${e}`); + }); + } + if (uuid.length > 0) { + await transactionalEntityManager.delete(FileEntity, { uuid: In(uuid) }); + return () => + this.minioClient.removeObjects(this.bucket, uuid).catch((e) => { + logger.error(`Failed to delete file [${uuid}]: ${e}`); + }); + } + return () => { + /* do nothing */ + }; + } + + /** + * Delete a user-uploaded file before calling finishUpload() + */ + deleteUnfinishedUploadedFile(uuid: string): void { + this.minioClient.removeObject(this.bucket, uuid).catch((e) => { + if (e.message === 'The specified key does not exist.') return; + logger.error(`Failed to delete unfinished uploaded file ${uuid}: ${e}`); + }); + } + + async getFileSizes(uuids: string[], transcationalEntityManager: EntityManager): Promise { + if (uuids.length === 0) return []; + const uniqueUuids = Array.from(new Set(uuids)); + const files = await transcationalEntityManager.find(FileEntity, { + uuid: In(uniqueUuids), + }); + const map = Object.fromEntries(files.map((file) => [file.uuid, file])); + return uuids.map((uuid) => map[uuid].size); + } + + async signDownloadLink({ + uuid, + downloadFilename, + noExpire, + useAlternativeEndpointFor, + }: { + uuid: string; + downloadFilename?: string; + noExpire?: boolean; + useAlternativeEndpointFor?: AlternativeUrlFor; + }): Promise { + const url = await this.minioClient.presignedGetObject( + this.bucket, + uuid, + // The maximum expire time is 7 days + noExpire ? 24 * 60 * 60 * 7 : FILE_DOWNLOAD_EXPIRE_TIME, + !downloadFilename + ? {} + : { + 'response-content-disposition': `attachment; filename="${encodeRFC5987ValueChars(downloadFilename)}"`, + }, + ); + + if (useAlternativeEndpointFor != null) return this.replaceWithAlternativeUrlFor[useAlternativeEndpointFor](url); + return url; + } + + async runMaintainceTasks(): Promise { + // Delete unused files + // TODO: Use listObjectsV2 instead, which returns at most 1000 objects in a time + const stream = this.minioClient.listObjects(this.bucket); + const deleteList: string[] = []; + await new Promise((resolve, reject) => { + const promises: Promise[] = []; + stream.on('data', (object) => { + promises.push( + (async () => { + const uuid = object.name; + if (!(await this.fileRepository.count({ uuid }))) { + deleteList.push(uuid); + } + })(), + ); + }); + stream.on('end', () => Promise.all(promises).then(resolve).catch(reject)); + stream.on('error', reject); + }); + await this.minioClient.removeObjects(this.bucket, deleteList); + } +} diff --git a/packages/hydrooj/src/utils.ts b/packages/hydrooj/src/utils.ts index a47df6f51..f17cb9c03 100644 --- a/packages/hydrooj/src/utils.ts +++ b/packages/hydrooj/src/utils.ts @@ -4,6 +4,7 @@ import cluster from 'cluster'; import path from 'path'; import serialize from 'serialize-javascript'; import Lru from 'lru-cache'; +import type { ContentNode } from './interface'; declare global { interface StringConstructor { @@ -224,48 +225,3 @@ function deepen(modifyString: (source: string) => string) { export const camelCase = deepen((source) => source.replace(/[_-][a-z]/g, (str) => str.slice(1).toUpperCase())); export const paramCase = deepen((source) => source.replace(/_/g, '-').replace(/(? `-${str.toLowerCase()}`)); export const snakeCase = deepen((source) => source.replace(/-/g, '_').replace(/(? `_${str.toLowerCase()}`)); - -interface ProblemSource { - description?: string, - input?: string, - output?: string, - samples?: [string, string][], - hint?: string, - source?: string, -} - -export function buildContent(source: ProblemSource, type: 'markdown' | 'html' = 'markdown', translate?: Function) { - const _ = translate || ((s: string) => s); - return type === 'html' - ? [ - ...source.description ? [`

${_('Description')}

`, source.description] : [], - ...source.input ? [`

${_('Input Format')}

`, source.input] : [], - ...source.output ? [`

'${_('Output Format')}

`, source.output] : [], - ...(source.samples).map((sample, i) => [ - `

${_('Sample Input')} ${i + 1}

`,
-                sample[0],
-                `

${_('Sample Output')} ${i + 1}

`,
-                sample[1],
-                '
', - ].join('\n')), - ...source.hint ? [`

${_('Hint')}

`, source.hint] : [], - ...source.source ? [`

${_('Source')}

`, source.source] : [], - ].join('\n') - : [ - ...source.description ? [`## ${_('Description')}`, source.description] : [], - ...source.input ? [`## ${_('Input Format')}`, source.input] : [], - ...source.output ? [`## '${_('Output Format')}`, source.output] : [], - ...(source.samples).map((sample, i) => [ - `## ${_('Sample Input')} ${i + 1}`, - '```', - sample[0], - '```', - `## ${_('Sample Output')} ${i + 1}`, - '```', - sample[1], - '```', - ].join('\n')), - ...source.hint ? [`## ${_('Hint')}`, source.hint] : [], - ...source.source ? [`## ${_('Source')}`, source.source] : [], - ].join('\n'); -} diff --git a/packages/import-qduoj/handler.ts b/packages/import-qduoj/handler.ts index 22256f3cf..8b9ba1a19 100644 --- a/packages/import-qduoj/handler.ts +++ b/packages/import-qduoj/handler.ts @@ -5,12 +5,12 @@ import fs from 'fs-extra'; import AdmZip from 'adm-zip'; import yaml from 'js-yaml'; import { ObjectID } from 'mongodb'; -import { LocalProblemConfig } from 'hydrooj'; +import { ContentNode, LocalProblemConfig } from 'hydrooj'; import { Route, Handler, param, Types, } from 'hydrooj/dist/service/server'; import { BadRequestError } from 'hydrooj/dist/error'; -import { buildContent, streamToBuffer } from 'hydrooj/dist/utils'; +import { streamToBuffer } from 'hydrooj/dist/utils'; import { ProblemAdd } from 'hydrooj/dist/lib/ui'; import * as file from 'hydrooj/dist/model/file'; import * as problem from 'hydrooj/dist/model/problem'; @@ -31,14 +31,53 @@ class ImportQduojHandler extends Handler { for (const folder of folders) { const buf = await fs.readFile(path.join(tmp, folder, 'problem.json')); const pdoc = JSON.parse(buf.toString()); - const content = buildContent({ - description: pdoc.description?.value, - input: pdoc.input_description?.value, - output: pdoc.output_description?.value, - samples: pdoc.samples.map((i) => [i.input, i.output]), - hint: pdoc.hint?.value, - source: pdoc.source?.value, - }, 'html'); + const content: ContentNode[] = []; + if (pdoc.description?.value) { + content.push({ + type: 'Text', + subType: 'html', + sectionTitle: this.translate('Description'), + text: pdoc.description.value, + }); + } + if (pdoc.input_description?.value) { + content.push({ + type: 'Text', + subType: 'html', + sectionTitle: this.translate('Input Format'), + text: pdoc.input_description.value, + }); + } + if (pdoc.output_description?.value) { + content.push({ + type: 'Text', + subType: 'html', + sectionTitle: this.translate('Output Format'), + text: pdoc.output_description.value, + }); + } + if (pdoc.samples?.length) { + content.push(...pdoc.samples.map((sample) => ({ + type: 'Sample', + payload: [sample.input, sample.output], + }))); + } + if (pdoc.hint?.value) { + content.push({ + type: 'Text', + subType: 'html', + sectionTitle: this.translate('Hint'), + text: pdoc.hint.value, + }); + } + if (pdoc.source?.value) { + content.push({ + type: 'Text', + subType: 'html', + sectionTitle: this.translate('Source'), + text: pdoc.source.value, + }); + } const pid = await problem.add(domainId, pdoc.display_id, pdoc.title, content, this.user._id, pdoc.tags); const testdata = new AdmZip(); const config: LocalProblemConfig = { diff --git a/packages/import-qduoj/package.json b/packages/import-qduoj/package.json index 7fa16abad..c38bac2b0 100644 --- a/packages/import-qduoj/package.json +++ b/packages/import-qduoj/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/import-qduoj", - "version": "1.0.0", + "version": "1.1.0", "description": "Import QDUOJ problem export", "main": "package.json", "repository": "https://github.com/hydro-dev/Hydro.git", @@ -15,10 +15,10 @@ "bson": "^4.1.0" }, "peerDependencies": { - "hydrooj": "^2.13.0" + "hydrooj": "^2.14.0" }, "scripts": { "build": "tsc", "lint": "eslint . --ext ts --fix" } -} +} \ No newline at end of file