From 7d1ddff47b209fb3e2e5a4141ddf3978c2866572 Mon Sep 17 00:00:00 2001 From: undefined Date: Mon, 7 Dec 2020 12:52:43 +0800 Subject: [PATCH] judge: update limit --- packages/hydrojudge/package.json | 2 +- packages/hydrojudge/src/cases.ts | 8 +- packages/hydrojudge/src/hosts/hydro.ts | 4 +- packages/hydrojudge/src/hosts/vj4.ts | 6 +- packages/hydrojudge/src/judge/hack.ts | 4 +- packages/hydrojudge/src/service.ts | 4 +- packages/hydrooj/package.json | 2 +- packages/hydrooj/src/handler/problem.ts | 2 +- .../hydrooj/src/lib/testdata.convert.ini.ts | 2 +- packages/hydrooj/src/service/minio.ts.i | 319 ------------------ 10 files changed, 17 insertions(+), 336 deletions(-) delete mode 100644 packages/hydrooj/src/service/minio.ts.i diff --git a/packages/hydrojudge/package.json b/packages/hydrojudge/package.json index dfa11886c..6bf9c3bf8 100644 --- a/packages/hydrojudge/package.json +++ b/packages/hydrojudge/package.json @@ -1,7 +1,7 @@ { "name": "@hydrooj/hydrojudge", "bin": "bin/hydrojudge.js", - "version": "2.1.30", + "version": "2.1.31", "main": "package.json", "author": "masnn", "repository": "https://github.com/hydro-dev/Hydro.git", diff --git a/packages/hydrojudge/src/cases.ts b/packages/hydrojudge/src/cases.ts index 29c75c5e4..002067a9d 100644 --- a/packages/hydrojudge/src/cases.ts +++ b/packages/hydrojudge/src/cases.ts @@ -87,7 +87,7 @@ async function read0(folder: string, files: string[], checkFile) { count: 0, subtasks: [{ time_limit_ms: 1000, - memory_limit_mb: 256, + memory_limit_mb: 512, type: 'sum', cases: [], score: Math.floor(100 / cases.length), @@ -104,7 +104,7 @@ async function read0(folder: string, files: string[], checkFile) { if (extra < cases.length) { config.subtasks.push({ time_limit_ms: 1000, - memory_limit_mb: 256, + memory_limit_mb: 512, type: 'sum', cases: [], score: Math.floor(100 / cases.length) + 1, @@ -133,7 +133,7 @@ async function read1(folder: string, files: string[], checkFile) { if (!subtask[REG.subtask(data)]) { subtask[REG.subtask(data)] = [{ time_limit_ms: 1000, - memory_limit_mb: 256, + memory_limit_mb: 512, type: 'min', cases: [c], }]; @@ -276,7 +276,7 @@ function convertIniConfig(ini: string) { cases: [{ input: `input/${input.toLowerCase()}`, output: `output/${output.toLowerCase()}` }], score: parseInt(score, 10), time: `${time}s`, - memory: '256m', + memory: '512m', }; if (!Number.isNaN(parseInt(memory, 10))) cur.memory = `${Math.floor(parseInt(memory, 10) / 1024)}m`; res.subtasks.push(cur); diff --git a/packages/hydrojudge/src/hosts/hydro.ts b/packages/hydrojudge/src/hosts/hydro.ts index 21c20785b..be8f7320e 100644 --- a/packages/hydrojudge/src/hosts/hydro.ts +++ b/packages/hydrojudge/src/hosts/hydro.ts @@ -77,7 +77,7 @@ class JudgeTask { this.tmpdir = path.resolve(TEMP_DIR, 'tmp', this.host, this.rid); this.clean = []; fs.ensureDirSync(this.tmpdir); - tmpfs.mount(this.tmpdir, '256m'); + tmpfs.mount(this.tmpdir, '512m'); log.info('Submission: %s/%s/%s pid=%s', this.host, this.domainId, this.rid, this.pid); try { if (this.type === 'judge') await this.doSubmission(); @@ -231,7 +231,7 @@ export default class Hydro { fs.ensureDirSync(path.dirname(savePath)); const zip = new AdmZip(tmpFilePath); const entries = zip.getEntries(); - if (entries.length > 256) throw new FormatError('Too many files'); + if (entries.length > 512) throw new FormatError('Too many files'); await new Promise((resolve, reject) => { zip.extractAllToAsync(savePath, true, (e) => { if (e) reject(e); diff --git a/packages/hydrojudge/src/hosts/vj4.ts b/packages/hydrojudge/src/hosts/vj4.ts index a66c64bec..7efc9bc08 100644 --- a/packages/hydrojudge/src/hosts/vj4.ts +++ b/packages/hydrojudge/src/hosts/vj4.ts @@ -74,7 +74,7 @@ class JudgeTask { this.tmpdir = path.resolve(TEMP_DIR, 'tmp', this.host, this.rid); this.clean = []; fs.ensureDirSync(this.tmpdir); - tmpfs.mount(this.tmpdir, '256m'); + tmpfs.mount(this.tmpdir, '512m'); log.info('Submission: %s/%s/%s pid=%s', this.host, this.domain_id, this.rid, this.pid); try { if (this.type === 0) await this.doSubmission(); @@ -256,7 +256,7 @@ export default class VJ4 { fs.ensureDirSync(path.dirname(savePath)); const zip = new AdmZip(tmpFilePath); const entries = zip.getEntries(); - if (entries.length > 256) throw new FormatError('Too many files'); + if (entries.length > 512) throw new FormatError('Too many files'); await new Promise((resolve, reject) => { zip.extractAllToAsync(savePath, true, (e) => { if (e) reject(e); @@ -286,7 +286,7 @@ export default class VJ4 { fs.ensureDirSync(path.dirname(savePath)); const zip = new AdmZip(tmpFilePath); const entries = zip.getEntries(); - if (entries.length > 256) throw new FormatError('Too many files'); + if (entries.length > 512) throw new FormatError('Too many files'); await new Promise((resolve, reject) => { zip.extractAllToAsync(savePath, true, (e) => { if (e) reject(e); diff --git a/packages/hydrojudge/src/judge/hack.ts b/packages/hydrojudge/src/judge/hack.ts index 2abe8d350..fd6acce64 100644 --- a/packages/hydrojudge/src/judge/hack.ts +++ b/packages/hydrojudge/src/judge/hack.ts @@ -78,7 +78,7 @@ export const judge = async (ctx) => { copyIn: execute.copyIn, copyOut, time_limit_ms: 1000, - memory_limit_mb: 256, + memory_limit_mb: 512, }, ); const { code, time_usage_ms, memory_usage_kb } = res; @@ -90,7 +90,7 @@ export const judge = async (ctx) => { if (status === STATUS.STATUS_ACCEPTED) { if (time_usage_ms > 1000) { status = STATUS.STATUS_TIME_LIMIT_EXCEEDED; - } else if (memory_usage_kb > 256 * 1024) { + } else if (memory_usage_kb > 512 * 1024) { status = STATUS.STATUS_MEMORY_LIMIT_EXCEEDED; } else { [status, , message] = await check({ diff --git a/packages/hydrojudge/src/service.ts b/packages/hydrojudge/src/service.ts index e52fdf85d..56e6e17d0 100644 --- a/packages/hydrojudge/src/service.ts +++ b/packages/hydrojudge/src/service.ts @@ -78,7 +78,7 @@ async function postInit() { fs.ensureDirSync(path.dirname(savePath)); const zip = new AdmZip(tmpFilePath); const entries = zip.getEntries(); - if (entries.length > 256) throw new FormatError('Too many files.'); + if (entries.length > 512) throw new FormatError('Too many files.'); await new Promise((resolve, reject) => { zip.extractAllToAsync(savePath, true, (e) => { if (e) reject(e); @@ -217,7 +217,7 @@ async function postInit() { this.tmpdir = path.resolve(tmpdir(), 'tmp', this.rid); this.clean = []; fs.ensureDirSync(this.tmpdir); - tmpfs.mount(this.tmpdir, '256m'); + tmpfs.mount(this.tmpdir, '512m'); logger.info(`Submission: ${this.rid}`, { pid: this.pid }); if (this.config.input) await this.run(); else if (this.config.hack) await this.hack(); diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index 02a2d91b5..d04a7972a 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "2.14.1", + "version": "2.14.2", "bin": "bin/hydrooj.js", "main": "dist/loader.js", "typings": "dist/loader.d.ts", diff --git a/packages/hydrooj/src/handler/problem.ts b/packages/hydrooj/src/handler/problem.ts index fe460d945..7e3103447 100644 --- a/packages/hydrooj/src/handler/problem.ts +++ b/packages/hydrooj/src/handler/problem.ts @@ -302,7 +302,7 @@ export class ProblemPretestHandler extends ProblemDetailHandler { { input, time: '1s', - memory: '256m', + memory: '512m', }, ); const rdoc = await record.get(domainId, rid); diff --git a/packages/hydrooj/src/lib/testdata.convert.ini.ts b/packages/hydrooj/src/lib/testdata.convert.ini.ts index 281f46a92..8a13249ee 100644 --- a/packages/hydrooj/src/lib/testdata.convert.ini.ts +++ b/packages/hydrooj/src/lib/testdata.convert.ini.ts @@ -10,7 +10,7 @@ function convert(ini: string) { cases: [{ input: `input/${input}`, output: `output/${output}` }], score: parseInt(score, 10), time: `${time}s`, - memory: '256m', + memory: '512m', }; if (!Number.isNaN(parseInt(memory, 10))) cur.memory = `${Math.floor(parseInt(memory, 10) / 1024)}m`; res.subtasks.push(cur); diff --git a/packages/hydrooj/src/service/minio.ts.i b/packages/hydrooj/src/service/minio.ts.i deleted file mode 100644 index 053cc492a..000000000 --- a/packages/hydrooj/src/service/minio.ts.i +++ /dev/null @@ -1,319 +0,0 @@ -import { URL } from 'url'; -import { Readable } from 'stream'; -import { v4 as UUID } from 'uuid'; -import { Client as MinioClient } from 'minio'; - -import { FileEntity } from './file.entity'; - -import { FileUploadInfoDto, SignedFileUploadRequestDto } from './dto'; - -// 10 minutes upload expire time -const FILE_UPLOAD_EXPIRE_TIME = 10 * 60; -// 20 minutes download expire time -const FILE_DOWNLOAD_EXPIRE_TIME = 20 * 60 * 60; - -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent -function encodeRFC5987ValueChars(str: string) { - return ( - encodeURIComponent(str) - // Note that although RFC3986 reserves "!", RFC5987 does not, - // so we do not need to escape it - .replace(/['()]/g, escape) // i.e., %27 %28 %29 - .replace(/\*/g, '%2A') - // The following are not required for percent-encoding per RFC5987, - // so we can allow for a little better readability over the wire: |`^ - .replace(/%(?:7C|60|5E)/g, unescape) - ); -} - -interface MinioEndpointConfig { - endPoint: string; - port: number; - useSSL: boolean; -} - -function parseMainEndpointUrl(endpoint: string): MinioEndpointConfig { - const url = new URL(endpoint); - const result: Partial = {}; - if (url.pathname !== '/') throw new Error('Main MinIO endpoint URL of a sub-directory is not supported.'); - if (url.username || url.password || url.hash || url.search) { - throw new Error('Authorization, search parameters and hash are not supported for main MinIO endpoint URL.'); - } - if (url.protocol === 'http:') result.useSSL = false; - else if (url.protocol === 'https:') result.useSSL = true; - else { - throw new Error( - `Invalid protocol "${url.protocol}" for main MinIO endpoint URL. Only HTTP and HTTPS are supported.`, - ); - } - result.endPoint = url.hostname; - result.port = url.port ? Number(url.port) : result.useSSL ? 443 : 80; - return result as MinioEndpointConfig; -} - -function parseAlternativeEndpointUrl(endpoint: string): (originalUrl: string) => string { - if (!endpoint) return (originalUrl) => originalUrl; - const url = new URL(endpoint); - if (url.hash || url.search) { - throw new Error('Search parameters and hash are not supported for alternative MinIO endpoint URL.'); - } - if (!url.pathname.endsWith('/')) throw new Error("Alternative MinIO endpoint URL's pathname must ends with '/'."); - return (originalUrl) => { - const parsedOriginUrl = new URL(originalUrl); - return new URL(parsedOriginUrl.pathname.slice(1) + parsedOriginUrl.search + parsedOriginUrl.hash, url).toString(); - }; -} - -export enum AlternativeUrlFor { - User, - Judge -} - -const minioClient = new MinioClient({ - ...parseMainEndpointUrl(config.endpoint), - accessKey: -}) - -export class FileService { - private readonly minioClient: MinioClient; - - private readonly bucket: string; - - private readonly replaceWithAlternativeUrlFor: Record string>; - - constructor( - private readonly connection: Connection, - private readonly fileRepository: Repository, - private readonly configService: ConfigService, - ) { - const config = this.configService.config.services.minio; - - this.minioClient = new MinioClient({ - ...parseMainEndpointUrl(config.endpoint), - accessKey: config.accessKey, - secretKey: config.secretKey, - }); - this.bucket = config.bucket; - this.replaceWithAlternativeUrlFor = { - [AlternativeUrlFor.User]: parseAlternativeEndpointUrl(config.endpointForUser), - [AlternativeUrlFor.Judge]: parseAlternativeEndpointUrl(config.endpointForJudge), - }; - } - - fileExistsInMinio(uuid: string): Promise { - return new Promise((resolve) => - this.minioClient - .statObject(this.bucket, uuid) - .then(() => resolve(true)) - .catch(() => resolve(false))); - } - - async uploadFile(uuid: string, streamOrBufferOrFile: string | Buffer | Readable, retryCount = 10): Promise { - for (let i = 0; i < retryCount; i++) { - try { - /* eslint-disable no-await-in-loop */ - if (typeof streamOrBufferOrFile === 'string') { await this.minioClient.fPutObject(this.bucket, uuid, streamOrBufferOrFile, {}); } else await this.minioClient.putObject(this.bucket, uuid, streamOrBufferOrFile, {}); - /* eslint-enable no-await-in-loop */ - } catch (e) { - if (i === retryCount - 1) throw e; - else { - // eslint-disable-next-line no-await-in-loop - await new Promise((resolve) => setTimeout(resolve, 1000)); - continue; - } - } - } - } - - async onModuleInit(): Promise { - let bucketExists: boolean; - try { - bucketExists = await this.minioClient.bucketExists(this.bucket); - } catch (e) { - throw new Error( - `Error initializing the MinIO client. Please check your configuration file and MinIO server. ${e}`, - ); - } - - if (!bucketExists) { - throw new Error( - `MinIO bucket ${this.bucket} doesn't exist. Please check your configuration file and MinIO server.`, - ); - } - } - - /** - * Process the client's request about uploading a file. - * - * * If the user has not uploaded the file, return a signed upload info object. - * * If the user has uploaded the file, check the file existance, file size and limits. - * - * @note - * The `checkLimit` function may return different result for each call with the same upload info. - * - * e.g. When a user uploaded other files before upload this file, the quota is enough before uploading but - * not enough after uploading. - * - * If the file is checked to be exceeding the limit after uploaded, it will be deleted. - * - * @return A `SignedFileUploadRequestDto` to return to the client if the file is not uploaded. - * @return `string` error message if client says the file is uploaded but we do not accept it according to some errors. - * @return A `FileEntity` if the file is uploaded successfully and saved to the database. - */ - async processUploadRequest( - uploadInfo: FileUploadInfoDto, - checkLimit: (size: number) => Promise | LimitCheckErrorType, - transactionalEntityManager: EntityManager, - ): Promise { - const limitCheckError = await checkLimit(uploadInfo.size); - if (limitCheckError) { - this.deleteUnfinishedUploadedFile(uploadInfo.uuid); - return limitCheckError; - } - - if (uploadInfo.uuid) { - // The client says the file is uploaded - - if ((await transactionalEntityManager.count(FileEntity, { uuid: uploadInfo.uuid })) !== 0) return 'FILE_UUID_EXISTS'; - - // Check file existance - try { - await this.minioClient.statObject(this.bucket, uploadInfo.uuid); - } catch (e) { - if (e.message === 'The specified key does not exist.') { - return 'FILE_NOT_UPLOADED'; - } - throw e; - } - - // Save to the database - const file = new FileEntity(); - file.uuid = uploadInfo.uuid; - file.size = uploadInfo.size; - file.uploadTime = new Date(); - - await transactionalEntityManager.save(FileEntity, file); - - return file; - } - // The client says it want to upload a file for this request - - return await this.signUploadRequest(uploadInfo.size, uploadInfo.size); - } - - /** - * Sign a upload request for given size. The alternative MinIO endpoint for user will be used in the POST URL. - */ - private async signUploadRequest(minSize?: number, maxSize?: number): Promise { - const uuid = UUID(); - const policy = this.minioClient.newPostPolicy(); - policy.setBucket(this.bucket); - policy.setKey(uuid); - policy.setExpires(new Date(Date.now() + FILE_UPLOAD_EXPIRE_TIME * 1000)); - if (minSize != null || maxSize != null) { - policy.setContentLengthRange(minSize || 0, maxSize || 0); - } - const policyResult = await this.minioClient.presignedPostPolicy(policy); - - return { - uuid, - method: 'POST', - url: this.replaceWithAlternativeUrlFor[AlternativeUrlFor.User](policyResult.postURL), - extraFormData: policyResult.formData, - fileFieldName: 'file', - }; - } - - /** - * @return A function to run after transaction, to delete the file(s) actually. - */ - async deleteFile(uuid: string | string[], transactionalEntityManager: EntityManager): Promise<() => void> { - if (typeof uuid === 'string') { - await transactionalEntityManager.delete(FileEntity, { uuid }); - return () => - this.minioClient.removeObject(this.bucket, uuid).catch((e) => { - logger.error(`Failed to delete file ${uuid}: ${e}`); - }); - } - if (uuid.length > 0) { - await transactionalEntityManager.delete(FileEntity, { uuid: In(uuid) }); - return () => - this.minioClient.removeObjects(this.bucket, uuid).catch((e) => { - logger.error(`Failed to delete file [${uuid}]: ${e}`); - }); - } - return () => { - /* do nothing */ - }; - } - - /** - * Delete a user-uploaded file before calling finishUpload() - */ - deleteUnfinishedUploadedFile(uuid: string): void { - this.minioClient.removeObject(this.bucket, uuid).catch((e) => { - if (e.message === 'The specified key does not exist.') return; - logger.error(`Failed to delete unfinished uploaded file ${uuid}: ${e}`); - }); - } - - async getFileSizes(uuids: string[], transcationalEntityManager: EntityManager): Promise { - if (uuids.length === 0) return []; - const uniqueUuids = Array.from(new Set(uuids)); - const files = await transcationalEntityManager.find(FileEntity, { - uuid: In(uniqueUuids), - }); - const map = Object.fromEntries(files.map((file) => [file.uuid, file])); - return uuids.map((uuid) => map[uuid].size); - } - - async signDownloadLink({ - uuid, - downloadFilename, - noExpire, - useAlternativeEndpointFor, - }: { - uuid: string; - downloadFilename?: string; - noExpire?: boolean; - useAlternativeEndpointFor?: AlternativeUrlFor; - }): Promise { - const url = await this.minioClient.presignedGetObject( - this.bucket, - uuid, - // The maximum expire time is 7 days - noExpire ? 24 * 60 * 60 * 7 : FILE_DOWNLOAD_EXPIRE_TIME, - !downloadFilename - ? {} - : { - 'response-content-disposition': `attachment; filename="${encodeRFC5987ValueChars(downloadFilename)}"`, - }, - ); - - if (useAlternativeEndpointFor != null) return this.replaceWithAlternativeUrlFor[useAlternativeEndpointFor](url); - return url; - } - - async runMaintainceTasks(): Promise { - // Delete unused files - // TODO: Use listObjectsV2 instead, which returns at most 1000 objects in a time - const stream = this.minioClient.listObjects(this.bucket); - const deleteList: string[] = []; - await new Promise((resolve, reject) => { - const promises: Promise[] = []; - stream.on('data', (object) => { - promises.push( - (async () => { - const uuid = object.name; - if (!(await this.fileRepository.count({ uuid }))) { - deleteList.push(uuid); - } - })(), - ); - }); - stream.on('end', () => Promise.all(promises).then(resolve).catch(reject)); - stream.on('error', reject); - }); - await this.minioClient.removeObjects(this.bucket, deleteList); - } -}