From a7fd32cac953a98d60d158ea922eff593bcc8dd9 Mon Sep 17 00:00:00 2001 From: undefined Date: Tue, 20 Apr 2021 21:56:31 +0800 Subject: [PATCH] judge: auto-remove extra files (#104) --- packages/hydrojudge/package.json | 2 +- packages/hydrojudge/src/hosts/hydro.ts | 5 +++++ packages/hydrojudge/src/service.ts | 6 ++++++ packages/hydrooj/package.json | 2 +- packages/hydrooj/src/handler/problem.ts | 1 + packages/hydrooj/src/service/storage.ts | 12 ++++++++++++ 6 files changed, 26 insertions(+), 2 deletions(-) diff --git a/packages/hydrojudge/package.json b/packages/hydrojudge/package.json index 03a546a48..927b69675 100644 --- a/packages/hydrojudge/package.json +++ b/packages/hydrojudge/package.json @@ -1,7 +1,7 @@ { "name": "@hydrooj/hydrojudge", "bin": "bin/hydrojudge.js", - "version": "2.5.8", + "version": "2.5.9", "main": "package.json", "author": "undefined ", "repository": "https://github.com/hydro-dev/Hydro.git", diff --git a/packages/hydrojudge/src/hosts/hydro.ts b/packages/hydrojudge/src/hosts/hydro.ts index 3cff1747c..177b32265 100644 --- a/packages/hydrojudge/src/hosts/hydro.ts +++ b/packages/hydrojudge/src/hosts/hydro.ts @@ -202,10 +202,15 @@ export default class Hydro { } catch (e) { /* ignore */ } const version = {}; const filenames = []; + const allFiles = new Set(); for (const file of files) { + allFiles.add(file.name); version[file.name] = file.etag; if (etags[file.name] !== file.etag) filenames.push(file.name); } + for (const name in etags) { + if (!allFiles.has(name)) await fs.rm(path.join(filePath, name)); + } if (filenames.length) { log.info(`Getting problem data: ${this.config.host}/${domainId}/${pid}`); if (next) next({ judge_text: '正在同步测试数据，请稍后' }); diff --git a/packages/hydrojudge/src/service.ts b/packages/hydrojudge/src/service.ts index 9253939b4..fa8b896ab 100644 --- a/packages/hydrojudge/src/service.ts +++ b/packages/hydrojudge/src/service.ts @@ -24,6 +24,7 @@ declare module 'hydrooj/dist/interface' { 'hydrojudge.tmpfs_size': string, 'hydrojudge.retry_delay_sec': number, 'hydrojudge.sandbox_host': string, + 'hydrojudge.memoryMax': string, 'hydrojudge.testcases_max': number, 'hydrojudge.total_time_limit': number, 'hydrojudge.parallelism': number, @@ -81,12 +82,17 @@ async function postInit() { etags = JSON.parse(fs.readFileSync(path.join(filePath, 'etags')).toString()); } catch (e) { /* ignore */ } const version = {}; + const filenames = new Set(); for (const file of files) { + filenames.add(file.name); version[file.name] = file.etag; if (etags[file.name] !== file.etag) { await storage.get(`problem/${domainId}/${pid}/testdata/${file.name}`, path.join(filePath, file.name)); } } + for (const name in etags) { + if (!filenames.has(name)) await fs.rm(path.join(filePath, name)); + } fs.writeFileSync(path.join(filePath, 'etags'), JSON.stringify(version)); fs.writeFileSync(path.join(filePath, 'lastUsage'), new Date().getTime().toString()); await processData(filePath).catch(noop); diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index 5d9498e44..d514060fc 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "2.23.9", + "version": "2.23.10", "bin": "bin/hydrooj.js", "main": "dist/loader.js", "typings": "dist/loader.d.ts", diff --git a/packages/hydrooj/src/handler/problem.ts b/packages/hydrooj/src/handler/problem.ts index fb60e6d36..9a979cdda 100644 --- a/packages/hydrooj/src/handler/problem.ts +++ b/packages/hydrooj/src/handler/problem.ts @@ -394,6 +394,7 @@ export class ProblemFilesHandler extends ProblemDetailHandler { async postUploadFile(domainId: string, filename: string, type = 'testdata') { if (!this.request.files.file) throw new ValidationError('file'); if (!filename) filename = this.request.files.file.name || String.random(16); + if (filename.includes('/')) throw new ValidationError('filename', 'Bad filename'); if (this.pdoc.owner !== this.user._id) this.checkPerm(PERM.PERM_EDIT_PROBLEM); if (filename.endsWith('.zip')) { const zip = new AdmZip(this.request.files.file.path); diff --git a/packages/hydrooj/src/service/storage.ts b/packages/hydrooj/src/service/storage.ts index 80d2cda9c..46407451e 100644 --- a/packages/hydrooj/src/service/storage.ts +++ b/packages/hydrooj/src/service/storage.ts @@ -115,6 +115,7 @@ class StorageService { } async put(target: string, file: string | Buffer | Readable, meta: ItemBucketMetadata = {}) { + if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); if (typeof file === 'string') file = createReadStream(file); try { return await this.client.putObject(this.opts.bucket, target, file, meta); @@ -125,6 +126,7 @@ class StorageService { } async get(target: string, path?: string) { + if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); try { if (path) return await this.client.fGetObject(this.opts.bucket, target, path); return await this.client.getObject(this.opts.bucket, target); @@ -135,6 +137,13 @@ class StorageService { } async del(target: string | string[]) { + if (typeof target === 'string') { + if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); + } else { + for (const t of target) { + if (t.includes('..') || t.includes('//')) throw new Error('Invalid path'); + } + } try { if (typeof target === 'string') return await this.client.removeObject(this.opts.bucket, target); return await this.client.removeObjects(this.opts.bucket, target); @@ -145,6 +154,7 @@ class StorageService { } async list(target: string, recursive = true) { + if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); try { const stream = this.client.listObjects(this.opts.bucket, target, recursive); return await new Promise((resolve, reject) => { @@ -168,6 +178,7 @@ class StorageService { } async getMeta(target: string) { + if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); try { const result = await this.client.statObject(this.opts.bucket, target); return { ...result.metaData, ...result }; @@ -178,6 +189,7 @@ class StorageService { } async signDownloadLink(target: string, filename?: string, noExpire = false, useAlternativeEndpointFor?: 'user' | 'judge'): Promise { + if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); try { const url = await this.client.presignedGetObject( this.opts.bucket,