From 566ebd9bb2515ece7b52dfc895305bc3140d6ff5 Mon Sep 17 00:00:00 2001 From: undefined Date: Fri, 3 Jan 2025 01:01:37 +0800 Subject: [PATCH] framework: auto cleanup uploaded files (#818) --- framework/framework/server.ts | 19 +++++++++++++++++++ packages/fps-importer/index.ts | 8 ++++---- packages/hydrooj/src/handler/contest.ts | 3 +-- packages/hydrooj/src/handler/homework.ts | 3 +-- packages/hydrooj/src/handler/misc.ts | 3 +-- packages/hydrooj/src/handler/problem.ts | 7 ++++--- packages/hydrooj/src/handler/training.ts | 3 +-- packages/hydrooj/src/service/watcher.ts | 2 +- packages/import-hoj/index.ts | 8 ++++---- packages/import-qduoj/index.ts | 8 ++++---- 10 files changed, 40 insertions(+), 24 deletions(-) diff --git a/framework/framework/server.ts b/framework/framework/server.ts index 8f8cf1ad4..2eb45920a 100644 --- a/framework/framework/server.ts +++ b/framework/framework/server.ts @@ -86,6 +86,7 @@ export type KoaContext = Koa.Context & { handler: any; request: Koa.Request & { body: any, files: Files }; session: Record; + holdFiles: (string | File)[]; }; const logger = new Logger('server'); @@ -199,6 +200,10 @@ export class Handler extends HandlerCommon { if (name) this.response.disposition = `attachment; filename="${encodeRFC5987ValueChars(name)}"`; } + holdFile(name: string | File) { + this.context.holdFiles.push(name); + } + async init() { if (this.request.method === 'post' && this.request.headers.referer && !this.context.cors && !this.allowCors) { try { @@ -394,6 +399,20 @@ ${c.response.status} ${endTime - startTime}ms ${c.response.length}`); keepExtensions: true, }, })); + this.server.use(async (c, next) => { + c.holdFiles = []; + try { + await next(); + } finally { + if (Object.keys(c.request.files || {}).length) { + for (const k in c.request.files) { + if (c.holdFiles.includes(k)) continue; + const files = Array.isArray(c.request.files[k]) ? c.request.files[k] : [c.request.files[k]]; + for (const f of files) if (!c.holdFiles.includes(f as any)) fs.rmSync(f.filepath); + } + } + } + }); this.ctx.on('dispose', () => { fs.emptyDirSync(uploadDir); }); diff --git a/packages/fps-importer/index.ts b/packages/fps-importer/index.ts index 801813b38..94c7f9336 100644 --- a/packages/fps-importer/index.ts +++ b/packages/fps-importer/index.ts @@ -82,12 +82,12 @@ class FpsProblemImportHandler extends Handler { } async post({ domainId }) { - if (!this.request.files.file) throw new ValidationError('file'); + const file = this.request.files.file; + if (!file) throw new ValidationError('file'); const tasks = []; try { - const file = await fs.stat(this.request.files.file.filepath); if (file.size > SystemModel.get('import-fps.limit')) throw new FileTooLargeError(); - const content = fs.readFileSync(this.request.files.file.filepath, 'utf-8'); + const content = await fs.readFile(file.filepath, 'utf-8'); const result = await xml2js.parseStringPromise(content); tasks.push(result); } catch (e) { @@ -95,7 +95,7 @@ class FpsProblemImportHandler extends Handler { console.log(e); let zip: AdmZip; try { - zip = new AdmZip(this.request.files.file.filepath); + zip = new AdmZip(file.filepath); } catch (err) { throw new ValidationError('zip', null, err.message); } diff --git a/packages/hydrooj/src/handler/contest.ts b/packages/hydrooj/src/handler/contest.ts index 446830568..9ea106fe2 100644 --- a/packages/hydrooj/src/handler/contest.ts +++ b/packages/hydrooj/src/handler/contest.ts @@ -476,8 +476,7 @@ export class ContestManagementHandler extends ContestManagementBaseHandler { } const file = this.request.files?.file; if (!file) throw new ValidationError('file'); - const f = statSync(file.filepath); - const size = Math.sum((this.tdoc.files || []).map((i) => i.size)) + f.size; + const size = Math.sum((this.tdoc.files || []).map((i) => i.size)) + file.size; if (size >= system.get('limit.contest_files_size')) { throw new FileLimitExceededError('size'); } diff --git a/packages/hydrooj/src/handler/homework.ts b/packages/hydrooj/src/handler/homework.ts index cc3f1687a..c0c67fe5c 100644 --- a/packages/hydrooj/src/handler/homework.ts +++ b/packages/hydrooj/src/handler/homework.ts @@ -273,8 +273,7 @@ export class HomeworkFilesHandler extends Handler { } const file = this.request.files?.file; if (!file) throw new ValidationError('file'); - const f = statSync(file.filepath); - const size = Math.sum((this.tdoc.files || []).map((i) => i.size)) + f.size; + const size = Math.sum((this.tdoc.files || []).map((i) => i.size)) + file.size; if (size >= system.get('limit.contest_files_size')) { throw new FileLimitExceededError('size'); } diff --git a/packages/hydrooj/src/handler/misc.ts b/packages/hydrooj/src/handler/misc.ts index d695969aa..c3c39383d 100644 --- a/packages/hydrooj/src/handler/misc.ts +++ b/packages/hydrooj/src/handler/misc.ts @@ -53,8 +53,7 @@ export class FilesHandler extends Handler { } const file = this.request.files?.file; if (!file) throw new ValidationError('file'); - const f = statSync(file.filepath); - const size = Math.sum((this.user._files || []).map((i) => i.size)) + f.size; + const size = Math.sum((this.user._files || []).map((i) => i.size)) + file.size; if (size >= system.get('limit.user_files_size')) { if (!this.user.hasPriv(PRIV.PRIV_UNLIMITED_QUOTA)) throw new FileLimitExceededError('size'); } diff --git a/packages/hydrooj/src/handler/problem.ts b/packages/hydrooj/src/handler/problem.ts index 9c6898d6e..c1ba31f52 100644 --- a/packages/hydrooj/src/handler/problem.ts +++ b/packages/hydrooj/src/handler/problem.ts @@ -705,13 +705,14 @@ export class ProblemFilesHandler extends ProblemDetailHandler { @post('filename', Types.Filename, true) @post('type', Types.Range(['testdata', 'additional_file']), true) async postUploadFile(domainId: string, filename: string, type = 'testdata') { - if (!this.request.files.file) throw new ValidationError('file'); - filename ||= this.request.files.file.originalFilename || String.random(16); + const file = this.request.files.file; + if (!file) throw new ValidationError('file'); + filename ||= file.originalFilename || String.random(16); const files = []; if (filename.endsWith('.zip') && type === 'testdata') { let zip: AdmZip; try { - zip = new AdmZip(this.request.files.file.filepath); + zip = new AdmZip(file.filepath); } catch (e) { throw new ValidationError('zip', null, e.message); } diff --git a/packages/hydrooj/src/handler/training.ts b/packages/hydrooj/src/handler/training.ts index c616ca978..242c843e0 100644 --- a/packages/hydrooj/src/handler/training.ts +++ b/packages/hydrooj/src/handler/training.ts @@ -261,8 +261,7 @@ export class TrainingFilesHandler extends Handler { } const file = this.request.files?.file; if (!file) throw new ValidationError('file'); - const f = statSync(file.filepath); - const size = Math.sum((this.tdoc.files || []).map((i) => i.size)) + f.size; + const size = Math.sum((this.tdoc.files || []).map((i) => i.size)) + file.size; if (size >= system.get('limit.contest_files_size')) { throw new FileLimitExceededError('size'); } diff --git a/packages/hydrooj/src/service/watcher.ts b/packages/hydrooj/src/service/watcher.ts index 8de73ed8d..aac9fefcd 100644 --- a/packages/hydrooj/src/service/watcher.ts +++ b/packages/hydrooj/src/service/watcher.ts @@ -102,7 +102,7 @@ export default class Watcher extends Service { let hasUpdate = false; while (index < pending.length) { const filename = pending[index]; - const { children } = require.cache[filename]; + const children = require.cache[filename]?.children || []; let isDeclined = true; let isAccepted = false; for (const { filename } of children) { diff --git a/packages/import-hoj/index.ts b/packages/import-hoj/index.ts index ae6077f27..973fce63c 100644 --- a/packages/import-hoj/index.ts +++ b/packages/import-hoj/index.ts @@ -124,10 +124,10 @@ class ImportHojHandler extends Handler { } async post({ domainId }) { - if (!this.request.files.file) throw new ValidationError('file'); - const stat = await fs.stat(this.request.files.file.filepath); - if (stat.size > 128 * 1024 * 1024) throw new ValidationError('file', 'File too large'); - await this.fromFile(domainId, this.request.files.file.filepath); + const file = this.request.files.file; + if (!file) throw new ValidationError('file'); + if (file.size > 128 * 1024 * 1024) throw new ValidationError('file', 'File too large'); + await this.fromFile(domainId, file.filepath); this.response.redirect = this.url('problem_main'); } } diff --git a/packages/import-qduoj/index.ts b/packages/import-qduoj/index.ts index 51888a9b3..7abca129d 100644 --- a/packages/import-qduoj/index.ts +++ b/packages/import-qduoj/index.ts @@ -129,10 +129,10 @@ class ImportQduojHandler extends Handler { } async post({ domainId }) { - if (!this.request.files.file) throw new ValidationError('file'); - const stat = await fs.stat(this.request.files.file.filepath); - if (stat.size > 256 * 1024 * 1024) throw new FileTooLargeError('256m'); - await this.fromFile(domainId, this.request.files.file.filepath); + const file = this.request.files.file; + if (!file) throw new ValidationError('file'); + if (file.size > 256 * 1024 * 1024) throw new FileTooLargeError('256m'); + await this.fromFile(domainId, file.filepath); this.response.redirect = this.url('problem_main'); } }