framework: auto cleanup uploaded files (#818)

framework/framework/server.ts

@@ -86,6 +86,7 @@ export type KoaContext = Koa.Context & {
     handler: any;
     request: Koa.Request & { body: any, files: Files };
     session: Record<string, any>;
+    holdFiles: (string | File)[];
 };
 
 const logger = new Logger('server');
@@ -199,6 +200,10 @@ export class Handler extends HandlerCommon {
         if (name) this.response.disposition = `attachment; filename="${encodeRFC5987ValueChars(name)}"`;
     }
 
+    holdFile(name: string | File) {
+        this.context.holdFiles.push(name);
+    }
+
     async init() {
         if (this.request.method === 'post' && this.request.headers.referer && !this.context.cors && !this.allowCors) {
             try {
@@ -394,6 +399,20 @@ ${c.response.status} ${endTime - startTime}ms ${c.response.length}`);
                     keepExtensions: true,
                 },
             }));
+            this.server.use(async (c, next) => {
+                c.holdFiles = [];
+                try {
+                    await next();
+                } finally {
+                    if (Object.keys(c.request.files || {}).length) {
+                        for (const k in c.request.files) {
+                            if (c.holdFiles.includes(k)) continue;
+                            const files = Array.isArray(c.request.files[k]) ? c.request.files[k] : [c.request.files[k]];
+                            for (const f of files) if (!c.holdFiles.includes(f as any)) fs.rmSync(f.filepath);
+                        }
+                    }
+                }
+            });
             this.ctx.on('dispose', () => {
                 fs.emptyDirSync(uploadDir);
             });

packages/fps-importer/index.ts

@@ -82,12 +82,12 @@ class FpsProblemImportHandler extends Handler {
     }
 
     async post({ domainId }) {
-        if (!this.request.files.file) throw new ValidationError('file');
+        const file = this.request.files.file;
+        if (!file) throw new ValidationError('file');
         const tasks = [];
         try {
-            const file = await fs.stat(this.request.files.file.filepath);
             if (file.size > SystemModel.get('import-fps.limit')) throw new FileTooLargeError();
-            const content = fs.readFileSync(this.request.files.file.filepath, 'utf-8');
+            const content = await fs.readFile(file.filepath, 'utf-8');
             const result = await xml2js.parseStringPromise(content);
             tasks.push(result);
         } catch (e) {
@@ -95,7 +95,7 @@ class FpsProblemImportHandler extends Handler {
             console.log(e);
             let zip: AdmZip;
             try {
-                zip = new AdmZip(this.request.files.file.filepath);
+                zip = new AdmZip(file.filepath);
             } catch (err) {
                 throw new ValidationError('zip', null, err.message);
             }

packages/hydrooj/src/handler/contest.ts

@@ -476,8 +476,7 @@ export class ContestManagementHandler extends ContestManagementBaseHandler {
         }
         const file = this.request.files?.file;
         if (!file) throw new ValidationError('file');
-        const f = statSync(file.filepath);
+        const size = Math.sum((this.tdoc.files || []).map((i) => i.size)) + file.size;
-        const size = Math.sum((this.tdoc.files || []).map((i) => i.size)) + f.size;
         if (size >= system.get('limit.contest_files_size')) {
             throw new FileLimitExceededError('size');
         }

packages/hydrooj/src/handler/homework.ts

@@ -273,8 +273,7 @@ export class HomeworkFilesHandler extends Handler {
         }
         const file = this.request.files?.file;
         if (!file) throw new ValidationError('file');
-        const f = statSync(file.filepath);
+        const size = Math.sum((this.tdoc.files || []).map((i) => i.size)) + file.size;
-        const size = Math.sum((this.tdoc.files || []).map((i) => i.size)) + f.size;
         if (size >= system.get('limit.contest_files_size')) {
             throw new FileLimitExceededError('size');
         }

packages/hydrooj/src/handler/misc.ts

@@ -53,8 +53,7 @@ export class FilesHandler extends Handler {
         }
         const file = this.request.files?.file;
         if (!file) throw new ValidationError('file');
-        const f = statSync(file.filepath);
+        const size = Math.sum((this.user._files || []).map((i) => i.size)) + file.size;
-        const size = Math.sum((this.user._files || []).map((i) => i.size)) + f.size;
         if (size >= system.get('limit.user_files_size')) {
             if (!this.user.hasPriv(PRIV.PRIV_UNLIMITED_QUOTA)) throw new FileLimitExceededError('size');
         }

packages/hydrooj/src/handler/problem.ts

@@ -705,13 +705,14 @@ export class ProblemFilesHandler extends ProblemDetailHandler {
     @post('filename', Types.Filename, true)
     @post('type', Types.Range(['testdata', 'additional_file']), true)
     async postUploadFile(domainId: string, filename: string, type = 'testdata') {
-        if (!this.request.files.file) throw new ValidationError('file');
+        const file = this.request.files.file;
-        filename ||= this.request.files.file.originalFilename || String.random(16);
+        if (!file) throw new ValidationError('file');
+        filename ||= file.originalFilename || String.random(16);
         const files = [];
         if (filename.endsWith('.zip') && type === 'testdata') {
             let zip: AdmZip;
             try {
-                zip = new AdmZip(this.request.files.file.filepath);
+                zip = new AdmZip(file.filepath);
             } catch (e) {
                 throw new ValidationError('zip', null, e.message);
             }

packages/hydrooj/src/handler/training.ts

@@ -261,8 +261,7 @@ export class TrainingFilesHandler extends Handler {
         }
         const file = this.request.files?.file;
         if (!file) throw new ValidationError('file');
-        const f = statSync(file.filepath);
+        const size = Math.sum((this.tdoc.files || []).map((i) => i.size)) + file.size;
-        const size = Math.sum((this.tdoc.files || []).map((i) => i.size)) + f.size;
         if (size >= system.get('limit.contest_files_size')) {
             throw new FileLimitExceededError('size');
         }

packages/hydrooj/src/service/watcher.ts

@@ -102,7 +102,7 @@ export default class Watcher extends Service {
             let hasUpdate = false;
             while (index < pending.length) {
                 const filename = pending[index];
-                const { children } = require.cache[filename];
+                const children = require.cache[filename]?.children || [];
                 let isDeclined = true;
                 let isAccepted = false;
                 for (const { filename } of children) {

packages/import-hoj/index.ts

@@ -124,10 +124,10 @@ class ImportHojHandler extends Handler {
     }
 
     async post({ domainId }) {
-        if (!this.request.files.file) throw new ValidationError('file');
+        const file = this.request.files.file;
-        const stat = await fs.stat(this.request.files.file.filepath);
+        if (!file) throw new ValidationError('file');
-        if (stat.size > 128 * 1024 * 1024) throw new ValidationError('file', 'File too large');
+        if (file.size > 128 * 1024 * 1024) throw new ValidationError('file', 'File too large');
-        await this.fromFile(domainId, this.request.files.file.filepath);
+        await this.fromFile(domainId, file.filepath);
         this.response.redirect = this.url('problem_main');
     }
 }

packages/import-qduoj/index.ts

@@ -129,10 +129,10 @@ class ImportQduojHandler extends Handler {
     }
 
     async post({ domainId }) {
-        if (!this.request.files.file) throw new ValidationError('file');
+        const file = this.request.files.file;
-        const stat = await fs.stat(this.request.files.file.filepath);
+        if (!file) throw new ValidationError('file');
-        if (stat.size > 256 * 1024 * 1024) throw new FileTooLargeError('256m');
+        if (file.size > 256 * 1024 * 1024) throw new FileTooLargeError('256m');
-        await this.fromFile(domainId, this.request.files.file.filepath);
+        await this.fromFile(domainId, file.filepath);
         this.response.redirect = this.url('problem_main');
     }
 }