From 902393e587d126fdbf9cf5946d2147b516adcbf2 Mon Sep 17 00:00:00 2001 From: undefined Date: Mon, 11 Apr 2022 16:20:50 +0800 Subject: [PATCH] core: record file operator --- packages/hydrooj/package.json | 2 +- packages/hydrooj/src/handler/misc.ts | 6 ++--- packages/hydrooj/src/handler/problem.ts | 14 +++++----- packages/hydrooj/src/interface.ts | 18 +++++++------ packages/hydrooj/src/model/problem.ts | 36 ++++++++++++++----------- packages/hydrooj/src/model/storage.ts | 24 ++++++++++++----- 6 files changed, 59 insertions(+), 41 deletions(-) diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index 801cbfe58..e692a0611 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "3.11.16", + "version": "3.11.17", "bin": "bin/hydrooj.js", "main": "src/loader", "module": "src/loader", diff --git a/packages/hydrooj/src/handler/misc.ts b/packages/hydrooj/src/handler/misc.ts index 384f9c95d..5c1a98d00 100644 --- a/packages/hydrooj/src/handler/misc.ts +++ b/packages/hydrooj/src/handler/misc.ts @@ -58,7 +58,7 @@ export class FilesHandler extends Handler { if (!filename) filename = this.request.files.file.name || String.random(16); if (filename.includes('/') || filename.includes('..')) throw new ValidationError('filename', null, 'Bad filename'); if (this.user._files.filter((i) => i.name === filename).length) throw new BadRequestError('file exists'); - await storage.put(`user/${this.user._id}/${filename}`, this.request.files.file.path); + await storage.put(`user/${this.user._id}/${filename}`, this.request.files.file.path, this.user._id); const meta = await storage.getMeta(`user/${this.user._id}/${filename}`); const payload = { name: filename, ...pick(meta, ['size', 'lastModified', 'etag']) }; if (!meta) throw new Error('Upload failed'); @@ -70,14 +70,14 @@ export class FilesHandler extends Handler { @post('from', Types.String) @post('to', Types.String) async postRename(domainId: string, from: string, to: string) { - await storage.rename(`user/${this.user._id}/${from}`, `user/${this.user._id}/${to}`); + await storage.rename(`user/${this.user._id}/${from}`, `user/${this.user._id}/${to}`, this.user._id); this.back(); } @post('files', Types.Array) async postDeleteFiles(domainId: string, files: string[]) { await Promise.all([ - storage.del(files.map((t) => `user/${this.user._id}/${t}`)), + storage.del(files.map((t) => `user/${this.user._id}/${t}`), this.user._id), user.setById(this.user._id, { _files: this.user._files.filter((i) => !files.includes(i.name)) }), ]); this.back(); diff --git a/packages/hydrooj/src/handler/problem.ts b/packages/hydrooj/src/handler/problem.ts index 5822f4fcc..496d16821 100644 --- a/packages/hydrooj/src/handler/problem.ts +++ b/packages/hydrooj/src/handler/problem.ts @@ -488,7 +488,7 @@ export class ProblemSubmitHandler extends ProblemDetailHandler { if (file.size > sizeLimit) throw new ValidationError('file'); if (file.size > 65535 || await isBinaryFile(file.path, file.size)) { const id = nanoid(); - await storage.put(`submission/${this.user._id}/${id}`, file.path); + await storage.put(`submission/${this.user._id}/${id}`, file.path, this.user._id); code = `@@hydro_submission_file@@${this.user._id}/${id}#${file.name}`; } else { // TODO auto detect & convert encoding @@ -606,7 +606,7 @@ export class ProblemFilesHandler extends ProblemDetailHandler { @post('filename', Types.Name, true) @post('type', Types.Range(['testdata', 'additional_file']), true) async postUploadFile(domainId: string, filename: string, type = 'testdata') { - if (this.pdoc.reference) throw new ForbiddenError('Cannot delete files of a referenced problem.'); + if (this.pdoc.reference) throw new ForbiddenError('Cannot edit files of a referenced problem.'); if (!this.request.files.file) throw new ValidationError('file'); if (!filename) filename = this.request.files.file.name || String.random(16); if (filename.includes('/') || filename.includes('..')) throw new ValidationError('filename', null, 'Bad filename'); @@ -651,10 +651,10 @@ export class ProblemFilesHandler extends ProblemDetailHandler { for (const entry of files) { if (entry.type === 'testdata') { // eslint-disable-next-line no-await-in-loop - await problem.addTestdata(domainId, this.pdoc.docId, entry.name, entry.data()); + await problem.addTestdata(domainId, this.pdoc.docId, entry.name, entry.data(), this.user._id); } else { // eslint-disable-next-line no-await-in-loop - await problem.addAdditionalFile(domainId, this.pdoc.docId, entry.name, entry.data()); + await problem.addAdditionalFile(domainId, this.pdoc.docId, entry.name, entry.data(), this.user._id); } } this.back(); @@ -665,8 +665,8 @@ export class ProblemFilesHandler extends ProblemDetailHandler { async postDeleteFiles(domainId: string, files: string[], type = 'testdata') { if (this.pdoc.reference) throw new ForbiddenError('Cannot delete files of a referenced problem.'); if (!this.user.own(this.pdoc, PERM.PERM_EDIT_PROBLEM_SELF)) this.checkPerm(PERM.PERM_EDIT_PROBLEM); - if (type === 'testdata') await problem.delTestdata(domainId, this.pdoc.docId, files); - else await problem.delAdditionalFile(domainId, this.pdoc.docId, files); + if (type === 'testdata') await problem.delTestdata(domainId, this.pdoc.docId, files, this.user._id); + else await problem.delAdditionalFile(domainId, this.pdoc.docId, files, this.user._id); this.back(); } } @@ -852,7 +852,7 @@ export class ProblemCreateHandler extends Handler { for (const file of files) { if (this.user._files.find((i) => i.name === file)) { tasks.push( - storage.rename(`user/${this.user._id}/${file}`, `problem/${domainId}/${docId}/additional_file/${file}`) + storage.rename(`user/${this.user._id}/${file}`, `problem/${domainId}/${docId}/additional_file/${file}`, this.user._id) .then(() => problem.addAdditionalFile(domainId, docId, file, '', true)), user.setById(this.user._id, { _files: this.user._files.filter((i) => i.name !== file) }), ); diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index b2492784e..f6ea50295 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -548,17 +548,19 @@ export interface BaseService { export interface FileNode { /** File Path In MinIO */ - _id: string + _id: string; /** Actual File Path */ - path: string - lastUsage?: Date - lastModified?: Date - etag?: string + path: string; + lastUsage?: Date; + lastModified?: Date; + etag?: string; /** Size: in bytes */ - size?: number + size?: number; /** AutoDelete */ - autoDelete?: Date - meta?: ItemBucketMetadata, + autoDelete?: Date; + owner?: number; + operator?: number[]; + meta?: ItemBucketMetadata; } export interface EventDoc { diff --git a/packages/hydrooj/src/model/problem.ts b/packages/hydrooj/src/model/problem.ts index 9cd261b3d..7af4d44aa 100644 --- a/packages/hydrooj/src/model/problem.ts +++ b/packages/hydrooj/src/model/problem.ts @@ -240,11 +240,11 @@ export class ProblemModel { return !!res[0][0].deletedCount; } - static async addTestdata(domainId: string, pid: number, name: string, f: Readable | Buffer | string) { + static async addTestdata(domainId: string, pid: number, name: string, f: Readable | Buffer | string, operator = 1) { if (!name) throw new ValidationError('name'); const [[, fileinfo]] = await Promise.all([ document.getSub(domainId, document.TYPE_PROBLEM, pid, 'data', name), - storage.put(`problem/${domainId}/${pid}/testdata/${name}`, f), + storage.put(`problem/${domainId}/${pid}/testdata/${name}`, f, operator), ]); const meta = await storage.getMeta(`problem/${domainId}/${pid}/testdata/${name}`); if (!meta) throw new Error('Upload failed'); @@ -255,17 +255,22 @@ export class ProblemModel { await bus.emit('problem/addTestdata', domainId, pid, name, payload); } - static async delTestdata(domainId: string, pid: number, name: string | string[]) { + static async delTestdata(domainId: string, pid: number, name: string | string[], operator = 1) { const names = (name instanceof Array) ? name : [name]; - await storage.del(names.map((t) => `problem/${domainId}/${pid}/testdata/${t}`)); - await ProblemModel.pull(domainId, pid, 'data', names); + await Promise.all([ + storage.del(names.map((t) => `problem/${domainId}/${pid}/testdata/${t}`), operator), + ProblemModel.pull(domainId, pid, 'data', names), + ]); await bus.emit('problem/delTestdata', domainId, pid, names); } - static async addAdditionalFile(domainId: string, pid: number, name: string, f: Readable | Buffer | string, skipUpload = false) { + static async addAdditionalFile( + domainId: string, pid: number, name: string, + f: Readable | Buffer | string, operator = 1, skipUpload = false, + ) { const [[, fileinfo]] = await Promise.all([ document.getSub(domainId, document.TYPE_PROBLEM, pid, 'additional_file', name), - skipUpload ? '' : storage.put(`problem/${domainId}/${pid}/additional_file/${name}`, f), + skipUpload ? '' : storage.put(`problem/${domainId}/${pid}/additional_file/${name}`, f, operator), ]); const meta = await storage.getMeta(`problem/${domainId}/${pid}/additional_file/${name}`); const payload = { name, ...pick(meta, ['size', 'lastModified', 'etag']) }; @@ -274,20 +279,21 @@ export class ProblemModel { await bus.emit('problem/addAdditionalFile', domainId, pid, name, payload); } - static async delAdditionalFile(domainId: string, pid: number, name: MaybeArray) { + static async delAdditionalFile(domainId: string, pid: number, name: MaybeArray, operator = 1) { const names = (name instanceof Array) ? name : [name]; - await storage.del(names.map((t) => `problem/${domainId}/${pid}/additional_file/${t}`)); - await ProblemModel.pull(domainId, pid, 'additional_file', names); + await Promise.all([ + storage.del(names.map((t) => `problem/${domainId}/${pid}/additional_file/${t}`), operator), + ProblemModel.pull(domainId, pid, 'additional_file', names), + ]); await bus.emit('problem/delAdditionalFile', domainId, pid, names); } - static async random(domainId: string, query: FilterQuery): Promise { + static async random(domainId: string, query: FilterQuery) { const cursor = document.getMulti(domainId, document.TYPE_PROBLEM, query); const pcount = await cursor.count(); - if (pcount) { - const pdoc = await cursor.skip(Math.floor(Math.random() * pcount)).limit(1).toArray(); - return pdoc[0].pid || pdoc[0].docId; - } return null; + if (!pcount) return null; + const pdoc = await cursor.skip(Math.floor(Math.random() * pcount)).limit(1).toArray(); + return pdoc[0].pid || pdoc[0].docId; } static async getList( diff --git a/packages/hydrooj/src/model/storage.ts b/packages/hydrooj/src/model/storage.ts index a589c70f3..eec781a54 100644 --- a/packages/hydrooj/src/model/storage.ts +++ b/packages/hydrooj/src/model/storage.ts @@ -8,21 +8,29 @@ import type { Readable } from 'stream'; import * as bus from '../service/bus'; import db from '../service/db'; import storage from '../service/storage'; +import * as system from './system'; import TaskModel from './task'; export class StorageModel { static coll = db.collection('storage'); - static async put(path: string, file: string | Buffer | Readable, meta: ItemBucketMetadata = {}) { + static async put(path: string, file: string | Buffer | Readable, meta?: ItemBucketMetadata, owner?: number); + static async put(path: string, file: string | Buffer | Readable, owner?: number); + static async put(path: string, file: string | Buffer | Readable, arg0?: ItemBucketMetadata | number, arg1?: number) { + const meta = typeof arg0 === 'object' ? arg0 : {}; + const owner = (typeof arg0 === 'number' ? arg0 : arg1) ?? 1; await StorageModel.del([path]); meta['Content-Type'] = (path.endsWith('.ans') || path.endsWith('.out')) ? 'text/plain' : lookup(path) || 'application/octet-stream'; - const _id = `${nanoid(3)}/${nanoid()}${extname(path)}`; + let _id = `${nanoid(3)}/${nanoid()}${extname(path)}`; + // Make sure id is not used + // eslint-disable-next-line no-await-in-loop + while (await StorageModel.coll.findOne({ _id })) _id = `${nanoid(3)}/${nanoid()}${extname(path)}`; await storage.put(_id, file, meta); const { metaData, size, etag } = await storage.getMeta(_id); await StorageModel.coll.insertOne({ - _id, meta: metaData, path, size, etag, lastModified: new Date(), + _id, meta: metaData, path, size, etag, lastModified: new Date(), owner, }); return path; } @@ -36,18 +44,19 @@ export class StorageModel { return await storage.get(value?._id || path, savePath); } - static async rename(path: string, newPath: string) { + static async rename(path: string, newPath: string, operator = 1) { return await StorageModel.coll.updateOne( { path, autoDelete: null }, - { $set: { path: newPath } }, + { $set: { path: newPath }, $push: { operator } }, ); } - static async del(path: string[]) { + static async del(path: string[], operator = 1) { + if (!path.length) return; const autoDelete = moment().add(7, 'day').toDate(); await StorageModel.coll.updateMany( { path: { $in: path }, autoDelete: null }, - { $set: { autoDelete } }, + { $set: { autoDelete }, $push: { operator } }, ); } @@ -93,6 +102,7 @@ export class StorageModel { } async function cleanFiles() { + if (system.get('server.keepFiles')) return; let res = await StorageModel.coll.findOneAndDelete({ autoDelete: { $lte: new Date() } }); while (res.value) { // eslint-disable-next-line no-await-in-loop