From fc2ac236570f8518b1709579be50a1efed20a31e Mon Sep 17 00:00:00 2001 From: undefined Date: Sat, 22 Jun 2024 23:35:07 +0800 Subject: [PATCH] core: storage: use soft copy (#830) --- packages/hydrooj/src/interface.ts | 2 ++ packages/hydrooj/src/model/problem.ts | 3 ++- packages/hydrooj/src/model/storage.ts | 26 ++++++++++++++++++------- packages/hydrooj/src/service/storage.ts | 22 ++------------------- 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 5ce54e0de..572ea162d 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -644,6 +644,8 @@ export interface FileNode { size?: number; /** AutoDelete */ autoDelete?: Date; + /** fileId if linked to an existing file */ + link?: string; owner?: number; operator?: number[]; meta?: Record; diff --git a/packages/hydrooj/src/model/problem.ts b/packages/hydrooj/src/model/problem.ts index b372c0fad..618c3f6ae 100644 --- a/packages/hydrooj/src/model/problem.ts +++ b/packages/hydrooj/src/model/problem.ts @@ -262,7 +262,8 @@ export class ProblemModel { const res = await Promise.all([ document.deleteOne(domainId, document.TYPE_PROBLEM, docId), document.deleteMultiStatus(domainId, document.TYPE_PROBLEM, { docId }), - storage.list(`problem/${domainId}/${docId}/`).then((items) => storage.del(items.map((item) => item.prefix + item.name))), + storage.list(`problem/${domainId}/${docId}/`) + .then((items) => storage.del(items.map((item) => `problem/${domainId}/${docId}/${item.name}`))), bus.parallel('problem/delete', domainId, docId), ]); await bus.emit('problem/del', domainId, docId); diff --git a/packages/hydrooj/src/model/storage.ts b/packages/hydrooj/src/model/storage.ts index cd9c6631d..c0f2972ed 100644 --- a/packages/hydrooj/src/model/storage.ts +++ b/packages/hydrooj/src/model/storage.ts @@ -1,5 +1,5 @@ import { extname } from 'path'; -import { escapeRegExp } from 'lodash'; +import { escapeRegExp, omit } from 'lodash'; import moment from 'moment-timezone'; import { nanoid } from 'nanoid'; import type { Readable } from 'stream'; @@ -39,7 +39,7 @@ export class StorageModel { { $set: { lastUsage: new Date() } }, { returnDocument: 'after' }, ); - return await storage.get(value?._id || path, savePath); + return await storage.get(value?.link || value?._id || path, savePath); } static async rename(path: string, newPath: string, operator = 1) { @@ -51,6 +51,20 @@ export class StorageModel { static async del(path: string[], operator = 1) { if (!path.length) return; + const affected = await StorageModel.coll.find({ path: { $in: path } }).toArray(); + if (!affected.length) return; + const linked = await StorageModel.coll.find({ link: { $in: affected.map((i) => i._id) }, path: { $nin: path } }).toArray(); + const processedIds = []; + for (const i of linked || []) { + if (processedIds.includes(i.link)) continue; + const current = affected.find((t) => t._id === t.link); // to be deleted + // eslint-disable-next-line no-await-in-loop + await Promise.all([ + StorageModel.coll.updateOne({ _id: current._id }, { $set: omit(i, ['_id']) }), + StorageModel.coll.updateOne({ _id: i._id }, { $set: omit(current, ['_id']), $unset: { link: '' } }), + ]); + processedIds.push(i.link); + } const autoDelete = moment().add(7, 'day').toDate(); await StorageModel.coll.updateMany( { path: { $in: path }, autoDelete: null }, @@ -66,7 +80,7 @@ export class StorageModel { autoDelete: null, }).toArray(); return results.map((i) => ({ - ...i, name: i.path.split(target)[1], prefix: target, + ...i, name: i.path.split(target)[1], })); } @@ -106,12 +120,10 @@ export class StorageModel { // Make sure id is not used // eslint-disable-next-line no-await-in-loop while (await StorageModel.coll.findOne({ _id })) _id = StorageModel.generateId(extname(dst)); - const result = await storage.copy(value._id, _id); - const { metaData, size, etag } = await storage.getMeta(_id); await StorageModel.coll.insertOne({ - _id, meta: metaData, path: dst, size, etag, lastModified: new Date(), owner: value.owner || 1, + ...value, _id, path: dst, link: value._id, lastModified: new Date(), owner: value.owner || 1, }); - return result; + return _id; } } diff --git a/packages/hydrooj/src/service/storage.ts b/packages/hydrooj/src/service/storage.ts index fdb15a668..06b7c54df 100644 --- a/packages/hydrooj/src/service/storage.ts +++ b/packages/hydrooj/src/service/storage.ts @@ -2,8 +2,8 @@ import { dirname, resolve } from 'path'; import { PassThrough, Readable } from 'stream'; import { URL } from 'url'; import { - CopyObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, - GetObjectCommand, HeadObjectCommand, PutObjectCommand, PutObjectCommandInput, S3Client, + DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, + HeadObjectCommand, PutObjectCommand, PutObjectCommandInput, S3Client, } from '@aws-sdk/client-s3'; import { Upload } from '@aws-sdk/lib-storage'; import { createPresignedPost } from '@aws-sdk/s3-presigned-post'; @@ -243,16 +243,6 @@ class RemoteStorageService { return { url, fields }; } - async copy(src: string, target: string) { - src = convertPath(src); - target = convertPath(target); - return await this.client.send(new CopyObjectCommand({ - Bucket: this.bucket, - Key: target, - CopySource: src, - })); - } - async status() { return { type: 'S3', @@ -332,14 +322,6 @@ class LocalStorageService { throw new Error('Not implemented'); } - async copy(src: string, target: string) { - src = resolve(this.dir, convertPath(src)); - target = resolve(this.dir, convertPath(target)); - await ensureDir(dirname(target)); - await copyFile(src, target); - return { etag: target, lastModified: new Date() }; - } - async status() { return { type: 'Local',