core: storage: use soft copy (#830)

packages/hydrooj/src/interface.ts

@@ -644,6 +644,8 @@ export interface FileNode {
     size?: number;
     /** AutoDelete */
     autoDelete?: Date;
+    /** fileId if linked to an existing file */
+    link?: string;
     owner?: number;
     operator?: number[];
     meta?: Record<string, string | number>;

packages/hydrooj/src/model/problem.ts

@@ -262,7 +262,8 @@ export class ProblemModel {
         const res = await Promise.all([
             document.deleteOne(domainId, document.TYPE_PROBLEM, docId),
             document.deleteMultiStatus(domainId, document.TYPE_PROBLEM, { docId }),
-            storage.list(`problem/${domainId}/${docId}/`).then((items) => storage.del(items.map((item) => item.prefix + item.name))),
+            storage.list(`problem/${domainId}/${docId}/`)
+                .then((items) => storage.del(items.map((item) => `problem/${domainId}/${docId}/${item.name}`))),
             bus.parallel('problem/delete', domainId, docId),
         ]);
         await bus.emit('problem/del', domainId, docId);

packages/hydrooj/src/model/storage.ts

@@ -1,5 +1,5 @@
 import { extname } from 'path';
-import { escapeRegExp } from 'lodash';
+import { escapeRegExp, omit } from 'lodash';
 import moment from 'moment-timezone';
 import { nanoid } from 'nanoid';
 import type { Readable } from 'stream';
@@ -39,7 +39,7 @@ export class StorageModel {
             { $set: { lastUsage: new Date() } },
             { returnDocument: 'after' },
         );
-        return await storage.get(value?._id || path, savePath);
+        return await storage.get(value?.link || value?._id || path, savePath);
     }
 
     static async rename(path: string, newPath: string, operator = 1) {
@@ -51,6 +51,20 @@ export class StorageModel {
 
     static async del(path: string[], operator = 1) {
         if (!path.length) return;
+        const affected = await StorageModel.coll.find({ path: { $in: path } }).toArray();
+        if (!affected.length) return;
+        const linked = await StorageModel.coll.find({ link: { $in: affected.map((i) => i._id) }, path: { $nin: path } }).toArray();
+        const processedIds = [];
+        for (const i of linked || []) {
+            if (processedIds.includes(i.link)) continue;
+            const current = affected.find((t) => t._id === t.link); // to be deleted
+            // eslint-disable-next-line no-await-in-loop
+            await Promise.all([
+                StorageModel.coll.updateOne({ _id: current._id }, { $set: omit(i, ['_id']) }),
+                StorageModel.coll.updateOne({ _id: i._id }, { $set: omit(current, ['_id']), $unset: { link: '' } }),
+            ]);
+            processedIds.push(i.link);
+        }
         const autoDelete = moment().add(7, 'day').toDate();
         await StorageModel.coll.updateMany(
             { path: { $in: path }, autoDelete: null },
@@ -66,7 +80,7 @@ export class StorageModel {
             autoDelete: null,
         }).toArray();
         return results.map((i) => ({
-            ...i, name: i.path.split(target)[1], prefix: target,
+            ...i, name: i.path.split(target)[1],
         }));
     }
 
@@ -106,12 +120,10 @@ export class StorageModel {
         // Make sure id is not used
         // eslint-disable-next-line no-await-in-loop
         while (await StorageModel.coll.findOne({ _id })) _id = StorageModel.generateId(extname(dst));
-        const result = await storage.copy(value._id, _id);
-        const { metaData, size, etag } = await storage.getMeta(_id);
         await StorageModel.coll.insertOne({
-            _id, meta: metaData, path: dst, size, etag, lastModified: new Date(), owner: value.owner || 1,
+            ...value, _id, path: dst, link: value._id, lastModified: new Date(), owner: value.owner || 1,
         });
-        return result;
+        return _id;
     }
 }
 

packages/hydrooj/src/service/storage.ts

@@ -2,8 +2,8 @@ import { dirname, resolve } from 'path';
 import { PassThrough, Readable } from 'stream';
 import { URL } from 'url';
 import {
-    CopyObjectCommand, DeleteObjectCommand, DeleteObjectsCommand,
+    DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand,
-    GetObjectCommand, HeadObjectCommand, PutObjectCommand, PutObjectCommandInput, S3Client,
+    HeadObjectCommand, PutObjectCommand, PutObjectCommandInput, S3Client,
 } from '@aws-sdk/client-s3';
 import { Upload } from '@aws-sdk/lib-storage';
 import { createPresignedPost } from '@aws-sdk/s3-presigned-post';
@@ -243,16 +243,6 @@ class RemoteStorageService {
         return { url, fields };
     }
 
-    async copy(src: string, target: string) {
-        src = convertPath(src);
-        target = convertPath(target);
-        return await this.client.send(new CopyObjectCommand({
-            Bucket: this.bucket,
-            Key: target,
-            CopySource: src,
-        }));
-    }
-
     async status() {
         return {
             type: 'S3',
@@ -332,14 +322,6 @@ class LocalStorageService {
         throw new Error('Not implemented');
     }
 
-    async copy(src: string, target: string) {
-        src = resolve(this.dir, convertPath(src));
-        target = resolve(this.dir, convertPath(target));
-        await ensureDir(dirname(target));
-        await copyFile(src, target);
-        return { etag: target, lastModified: new Date() };
-    }
-
     async status() {
         return {
             type: 'Local',