core: file rename api

packages/hydrooj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "hydrooj",
-    "version": "2.25.15",
+    "version": "2.26.0",
     "bin": "bin/hydrooj.js",
     "main": "dist/loader.js",
     "typings": "dist/loader.d.ts",

packages/hydrooj/src/entry/common.ts

@@ -18,9 +18,9 @@ export const builtinLib = [
 
 export const builtinModel = [
     'builtin', 'document', 'domain', 'blacklist', 'opcount',
-    'setting', 'token', 'user', 'problem', 'record',
+    'setting', 'token', 'user', 'storage', 'problem',
-    'contest', 'message', 'solution', 'training', 'oplog',
+    'record', 'contest', 'message', 'solution', 'training',
-    'discussion', 'system',
+    'discussion', 'system', 'oplog',
 ];
 
 export const builtinHandler = [

packages/hydrooj/src/handler/remote.ts

@@ -10,10 +10,10 @@ import { PERM, PRIV } from '../model/builtin';
 import token from '../model/token';
 import problem from '../model/problem';
 import * as system from '../model/system';
+import storage from '../model/storage';
 import {
     Handler, Types, Route, post,
 } from '../service/server';
-import storage from '../service/storage';
 
 const logger = new Logger('remote');
 

packages/hydrooj/src/interface.ts

@@ -3,6 +3,7 @@ import type { Cursor, ObjectID } from 'mongodb';
 import type fs from 'fs';
 import type { Dictionary, NumericDictionary } from 'lodash';
 import type * as Koa from 'koa';
+import { ItemBucketMetadata } from 'minio';
 import type { ProblemDoc } from './model/problem';
 
 type document = typeof import('./model/document');
@@ -472,6 +473,21 @@ export interface BaseService {
     stop?: Function;
 }
 
+export interface FileNode {
+    /** File Path In MinIO */
+    _id: string
+    /** Actual File Path */
+    path: string
+    lastUsage?: Date
+    lastModified?: Date
+    etag?: string
+    /** Size: in bytes */
+    size?: number
+    /** AutoDelete */
+    autoDelete?: Date
+    meta?: ItemBucketMetadata,
+}
+
 export interface Collections {
     'blacklist': BlacklistDoc,
     'contest': Tdoc,
@@ -488,6 +504,7 @@ export interface Collections {
     'oauth': any,
     'system': System,
     'task': Task,
+    'storage': FileNode,
     'oplog': OplogDoc,
     'opcount': any,
     'fs.chunks': any,
@@ -515,6 +532,7 @@ export interface Model {
     training: typeof import('./model/training'),
     user: typeof import('./model/user').default,
     oauth: typeof import('./model/oauth').default,
+    storage: typeof import('./model/storage').default,
 }
 
 export interface Service {

packages/hydrooj/src/model/problem.ts

@@ -4,13 +4,13 @@ import { Dictionary, escapeRegExp, pick } from 'lodash';
 import { streamToBuffer } from '@hydrooj/utils/lib/utils';
 import * as document from './document';
 import { STATUS } from './builtin';
+import storage from './storage';
 import { buildProjection } from '../utils';
 import type { ProblemStatusDoc, ProblemDict, Document } from '../interface';
 import {
     ArrayKeys, MaybeArray, NumberKeys, Projection,
 } from '../typeutils';
 import { ProblemNotFoundError, ValidationError } from '../error';
-import storage from '../service/storage';
 import * as bus from '../service/bus';
 
 export interface ProblemDoc extends Document { }

packages/hydrooj/src/model/storage.ts

@@ -0,0 +1,82 @@
+import type { Readable } from 'stream';
+import { escapeRegExp } from 'lodash';
+import { ItemBucketMetadata } from 'minio';
+import moment from 'moment';
+import storage from '../service/storage';
+import * as bus from '../service/bus';
+import db from '../service/db';
+
+export class StorageModel {
+    static coll = db.collection('storage');
+
+    static async put(path: string, file: string | Buffer | Readable, meta: ItemBucketMetadata = {}) {
+        const [current, place] = await Promise.all([
+            StorageModel.coll.findOne({ path }),
+            StorageModel.coll.findOne({ _id: path }),
+        ]);
+        if (current) {
+            await storage.put(current._id, file, meta);
+            const { metaData, size, etag } = await storage.getMeta(current._id);
+            await StorageModel.coll.updateOne({ path }, {
+                $set: {
+                    meta: metaData, size, etag, lastModified: new Date(),
+                },
+            });
+            return path;
+        }
+        const target = place ? path + Math.random().toString(16) : path;
+        await storage.put(target, file, meta);
+        const { metaData, size, etag } = await storage.getMeta(target);
+        await StorageModel.coll.insertOne({
+            _id: target, meta: metaData, path, size, etag, lastModified: new Date(),
+        });
+        return path;
+    }
+
+    static async get(path: string, savePath?: string) {
+        const { value } = await StorageModel.coll.findOneAndUpdate({ path }, { $set: { lastUsage: new Date() } }, { returnOriginal: false });
+        if (value) return await storage.get(value._id, savePath);
+        return await storage.get(path, savePath);
+    }
+
+    static async rename(path: string, newPath: string) {
+        return await StorageModel.coll.updateOne({ path }, { $set: { path: newPath } });
+    }
+
+    static async del(path: string[]) {
+        const autoDelete = moment().add(7, 'day').toDate();
+        await StorageModel.coll.updateMany({ path: { $in: path } }, { $set: { autoDelete } });
+    }
+
+    static async list(target: string, recursive = true) {
+        if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
+        if (target?.length && !target.endsWith('/')) target += '/';
+        const results = recursive
+            ? await StorageModel.coll.find({ path: { $regex: new RegExp(`^${escapeRegExp(target)}`, 'i') } }).toArray()
+            : await StorageModel.coll.find({ path: { $regex: new RegExp(`^${escapeRegExp(target)}[^/]+$`) } }).toArray();
+        return results.map((i) => ({
+            ...i, name: i.path, prefix: i.path.split(target)[1],
+        }));
+    }
+
+    static async getMeta(path: string) {
+        const { value } = await StorageModel.coll.findOneAndUpdate({ path }, { $set: { lastUsage: new Date() } }, { returnOriginal: false });
+        if (!value) return null;
+        return {
+            ...value.meta,
+            size: value.size,
+            lastModified: value.lastModified,
+            etag: value.etag,
+        };
+    }
+
+    static async signDownloadLink(target: string, filename?: string, noExpire = false, useAlternativeEndpointFor?: 'user' | 'judge') {
+        await StorageModel.coll.updateOne({ path: target }, { $set: { lastUsage: new Date() } });
+        return await storage.signDownloadLink(target, filename, noExpire, useAlternativeEndpointFor);
+    }
+}
+
+bus.on('app/started', () => StorageModel.coll.createIndex({ path: 1 }, { unique: true }));
+
+global.Hydro.model.storage = StorageModel;
+export default StorageModel;

packages/hydrooj/src/script/updateFilelist.ts

@@ -1,4 +1,4 @@
-import storage from '../service/storage';
+import storage from '../model/storage';
 import { iterateAllProblem, iterateAllProblemInDomain } from '../pipelineUtils';
 
 export const description = 'Sync problem filelist from s3 service';

packages/hydrooj/src/service/storage.ts

@@ -76,7 +76,7 @@ function encodeRFC5987ValueChars(str: string) {
 class StorageService {
     public client: Client;
     public error = '';
-    private opts: StorageOptions;
+    public opts: StorageOptions;
     private replaceWithAlternativeUrlFor: Record<'user' | 'judge', (originalUrl: string) => string>;
 
     async start() {
@@ -153,6 +153,7 @@ class StorageService {
         }
     }
 
+    /** @deprecated use StorageModel.list instead. */
     async list(target: string, recursive = true) {
         if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
         try {

packages/hydrooj/src/upgrade.ts

@@ -6,6 +6,7 @@ import Queue from 'p-queue';
 import yaml from 'js-yaml';
 import { pick } from 'lodash';
 import { convertIniConfig } from '@hydrooj/utils/lib/cases';
+import { BucketItem } from 'minio';
 import { Progress } from './ui';
 import { RecordDoc } from './interface';
 import { Logger } from './logger';
@@ -24,6 +25,8 @@ import * as document from './model/document';
 import * as system from './model/system';
 import { STATUS } from './model/builtin';
 import RecordModel from './model/record';
+import StorageModel from './model/storage';
+import { size } from './lib/misc';
 
 const logger = new Logger('upgrade');
 type UpgradeScript = void | (() => Promise<boolean | void>);
@@ -327,6 +330,28 @@ const scripts: UpgradeScript[] = [
         });
         return true;
     },
+    async function _26_27() {
+        const stream = storage.client.listObjects(storage.opts.bucket, '', true);
+        await new Promise<BucketItem[]>((resolve, reject) => {
+            stream.on('data', (result) => {
+                if (result.size) {
+                    logger.debug('File found: %s %s', result.name, size(result.size));
+                    StorageModel.coll.insertOne({
+                        _id: result.name,
+                        path: result.name,
+                        size: result.size,
+                        lastModified: result.lastModified,
+                        lastUsage: new Date(),
+                        etag: result.etag,
+                        meta: {},
+                    });
+                }
+            });
+            stream.on('end', () => resolve(null));
+            stream.on('error', reject);
+        });
+        return true;
+    },
 ];
 
 export default scripts;