From e5bfb78f9409b9af2a459cf6621d1e96211a571e Mon Sep 17 00:00:00 2001 From: undefined Date: Sat, 15 May 2021 00:41:11 +0800 Subject: [PATCH] core: file rename api --- packages/hydrooj/package.json | 2 +- packages/hydrooj/src/entry/common.ts | 6 +- packages/hydrooj/src/handler/remote.ts | 2 +- packages/hydrooj/src/interface.ts | 18 ++++ packages/hydrooj/src/model/problem.ts | 2 +- packages/hydrooj/src/model/storage.ts | 82 +++++++++++++++++++ packages/hydrooj/src/script/updateFilelist.ts | 2 +- packages/hydrooj/src/service/storage.ts | 3 +- packages/hydrooj/src/upgrade.ts | 25 ++++++ 9 files changed, 134 insertions(+), 8 deletions(-) create mode 100644 packages/hydrooj/src/model/storage.ts diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index f5791a5da..6a99df0ee 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "2.25.15", + "version": "2.26.0", "bin": "bin/hydrooj.js", "main": "dist/loader.js", "typings": "dist/loader.d.ts", diff --git a/packages/hydrooj/src/entry/common.ts b/packages/hydrooj/src/entry/common.ts index 94fb3800a..a3fec3a02 100644 --- a/packages/hydrooj/src/entry/common.ts +++ b/packages/hydrooj/src/entry/common.ts @@ -18,9 +18,9 @@ export const builtinLib = [ export const builtinModel = [ 'builtin', 'document', 'domain', 'blacklist', 'opcount', - 'setting', 'token', 'user', 'problem', 'record', - 'contest', 'message', 'solution', 'training', 'oplog', - 'discussion', 'system', + 'setting', 'token', 'user', 'storage', 'problem', + 'record', 'contest', 'message', 'solution', 'training', + 'discussion', 'system', 'oplog', ]; export const builtinHandler = [ diff --git a/packages/hydrooj/src/handler/remote.ts b/packages/hydrooj/src/handler/remote.ts index 46da897fc..29f85f908 100644 --- a/packages/hydrooj/src/handler/remote.ts +++ b/packages/hydrooj/src/handler/remote.ts @@ -10,10 +10,10 @@ import { PERM, PRIV } from '../model/builtin'; import token from '../model/token'; import problem from '../model/problem'; import * as system from '../model/system'; +import storage from '../model/storage'; import { Handler, Types, Route, post, } from '../service/server'; -import storage from '../service/storage'; const logger = new Logger('remote'); diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index ea9c2cac3..1f9bd4d05 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -3,6 +3,7 @@ import type { Cursor, ObjectID } from 'mongodb'; import type fs from 'fs'; import type { Dictionary, NumericDictionary } from 'lodash'; import type * as Koa from 'koa'; +import { ItemBucketMetadata } from 'minio'; import type { ProblemDoc } from './model/problem'; type document = typeof import('./model/document'); @@ -472,6 +473,21 @@ export interface BaseService { stop?: Function; } +export interface FileNode { + /** File Path In MinIO */ + _id: string + /** Actual File Path */ + path: string + lastUsage?: Date + lastModified?: Date + etag?: string + /** Size: in bytes */ + size?: number + /** AutoDelete */ + autoDelete?: Date + meta?: ItemBucketMetadata, +} + export interface Collections { 'blacklist': BlacklistDoc, 'contest': Tdoc, @@ -488,6 +504,7 @@ export interface Collections { 'oauth': any, 'system': System, 'task': Task, + 'storage': FileNode, 'oplog': OplogDoc, 'opcount': any, 'fs.chunks': any, @@ -515,6 +532,7 @@ export interface Model { training: typeof import('./model/training'), user: typeof import('./model/user').default, oauth: typeof import('./model/oauth').default, + storage: typeof import('./model/storage').default, } export interface Service { diff --git a/packages/hydrooj/src/model/problem.ts b/packages/hydrooj/src/model/problem.ts index e5f2d0d74..d32c3d08c 100644 --- a/packages/hydrooj/src/model/problem.ts +++ b/packages/hydrooj/src/model/problem.ts @@ -4,13 +4,13 @@ import { Dictionary, escapeRegExp, pick } from 'lodash'; import { streamToBuffer } from '@hydrooj/utils/lib/utils'; import * as document from './document'; import { STATUS } from './builtin'; +import storage from './storage'; import { buildProjection } from '../utils'; import type { ProblemStatusDoc, ProblemDict, Document } from '../interface'; import { ArrayKeys, MaybeArray, NumberKeys, Projection, } from '../typeutils'; import { ProblemNotFoundError, ValidationError } from '../error'; -import storage from '../service/storage'; import * as bus from '../service/bus'; export interface ProblemDoc extends Document { } diff --git a/packages/hydrooj/src/model/storage.ts b/packages/hydrooj/src/model/storage.ts new file mode 100644 index 000000000..3d43acfec --- /dev/null +++ b/packages/hydrooj/src/model/storage.ts @@ -0,0 +1,82 @@ +import type { Readable } from 'stream'; +import { escapeRegExp } from 'lodash'; +import { ItemBucketMetadata } from 'minio'; +import moment from 'moment'; +import storage from '../service/storage'; +import * as bus from '../service/bus'; +import db from '../service/db'; + +export class StorageModel { + static coll = db.collection('storage'); + + static async put(path: string, file: string | Buffer | Readable, meta: ItemBucketMetadata = {}) { + const [current, place] = await Promise.all([ + StorageModel.coll.findOne({ path }), + StorageModel.coll.findOne({ _id: path }), + ]); + if (current) { + await storage.put(current._id, file, meta); + const { metaData, size, etag } = await storage.getMeta(current._id); + await StorageModel.coll.updateOne({ path }, { + $set: { + meta: metaData, size, etag, lastModified: new Date(), + }, + }); + return path; + } + const target = place ? path + Math.random().toString(16) : path; + await storage.put(target, file, meta); + const { metaData, size, etag } = await storage.getMeta(target); + await StorageModel.coll.insertOne({ + _id: target, meta: metaData, path, size, etag, lastModified: new Date(), + }); + return path; + } + + static async get(path: string, savePath?: string) { + const { value } = await StorageModel.coll.findOneAndUpdate({ path }, { $set: { lastUsage: new Date() } }, { returnOriginal: false }); + if (value) return await storage.get(value._id, savePath); + return await storage.get(path, savePath); + } + + static async rename(path: string, newPath: string) { + return await StorageModel.coll.updateOne({ path }, { $set: { path: newPath } }); + } + + static async del(path: string[]) { + const autoDelete = moment().add(7, 'day').toDate(); + await StorageModel.coll.updateMany({ path: { $in: path } }, { $set: { autoDelete } }); + } + + static async list(target: string, recursive = true) { + if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); + if (target?.length && !target.endsWith('/')) target += '/'; + const results = recursive + ? await StorageModel.coll.find({ path: { $regex: new RegExp(`^${escapeRegExp(target)}`, 'i') } }).toArray() + : await StorageModel.coll.find({ path: { $regex: new RegExp(`^${escapeRegExp(target)}[^/]+$`) } }).toArray(); + return results.map((i) => ({ + ...i, name: i.path, prefix: i.path.split(target)[1], + })); + } + + static async getMeta(path: string) { + const { value } = await StorageModel.coll.findOneAndUpdate({ path }, { $set: { lastUsage: new Date() } }, { returnOriginal: false }); + if (!value) return null; + return { + ...value.meta, + size: value.size, + lastModified: value.lastModified, + etag: value.etag, + }; + } + + static async signDownloadLink(target: string, filename?: string, noExpire = false, useAlternativeEndpointFor?: 'user' | 'judge') { + await StorageModel.coll.updateOne({ path: target }, { $set: { lastUsage: new Date() } }); + return await storage.signDownloadLink(target, filename, noExpire, useAlternativeEndpointFor); + } +} + +bus.on('app/started', () => StorageModel.coll.createIndex({ path: 1 }, { unique: true })); + +global.Hydro.model.storage = StorageModel; +export default StorageModel; diff --git a/packages/hydrooj/src/script/updateFilelist.ts b/packages/hydrooj/src/script/updateFilelist.ts index fa3ced8c4..f79c1b6da 100644 --- a/packages/hydrooj/src/script/updateFilelist.ts +++ b/packages/hydrooj/src/script/updateFilelist.ts @@ -1,4 +1,4 @@ -import storage from '../service/storage'; +import storage from '../model/storage'; import { iterateAllProblem, iterateAllProblemInDomain } from '../pipelineUtils'; export const description = 'Sync problem filelist from s3 service'; diff --git a/packages/hydrooj/src/service/storage.ts b/packages/hydrooj/src/service/storage.ts index 46407451e..36d19f20a 100644 --- a/packages/hydrooj/src/service/storage.ts +++ b/packages/hydrooj/src/service/storage.ts @@ -76,7 +76,7 @@ function encodeRFC5987ValueChars(str: string) { class StorageService { public client: Client; public error = ''; - private opts: StorageOptions; + public opts: StorageOptions; private replaceWithAlternativeUrlFor: Record<'user' | 'judge', (originalUrl: string) => string>; async start() { @@ -153,6 +153,7 @@ class StorageService { } } + /** @deprecated use StorageModel.list instead. */ async list(target: string, recursive = true) { if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); try { diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index c7c6d84ea..f2127e3ac 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -6,6 +6,7 @@ import Queue from 'p-queue'; import yaml from 'js-yaml'; import { pick } from 'lodash'; import { convertIniConfig } from '@hydrooj/utils/lib/cases'; +import { BucketItem } from 'minio'; import { Progress } from './ui'; import { RecordDoc } from './interface'; import { Logger } from './logger'; @@ -24,6 +25,8 @@ import * as document from './model/document'; import * as system from './model/system'; import { STATUS } from './model/builtin'; import RecordModel from './model/record'; +import StorageModel from './model/storage'; +import { size } from './lib/misc'; const logger = new Logger('upgrade'); type UpgradeScript = void | (() => Promise); @@ -327,6 +330,28 @@ const scripts: UpgradeScript[] = [ }); return true; }, + async function _26_27() { + const stream = storage.client.listObjects(storage.opts.bucket, '', true); + await new Promise((resolve, reject) => { + stream.on('data', (result) => { + if (result.size) { + logger.debug('File found: %s %s', result.name, size(result.size)); + StorageModel.coll.insertOne({ + _id: result.name, + path: result.name, + size: result.size, + lastModified: result.lastModified, + lastUsage: new Date(), + etag: result.etag, + meta: {}, + }); + } + }); + stream.on('end', () => resolve(null)); + stream.on('error', reject); + }); + return true; + }, ]; export default scripts;