feat: minio storage support framework

packages/hydrojudge/package.json

@@ -15,7 +15,7 @@
         "p-queue": "^6.6.1",
         "shell-quote": "^1.7.2",
         "supports-color": "^8.0.0",
-        "systeminformation": "^4.31.0",
+        "systeminformation": "^4.31.1",
         "yargs": "^16.2.0"
     },
     "peerDependencies": {
@@ -24,7 +24,7 @@
     "license": "AGPL-3.0-only",
     "devDependencies": {
         "@types/bson": "^4.0.2",
-        "@types/fs-extra": "^9.0.4",
+        "@types/fs-extra": "^9.0.5",
         "@types/js-yaml": "^3.12.5",
         "@types/shell-quote": "^1.7.0"
     },
@@ -32,4 +32,4 @@
         "build": "tsc",
         "lint": "eslint . --ext ts --fix"
     }
-}
+}

packages/hydrooj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "hydrooj",
-    "version": "2.14.8",
+    "version": "2.14.9",
     "bin": "bin/hydrooj.js",
     "main": "dist/loader.js",
     "typings": "dist/loader.d.ts",
@@ -23,30 +23,32 @@
         "koa-static-cache": "^5.1.3",
         "lodash": "^4.17.20",
         "lru-cache": "^6.0.0",
+        "minio": "^7.0.17",
         "moment-timezone": "^0.5.32",
         "mongodb": "^3.6.3",
-        "nodemailer": "^6.4.16",
+        "nodemailer": "^6.4.17",
         "ps-list": "^7.2.0",
         "serialize-javascript": "^5.0.1",
         "sockjs": "^0.3.20",
         "superagent": "^6.1.0",
         "supports-color": "^8.0.0",
-        "systeminformation": "^4.31.0",
+        "systeminformation": "^4.31.1",
         "wtfnode": "^0.8.4",
         "yargs": "^16.2.0"
     },
     "devDependencies": {
         "@types/adm-zip": "^0.4.33",
-        "@types/fs-extra": "^9.0.4",
+        "@types/fs-extra": "^9.0.5",
         "@types/js-yaml": "^3.12.5",
         "@types/koa": "^2.11.4",
         "@types/koa-router": "^7.4.1",
         "@types/koa-static-cache": "^5.1.0",
         "@types/lodash": "^4.14.165",
         "@types/lru-cache": "^5.1.0",
+        "@types/minio": "^7.0.6",
         "@types/mongodb": "^3.5.33",
         "@types/nodemailer": "^6.4.0",
-        "@types/serialize-javascript": "^4.0.0",
+        "@types/serialize-javascript": "^5.0.0",
         "@types/sockjs": "^0.3.32",
         "@types/superagent": "^4.1.8",
         "@types/supports-color": "^7.2.0",

packages/hydrooj/src/check.ts

@@ -3,7 +3,7 @@
 import os from 'os';
 import { Dictionary } from 'lodash';
 import * as system from './model/system';
-import * as db from './service/db';
+import db from './service/db';
 
 const c = {};
 

packages/hydrooj/src/entry/cli.ts

@@ -5,7 +5,9 @@ import {
     lib, service, model,
     builtinLib, builtinModel,
 } from './common';
+import options from '../options';
 import * as bus from '../service/bus';
+import db from '../service/db';
 
 const COMMENTS = /((\/\/.*$)|(\/\*[\s\S]*?\*\/))/mg;
 const ARR = /=>.*$/mg;
@@ -52,10 +54,8 @@ export async function load() {
     require('../utils');
     require('../error');
     require('../options');
-    await new Promise((resolve) => {
-        bus.once('database/connect', resolve);
-        require('../service/db');
-    });
+    const opts = options();
+    await db.start(opts);
     for (const i of builtinLib) require(`../lib/${i}`);
     await lib(pending, fail);
     require('../service/gridfs');

packages/hydrooj/src/entry/master.ts

@@ -8,7 +8,9 @@ import { argv } from 'yargs';
 import { builtinModel } from './common';
 import { Entry } from '../loader';
 import { Logger } from '../logger';
+import options from '../options';
 import * as bus from '../service/bus';
+import db from '../service/db';
 
 const logger = new Logger('entry/master');
 const tmpdir = path.resolve(os.tmpdir(), 'hydro');
@@ -48,13 +50,9 @@ export async function load(call: Entry) {
     bus.once('app/exit', () => {
         fs.removeSync(lockfile);
     });
-    await new Promise((resolve) => {
-        bus.once('database/connect', () => {
-            bus.once('database/config', resolve);
-            require('../model/system');
-        });
-        require('../service/db');
-    });
+    const opts = options();
+    await db.start(opts);
+    await require('../model/system').runConfig();
     require('../service/monitor');
     for (const i of builtinModel) require(`../model/${i}`);
     const modelSystem = require('../model/system');

packages/hydrooj/src/entry/worker.ts

@@ -4,7 +4,9 @@ import {
     locale, template, lib, service, model, handler, script, setting, uistatic,
     builtinLib, builtinScript, builtinHandler, builtinModel,
 } from './common';
+import options from '../options';
 import * as bus from '../service/bus';
+import db from '../service/db';
 
 export async function load() {
     const pending = global.addons;
@@ -19,12 +21,11 @@ export async function load() {
         template(pending, fail),
         uistatic(pending, fail),
     ]);
+    const opts = options();
+    await db.start(opts);
     await new Promise((resolve) => {
-        bus.once('database/connect', () => {
-            bus.once('database/config', resolve);
-            require('../model/system');
-        });
-        require('../service/db');
+        bus.once('database/config', resolve);
+        require('../model/system');
     });
     for (const i of builtinLib) require(`../lib/${i}`);
     await lib(pending, fail);

packages/hydrooj/src/handler/misc.ts

@@ -5,7 +5,7 @@ import { PRIV } from '../model/builtin';
 import * as system from '../model/system';
 import * as file from '../model/file';
 import * as user from '../model/user';
-import * as db from '../service/db';
+import db from '../service/db';
 import {
     Route, Handler, Types, param,
 } from '../service/server';

packages/hydrooj/src/interface.ts

@@ -537,6 +537,7 @@ export interface Service {
     fs: HydroFileSystem,
     monitor: typeof import('./service/monitor'),
     server: typeof import('./service/server'),
+    storage: typeof import('./service/storage'),
 }
 
 interface GeoIP {

packages/hydrooj/src/model/blacklist.ts

@@ -1,4 +1,4 @@
-import * as db from '../service/db';
+import db from '../service/db';
 import * as bus from '../service/bus';
 
 const coll = db.collection('blacklist');

packages/hydrooj/src/model/document.ts

@@ -8,7 +8,7 @@ import {
 } from '../interface';
 import { buildProjection } from '../utils';
 import { NumberKeys, ArrayKeys, Projection } from '../typeutils';
-import * as db from '../service/db';
+import db from '../service/db';
 import * as bus from '../service/bus';
 import { Content } from '../loader';
 

packages/hydrooj/src/model/domain.ts

@@ -3,7 +3,7 @@ import { FilterQuery } from 'mongodb';
 import { BUILTIN_ROLES, PRIV } from './builtin';
 import { DomainDoc } from '../interface';
 import * as bus from '../service/bus';
-import * as db from '../service/db';
+import db from '../service/db';
 
 const coll = db.collection('domain');
 const collUser = db.collection('domain.user');

packages/hydrooj/src/model/file.ts

@@ -3,7 +3,7 @@ import * as fs from 'fs';
 import { Dictionary } from 'lodash';
 import { ForbiddenError, NotFoundError } from '../error';
 import { Ufdoc } from '../interface';
-import * as db from '../service/db';
+import db from '../service/db';
 import * as bus from '../service/bus';
 import gridfs from '../service/gridfs';
 import hash from '../lib/hash.hydro';

packages/hydrooj/src/model/message.ts

@@ -1,7 +1,7 @@
 import { FilterQuery, ObjectID } from 'mongodb';
 import * as user from './user';
 import { Mdoc } from '../interface';
-import * as db from '../service/db';
+import db from '../service/db';
 import * as bus from '../service/bus';
 
 const coll = db.collection('message');

packages/hydrooj/src/model/oauth.ts

@@ -1,6 +1,6 @@
 import Lru from 'lru-cache';
 import { Collection } from 'mongodb';
-import * as db from '../service/db';
+import db from '../service/db';
 
 interface OauthMap {
     _id: string, // source openId

packages/hydrooj/src/model/opcount.ts

@@ -1,6 +1,6 @@
 import { OpcountExceededError } from '../error';
 import * as bus from '../service/bus';
-import * as db from '../service/db';
+import db from '../service/db';
 
 const coll = db.collection('opcount');
 

packages/hydrooj/src/model/oplog.ts

@@ -1,6 +1,6 @@
 import { ObjectID } from 'mongodb';
 import { OplogDoc } from '../interface';
-import * as db from '../service/db';
+import db from '../service/db';
 
 export const coll = db.collection('oplog');
 

packages/hydrooj/src/model/record.ts

@@ -8,7 +8,7 @@ import * as problem from './problem';
 import {
     Rdoc, TestCase, RunConfig, ContestInfo, ProblemConfig,
 } from '../interface';
-import * as db from '../service/db';
+import db from '../service/db';
 
 export const coll: Collection<Rdoc> = db.collection('record');
 

packages/hydrooj/src/model/system.ts

@@ -1,7 +1,7 @@
 import { SYSTEM_SETTINGS } from './setting';
 import { NumberKeys } from '../typeutils';
 import { SystemKeys } from '../interface';
-import * as db from '../service/db';
+import db from '../service/db';
 import * as bus from '../service/bus';
 
 const coll = db.collection('system');
@@ -56,16 +56,17 @@ export async function inc<K extends NumberKeys<SystemKeys>>(_id: K) {
     return res.value.value;
 }
 
-(async () => {
+export async function runConfig() {
     for (const setting of SYSTEM_SETTINGS) {
         if (setting.value) cache[setting.key] = setting.value;
     }
     const config = await coll.find({}).toArray();
     for (const i of config) cache[i._id] = i.value;
     bus.emit('database/config');
-})();
+}
 
 global.Hydro.model.system = {
+    runConfig,
     get,
     getMany,
     inc,

packages/hydrooj/src/model/task.ts

@@ -2,7 +2,7 @@ import moment from 'moment-timezone';
 import { FilterQuery, ObjectID } from 'mongodb';
 import { Task } from '../interface';
 import { Logger } from '../logger';
-import * as db from '../service/db';
+import db from '../service/db';
 
 const logger = new Logger('model/task');
 const coll = db.collection('task');

packages/hydrooj/src/model/token.ts

@@ -1,5 +1,5 @@
 import { TokenDoc } from '../interface';
-import * as db from '../service/db';
+import db from '../service/db';
 import * as bus from '../service/bus';
 
 export const coll = db.collection('token');

packages/hydrooj/src/model/user.ts

@@ -9,7 +9,7 @@ import { User as _User, Udoc, Udict } from '../interface';
 import { Value } from '../typeutils';
 import { Logger } from '../logger';
 import pwhash from '../lib/hash.hydro';
-import * as db from '../service/db';
+import db from '../service/db';
 import * as bus from '../service/bus';
 
 const coll: Collection<Udoc> = db.collection('user');

packages/hydrooj/src/script/blacklist.ts

@@ -4,7 +4,7 @@ import * as document from '../model/document';
 import * as discussion from '../model/discussion';
 import * as user from '../model/user';
 import * as blacklist from '../model/blacklist';
-import * as db from '../service/db';
+import db from '../service/db';
 
 export const description = 'Add blacklist by ip, uid';
 

packages/hydrooj/src/script/deleteUser.ts

@@ -4,7 +4,7 @@ import { Value } from '../typeutils';
 import * as user from '../model/user';
 import * as document from '../model/document';
 import * as setting from '../model/setting';
-import * as db from '../service/db';
+import db from '../service/db';
 import paginate from '../lib/paginate';
 import { STATUS, PRIV } from '../model/builtin';
 

packages/hydrooj/src/script/problemStat.ts

@@ -1,4 +1,4 @@
-import * as db from '../service/db';
+import db from '../service/db';
 import { STATUS } from '../model/builtin';
 import * as document from '../model/document';
 

packages/hydrooj/src/service/db.ts

@@ -1,34 +1,43 @@
-import mongodb, { Collection } from 'mongodb';
+import { Collection, Db, MongoClient } from 'mongodb';
 import * as bus from './bus';
-import options from '../options';
 import { Collections } from '../interface';
 
-const opts = options();
-
-let mongourl = 'mongodb://';
-if (opts.username) mongourl += `${opts.username}:${opts.password}@`;
-mongourl += `${opts.host}:${opts.port}/${opts.name}`;
-
-// eslint-disable-next-line import/no-mutable-exports
-export let db: mongodb.Db = null;
-// eslint-disable-next-line import/no-mutable-exports
-export let db2: mongodb.Db = null;
-
-mongodb.MongoClient.connect(mongourl, { useNewUrlParser: true, useUnifiedTopology: true })
-    .then((Client) => {
-        db = Client.db(opts.name);
-        global.Hydro.service.db.db = db;
-        mongodb.MongoClient.connect(mongourl, { useNewUrlParser: true, useUnifiedTopology: true })
-            .then((Client1) => {
-                db2 = Client1.db(opts.name);
-                global.Hydro.service.db.db2 = db2;
-                bus.parallel('database/connect', db);
-            });
-    });
-
-export function collection<K extends keyof Collections>(c: K): Collection<Collections[K]> {
-    if (opts.prefix) return db.collection(`${opts.prefix}.${c}`);
-    return db.collection(c);
+interface MongoConfig {
+    username?: string,
+    password?: string,
+    host?: string,
+    port?: string,
+    name?: string,
+    url?: string,
+    prefix?: string,
 }
 
-global.Hydro.service.db = { collection, db, db2 };
+class MongoService {
+    public client: MongoClient;
+    public client2: MongoClient;
+    public db: Db;
+    public db2: Db;
+    private opts: MongoConfig;
+
+    async start(opts: MongoConfig) {
+        this.opts = opts;
+        let mongourl = 'mongodb://';
+        if (opts.username) mongourl += `${opts.username}:${opts.password}@`;
+        mongourl += `${opts.host}:${opts.port}/${opts.name}`;
+        if (opts.url) mongourl = opts.url;
+        this.client = await MongoClient.connect(mongourl, { useNewUrlParser: true, useUnifiedTopology: true });
+        this.db = this.client.db(opts.name);
+        this.client2 = await MongoClient.connect(mongourl, { useNewUrlParser: true, useUnifiedTopology: true });
+        this.db2 = this.client2.db(opts.name);
+        bus.parallel('database/connect', this.db);
+    }
+
+    public collection<K extends keyof Collections>(c: K): Collection<Collections[K]> {
+        if (this.opts.prefix) return this.db.collection(`${this.opts.prefix}.${c}`);
+        return this.db.collection(c);
+    }
+}
+
+const service = new MongoService();
+global.Hydro.service.db = service;
+export = service;

packages/hydrooj/src/service/gridfs.ts

@@ -1,8 +1,8 @@
 import { GridFSBucket } from 'mongodb';
-import { db2 } from './db';
+import db from './db';
 import { HydroFileSystem } from '../interface';
 
-const fs = new GridFSBucket(db2);
+const fs = new GridFSBucket(db.db2);
 
 // @ts-ignore
 global.Hydro.service.fs = fs;

packages/hydrooj/src/service/monitor.ts

@@ -2,7 +2,7 @@ import cluster from 'cluster';
 import crypto from 'crypto';
 import axios from 'axios';
 import { safeDump } from 'js-yaml';
-import * as db from './db';
+import db from './db';
 import * as bus from './bus';
 import * as sysinfo from '../lib/sysinfo';
 

packages/hydrooj/src/service/storage.ts

@@ -0,0 +1,68 @@
+import { Readable } from 'stream';
+import Minio, { BucketItem, ItemBucketMetadata } from 'minio';
+
+interface StorageOptions {
+    host?: string,
+    port?: number,
+    useSSL?: boolean,
+    accessKey?: string,
+    secretKey?: string,
+    bucket?: string,
+    region?: string,
+}
+
+const defaultConfig: StorageOptions = {
+    host: 'play.min.io',
+    port: 9000,
+    useSSL: true,
+    accessKey: 'Q3AM3UQ867SPQQA43P2F',
+    secretKey: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG',
+    bucket: 'hydro',
+    region: 'us-east-1',
+};
+
+class StorageService {
+    public client: Minio.Client;
+    private opts: StorageOptions;
+
+    async start(opts: StorageOptions) {
+        this.opts = { ...defaultConfig, ...opts };
+        this.client = new Minio.Client({
+            endPoint: this.opts.host,
+            port: this.opts.port,
+            useSSL: this.opts.useSSL,
+            accessKey: this.opts.accessKey,
+            secretKey: this.opts.secretKey,
+        });
+        const exists = await this.client.bucketExists(this.opts.bucket);
+        if (!exists) await this.client.makeBucket(this.opts.bucket, this.opts.region);
+    }
+
+    async put(target: string, file: string | Buffer | Readable, meta: ItemBucketMetadata = {}) {
+        if (typeof file === 'string') return await this.client.fPutObject(this.opts.bucket, target, file, meta);
+        return await this.client.putObject(this.opts.bucket, target, file, meta);
+    }
+
+    async get(target: string) {
+        return await this.client.getObject(this.opts.bucket, target);
+    }
+
+    async del(target: string | string[]) {
+        if (typeof target === 'string') return await this.client.removeObject(this.opts.bucket, target);
+        return await this.client.removeObjects(this.opts.bucket, target);
+    }
+
+    async list(target: string, recursive = false) {
+        const stream = this.client.listObjects(this.opts.bucket, target, recursive);
+        return await new Promise<BucketItem[]>((resolve, reject) => {
+            const results: BucketItem[] = [];
+            stream.on('data', (result) => results.push(result));
+            stream.on('end', () => resolve(results));
+            stream.on('error', reject);
+        });
+    }
+}
+
+const service = new StorageService();
+global.Hydro.service.storage = service;
+export = service;

packages/hydrooj/src/upgrade.ts

@@ -1,7 +1,7 @@
 /* eslint-disable no-await-in-loop */
 import * as domain from './model/domain';
 import * as document from './model/document';
-import { db } from './service/db';
+import db from './service/db';
 
 type UpgradeScript = () => Promise<void>;