diff --git a/.eslintrc.js b/.eslintrc.js index 4a9f347a3..3b25b3a03 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -27,6 +27,7 @@ module.exports = { 'import/no-named-as-default-member': 'off', 'import/prefer-default-export': 'off', indent: ['warn', 4], + 'lines-between-class-members': ['error', 'always', { 'exceptAfterSingleLine': true }], 'max-classes-per-file': 'off', 'max-len': ['warn', 150], 'no-bitwise': 'off', diff --git a/docs/addon-ts.md b/docs/addon-ts.md deleted file mode 100644 index 8a55e8732..000000000 --- a/docs/addon-ts.md +++ /dev/null @@ -1,208 +0,0 @@ -# Hydro 附加组件开发 - -前置条件：NodeJS>10.10 -此教程将以编写剪贴板插件为例进行说明。 - -## Step1 初始化项目 - -在一个空文件夹中运行 `yarn init` 并按照提示填写相关信息。 - -```sh -/workspace/hydro-plugin $ yarn init -yarn init v1.22.4 -question name (hydro-plugin): @hydrooj/pastebin -question version (1.0.0): 0.0.1 -question description: HydroOJ的剪贴板组件 -question entry point (index.js): package.json -question repository url: https://github.com/hydro-dev/pastebin.git -question author: undefined -question license (MIT): MIT -question private: -success Saved package.json -``` - -## Step2 准备编写组件 - -分析：剪贴板组件需要以下功能： - -- 与数据库交互来存储/检索相应文档。 -- 提供 /paste/create 路由以创建新文档。 -- 提供 /paste/show/:ID 来查看已创建的文档。 -- 根据用户ID进行鉴权，允许将文档设置为私密以防止他人查看。 - -Hydro的推荐架构如下： - -- handler.ts: 用于处理路由 -- model.ts: 数据库模型 -- lib.ts: 不依赖于数据库等的库（如md5函数） -- script.ts: 可能会被用户多次使用到的脚本（如重新计算rp） -- locale/: 翻译文件 -- template/: 页面模板 -- setting.yaml: 模块所用到的设置，格式在下方说明 - -## Step3 tsconfig.json - -```json -{ - "compilerOptions": { - "target": "es2019", - "module": "commonjs", - "esModuleInterop": true, - "moduleResolution": "node", - "declaration": true, - "sourceMap": true, - "composite": true, - "strictBindCallApply": true, - "experimentalDecorators": true, - "outDir": ".", - "rootDir": "." - }, - "include": [ - "*.ts" - ], - "exclude": [] -} -``` - -## Step3 model.js - -提示：若不便于使用 import 引入 Hydro 的文件，可以从 global.Hydro 中取出需要的模块。 - -```ts -import 'hydrooj'; -import * as db from 'hydrooj/dist/service/db'; - -const coll = db.collection('paste'); - -interface Paste { - _id: string, - owner: number, - content: string, - isPrivate: boolean, -} - -declare module 'hydrooj' { - interface Collections { - paste: Paste, - } -} - -export async function add(userId: number, content: string, isPrivate: boolean): Promise { - const pasteId = String.random(16); // Hydro提供了此方法，创建一个长度为16的随机字符串 - // 使用 mongodb 为数据库驱动，相关操作参照其文档 - const result = await coll.insertOne({ - _id: pasteId, - owner: userId, - content, - isPrivate, - }); - return result.insertedId; // 返回插入的文档ID -} - -export async function get(pasteId: string): Promise { - return await coll.findOne({ _id: pasteId }); -} - -// 暴露这些接口 -global.Hydro.model.pastebin = { add, get }; - -``` - -## Step4 handler.js - -在路由中定义所有的函数应均为异步函数，支持的函数有：prepare, get, post, post[Operation], cleanup -具体流程如下： - -``` -先执行 prepare(args) （如果存在） -args 为传入的参数集合（包括 QueryString, Body, Path）中的全部参数， -再执行 prepare(args) （如果存在） -检查请求类型： - -为 GET ？ - -> 执行 get(args) -为 POST ? - -> 执行 post(args) - -> 含有 operation 字段？ - -> 执行 post[Operation] -``` - -执行 cleanup() - -如果在 this.response.template 指定模板则渲染，否则直接返回 this.response.body 中的内容。 - -* 在表单提交时的 operation 字段使用下划线，函数名使用驼峰命名。 - -如 `` 对应 `postConfirmDelete` 函数。 - -应当提供 `apply` 函数，并与定义的 Handler 一同挂载到 `global.Hydro.handler[模块名]` 位置。 -`apply` 函数将在初始化阶段被调用。 - -```ts -import { Route, Handler } from 'hydrooj/dist/service/server'; -import { PRIV } from 'hydrooj/dist/model/builtin'; // 内置 Privilege 权限节点 -import { isContent } from 'hydrooj/dist/lib/validator'; // 用于检查用户输入是否合法 -import { NotFoundError } from 'hydrooj/dist/error'; -import * as pastebin from './pastebin'; // 刚刚编写的pastebin模型 - -// 创建新路由 -class PasteCreateHandler extends Handler { - // Get请求时触发该函数 - async get() { - // 检查用户是否登录，此处为多余（因为底部注册路由时已声明所需权限） - // 此方法适用于权限的动态检查 - // this.checkPriv(PRIV.PRIV_USER_PROFILE); - this.response.template = 'paste_create.html'; // 返回此页面 - } - - // 使用 isContent 检查输入 - @param('content', Types.String, isContent) - @param('private', Types.Boolean) - // 从用户提交的表单中取出content和private字段 - // domainId 为固定传入参数 - async post(domainId: string, content: string, private = false) { - // 在HTML表单提交的多选框中，选中值为 'on'，未选中则为空，需要进行转换 - await pastebin.add(this.user._id, content, !!private); - // 将用户重定向到创建完成的url - this.response.redirect = this.url('paste_show', { id: pasteid }); - } -} - -class PasteShowHandler extends Handler { - @param('id', Types.String) - async get(domainId: string, id: string) { - const doc = await pastebin.get(id); - if (!doc) throw new NotFoundError(id); - if (doc.isPrivate && this.user._id !== doc.owner) { - throw new PermissionError(); - } - this.response.body = { doc }; - this.response.template = 'paste_show.html'; - } - - @param('id', Types.String) - async postDelete(domainId: string, id: string){ - // 当提交表单并存在 operation 值为 delete 时执行。 - // 本例中未实现删除功能，仅作为说明。 - } -} - -// Hydro会在服务初始化完成后调用该函数。 -export async function apply(){ - // 注册一个名为 paste_create 的路由，匹配 '/paste/create'， - // 使用PasteCreateHandler处理，访问改路由需要PRIV.PRIV_USER_PROFILE权限 - // 提示：路由匹配基于 path-to-regexp - Route('paste_create', '/paste/create', PasteCreateHandler, PRIV.PRIV_USER_PROFILE); - Route('paste_show', '/paste/show/:id', PasteShowHandler); -} - -global.Hydro.handler.pastebin = apply; -``` - -## Step5 template - -TODO - -## Step6 Locale - -用于提供多国翻译。格式与 Hydro 的 locale 文件夹格式相同。 diff --git a/docs/addon.md b/docs/addon.md deleted file mode 100644 index d603e98f8..000000000 --- a/docs/addon.md +++ /dev/null @@ -1,175 +0,0 @@ -# Hydro 附加组件开发 - -前置条件：NodeJS>10.10 -此教程将以编写剪贴板插件为例进行说明。 - -## Step1 初始化项目 - -在一个空文件夹中运行 `yarn init` 并按照提示填写相关信息。 - -```sh -/workspace/hydro-plugin $ yarn init -yarn init v1.22.4 -question name (hydro-plugin): @hydrooj/pastebin -question version (1.0.0): 0.0.1 -question description: HydroOJ的剪贴板组件 -question entry point (index.js): package.json -question repository url: https://github.com/hydro-dev/pastebin.git -question author: undefined -question license (MIT): MIT -question private: -success Saved package.json -``` - -## Step2 准备编写组件 - -分析：剪贴板组件需要以下功能： - -- 与数据库交互来存储/检索相应文档。 -- 提供 /paste/create 路由以创建新文档。 -- 提供 /paste/show/:ID 来查看已创建的文档。 -- 根据用户ID进行鉴权，允许将文档设置为私密以防止他人查看。 - -Hydro的推荐架构如下： - -- handler.js: 用于处理路由 -- model.js: 数据库模型 -- lib.js: 不依赖于数据库等的库（如md5函数） -- script.js: 可能会被用户多次使用到的脚本（如重新计算rp） -- locale/: 翻译文件 -- template/: 页面模板 -- setting.yaml: 模块所用到的设置，格式在下方说明 - -## Step3 model.js - -提示：由于模块中不便于使用 require() 引入 Hydro 的文件，可以从 global.Hydro 中取出需要的模块。 - -```js -const { db } = global.Hydro.service; // 数据库连接 -const coll = db.collection('paste'); - -/** - * 添加一个文档 - * @param {number} userId - * @param {string} content - * @param {boolean} isPrivate - * @return {Promise} - */ -async function add(userId, content, isPrivate) { - const pasteId = String.random(16); // Hydro提供了此方法，创建一个长度为16的随机字符串 - // 使用 mongodb 为数据库驱动，相关操作参照其文档 - const result = await coll.insertOne({ - _id: pasteId, - owner: userId, - content, - isPrivate, - }); - return result.insertedId; // 返回插入的文档ID -} - -/** - * 查询一个文档 - * @param {string} pasteId - * @return {Promise} - */ -async function get(pasteId) { - return await coll.findOne({ _id: pasteId }); -} - -// 暴露这些接口 -global.Hydro.model.pastebin = { add, get }; - -``` - -## Step4 handler.js - -在路由中定义所有的函数应均为异步函数，支持的函数有：prepare, get, post, post[Operation], cleanup -具体流程如下： - -``` -先执行 prepare(args) （如果存在） -args 为传入的参数集合（包括 QueryString, Body, Path）中的全部参数， -再执行 prepare(args) （如果存在） -检查请求类型： - -为 GET ？ - -> 执行 get(args) -为 POST ? - -> 执行 post(args) - -> 含有 operation 字段？ - -> 执行 post[Operation] -``` - -执行 cleanup() - -如果在 this.response.template 指定模板则渲染，否则直接返回 this.response.body 中的内容。 - -* 在表单提交时的 operation 字段使用下划线，函数名使用驼峰命名。 - -如 `` 对应 `postConfirmDelete` 函数。 - -应当提供 `apply` 函数，并与定义的 Handler 一同挂载到 `global.Hydro.handler[模块名]` 位置。 -`apply` 函数将在初始化阶段被调用。 - -```js -const { Route, Handler } = global.Hydro.service.server; // 注册路由所用工具 -const { PRIV } = global.Hydro.model.builtin; // 内置 Privilege 权限节点 -const pastebin = global.Hydro.model.pastebin; // 刚刚编写的pastebin模型 -const { checkContent } = global.Hydro.lib.validator; // 用于检查用户输入是否合法 -const { NotFoundError } = global.Hydro.error; - -// 创建新路由 -class PasteCreateHandler extends Handler { - // Get请求时触发该函数 - async get() { - // 检查用户是否登录，此处为多余（因为底部注册路由时已声明所需权限） - // 此方法适用于权限的动态检查 - // this.checkPriv(PRIV.PRIV_USER_PROFILE); - this.response.template = 'paste_create.html'; // 返回此页面 - } - - async post({ content, private = false }) { // 从用户提交的表单中取出content和private字段 - checkContent(content); // 检查输入 - // 在HTML表单提交的多选框中，选中值为 'on'，未选中则为空，需要进行转换 - await pastebin.add(this.user._id, content, !!private); - // 将用户重定向到创建完成的url - this.response.redirect = this.url('paste_show', { id: pasteid }); - } -} - -class PasteShowHandler extends Handler { - async get({ id }) { - const doc = await pastebin.get(id); - if (!doc) throw new NotFoundError(id); - if (doc.isPrivate) { - if (this.user._id !== doc.owner) throw new PermissionError(); - } - this.response.body = { doc }; - this.response.template = 'paste_show.html'; - } - - async postDelete({ id }){ - // 当提交表单并存在 operation 值为 delete 时执行。 - // 本例中未实现删除功能，仅作为说明。 - } -} - -// Hydro会在服务初始化完成后调用该函数。 -async function apply(){ - // 注册一个名为 paste_create 的路由，匹配 '/paste/create'， - // 使用PasteCreateHandler处理，访问改路由需要PRIV.PRIV_USER_PROFILE权限 - // 提示：路由匹配基于 path-to-regexp - Route('paste_create', '/paste/create', PasteCreateHandler, PRIV.PRIV_USER_PROFILE); - Route('paste_show', '/paste/show/:id', PasteShowHandler); -} - -global.Hydro.handler.pastebin = apply; -``` - -## Step5 template - -TODO - -## Step6 Locale - -用于提供多国翻译。格式与 Hydro 的 locale 文件夹格式相同。 diff --git a/docs/development.md b/docs/development.md deleted file mode 100644 index 72953b184..000000000 --- a/docs/development.md +++ /dev/null @@ -1,4 +0,0 @@ -# Hydro 开发 - -Hydro 使用 Typescript 开发。代码风格遵循 airbnb 标准，indent=4spaces。 -详见 .eslintrc.js diff --git a/package.json b/package.json index ea3ba5e0c..13a9c1675 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,8 @@ "devDependencies": { "@shelf/jest-mongodb": "^1.2.3", "@types/cross-spawn": "^6.0.2", - "@types/jest": "^26.0.18", - "@types/node": "^14.14.10", + "@types/jest": "^26.0.19", + "@types/node": "^14.14.13", "@types/semver": "^7.3.4", "@typescript-eslint/eslint-plugin": "^3.9.1", "@typescript-eslint/parser": "^3.9.1", @@ -42,4 +42,4 @@ "fs-extra": "^9.0.1", "yargs": "^16.2.0" } -} \ No newline at end of file +} diff --git a/packages/hydrojudge/package.json b/packages/hydrojudge/package.json index 2ce13a999..75ac91fc6 100644 --- a/packages/hydrojudge/package.json +++ b/packages/hydrojudge/package.json @@ -15,7 +15,7 @@ "p-queue": "^6.6.1", "shell-quote": "^1.7.2", "supports-color": "^8.0.0", - "systeminformation": "^4.31.0", + "systeminformation": "^4.31.1", "yargs": "^16.2.0" }, "peerDependencies": { @@ -24,7 +24,7 @@ "license": "AGPL-3.0-only", "devDependencies": { "@types/bson": "^4.0.2", - "@types/fs-extra": "^9.0.4", + "@types/fs-extra": "^9.0.5", "@types/js-yaml": "^3.12.5", "@types/shell-quote": "^1.7.0" }, @@ -32,4 +32,4 @@ "build": "tsc", "lint": "eslint . --ext ts --fix" } -} \ No newline at end of file +} diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index 96847866d..01f821f4f 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "2.14.8", + "version": "2.14.9", "bin": "bin/hydrooj.js", "main": "dist/loader.js", "typings": "dist/loader.d.ts", @@ -23,30 +23,32 @@ "koa-static-cache": "^5.1.3", "lodash": "^4.17.20", "lru-cache": "^6.0.0", + "minio": "^7.0.17", "moment-timezone": "^0.5.32", "mongodb": "^3.6.3", - "nodemailer": "^6.4.16", + "nodemailer": "^6.4.17", "ps-list": "^7.2.0", "serialize-javascript": "^5.0.1", "sockjs": "^0.3.20", "superagent": "^6.1.0", "supports-color": "^8.0.0", - "systeminformation": "^4.31.0", + "systeminformation": "^4.31.1", "wtfnode": "^0.8.4", "yargs": "^16.2.0" }, "devDependencies": { "@types/adm-zip": "^0.4.33", - "@types/fs-extra": "^9.0.4", + "@types/fs-extra": "^9.0.5", "@types/js-yaml": "^3.12.5", "@types/koa": "^2.11.4", "@types/koa-router": "^7.4.1", "@types/koa-static-cache": "^5.1.0", "@types/lodash": "^4.14.165", "@types/lru-cache": "^5.1.0", + "@types/minio": "^7.0.6", "@types/mongodb": "^3.5.33", "@types/nodemailer": "^6.4.0", - "@types/serialize-javascript": "^4.0.0", + "@types/serialize-javascript": "^5.0.0", "@types/sockjs": "^0.3.32", "@types/superagent": "^4.1.8", "@types/supports-color": "^7.2.0", diff --git a/packages/hydrooj/src/check.ts b/packages/hydrooj/src/check.ts index 86684c2b1..eecaf85b3 100644 --- a/packages/hydrooj/src/check.ts +++ b/packages/hydrooj/src/check.ts @@ -3,7 +3,7 @@ import os from 'os'; import { Dictionary } from 'lodash'; import * as system from './model/system'; -import * as db from './service/db'; +import db from './service/db'; const c = {}; diff --git a/packages/hydrooj/src/entry/cli.ts b/packages/hydrooj/src/entry/cli.ts index 275b820ce..c5b79a816 100644 --- a/packages/hydrooj/src/entry/cli.ts +++ b/packages/hydrooj/src/entry/cli.ts @@ -5,7 +5,9 @@ import { lib, service, model, builtinLib, builtinModel, } from './common'; +import options from '../options'; import * as bus from '../service/bus'; +import db from '../service/db'; const COMMENTS = /((\/\/.*$)|(\/\*[\s\S]*?\*\/))/mg; const ARR = /=>.*$/mg; @@ -52,10 +54,8 @@ export async function load() { require('../utils'); require('../error'); require('../options'); - await new Promise((resolve) => { - bus.once('database/connect', resolve); - require('../service/db'); - }); + const opts = options(); + await db.start(opts); for (const i of builtinLib) require(`../lib/${i}`); await lib(pending, fail); require('../service/gridfs'); diff --git a/packages/hydrooj/src/entry/master.ts b/packages/hydrooj/src/entry/master.ts index 41132e17c..40f9e2357 100644 --- a/packages/hydrooj/src/entry/master.ts +++ b/packages/hydrooj/src/entry/master.ts @@ -8,7 +8,9 @@ import { argv } from 'yargs'; import { builtinModel } from './common'; import { Entry } from '../loader'; import { Logger } from '../logger'; +import options from '../options'; import * as bus from '../service/bus'; +import db from '../service/db'; const logger = new Logger('entry/master'); const tmpdir = path.resolve(os.tmpdir(), 'hydro'); @@ -48,13 +50,9 @@ export async function load(call: Entry) { bus.once('app/exit', () => { fs.removeSync(lockfile); }); - await new Promise((resolve) => { - bus.once('database/connect', () => { - bus.once('database/config', resolve); - require('../model/system'); - }); - require('../service/db'); - }); + const opts = options(); + await db.start(opts); + await require('../model/system').runConfig(); require('../service/monitor'); for (const i of builtinModel) require(`../model/${i}`); const modelSystem = require('../model/system'); diff --git a/packages/hydrooj/src/entry/worker.ts b/packages/hydrooj/src/entry/worker.ts index 7b28431d5..29f2611bd 100644 --- a/packages/hydrooj/src/entry/worker.ts +++ b/packages/hydrooj/src/entry/worker.ts @@ -4,7 +4,9 @@ import { locale, template, lib, service, model, handler, script, setting, uistatic, builtinLib, builtinScript, builtinHandler, builtinModel, } from './common'; +import options from '../options'; import * as bus from '../service/bus'; +import db from '../service/db'; export async function load() { const pending = global.addons; @@ -19,12 +21,11 @@ export async function load() { template(pending, fail), uistatic(pending, fail), ]); + const opts = options(); + await db.start(opts); await new Promise((resolve) => { - bus.once('database/connect', () => { - bus.once('database/config', resolve); - require('../model/system'); - }); - require('../service/db'); + bus.once('database/config', resolve); + require('../model/system'); }); for (const i of builtinLib) require(`../lib/${i}`); await lib(pending, fail); diff --git a/packages/hydrooj/src/handler/misc.ts b/packages/hydrooj/src/handler/misc.ts index 90aff1dcc..ea78a5a96 100644 --- a/packages/hydrooj/src/handler/misc.ts +++ b/packages/hydrooj/src/handler/misc.ts @@ -5,7 +5,7 @@ import { PRIV } from '../model/builtin'; import * as system from '../model/system'; import * as file from '../model/file'; import * as user from '../model/user'; -import * as db from '../service/db'; +import db from '../service/db'; import { Route, Handler, Types, param, } from '../service/server'; diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 895a9f141..00d7d0761 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -537,6 +537,7 @@ export interface Service { fs: HydroFileSystem, monitor: typeof import('./service/monitor'), server: typeof import('./service/server'), + storage: typeof import('./service/storage'), } interface GeoIP { diff --git a/packages/hydrooj/src/model/blacklist.ts b/packages/hydrooj/src/model/blacklist.ts index c6e3f4b22..7164f2c55 100644 --- a/packages/hydrooj/src/model/blacklist.ts +++ b/packages/hydrooj/src/model/blacklist.ts @@ -1,4 +1,4 @@ -import * as db from '../service/db'; +import db from '../service/db'; import * as bus from '../service/bus'; const coll = db.collection('blacklist'); diff --git a/packages/hydrooj/src/model/document.ts b/packages/hydrooj/src/model/document.ts index 55b4e07f0..18f45de54 100644 --- a/packages/hydrooj/src/model/document.ts +++ b/packages/hydrooj/src/model/document.ts @@ -8,7 +8,7 @@ import { } from '../interface'; import { buildProjection } from '../utils'; import { NumberKeys, ArrayKeys, Projection } from '../typeutils'; -import * as db from '../service/db'; +import db from '../service/db'; import * as bus from '../service/bus'; import { Content } from '../loader'; diff --git a/packages/hydrooj/src/model/domain.ts b/packages/hydrooj/src/model/domain.ts index c3ea675e3..7efb668e3 100644 --- a/packages/hydrooj/src/model/domain.ts +++ b/packages/hydrooj/src/model/domain.ts @@ -3,7 +3,7 @@ import { FilterQuery } from 'mongodb'; import { BUILTIN_ROLES, PRIV } from './builtin'; import { DomainDoc } from '../interface'; import * as bus from '../service/bus'; -import * as db from '../service/db'; +import db from '../service/db'; const coll = db.collection('domain'); const collUser = db.collection('domain.user'); diff --git a/packages/hydrooj/src/model/file.ts b/packages/hydrooj/src/model/file.ts index 2bbf4ed06..5dd20a55b 100644 --- a/packages/hydrooj/src/model/file.ts +++ b/packages/hydrooj/src/model/file.ts @@ -3,7 +3,7 @@ import * as fs from 'fs'; import { Dictionary } from 'lodash'; import { ForbiddenError, NotFoundError } from '../error'; import { Ufdoc } from '../interface'; -import * as db from '../service/db'; +import db from '../service/db'; import * as bus from '../service/bus'; import gridfs from '../service/gridfs'; import hash from '../lib/hash.hydro'; diff --git a/packages/hydrooj/src/model/message.ts b/packages/hydrooj/src/model/message.ts index a21d907e8..fa2781257 100644 --- a/packages/hydrooj/src/model/message.ts +++ b/packages/hydrooj/src/model/message.ts @@ -1,7 +1,7 @@ import { FilterQuery, ObjectID } from 'mongodb'; import * as user from './user'; import { Mdoc } from '../interface'; -import * as db from '../service/db'; +import db from '../service/db'; import * as bus from '../service/bus'; const coll = db.collection('message'); diff --git a/packages/hydrooj/src/model/oauth.ts b/packages/hydrooj/src/model/oauth.ts index f0f0c9b8f..2734f0b51 100644 --- a/packages/hydrooj/src/model/oauth.ts +++ b/packages/hydrooj/src/model/oauth.ts @@ -1,6 +1,6 @@ import Lru from 'lru-cache'; import { Collection } from 'mongodb'; -import * as db from '../service/db'; +import db from '../service/db'; interface OauthMap { _id: string, // source openId diff --git a/packages/hydrooj/src/model/opcount.ts b/packages/hydrooj/src/model/opcount.ts index 0acc17e11..09e72666b 100644 --- a/packages/hydrooj/src/model/opcount.ts +++ b/packages/hydrooj/src/model/opcount.ts @@ -1,6 +1,6 @@ import { OpcountExceededError } from '../error'; import * as bus from '../service/bus'; -import * as db from '../service/db'; +import db from '../service/db'; const coll = db.collection('opcount'); diff --git a/packages/hydrooj/src/model/oplog.ts b/packages/hydrooj/src/model/oplog.ts index 39a611a08..2b8f19fcc 100644 --- a/packages/hydrooj/src/model/oplog.ts +++ b/packages/hydrooj/src/model/oplog.ts @@ -1,6 +1,6 @@ import { ObjectID } from 'mongodb'; import { OplogDoc } from '../interface'; -import * as db from '../service/db'; +import db from '../service/db'; export const coll = db.collection('oplog'); diff --git a/packages/hydrooj/src/model/record.ts b/packages/hydrooj/src/model/record.ts index 61091376a..07e8f0097 100644 --- a/packages/hydrooj/src/model/record.ts +++ b/packages/hydrooj/src/model/record.ts @@ -8,7 +8,7 @@ import * as problem from './problem'; import { Rdoc, TestCase, RunConfig, ContestInfo, ProblemConfig, } from '../interface'; -import * as db from '../service/db'; +import db from '../service/db'; export const coll: Collection = db.collection('record'); diff --git a/packages/hydrooj/src/model/system.ts b/packages/hydrooj/src/model/system.ts index 195cd91e2..6a3c95ef6 100644 --- a/packages/hydrooj/src/model/system.ts +++ b/packages/hydrooj/src/model/system.ts @@ -1,7 +1,7 @@ import { SYSTEM_SETTINGS } from './setting'; import { NumberKeys } from '../typeutils'; import { SystemKeys } from '../interface'; -import * as db from '../service/db'; +import db from '../service/db'; import * as bus from '../service/bus'; const coll = db.collection('system'); @@ -56,16 +56,17 @@ export async function inc>(_id: K) { return res.value.value; } -(async () => { +export async function runConfig() { for (const setting of SYSTEM_SETTINGS) { if (setting.value) cache[setting.key] = setting.value; } const config = await coll.find({}).toArray(); for (const i of config) cache[i._id] = i.value; bus.emit('database/config'); -})(); +} global.Hydro.model.system = { + runConfig, get, getMany, inc, diff --git a/packages/hydrooj/src/model/task.ts b/packages/hydrooj/src/model/task.ts index 21f71bce7..bdbcc111c 100644 --- a/packages/hydrooj/src/model/task.ts +++ b/packages/hydrooj/src/model/task.ts @@ -2,7 +2,7 @@ import moment from 'moment-timezone'; import { FilterQuery, ObjectID } from 'mongodb'; import { Task } from '../interface'; import { Logger } from '../logger'; -import * as db from '../service/db'; +import db from '../service/db'; const logger = new Logger('model/task'); const coll = db.collection('task'); diff --git a/packages/hydrooj/src/model/token.ts b/packages/hydrooj/src/model/token.ts index e76a75c8f..eef72c3df 100644 --- a/packages/hydrooj/src/model/token.ts +++ b/packages/hydrooj/src/model/token.ts @@ -1,5 +1,5 @@ import { TokenDoc } from '../interface'; -import * as db from '../service/db'; +import db from '../service/db'; import * as bus from '../service/bus'; export const coll = db.collection('token'); diff --git a/packages/hydrooj/src/model/user.ts b/packages/hydrooj/src/model/user.ts index ea21eaf48..06eab1461 100644 --- a/packages/hydrooj/src/model/user.ts +++ b/packages/hydrooj/src/model/user.ts @@ -9,7 +9,7 @@ import { User as _User, Udoc, Udict } from '../interface'; import { Value } from '../typeutils'; import { Logger } from '../logger'; import pwhash from '../lib/hash.hydro'; -import * as db from '../service/db'; +import db from '../service/db'; import * as bus from '../service/bus'; const coll: Collection = db.collection('user'); diff --git a/packages/hydrooj/src/script/blacklist.ts b/packages/hydrooj/src/script/blacklist.ts index e284000a2..f9b53daea 100644 --- a/packages/hydrooj/src/script/blacklist.ts +++ b/packages/hydrooj/src/script/blacklist.ts @@ -4,7 +4,7 @@ import * as document from '../model/document'; import * as discussion from '../model/discussion'; import * as user from '../model/user'; import * as blacklist from '../model/blacklist'; -import * as db from '../service/db'; +import db from '../service/db'; export const description = 'Add blacklist by ip, uid'; diff --git a/packages/hydrooj/src/script/deleteUser.ts b/packages/hydrooj/src/script/deleteUser.ts index 27e6d0eb2..e547ecca1 100644 --- a/packages/hydrooj/src/script/deleteUser.ts +++ b/packages/hydrooj/src/script/deleteUser.ts @@ -4,7 +4,7 @@ import { Value } from '../typeutils'; import * as user from '../model/user'; import * as document from '../model/document'; import * as setting from '../model/setting'; -import * as db from '../service/db'; +import db from '../service/db'; import paginate from '../lib/paginate'; import { STATUS, PRIV } from '../model/builtin'; diff --git a/packages/hydrooj/src/script/problemStat.ts b/packages/hydrooj/src/script/problemStat.ts index 265ebf63a..6bd007140 100644 --- a/packages/hydrooj/src/script/problemStat.ts +++ b/packages/hydrooj/src/script/problemStat.ts @@ -1,4 +1,4 @@ -import * as db from '../service/db'; +import db from '../service/db'; import { STATUS } from '../model/builtin'; import * as document from '../model/document'; diff --git a/packages/hydrooj/src/service/db.ts b/packages/hydrooj/src/service/db.ts index d763b58a3..424e12e17 100644 --- a/packages/hydrooj/src/service/db.ts +++ b/packages/hydrooj/src/service/db.ts @@ -1,34 +1,43 @@ -import mongodb, { Collection } from 'mongodb'; +import { Collection, Db, MongoClient } from 'mongodb'; import * as bus from './bus'; -import options from '../options'; import { Collections } from '../interface'; -const opts = options(); - -let mongourl = 'mongodb://'; -if (opts.username) mongourl += `${opts.username}:${opts.password}@`; -mongourl += `${opts.host}:${opts.port}/${opts.name}`; - -// eslint-disable-next-line import/no-mutable-exports -export let db: mongodb.Db = null; -// eslint-disable-next-line import/no-mutable-exports -export let db2: mongodb.Db = null; - -mongodb.MongoClient.connect(mongourl, { useNewUrlParser: true, useUnifiedTopology: true }) - .then((Client) => { - db = Client.db(opts.name); - global.Hydro.service.db.db = db; - mongodb.MongoClient.connect(mongourl, { useNewUrlParser: true, useUnifiedTopology: true }) - .then((Client1) => { - db2 = Client1.db(opts.name); - global.Hydro.service.db.db2 = db2; - bus.parallel('database/connect', db); - }); - }); - -export function collection(c: K): Collection { - if (opts.prefix) return db.collection(`${opts.prefix}.${c}`); - return db.collection(c); +interface MongoConfig { + username?: string, + password?: string, + host?: string, + port?: string, + name?: string, + url?: string, + prefix?: string, } -global.Hydro.service.db = { collection, db, db2 }; +class MongoService { + public client: MongoClient; + public client2: MongoClient; + public db: Db; + public db2: Db; + private opts: MongoConfig; + + async start(opts: MongoConfig) { + this.opts = opts; + let mongourl = 'mongodb://'; + if (opts.username) mongourl += `${opts.username}:${opts.password}@`; + mongourl += `${opts.host}:${opts.port}/${opts.name}`; + if (opts.url) mongourl = opts.url; + this.client = await MongoClient.connect(mongourl, { useNewUrlParser: true, useUnifiedTopology: true }); + this.db = this.client.db(opts.name); + this.client2 = await MongoClient.connect(mongourl, { useNewUrlParser: true, useUnifiedTopology: true }); + this.db2 = this.client2.db(opts.name); + bus.parallel('database/connect', this.db); + } + + public collection(c: K): Collection { + if (this.opts.prefix) return this.db.collection(`${this.opts.prefix}.${c}`); + return this.db.collection(c); + } +} + +const service = new MongoService(); +global.Hydro.service.db = service; +export = service; diff --git a/packages/hydrooj/src/service/gridfs.ts b/packages/hydrooj/src/service/gridfs.ts index b0779045d..cae1012fe 100644 --- a/packages/hydrooj/src/service/gridfs.ts +++ b/packages/hydrooj/src/service/gridfs.ts @@ -1,8 +1,8 @@ import { GridFSBucket } from 'mongodb'; -import { db2 } from './db'; +import db from './db'; import { HydroFileSystem } from '../interface'; -const fs = new GridFSBucket(db2); +const fs = new GridFSBucket(db.db2); // @ts-ignore global.Hydro.service.fs = fs; diff --git a/packages/hydrooj/src/service/monitor.ts b/packages/hydrooj/src/service/monitor.ts index e16b2f204..802def325 100644 --- a/packages/hydrooj/src/service/monitor.ts +++ b/packages/hydrooj/src/service/monitor.ts @@ -2,7 +2,7 @@ import cluster from 'cluster'; import crypto from 'crypto'; import axios from 'axios'; import { safeDump } from 'js-yaml'; -import * as db from './db'; +import db from './db'; import * as bus from './bus'; import * as sysinfo from '../lib/sysinfo'; diff --git a/packages/hydrooj/src/service/storage.ts b/packages/hydrooj/src/service/storage.ts new file mode 100644 index 000000000..eea40e3fa --- /dev/null +++ b/packages/hydrooj/src/service/storage.ts @@ -0,0 +1,68 @@ +import { Readable } from 'stream'; +import Minio, { BucketItem, ItemBucketMetadata } from 'minio'; + +interface StorageOptions { + host?: string, + port?: number, + useSSL?: boolean, + accessKey?: string, + secretKey?: string, + bucket?: string, + region?: string, +} + +const defaultConfig: StorageOptions = { + host: 'play.min.io', + port: 9000, + useSSL: true, + accessKey: 'Q3AM3UQ867SPQQA43P2F', + secretKey: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG', + bucket: 'hydro', + region: 'us-east-1', +}; + +class StorageService { + public client: Minio.Client; + private opts: StorageOptions; + + async start(opts: StorageOptions) { + this.opts = { ...defaultConfig, ...opts }; + this.client = new Minio.Client({ + endPoint: this.opts.host, + port: this.opts.port, + useSSL: this.opts.useSSL, + accessKey: this.opts.accessKey, + secretKey: this.opts.secretKey, + }); + const exists = await this.client.bucketExists(this.opts.bucket); + if (!exists) await this.client.makeBucket(this.opts.bucket, this.opts.region); + } + + async put(target: string, file: string | Buffer | Readable, meta: ItemBucketMetadata = {}) { + if (typeof file === 'string') return await this.client.fPutObject(this.opts.bucket, target, file, meta); + return await this.client.putObject(this.opts.bucket, target, file, meta); + } + + async get(target: string) { + return await this.client.getObject(this.opts.bucket, target); + } + + async del(target: string | string[]) { + if (typeof target === 'string') return await this.client.removeObject(this.opts.bucket, target); + return await this.client.removeObjects(this.opts.bucket, target); + } + + async list(target: string, recursive = false) { + const stream = this.client.listObjects(this.opts.bucket, target, recursive); + return await new Promise((resolve, reject) => { + const results: BucketItem[] = []; + stream.on('data', (result) => results.push(result)); + stream.on('end', () => resolve(results)); + stream.on('error', reject); + }); + } +} + +const service = new StorageService(); +global.Hydro.service.storage = service; +export = service; diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index e745a0d69..e7fe694a5 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -1,7 +1,7 @@ /* eslint-disable no-await-in-loop */ import * as domain from './model/domain'; import * as document from './model/document'; -import { db } from './service/db'; +import db from './service/db'; type UpgradeScript = () => Promise;