update

.eslintrc.js

@@ -20,6 +20,7 @@ module.exports = {
         'guard-for-in': 'off',
         'implicit-arrow-linebreak': 'off',
         'import/extensions': 'off',
+        'import/no-named-as-default-member': 'off',
         'import/prefer-default-export': 'off',
         indent: ['warn', 4],
         'max-classes-per-file': 'off',

.github/workflows/build.yml

@@ -11,6 +11,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     env:
+      LICENSE_KEY: ${{ secrets.MAXMIND_KEY }}
       NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
     steps:
     - name: Check out

.gitignore

@@ -14,9 +14,12 @@ build/*.js
 *.js.map
 *.hydro
 !.gitkeep
-tsconfig.tsbuildinfo
+*.tsbuildinfo
 yarn.lock
 
 packages/**/*.js
+!**/bin/*.js
 packages/**/*.js.map
 packages/**/*.d.ts
+
+*.mmdb

build/start.ts

@@ -0,0 +1,39 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import os from 'os';
+import path from 'path';
+import cluster from 'cluster';
+import fs from 'fs-extra';
+import { filter } from 'lodash';
+import { argv } from 'yargs';
+import * as hydro from 'hydrooj';
+
+if (!cluster.isMaster) {
+    // Forked by hydro
+    hydro.load().catch((e) => {
+        console.error(e);
+        process.exit(1);
+    });
+} else {
+    fs.ensureDirSync(path.resolve(os.homedir(), '.hydro'));
+    const addonPath = path.resolve(os.homedir(), '.hydro', 'addon.json');
+    let addons = filter(
+        fs.readdirSync(path.resolve(__dirname, '../packages')),
+        (i) => i !== 'hydrooj',
+    ).map((i) => `@hydrooj/${i}`);
+    fs.writeFileSync(addonPath, JSON.stringify(addons, null, 2));
+
+    try {
+        const ui = argv.ui as string || '@hydrooj/ui-default';
+        require.resolve(ui);
+        addons.push(ui);
+    } catch (e) {
+        console.error('Please also install @hydrooj/ui-default');
+    }
+
+    addons = Array.from(new Set(addons));
+    for (const addon of addons) hydro.addon(addon);
+    (argv._[0] === 'cli' ? hydro.loadCli : hydro.load)().catch((e) => {
+        console.error(e);
+        process.exit(1);
+    });
+}

package.json

@@ -31,5 +31,9 @@
         "semver": "^7.3.2",
         "ts-jest": "^26.1.4",
         "typescript": "^3.9.7"
+    },
+    "dependencies": {
+        "fs-extra": "^9.0.1",
+        "yargs": "^15.4.1"
     }
 }

packages/geoip/README.md

@@ -0,0 +1,5 @@
+# GeoIp
+
+显示用户登录地。  
+
+IP geo-location data is provided by [Maxmind](http://www.maxmind.com)  

packages/geoip/lib.ts

@@ -0,0 +1,26 @@
+import fs from 'fs';
+import path from 'path';
+import { Reader } from 'maxmind';
+
+const buffer = fs.readFileSync(path.resolve(__dirname, 'GeoLite2-City.mmdb'));
+const reader = new Reader(buffer);
+
+function lookup(ip: string, locale: string) {
+    const res: any = reader.get(ip);
+    if (!res) return {};
+    const ret: any = {};
+    if (res.location) ret.location = res.location;
+    if (res.continent) ret.continent = res.continent.names[locale] || res.continent.names.en;
+    if (res.country || res.registered_country) {
+        ret.country = (res.country || res.registered_country).names[locale]
+            || (res.country || res.registered_country).names.en;
+    }
+    if (res.city) ret.city = res.city.names[locale] || res.city.names.en;
+    ret.display = `${ret.continent} ${ret.country} ${ret.city}`;
+    return ret;
+}
+
+global.Hydro.lib.geoip = exports = {
+    provider: '<a href="http://www.maxmind.com" target="_blank">MaxMind</a>',
+    lookup,
+};

packages/geoip/locale/zh_CN.yaml

@@ -0,0 +1 @@
+geoip_locale: zh-CN

packages/geoip/locale/zh_TW.yaml

@@ -0,0 +1 @@
+geoip_locale: zh-CN

packages/geoip/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "@hydrooj/geoip",
+  "version": "1.1.6",
+  "main": "package.json",
+  "repository": "git@github.com:hydro-dev/Hydro.git",
+  "author": "undefined <masnn0@outlook.com>",
+  "license": "MIT",
+  "dependencies": {
+    "maxmind": "^4.1.3"
+  },
+  "peerDependencies": {
+    "hydrooj": "^2.12.0"
+  }
+}

packages/geoip/prepublish.sh

@@ -0,0 +1,4 @@
+wget -O ./GeoLite2-City.tar.gz "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${LICENSE_KEY}&suffix=tar.gz"
+tar zxvf ./GeoLite2-City.tar.gz -C .
+mv ./GeoLite2-City_*/GeoLite2-City.mmdb ./GeoLite2-City.mmdb
+rm -r ./GeoLite2-City_* ./GeoLite2-City.tar.gz

packages/hydrooj/bin/hydrooj.js

@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+
+const os = require('os');
+const path = require('path');
+const cluster = require('cluster');
+const fs = require('fs-extra');
+const { argv } = require('yargs');
+const hydro = require('../dist/loader');
+
+if (!cluster.isMaster) {
+    // Forked by hydro
+    hydro.load().catch((e) => {
+        console.error(e);
+        process.exit(1);
+    });
+} else {
+    fs.ensureDirSync(path.resolve(os.homedir(), '.hydro'));
+    const addonPath = path.resolve(os.homedir(), '.hydro', 'addon.json');
+    if (!fs.existsSync(addonPath)) fs.writeFileSync(addonPath, '[]');
+    let addons = JSON.parse(fs.readFileSync(addonPath).toString());
+
+    try {
+        const ui = argv.ui || '@hydrooj/ui-default';
+        require.resolve(ui);
+        addons.push(ui);
+    } catch (e) {
+        console.error('Please also install @hydrooj/ui-default');
+    }
+
+    if (argv._[0] && argv._[0] !== 'cli') {
+        const operation = argv._[0];
+        const arg1 = argv._[1];
+        const arg2 = argv._[2];
+        if (operation === 'addon') {
+            if (arg1 === 'add') addons.push(arg2);
+            else if (arg1 === 'remove') {
+                for (let i = 0; i < addons.length; i++) {
+                    if (addons[i] === arg2) {
+                        addons.splice(i, 1);
+                        break;
+                    }
+                }
+            }
+            addons = Array.from(new Set(addons));
+            console.log('Current Addons: ', addons);
+            fs.writeFileSync(addonPath, JSON.stringify(addons, null, 2));
+        }
+    } else {
+        addons = Array.from(new Set(addons));
+        for (const addon of addons) hydro.addon(addon);
+        (argv._[0] === 'cli' ? hydro.loadCli : hydro.load)().catch((e) => {
+            console.error(e);
+            process.exit(1);
+        });
+    }
+}

packages/hydrooj/package.json

@@ -1,8 +1,8 @@
 {
     "name": "hydrooj",
-    "version": "2.12.2",
+    "version": "2.12.3",
-    "main": "dist/loader.js",
     "bin": "bin/hydrooj.js",
+    "main": "dist/loader.js",
     "repository": "https://github.com/hydro-dev/Hydro.git",
     "author": "masnn",
     "license": "GPL-3.0-only",
@@ -40,7 +40,7 @@
         "ps-list": "^7.2.0",
         "serialize-javascript": "^4.0.0",
         "sockjs": "^0.3.20",
-        "superagent": "^5.2.2",
+        "superagent": "^5.3.1",
         "systeminformation": "^4.26.10",
         "yargs": "^15.4.1"
     },

packages/hydrooj/src/entry/master.ts

@@ -58,7 +58,10 @@ export async function load(call: Entry) {
         await ins.run({ username: 'Root', password: 'rootroot' });
     }
     for (const postInit of global.Hydro.postInit) {
-        await postInit().catch(console.error);
+        await postInit().catch((e) => {
+            console.error('Error running postInit:', postInit.toString());
+            console.error(e);
+        });
     }
     return await modelSystem.get('server.worker');
 }

packages/hydrooj/src/handler/contest.ts

@@ -118,7 +118,7 @@ class ContestBoardcastHandler extends ContestHandler {
         const tdoc = await contest.get(domainId, tid);
         if (tdoc.owner !== this.user._id) throw new PermissionError('Boardcast Message');
         const tsdocs = await contest.getMultiStatus(domainId, { docId: tid }).toArray();
-        const uids = Array.from(new Set(tsdocs.map((tsdoc) => tsdoc.uid)));
+        const uids: number[] = Array.from(new Set(tsdocs.map((tsdoc) => tsdoc.uid)));
         await Promise.all(
             uids.map((uid) => message.send(this.user._id, uid, content, message.FLAG_ALERT)),
         );
@@ -343,7 +343,7 @@ class ContestCodeHandler extends ContestHandler {
             },
         }).toArray();
         for (const rdoc of rdocs) {
-            zip.addFile(`${rnames[rdoc._id]}.${rdoc.lang}`, rdoc.code);
+            zip.addFile(`${rnames[rdoc._id.toHexString()]}.${rdoc.lang}`, Buffer.from(rdoc.code));
         }
         await this.binary(zip.toBuffer(), `${tdoc.title}.zip`);
     }

packages/hydrooj/src/handler/homework.ts

@@ -395,7 +395,7 @@ class HomeworkCodeHandler extends HomeworkHandler {
             },
         }).toArray();
         for (const rdoc of rdocs) {
-            zip.addFile(`${rnames[rdoc._id]}.${rdoc.lang}`, rdoc.code);
+            zip.addFile(`${rnames[rdoc._id.toHexString()]}.${rdoc.lang}`, Buffer.from(rdoc.code));
         }
         await this.binary(zip.toBuffer(), `${tdoc.title}.zip`);
     }

packages/hydrooj/src/handler/user.ts

@@ -18,6 +18,7 @@ import { isEmail, isPassword, isUname } from '../lib/validator';
 import { sendMail } from '../lib/mail';
 import * as misc from '../lib/misc';
 import paginate from '../lib/paginate';
+import { User } from '../interface';
 
 class UserLoginHandler extends Handler {
     async get() {
@@ -201,12 +202,12 @@ class UserSearchHandler extends Handler {
     @param('exectMatch', Types.Boolean)
     async get(domainId: string, q: string, exactMatch = false) {
         let udoc = await user.getById(domainId, parseInt(q, 10));
-        const udocs = udoc ? [udoc] : [];
+        const udocs: User[] = udoc ? [udoc] : [];
         udoc = await user.getByUname(domainId, q);
         if (udoc) udocs.push(udoc);
         udoc = await user.getByEmail(domainId, q);
         if (udoc) udocs.push(udoc);
-        if (!exactMatch) udocs.push(...await user.getPrefixList(q, 20));
+        if (!exactMatch) udocs.push(...await user.getPrefixList(domainId, q, 20));
         for (const i in udocs) {
             udocs[i].gravatar = misc.gravatar(udocs[i].gravatar || '');
         }

packages/hydrooj/src/interface.ts

@@ -247,9 +247,10 @@ export interface Mdoc {
 export interface Ufdoc {
     _id: ObjectID,
     secret: string,
-    md5: string,
+    md5?: string,
     owner: number,
-    size: number,
+    size?: number,
+    filename: string,
 }
 
 // Blacklist
@@ -357,11 +358,7 @@ export interface JudgeResultBody {
 
 export type PathComponent = [string, string, any?, boolean?];
 
-declare global {
+export interface Model {
-    namespace NodeJS {
-        interface Global {
-            Hydro: {
-                model: {
     blacklist: typeof import('./model/blacklist'),
     builtin: typeof import('./model/builtin'),
     contest: typeof import('./model/contest'),
@@ -381,19 +378,20 @@ declare global {
     training: typeof import('./model/training'),
     user: typeof import('./model/user'),
     [key: string]: any,
-                },
+}
-                handler: { [key: string]: Function },
+
-                script: { [key: string]: Script },
+export interface Service {
-                service: {
     bus: typeof import('./service/bus'),
     db: typeof import('./service/db'),
     gridfs: GridFSBucket,
     monitor: typeof import('./service/monitor'),
     server: typeof import('./service/server'),
     [key: string]: any,
-                },
+}
-                lib: {
+
+export interface Lib {
     download: typeof import('./lib/download').default,
+    superagent: typeof import('./lib/superagent'),
     'hash.hydro': typeof import('./lib/hash.hydro').default,
     i18n: typeof import('./lib/i18n').default,
     'import.syzoj': typeof import('./lib/import.syzoj').syzoj,
@@ -413,12 +411,22 @@ declare global {
     useragent: typeof import('./lib/useragent'),
     validator: typeof import('./lib/validator'),
     [key: string]: any
-                },
+}
+
+declare global {
+    namespace NodeJS {
+        interface Global {
+            Hydro: {
+                model: Model,
+                handler: Dict<Function>,
+                script: Dict<Script>,
+                service: Service,
+                lib: Lib,
                 stat: any,
                 ui: {
                     manifest: Dict<string>,
+                    template: Dict<string>,
                     nav: any,
-                    template: { [key: string]: string },
                 },
                 error: typeof import('./error'),
                 locales: Dict<Dict<string>>,

packages/hydrooj/src/lib/superagent.ts

@@ -0,0 +1,52 @@
+import superagent from 'superagent';
+import proxyAgent from 'proxy-agent';
+
+namespace Proxy {
+    function getUrl() {
+        return this._url;
+    }
+    function setUrl(v: string) {
+        this._url = v;
+        // eslint-disable-next-line no-use-before-define
+        proxy.call(this, this._proxyUri);
+    }
+    function setupUrl(req: superagent.Request) {
+        const desc = Object.getOwnPropertyDescriptor(req, 'url');
+        if (desc.get === getUrl && desc.set === setUrl) return;
+        // @ts-ignore
+        req._url = req.url;
+        desc.get = getUrl;
+        desc.set = setUrl;
+        delete desc.value;
+        delete desc.writable;
+        Object.defineProperty(req, 'url', desc);
+    }
+    function proxy(uri: string) {
+        if (!uri) return this;
+        setupUrl(this);
+        const agent = proxyAgent(uri);
+        if (agent) this.agent(agent);
+        this._proxyUri = uri;
+        return this;
+    }
+    export function setup(agent, uri = '') {
+        const Request = agent.Request;
+        if (Request) {
+            Request.prototype.proxy = proxy;
+            return agent;
+        }
+        return proxy.call(agent, uri);
+    }
+}
+
+declare module 'superagent' {
+    namespace request {
+        interface Request {
+            proxy(url: string): this;
+        }
+    }
+}
+
+Proxy.setup(superagent);
+
+export = superagent;

packages/hydrooj/src/model/blacklist.ts

@@ -1,9 +1,10 @@
+import { Collection } from 'mongodb';
 import * as db from '../service/db';
 import { Bdoc } from '../interface';
 
-const coll = db.collection('blacklist');
+const coll: Collection<Bdoc> = db.collection('blacklist');
 
-export async function add(ip: string): Promise<Bdoc> {
+export async function add(ip: string) {
     /**
      * Add a ip into blacklist.
      * @param {string} ip
@@ -18,7 +19,7 @@ export async function add(ip: string): Promise<Bdoc> {
     return res.value;
 }
 
-export function get(ip: string): Promise<Bdoc | null> {
+export function get(ip: string) {
     /**
      * Get a ip, return null if not.
      * @param {string} ip

packages/hydrooj/src/model/file.ts

@@ -1,4 +1,4 @@
-import { ObjectID } from 'mongodb';
+import { ObjectID, Collection } from 'mongodb';
 import * as fs from 'fs';
 import { Dictionary } from 'lodash';
 import { ForbiddenError, NotFoundError } from '../error';
@@ -7,7 +7,7 @@ import gridfs from '../service/gridfs';
 import hash from '../lib/hash.hydro';
 import { Ufdoc } from '../interface';
 
-const coll = db.collection('file');
+const coll: Collection<Ufdoc> = db.collection('file');
 const collFile = db.collection('fs.files');
 const collChunk = db.collection('fs.chunks');
 
@@ -25,11 +25,10 @@ export async function add(
     const w = gridfs.openUploadStream(filename);
     await coll.insertOne({
         ...meta,
-        _id: w.id,
+        _id: w.id as ObjectID,
         secret: String.random(32),
         owner,
         filename,
-        count: 1,
     });
     await new Promise((resolve, reject) => {
         w.on('error', reject);
@@ -37,7 +36,7 @@ export async function add(
         stream.pipe(w);
     });
     const c = await gridfs.find({ _id: w.id }).toArray();
-    await coll.updateOne({ _id: w.id }, { $set: { md5: c[0].md5, size: c[0].length } });
+    await coll.updateOne({ _id: w.id as ObjectID }, { $set: { md5: c[0].md5, size: c[0].length } });
     return w.id as ObjectID;
 }
 
@@ -49,26 +48,6 @@ export function del(_id: ObjectID) {
     ]);
 }
 
-export async function inc(_id: ObjectID): Promise<number> {
-    const doc = await coll.findOneAndUpdate(
-        { _id },
-        { $inc: { count: 1 } },
-        { returnOriginal: false },
-    );
-    return doc.value?.count;
-}
-
-export async function dec(_id: ObjectID): Promise<number> {
-    const file = await coll.findOneAndUpdate(
-        { _id },
-        { $inc: { count: -1 } },
-        { returnOriginal: false },
-    );
-    if (!file.value) return 0;
-    if (!file.value.count) await del(_id);
-    return file.value.count;
-}
-
 export async function getWithSecret(_id: ObjectID, secret: string, reject?: Function) {
     const file = await coll.findOne({ _id });
     if (!file) throw new NotFoundError(_id);
@@ -144,5 +123,5 @@ function ensureIndexes() {
 
 global.Hydro.postInit.push(ensureIndexes);
 global.Hydro.model.file = {
-    add, get, del, inc, dec, getMeta, getWithSecret, getMulti, getMetaDict, url,
+    add, get, del, getMeta, getWithSecret, getMulti, getMetaDict, url,
 };

packages/hydrooj/src/model/record.ts

@@ -1,4 +1,4 @@
-import { ObjectID } from 'mongodb';
+import { ObjectID, Collection } from 'mongodb';
 import { Dictionary } from 'lodash';
 import { STATUS } from './builtin';
 import * as task from './task';
@@ -8,7 +8,7 @@ import {
 } from '../interface';
 import * as db from '../service/db';
 
-const coll = db.collection('record');
+const coll: Collection<Rdoc> = db.collection('record');
 
 export interface RdocBase {
     _id?: ObjectID,

packages/hydrooj/src/model/token.ts

@@ -1,7 +1,8 @@
+import { Collection } from 'mongodb';
 import { TokenDoc } from '../interface';
 import * as db from '../service/db';
 
-const coll = db.collection('token');
+const coll: Collection<TokenDoc> = db.collection('token');
 
 export const TYPE_SESSION = 0;
 export const TYPE_REGISTRATION = 2;
@@ -11,7 +12,7 @@ export const TYPE_LOSTPASS = 5;
 
 function ensureIndexes() {
     return Promise.all([
-        coll.createIndex([{ uid: 1 }, { tokenType: 1 }, { updateAt: -1 }], { sparse: true }),
+        coll.createIndex({ uid: 1, tokenType: 1, updateAt: -1 }, { sparse: true }),
         coll.createIndex('expireAt', { expireAfterSeconds: 0 }),
     ]);
 }

packages/hydrooj/src/model/user.ts

@@ -1,3 +1,4 @@
+import { Collection } from 'mongodb';
 import * as system from './system';
 import * as token from './token';
 import * as setting from './setting';
@@ -8,7 +9,7 @@ import pwhash from '../lib/hash.hydro';
 import * as db from '../service/db';
 import { User as _User, Udoc, Udict } from '../interface';
 
-const coll = db.collection('user');
+const coll: Collection<Udoc> = db.collection('user');
 
 export async function setPassword(uid: number, password: string): Promise<Udoc> {
     const salt = String.random();
@@ -203,11 +204,14 @@ export function getMulti(params: any) {
     return coll.find(params);
 }
 
-export async function getPrefixList(prefix: string, limit = 50) {
+export async function getPrefixList(domainId: string, prefix: string, limit = 50) {
     prefix = prefix.toLowerCase();
     const $regex = new RegExp(`\\A\\Q${prefix}\\E`, 'gmi');
-    const udocs = await coll.find({ unameLower: { $regex } }).limit(limit).toArray();
+    const uids = await coll.find({ unameLower: { $regex } })
-    return udocs;
+        .limit(limit).map((doc) => doc._id).toArray();
+    const users = [];
+    for (const uid of uids) users.push(getById(domainId, uid));
+    return await Promise.all(users);
 }
 
 export async function setPriv(uid: number, priv: number): Promise<Udoc | null> {

packages/hydrooj/src/utils.ts

@@ -1,8 +1,6 @@
 import fs from 'fs';
 import cluster from 'cluster';
 import path from 'path';
-import * as superagent from 'superagent';
-import proxyAgent from 'proxy-agent';
 import serialize from 'serialize-javascript';
 import Lru from 'lru-cache';
 
@@ -41,52 +39,6 @@ export function lrucache(target: any, funcName: string, obj: any) {
     return obj;
 }
 
-namespace Proxy {
-    function getUrl() {
-        return this._url;
-    }
-    function setUrl(v: string) {
-        this._url = v;
-        // eslint-disable-next-line no-use-before-define
-        proxy.call(this, this._proxyUri);
-    }
-    function setupUrl(req: superagent.Request) {
-        const desc = Object.getOwnPropertyDescriptor(req, 'url');
-        if (desc.get === getUrl && desc.set === setUrl) return;
-        // @ts-ignore
-        req._url = req.url;
-        desc.get = getUrl;
-        desc.set = setUrl;
-        delete desc.value;
-        delete desc.writable;
-        Object.defineProperty(req, 'url', desc);
-    }
-    function proxy(uri: string) {
-        if (!uri) return this;
-        setupUrl(this);
-        const agent = proxyAgent(uri);
-        if (agent) this.agent(agent);
-        this._proxyUri = uri;
-        return this;
-    }
-    export function setup(agent, uri = '') {
-        const Request = agent.Request;
-        if (Request) {
-            Request.prototype.proxy = proxy;
-            return agent;
-        }
-        return proxy.call(agent, uri);
-    }
-}
-
-declare module 'superagent' {
-    interface SuperAgentRequest {
-        proxy(url: string): this;
-    }
-}
-
-Proxy.setup(superagent);
-
 if (!cluster.worker) {
     // @ts-ignore
     cluster.worker = { id: 0 };

packages/hydrooj/tsconfig.json

@@ -6,5 +6,9 @@
     },
     "include": [
         "src",
+    ],
+    "exclude": [
+        "dist",
+        "bin"
     ]
 }

packages/login-with-github/lib.ts

@@ -1,5 +1,8 @@
 import 'hydrooj';
-import superagent from 'superagent';
+import * as superagent from 'superagent';
+import superagentProxy from 'superagent-proxy';
+
+superagentProxy(superagent);
 
 async function get() {
     const { system, token } = global.Hydro.model;

packages/login-with-github/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hydrooj/login-with-github",
-    "version": "0.0.2",
+    "version": "0.0.3",
     "main": "package.json",
     "repository": "git@github.com:hydro-dev/Hydro.git",
     "author": "undefined <masnn0@outlook.com>",
@@ -11,12 +11,14 @@
     },
     "devDependencies": {
         "@types/superagent": "^4.1.8",
+        "@types/superagent-proxy": "^2.0.0",
         "hydrooj": "2.12.1"
     },
     "peerDependencies": {
         "hydrooj": "^2.12.1"
     },
     "dependencies": {
-        "superagent": "^5.3.1"
+        "superagent": "^5.3.1",
+        "superagent-proxy": "^2.0.0"
     }
 }

packages/login-with-google/lib.ts

@@ -1,5 +1,8 @@
 import 'hydrooj';
-import superagent from 'superagent';
+import * as superagent from 'superagent';
+import proxy from 'superagent-proxy';
+
+proxy(superagent);
 
 async function get() {
     const { system, token } = global.Hydro.model;

packages/login-with-google/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hydrooj/login-with-google",
-    "version": "0.0.1",
+    "version": "0.0.2",
     "main": "package.json",
     "repository": "git@github.com:hydro-dev/Hydro.git",
     "author": "undefined <masnn0@outlook.com>",
@@ -11,12 +11,14 @@
     },
     "devDependencies": {
         "@types/superagent": "^4.1.8",
+        "@types/superagent-proxy": "^2.0.0",
         "hydrooj": "2.12.1"
     },
     "peerDependencies": {
         "hydrooj": "^2.12.1"
     },
     "dependencies": {
-        "superagent": "^5.3.1"
+        "superagent": "^5.3.1",
+        "superagent-proxy": "^2.0.0"
     }
 }

packages/migrate-vijos/README.md

@@ -0,0 +1,17 @@
+# migrate-vijos
+
+这个模块用于从Vijos4.0的数据库导入数据。
+
+## 迁移须知
+
+迁移前，您需要指定数据来源。  
+**请不要**将 Hydro 正在使用的数据库设置为数据来源的数据库。  
+迁移过程中，Hydro 的以下内容会被**清空**：
+
+题目列表，提交记录，用户列表，比赛列表，比赛成绩表，训练列表，训练进度，站内消息，题解，讨论  
+
+以下内容将被迁移：  
+
+题目与测试数据、题解，讨论与回复，比赛、训练、作业相关数据，提交记录，用户列表，站内消息。
+
+迁移过程中请保持网络畅通。

packages/migrate-vijos/lib.ts

@@ -0,0 +1,17 @@
+import 'hydrooj';
+
+const { md5, sha1 } = global.Hydro.lib;
+
+interface Udoc {
+    uname: string,
+    unameLower: string,
+}
+
+function hash(password: string, salt: string, udoc: Udoc) {
+    const { uname, unameLower } = udoc;
+    const pmd5 = md5(password);
+    const mixedSha1 = sha1(md5(unameLower + pmd5) + salt + sha1(pmd5 + salt));
+    return `${Buffer.from(uname).toString('base64')}|${mixedSha1}`;
+}
+
+global.Hydro.lib['hash.vj2'] = hash;

packages/migrate-vijos/package.json

@@ -0,0 +1,24 @@
+{
+    "name": "@hydrooj/migrate-vijos",
+    "version": "1.3.3",
+    "main": "package.json",
+    "repository": "git@github.com:hydro-dev/Hydro.git",
+    "author": "undefined <masnn0@outlook.com>",
+    "license": "MIT",
+    "scripts": {
+        "lint": "eslint lib.ts script.ts --fix",
+        "build": "tsc"
+    },
+    "devDependencies": {
+        "@types/js-yaml": "^3.12.5",
+        "@types/mongodb": "^3.5.25",
+        "hydrooj": "2.11.5"
+    },
+    "peerDependencies": {
+        "hydrooj": "^2.11.3"
+    },
+    "dependencies": {
+        "js-yaml": "^3.14.0",
+        "mongodb": "^3.6.0"
+    }
+}

packages/migrate-vijos/script.ts

@@ -0,0 +1,524 @@
+/* eslint-disable no-await-in-loop */
+import {
+    Rdoc, TestCase, Mdoc, Drrdoc,
+} from 'hydrooj';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import yaml from 'js-yaml';
+import mongodb, { ObjectID, Db, Cursor } from 'mongodb';
+
+const dst = global.Hydro.service.db;
+const { file, discussion, document } = global.Hydro.model;
+const { testdataConfig } = global.Hydro.lib;
+
+// TODO output enhancement
+const map = {};
+
+const pid = (id) => {
+    if (map[id.toString()]) return map[id.toString()];
+    return id;
+};
+
+const tasks = {
+    user: async (doc) => ({
+        _id: doc._id,
+        uname: doc.uname,
+        unameLower: doc.uname_lower,
+        salt: doc.salt,
+        hash: doc.hash.split('|')[1],
+        hashType: doc.hash.split('|')[0] === 'vj4' ? 'hydro' : doc.hash.split('|')[0],
+        priv: doc.priv,
+        gravatar: doc.gravatar,
+        mail: doc.mail,
+        mailLower: doc.mail_lower,
+        regat: doc.regat,
+        regip: doc.regip,
+        loginat: doc.loginat,
+        loginip: doc.loginip,
+        codeLang: doc.code_lang,
+        codeTemplate: doc.code_template,
+        timezone: doc.timezone,
+        viewLang: doc.viewLang,
+        bio: doc.bio,
+        gender: doc.gender,
+        qq: doc.qq,
+    }),
+    document: {
+        _id: '_id',
+        doc_id: 'docId',
+        doc_type: 'docType',
+        num_submit: 'nSubmit',
+        num_accept: 'nAccept',
+        difficulty: 'difficulty',
+        difficulty_admin: null,
+        difficulty_algo: 'difficultyAlgo',
+        difficulty_setting: 'difficultySetting',
+        pname: 'pid',
+        title: 'title',
+        content: 'content',
+        owner_uid: 'owner',
+        category: 'category',
+        hidden: 'hidden',
+        data: 'data',
+        tag: 'tag',
+        vote: 'vote',
+        reply: {
+            field: 'reply',
+            processer: (reply) => {
+                const res = [];
+                for (const r of reply) {
+                    const drrdoc: Drrdoc = {
+                        _id: r._id,
+                        content: r.content,
+                        owner: r.owner_uid,
+                        ip: r.ip,
+                    };
+                    res.push(drrdoc);
+                }
+                return res;
+            },
+        },
+        ac_msg: 'acMsg',
+        parent_doc_id: {
+            field: 'parentId',
+            processer: (parentId, doc) => {
+                if (doc.parent_doc_type === global.Hydro.model.document.TYPE_PROBLEM) {
+                    return pid(parentId);
+                }
+                return parentId;
+            },
+        },
+        parent_doc_type: 'parentType',
+        num_replies: 'nReply',
+        views: 'views',
+        highlight: 'highlight',
+        ip: 'ip',
+        domain_id: 'domainId',
+        update_at: 'updateAt',
+        begin_at: 'beginAt',
+        end_at: 'endAt',
+        penalty_since: 'penaltySince',
+        penalty_rules: {
+            field: 'penaltyRules',
+            processer: (rule) => {
+                const n = {};
+                for (const key in rule) {
+                    n[parseInt(key, 10) / 3600] = rule;
+                }
+                return n;
+            },
+        },
+        rule: {
+            field: 'rule',
+            processer: (rule: number) => {
+                const rules = {
+                    2: 'oi',
+                    3: 'acm',
+                    11: 'homework',
+                };
+                return rules[rule];
+            },
+        },
+        pids: {
+            field: 'pids',
+            processer: (pids) => pids.map((p) => pid(p)),
+        },
+        attend: 'attend',
+        desc: 'description',
+        enroll: 'enroll',
+        rated: 'rated',
+        dag: {
+            field: 'dag',
+            processer: (dag) => {
+                const r = [];
+                for (const t of dag) {
+                    r.push({
+                        _id: t._id,
+                        title: t.title,
+                        requireNids: t.require_nids,
+                        pids: t.pids.map((id) => pid(id)),
+                    });
+                }
+                return r;
+            },
+        },
+    },
+    'document.status': {
+        _id: '_id',
+        doc_id: 'docId',
+        doc_type: 'docType',
+        uid: 'uid',
+        domain_id: 'domainId',
+        num_accept: 'nAccept',
+        num_submit: 'nSubmit',
+        status: 'status',
+        accept: 'accept',
+        vote: 'vote',
+        star: 'star',
+        enroll: 'enroll',
+        rid: 'rid',
+        rev: 'rev',
+        attend: 'attend',
+        journal: {
+            field: 'journal',
+            processer: (journal) => {
+                const r = [];
+                for (const i of journal) {
+                    r.push({ ...i, pid: pid(i.pid) });
+                }
+                return r;
+            },
+        },
+        penalty_score: 'penaltyScore',
+        detail: {
+            field: 'detail',
+            processer: (detail) => {
+                const r = [];
+                for (const i of detail) {
+                    r.push({ ...i, pid: pid(i.pid) });
+                }
+                return r;
+            },
+        },
+        score: 'score',
+        time: 'time',
+        done: 'done',
+        done_nids: 'doneNids',
+        done_pids: {
+            field: 'donePids',
+            processer: (pids) => pids.map((id) => pid(id)),
+        },
+        rp: null,
+    },
+    'domain.user': {
+        _id: '_id',
+        domain_id: 'domainId',
+        uid: 'uid',
+        num_problems: 'nProblem',
+        num_submit: 'nSubmit',
+        num_accept: 'nAccept',
+        num_liked: 'nLike',
+        level: 'level',
+        role: 'role',
+        join_at: 'joinAt',
+        display_name: 'displayName',
+        rp: null,
+        rank: null,
+    },
+    record: async (doc) => {
+        const testCases: TestCase[] = [];
+        for (const c of doc.cases || []) {
+            testCases.push({
+                status: c.status,
+                time: c.time_ms || c.time,
+                memory: c.memory_kb || c.memory,
+                message: (c.judge_text || '') + (c.message || ''),
+            });
+        }
+        const rdoc: Rdoc = {
+            _id: doc._id,
+            status: doc.status,
+            score: doc.score,
+            time: doc.time_ms,
+            memory: doc.memory_kb,
+            code: doc.code,
+            lang: doc.lang,
+            uid: doc.uid,
+            pid: pid(doc.pid),
+            domainId: doc.domain_id,
+            judger: doc.judge_uid || 1,
+            judgeAt: doc.judge_at || new Date(),
+            judgeTexts: doc.judge_texts || [],
+            compilerTexts: doc.compiler_texts || [],
+            testCases,
+            hidden: !!doc.hidden,
+            rejudged: !!doc.rejudged,
+        };
+        if (doc.rejudged) rdoc.rejudged = true;
+        if (doc.tid) {
+            rdoc.contest = {
+                type: doc.ttype || document.TYPE_CONTEST,
+                tid: doc.tid,
+            };
+        }
+        return rdoc;
+    },
+    domain: {
+        _id: '_id',
+        doc_id: 'docId',
+        doc_type: 'docType',
+        owner_uid: 'owner',
+        name: 'name',
+        roles: {
+            field: 'roles',
+            processer: (roles) => {
+                const res = {};
+                for (const role in roles) {
+                    res[role] = roles[role].toString();
+                }
+                return res;
+            },
+        },
+        gravatar: 'gravatar',
+        bulletin: 'bulletin',
+        pid_counter: 'pidCounter',
+    },
+    'fs.files': async (doc) => doc,
+    'fs.chunks': async (doc) => doc,
+    file: async (doc) => ({
+        _id: doc._id,
+        count: 0,
+        secret: doc.metadata.secret,
+        size: doc.length,
+        md5: doc.md5,
+    }),
+};
+
+type CursorGetter = (s: Db) => Cursor<any>;
+
+const cursor: NodeJS.Dict<CursorGetter> = {
+    user: (s) => s.collection('user').find(),
+    document: (s) => s.collection('document').find({ doc_type: { $ne: 20 } }),
+    'document.status': (s) => s.collection('document.status').find(),
+    'domain.user': (s) => s.collection('domain.user').find(),
+    record: (s) => s.collection('record').find(),
+    domain: (s) => s.collection('domain').find(),
+    'fs.files': (s) => s.collection('fs.files').find(),
+    'fs.chunks': (s) => s.collection('fs.chunks').find(),
+    file: (s) => s.collection('fs.files').find(),
+};
+
+async function discussionNode(src: Db, report: Function) {
+    const count = await src.collection('document').find({ doc_type: 20 }).count();
+    await report({ progress: 1, message: `discussion.node: ${count}` });
+    const total = Math.floor(count / 5);
+    for (let i = 0; i <= total; i++) {
+        const docs = await src.collection('document')
+            .find({ doc_type: 20 }).skip(i * 5).limit(5)
+            .toArray();
+        for (const doc of docs) {
+            const t = [];
+            for (const item of doc.content || []) {
+                const category = item[0];
+                const nodes = item[1];
+                for (const node of nodes || []) {
+                    if (node.pic) {
+                        t.push(discussion.addNode(
+                            doc.domain_id, node.name, category, { pic: node.pic },
+                        ));
+                    } else {
+                        t.push(discussion.addNode(doc.domain_id, node.name, category, {}));
+                    }
+                }
+            }
+            await Promise.all(t).catch((e) => e);
+        }
+        await report({ progress: Math.round(100 * ((i + 1) / (total + 1))) });
+    }
+}
+
+function addSpace(content: string) {
+    const lines = content.split('\r\n');
+    for (let i = 0; i < lines.length; i++) {
+        // eslint-disable-next-line no-continue
+        if (lines[i].endsWith('|')) continue; // Markdown table;
+        if (!lines[i].endsWith('  ')) lines[i] = `${lines[i]}  `;
+    }
+    return lines.join('\n');
+}
+
+async function fix(doc) {
+    await dst.collection('document').updateOne(
+        { _id: doc._id },
+        { $set: { pid: doc.pid || doc.docId.toString(), content: addSpace(doc.content) } },
+    );
+    if (doc.data && doc.data instanceof ObjectID) {
+        const r = await file.get(doc.data);
+        const p = path.resolve(os.tmpdir(), 'hydro', `migrate.vijos.${doc._id}.zip`);
+        const w = fs.createWriteStream(p);
+        await new Promise((resolve, reject) => {
+            w.on('finish', resolve);
+            w.on('error', reject);
+            r.pipe(w);
+        });
+        const config = yaml.safeLoad(await testdataConfig.readConfig(p));
+        await dst.collection('document').updateOne(
+            { _id: doc._id },
+            { $set: { config } },
+        );
+        fs.unlinkSync(p);
+    }
+}
+
+async function fixProblem(report: Function) {
+    const count = await dst.collection('document').find({ docType: 10 }).count();
+    await report({ progress: 1, message: `Fix pid: ${count}` });
+    const total = Math.floor(count / 50);
+    for (let i = 0; i <= total; i++) {
+        const docs = await dst.collection('document')
+            .find({ docType: 10 }).skip(i * 50).limit(50)
+            .toArray();
+        for (const doc of docs) {
+            await fix(doc).catch((e) => report({ message: `${e.toString()}\n${e.stack}` }));
+        }
+        await report({ progress: Math.round(100 * ((i + 1) / (total + 1))) });
+    }
+}
+
+function objid(ts: Date) {
+    const p = Math.floor(ts.getTime() / 1000).toString(16);
+    const id = new mongodb.ObjectID();
+    return new mongodb.ObjectID(p + id.toHexString().slice(8, 8 + 6 + 4 + 6));
+}
+
+// FIXME this seems not working
+async function message(src: Db, report: Function) {
+    const count = await src.collection('message').find().count();
+    await report({ progress: 1, message: `Messages: ${count}` });
+    const total = Math.floor(count / 50);
+    for (let i = 0; i <= total; i++) {
+        const docs = await src.collection('message')
+            .find().skip(i * 50).limit(50)
+            .toArray();
+        for (const doc of docs) {
+            for (const msg of doc.reply) {
+                const mdoc: Mdoc = {
+                    _id: objid(msg.at),
+                    from: msg.sender_uid,
+                    to: msg.sender_uid === doc.sender_uid ? doc.sendee_uid : doc.sender_uid,
+                    content: msg.content,
+                    // Mark all as read
+                    flag: 0,
+                };
+                await dst.collection('message').insertOne(mdoc);
+            }
+        }
+        await report({ progress: Math.round(100 * ((i + 1) / (total + 1))) });
+    }
+}
+
+async function task(name: string, src: Db, report: Function) {
+    const count = await cursor[name](src).count();
+    await report({ progress: 1, message: `${name}: ${count}` });
+    const total = Math.floor(count / 50);
+    let lastProgress = -1;
+    for (let i = 0; i <= total; i++) {
+        const docs = await cursor[name](src).skip(i * 50).limit(50).toArray();
+        const res = [];
+        for (const doc of docs) {
+            let d: any = {};
+            if (typeof tasks[name].call === 'function') {
+                d = await tasks[name](doc);
+            } else {
+                const mapper = tasks[name];
+                for (const key in doc) {
+                    if (typeof mapper[key] === 'string') {
+                        d[mapper[key]] = doc[key];
+                    } else if (mapper[key] === null) {
+                        // Ignore this key
+                    } else if (typeof mapper[key] === 'object') {
+                        d[mapper[key].field] = mapper[key].processer(doc[key], doc);
+                    } else {
+                        await report({ message: `Unknown key ${key} in collection ${name}` });
+                    }
+                }
+            }
+            if (d) {
+                if (d.rule) d.rated = true; // Hack contest rated field
+                const docWithoutId = {};
+                const docWithoutDid = {};
+                for (const key in d) {
+                    if (key !== '_id') {
+                        if (key !== 'domainId' && key !== 'docId' && key !== 'docType' && key !== 'uid') {
+                            docWithoutDid[key] = d[key];
+                        }
+                        docWithoutId[key] = d[key];
+                    }
+                }
+                if (d.domainId && d.docId && d.docType) {
+                    const query: any = { domainId: d.domainId, docId: d.docId, docType: d.docType };
+                    if (d.uid) query.uid = d.uid;
+                    res.push((async () => {
+                        const data = await dst.collection(name).findOne(query);
+                        if (data) {
+                            await dst.collection(name).updateOne(query, { $set: docWithoutDid });
+                        } else if (d._id) {
+                            const dat = await dst.collection(name).findOne({ _id: d._id });
+                            if (dat) {
+                                await dst.collection(name).updateOne({
+                                    _id: d._id,
+                                }, { $set: docWithoutId });
+                            } else {
+                                await dst.collection(name).insertOne(d);
+                            }
+                        } else {
+                            await dst.collection(name).insertOne(d);
+                        }
+                    })());
+                } else if (d._id) {
+                    res.push(dst.collection(name).updateOne({
+                        _id: d._id,
+                    }, { $set: docWithoutId }, { upsert: true }));
+                } else res.push(dst.collection(name).insertOne(d));
+            }
+        }
+        await Promise.all(res).catch((e) => report({ message: `${e}\n${e.stack}` }));
+        const progress = Math.round(100 * ((i + 1) / (total + 1)));
+        if (progress > lastProgress) {
+            await report({ progress });
+            lastProgress = progress;
+        }
+    }
+}
+
+export async function run({
+    host = 'localhost', port = 27017, name = 'vijos4', username, password,
+}, report: Function) {
+    let mongourl = 'mongodb://';
+    if (username) mongourl += `${username}:${password}@`;
+    mongourl += `${host}:${port}/${name}`;
+    const Database = await mongodb.MongoClient.connect(mongourl, {
+        useNewUrlParser: true, useUnifiedTopology: true,
+    });
+    const src = Database.db(name);
+    await report({ progress: 0, message: 'Database connected.' });
+    const userCounter = await src.collection('system').findOne({ _id: 'user_counter' });
+    if (!userCounter) {
+        report({ message: 'No valid installation found' });
+        return false;
+    }
+    await dst.collection('system').updateOne(
+        { _id: 'user' },
+        {
+            $set: {
+                value: userCounter.value,
+            },
+        },
+        { upsert: true },
+    );
+    await report({ progress: 1, message: 'Collection:system done.' });
+    if (!await dst.collection('system').findOne({ _id: 'migrateVijosFs' })) {
+        const f = ['fs.files', 'fs.chunks'];
+        for (const i of f) {
+            await dst.collection(i).deleteMany({});
+            await task(i, src, report);
+        }
+        await dst.collection('system').insertOne({ _id: 'migrateVijosFs', value: true });
+    }
+    await dst.collection('user').deleteMany({ _id: { $nin: [0, 1] } });
+    await dst.collection('message').deleteMany({});
+    const d = ['domain', 'user', 'document', 'document.status', 'domain.user', 'record', 'file'];
+    for (const i of d) await task(i, src, report);
+    await fixProblem(report);
+    await discussionNode(src, report);
+    await message(src, report);
+    return true;
+}
+
+export const description = 'migrate from vijos';
+export const validate = {
+    host: 'string', port: 'number', name: 'string', username: 'string', password: 'string',
+};
+
+global.Hydro.script.migrateVijos = { run, description, validate };

packages/migrate-vijos/tsconfig.json

@@ -0,0 +1,11 @@
+{
+    "extends": "../../tsconfig.base",
+    "compilerOptions": {
+        "outDir": ".",
+        "rootDir": "."
+    },
+    "include": [
+        "*.ts"
+    ],
+    "exclude": []
+}

tsconfig.build.json

@@ -0,0 +1,13 @@
+{
+    "extends": "./tsconfig.base",
+    "compilerOptions": {
+        "outDir": "build",
+        "rootDir": "build",
+    },
+    "include": [
+        "build"
+    ],
+    "exclude": [
+        "packages"
+    ]
+}

tsconfig.json

@@ -8,8 +8,10 @@
         "esModuleInterop": true
     },
     "references": [
+        { "path": "./tsconfig.build.json" },
         { "path": "./packages/hydrooj" },
         { "path": "./packages/login-with-github" },
-        { "path": "./packages/login-with-google" }
+        { "path": "./packages/login-with-google" },
+        { "path": "./packages/migrate-vijos" },
     ],
 }