core: add task manager

packages/hydrooj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "hydrooj",
-    "version": "2.26.8",
+    "version": "2.26.9",
     "bin": "bin/hydrooj.js",
     "main": "dist/loader.js",
     "typings": "dist/loader.d.ts",
@@ -17,7 +17,6 @@
         "cookies": "^0.8.0",
         "detect-browser": "^5.1.1",
         "fs-extra": "^10.0.0",
-        "inquirer": "^8.0.0",
         "js-yaml": "^4.0.0",
         "koa": "^2.13.1",
         "koa-body": "^4.2.0",
@@ -41,7 +40,6 @@
     "devDependencies": {
         "@types/adm-zip": "^0.4.34",
         "@types/fs-extra": "^9.0.10",
-        "@types/inquirer": "^7.3.1",
         "@types/js-yaml": "^4.0.0",
         "@types/koa": "^2.13.1",
         "@types/koa-compress": "^4.0.1",

packages/hydrooj/src/handler/import.syzoj.ts

@@ -11,6 +11,7 @@ import { ValidationError, RemoteOnlineJudgeError } from '../error';
 import { Logger } from '../logger';
 import type { ContentNode } from '../interface';
 import problem from '../model/problem';
+import TaskModel from '../model/task';
 import { PERM } from '../model/builtin';
 import {
     Route, Handler, Types, post,
@@ -21,6 +22,52 @@ import download from '../lib/download';
 const RE_SYZOJ = /(https?):\/\/([^/]+)\/(problem|p)\/([0-9]+)\/?/i;
 const logger = new Logger('import.syzoj');
 
+async function syncFiles(info) {
+    const {
+        protocol, host, body, pid, domainId, docId,
+    } = info;
+    const judge = body.judgeInfo;
+    const r = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/downloadProblemFiles`)
+        .send({
+            problemId: +pid,
+            type: 'TestData',
+            filenameList: body.testData.map((node) => node.filename),
+        });
+    const urls = {};
+    if (r.body.error) return;
+    for (const t of r.body.downloadInfo) urls[t.filename] = t.downloadUrl;
+    for (const f of body.testData) {
+        const p = new PassThrough();
+        superagent.get(urls[f.filename]).pipe(p);
+        // eslint-disable-next-line no-await-in-loop
+        await problem.addTestdata(domainId, docId, f.filename, p);
+    }
+    if (judge) {
+        const config = {
+            time: `${judge.timeLimit}ms`,
+            memory: `${judge.memoryLimit}m`,
+            // TODO other config
+        };
+        await problem.addTestdata(domainId, docId, 'config.yaml', Buffer.from(yaml.dump(config)));
+    }
+    const a = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/downloadProblemFiles`)
+        .send({
+            problemId: +pid,
+            type: 'AdditionalFile',
+            filenameList: body.additionalFiles.map((node) => node.filename),
+        });
+    const aurls = {};
+    if (a.body.error) return;
+    for (const t of a.body.downloadInfo) aurls[t.filename] = t.downloadUrl;
+    for (const f of body.additionalFiles) {
+        const p = new PassThrough();
+        superagent.get(aurls[f.filename]).pipe(p);
+        // eslint-disable-next-line no-await-in-loop
+        await problem.addAdditionalFile(domainId, docId, f.filename, p);
+    }
+}
+TaskModel.Worker.addHandler('import.syzoj', syncFiles);
+
 class ProblemImportSYZOJHandler extends Handler {
     async get() {
         this.response.template = 'problem_import_syzoj.html';
@@ -174,49 +221,11 @@ class ProblemImportSYZOJHandler extends Handler {
         const docId = await problem.add(
             domainId, target, title, JSON.stringify(content), this.user._id, tags || [], hidden,
         );
-        const syncFiles = async () => {
-            const judge = result.body.judgeInfo;
-            const r = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/downloadProblemFiles`)
-                .send({
-                    problemId: +pid,
-                    type: 'TestData',
-                    filenameList: result.body.testData.map((node) => node.filename),
-                });
-            const urls = {};
-            if (r.body.error) return;
-            for (const t of r.body.downloadInfo) urls[t.filename] = t.downloadUrl;
-            for (const f of result.body.testData) {
-                const p = new PassThrough();
-                superagent.get(urls[f.filename]).pipe(p);
-                // eslint-disable-next-line no-await-in-loop
-                await problem.addTestdata(domainId, docId, f.filename, p);
-            }
-            if (judge) {
-                const config = {
-                    time: `${judge.timeLimit}ms`,
-                    memory: `${judge.memoryLimit}m`,
-                    // TODO other config
-                };
-                await problem.addTestdata(domainId, docId, 'config.yaml', Buffer.from(yaml.dump(config)));
-            }
-            const a = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/downloadProblemFiles`)
-                .send({
-                    problemId: +pid,
-                    type: 'AdditionalFile',
-                    filenameList: result.body.additionalFiles.map((node) => node.filename),
-                });
-            const aurls = {};
-            if (a.body.error) return;
-            for (const t of a.body.downloadInfo) aurls[t.filename] = t.downloadUrl;
-            for (const f of result.body.additionalFiles) {
-                const p = new PassThrough();
-                superagent.get(aurls[f.filename]).pipe(p);
-                // eslint-disable-next-line no-await-in-loop
-                await problem.addAdditionalFile(domainId, docId, f.filename, p);
-            }
+        const payload = {
+            protocol, host, pid, domainId, docId, body: result.body,
         };
-        if (wait) await syncFiles();
-        else syncFiles().catch(logger.error);
+        if (wait) await syncFiles(payload);
+        else await TaskModel.add({ ...payload, type: 'schedule', subType: 'import.syzoj' });
         return docId;
     }
 
@@ -252,18 +261,11 @@ class ProblemImportSYZOJHandler extends Handler {
             assert(url.match(RE_SYZOJ), new ValidationError('url'));
             if (!url.endsWith('/')) url += '/';
             const [, protocol, host, n, pid] = RE_SYZOJ.exec(url);
-            if (n === 'p') {
-                const docId = await this.v3(
-                    domainId, targetPid, hidden,
-                    protocol, host, pid, false,
-                );
-                this.response.body = { pid: targetPid || docId };
-                this.response.redirect = this.url('problem_files', { pid: targetPid || docId });
-            } else {
-                const docId = await this.v2(domainId, targetPid, hidden, url);
-                this.response.body = { pid: targetPid || docId };
-                this.response.redirect = this.url('problem_files', { pid: targetPid || docId });
-            }
+            const docId = n === 'p'
+                ? await this.v3(domainId, targetPid, hidden, protocol, host, pid, false)
+                : await this.v2(domainId, targetPid, hidden, url);
+            this.response.body = { pid: targetPid || docId };
+            this.response.redirect = this.url('problem_detail', { pid: targetPid || docId });
         }
     }
 }

packages/hydrooj/src/model/task.ts

@@ -1,12 +1,89 @@
 import moment from 'moment-timezone';
 import { FilterQuery, ObjectID } from 'mongodb';
-import { Task } from '../interface';
+import { BaseService, Task } from '../interface';
 import { Logger } from '../logger';
 import db from '../service/db';
 
 const logger = new Logger('model/task');
 const coll = db.collection('task');
 
+async function getFirst(query: FilterQuery<Task>) {
+    const q = { ...query };
+    q.executeAfter = q.executeAfter || { $lt: new Date() };
+    const res = await coll.findOneAndDelete(q, { sort: { priority: -1 } });
+    if (res.value) {
+        logger.debug('%o', res.value);
+        if (res.value.interval) {
+            const executeAfter = moment(res.value.executeAfter).add(...res.value.interval).toDate();
+            await coll.insertOne({ ...res.value, executeAfter });
+        }
+        return res.value;
+    }
+    return null;
+}
+
+class Consumer {
+    consuming: boolean;
+    running?: any;
+    filter: any;
+    func: Function;
+    timeout: NodeJS.Timeout;
+
+    constructor(filter: any, func: Function) {
+        this.consuming = true;
+        this.filter = filter;
+        this.func = func;
+        this.get = this.get.bind(this);
+        this.get();
+    }
+
+    async get() {
+        if (this.running || !this.consuming) return;
+        try {
+            const res = await getFirst(this.filter);
+            if (res) {
+                this.running = res;
+                await this.func(res);
+                this.running = null;
+            }
+        } catch (e) {
+            logger.error(e);
+            this.destory();
+        }
+        this.timeout = setTimeout(this.get, 100);
+    }
+
+    async destory() {
+        this.consuming = false;
+        clearTimeout(this.timeout);
+    }
+}
+
+class WorkerService implements BaseService {
+    private handlers: Record<string, Function> = {};
+    public consumer: Consumer;
+    public started = false;
+
+    public start() {
+        this.consumer = new Consumer(
+            { type: 'schedule', subType: { $in: Object.keys(this.handlers) } },
+            (doc) => {
+                logger.debug('Worker task: %o', doc);
+                return this.handlers[doc.subType](doc);
+            },
+        );
+        this.started = true;
+    }
+
+    public addHandler(type: string, handler: Function) {
+        this.handlers[type] = handler;
+        this.consumer.filter = { type: 'schedule', subType: { $in: Object.keys(this.handlers) } };
+    }
+}
+
+const Worker = new WorkerService();
+Worker.start();
+
 class TaskModel {
     static async add(task: Partial<Task> & { type: string }) {
         const t: Task = {
@@ -32,21 +109,7 @@ class TaskModel {
         return coll.deleteOne({ _id });
     }
 
-    static async getFirst(query: FilterQuery<Task>) {
-        const q = { ...query };
-        q.executeAfter = q.executeAfter || { $lt: new Date() };
-        const res = await coll.findOneAndDelete(q, { sort: { priority: -1 } });
-        if (res.value) {
-            logger.debug('%o', res.value);
-            if (res.value.interval) {
-                await coll.insertOne({
-                    ...res.value, executeAfter: moment().add(...res.value.interval).toDate(),
-                });
-            }
-            return res.value;
-        }
-        return null;
-    }
+    static getFirst = getFirst;
 
     static async getDelay(query?: FilterQuery<Task>) {
         const now = new Date();
@@ -56,52 +119,12 @@ class TaskModel {
     }
 
     static async consume(query: any, cb: Function) {
-        let isRunning = false;
-        const interval = setInterval(async () => {
-            if (isRunning) return;
-            isRunning = true;
-            const res = await TaskModel.getFirst(query);
-            if (res) {
-                try {
-                    await cb(res);
-                } catch (e) {
-                    clearInterval(interval);
-                }
-            }
-            isRunning = false;
-        }, 100);
+        return new Consumer(query, cb);
     }
 
-    Consumer = class Consumer {
-        consuming: boolean;
-        running?: Promise<any>;
-        interval: NodeJS.Timeout;
-        filter: any;
-        func: Function;
-
-        constructor(filter: any, func: Function) {
-            this.consuming = true;
-            this.filter = filter;
-            this.func = func;
-            this.interval = setInterval(this.consume.bind(this), 100);
-        }
-
-        async consume() {
-            if (this.running || !this.consuming) return;
-            const res = await TaskModel.getFirst(this.filter);
-            if (res) {
-                this.running = this.func(res);
-                if (this.running instanceof Promise) await this.running;
-                this.running = null;
-            }
-        }
-
-        async destory() {
-            this.consuming = false;
-            clearInterval(this.interval);
-            if (this.running) await this.running;
-        }
-    };
+    static Consumer = Consumer;
+    static WorkerService = WorkerService;
+    static Worker = Worker;
 }
 
 export default TaskModel;

packages/hydrooj/src/ui.ts

@@ -1,12 +1,8 @@
 /* eslint-disable @typescript-eslint/no-shadow */
 import cluster from 'cluster';
 import { argv } from 'yargs';
-import BottomBar from 'inquirer/lib/ui/bottom-bar';
-import { Logger } from './logger';
 import * as bus from './service/bus';
 
-const logger = new Logger('ui');
-const useTerminal = cluster.isMaster && process.stdout.isTTY;
 const disabledTerminal = argv.legacy || argv._.length || process.env.NODE_ENV === 'test';
 
 export namespace Progress {
@@ -43,45 +39,7 @@ async function terminate() {
     process.exit(hasError ? 1 : 0);
 }
 process.on('SIGINT', terminate);
-
-if (useTerminal && !disabledTerminal) {
-    let current = 0;
-    const history = [''];
-    let input = '';
-    const ui = new BottomBar({ bottomBar: '>' });
-    process.stdin.on('data', (data) => {
-        // Control Seq?
-        const seq = data.toString('hex');
-        if (seq === '7f') input = input.substr(0, input.length - 1);
-        else if (seq === '1b5b41') {
-            // Arrow up
-            current--;
-            input = history[current];
-        } else if (seq === '1b5b42') {
-            // Arrow down
-            current++;
-            input = history[current];
-        } else if (seq === '0d') {
-            // Enter
-            history.push(input);
-            current = history.length;
-            if (input[0] === '@') {
-                for (const i in cluster.workers) {
-                    cluster.workers[i].send({ event: 'message/run', payload: [input.substr(1, input.length - 1)] });
-                    break;
-                }
-            } else bus.parallel('message/run', input);
-            input = '';
-        } else input += data.toString();
-        ui.updateBottomBar(`>${input}`);
-    });
-    bus.on('message/log', (message) => {
-        ui.log.write(message);
-    });
-    if (process.env.SSH_CLIENT || process.env.SSH_TTY || process.env.SSH_CONNECTION) {
-        logger.warn('Running over ssh detected. Add a --legacy when starting if GUI mode doesn\'t work properly.');
-    }
-} else if (cluster.isMaster) {
+if (cluster.isMaster) {
     if (!disabledTerminal) console.log('Not running in a terminal environment. Interactive mode disabled.');
     bus.on('message/log', (message) => {
         process.stdout.write(`${message}\n`);