fix some bugs, bump ver(2.15.0), update install

packages/hydrooj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "hydrooj",
-    "version": "2.14.9",
+    "version": "2.15.0",
     "bin": "bin/hydrooj.js",
     "main": "dist/loader.js",
     "typings": "dist/loader.d.ts",
@@ -19,11 +19,12 @@
         "js-yaml": "^3.14.1",
         "koa": "^2.13.0",
         "koa-body": "^4.2.0",
+        "koa-proxies": "^0.11.0",
         "koa-router": "^10.0.0",
         "koa-static-cache": "^5.1.3",
         "lodash": "^4.17.20",
         "lru-cache": "^6.0.0",
-        "minio": "^7.0.17",
+        "minio": "7.0.17",
         "moment-timezone": "^0.5.32",
         "mongodb": "^3.6.3",
         "nodemailer": "^6.4.17",
@@ -31,8 +32,7 @@
         "serialize-javascript": "^5.0.1",
         "sockjs": "^0.3.20",
         "superagent": "^6.1.0",
-        "supports-color": "^8.0.0",
-        "systeminformation": "^4.31.1",
+        "systeminformation": "^4.32.0",
         "terminal-kit": "^1.44.0",
         "yargs": "^16.2.0"
     },
@@ -51,12 +51,11 @@
         "@types/serialize-javascript": "^5.0.0",
         "@types/sockjs": "^0.3.32",
         "@types/superagent": "^4.1.8",
-        "@types/supports-color": "^7.2.0",
         "@types/terminal-kit": "^1.28.2",
-        "@types/yargs": "^15.0.11"
+        "@types/yargs": "^15.0.12"
     },
     "scripts": {
         "build": "tsc",
         "build:watch": "tsc --watch"
     }
-}
+}

packages/hydrooj/src/entry/master.ts

@@ -52,6 +52,11 @@ export async function load(call: Entry) {
     await db.start(opts);
     const modelSystem = require('../model/system');
     await modelSystem.runConfig();
+    if (process.env.MINIO_ACCESS_KEY && process.env.MINIO_SECRET_KEY) {
+        await modelSystem.set('file.accessKey', process.env.MINIO_ACCESS_KEY);
+        await modelSystem.set('file.secretKey', process.env.MINIO_SECRET_KEY);
+        await modelSystem.set('file.endPoint', 'http://localhost:9000/');
+    }
     const [endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge] = modelSystem.getMany([
         'file.endPoint', 'file.accessKey', 'file.secretKey', 'file.bucket', 'file.region',
         'file.endPointForUser', 'file.endPointForJudge',

packages/hydrooj/src/handler/misc.ts

@@ -125,7 +125,7 @@ class SwitchLanguageHandler extends Handler {
 }
 
 export async function apply() {
-    Route('file_upload', '/fs/upload', FileUploadHandler, PRIV.PRIV_CREATE_FILE);
+    Route('file_upload', '/upload', FileUploadHandler, PRIV.PRIV_CREATE_FILE);
     Route('status', '/status', StatusHandler);
     Route('status_update', '/status/update', StatusUpdateHandler);
     Route('check_in', '/checkin', CheckInHandler, PRIV.PRIV_USER_PROFILE);

packages/hydrooj/src/handler/problem.ts

@@ -669,6 +669,7 @@ export class ProblemCreateHandler extends Handler {
 
 export class ProblemImportHandler extends Handler {
     async get() {
+        this.response.body = { type: 'Hydro' };
         this.response.template = 'problem_import.html';
     }
 

packages/hydrooj/src/interface.ts

@@ -599,6 +599,7 @@ declare global {
                 ui: UI,
                 error: typeof import('./error'),
                 Logger: typeof import('./logger').Logger,
+                logger: typeof import('./logger').logger,
                 locales: Dict<Dict<string>>,
                 isFirstWorker: boolean,
             },

packages/hydrooj/src/logger.ts

@@ -75,3 +75,5 @@ export class Logger {
 }
 
 global.Hydro.Logger = Logger;
+export const logger = new Logger('*');
+global.Hydro.logger = logger;

packages/hydrooj/src/model/record.ts

@@ -61,7 +61,7 @@ export async function judge(domainId: string, rid: ObjectID, priority = 1) {
     let data = [];
     if (rdoc.pid) {
         const pdoc = await problem.get(domainId, rdoc.pid);
-        data = await storage.list(`problem/${domainId}/${rdoc.pid}/testdata/`);
+        data = await storage.list(`problem/${domainId}/${rdoc.pid}/testdata/`, true);
         if (!config) config = pdoc?.config || {};
     }
     delete rdoc._id;

packages/hydrooj/src/model/setting.ts

@@ -125,8 +125,8 @@ SystemSetting(
     Setting('setting_file', 'file.secretKey', null, null, 'password', 'Storage engine secret', null, FLAG_SECRET),
     Setting('setting_file', 'file.bucket', null, 'hydro', 'text', 'Storage engine bucket'),
     Setting('setting_file', 'file.region', null, 'us-east-1', 'text', 'Storage engine region'),
-    Setting('setting_file', 'file.endPointForUser', null, null, 'text', 'EndPoint for user'),
-    Setting('setting_file', 'file.endPointForJudge', null, null, 'text', 'EndPoint for judge'),
+    Setting('setting_file', 'file.endPointForUser', null, '/fs/', 'text', 'EndPoint for user'),
+    Setting('setting_file', 'file.endPointForJudge', null, '/fs/', 'text', 'EndPoint for judge'),
     Setting('setting_smtp', 'smtp.user', null, null, 'text', 'SMTP Username'),
     Setting('setting_smtp', 'smtp.pass', null, null, 'password', 'SMTP Password', null, FLAG_SECRET),
     Setting('setting_smtp', 'smtp.host', null, null, 'text', 'SMTP Server Host'),

packages/hydrooj/src/service/server.ts

@@ -10,6 +10,7 @@ import { ObjectID } from 'mongodb';
 import Koa, { Context } from 'koa';
 import Body from 'koa-body';
 import Router from 'koa-router';
+import proxy from 'koa-proxies';
 import cache from 'koa-static-cache';
 import sockjs from 'sockjs';
 import { SetOption } from 'cookies';
@@ -270,6 +271,11 @@ export function requireCsrfToken(target: any, funcName: string, obj: any) {
 
 export async function prepare() {
     app.keys = system.get('session.keys') as unknown as string[];
+    app.use(proxy('/fs', {
+        target: system.get('file.endPoint'),
+        changeOrigin: true,
+        rewrite: (p) => p.replace('/fs', ''),
+    }));
     if (argv.public) {
         app.use(cache(argv.public, {
             maxAge: 0,
@@ -894,7 +900,6 @@ export function start() {
             const endTime = new Date().getTime();
             if (ctx.response.headers.nolog) return;
             ctx._remoteAddress = ctx.request.ip;
-            const status = ctx.response.status;
             logger.debug(`${ctx.request.method} ${ctx.request.path} ${ctx.response.status} ${endTime - startTime}ms ${ctx.response.length}`);
         });
     }

packages/hydrooj/src/service/storage.ts

@@ -1,6 +1,11 @@
 import { Readable } from 'stream';
+import assert from 'assert';
 import { URL } from 'url';
 import { Client, BucketItem, ItemBucketMetadata } from 'minio';
+import { Logger } from '../logger';
+import { streamToBuffer } from '../utils';
+
+const logger = new Logger('storage');
 
 interface StorageOptions {
     endPoint: string;
@@ -38,12 +43,17 @@ function parseMainEndpointUrl(endpoint: string): MinioEndpointConfig {
 }
 function parseAlternativeEndpointUrl(endpoint: string): (originalUrl: string) => string {
     if (!endpoint) return (originalUrl) => originalUrl;
+    const pathonly = endpoint.startsWith('/');
+    if (pathonly) endpoint = `https://localhost${endpoint}`;
     const url = new URL(endpoint);
     if (url.hash || url.search) throw new Error('Search parameters and hash are not supported for alternative MinIO endpoint URL.');
     if (!url.pathname.endsWith('/')) throw new Error("Alternative MinIO endpoint URL's pathname must ends with '/'.");
     return (originalUrl) => {
         const parsedOriginUrl = new URL(originalUrl);
-        return new URL(parsedOriginUrl.pathname.slice(1) + parsedOriginUrl.search + parsedOriginUrl.hash, url).toString();
+        const replaced = new URL(parsedOriginUrl.pathname.slice(1) + parsedOriginUrl.search + parsedOriginUrl.hash, url).toString();
+        return pathonly
+            ? replaced.replace('https://localhost', '')
+            : replaced;
     };
 }
 // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent
@@ -80,7 +90,14 @@ class StorageService {
                 user: parseAlternativeEndpointUrl(this.opts.endPointForUser),
                 judge: parseAlternativeEndpointUrl(this.opts.endPointForJudge),
             };
+            await this.put('storage.test', Buffer.from('test'));
+            const result = await streamToBuffer(await this.get('storage.test'));
+            assert(result.toString() === 'test');
+            await this.del('storage.test');
+            logger.success('Storage connected.');
         } catch (e) {
+            logger.error('Storage init fail.');
+            logger.error(e);
             this.error = e.toString();
         }
     }
@@ -104,11 +121,15 @@ class StorageService {
         const stream = this.client.listObjects(this.opts.bucket, target, recursive);
         return await new Promise<BucketItem[]>((resolve, reject) => {
             const results: BucketItem[] = [];
-            stream.on('data', (result) => results.push({
-                ...result,
-                prefix: target,
-                name: result.name.split(target)[1],
-            }));
+            stream.on('data', (result) => {
+                if (result.size) {
+                    results.push({
+                        ...result,
+                        prefix: target,
+                        name: result.name.split(target)[1],
+                    });
+                }
+            });
             stream.on('end', () => resolve(results));
             stream.on('error', reject);
         });

packages/hydrooj/src/ui.ts

@@ -5,6 +5,36 @@ import {
 } from 'terminal-kit';
 import * as bus from './service/bus';
 
+declare module 'terminal-kit/Terminal' {
+    interface ProgressBarOptions {
+        y?: number;
+    }
+}
+
+export namespace Progress {
+    export class Progress {
+        constructor(public args) {
+            console.log('progress start: ', args);
+        }
+
+        startItem(args) {
+            console.log('progress: ', this.args, args);
+        }
+
+        itemDone(args) {
+            console.log('done: ', this.args, args);
+        }
+
+        stop: () => {}
+    }
+
+    export function create(args) {
+        return process.stdout.isTTY
+            ? terminal.progressBar(args)
+            : new Progress(args);
+    }
+}
+
 async function terminate() {
     let hasError = false;
     try {
@@ -12,7 +42,7 @@ async function terminate() {
     } catch (e) {
         hasError = true;
     }
-    if (cluster.isMaster) {
+    if (cluster.isMaster && process.stdout.isTTY) {
         terminal.hideCursor(false);
         terminal.styleReset();
         terminal.resetScrollingRegion();
@@ -22,7 +52,8 @@ async function terminate() {
     process.exit(hasError ? 1 : 0);
 }
 process.on('SIGINT', terminate);
-if (cluster.isMaster) {
+
+if (cluster.isMaster && process.stdout.isTTY) {
     terminal.clear();
     terminal.grabInput();
     terminal.hideCursor();
@@ -76,7 +107,22 @@ if (cluster.isMaster) {
             ShellInput.input.setContent(history[current] || '', false, false);
         }
     });
-    terminal.on('key', (key, matches, data) => {
+    terminal.on('key', (key) => {
         if (key === 'CTRL_C') terminate();
     });
+} else if (cluster.isMaster) {
+    console.log('Not running in a terminal environment. Interactive mode disabled.');
+    bus.on('message/log', (message) => {
+        process.stdout.write(`${message}\n`);
+    });
+    process.stdin.setEncoding('utf-8');
+    process.stdin.on('data', (buf) => {
+        const input = buf.toString();
+        if (input[0] === '@') {
+            for (const i in cluster.workers) {
+                cluster.workers[i].send({ event: 'message/run', payload: [input.substr(1, input.length - 1)] });
+                break;
+            }
+        } else bus.parallel('message/run', input);
+    });
 }

packages/hydrooj/src/upgrade.ts

@@ -1,9 +1,8 @@
 /* eslint-disable no-await-in-loop */
 import { ObjectID, Collection } from 'mongodb';
-// @ts-ignore
-import { terminal } from 'terminal-kit';
 import AdmZip from 'adm-zip';
 import Queue from 'p-queue';
+import { Progress } from './ui';
 import { Ufdoc } from './interface';
 import { Logger } from './logger';
 import { streamToBuffer } from './utils';
@@ -16,12 +15,6 @@ import * as domain from './model/domain';
 import * as document from './model/document';
 import * as system from './model/system';
 
-declare module 'terminal-kit/Terminal' {
-    interface ProgressBarOptions {
-        y?: number;
-    }
-}
-
 const logger = new Logger('upgrade');
 type UpgradeScript = () => Promise<boolean | void>;
 
@@ -73,7 +66,7 @@ const scripts: UpgradeScript[] = [
             const ddoc = ddocs[i];
             logger.info('Domain %s (%d/%d)', ddoc._id, i + 1, ddocs.length);
             const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray();
-            const domainProgress = terminal.progressBar({ items: pdocs.length, title: 'Problems', inline: true });
+            const domainProgress = Progress.create({ items: pdocs.length, title: 'Problems', inline: true });
             for (let j = 0; j < pdocs.length; j++) {
                 const pdoc = pdocs[j];
                 domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`);
@@ -110,13 +103,13 @@ const scripts: UpgradeScript[] = [
             domainProgress.stop();
         }
         const udocs = await user.getMulti().project({ _id: 1, uname: 1 }).toArray();
-        const userfileProgress = terminal.progressBar({
+        const userfileProgress = Progress.create({
             items: udocs.length, title: 'Users', y: 3,
         });
         for (const udoc of udocs) {
             userfileProgress.startItem(`${udoc._id}: ${udoc.uname}`);
             const ufdocs = await coll.find({ owner: udoc._id }).toArray();
-            const currentUser = terminal.progressBar({
+            const currentUser = Progress.create({
                 items: ufdocs.length, title: `${udoc._id}: ${udoc.uname}`, y: 4, eta: true,
             });
             const namelist = [];