From bd8ff053874150bec65d1cd128d5441a1ef35418 Mon Sep 17 00:00:00 2001 From: undefined Date: Tue, 22 Dec 2020 13:40:47 +0800 Subject: [PATCH] fix some bugs, bump ver(2.15.0), update install --- install/centos-7.sh | 9 ++ install/ubuntu-1604.sh | 22 +++-- install/ubuntu-1804.sh | 22 +++-- install/ubuntu-2004.sh | 25 +++-- package.json | 4 +- packages/fps-importer/handler.ts | 98 ++++++++----------- packages/fps-importer/package.json | 4 +- packages/geoip/package.json | 4 +- packages/hydrojudge/package.json | 9 +- packages/hydrojudge/src/log.ts | 5 +- packages/hydrooj/package.json | 13 ++- packages/hydrooj/src/entry/master.ts | 5 + packages/hydrooj/src/handler/misc.ts | 2 +- packages/hydrooj/src/handler/problem.ts | 1 + packages/hydrooj/src/interface.ts | 1 + packages/hydrooj/src/logger.ts | 2 + packages/hydrooj/src/model/record.ts | 2 +- packages/hydrooj/src/model/setting.ts | 4 +- packages/hydrooj/src/service/server.ts | 7 +- packages/hydrooj/src/service/storage.ts | 33 +++++-- packages/hydrooj/src/ui.ts | 52 +++++++++- packages/hydrooj/src/upgrade.ts | 15 +-- packages/import-qduoj/handler.ts | 1 + packages/import-qduoj/package.json | 4 +- .../template/problem_import_qduoj.html | 28 ------ packages/login-with-github/package.json | 4 +- packages/login-with-google/package.json | 4 +- packages/login-with-osu/package.json | 4 +- packages/login-with-qq/package.json | 4 +- 29 files changed, 226 insertions(+), 162 deletions(-) delete mode 100644 packages/import-qduoj/template/problem_import_qduoj.html diff --git a/install/centos-7.sh b/install/centos-7.sh index 7ebd7f56d..bfffea545 100644 --- a/install/centos-7.sh +++ b/install/centos-7.sh @@ -2,6 +2,8 @@ echo "Running Hydro Installer for centos 7" db_password=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) +MINIO_ACCESS_KEY=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) +MINIO_SECRET_KEY=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) # Basic mkdir -p /etc/yum.repos.d @@ -52,6 +54,13 @@ pm2 start "mongod --auth" echo 'Installing g++' apt-get install -y g++ >/dev/null +# Install MinIO +echo 'Installing MinIO' +wget https://dl.min.io/server/minio/release/linux-amd64/minio +chmod +x minio +mkdir -p /data/file +pm2 start "./minio server /data/file" --name minio + # Install HydroOJ # TODO: install basic addons? echo "Installing Hydro" diff --git a/install/ubuntu-1604.sh b/install/ubuntu-1604.sh index a739ca35b..c11926797 100644 --- a/install/ubuntu-1604.sh +++ b/install/ubuntu-1604.sh @@ -1,7 +1,8 @@ #!/bin/bash - +name=xenial echo "Running Hydro Installer for ubuntu 16.04" -db_password=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) +MINIO_ACCESS_KEY=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) +MINIO_SECRET_KEY=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) # Basic echo "apt-get update" @@ -9,9 +10,10 @@ apt-get -qq update echo "apt-get install curl wget gnupg -y" apt-get install -y curl wget gnupg >/dev/null apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 -echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list +echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu $name/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list echo "apt-get update" apt-get -qq update +mkdir -p /data/db /data/file # Install NodeJS echo "Installing NodeJS" @@ -26,8 +28,8 @@ yarn global add pm2 # Install MongoDB echo "Installing MongoDB" +db_password=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) apt-get -qq install -y mongodb-org -mkdir -p /data/db echo "db.createUser({ user: 'hydro', pwd: '$db_password', @@ -46,6 +48,11 @@ pm2 del mongod >/dev/null echo 'Starting mongodb' pm2 start "mongod --auth --bind_ip 0.0.0.0" --name mongodb +# Install MinIO +wget https://dl.min.io/server/minio/release/linux-amd64/minio +chmod +x minio +pm2 start "./minio server /data/file" --name minio + # Install Compiler echo 'Installing g++' apt-get install -y g++ >/dev/null @@ -54,7 +61,7 @@ apt-get install -y g++ >/dev/null # TODO: install basic addons? echo "Installing Hydro" yarn global add hydrooj @hydrooj/ui-default @hydrooj/hydrojudge -wget https://github.com/criyle/go-judge/releases/download/v0.7.1/executorserver-amd64 -O /usr/bin/sandbox +wget https://github.com/criyle/go-judge/releases/download/v0.9.4/executorserver-amd64 -O /usr/bin/sandbox chmod +x /usr/bin/sandbox pm2 start sandbox mkdir ~/.hydro @@ -62,7 +69,10 @@ echo "{\"host\":\"127.0.0.1\",\"port\":\"27017\",\"name\":\"hydro\",\"username\" echo '["@hydrooj/ui-default","@hydrooj/hydrojudge"]' >~/.hydro/addon.json pm2 start hydrooj +pm2 startup +pm2 save + # Done echo "Done" echo "Database username: hydro" -echo "Database password: $db_password" +echo "Database password: $db_password" \ No newline at end of file diff --git a/install/ubuntu-1804.sh b/install/ubuntu-1804.sh index 25c4d9c03..86fe4ee26 100644 --- a/install/ubuntu-1804.sh +++ b/install/ubuntu-1804.sh @@ -1,7 +1,8 @@ #!/bin/bash - +name=bionic echo "Running Hydro Installer for ubuntu 18.04" -db_password=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) +MINIO_ACCESS_KEY=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) +MINIO_SECRET_KEY=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) # Basic echo "apt-get update" @@ -9,9 +10,10 @@ apt-get -qq update echo "apt-get install curl wget gnupg -y" apt-get install -y curl wget gnupg >/dev/null apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 -echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list +echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu $name/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list echo "apt-get update" apt-get -qq update +mkdir -p /data/db /data/file # Install NodeJS echo "Installing NodeJS" @@ -26,8 +28,8 @@ yarn global add pm2 # Install MongoDB echo "Installing MongoDB" +db_password=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) apt-get -qq install -y mongodb-org -mkdir -p /data/db echo "db.createUser({ user: 'hydro', pwd: '$db_password', @@ -46,6 +48,11 @@ pm2 del mongod >/dev/null echo 'Starting mongodb' pm2 start "mongod --auth --bind_ip 0.0.0.0" --name mongodb +# Install MinIO +wget https://dl.min.io/server/minio/release/linux-amd64/minio +chmod +x minio +pm2 start "./minio server /data/file" --name minio + # Install Compiler echo 'Installing g++' apt-get install -y g++ >/dev/null @@ -54,7 +61,7 @@ apt-get install -y g++ >/dev/null # TODO: install basic addons? echo "Installing Hydro" yarn global add hydrooj @hydrooj/ui-default @hydrooj/hydrojudge -wget https://github.com/criyle/go-judge/releases/download/v0.7.1/executorserver-amd64 -O /usr/bin/sandbox +wget https://github.com/criyle/go-judge/releases/download/v0.9.4/executorserver-amd64 -O /usr/bin/sandbox chmod +x /usr/bin/sandbox pm2 start sandbox mkdir ~/.hydro @@ -62,7 +69,10 @@ echo "{\"host\":\"127.0.0.1\",\"port\":\"27017\",\"name\":\"hydro\",\"username\" echo '["@hydrooj/ui-default","@hydrooj/hydrojudge"]' >~/.hydro/addon.json pm2 start hydrooj +pm2 startup +pm2 save + # Done echo "Done" echo "Database username: hydro" -echo "Database password: $db_password" +echo "Database password: $db_password" \ No newline at end of file diff --git a/install/ubuntu-2004.sh b/install/ubuntu-2004.sh index 706fd5f34..a9a3f0693 100644 --- a/install/ubuntu-2004.sh +++ b/install/ubuntu-2004.sh @@ -1,16 +1,19 @@ #!/bin/bash - +name=focal echo "Running Hydro Installer for ubuntu 20.04" -db_password=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) +MINIO_ACCESS_KEY=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) +MINIO_SECRET_KEY=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) # Basic echo "apt-get update" apt-get -qq update echo "apt-get install curl wget gnupg -y" apt-get install -y curl wget gnupg >/dev/null -wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | sudo apt-key add - -echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/4.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.4.list +apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 +echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu $name/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list +echo "apt-get update" apt-get -qq update +mkdir -p /data/db /data/file # Install NodeJS echo "Installing NodeJS" @@ -25,8 +28,8 @@ yarn global add pm2 # Install MongoDB echo "Installing MongoDB" +db_password=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) apt-get -qq install -y mongodb-org -mkdir -p /data/db echo "db.createUser({ user: 'hydro', pwd: '$db_password', @@ -45,6 +48,11 @@ pm2 del mongod >/dev/null echo 'Starting mongodb' pm2 start "mongod --auth --bind_ip 0.0.0.0" --name mongodb +# Install MinIO +wget https://dl.min.io/server/minio/release/linux-amd64/minio +chmod +x minio +pm2 start "./minio server /data/file" --name minio + # Install Compiler echo 'Installing g++' apt-get install -y g++ >/dev/null @@ -53,7 +61,7 @@ apt-get install -y g++ >/dev/null # TODO: install basic addons? echo "Installing Hydro" yarn global add hydrooj @hydrooj/ui-default @hydrooj/hydrojudge -wget https://github.com/criyle/go-judge/releases/download/v0.7.1/executorserver-amd64 -O /usr/bin/sandbox +wget https://github.com/criyle/go-judge/releases/download/v0.9.4/executorserver-amd64 -O /usr/bin/sandbox chmod +x /usr/bin/sandbox pm2 start sandbox mkdir ~/.hydro @@ -61,7 +69,10 @@ echo "{\"host\":\"127.0.0.1\",\"port\":\"27017\",\"name\":\"hydro\",\"username\" echo '["@hydrooj/ui-default","@hydrooj/hydrojudge"]' >~/.hydro/addon.json pm2 start hydrooj +pm2 startup +pm2 save + # Done echo "Done" echo "Database username: hydro" -echo "Database password: $db_password" +echo "Database password: $db_password" \ No newline at end of file diff --git a/package.json b/package.json index 13a9c1675..d0d7706a3 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "@shelf/jest-mongodb": "^1.2.3", "@types/cross-spawn": "^6.0.2", "@types/jest": "^26.0.19", - "@types/node": "^14.14.13", + "@types/node": "^14.14.14", "@types/semver": "^7.3.4", "@typescript-eslint/eslint-plugin": "^3.9.1", "@typescript-eslint/parser": "^3.9.1", @@ -42,4 +42,4 @@ "fs-extra": "^9.0.1", "yargs": "^16.2.0" } -} +} \ No newline at end of file diff --git a/packages/fps-importer/handler.ts b/packages/fps-importer/handler.ts index 6d4a3928a..129a991e4 100644 --- a/packages/fps-importer/handler.ts +++ b/packages/fps-importer/handler.ts @@ -1,66 +1,28 @@ /* eslint-disable no-await-in-loop */ -import path from 'path'; -import os from 'os'; -import fs from 'fs-extra'; import AdmZip from 'adm-zip'; import xml2js from 'xml2js'; -import { ObjectID } from 'mongodb'; +import fs from 'fs-extra'; import { filter } from 'lodash'; import decodeHTML from 'decode-html'; import { LocalProblemConfig } from 'hydrooj'; import { Route, Handler, param, Types, } from 'hydrooj/dist/service/server'; -import { streamToBuffer } from 'hydrooj/dist/utils'; +import * as storage from 'hydrooj/dist/service/storage'; import { buildContent } from 'hydrooj/dist/lib/content'; import { ProblemAdd } from 'hydrooj/dist/lib/ui'; -import * as file from 'hydrooj/dist/model/file'; import * as solution from 'hydrooj/dist/model/solution'; import { PERM } from 'hydrooj/dist/model/builtin'; - -const processing = {}; +import { ValidationError } from 'hydrooj/dist/error'; class FpsProblemImportHandler extends Handler { - @param('ufid', Types.ObjectID, true) - async get(domainId: string, ufid?: ObjectID) { - if (ufid) { - if (processing[ufid.toHexString()]) { - this.response.body = 'Processing'; - return; - } - const stream = await file.get(ufid); - processing[ufid.toHexString()] = true; - try { - const buf = await streamToBuffer(stream); - let input = buf.toString(); - try { - await xml2js.parseStringPromise(input); - } catch (e) { - const zip = new AdmZip(buf); - const entries = zip.getEntries(); - for (const entry of entries) { - if (entry.entryName.endsWith('.xml')) { - input = entry.getData().toString(); - break; - } - } - } - // @ts-ignore - await this.post({ domainId, input }); - await file.del(ufid); - } catch (e) { - processing[ufid.toHexString()] = false; - throw e; - } - } else this.response.template = 'problem_import_fps.html'; + async get() { + this.response.template = 'problem_import.html'; } - @param('input', Types.String) - async post(domainId: string, input: string) { - const result = await xml2js.parseStringPromise(input); + async run(domainId: string, result: any) { const problem = global.Hydro.model.problem; for (const p of result.fps.item) { - const testdata = new AdmZip(); const content = buildContent({ description: p.description, input: p.input, @@ -77,26 +39,18 @@ class FpsProblemImportHandler extends Handler { const title = decodeHTML(p.title.join(' ')); const tags = filter(p.source, (i: string) => i.trim()); const pid = await problem.add(domainId, null, title, content, this.user._id, tags, []); - testdata.addFile('config.yaml', Buffer.from(`time: ${config.time}\nmemory: ${config.memory}`)); + await storage.put(`problem/${domainId}/${pid}/testdata/config.yaml`, Buffer.from(`time: ${config.time}\nmemory: ${config.memory}`)); if (p.test_output) { for (let i = 0; i < p.test_input.length; i++) { - testdata.addFile(`${i + 1}.in`, Buffer.from(p.test_input[i])); - testdata.addFile(`${i + 1}.out`, Buffer.from(p.test_output[i])); + await storage.put(`problem/${domainId}/${pid}/testdata/${i + 1}.in`, Buffer.from(p.test_input[i])); + await storage.put(`problem/${domainId}/${pid}/testdata/${i + 1}.out`, Buffer.from(p.test_output[i])); } } else if (p.test_input) { for (let i = 0; i < p.test_input.length / 2; i++) { - testdata.addFile(`${i + 1}.in`, Buffer.from(p.test_input[2 * i])); - testdata.addFile(`${i + 1}.out`, Buffer.from(p.test_input[2 * i + 1])); + await storage.put(`problem/${domainId}/${pid}/testdata/${i + 1}.in`, Buffer.from(p.test_input[2 * i])); + await storage.put(`problem/${domainId}/${pid}/testdata/${i + 1}.out`, Buffer.from(p.test_input[2 * i + 1])); } } - const f = path.resolve(os.tmpdir(), 'hydro', `${Math.random()}.zip`); - await new Promise((resolve, reject) => { - testdata.writeZip(f, (err) => { - if (err) reject(err); - resolve(); - }); - }); - await problem.setTestdata(domainId, pid, f); await problem.edit(domainId, pid, { html: true }); if (p.solution) { let s = ''; @@ -105,9 +59,35 @@ class FpsProblemImportHandler extends Handler { } await solution.add(domainId, pid, this.user._id, s); } - await fs.unlink(f); } - this.response.body = { count: result.fps.item.length }; + } + + @param('input', Types.String, true) + async post(domainId: string, input: string) { + if (!input && !this.request.files.file) throw new ValidationError('file'); + const tasks = []; + if (this.request.files.file) { + try { + const content = fs.readFile(this.request.files.file.path).toString(); + const result = await xml2js.parseStringPromise(content); + tasks.push(result); + } catch (e) { + const zip = new AdmZip(this.request.files.file.path); + for (const entry of zip.getEntries()) { + try { + const content = entry.getData().toString(); + const result = await xml2js.parseStringPromise(content); + tasks.push(result); + } catch { } // eslint-disable-line no-empty + } + } + } else { + tasks.push(await xml2js.parseStringPromise(input)); + } + if (!tasks.length) throw new ValidationError('file'); + for (const task of tasks) { + await this.run(domainId, task); + } this.response.redirect = this.url('problem_main'); } } diff --git a/packages/fps-importer/package.json b/packages/fps-importer/package.json index aa760cc02..7b7b16d81 100644 --- a/packages/fps-importer/package.json +++ b/packages/fps-importer/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/fps-importer", - "version": "1.1.3", + "version": "1.1.4", "description": "Import FPS problems", "main": "package.json", "repository": "https://github.com/hydro-dev/Hydro.git", @@ -17,7 +17,7 @@ "bson": "^4.2.2" }, "peerDependencies": { - "hydrooj": "^2.14.0" + "hydrooj": "^2.15.0" }, "scripts": { "build": "tsc", diff --git a/packages/geoip/package.json b/packages/geoip/package.json index 07d80dc79..99b3de89b 100644 --- a/packages/geoip/package.json +++ b/packages/geoip/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/geoip", - "version": "1.1.16", + "version": "1.1.17", "main": "package.json", "repository": "git@github.com:hydro-dev/Hydro.git", "author": "undefined ", @@ -12,6 +12,6 @@ "prepublish": "./prepublish.sh" }, "peerDependencies": { - "hydrooj": "^2.12.0||^2.13.0" + "hydrooj": "^2.15.0" } } \ No newline at end of file diff --git a/packages/hydrojudge/package.json b/packages/hydrojudge/package.json index 75ac91fc6..b5f04cb7c 100644 --- a/packages/hydrojudge/package.json +++ b/packages/hydrojudge/package.json @@ -1,7 +1,7 @@ { "name": "@hydrooj/hydrojudge", "bin": "bin/hydrojudge.js", - "version": "2.1.34", + "version": "2.2.0", "main": "package.json", "author": "undefined ", "repository": "https://github.com/hydro-dev/Hydro.git", @@ -14,12 +14,11 @@ "lodash": "^4.17.20", "p-queue": "^6.6.1", "shell-quote": "^1.7.2", - "supports-color": "^8.0.0", - "systeminformation": "^4.31.1", + "systeminformation": "^4.32.0", "yargs": "^16.2.0" }, "peerDependencies": { - "hydrooj": "^2.13.3" + "hydrooj": "^2.15.0" }, "license": "AGPL-3.0-only", "devDependencies": { @@ -32,4 +31,4 @@ "build": "tsc", "lint": "eslint . --ext ts --fix" } -} +} \ No newline at end of file diff --git a/packages/hydrojudge/src/log.ts b/packages/hydrojudge/src/log.ts index 63ebefb73..bb23607c3 100644 --- a/packages/hydrojudge/src/log.ts +++ b/packages/hydrojudge/src/log.ts @@ -1,6 +1,5 @@ import { inspect, InspectOptions, format } from 'util'; import { argv } from 'yargs'; -import { stderr } from 'supports-color'; namespace Time { export const second = 1000; @@ -23,7 +22,7 @@ namespace Time { } } -const colors = stderr.level < 2 ? [6, 2, 3, 4, 5, 1] : [ +const colors = [ 20, 21, 26, 27, 32, 33, 38, 39, 40, 41, 42, 43, 44, 45, 56, 57, 62, 63, 68, 69, 74, 75, 76, 77, 78, 79, 80, 81, 92, 93, 98, 99, 112, 113, 129, 134, 135, 148, 149, 160, 161, 162, 163, 164, 165, 166, 167, 168, @@ -59,7 +58,7 @@ export class Logger { static lastTime = 0 static options: InspectOptions = { - colors: stderr.hasBasic, + colors: process.stderr.isTTY, } static formatters: Record string> = { diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index fc2f31601..c620235ad 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "2.14.9", + "version": "2.15.0", "bin": "bin/hydrooj.js", "main": "dist/loader.js", "typings": "dist/loader.d.ts", @@ -19,11 +19,12 @@ "js-yaml": "^3.14.1", "koa": "^2.13.0", "koa-body": "^4.2.0", + "koa-proxies": "^0.11.0", "koa-router": "^10.0.0", "koa-static-cache": "^5.1.3", "lodash": "^4.17.20", "lru-cache": "^6.0.0", - "minio": "^7.0.17", + "minio": "7.0.17", "moment-timezone": "^0.5.32", "mongodb": "^3.6.3", "nodemailer": "^6.4.17", @@ -31,8 +32,7 @@ "serialize-javascript": "^5.0.1", "sockjs": "^0.3.20", "superagent": "^6.1.0", - "supports-color": "^8.0.0", - "systeminformation": "^4.31.1", + "systeminformation": "^4.32.0", "terminal-kit": "^1.44.0", "yargs": "^16.2.0" }, @@ -51,12 +51,11 @@ "@types/serialize-javascript": "^5.0.0", "@types/sockjs": "^0.3.32", "@types/superagent": "^4.1.8", - "@types/supports-color": "^7.2.0", "@types/terminal-kit": "^1.28.2", - "@types/yargs": "^15.0.11" + "@types/yargs": "^15.0.12" }, "scripts": { "build": "tsc", "build:watch": "tsc --watch" } -} +} \ No newline at end of file diff --git a/packages/hydrooj/src/entry/master.ts b/packages/hydrooj/src/entry/master.ts index e69993a2f..cec349439 100644 --- a/packages/hydrooj/src/entry/master.ts +++ b/packages/hydrooj/src/entry/master.ts @@ -52,6 +52,11 @@ export async function load(call: Entry) { await db.start(opts); const modelSystem = require('../model/system'); await modelSystem.runConfig(); + if (process.env.MINIO_ACCESS_KEY && process.env.MINIO_SECRET_KEY) { + await modelSystem.set('file.accessKey', process.env.MINIO_ACCESS_KEY); + await modelSystem.set('file.secretKey', process.env.MINIO_SECRET_KEY); + await modelSystem.set('file.endPoint', 'http://localhost:9000/'); + } const [endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge] = modelSystem.getMany([ 'file.endPoint', 'file.accessKey', 'file.secretKey', 'file.bucket', 'file.region', 'file.endPointForUser', 'file.endPointForJudge', diff --git a/packages/hydrooj/src/handler/misc.ts b/packages/hydrooj/src/handler/misc.ts index 7e628d221..d34be57de 100644 --- a/packages/hydrooj/src/handler/misc.ts +++ b/packages/hydrooj/src/handler/misc.ts @@ -125,7 +125,7 @@ class SwitchLanguageHandler extends Handler { } export async function apply() { - Route('file_upload', '/fs/upload', FileUploadHandler, PRIV.PRIV_CREATE_FILE); + Route('file_upload', '/upload', FileUploadHandler, PRIV.PRIV_CREATE_FILE); Route('status', '/status', StatusHandler); Route('status_update', '/status/update', StatusUpdateHandler); Route('check_in', '/checkin', CheckInHandler, PRIV.PRIV_USER_PROFILE); diff --git a/packages/hydrooj/src/handler/problem.ts b/packages/hydrooj/src/handler/problem.ts index c56f5aa52..b5f0aa088 100644 --- a/packages/hydrooj/src/handler/problem.ts +++ b/packages/hydrooj/src/handler/problem.ts @@ -669,6 +669,7 @@ export class ProblemCreateHandler extends Handler { export class ProblemImportHandler extends Handler { async get() { + this.response.body = { type: 'Hydro' }; this.response.template = 'problem_import.html'; } diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 30e7adcdd..2fca1aff2 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -599,6 +599,7 @@ declare global { ui: UI, error: typeof import('./error'), Logger: typeof import('./logger').Logger, + logger: typeof import('./logger').logger, locales: Dict>, isFirstWorker: boolean, }, diff --git a/packages/hydrooj/src/logger.ts b/packages/hydrooj/src/logger.ts index 48529f35f..0448056c6 100644 --- a/packages/hydrooj/src/logger.ts +++ b/packages/hydrooj/src/logger.ts @@ -75,3 +75,5 @@ export class Logger { } global.Hydro.Logger = Logger; +export const logger = new Logger('*'); +global.Hydro.logger = logger; diff --git a/packages/hydrooj/src/model/record.ts b/packages/hydrooj/src/model/record.ts index 74cbaaa14..4de171bf8 100644 --- a/packages/hydrooj/src/model/record.ts +++ b/packages/hydrooj/src/model/record.ts @@ -61,7 +61,7 @@ export async function judge(domainId: string, rid: ObjectID, priority = 1) { let data = []; if (rdoc.pid) { const pdoc = await problem.get(domainId, rdoc.pid); - data = await storage.list(`problem/${domainId}/${rdoc.pid}/testdata/`); + data = await storage.list(`problem/${domainId}/${rdoc.pid}/testdata/`, true); if (!config) config = pdoc?.config || {}; } delete rdoc._id; diff --git a/packages/hydrooj/src/model/setting.ts b/packages/hydrooj/src/model/setting.ts index 376962f85..18a4aba2b 100644 --- a/packages/hydrooj/src/model/setting.ts +++ b/packages/hydrooj/src/model/setting.ts @@ -125,8 +125,8 @@ SystemSetting( Setting('setting_file', 'file.secretKey', null, null, 'password', 'Storage engine secret', null, FLAG_SECRET), Setting('setting_file', 'file.bucket', null, 'hydro', 'text', 'Storage engine bucket'), Setting('setting_file', 'file.region', null, 'us-east-1', 'text', 'Storage engine region'), - Setting('setting_file', 'file.endPointForUser', null, null, 'text', 'EndPoint for user'), - Setting('setting_file', 'file.endPointForJudge', null, null, 'text', 'EndPoint for judge'), + Setting('setting_file', 'file.endPointForUser', null, '/fs/', 'text', 'EndPoint for user'), + Setting('setting_file', 'file.endPointForJudge', null, '/fs/', 'text', 'EndPoint for judge'), Setting('setting_smtp', 'smtp.user', null, null, 'text', 'SMTP Username'), Setting('setting_smtp', 'smtp.pass', null, null, 'password', 'SMTP Password', null, FLAG_SECRET), Setting('setting_smtp', 'smtp.host', null, null, 'text', 'SMTP Server Host'), diff --git a/packages/hydrooj/src/service/server.ts b/packages/hydrooj/src/service/server.ts index 6c2c9a374..ba20c4d28 100644 --- a/packages/hydrooj/src/service/server.ts +++ b/packages/hydrooj/src/service/server.ts @@ -10,6 +10,7 @@ import { ObjectID } from 'mongodb'; import Koa, { Context } from 'koa'; import Body from 'koa-body'; import Router from 'koa-router'; +import proxy from 'koa-proxies'; import cache from 'koa-static-cache'; import sockjs from 'sockjs'; import { SetOption } from 'cookies'; @@ -270,6 +271,11 @@ export function requireCsrfToken(target: any, funcName: string, obj: any) { export async function prepare() { app.keys = system.get('session.keys') as unknown as string[]; + app.use(proxy('/fs', { + target: system.get('file.endPoint'), + changeOrigin: true, + rewrite: (p) => p.replace('/fs', ''), + })); if (argv.public) { app.use(cache(argv.public, { maxAge: 0, @@ -894,7 +900,6 @@ export function start() { const endTime = new Date().getTime(); if (ctx.response.headers.nolog) return; ctx._remoteAddress = ctx.request.ip; - const status = ctx.response.status; logger.debug(`${ctx.request.method} ${ctx.request.path} ${ctx.response.status} ${endTime - startTime}ms ${ctx.response.length}`); }); } diff --git a/packages/hydrooj/src/service/storage.ts b/packages/hydrooj/src/service/storage.ts index edbc2a115..815fe52e1 100644 --- a/packages/hydrooj/src/service/storage.ts +++ b/packages/hydrooj/src/service/storage.ts @@ -1,6 +1,11 @@ import { Readable } from 'stream'; +import assert from 'assert'; import { URL } from 'url'; import { Client, BucketItem, ItemBucketMetadata } from 'minio'; +import { Logger } from '../logger'; +import { streamToBuffer } from '../utils'; + +const logger = new Logger('storage'); interface StorageOptions { endPoint: string; @@ -38,12 +43,17 @@ function parseMainEndpointUrl(endpoint: string): MinioEndpointConfig { } function parseAlternativeEndpointUrl(endpoint: string): (originalUrl: string) => string { if (!endpoint) return (originalUrl) => originalUrl; + const pathonly = endpoint.startsWith('/'); + if (pathonly) endpoint = `https://localhost${endpoint}`; const url = new URL(endpoint); if (url.hash || url.search) throw new Error('Search parameters and hash are not supported for alternative MinIO endpoint URL.'); if (!url.pathname.endsWith('/')) throw new Error("Alternative MinIO endpoint URL's pathname must ends with '/'."); return (originalUrl) => { const parsedOriginUrl = new URL(originalUrl); - return new URL(parsedOriginUrl.pathname.slice(1) + parsedOriginUrl.search + parsedOriginUrl.hash, url).toString(); + const replaced = new URL(parsedOriginUrl.pathname.slice(1) + parsedOriginUrl.search + parsedOriginUrl.hash, url).toString(); + return pathonly + ? replaced.replace('https://localhost', '') + : replaced; }; } // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent @@ -80,7 +90,14 @@ class StorageService { user: parseAlternativeEndpointUrl(this.opts.endPointForUser), judge: parseAlternativeEndpointUrl(this.opts.endPointForJudge), }; + await this.put('storage.test', Buffer.from('test')); + const result = await streamToBuffer(await this.get('storage.test')); + assert(result.toString() === 'test'); + await this.del('storage.test'); + logger.success('Storage connected.'); } catch (e) { + logger.error('Storage init fail.'); + logger.error(e); this.error = e.toString(); } } @@ -104,11 +121,15 @@ class StorageService { const stream = this.client.listObjects(this.opts.bucket, target, recursive); return await new Promise((resolve, reject) => { const results: BucketItem[] = []; - stream.on('data', (result) => results.push({ - ...result, - prefix: target, - name: result.name.split(target)[1], - })); + stream.on('data', (result) => { + if (result.size) { + results.push({ + ...result, + prefix: target, + name: result.name.split(target)[1], + }); + } + }); stream.on('end', () => resolve(results)); stream.on('error', reject); }); diff --git a/packages/hydrooj/src/ui.ts b/packages/hydrooj/src/ui.ts index 2774af5dd..73042abe9 100644 --- a/packages/hydrooj/src/ui.ts +++ b/packages/hydrooj/src/ui.ts @@ -5,6 +5,36 @@ import { } from 'terminal-kit'; import * as bus from './service/bus'; +declare module 'terminal-kit/Terminal' { + interface ProgressBarOptions { + y?: number; + } +} + +export namespace Progress { + export class Progress { + constructor(public args) { + console.log('progress start: ', args); + } + + startItem(args) { + console.log('progress: ', this.args, args); + } + + itemDone(args) { + console.log('done: ', this.args, args); + } + + stop: () => {} + } + + export function create(args) { + return process.stdout.isTTY + ? terminal.progressBar(args) + : new Progress(args); + } +} + async function terminate() { let hasError = false; try { @@ -12,7 +42,7 @@ async function terminate() { } catch (e) { hasError = true; } - if (cluster.isMaster) { + if (cluster.isMaster && process.stdout.isTTY) { terminal.hideCursor(false); terminal.styleReset(); terminal.resetScrollingRegion(); @@ -22,7 +52,8 @@ async function terminate() { process.exit(hasError ? 1 : 0); } process.on('SIGINT', terminate); -if (cluster.isMaster) { + +if (cluster.isMaster && process.stdout.isTTY) { terminal.clear(); terminal.grabInput(); terminal.hideCursor(); @@ -76,7 +107,22 @@ if (cluster.isMaster) { ShellInput.input.setContent(history[current] || '', false, false); } }); - terminal.on('key', (key, matches, data) => { + terminal.on('key', (key) => { if (key === 'CTRL_C') terminate(); }); +} else if (cluster.isMaster) { + console.log('Not running in a terminal environment. Interactive mode disabled.'); + bus.on('message/log', (message) => { + process.stdout.write(`${message}\n`); + }); + process.stdin.setEncoding('utf-8'); + process.stdin.on('data', (buf) => { + const input = buf.toString(); + if (input[0] === '@') { + for (const i in cluster.workers) { + cluster.workers[i].send({ event: 'message/run', payload: [input.substr(1, input.length - 1)] }); + break; + } + } else bus.parallel('message/run', input); + }); } diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index e52d32023..cf61a985a 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -1,9 +1,8 @@ /* eslint-disable no-await-in-loop */ import { ObjectID, Collection } from 'mongodb'; -// @ts-ignore -import { terminal } from 'terminal-kit'; import AdmZip from 'adm-zip'; import Queue from 'p-queue'; +import { Progress } from './ui'; import { Ufdoc } from './interface'; import { Logger } from './logger'; import { streamToBuffer } from './utils'; @@ -16,12 +15,6 @@ import * as domain from './model/domain'; import * as document from './model/document'; import * as system from './model/system'; -declare module 'terminal-kit/Terminal' { - interface ProgressBarOptions { - y?: number; - } -} - const logger = new Logger('upgrade'); type UpgradeScript = () => Promise; @@ -73,7 +66,7 @@ const scripts: UpgradeScript[] = [ const ddoc = ddocs[i]; logger.info('Domain %s (%d/%d)', ddoc._id, i + 1, ddocs.length); const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray(); - const domainProgress = terminal.progressBar({ items: pdocs.length, title: 'Problems', inline: true }); + const domainProgress = Progress.create({ items: pdocs.length, title: 'Problems', inline: true }); for (let j = 0; j < pdocs.length; j++) { const pdoc = pdocs[j]; domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`); @@ -110,13 +103,13 @@ const scripts: UpgradeScript[] = [ domainProgress.stop(); } const udocs = await user.getMulti().project({ _id: 1, uname: 1 }).toArray(); - const userfileProgress = terminal.progressBar({ + const userfileProgress = Progress.create({ items: udocs.length, title: 'Users', y: 3, }); for (const udoc of udocs) { userfileProgress.startItem(`${udoc._id}: ${udoc.uname}`); const ufdocs = await coll.find({ owner: udoc._id }).toArray(); - const currentUser = terminal.progressBar({ + const currentUser = Progress.create({ items: ufdocs.length, title: `${udoc._id}: ${udoc.uname}`, y: 4, eta: true, }); const namelist = []; diff --git a/packages/import-qduoj/handler.ts b/packages/import-qduoj/handler.ts index 87cb8d04a..33a483dbc 100644 --- a/packages/import-qduoj/handler.ts +++ b/packages/import-qduoj/handler.ts @@ -108,6 +108,7 @@ class ImportQduojHandler extends Handler { } async get() { + this.response.body = { type: 'QDUOJ' }; this.response.template = 'problem_import.html'; } diff --git a/packages/import-qduoj/package.json b/packages/import-qduoj/package.json index cdba461e3..33b0039af 100644 --- a/packages/import-qduoj/package.json +++ b/packages/import-qduoj/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/import-qduoj", - "version": "1.1.0", + "version": "1.1.1", "description": "Import QDUOJ problem export", "main": "package.json", "repository": "https://github.com/hydro-dev/Hydro.git", @@ -15,7 +15,7 @@ "bson": "^4.2.2" }, "peerDependencies": { - "hydrooj": "^2.14.0" + "hydrooj": "^2.15.0" }, "scripts": { "build": "tsc", diff --git a/packages/import-qduoj/template/problem_import_qduoj.html b/packages/import-qduoj/template/problem_import_qduoj.html deleted file mode 100644 index a23a85bf6..000000000 --- a/packages/import-qduoj/template/problem_import_qduoj.html +++ /dev/null @@ -1,28 +0,0 @@ -{% extends "layout/basic.html" %} -{% block content %} -
-
-
-
-
- -

{{ _('Upload zipfile') }}: - - - - -

-
-
-
-
-
-
-
-

{{ _('What is this?') }}

-

{{ _('With this feature, you can import problems that you can view from a site to here. Their title, content, tags and categories will be imported.') }}

-
-
-
-
-{% endblock %} diff --git a/packages/login-with-github/package.json b/packages/login-with-github/package.json index 12f017096..6604380cd 100644 --- a/packages/login-with-github/package.json +++ b/packages/login-with-github/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/login-with-github", - "version": "0.0.7", + "version": "0.0.8", "main": "package.json", "repository": "git@github.com:hydro-dev/Hydro.git", "author": "undefined ", @@ -14,7 +14,7 @@ "@types/superagent-proxy": "^2.0.0" }, "peerDependencies": { - "hydrooj": "^2.12.1||^2.13.0" + "hydrooj": "^2.15.0" }, "dependencies": { "superagent": "^6.1.0", diff --git a/packages/login-with-google/package.json b/packages/login-with-google/package.json index 44cd1ca59..12462b6fa 100644 --- a/packages/login-with-google/package.json +++ b/packages/login-with-google/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/login-with-google", - "version": "0.0.8", + "version": "0.0.9", "main": "package.json", "repository": "git@github.com:hydro-dev/Hydro.git", "author": "undefined ", @@ -14,7 +14,7 @@ "@types/superagent-proxy": "^2.0.0" }, "peerDependencies": { - "hydrooj": "^2.12.1||^2.13.0" + "hydrooj": "^2.15.0" }, "dependencies": { "superagent": "^6.1.0", diff --git a/packages/login-with-osu/package.json b/packages/login-with-osu/package.json index 1745d7b37..931c87a84 100644 --- a/packages/login-with-osu/package.json +++ b/packages/login-with-osu/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/login-with-osu", - "version": "0.0.7", + "version": "0.0.8", "main": "package.json", "repository": "git@github.com:hydro-dev/Hydro.git", "author": "undefined ", @@ -14,7 +14,7 @@ "@types/superagent-proxy": "^2.0.0" }, "peerDependencies": { - "hydrooj": "^2.12.1||^2.13.0" + "hydrooj": "^2.15.0" }, "dependencies": { "superagent": "^6.1.0", diff --git a/packages/login-with-qq/package.json b/packages/login-with-qq/package.json index 3dd7be308..a7550b10b 100644 --- a/packages/login-with-qq/package.json +++ b/packages/login-with-qq/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/login-with-qq", - "version": "0.0.4", + "version": "0.0.5", "main": "package.json", "repository": "git@github.com:hydro-dev/Hydro.git", "author": "undefined ", @@ -10,7 +10,7 @@ "build": "tsc" }, "peerDependencies": { - "hydrooj": "^2.13.1" + "hydrooj": "^2.15.0" }, "dependencies": { "ws": "^7.4.1"