mirror of
https://gitcode.com/gh_mirrors/hy/Hydro.git
synced 2026-10-17 09:36:08 +08:00
core&ui: goodbye, minio (#417)
This commit is contained in:
undefined authored and GitHub committed
1 parent
7abc0a7d1b
commit
e06c86e9dd
34 files changed
+410
-322
No files matched your search
@@ -18,4 +18,3 @@ RUN npm i pm2 yarn -g && \
|
||||
sudo apt-get install clang -y --no-install-recommends && \
|
||||
mkdir -p ~/.hydro && \
|
||||
echo '{"host":"127.0.0.1","port":"27017","name":"hydro","username":"","password":""}' >~/.hydro/config.json && \
|
||||
echo "MINIO_ACCESS_KEY=minioadmin\nMINIO_SECRET_KEY=minioadmin" >~/.hydro/env
|
||||
@@ -9,6 +9,7 @@ services:
|
||||
VARIANT: 16
|
||||
volumes:
|
||||
- ..:/workspace:cached
|
||||
- testdata:/data/file
|
||||
command: sleep infinity
|
||||
network_mode: service:db
|
||||
db:
|
||||
@@ -16,14 +17,7 @@ services:
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- mongodb-data:/data/db
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- minio-data:/data
|
||||
network_mode: service:db
|
||||
command: minio server /data --console-address ":9001"
|
||||
|
||||
volumes:
|
||||
mongodb-data:
|
||||
minio-data:
|
||||
testdata:
|
||||
Vendored
+2
-3
@@ -1,10 +1,9 @@
|
||||
FROM gitpod/workspace-mongodb
|
||||
RUN npm i pm2 -g && \
|
||||
sudo wget https://dl.min.io/server/minio/release/linux-amd64/minio -O /usr/bin/minio && \
|
||||
sudo chmod 755 /usr/bin/minio && \
|
||||
sudo apt-get update && \
|
||||
sudo apt-get install clang -y && \
|
||||
cargo install sonic-server --version 1.3.0 && \
|
||||
mkdir -p /home/gitpod/.hydro && \
|
||||
echo '{"host":"127.0.0.1","port":"27017","name":"hydro","username":"","password":""}' >/home/gitpod/.hydro/config.json && \
|
||||
echo "MINIO_ACCESS_KEY=hydro\nMINIO_SECRET_KEY=hydrohydro" >/home/gitpod/.hydro/env
|
||||
mkdir /data/file -p
|
||||
chmod 777 /data/file
|
||||
@@ -4,7 +4,6 @@ tasks:
|
||||
- init: |
|
||||
yarn
|
||||
pm2 start mongod
|
||||
pm2 start "MINIO_ACCESS_KEY=hydro MINIO_SECRET_KEY=hydrohydro minio server /home/gitpod/file" --name minio
|
||||
yarn build:ui
|
||||
npx hydrooj cli system set server.port 2333
|
||||
npx hydrooj cli user create root@hydro.local root rootroot 2
|
||||
|
||||
@@ -12,7 +12,6 @@ fi
|
||||
|
||||
if [ ! -f "$ROOT/first" ]; then
|
||||
echo "for marking use only!" > "$ROOT/first"
|
||||
hydrooj cli system set file.endPoint http://oj-minio:9000/
|
||||
|
||||
hydrooj cli user create systemjudge@systemjudge.local root rootroot
|
||||
hydrooj cli user setSuperAdmin 2
|
||||
|
||||
@@ -1,16 +1,6 @@
|
||||
version: '3.7'
|
||||
|
||||
services:
|
||||
oj-minio:
|
||||
image: minio/minio
|
||||
container_name: oj-minio
|
||||
command: server /data
|
||||
restart: always
|
||||
volumes:
|
||||
- ./data/minio:/data
|
||||
environment:
|
||||
- MINIO_ACCESS_KEY=CHANGE_THIS
|
||||
- MINIO_SECRET_KEY=CHANGE_THIS
|
||||
|
||||
# Warning: mongodb here is not password-protected.
|
||||
# DO NOT EXPOSE THIS SERVICE TO THE PUBLIC.
|
||||
@@ -26,13 +16,10 @@ services:
|
||||
container_name: oj-backend
|
||||
restart: always
|
||||
depends_on:
|
||||
- oj-minio
|
||||
- oj-mongo
|
||||
volumes:
|
||||
- ./data/file:/data/file
|
||||
- ./data/backend:/root/.hydro
|
||||
environment:
|
||||
- MINIO_ACCESS_KEY=CHANGE_THIS
|
||||
- MINIO_SECRET_KEY=CHANGE_THIS
|
||||
ports:
|
||||
- "0.0.0.0:80:8888" # In docker mode, change THIS port instead of port in system settings!
|
||||
|
||||
|
||||
@@ -20,7 +20,6 @@ const locales = {
|
||||
'error.nodeVersionPraseFail': '无法解析 Node 版本号，请尝试手动安装。',
|
||||
'install.pm2': '正在安装 PM2...',
|
||||
'install.createDatabaseUser': '正在创建数据库用户...',
|
||||
'install.minio': '正在安装 MinIO...',
|
||||
'install.compiler': '正在安装编译器...',
|
||||
'install.hydro': '正在安装 Hydro...',
|
||||
'install.done': 'Hydro 安装成功！',
|
||||
@@ -45,7 +44,6 @@ const locales = {
|
||||
'error.nodeVersionPraseFail': 'Unable to parse Node version, please try to install manually.',
|
||||
'install.pm2': 'Installing PM2...',
|
||||
'install.createDatabaseUser': 'Creating database user...',
|
||||
'install.minio': 'Installing MinIO...',
|
||||
'install.compiler': 'Installing compiler...',
|
||||
'install.hydro': 'Installing Hydro...',
|
||||
'install.done': 'Hydro installation completed!',
|
||||
@@ -83,8 +81,6 @@ if (!cpuInfoFile.includes('avx2')) {
|
||||
let migration;
|
||||
let retry = 0;
|
||||
log.info('install.start');
|
||||
const MINIO_ACCESS_KEY = randomstring(32);
|
||||
const MINIO_SECRET_KEY = randomstring(32);
|
||||
let DATABASE_PASSWORD = randomstring(32);
|
||||
// TODO read from args
|
||||
const CN = true;
|
||||
@@ -193,13 +189,6 @@ To disable this feature, checkout our sourcecode.`);
|
||||
skip: () => !exec('pm2 -v').code,
|
||||
operations: ['yarn global add pm2'],
|
||||
},
|
||||
{
|
||||
init: 'install.minio',
|
||||
skip: () => !exec('minio -v').code,
|
||||
operations: [
|
||||
'nix-env -iA nixpkgs.minio',
|
||||
],
|
||||
},
|
||||
{
|
||||
init: 'install.compiler',
|
||||
operations: [
|
||||
@@ -249,8 +238,6 @@ To disable this feature, checkout our sourcecode.`);
|
||||
operations: [
|
||||
['pm2 stop all', { ignore: true }],
|
||||
() => fs.writefile(`${__env.HOME}/.hydro/mount.yaml`, mount),
|
||||
`echo "MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}\nMINIO_SECRET_KEY=${MINIO_SECRET_KEY}" >/root/.hydro/env`,
|
||||
`pm2 start "MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY} MINIO_SECRET_KEY=${MINIO_SECRET_KEY} minio server /data/file" --name minio`,
|
||||
'pm2 start mongod --name mongodb -- --auth --bind_ip 0.0.0.0',
|
||||
() => sleep(1000),
|
||||
`pm2 start bash --name hydro-sandbox -- -c "ulimit -s unlimited && hydro-sandbox -mount-conf ${__env.HOME}/.hydro/mount.yaml"`,
|
||||
@@ -291,8 +278,6 @@ To disable this feature, checkout our sourcecode.`);
|
||||
() => log.info('extra.restartTerm'),
|
||||
() => log.info('extra.dbUser'),
|
||||
() => log.info('extra.dbPassword', DATABASE_PASSWORD),
|
||||
() => log.info('MINIO_ACCESS_KEY=%s', MINIO_ACCESS_KEY),
|
||||
() => log.info('MINIO_SECRET_KEY=%s', MINIO_SECRET_KEY),
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
@@ -15,7 +15,6 @@ in pkgs.dockerTools.buildImage {
|
||||
name = "hydro-web";
|
||||
paths = [
|
||||
hydro.mongodb4
|
||||
pkgs.minio
|
||||
pkgs.nodejs
|
||||
pkgs.yarn
|
||||
];
|
||||
|
||||
@@ -6,7 +6,6 @@ in pkgs.buildEnv {
|
||||
name = "hydro-env";
|
||||
paths = [
|
||||
mongo
|
||||
pkgs.minio
|
||||
pkgs.nodejs
|
||||
pkgs.yarn
|
||||
pkgs.git
|
||||
|
||||
+1
-2
@@ -15,7 +15,6 @@ mongo 127.0.0.1:27017/hydro /tmp/createUser.js
|
||||
echo "{\"host\":\"127.0.0.1\",\"port\":\"27017\",\"name\":\"hydro\",\"username\":\"hydro\",\"password\":\"$db_password\"}" >~/.hydro/config.json
|
||||
pm2 stop mongod
|
||||
pm2 del mongod
|
||||
pm2 restart minio
|
||||
pm2 start mongodb
|
||||
pm2 restart hydrooj
|
||||
pm2 restart all
|
||||
pm2 save
|
||||
@@ -7,8 +7,6 @@ echo "详情请参阅文档 -> https://hydro.js.org"
|
||||
echo "使用 Ctrl-C 退出该脚本，或是等待十秒后继续。"
|
||||
echo "Will continue installation in 10 secs, press Ctrl-C to exit"
|
||||
sleep 10
|
||||
MINIO_ACCESS_KEY=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20)
|
||||
MINIO_SECRET_KEY=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20)
|
||||
|
||||
# Basic
|
||||
echo "apt-get update"
|
||||
@@ -54,11 +52,6 @@ pm2 del mongod >/dev/null
|
||||
echo 'Starting mongodb'
|
||||
pm2 start "mongod --auth --bind_ip 0.0.0.0" --name mongodb
|
||||
|
||||
# Install MinIO
|
||||
wget http://dl.minio.org.cn/server/minio/release/linux-amd64/minio
|
||||
chmod +x minio
|
||||
pm2 start "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY MINIO_SECRET_KEY=$MINIO_SECRET_KEY ./minio server /data/file" --name minio
|
||||
|
||||
# Install Compiler
|
||||
echo 'Installing g++'
|
||||
apt-get install -y g++ >/dev/null
|
||||
@@ -81,5 +74,3 @@ pm2 save
|
||||
echo "Done"
|
||||
echo "Database username: hydro"
|
||||
echo "Database password: $db_password"
|
||||
echo "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY
|
||||
MINIO_SECRET_KEY=$MINIO_SECRET_KEY" >~/.hydro/env
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "hydrooj",
|
||||
"version": "3.15.19",
|
||||
"version": "3.16.0",
|
||||
"bin": "bin/hydrooj.js",
|
||||
"main": "src/loader",
|
||||
"module": "src/loader",
|
||||
@@ -15,7 +15,6 @@
|
||||
"@graphql-tools/schema": "^8.5.1",
|
||||
"@hydrooj/utils": "workspace:*",
|
||||
"adm-zip": "0.5.5",
|
||||
"ajv": "^8.11.0",
|
||||
"cac": "^6.7.12",
|
||||
"cookies": "^0.8.0",
|
||||
"detect-browser": "^5.3.0",
|
||||
|
||||
@@ -88,8 +88,9 @@ export async function load() {
|
||||
require('../options');
|
||||
const opts = options();
|
||||
await db.start(opts);
|
||||
await require('../settings').loadConfig();
|
||||
const storage = require('../service/storage');
|
||||
await storage.start();
|
||||
await storage.loadStorageService();
|
||||
require('../lib/index');
|
||||
await lib(pending, fail);
|
||||
const systemModel = require('../model/system');
|
||||
|
||||
@@ -37,12 +37,13 @@ export async function load() {
|
||||
if (detail) logger.info('finish: locale/template/static');
|
||||
const opts = options();
|
||||
await db.start(opts);
|
||||
await require('../settings').loadConfig();
|
||||
if (detail) logger.info('finish: db.connect');
|
||||
const modelSystem = require('../model/system');
|
||||
await modelSystem.runConfig();
|
||||
if (detail) logger.info('finish: config');
|
||||
const storage = require('../service/storage');
|
||||
await storage.start();
|
||||
await storage.loadStorageService();
|
||||
if (detail) logger.info('finish: storage.connect');
|
||||
require('../lib/index');
|
||||
if (detail) logger.info('finish: lib.builtin');
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { exec } from 'child_process';
|
||||
import { inspect } from 'util';
|
||||
import Ajv from 'ajv';
|
||||
import * as yaml from 'js-yaml';
|
||||
import Schema from 'schemastery';
|
||||
import * as check from '../check';
|
||||
import { BadRequestError, ValidationError } from '../error';
|
||||
import {
|
||||
@@ -19,12 +19,10 @@ import {
|
||||
Connection, ConnectionHandler, Handler,
|
||||
param, Route, Types,
|
||||
} from '../service/server';
|
||||
import { schema } from '../settings';
|
||||
import { configSource, saveConfig, SystemSettings } from '../settings';
|
||||
import * as judge from './judge';
|
||||
|
||||
const logger = new Logger('manage');
|
||||
const ajv = new Ajv({ useDefaults: true });
|
||||
const validator = ajv.compile<any>(schema);
|
||||
|
||||
function set(key: string, value: any) {
|
||||
if (setting.SYSTEM_SETTINGS_BY_KEY[key]) {
|
||||
@@ -156,55 +154,12 @@ class SystemSettingHandler extends SystemHandler {
|
||||
for (const s of this.response.body.settings) {
|
||||
this.response.body.current[s.key] = system.get(s.key);
|
||||
}
|
||||
const hide = [];
|
||||
const raw = system.get('_') || '';
|
||||
let config = yaml.load(raw);
|
||||
const valid = validator(config);
|
||||
if (!(raw.trim() && valid)) {
|
||||
const data = {};
|
||||
for (const key in schema.properties) {
|
||||
data[key] = {};
|
||||
for (const subkey in (schema.definitions[key] as any).properties || {}) {
|
||||
if (system.get(`${key}.${subkey}`)) {
|
||||
data[key][subkey] = system.get(`${key}.${subkey}`);
|
||||
if ((schema.definitions[key] as any).properties[subkey]?.writeOnly) {
|
||||
if (data[key][subkey] instanceof Array) {
|
||||
hide.push(...data[key][subkey]);
|
||||
} else hide.push(data[key][subkey]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
validator(data);
|
||||
config = yaml.dump(data);
|
||||
} else config = yaml.dump(config);
|
||||
this.response.body.hide = hide;
|
||||
this.response.body.config = config;
|
||||
}
|
||||
|
||||
async post(args: any) {
|
||||
const tasks = [];
|
||||
const booleanKeys = args.booleanKeys || {};
|
||||
delete args.booleanKeys;
|
||||
if (args._) {
|
||||
if (typeof args._ !== 'string') throw new ValidationError('config');
|
||||
try {
|
||||
const payload = yaml.load(args._);
|
||||
const valid = validator(payload);
|
||||
if (!valid) {
|
||||
throw new ValidationError('config', null, validator.errors[0]);
|
||||
}
|
||||
for (const key in payload) {
|
||||
for (const subkey in payload[key]) {
|
||||
tasks.push(system.set(`${key}.${subkey}`, payload[key][subkey]));
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
throw new ValidationError('config', null, e.message);
|
||||
}
|
||||
await system.set('_', args._);
|
||||
delete args._;
|
||||
}
|
||||
for (const key in args) {
|
||||
if (typeof args[key] === 'object') {
|
||||
for (const subkey in args[key]) {
|
||||
@@ -228,10 +183,25 @@ class SystemSettingHandler extends SystemHandler {
|
||||
}
|
||||
}
|
||||
|
||||
class SystemSettingSchemaHandler extends SystemHandler {
|
||||
class SystemConfigHandler extends SystemHandler {
|
||||
async get() {
|
||||
this.response.body = JSON.stringify(schema);
|
||||
this.response.type = 'application/json';
|
||||
this.response.template = 'manage_config.html';
|
||||
this.response.body = {
|
||||
schema: Schema.intersect(SystemSettings).toJSON(),
|
||||
value: configSource,
|
||||
};
|
||||
}
|
||||
|
||||
@param('value', Types.String)
|
||||
async post(domainId: string, value: string) {
|
||||
let config;
|
||||
try {
|
||||
config = yaml.load(value);
|
||||
Schema.intersect(SystemSettings)(config);
|
||||
} catch (e) {
|
||||
throw new ValidationError('value');
|
||||
}
|
||||
await saveConfig(config);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -294,7 +264,7 @@ async function apply() {
|
||||
Route('manage_dashboard', '/manage/dashboard', SystemDashboardHandler);
|
||||
Route('manage_script', '/manage/script', SystemScriptHandler);
|
||||
Route('manage_setting', '/manage/setting', SystemSettingHandler);
|
||||
Route('manage_setting_schema', '/manage/setting/schema.json', SystemSettingSchemaHandler);
|
||||
Route('manage_config', '/manage/config', SystemConfigHandler);
|
||||
Route('manage_user_import', '/manage/userimport', SystemUserImportHandler);
|
||||
Connection('manage_check', '/manage/check-conn', SystemCheckConnHandler);
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
/* eslint-disable camelcase */
|
||||
import { statSync } from 'fs';
|
||||
import { pick } from 'lodash';
|
||||
import { lookup } from 'mime-types';
|
||||
import {
|
||||
BadRequestError, ForbiddenError, ValidationError,
|
||||
} from '../error';
|
||||
import { md5 } from '../lib/crypto';
|
||||
import { PRIV } from '../model/builtin';
|
||||
import * as oplog from '../model/oplog';
|
||||
import storage from '../model/storage';
|
||||
@@ -12,6 +14,8 @@ import user from '../model/user';
|
||||
import {
|
||||
Handler, param, post, Route, Types,
|
||||
} from '../service/server';
|
||||
import { encodeRFC5987ValueChars } from '../service/storage';
|
||||
import { builtinConfig } from '../settings';
|
||||
import { sortFiles } from '../utils';
|
||||
|
||||
class SwitchLanguageHandler extends Handler {
|
||||
@@ -101,6 +105,23 @@ export class FSDownloadHandler extends Handler {
|
||||
}
|
||||
}
|
||||
|
||||
export class StorageHandler extends Handler {
|
||||
@param('target', Types.Name)
|
||||
@param('filename', Types.Name, true)
|
||||
@param('expire', Types.UnsignedInt)
|
||||
@param('secret', Types.String)
|
||||
async get(domainId: string, target: string, filename = '', expire: number, secret: string) {
|
||||
const expected = md5(`${target}/${expire}/${builtinConfig.file.secret}`);
|
||||
if (expire < Date.now()) throw new ForbiddenError('Link expired');
|
||||
if (secret !== expected) throw new ForbiddenError('Invalid secret');
|
||||
this.response.body = await storage.get(target);
|
||||
this.response.type = (target.endsWith('.out') || target.endsWith('.ans'))
|
||||
? 'text/plain'
|
||||
: lookup(target) || 'application/octet-stream';
|
||||
if (filename) this.response.disposition = `attachment; filename="${encodeRFC5987ValueChars(filename)}"`;
|
||||
}
|
||||
}
|
||||
|
||||
export class SwitchAccountHandler extends Handler {
|
||||
@param('uid', Types.Int)
|
||||
async get(domainId: string, uid: number) {
|
||||
@@ -113,6 +134,7 @@ export async function apply() {
|
||||
Route('switch_language', '/language/:lang', SwitchLanguageHandler);
|
||||
Route('home_files', '/file', FilesHandler);
|
||||
Route('fs_download', '/file/:uid/:filename', FSDownloadHandler);
|
||||
Route('storage', '/storage', StorageHandler);
|
||||
Route('switch_account', '/account', SwitchAccountHandler, PRIV.PRIV_EDIT_SYSTEM);
|
||||
}
|
||||
|
||||
|
||||
@@ -183,14 +183,6 @@ const ignoreUA = [
|
||||
].join('\n');
|
||||
|
||||
SystemSetting(
|
||||
Setting('setting_file', 'file.endPoint', 'http://127.0.0.1:9000', 'text', 'file.endPoint', 'Storage engine endPoint'),
|
||||
Setting('setting_file', 'file.accessKey', null, 'text', 'file.accessKey', 'Storage engine accessKey'),
|
||||
Setting('setting_file', 'file.secretKey', null, 'password', 'file.secretKey', 'Storage engine secret', FLAG_SECRET),
|
||||
Setting('setting_file', 'file.bucket', 'hydro', 'text', 'file.bucket', 'Storage engine bucket'),
|
||||
Setting('setting_file', 'file.region', 'us-east-1', 'text', 'file.region', 'Storage engine region'),
|
||||
Setting('setting_file', 'file.pathStyle', true, 'boolean', 'file.pathStyle', 'pathStyle endpoint'),
|
||||
Setting('setting_file', 'file.endPointForUser', '/fs/', 'text', 'file.endPointForUser', 'EndPoint for user'),
|
||||
Setting('setting_file', 'file.endPointForJudge', '/fs/', 'text', 'file.endPointForJudge', 'EndPoint for judge'),
|
||||
Setting('setting_smtp', 'smtp.user', null, 'text', 'smtp.user', 'SMTP Username'),
|
||||
Setting('setting_smtp', 'smtp.pass', null, 'password', 'smtp.pass', 'SMTP Password', FLAG_SECRET),
|
||||
Setting('setting_smtp', 'smtp.host', null, 'text', 'smtp.host', 'SMTP Server Host'),
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { extname } from 'path';
|
||||
import { escapeRegExp } from 'lodash';
|
||||
import { lookup } from 'mime-types';
|
||||
import { ItemBucketMetadata } from 'minio';
|
||||
import moment from 'moment';
|
||||
import { nanoid } from 'nanoid';
|
||||
import type { Readable } from 'stream';
|
||||
@@ -14,11 +13,8 @@ import TaskModel from './task';
|
||||
export class StorageModel {
|
||||
static coll = db.collection('storage');
|
||||
|
||||
static async put(path: string, file: string | Buffer | Readable, meta?: ItemBucketMetadata, owner?: number);
|
||||
static async put(path: string, file: string | Buffer | Readable, owner?: number);
|
||||
static async put(path: string, file: string | Buffer | Readable, arg0?: ItemBucketMetadata | number, arg1?: number) {
|
||||
const meta = typeof arg0 === 'object' ? arg0 : {};
|
||||
const owner = (typeof arg0 === 'number' ? arg0 : arg1) ?? 1;
|
||||
static async put(path: string, file: string | Buffer | Readable, owner?: number) {
|
||||
const meta = {};
|
||||
await StorageModel.del([path]);
|
||||
meta['Content-Type'] = (path.endsWith('.ans') || path.endsWith('.out'))
|
||||
? 'text/plain'
|
||||
@@ -27,7 +23,7 @@ export class StorageModel {
|
||||
// Make sure id is not used
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
while (await StorageModel.coll.findOne({ _id })) _id = `${nanoid(3)}/${nanoid()}${extname(path)}`;
|
||||
await storage.put(_id, file, meta);
|
||||
await storage.put(_id, file);
|
||||
const { metaData, size, etag } = await storage.getMeta(_id);
|
||||
await StorageModel.coll.insertOne({
|
||||
_id, meta: metaData, path, size, etag, lastModified: new Date(), owner,
|
||||
@@ -63,15 +59,10 @@ export class StorageModel {
|
||||
static async list(target: string, recursive = true) {
|
||||
if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
|
||||
if (target.length && !target.endsWith('/')) target += '/';
|
||||
const results = recursive
|
||||
? await StorageModel.coll.find({
|
||||
path: { $regex: new RegExp(`^${escapeRegExp(target)}`, 'i') },
|
||||
autoDelete: null,
|
||||
}).toArray()
|
||||
: await StorageModel.coll.find({
|
||||
path: { $regex: new RegExp(`^${escapeRegExp(target)}[^/]+$`) },
|
||||
autoDelete: null,
|
||||
}).toArray();
|
||||
const results = await StorageModel.coll.find({
|
||||
path: { $regex: new RegExp(`^${escapeRegExp(target)}${recursive ? '' : '[^/]+$'}`, 'i') },
|
||||
autoDelete: null,
|
||||
}).toArray();
|
||||
return results.map((i) => ({
|
||||
...i, name: i.path.split(target)[1], prefix: target,
|
||||
}));
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { existsSync, readFileSync } from 'fs-extra';
|
||||
import { findFileSync } from '@hydrooj/utils/lib/utils';
|
||||
import { Logger } from './logger';
|
||||
|
||||
@@ -8,8 +8,8 @@ const logger = new Logger('options');
|
||||
|
||||
export = function load() {
|
||||
const envFile = path.resolve(os.homedir(), '.hydro', 'env');
|
||||
if (fs.existsSync(envFile)) {
|
||||
const content = fs.readFileSync(envFile).toString().replace(/\r/g, '');
|
||||
if (existsSync(envFile)) {
|
||||
const content = readFileSync(envFile).toString().replace(/\r/g, '');
|
||||
for (const line of content.split('\n')) {
|
||||
if (!line.includes('=')) continue;
|
||||
process.env[line.split('=')[0]] = line.split('=')[1].trim();
|
||||
@@ -19,7 +19,7 @@ export = function load() {
|
||||
if (!f) return null;
|
||||
let result: any = {};
|
||||
try {
|
||||
result = JSON.parse(fs.readFileSync(f).toString());
|
||||
result = JSON.parse(readFileSync(f, 'utf-8'));
|
||||
} catch (e) {
|
||||
logger.error('Cannot read config file %o', e);
|
||||
result = {};
|
||||
|
||||
@@ -20,6 +20,7 @@ import { PERM, PRIV } from '../model/builtin';
|
||||
import * as opcount from '../model/opcount';
|
||||
import * as system from '../model/system';
|
||||
import { User } from '../model/user';
|
||||
import { builtinConfig } from '../settings';
|
||||
import { errorMessage } from '../utils';
|
||||
import * as bus from './bus';
|
||||
import * as decorators from './decorators';
|
||||
@@ -29,6 +30,7 @@ import rendererLayer from './layers/renderer';
|
||||
import responseLayer from './layers/response';
|
||||
import userLayer from './layers/user';
|
||||
import { Router } from './router';
|
||||
import { encodeRFC5987ValueChars } from './storage';
|
||||
|
||||
export * from './decorators';
|
||||
|
||||
@@ -109,11 +111,17 @@ const serializer = (showDisplayName = false) => (k: string, v: any) => {
|
||||
|
||||
export async function prepare() {
|
||||
app.keys = system.get('session.keys') as unknown as string[];
|
||||
app.use(proxy('/fs', {
|
||||
target: system.get('file.endPoint'),
|
||||
const proxyMiddleware = proxy('/fs', {
|
||||
target: builtinConfig.file.endPoint,
|
||||
changeOrigin: true,
|
||||
rewrite: (p) => p.replace('/fs', ''),
|
||||
}));
|
||||
});
|
||||
app.use(async (ctx, next) => {
|
||||
if (!ctx.path.startsWith('/fs/')) return await next();
|
||||
if (ctx.request.search.toLowerCase().includes('x-amz-credential')) return await proxyMiddleware(ctx, next);
|
||||
ctx.request.path = ctx.path = ctx.path.split('/fs')[1];
|
||||
return await next();
|
||||
});
|
||||
app.use(Compress());
|
||||
for (const dir of global.publicDirs) {
|
||||
app.use(cache(dir, {
|
||||
@@ -122,9 +130,9 @@ export async function prepare() {
|
||||
}
|
||||
if (process.env.DEV) {
|
||||
app.use(async (ctx: Context, next: Function) => {
|
||||
const startTime = new Date().getTime();
|
||||
const startTime = Date.now();
|
||||
await next();
|
||||
const endTime = new Date().getTime();
|
||||
const endTime = Date.now();
|
||||
if (ctx.nolog || ctx.response.headers.nolog) return;
|
||||
ctx._remoteAddress = ctx.request.ip;
|
||||
logger.debug(`${ctx.request.method} /${ctx.domainId || 'system'}${ctx.request.path} \
|
||||
@@ -216,7 +224,7 @@ export class Handler extends HandlerCommon {
|
||||
this.response.body = data;
|
||||
this.response.template = null;
|
||||
this.response.type = 'application/octet-stream';
|
||||
if (name) this.response.disposition = `attachment; filename="${encodeURIComponent(name)}"`;
|
||||
if (name) this.response.disposition = `attachment; filename="${encodeRFC5987ValueChars(name)}"`;
|
||||
}
|
||||
|
||||
async init() {
|
||||
|
||||
@@ -1,9 +1,16 @@
|
||||
import { dirname, resolve } from 'path';
|
||||
import { Readable } from 'stream';
|
||||
import { URL } from 'url';
|
||||
import { createReadStream } from 'fs-extra';
|
||||
import {
|
||||
copyFile, createReadStream, ensureDir,
|
||||
remove, stat, writeFile,
|
||||
} from 'fs-extra';
|
||||
import { lookup } from 'mime-types';
|
||||
import { BucketItem, Client, ItemBucketMetadata } from 'minio';
|
||||
import { md5 } from '../lib/crypto';
|
||||
import { Logger } from '../logger';
|
||||
import * as system from '../model/system';
|
||||
import { builtinConfig } from '../settings';
|
||||
import { MaybeArray } from '../typeutils';
|
||||
|
||||
const logger = new Logger('storage');
|
||||
|
||||
@@ -18,38 +25,38 @@ interface StorageOptions {
|
||||
endPointForJudge?: string;
|
||||
}
|
||||
|
||||
interface MinioEndpointConfig {
|
||||
interface EndpointConfig {
|
||||
endPoint: string;
|
||||
port: number;
|
||||
useSSL: boolean;
|
||||
}
|
||||
|
||||
function parseMainEndpointUrl(endpoint: string): MinioEndpointConfig {
|
||||
function parseMainEndpointUrl(endpoint: string): EndpointConfig {
|
||||
if (!endpoint) throw new Error('Empty endpoint');
|
||||
const url = new URL(endpoint);
|
||||
const result: Partial<MinioEndpointConfig> = {};
|
||||
if (url.pathname !== '/') throw new Error('Main MinIO endpoint URL of a sub-directory is not supported.');
|
||||
const result: Partial<EndpointConfig> = {};
|
||||
if (url.pathname !== '/') throw new Error('Main endpoint URL of a sub-directory is not supported.');
|
||||
if (url.username || url.password || url.hash || url.search) {
|
||||
throw new Error('Authorization, search parameters and hash are not supported for main MinIO endpoint URL.');
|
||||
throw new Error('Authorization, search parameters and hash are not supported for main endpoint URL.');
|
||||
}
|
||||
if (url.protocol === 'http:') result.useSSL = false;
|
||||
else if (url.protocol === 'https:') result.useSSL = true;
|
||||
else {
|
||||
throw new Error(
|
||||
`Invalid protocol "${url.protocol}" for main MinIO endpoint URL. Only HTTP and HTTPS are supported.`,
|
||||
`Invalid protocol "${url.protocol}" for main endpoint URL. Only HTTP and HTTPS are supported.`,
|
||||
);
|
||||
}
|
||||
result.endPoint = url.hostname;
|
||||
result.port = url.port ? Number(url.port) : result.useSSL ? 443 : 80;
|
||||
return result as MinioEndpointConfig;
|
||||
return result as EndpointConfig;
|
||||
}
|
||||
function parseAlternativeEndpointUrl(endpoint: string): (originalUrl: string) => string {
|
||||
if (!endpoint) return (originalUrl) => originalUrl;
|
||||
const pathonly = endpoint.startsWith('/');
|
||||
if (pathonly) endpoint = `https://localhost${endpoint}`;
|
||||
const url = new URL(endpoint);
|
||||
if (url.hash || url.search) throw new Error('Search parameters and hash are not supported for alternative MinIO endpoint URL.');
|
||||
if (!url.pathname.endsWith('/')) throw new Error("Alternative MinIO endpoint URL's pathname must ends with '/'.");
|
||||
if (url.hash || url.search) throw new Error('Search parameters and hash are not supported for alternative endpoint URL.');
|
||||
if (!url.pathname.endsWith('/')) throw new Error("Alternative endpoint URL's pathname must ends with '/'.");
|
||||
return (originalUrl) => {
|
||||
const parsedOriginUrl = new URL(originalUrl);
|
||||
const replaced = new URL(parsedOriginUrl.pathname.slice(1) + parsedOriginUrl.search + parsedOriginUrl.hash, url).toString();
|
||||
@@ -59,7 +66,7 @@ function parseAlternativeEndpointUrl(endpoint: string): (originalUrl: string) =>
|
||||
};
|
||||
}
|
||||
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent
|
||||
function encodeRFC5987ValueChars(str: string) {
|
||||
export function encodeRFC5987ValueChars(str: string) {
|
||||
return (
|
||||
encodeURIComponent(str)
|
||||
// Note that although RFC3986 reserves "!", RFC5987 does not,
|
||||
@@ -72,7 +79,7 @@ function encodeRFC5987ValueChars(str: string) {
|
||||
);
|
||||
}
|
||||
|
||||
class StorageService {
|
||||
class RemoteStorageService {
|
||||
public client: Client;
|
||||
public error = '';
|
||||
public opts: StorageOptions;
|
||||
@@ -80,13 +87,17 @@ class StorageService {
|
||||
|
||||
async start() {
|
||||
try {
|
||||
const [
|
||||
endPoint, accessKey, secretKey, bucket, region,
|
||||
pathStyle, endPointForUser, endPointForJudge,
|
||||
] = system.getMany([
|
||||
'file.endPoint', 'file.accessKey', 'file.secretKey', 'file.bucket', 'file.region',
|
||||
'file.pathStyle', 'file.endPointForUser', 'file.endPointForJudge',
|
||||
]);
|
||||
logger.info('Starting storage service with endpoint:', builtinConfig.file.endPoint);
|
||||
const {
|
||||
endPoint,
|
||||
accessKey,
|
||||
secretKey,
|
||||
bucket,
|
||||
region,
|
||||
pathStyle,
|
||||
endPointForUser,
|
||||
endPointForJudge,
|
||||
} = builtinConfig.file;
|
||||
this.opts = {
|
||||
endPoint,
|
||||
accessKey,
|
||||
@@ -97,11 +108,6 @@ class StorageService {
|
||||
endPointForUser,
|
||||
endPointForJudge,
|
||||
};
|
||||
if (process.env.MINIO_ACCESS_KEY) {
|
||||
logger.info('Using MinIO key from environment variables');
|
||||
this.opts.accessKey = process.env.MINIO_ACCESS_KEY;
|
||||
this.opts.secretKey = process.env.MINIO_SECRET_KEY;
|
||||
}
|
||||
this.client = new Client({
|
||||
...parseMainEndpointUrl(this.opts.endPoint),
|
||||
pathStyle: this.opts.pathStyle,
|
||||
@@ -133,7 +139,7 @@ class StorageService {
|
||||
if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
|
||||
if (typeof file === 'string') file = createReadStream(file);
|
||||
try {
|
||||
return await this.client.putObject(this.opts.bucket, target, file, meta);
|
||||
await this.client.putObject(this.opts.bucket, target, file, meta);
|
||||
} catch (e) {
|
||||
e.stack = new Error().stack;
|
||||
throw e;
|
||||
@@ -154,11 +160,7 @@ class StorageService {
|
||||
async del(target: string | string[]) {
|
||||
if (typeof target === 'string') {
|
||||
if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
|
||||
} else {
|
||||
for (const t of target) {
|
||||
if (t.includes('..') || t.includes('//')) throw new Error('Invalid path');
|
||||
}
|
||||
}
|
||||
} else if (target.find((t) => t.includes('..') || t.includes('//'))) throw new Error('Invalid path');
|
||||
try {
|
||||
if (typeof target === 'string') return await this.client.removeObject(this.opts.bucket, target);
|
||||
return await this.client.removeObjects(this.opts.bucket, target);
|
||||
@@ -173,7 +175,7 @@ class StorageService {
|
||||
if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
|
||||
try {
|
||||
const stream = this.client.listObjects(this.opts.bucket, target, recursive);
|
||||
return await new Promise<BucketItem[]>((resolve, reject) => {
|
||||
return await new Promise<BucketItem[]>((r, reject) => {
|
||||
const results: BucketItem[] = [];
|
||||
stream.on('data', (result) => {
|
||||
if (result.size) {
|
||||
@@ -184,7 +186,7 @@ class StorageService {
|
||||
});
|
||||
}
|
||||
});
|
||||
stream.on('end', () => resolve(results));
|
||||
stream.on('end', () => r(results));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
} catch (e) {
|
||||
@@ -237,6 +239,92 @@ class StorageService {
|
||||
}
|
||||
}
|
||||
|
||||
const service = new StorageService();
|
||||
global.Hydro.service.storage = service;
|
||||
export = service;
|
||||
class LocalStorageService {
|
||||
client: null;
|
||||
error: null;
|
||||
dir: string;
|
||||
opts: null;
|
||||
private replaceWithAlternativeUrlFor: Record<'user' | 'judge', (originalUrl: string) => string>;
|
||||
|
||||
async start() {
|
||||
logger.debug('Loading local storage service with path:', builtinConfig.file.path);
|
||||
await ensureDir(builtinConfig.file.path);
|
||||
this.dir = builtinConfig.file.path;
|
||||
this.replaceWithAlternativeUrlFor = {
|
||||
user: parseAlternativeEndpointUrl(builtinConfig.file.endPointForUser),
|
||||
judge: parseAlternativeEndpointUrl(builtinConfig.file.endPointForJudge),
|
||||
};
|
||||
}
|
||||
|
||||
async put(target: string, file: string | Buffer | Readable) {
|
||||
if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
|
||||
target = resolve(this.dir, target);
|
||||
await ensureDir(dirname(target));
|
||||
if (typeof file === 'string') await copyFile(file, target);
|
||||
else await writeFile(target, file);
|
||||
}
|
||||
|
||||
async get(target: string, path?: string) {
|
||||
if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
|
||||
target = resolve(this.dir, target);
|
||||
if (path) await copyFile(target, path);
|
||||
return createReadStream(target);
|
||||
}
|
||||
|
||||
async del(target: MaybeArray<string>) {
|
||||
const targets = typeof target === 'string' ? [target] : target;
|
||||
if (targets.find((i) => i.includes('..') || i.includes('//'))) throw new Error('Invalid path');
|
||||
await Promise.all(targets.map((i) => remove(resolve(this.dir, i))));
|
||||
}
|
||||
|
||||
async getMeta(target: string) {
|
||||
if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
|
||||
target = resolve(this.dir, target);
|
||||
const file = await stat(target);
|
||||
return {
|
||||
size: file.size,
|
||||
etag: Buffer.from(target).toString('base64'),
|
||||
lastModified: file.mtime,
|
||||
metaData: {
|
||||
'Content-Type': (target.endsWith('.ans') || target.endsWith('.out'))
|
||||
? 'text/plain'
|
||||
: lookup(target) || 'application/octet-stream',
|
||||
'Content-Length': file.size,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async signDownloadLink(target: string, filename = '', noExpire = false, useAlternativeEndpointFor?: 'user' | 'judge'): Promise<string> {
|
||||
if (target.includes('..') || target.includes('//')) throw new Error('Invalid path');
|
||||
const url = new URL('https://localhost/storage');
|
||||
url.searchParams.set('target', target);
|
||||
if (filename) url.searchParams.set('filename', filename);
|
||||
const expire = (Date.now() + (noExpire ? 7 * 24 * 3600 : 600) * 1000).toString();
|
||||
url.searchParams.set('expire', expire);
|
||||
url.searchParams.set('secret', md5(`${target}/${expire}/${builtinConfig.file.secret}`));
|
||||
if (useAlternativeEndpointFor) return this.replaceWithAlternativeUrlFor[useAlternativeEndpointFor](url.toString());
|
||||
return `/${url.toString().split('localhost/')[1]}`;
|
||||
}
|
||||
|
||||
async signUpload() {
|
||||
throw new Error('Not implemented');
|
||||
}
|
||||
|
||||
async list() {
|
||||
throw new Error('deprecated');
|
||||
}
|
||||
}
|
||||
|
||||
let service; // eslint-disable-line import/no-mutable-exports
|
||||
|
||||
export async function loadStorageService() {
|
||||
service = builtinConfig.file.type === 's3' ? new RemoteStorageService() : new LocalStorageService();
|
||||
global.Hydro.service.storage = service;
|
||||
await service.start();
|
||||
}
|
||||
|
||||
export default new Proxy({}, {
|
||||
get(self, key) {
|
||||
return service[key];
|
||||
},
|
||||
}) as RemoteStorageService | LocalStorageService;
|
||||
+104
-129
@@ -1,136 +1,111 @@
|
||||
import { JSONSchema7Definition } from 'json-schema';
|
||||
import yaml from 'js-yaml';
|
||||
import { nanoid } from 'nanoid';
|
||||
import Schema from 'schemastery';
|
||||
import * as bus from 'hydrooj/src/service/bus';
|
||||
import { Logger } from './logger';
|
||||
import { NestKeys } from './typeutils';
|
||||
|
||||
type Def = Exclude<JSONSchema7Definition, boolean>;
|
||||
const defaultPath = process.env.CI ? '/tmp/file' : '/data/file/hydro';
|
||||
const FileSetting = Schema.intersect([
|
||||
Schema.object({
|
||||
type: Schema.union([
|
||||
Schema.const('file').description('local file provider').required(),
|
||||
Schema.const('s3').description('s3 provider').required(),
|
||||
] as const).description('provider type').default('file'),
|
||||
endPointForUser: Schema.string().default('/fs/').required(),
|
||||
endPointForJudge: Schema.string().default('/fs/').required(),
|
||||
}).description('setting_file'),
|
||||
Schema.union([
|
||||
Schema.object({
|
||||
type: Schema.const('file').required(),
|
||||
path: Schema.string().default(defaultPath).description('Storage path').required(),
|
||||
secret: Schema.string().description('Download file sign secret').default(nanoid()),
|
||||
}),
|
||||
Schema.object({
|
||||
type: Schema.const('s3').required(),
|
||||
endPoint: Schema.string().required(),
|
||||
accessKey: Schema.string().required().description('access key'),
|
||||
secretKey: Schema.string().required().description('secret key').role('secret'),
|
||||
bucket: Schema.string().default('hydro').required(),
|
||||
region: Schema.string().default('us-east-1').required(),
|
||||
pathStyle: Schema.boolean().default(true).required(),
|
||||
}),
|
||||
] as const),
|
||||
] as const).default({
|
||||
type: 'file',
|
||||
path: defaultPath,
|
||||
endPointForUser: '/fs/',
|
||||
endPointForJudge: '/fs/',
|
||||
secret: nanoid(),
|
||||
});
|
||||
|
||||
function port(examples: number[] = []) {
|
||||
const res: Def = {
|
||||
type: 'integer', minimum: 1, maximum: 65535,
|
||||
};
|
||||
if (examples.length) {
|
||||
res.default = examples[0];
|
||||
res.examples = examples;
|
||||
const builtinSettings = Schema.object({
|
||||
file: FileSetting,
|
||||
});
|
||||
export const SystemSettings: Schema[] = [builtinSettings];
|
||||
export let configSource = ''; // eslint-disable-line import/no-mutable-exports
|
||||
export let systemConfig: any = {}; // eslint-disable-line import/no-mutable-exports
|
||||
const logger = new Logger('settings');
|
||||
const update = [];
|
||||
|
||||
export async function loadConfig() {
|
||||
const config = await global.Hydro.service.db.collection('system').findOne({ _id: 'config' });
|
||||
try {
|
||||
configSource = config?.value || '{}';
|
||||
systemConfig = yaml.load(configSource);
|
||||
logger.info('Successfully loaded config');
|
||||
for (const u of update) u();
|
||||
} catch (e) {
|
||||
logger.error('Failed to load config', e.message);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
export async function saveConfig(config: any) {
|
||||
Schema.intersect(SystemSettings)(config);
|
||||
const value = yaml.dump(config);
|
||||
await global.Hydro.service.db.collection('system').updateOne({ _id: 'config' }, { $set: { value } }, { upsert: true });
|
||||
bus.broadcast('config/update');
|
||||
}
|
||||
export async function setConfig(key: string, value: any) {
|
||||
const path = key.split('.');
|
||||
const t = path.pop();
|
||||
let cursor = systemConfig;
|
||||
for (const p of path) {
|
||||
if (!cursor[p]) cursor[p] = {};
|
||||
cursor = cursor[p];
|
||||
}
|
||||
cursor[t] = value;
|
||||
await saveConfig(systemConfig);
|
||||
}
|
||||
|
||||
export const Schema = {
|
||||
string<T extends Def>(title: string, defaultValue: string, extra?: T) {
|
||||
return {
|
||||
type: 'string' as 'string',
|
||||
default: defaultValue,
|
||||
title,
|
||||
...extra,
|
||||
};
|
||||
},
|
||||
boolean<T extends Def>(title: string, defaultValue: boolean, extra?: T) {
|
||||
return {
|
||||
type: 'boolean' as 'boolean',
|
||||
default: defaultValue,
|
||||
title,
|
||||
...extra,
|
||||
};
|
||||
},
|
||||
integer<T extends Def>(title: string, defaultValue: number, extra?: T) {
|
||||
return {
|
||||
type: 'integer' as 'integer',
|
||||
default: defaultValue,
|
||||
title,
|
||||
...extra,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const definitions: Record<string, Def> = {
|
||||
smtp: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
user: Schema.string('SMTP Username', 'noreply@hydro.ac'),
|
||||
from: Schema.string('Mail From', 'Hydro <noreply@hydro.ac>'),
|
||||
pass: Schema.string('SMTP Password', '', { writeOnly: true }),
|
||||
host: Schema.string('SMTP Server Host', 'smtp.hydro.ac', { pattern: '^[a-zA-Z0-9\\-\\.]+$' }),
|
||||
port: Schema.integer('SMTP Server Port', 25, { examples: [25, 465], minimum: 1, maximum: 65535 }),
|
||||
secure: Schema.boolean('Use SSL', false),
|
||||
verify: Schema.boolean('Verify register email', false),
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
file: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
endPoint: Schema.string('Storage engine endPoint', 'http://localhost:9000', {
|
||||
pattern: '^https?://[a-zA-Z0-9\\-\\.]+/?$',
|
||||
}),
|
||||
accessKey: Schema.string('Storage engine accessKey', ''),
|
||||
secretKey: Schema.string('Storage engine secretKey', '', { writeOnly: true }),
|
||||
bucket: Schema.string('Storage engine bucket', 'hydro'),
|
||||
region: Schema.string('Storage engine region', 'us-east-1'),
|
||||
pathStyle: Schema.boolean('pathStyle endpoint', true),
|
||||
endPointForUser: Schema.string('EndPoint for user', '/fs/'),
|
||||
endPointForJudge: Schema.string('EndPoint for judge', '/fs/'),
|
||||
},
|
||||
required: ['endPoint', 'accessKey', 'secretKey'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
server: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
name: Schema.string('Server Name', 'Hydro'),
|
||||
url: Schema.string('Self URL', 'https://hydro.ac/', { pattern: '/$' }),
|
||||
cdn: Schema.string('CDN prefix', '/', {
|
||||
pattern: '/$', examples: ['/', 'https://cdn.hydro.ac/'],
|
||||
}),
|
||||
port: port([8888, 80, 443]),
|
||||
xff: Schema.string('IP Header', '', { examples: ['x-forwarded-for', 'x-real-ip'], pattern: '^[a-z-]+$' }),
|
||||
xhost: Schema.string('Host Header', '', { examples: ['x-real-host'], pattern: '^[a-z-]+$' }),
|
||||
language: { type: 'string', enum: Object.keys(global.Hydro.locales) },
|
||||
upload: Schema.string('Upload size limit', '256m', { pattern: '^[0-9]+[mkg]b?$' }),
|
||||
login: Schema.boolean('Enable builtin login', true),
|
||||
message: Schema.boolean('Enable message', true),
|
||||
blog: Schema.boolean('Enable blog', true),
|
||||
checkUpdate: Schema.boolean('Daily update check', true),
|
||||
},
|
||||
required: ['url', 'port', 'language'],
|
||||
},
|
||||
limit: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
problem_files_max: { type: 'integer', minimum: 0 },
|
||||
},
|
||||
},
|
||||
session: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
keys: {
|
||||
type: 'array', items: { type: 'string' }, default: [String.random(32)], writeOnly: true,
|
||||
export function requestConfig<T, S>(s: Schema<T, S>): {
|
||||
config: ReturnType<Schema<T, S>>,
|
||||
setConfig: (key: NestKeys<ReturnType<Schema<T, S>>>, value: any) => Promise<void>,
|
||||
} {
|
||||
SystemSettings.push(s);
|
||||
let curValue = s(systemConfig);
|
||||
update.push(() => {
|
||||
try {
|
||||
curValue = s(systemConfig);
|
||||
} catch (e) {
|
||||
logger.warn('Cannot read config: ', e.message);
|
||||
curValue = null;
|
||||
}
|
||||
});
|
||||
return {
|
||||
config: new Proxy(curValue as any, {
|
||||
get(self, key: string) {
|
||||
return curValue?.[key];
|
||||
},
|
||||
secure: { type: 'boolean', default: false },
|
||||
saved_expire_seconds: { type: 'integer', minimum: 300, default: 3600 * 24 * 30 },
|
||||
unsaved_expire_seconds: { type: 'integer', minimum: 60, default: 3600 * 3 },
|
||||
},
|
||||
},
|
||||
user: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
quota: { type: 'integer', minimum: 0 },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
export const schema: Def = {
|
||||
type: 'object',
|
||||
definitions,
|
||||
properties: {
|
||||
smtp: definitions.smtp,
|
||||
file: definitions.file,
|
||||
server: definitions.server,
|
||||
limit: definitions.limit,
|
||||
session: definitions.session,
|
||||
user: definitions.user,
|
||||
},
|
||||
additionalProperties: true,
|
||||
};
|
||||
|
||||
export function addDef(key: string, def: Def) {
|
||||
definitions[key] = def;
|
||||
schema.properties[key] = definitions[key];
|
||||
set(self) {
|
||||
throw new Error(`Not allowed to set setting ${self.p.join('.')}`);
|
||||
},
|
||||
}),
|
||||
setConfig,
|
||||
};
|
||||
}
|
||||
|
||||
const builtin = requestConfig(builtinSettings);
|
||||
export const builtinConfig = builtin.config;
|
||||
export const setBuiltinConfig = builtin.setConfig;
|
||||
|
||||
bus.on('config/update', loadConfig);
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
} from './pipelineUtils';
|
||||
import db from './service/db';
|
||||
import storage from './service/storage';
|
||||
import { setBuiltinConfig } from './settings';
|
||||
import { streamToBuffer } from './utils';
|
||||
import welcome from './welcome';
|
||||
|
||||
@@ -94,7 +95,7 @@ const scripts: UpgradeScript[] = [
|
||||
try {
|
||||
const [file, current] = await Promise.all([
|
||||
streamToBuffer(gridfs.openDownloadStream(pdoc.data)),
|
||||
storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`),
|
||||
storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`) as any,
|
||||
]);
|
||||
const zip = new AdmZip(file);
|
||||
const entries = zip.getEntries();
|
||||
@@ -146,8 +147,8 @@ const scripts: UpgradeScript[] = [
|
||||
await iterateAllProblem(['docId', 'domainId', 'config'], async (pdoc) => {
|
||||
logger.info('%s/%s', pdoc.domainId, pdoc.docId);
|
||||
const [data, additional_file] = await Promise.all([
|
||||
storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`),
|
||||
storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/additional_file/`),
|
||||
storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`) as any,
|
||||
storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/additional_file/`) as any,
|
||||
]);
|
||||
await problem.edit(
|
||||
pdoc.domainId, pdoc.docId,
|
||||
@@ -802,6 +803,33 @@ const scripts: UpgradeScript[] = [
|
||||
});
|
||||
return true;
|
||||
},
|
||||
async function _66_67() {
|
||||
const [
|
||||
endPoint, accessKey, secretKey, bucket, region,
|
||||
pathStyle, endPointForUser, endPointForJudge,
|
||||
] = system.getMany([
|
||||
'file.endPoint', 'file.accessKey', 'file.secretKey', 'file.bucket', 'file.region',
|
||||
'file.pathStyle', 'file.endPointForUser', 'file.endPointForJudge',
|
||||
]);
|
||||
if ((endPoint && accessKey) || process.env.MINIO_ACCESS_KEY) {
|
||||
await setBuiltinConfig('file', {
|
||||
type: 's3',
|
||||
endPoint: process.env.MINIO_ACCESS_KEY ? 'http://127.0.0.1:9000/' : endPoint,
|
||||
accessKey: process.env.MINIO_ACCESS_KEY || accessKey,
|
||||
secretKey: process.env.MINIO_SECRET_KEY || secretKey,
|
||||
bucket,
|
||||
region,
|
||||
pathStyle,
|
||||
endPointForUser,
|
||||
endPointForJudge,
|
||||
});
|
||||
setTimeout(() => {
|
||||
logger.info('Upgrade done. please restart the server.');
|
||||
process.exit(0);
|
||||
}, 1000);
|
||||
}
|
||||
return true;
|
||||
},
|
||||
];
|
||||
|
||||
export default scripts;
|
||||
@@ -96,7 +96,7 @@ export default function (env: { production?: boolean, measure?: boolean } = {})
|
||||
chunkFilename: '[name].[chunkhash].chunk.js',
|
||||
},
|
||||
resolve: {
|
||||
extensions: ['.js', '.jsx', '.ts', '.tsx'],
|
||||
extensions: ['.js', '.jsx', '.ts', '.tsx', '.cjs'],
|
||||
alias: {
|
||||
vj: root(),
|
||||
},
|
||||
@@ -148,8 +148,9 @@ export default function (env: { production?: boolean, measure?: boolean } = {})
|
||||
},
|
||||
},
|
||||
{
|
||||
test: /\.[jt]sx?$/,
|
||||
test: /\.[mc]?[jt]sx?$/,
|
||||
exclude: /@types\//,
|
||||
type: 'javascript/auto',
|
||||
use: [esbuildLoader()],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -107,7 +107,6 @@ async function onCommentClickReplyComment(ev, options: any = {}) {
|
||||
}
|
||||
|
||||
async function onCommentClickReplyReply(ev) {
|
||||
console.log(ev);
|
||||
const $evTarget = $(ev.currentTarget);
|
||||
const $mediaBody = $evTarget.closest('.media__body');
|
||||
const uid = $mediaBody
|
||||
|
||||
@@ -7,7 +7,6 @@ function runSubstitute($container: JQuery<Document | HTMLElement>) {
|
||||
$container.find(`textarea[data-${language}]`).get().forEach((element) => {
|
||||
const config: any = { language };
|
||||
if ($(element).data('model')) config.model = $(element).data('model');
|
||||
if ($(element).data('hide')) config.hide = $(element).data('hide');
|
||||
CmEditor.getOrConstruct($(element), config);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -98,7 +98,7 @@ setDiagnosticsOptions({
|
||||
schema: problemConfigSchema as any,
|
||||
},
|
||||
{
|
||||
uri: `${UiContext.cdn_prefix}manage/setting/schema.json`,
|
||||
uri: '/manage/config/schema.json',
|
||||
fileMatch: ['hydro://system/setting.yaml'],
|
||||
},
|
||||
],
|
||||
|
||||
@@ -11,9 +11,12 @@ import user from 'hydrooj/src/model/user';
|
||||
import * as bus from 'hydrooj/src/service/bus';
|
||||
import { UiContextBase } from 'hydrooj/src/service/layers/base';
|
||||
import { Handler, Route } from 'hydrooj/src/service/server';
|
||||
import { SystemSettings } from 'hydrooj/src/settings';
|
||||
import { ObjectID } from 'mongodb';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import Schema from 'schemastery';
|
||||
import convert from 'schemastery-jsonschema';
|
||||
import markdown from './backendlib/markdown';
|
||||
|
||||
declare module 'hydrooj/src/interface' {
|
||||
@@ -157,6 +160,13 @@ class LanguageHandler extends ResourceHandler {
|
||||
}
|
||||
}
|
||||
|
||||
class SystemConfigSchemaHandler extends Handler {
|
||||
async get() {
|
||||
const schema = convert(Schema.intersect(SystemSettings) as any, true);
|
||||
this.response.body = schema;
|
||||
}
|
||||
}
|
||||
|
||||
class RichMediaHandler extends Handler {
|
||||
async renderUser(domainId, payload) {
|
||||
let d = payload.domainId || domainId;
|
||||
@@ -213,6 +223,7 @@ global.Hydro.handler.ui = async () => {
|
||||
Route('set_theme', '/set_theme/:theme', SetThemeHandler);
|
||||
Route('constant', '/constant/:version', UiConstantsHandler);
|
||||
Route('markdown', '/markdown', MarkdownHandler);
|
||||
Route('config_schema', '/manage/config/schema.json', SystemConfigSchemaHandler, PRIV.PRIV_EDIT_SYSTEM);
|
||||
Route('lang', '/l/:lang', LanguageHandler);
|
||||
Route('media', '/media', RichMediaHandler);
|
||||
};
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@hydrooj/ui-default",
|
||||
"version": "4.39.8",
|
||||
"version": "4.39.9",
|
||||
"author": "undefined <i@undefined.moe>",
|
||||
"license": "AGPL-3.0",
|
||||
"main": "hydro.js",
|
||||
@@ -129,6 +129,7 @@
|
||||
"mongodb": "^3.7.3",
|
||||
"nunjucks": "^3.2.3",
|
||||
"p-queue": "^7.3.0",
|
||||
"schemastery-jsonschema": "^1.0.3",
|
||||
"streamsaver": "^2.0.6",
|
||||
"xss": "^1.0.14"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import yaml from 'js-yaml';
|
||||
import Schema from 'schemastery';
|
||||
import Notification from 'vj/components/notification';
|
||||
import { NamedPage } from 'vj/misc/Page';
|
||||
import request from 'vj/utils/request';
|
||||
|
||||
const page = new NamedPage('manage_config', async () => {
|
||||
const schema = new Schema(UiContext.schema);
|
||||
setInterval(() => {
|
||||
try {
|
||||
const v = yaml.load($('#config').val().toString());
|
||||
schema(v);
|
||||
$('#info').text('');
|
||||
} catch (e) {
|
||||
console.debug(e);
|
||||
$('#info').text(e.message);
|
||||
}
|
||||
}, 1000);
|
||||
$('#submit').on('click', () => {
|
||||
const value = $('#config').val();
|
||||
request.post('', { value }).then(() => {
|
||||
Notification.success('保存成功');
|
||||
}).catch((e) => {
|
||||
Notification.error('保存失败:', e.message);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
export default page;
|
||||
@@ -16,6 +16,7 @@
|
||||
{{ sidemenu.render_item(null, 'manage_script') }}
|
||||
{{ sidemenu.render_item(null, 'manage_user_import') }}
|
||||
{{ sidemenu.render_item(null, 'manage_setting') }}
|
||||
{{ sidemenu.render_item(null, 'manage_config') }}
|
||||
</ol>
|
||||
</li>
|
||||
</ol>
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
{% extends "manage_base.html" %}
|
||||
{% block manage_content %}
|
||||
{{ set(UiContext, 'schema', schema) }}
|
||||
<div class="section">
|
||||
<div class="section__body typo">
|
||||
<div class="row">
|
||||
<div class="medium-10 columns form__item">
|
||||
<label>
|
||||
config
|
||||
<div name="form_item_config" class="textarea-container">
|
||||
<textarea id="config" data-yaml data-model="hydro://system/setting.yaml" placeholder="" class="textbox">{{ value }}</textarea>
|
||||
</div>
|
||||
</label>
|
||||
<p class="help-text" id="info"></p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="medium-10 columns form__item end">
|
||||
<button id="submit" class="rounded primary button">{{ _('Save All Changes') }}</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -8,7 +8,6 @@
|
||||
{% if not setting.flag|bitand(model.setting.FLAG_HIDDEN) %}
|
||||
{% set secret = setting.flag|bitand(model.setting.FLAG_SECRET) != 0 %}
|
||||
{% if setting.type == 'text' or setting.type == 'password' or setting.type == 'number' or setting.type == 'float' %}
|
||||
{% set isFileConfig = (setting.name === 'file.accessKey' or setting.name === 'file.secretKey') and process.env.MINIO_ACCESS_KEY %}
|
||||
{{ form.form_text({
|
||||
type:setting.type,
|
||||
label:setting.name,
|
||||
@@ -16,7 +15,7 @@
|
||||
name:setting.key,
|
||||
value:'' if (secret or isFileConfig) else (current[setting.key]|default(setting.value)),
|
||||
disabled:setting.flag|bitand(2),
|
||||
placeholder:'Please edit in ~/.hydro/env instead' if isFileConfig else (_('(Not changed)') if secret else '')
|
||||
placeholder:_('(Not changed)') if secret else ''
|
||||
}) }}
|
||||
{% elif setting.type == 'select' %}
|
||||
{{ form.form_select({
|
||||
|
||||
@@ -214,7 +214,6 @@ export default class CodeforcesProvider implements IBasicProvider {
|
||||
await page.waitForRequest((req) => {
|
||||
if (req.method() !== 'POST') return false;
|
||||
if (!req.url().endsWith('/enter')) return false;
|
||||
console.log(req);
|
||||
return true;
|
||||
}, { timeout: 24 * 3600 * 1000 });
|
||||
await page.waitForTimeout(10 * 1000);
|
||||
|
||||
Reference in new issue
Block a user