Weishen/Hydro
2
0 Fork 0
mirror of https://gitcode.com/gh_mirrors/hy/Hydro.git synced 2026-10-17 09:36:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

core: logger upd

Browse Source
This commit is contained in:
undefined committed 2020-12-21 03:06:41 +08:00
1 parent d4088f429a
commit 4d031eb9ab
9 files changed
+141 -131
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/hydrojudge/src/service.ts
+1 -1
View File
@@ -66,7 +66,7 @@ async function postInit() {
const filePath = path.join(homedir(), '.cache', 'hydro', 'judge', domainId, pid);
await fs.ensureDir(filePath);
if (!files.length) throw new SystemError('Problem data not found.');
let etags: Record<string, string>;
let etags: Record<string, string> = {};
try {
etags = JSON.parse(fs.readFileSync(path.join(filePath, 'etags')).toString());
} catch (e) { /* ignore */ }
packages/hydrojudge/src/utils.ts
+8
View File
@@ -109,6 +109,14 @@ export function restrictFile(p: string) {
export function ensureFile(folder: string) {
return (file: string, message: string) => {
// Historical issue
if (file.includes('/')) {
const f = path.join(folder, restrictFile(file.split('/')[1]));
if (fs.existsSync(f)) {
const stat = fs.statSync(f);
if (stat.isFile()) return f;
}
}
const f = path.join(folder, restrictFile(file));
if (!fs.existsSync(f)) throw new FormatError(message, [file]);
const stat = fs.statSync(f);
packages/hydrooj/package.json
+2 -1
View File
@@ -27,6 +27,7 @@
"moment-timezone": "^0.5.32",
"mongodb": "^3.6.3",
"nodemailer": "^6.4.17",
"p-queue": "^6.6.2",
"serialize-javascript": "^5.0.1",
"sockjs": "^0.3.20",
"superagent": "^6.1.0",
@@ -58,4 +59,4 @@
"build": "tsc",
"build:watch": "tsc --watch"
}
}
}
packages/hydrooj/src/handler/problem.ts
+5 -4
View File
@@ -453,12 +453,13 @@ export class ProblemFilesHandler extends ProblemDetailHandler {
@param('testdata', Types.Boolean)
@param('additional_file', Types.Boolean)
async get(domainId: string, getTestdata = true, getAdditionalFile = true) {
const canReadData = this.user._id === this.pdoc.owner || this.user.hasPerm(PERM.PERM_READ_PROBLEM_DATA);
const [testdata, additional_file] = await Promise.all([
getTestdata
? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/testdata`, true)
getTestdata && canReadData
? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/testdata/`, true)
: [],
getAdditionalFile
? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file`, true)
? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/`, true)
: [],
]);
this.response.body.testdata = testdata;
@@ -671,7 +672,7 @@ export class ProblemImportHandler extends Handler {
this.response.template = 'problem_import.html';
}
async post(domainId: string) {
async post({ domainId }) {
if (!this.request.files.file) throw new ValidationError('file');
const stat = await fs.stat(this.request.files.file.path);
if (stat.size > 128 * 1024 * 1024) throw new BadRequestError('File too large');
packages/hydrooj/src/model/builtin.ts
+6
View File
@@ -18,11 +18,17 @@ export const PERM = {
// Problem and Record
PERM_CREATE_PROBLEM: 1n << 4n,
PERM_EDIT_PROBLEM: 1n << 5n,
/**
* @deprecated
*/
PERM_EDIT_PROBLEM_SELF: 1n << 6n,
PERM_VIEW_PROBLEM: 1n << 7n,
PERM_VIEW_PROBLEM_HIDDEN: 1n << 8n,
PERM_SUBMIT_PROBLEM: 1n << 9n,
PERM_READ_PROBLEM_DATA: 1n << 10n,
/**
* @deprecated
*/
PERM_READ_PROBLEM_DATA_SELF: 1n << 11n,
PERM_READ_RECORD_CODE: 1n << 12n,
PERM_REJUDGE_PROBLEM: 1n << 13n,
packages/hydrooj/src/service/server.ts
+5 -11
View File
@@ -520,7 +520,7 @@ export class Handler {
try {
await this.render(error instanceof UserFacingError ? 'error.html' : 'bsod.html', { error });
} catch (e) {
console.error(e);
logger.error(e);
// this.response.body.error = {};
}
}
@@ -615,8 +615,8 @@ export class Handler {
async onerror(error: HydroError) {
if (!error.msg) error.msg = () => error.message;
if (!(error instanceof NotFoundError)) {
console.error(error.msg(), error.params);
console.error(error.stack);
logger.error(error.msg(), error.params);
logger.error(error.stack);
}
this.response.status = error instanceof UserFacingError ? error.code : 500;
this.response.template = error instanceof UserFacingError ? 'error.html' : 'bsod.html';
@@ -807,7 +807,7 @@ export class ConnectionHandler {
async message(message: any) { } // eslint-disable-line
onerror(err: HydroError) {
if (!(err instanceof NotFoundError)) console.error(err);
if (!(err instanceof NotFoundError)) logger.error(err);
this.send({
error: {
name: err.name,
@@ -895,13 +895,7 @@ export function start() {
if (ctx.response.headers.nolog) return;
ctx._remoteAddress = ctx.request.ip;
const status = ctx.response.status;
const color = status >= 500 ? 31 // red
: status >= 400 ? 33 // yellow
: status >= 300 ? 36 // cyan
: status >= 200 ? 32 // green
: 0; // no color
logger.debug(`\
\x1b[0m${ctx.request.method} ${ctx.request.path} \x1b[${color}m${ctx.response.status}\x1b[0m ${endTime - startTime}ms ${ctx.response.length}\x1b[0m`);
logger.debug(`${ctx.request.method} ${ctx.request.path} ${ctx.response.status} ${endTime - startTime}ms ${ctx.response.length}`);
});
}
app.use(async (ctx, next) => {
packages/hydrooj/src/ui.ts
+15
View File
@@ -51,7 +51,11 @@ if (cluster.isMaster) {
bus.on('message/log', (message) => {
LogBox.appendLog(message);
});
const history = [''];
let current = 0;
ShellInput.on('submit', async (input) => {
history.push(input);
current = history.length;
ShellInput.input.setContent(' '.repeat(terminal.width), false, false);
ShellInput.input.setContent('', false, false);
if (input[0] === '@') {
@@ -61,6 +65,17 @@ if (cluster.isMaster) {
}
} else bus.parallel('message/run', input);
});
ShellInput.on('key', (key) => {
if (key === 'UP') {
current--;
ShellInput.input.setContent(' '.repeat(terminal.width), false, false);
ShellInput.input.setContent(history[current] || '', false, false);
} else if (key === 'DOWN') {
current++;
ShellInput.input.setContent(' '.repeat(terminal.width), false, false);
ShellInput.input.setContent(history[current] || '', false, false);
}
});
terminal.on('key', (key, matches, data) => {
if (key === 'CTRL_C') terminate();
});
packages/hydrooj/src/upgrade.ts
+71 -70
View File
@@ -1,7 +1,9 @@
/* eslint-disable no-await-in-loop */
import { ObjectID, Collection } from 'mongodb';
// @ts-ignore
import { terminal } from 'terminal-kit';
import AdmZip from 'adm-zip';
import Queue from 'p-queue';
import { Ufdoc } from './interface';
import { Logger } from './logger';
import { streamToBuffer } from './utils';
@@ -60,83 +62,82 @@ const scripts: UpgradeScript[] = [
return false;
}
const coll: Collection<Ufdoc> = db.collection('file');
const collFile = db.collection('fs.files');
const collChunk = db.collection('fs.chunks');
function del(_id: ObjectID) {
return Promise.all([
coll.deleteOne({ _id }),
collFile.deleteOne({ _id }),
collChunk.deleteMany({ files_id: _id }),
]);
}
logger.info('Changing storage engine. This may take a long time.');
// Problem file and User file
try {
let savedProgress = system.get('upgrade.file.progress.domain');
if (savedProgress) savedProgress = JSON.parse(savedProgress);
else savedProgress = { pdocs: [] };
const ddocs = await domain.getMulti().project({ _id: 1 }).toArray();
logger.info('Total found %d domains.', ddocs.length);
for (let i = 0; i < ddocs.length; i++) {
const ddoc = ddocs[i];
logger.info('Domain %s (%d/%d)', ddoc._id, i + 1, ddocs.length);
const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray();
const domainProgress = terminal.progressBar({ items: pdocs.length, title: 'Problems', inline: true });
for (let j = 0; j < pdocs.length; j++) {
const pdoc = pdocs[j];
domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`);
if (!savedProgress.pdocs.includes(`${pdoc.domainId}/${pdoc.docId}`) && pdoc.data instanceof ObjectID) {
try {
const file = await streamToBuffer(gridfs.openDownloadStream(pdoc.data));
const zip = new AdmZip(file);
const entries = zip.getEntries();
await Promise.all(entries.map(
(entry) => storage.put(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/${entry.name}`, entry.getData()),
));
} catch (e) {
if (e.toString().includes('FileNotFound')) {
logger.error('Problem data not found %s/%s', pdoc.domainId, pdoc.docId);
} else throw e;
let savedProgress = system.get('upgrade.file.progress.domain');
if (savedProgress) savedProgress = JSON.parse(savedProgress);
else savedProgress = { pdocs: [] };
const ddocs = await domain.getMulti().project({ _id: 1 }).toArray();
logger.info('Total found %d domains.', ddocs.length);
for (let i = 0; i < ddocs.length; i++) {
const ddoc = ddocs[i];
logger.info('Domain %s (%d/%d)', ddoc._id, i + 1, ddocs.length);
const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray();
const domainProgress = terminal.progressBar({ items: pdocs.length, title: 'Problems', inline: true });
for (let j = 0; j < pdocs.length; j++) {
const pdoc = pdocs[j];
domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`);
if (!savedProgress.pdocs.includes(`${pdoc.domainId}/${pdoc.docId}`) && pdoc.data instanceof ObjectID) {
try {
const [file, current] = await Promise.all([
streamToBuffer(gridfs.openDownloadStream(pdoc.data)),
storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`, true),
]);
const zip = new AdmZip(file);
const entries = zip.getEntries();
if (entries.map((entry) => entry.entryName).sort().join('?') !== current.map((f) => f.name).sort().join('?')) {
await storage.del(current.map((entry) => entry.prefix + entry.name));
const queue = new Queue({ concurrency: 5 });
const tasks = [];
for (const entry of entries) {
tasks.push(queue.add(() => storage.put(
`problem/${pdoc.domainId}/${pdoc.docId}/testdata/${entry.entryName}`,
entry.getData(),
)));
}
await Promise.all(tasks);
}
savedProgress.pdocs.push(`${pdoc.domainId}/${pdoc.docId}`);
} catch (e) {
if (e.toString().includes('FileNotFound')) {
logger.error('Problem data not found %s/%s', pdoc.domainId, pdoc.docId);
} else throw e;
}
system.set('upgrade.file.progress.domain', JSON.stringify(savedProgress));
domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`);
savedProgress.pdocs.push(`${pdoc.domainId}/${pdoc.docId}`);
}
domainProgress.stop();
system.set('upgrade.file.progress.domain', JSON.stringify(savedProgress));
domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`);
}
const udocs = await user.getMulti().project({ _id: 1, uname: 1 }).toArray();
const userfileProgress = terminal.progressBar({
items: udocs.length, title: 'Users', y: 3,
});
for (const udoc of udocs) {
userfileProgress.startItem(`${udoc._id}: ${udoc.uname}`);
const ufdocs = await coll.find({ owner: udoc._id }).toArray();
const currentUser = terminal.progressBar({
items: ufdocs.length, title: `${udoc._id}: ${udoc.uname}`, y: 4, eta: true,
});
const namelist = [];
for (const ufdoc of ufdocs) {
currentUser.startItem(`${ufdoc._id}: ${ufdoc.filename || ''}`);
ufdoc.filename = ufdoc.filename || ufdoc._id.toHexString();
const file = await streamToBuffer(gridfs.openDownloadStream(ufdoc._id));
if (!namelist.includes(ufdoc.filename)) {
await storage.put(`user/${udoc._id}/${ufdoc.filename}`, file);
} else {
await storage.put(`user/${udoc._id}/${ufdoc._id}`, file);
}
namelist.push(ufdoc.filename);
currentUser.itemDone(`${ufdoc._id}: ${ufdoc.filename || ''}`);
}
currentUser.stop();
userfileProgress.itemDone(`${udoc._id}: ${udoc.uname}`);
}
userfileProgress.stop();
return true;
} catch (e) {
logger.error(e);
return false;
domainProgress.stop();
}
const udocs = await user.getMulti().project({ _id: 1, uname: 1 }).toArray();
const userfileProgress = terminal.progressBar({
items: udocs.length, title: 'Users', y: 3,
});
for (const udoc of udocs) {
userfileProgress.startItem(`${udoc._id}: ${udoc.uname}`);
const ufdocs = await coll.find({ owner: udoc._id }).toArray();
const currentUser = terminal.progressBar({
items: ufdocs.length, title: `${udoc._id}: ${udoc.uname}`, y: 4, eta: true,
});
const namelist = [];
for (const ufdoc of ufdocs) {
currentUser.startItem(`${ufdoc._id}: ${ufdoc.filename || ''}`);
ufdoc.filename = ufdoc.filename || ufdoc._id.toHexString();
const file = await streamToBuffer(gridfs.openDownloadStream(ufdoc._id));
if (!namelist.includes(ufdoc.filename)) {
await storage.put(`user/${udoc._id}/${ufdoc.filename}`, file);
} else {
await storage.put(`user/${udoc._id}/${ufdoc._id}`, file);
}
namelist.push(ufdoc.filename);
currentUser.itemDone(`${ufdoc._id}: ${ufdoc.filename || ''}`);
}
currentUser.stop();
userfileProgress.itemDone(`${udoc._id}: ${udoc.uname}`);
}
userfileProgress.stop();
logger.success('Files copied successfully. You can now remove collection `file` `fs.files` `fs.chunks` in the database.');
return true;
},
];
packages/import-qduoj/handler.ts
+28 -44
View File
@@ -4,25 +4,20 @@ import os from 'os';
import fs from 'fs-extra';
import AdmZip from 'adm-zip';
import yaml from 'js-yaml';
import { ObjectID } from 'mongodb';
import { ContentNode, LocalProblemConfig } from 'hydrooj';
import {
Route, Handler, param, Types,
} from 'hydrooj/dist/service/server';
import { BadRequestError } from 'hydrooj/dist/error';
import { streamToBuffer } from 'hydrooj/dist/utils';
import { Route, Handler } from 'hydrooj/dist/service/server';
import * as storage from 'hydrooj/dist/service/storage';
import { BadRequestError, ValidationError } from 'hydrooj/dist/error';
import { ProblemAdd } from 'hydrooj/dist/lib/ui';
import * as file from 'hydrooj/dist/model/file';
import * as problem from 'hydrooj/dist/model/problem';
import { PERM } from 'hydrooj/dist/model/builtin';
const processing = {};
fs.ensureDirSync('/tmp/hydro/import-qduoj');
class ImportQduojHandler extends Handler {
async ImportFromFile(domainId: string, id: string, zipfile: Buffer) {
async ImportFromFile(domainId: string, zipfile: string) {
const zip = new AdmZip(zipfile);
const tmp = path.resolve(os.tmpdir(), 'hydro', 'import-qduoj', id);
const tmp = path.resolve(os.tmpdir(), 'hydro', 'import-qduoj', String.random(32));
await new Promise((resolve, reject) => {
zip.extractAllToAsync(tmp, true, (err) => (err ? resolve() : reject(err)));
});
@@ -79,15 +74,20 @@ class ImportQduojHandler extends Handler {
});
}
const pid = await problem.add(domainId, pdoc.display_id, pdoc.title, content, this.user._id, pdoc.tags);
const testdata = new AdmZip();
const config: LocalProblemConfig = {
time: `${pdoc.time_limit}ms`,
memory: `${pdoc.memory_limit}m`,
subtasks: [],
};
for (const tc of pdoc.test_case_score) {
testdata.addLocalFile(path.join(tmp, 'testcase', tc.input_name));
testdata.addLocalFile(path.join(tmp, 'testcase', tc.output_name));
await storage.put(
`problem/${domainId}/${pid}/testdata/${tc.input_name}`,
path.join(tmp, 'testcase', tc.input_name),
);
await storage.put(
`problem/${domainId}/${pid}/testdata/${tc.output_name}`,
path.join(tmp, 'testcase', tc.output_name),
);
config.subtasks.push({
score: tc.score,
cases: [{
@@ -96,43 +96,27 @@ class ImportQduojHandler extends Handler {
}],
});
}
testdata.addFile('config.yaml', Buffer.from(yaml.safeDump(config)));
const f = path.resolve(os.tmpdir(), 'hydro', `${Math.random()}.zip`);
await new Promise((resolve, reject) => {
testdata.writeZip(f, (err) => (err ? resolve() : reject(err)));
});
try {
await problem.setTestdata(domainId, pid, f);
await problem.edit(domainId, pid, { html: true });
} finally {
await fs.unlink(f);
}
await storage.put(
`problem/${domainId}/${pid}/testdata/config.yaml`,
Buffer.from(yaml.safeDump(config)),
);
await problem.edit(domainId, pid, { html: true });
}
} finally {
await fs.remove(tmp);
}
}
@param('ufid', Types.ObjectID, true)
async get(domainId: string, ufid?: ObjectID) {
if (ufid) {
if (processing[ufid.toHexString()]) {
this.response.body = 'Processing';
return;
}
const stat = await file.getMeta(ufid);
if (stat.size > 128 * 1024 * 1024) throw new BadRequestError('File too large');
const stream = await file.get(ufid);
processing[ufid.toHexString()] = true;
try {
const buf = await streamToBuffer(stream);
await this.ImportFromFile(domainId, ufid.toHexString(), buf);
await file.del(ufid);
} catch (e) {
processing[ufid.toHexString()] = false;
throw e;
}
} else this.response.template = 'problem_import_qduoj.html';
async get() {
this.response.template = 'problem_import.html';
}
async post({ domainId }) {
if (!this.request.files.file) throw new ValidationError('file');
const stat = await fs.stat(this.request.files.file.path);
if (stat.size > 128 * 1024 * 1024) throw new BadRequestError('File too large');
await this.ImportFromFile(domainId, this.request.files.file.path);
this.response.redirect = this.url('problem_main');
}
}