Weishen/Hydro
2
0 Fork 0
mirror of https://gitcode.com/gh_mirrors/hy/Hydro.git synced 2026-10-20 01:38:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

core: logger upd

Browse Source
This commit is contained in:
undefined committed 2020-12-21 03:06:41 +08:00
1 parent d4088f429a
commit 4d031eb9ab
9 files changed
+141 -131
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/hydrooj/package.json
+2 -1
View File
@@ -27,6 +27,7 @@
"moment-timezone": "^0.5.32",
"mongodb": "^3.6.3",
"nodemailer": "^6.4.17",
"p-queue": "^6.6.2",
"serialize-javascript": "^5.0.1",
"sockjs": "^0.3.20",
"superagent": "^6.1.0",
@@ -58,4 +59,4 @@
"build": "tsc",
"build:watch": "tsc --watch"
}
}
}
packages/hydrooj/src/handler/problem.ts
+5 -4
View File
@@ -453,12 +453,13 @@ export class ProblemFilesHandler extends ProblemDetailHandler {
@param('testdata', Types.Boolean)
@param('additional_file', Types.Boolean)
async get(domainId: string, getTestdata = true, getAdditionalFile = true) {
const canReadData = this.user._id === this.pdoc.owner || this.user.hasPerm(PERM.PERM_READ_PROBLEM_DATA);
const [testdata, additional_file] = await Promise.all([
getTestdata
? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/testdata`, true)
getTestdata && canReadData
? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/testdata/`, true)
: [],
getAdditionalFile
? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file`, true)
? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/`, true)
: [],
]);
this.response.body.testdata = testdata;
@@ -671,7 +672,7 @@ export class ProblemImportHandler extends Handler {
this.response.template = 'problem_import.html';
}
async post(domainId: string) {
async post({ domainId }) {
if (!this.request.files.file) throw new ValidationError('file');
const stat = await fs.stat(this.request.files.file.path);
if (stat.size > 128 * 1024 * 1024) throw new BadRequestError('File too large');
packages/hydrooj/src/model/builtin.ts
+6
View File
@@ -18,11 +18,17 @@ export const PERM = {
// Problem and Record
PERM_CREATE_PROBLEM: 1n << 4n,
PERM_EDIT_PROBLEM: 1n << 5n,
/**
* @deprecated
*/
PERM_EDIT_PROBLEM_SELF: 1n << 6n,
PERM_VIEW_PROBLEM: 1n << 7n,
PERM_VIEW_PROBLEM_HIDDEN: 1n << 8n,
PERM_SUBMIT_PROBLEM: 1n << 9n,
PERM_READ_PROBLEM_DATA: 1n << 10n,
/**
* @deprecated
*/
PERM_READ_PROBLEM_DATA_SELF: 1n << 11n,
PERM_READ_RECORD_CODE: 1n << 12n,
PERM_REJUDGE_PROBLEM: 1n << 13n,
packages/hydrooj/src/service/server.ts
+5 -11
View File
@@ -520,7 +520,7 @@ export class Handler {
try {
await this.render(error instanceof UserFacingError ? 'error.html' : 'bsod.html', { error });
} catch (e) {
console.error(e);
logger.error(e);
// this.response.body.error = {};
}
}
@@ -615,8 +615,8 @@ export class Handler {
async onerror(error: HydroError) {
if (!error.msg) error.msg = () => error.message;
if (!(error instanceof NotFoundError)) {
console.error(error.msg(), error.params);
console.error(error.stack);
logger.error(error.msg(), error.params);
logger.error(error.stack);
}
this.response.status = error instanceof UserFacingError ? error.code : 500;
this.response.template = error instanceof UserFacingError ? 'error.html' : 'bsod.html';
@@ -807,7 +807,7 @@ export class ConnectionHandler {
async message(message: any) { } // eslint-disable-line
onerror(err: HydroError) {
if (!(err instanceof NotFoundError)) console.error(err);
if (!(err instanceof NotFoundError)) logger.error(err);
this.send({
error: {
name: err.name,
@@ -895,13 +895,7 @@ export function start() {
if (ctx.response.headers.nolog) return;
ctx._remoteAddress = ctx.request.ip;
const status = ctx.response.status;
const color = status >= 500 ? 31 // red
: status >= 400 ? 33 // yellow
: status >= 300 ? 36 // cyan
: status >= 200 ? 32 // green
: 0; // no color
logger.debug(`\
\x1b[0m${ctx.request.method} ${ctx.request.path} \x1b[${color}m${ctx.response.status}\x1b[0m ${endTime - startTime}ms ${ctx.response.length}\x1b[0m`);
logger.debug(`${ctx.request.method} ${ctx.request.path} ${ctx.response.status} ${endTime - startTime}ms ${ctx.response.length}`);
});
}
app.use(async (ctx, next) => {
packages/hydrooj/src/ui.ts
+15
View File
@@ -51,7 +51,11 @@ if (cluster.isMaster) {
bus.on('message/log', (message) => {
LogBox.appendLog(message);
});
const history = [''];
let current = 0;
ShellInput.on('submit', async (input) => {
history.push(input);
current = history.length;
ShellInput.input.setContent(' '.repeat(terminal.width), false, false);
ShellInput.input.setContent('', false, false);
if (input[0] === '@') {
@@ -61,6 +65,17 @@ if (cluster.isMaster) {
}
} else bus.parallel('message/run', input);
});
ShellInput.on('key', (key) => {
if (key === 'UP') {
current--;
ShellInput.input.setContent(' '.repeat(terminal.width), false, false);
ShellInput.input.setContent(history[current] || '', false, false);
} else if (key === 'DOWN') {
current++;
ShellInput.input.setContent(' '.repeat(terminal.width), false, false);
ShellInput.input.setContent(history[current] || '', false, false);
}
});
terminal.on('key', (key, matches, data) => {
if (key === 'CTRL_C') terminate();
});
packages/hydrooj/src/upgrade.ts
+71 -70
View File
@@ -1,7 +1,9 @@
/* eslint-disable no-await-in-loop */
import { ObjectID, Collection } from 'mongodb';
// @ts-ignore
import { terminal } from 'terminal-kit';
import AdmZip from 'adm-zip';
import Queue from 'p-queue';
import { Ufdoc } from './interface';
import { Logger } from './logger';
import { streamToBuffer } from './utils';
@@ -60,83 +62,82 @@ const scripts: UpgradeScript[] = [
return false;
}
const coll: Collection<Ufdoc> = db.collection('file');
const collFile = db.collection('fs.files');
const collChunk = db.collection('fs.chunks');
function del(_id: ObjectID) {
return Promise.all([
coll.deleteOne({ _id }),
collFile.deleteOne({ _id }),
collChunk.deleteMany({ files_id: _id }),
]);
}
logger.info('Changing storage engine. This may take a long time.');
// Problem file and User file
try {
let savedProgress = system.get('upgrade.file.progress.domain');
if (savedProgress) savedProgress = JSON.parse(savedProgress);
else savedProgress = { pdocs: [] };
const ddocs = await domain.getMulti().project({ _id: 1 }).toArray();
logger.info('Total found %d domains.', ddocs.length);
for (let i = 0; i < ddocs.length; i++) {
const ddoc = ddocs[i];
logger.info('Domain %s (%d/%d)', ddoc._id, i + 1, ddocs.length);
const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray();
const domainProgress = terminal.progressBar({ items: pdocs.length, title: 'Problems', inline: true });
for (let j = 0; j < pdocs.length; j++) {
const pdoc = pdocs[j];
domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`);
if (!savedProgress.pdocs.includes(`${pdoc.domainId}/${pdoc.docId}`) && pdoc.data instanceof ObjectID) {
try {
const file = await streamToBuffer(gridfs.openDownloadStream(pdoc.data));
const zip = new AdmZip(file);
const entries = zip.getEntries();
await Promise.all(entries.map(
(entry) => storage.put(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/${entry.name}`, entry.getData()),
));
} catch (e) {
if (e.toString().includes('FileNotFound')) {
logger.error('Problem data not found %s/%s', pdoc.domainId, pdoc.docId);
} else throw e;
let savedProgress = system.get('upgrade.file.progress.domain');
if (savedProgress) savedProgress = JSON.parse(savedProgress);
else savedProgress = { pdocs: [] };
const ddocs = await domain.getMulti().project({ _id: 1 }).toArray();
logger.info('Total found %d domains.', ddocs.length);
for (let i = 0; i < ddocs.length; i++) {
const ddoc = ddocs[i];
logger.info('Domain %s (%d/%d)', ddoc._id, i + 1, ddocs.length);
const pdocs = await problem.getMulti(ddoc._id, { data: { $ne: null } }, ['domainId', 'docId', 'data', 'title']).toArray();
const domainProgress = terminal.progressBar({ items: pdocs.length, title: 'Problems', inline: true });
for (let j = 0; j < pdocs.length; j++) {
const pdoc = pdocs[j];
domainProgress.startItem(`${pdoc.docId}: ${pdoc.title}`);
if (!savedProgress.pdocs.includes(`${pdoc.domainId}/${pdoc.docId}`) && pdoc.data instanceof ObjectID) {
try {
const [file, current] = await Promise.all([
streamToBuffer(gridfs.openDownloadStream(pdoc.data)),
storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`, true),
]);
const zip = new AdmZip(file);
const entries = zip.getEntries();
if (entries.map((entry) => entry.entryName).sort().join('?') !== current.map((f) => f.name).sort().join('?')) {
await storage.del(current.map((entry) => entry.prefix + entry.name));
const queue = new Queue({ concurrency: 5 });
const tasks = [];
for (const entry of entries) {
tasks.push(queue.add(() => storage.put(
`problem/${pdoc.domainId}/${pdoc.docId}/testdata/${entry.entryName}`,
entry.getData(),
)));
}
await Promise.all(tasks);
}
savedProgress.pdocs.push(`${pdoc.domainId}/${pdoc.docId}`);
} catch (e) {
if (e.toString().includes('FileNotFound')) {
logger.error('Problem data not found %s/%s', pdoc.domainId, pdoc.docId);
} else throw e;
}
system.set('upgrade.file.progress.domain', JSON.stringify(savedProgress));
domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`);
savedProgress.pdocs.push(`${pdoc.domainId}/${pdoc.docId}`);
}
domainProgress.stop();
system.set('upgrade.file.progress.domain', JSON.stringify(savedProgress));
domainProgress.itemDone(`${pdoc.docId}: ${pdoc.title}`);
}
const udocs = await user.getMulti().project({ _id: 1, uname: 1 }).toArray();
const userfileProgress = terminal.progressBar({
items: udocs.length, title: 'Users', y: 3,
});
for (const udoc of udocs) {
userfileProgress.startItem(`${udoc._id}: ${udoc.uname}`);
const ufdocs = await coll.find({ owner: udoc._id }).toArray();
const currentUser = terminal.progressBar({
items: ufdocs.length, title: `${udoc._id}: ${udoc.uname}`, y: 4, eta: true,
});
const namelist = [];
for (const ufdoc of ufdocs) {
currentUser.startItem(`${ufdoc._id}: ${ufdoc.filename || ''}`);
ufdoc.filename = ufdoc.filename || ufdoc._id.toHexString();
const file = await streamToBuffer(gridfs.openDownloadStream(ufdoc._id));
if (!namelist.includes(ufdoc.filename)) {
await storage.put(`user/${udoc._id}/${ufdoc.filename}`, file);
} else {
await storage.put(`user/${udoc._id}/${ufdoc._id}`, file);
}
namelist.push(ufdoc.filename);
currentUser.itemDone(`${ufdoc._id}: ${ufdoc.filename || ''}`);
}
currentUser.stop();
userfileProgress.itemDone(`${udoc._id}: ${udoc.uname}`);
}
userfileProgress.stop();
return true;
} catch (e) {
logger.error(e);
return false;
domainProgress.stop();
}
const udocs = await user.getMulti().project({ _id: 1, uname: 1 }).toArray();
const userfileProgress = terminal.progressBar({
items: udocs.length, title: 'Users', y: 3,
});
for (const udoc of udocs) {
userfileProgress.startItem(`${udoc._id}: ${udoc.uname}`);
const ufdocs = await coll.find({ owner: udoc._id }).toArray();
const currentUser = terminal.progressBar({
items: ufdocs.length, title: `${udoc._id}: ${udoc.uname}`, y: 4, eta: true,
});
const namelist = [];
for (const ufdoc of ufdocs) {
currentUser.startItem(`${ufdoc._id}: ${ufdoc.filename || ''}`);
ufdoc.filename = ufdoc.filename || ufdoc._id.toHexString();
const file = await streamToBuffer(gridfs.openDownloadStream(ufdoc._id));
if (!namelist.includes(ufdoc.filename)) {
await storage.put(`user/${udoc._id}/${ufdoc.filename}`, file);
} else {
await storage.put(`user/${udoc._id}/${ufdoc._id}`, file);
}
namelist.push(ufdoc.filename);
currentUser.itemDone(`${ufdoc._id}: ${ufdoc.filename || ''}`);
}
currentUser.stop();
userfileProgress.itemDone(`${udoc._id}: ${udoc.uname}`);
}
userfileProgress.stop();
logger.success('Files copied successfully. You can now remove collection `file` `fs.files` `fs.chunks` in the database.');
return true;
},
];