From ee0f5deedcfb7fbaff7579a1a0380928858223b6 Mon Sep 17 00:00:00 2001 From: undefined Date: Sat, 7 Nov 2020 23:47:35 +0800 Subject: [PATCH] =?UTF-8?q?core:=20=E6=94=AF=E6=8C=81=E9=A2=98=E7=9B=AE?= =?UTF-8?q?=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 8 ++-- packages/hydrojudge/package.json | 4 +- packages/hydrooj/package.json | 16 ++++---- packages/hydrooj/src/handler/problem.ts | 47 ++++++++++++++++++++-- packages/hydrooj/src/lib/testdataConfig.ts | 4 +- packages/hydrooj/src/model/file.ts | 10 +++-- packages/hydrooj/src/model/problem.ts | 6 +-- packages/hydrooj/src/service/server.ts | 9 +++-- packages/hydrooj/src/utils.ts | 8 ++++ packages/login-with-qq/package.json | 4 +- packages/migrate-vijos/package.json | 4 +- 11 files changed, 87 insertions(+), 33 deletions(-) diff --git a/package.json b/package.json index 826b2c556..5b0d49069 100644 --- a/package.json +++ b/package.json @@ -20,17 +20,17 @@ "@shelf/jest-mongodb": "^1.2.3", "@types/cross-spawn": "^6.0.2", "@types/jest": "^26.0.14", - "@types/node": "^14.14.5", + "@types/node": "^14.14.6", "@types/semver": "^7.3.4", "@typescript-eslint/eslint-plugin": "^3.9.1", "@typescript-eslint/parser": "^3.9.1", "cross-spawn": "^7.0.3", - "eslint": "^7.12.1", - "eslint-config-airbnb-base": "^14.2.0", + "eslint": "^7.13.0", + "eslint-config-airbnb-base": "^14.2.1", "eslint-import-resolver-typescript": "^2.3.0", "eslint-plugin-import": "^2.22.1", "globby": "^11.0.1", - "jest": "^26.6.1", + "jest": "^26.6.3", "latest-version": "^5.1.0", "ora": "^5.1.0", "semver": "^7.3.2", diff --git a/packages/hydrojudge/package.json b/packages/hydrojudge/package.json index 69dce590e..31d2666b5 100644 --- a/packages/hydrojudge/package.json +++ b/packages/hydrojudge/package.json @@ -15,7 +15,7 @@ "p-queue": "^6.6.1", "shell-quote": "^1.7.2", "supports-color": "^7.2.0", - "systeminformation": "^4.27.11", + "systeminformation": "^4.28.1", "yargs": "^16.1.0" }, "peerDependencies": { @@ -24,7 +24,7 @@ "license": "AGPL-3.0-only", "devDependencies": { "@types/bson": "^4.0.2", - "@types/fs-extra": "^9.0.1", + "@types/fs-extra": "^9.0.3", "@types/js-yaml": "^3.12.5", "@types/shell-quote": "^1.7.0" }, diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index c177bd5f9..e034bdbe0 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "2.13.40", + "version": "2.13.41", "bin": "bin/hydrooj.js", "main": "dist/loader.js", "typings": "dist/loader.d.ts", @@ -19,31 +19,31 @@ "js-yaml": "^3.13.1", "koa": "^2.13.0", "koa-body": "^4.2.0", - "koa-router": "^9.4.0", + "koa-router": "^10.0.0", "koa-static-cache": "^5.1.3", "lodash": "^4.17.20", "lru-cache": "^6.0.0", "moment-timezone": "^0.5.31", - "mongodb": "^3.6.2", - "nodemailer": "^6.4.13", + "mongodb": "^3.6.3", + "nodemailer": "^6.4.15", "ps-list": "^7.2.0", "serialize-javascript": "^5.0.1", "sockjs": "^0.3.20", "superagent": "^6.1.0", "supports-color": "^7.2.0", - "systeminformation": "^4.27.11", + "systeminformation": "^4.28.1", "yargs": "^16.0.3" }, "devDependencies": { "@types/adm-zip": "^0.4.33", - "@types/fs-extra": "^9.0.1", + "@types/fs-extra": "^9.0.3", "@types/js-yaml": "^3.12.5", "@types/koa": "^2.11.4", "@types/koa-router": "^7.4.1", "@types/koa-static-cache": "^5.1.0", - "@types/lodash": "^4.14.161", + "@types/lodash": "^4.14.165", "@types/lru-cache": "^5.1.0", - "@types/mongodb": "^3.5.27", + "@types/mongodb": "^3.5.33", "@types/nodemailer": "^6.4.0", "@types/serialize-javascript": "^4.0.0", "@types/sockjs": "^0.3.32", diff --git a/packages/hydrooj/src/handler/problem.ts b/packages/hydrooj/src/handler/problem.ts index 88dc8930a..5b100f755 100644 --- a/packages/hydrooj/src/handler/problem.ts +++ b/packages/hydrooj/src/handler/problem.ts @@ -1,11 +1,13 @@ -import { isSafeInteger, flatten } from 'lodash'; +import { isSafeInteger, flatten, pick } from 'lodash'; import yaml from 'js-yaml'; import { FilterQuery, ObjectID } from 'mongodb'; +import AdmZip from 'adm-zip'; import { NoProblemError, ProblemDataNotFoundError, BadRequestError, SolutionNotFoundError, ProblemNotFoundError, ValidationError, PermissionError, } from '../error'; +import { streamToBuffer } from '../utils'; import { Pdoc, User, Rdoc, PathComponent, } from '../interface'; @@ -13,6 +15,7 @@ import paginate from '../lib/paginate'; import { checkPid, isTitle, isContent, isPid, } from '../lib/validator'; +import { ProblemAdd } from '../lib/ui'; import * as file from '../model/file'; import * as problem from '../model/problem'; import * as record from '../model/record'; @@ -233,6 +236,23 @@ export class ProblemDetailHandler extends ProblemHandler { } } +export class ProblemExportHandler extends ProblemDetailHandler { + async get() { + const hasPerm = (this.user._id === this.pdoc.owner && this.user.hasPerm(PERM.PERM_READ_PROBLEM_DATA_SELF)) + || this.user.hasPerm(PERM.PERM_READ_PROBLEM_DATA_SELF); + const pdoc = pick(this.pdoc, ['pid', 'acMsg', 'content', 'config', 'title', 'html', 'tag', 'category']); + let zip: AdmZip; + if (hasPerm) { + if (this.pdoc.data instanceof ObjectID) { + const buf = await streamToBuffer(await file.get(this.pdoc.data)); + zip = new AdmZip(buf); + } else zip = new AdmZip(); + } else zip = new AdmZip(); + zip.addFile('problem.json', Buffer.from(JSON.stringify(pdoc))); + this.response.attachment(`${this.pdoc.title}.zip`, zip.toBuffer()); + } +} + export class ProblemSubmitHandler extends ProblemDetailHandler { @param('pid', Types.String, null, parsePid) async get(domainId: string, pid: string | number) { @@ -615,7 +635,6 @@ export class ProblemSolutionReplyRawHandler extends ProblemDetailHandler { export class ProblemCreateHandler extends Handler { async get() { this.response.template = 'problem_edit.html'; - this.checkPerm(PERM.PERM_CREATE_PROBLEM); this.response.body = { path: [ ['Hydro', 'homepage'], @@ -640,11 +659,32 @@ export class ProblemCreateHandler extends Handler { } } +export class ProblemImportHandler extends Handler { + @param('ufid', Types.ObjectID, true) + async get(domainId: string, ufid?: ObjectID) { + if (ufid) { + const stat = await file.getMeta(ufid); + if (stat.size > 128 * 1024 * 1024) throw new BadRequestError('File too large'); + const stream = await file.get(ufid); + const buf = await streamToBuffer(stream); + const zip = new AdmZip(buf); + const pdoc = JSON.parse(zip.getEntry('problem.json').getData().toString()); + const pid = await problem.add(domainId, pdoc.pid, pdoc.title, pdoc.content, this.user._id, pdoc.tags, pdoc.category); + await problem.setTestdata(domainId, pid, buf); + await problem.edit(domainId, pid, { html: pdoc.html }); + this.response.redirect = this.url('problem_detail', { pid }); + await file.del(ufid); + } else this.response.template = 'problem_import.html'; + } +} + export async function apply() { + ProblemAdd('problem_import', {}, 'copy', 'Import From Hydro'); Route('problem_main', '/p', ProblemMainHandler); Route('problem_category', '/p/category/:category', ProblemCategoryHandler); Route('problem_random', '/problem/random', ProblemRandomHandler); Route('problem_detail', '/p/:pid', ProblemDetailHandler); + Route('problem_export', '/p/:pid/export', ProblemExportHandler); Route('problem_submit', '/p/:pid/submit', ProblemSubmitHandler, PERM.PERM_SUBMIT_PROBLEM); Route('problem_pretest', '/p/:pid/pretest', ProblemPretestHandler); Route('problem_settings', '/p/:pid/settings', ProblemSettingsHandler); @@ -655,7 +695,8 @@ export async function apply() { Route('problem_solution', '/p/:pid/solution', ProblemSolutionHandler); Route('problem_solution_raw', '/p/:pid/solution/:psid/raw', ProblemSolutionRawHandler); Route('problem_solution_reply_raw', '/p/:pid/solution/:psid/:psrid/raw', ProblemSolutionReplyRawHandler); - Route('problem_create', '/problem/create', ProblemCreateHandler); + Route('problem_create', '/problem/create', ProblemCreateHandler, PERM.PERM_CREATE_PROBLEM); + Route('problem_import', '/problem/import', ProblemImportHandler, PERM.PERM_CREATE_PROBLEM); Connection('problem_pretest_conn', '/conn/pretest', ProblemPretestConnectionHandler); } diff --git a/packages/hydrooj/src/lib/testdataConfig.ts b/packages/hydrooj/src/lib/testdataConfig.ts index 5ab43945b..7046cd79f 100644 --- a/packages/hydrooj/src/lib/testdataConfig.ts +++ b/packages/hydrooj/src/lib/testdataConfig.ts @@ -2,8 +2,8 @@ import Zip from 'adm-zip'; import { safeLoad } from 'js-yaml'; import { LocalProblemConfig } from '../interface'; -export async function readConfig(filePath: string) { - const data = new Zip(filePath); +export async function readConfig(file: string | Buffer) { + const data = new Zip(file); const entries = data.getEntries(); for (const entry of entries) { if (entry.name.toLowerCase() === 'config.yaml') { diff --git a/packages/hydrooj/src/model/file.ts b/packages/hydrooj/src/model/file.ts index 5ab8eee1f..cfa514aec 100644 --- a/packages/hydrooj/src/model/file.ts +++ b/packages/hydrooj/src/model/file.ts @@ -7,6 +7,7 @@ import * as db from '../service/db'; import * as bus from '../service/bus'; import gridfs from '../service/gridfs'; import hash from '../lib/hash.hydro'; +import { bufferToStream } from '../utils'; const coll: Collection = db.collection('file'); const collFile = db.collection('fs.files'); @@ -17,12 +18,13 @@ function _timestamp() { } export async function add( - streamOrPath: fs.ReadStream | string, filename: string, + f: NodeJS.ReadableStream | Buffer | string, filename: string, owner = 1, meta = {}, ) { - let stream: fs.ReadStream; - if (typeof streamOrPath === 'string') stream = fs.createReadStream(streamOrPath); - else stream = streamOrPath; + let stream: NodeJS.ReadableStream; + if (typeof f === 'string') stream = fs.createReadStream(f); + else if (f instanceof Buffer) stream = bufferToStream(f); + else stream = f; const w = gridfs.openUploadStream(filename); await coll.insertOne({ ...meta, diff --git a/packages/hydrooj/src/model/problem.ts b/packages/hydrooj/src/model/problem.ts index a6f91ebe9..f40e59528 100644 --- a/packages/hydrooj/src/model/problem.ts +++ b/packages/hydrooj/src/model/problem.ts @@ -187,10 +187,10 @@ export function setStar(domainId: string, pid: number, uid: number, star: boolea return document.setStatus(domainId, document.TYPE_PROBLEM, pid, uid, { star }); } -export async function setTestdata(domainId: string, _id: number, filePath: string) { +export async function setTestdata(domainId: string, _id: number, f: string | Buffer) { const pdoc = await get(domainId, _id); - const config = await testdataConfig.readConfig(filePath); - const id = await file.add(filePath, 'data.zip', 1); + const config = await testdataConfig.readConfig(f); + const id = await file.add(f, 'data.zip', 1); if (pdoc.data instanceof ObjectID) file.del(pdoc.data); return await edit(domainId, _id, { data: id, config: safeLoad(config) as any }); } diff --git a/packages/hydrooj/src/service/server.ts b/packages/hydrooj/src/service/server.ts index dc16f9a06..e7d59dd32 100644 --- a/packages/hydrooj/src/service/server.ts +++ b/packages/hydrooj/src/service/server.ts @@ -357,11 +357,14 @@ export class Handler { status: null, template: null, redirect: null, - attachment: (name, stream) => { + attachment: (name, streamOrBuffer) => { ctx.attachment(name); - if (stream) { + if (streamOrBuffer instanceof Buffer) { this.response.body = null; - ctx.body = stream.pipe(new PassThrough()); + ctx.body = streamOrBuffer; + } else { + this.response.body = null; + ctx.body = streamOrBuffer.pipe(new PassThrough()); } }, disposition: null, diff --git a/packages/hydrooj/src/utils.ts b/packages/hydrooj/src/utils.ts index 8f7053a44..311ce5bad 100644 --- a/packages/hydrooj/src/utils.ts +++ b/packages/hydrooj/src/utils.ts @@ -1,4 +1,5 @@ import fs from 'fs'; +import { Duplex } from 'stream'; import cluster from 'cluster'; import path from 'path'; import serialize from 'serialize-javascript'; @@ -180,6 +181,13 @@ export function streamToBuffer(stream): Promise { }); } +export function bufferToStream(buffer: Buffer): NodeJS.ReadableStream { + const stream = new Duplex(); + stream.push(buffer); + stream.push(null); + return stream; +} + export function buildProjection(fields: string[]): Record { const o = {}; for (const k of fields) o[k] = 1; diff --git a/packages/login-with-qq/package.json b/packages/login-with-qq/package.json index 85d554aa5..1e3dbaddb 100644 --- a/packages/login-with-qq/package.json +++ b/packages/login-with-qq/package.json @@ -16,6 +16,6 @@ "ws": "^7.3.1" }, "devDependencies": { - "@types/ws": "^7.2.8" + "@types/ws": "^7.2.9" } -} +} \ No newline at end of file diff --git a/packages/migrate-vijos/package.json b/packages/migrate-vijos/package.json index fd270bfbc..060d698cb 100644 --- a/packages/migrate-vijos/package.json +++ b/packages/migrate-vijos/package.json @@ -11,13 +11,13 @@ }, "devDependencies": { "@types/js-yaml": "^3.12.5", - "@types/mongodb": "^3.5.27" + "@types/mongodb": "^3.5.33" }, "peerDependencies": { "hydrooj": "^2.13.25" }, "dependencies": { "js-yaml": "^3.14.0", - "mongodb": "^3.6.2" + "mongodb": "^3.6.3" } } \ No newline at end of file