From 1bf6fcd423a4ca16500eec9e0734e84a8510c42e Mon Sep 17 00:00:00 2001 From: undefined Date: Wed, 23 Dec 2020 01:33:55 +0800 Subject: [PATCH] core: remove problem_copy --- packages/fps-importer/handler.ts | 11 ++-- packages/hydrooj/package.json | 2 +- packages/hydrooj/src/handler/import.syzoj.ts | 28 +++++++--- packages/hydrooj/src/handler/problem.ts | 56 +++++++------------- packages/hydrooj/src/interface.ts | 12 ++--- packages/hydrooj/src/model/document.ts | 21 ++++++-- packages/hydrooj/src/model/problem.ts | 36 +++++++++++-- packages/hydrooj/src/upgrade.ts | 13 +++++ packages/import-qduoj/handler.ts | 3 +- packages/migrate-hustoj/script.ts | 2 +- 10 files changed, 114 insertions(+), 70 deletions(-) diff --git a/packages/fps-importer/handler.ts b/packages/fps-importer/handler.ts index 129a991e4..81df7d527 100644 --- a/packages/fps-importer/handler.ts +++ b/packages/fps-importer/handler.ts @@ -8,7 +8,6 @@ import { LocalProblemConfig } from 'hydrooj'; import { Route, Handler, param, Types, } from 'hydrooj/dist/service/server'; -import * as storage from 'hydrooj/dist/service/storage'; import { buildContent } from 'hydrooj/dist/lib/content'; import { ProblemAdd } from 'hydrooj/dist/lib/ui'; import * as solution from 'hydrooj/dist/model/solution'; @@ -39,16 +38,16 @@ class FpsProblemImportHandler extends Handler { const title = decodeHTML(p.title.join(' ')); const tags = filter(p.source, (i: string) => i.trim()); const pid = await problem.add(domainId, null, title, content, this.user._id, tags, []); - await storage.put(`problem/${domainId}/${pid}/testdata/config.yaml`, Buffer.from(`time: ${config.time}\nmemory: ${config.memory}`)); + await problem.addTestdata(domainId, pid, 'config.yaml', Buffer.from(`time: ${config.time}\nmemory: ${config.memory}`)); if (p.test_output) { for (let i = 0; i < p.test_input.length; i++) { - await storage.put(`problem/${domainId}/${pid}/testdata/${i + 1}.in`, Buffer.from(p.test_input[i])); - await storage.put(`problem/${domainId}/${pid}/testdata/${i + 1}.out`, Buffer.from(p.test_output[i])); + await problem.addTestdata(domainId, pid, `${i + 1}.in`, Buffer.from(p.test_input[i])); + await problem.addTestdata(domainId, pid, `${i + 1}.out`, Buffer.from(p.test_output[i])); } } else if (p.test_input) { for (let i = 0; i < p.test_input.length / 2; i++) { - await storage.put(`problem/${domainId}/${pid}/testdata/${i + 1}.in`, Buffer.from(p.test_input[2 * i])); - await storage.put(`problem/${domainId}/${pid}/testdata/${i + 1}.out`, Buffer.from(p.test_input[2 * i + 1])); + await problem.addTestdata(domainId, pid, `${i + 1}.in`, Buffer.from(p.test_input[2 * i])); + await problem.addTestdata(domainId, pid, `${i + 1}.out`, Buffer.from(p.test_input[2 * i + 1])); } } await problem.edit(domainId, pid, { html: true }); diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index f03d3ef9e..2f8a46087 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "2.15.3", + "version": "2.15.4", "bin": "bin/hydrooj.js", "main": "dist/loader.js", "typings": "dist/loader.d.ts", diff --git a/packages/hydrooj/src/handler/import.syzoj.ts b/packages/hydrooj/src/handler/import.syzoj.ts index 4531d231b..333f8f854 100644 --- a/packages/hydrooj/src/handler/import.syzoj.ts +++ b/packages/hydrooj/src/handler/import.syzoj.ts @@ -75,8 +75,8 @@ class ProblemImportSYZOJHandler extends Handler { ...result.body.localizedContentsOfAllLocales, ][0].title; const docId = await problem.add( - domainId, targetPid || `${host}_${pid}`, title, content, this.user._id, - tags || [], [], null, hidden, + domainId, targetPid, title, content, this.user._id, + tags || [], [], hidden, ); (async () => { const judge = result.body.judgeInfo; @@ -93,9 +93,8 @@ class ProblemImportSYZOJHandler extends Handler { const p = new PassThrough(); superagent.get(urls[f.filename]).pipe(p); // eslint-disable-next-line no-await-in-loop - await storage.put(`problem/${domainId}/${docId}/testdata/${f.filename}`, p); + await problem.addTestdata(domainId, docId, f.filename, p); } - // TODO additional_file if (judge) { await problem.edit(domainId, docId, { config: { @@ -105,6 +104,21 @@ class ProblemImportSYZOJHandler extends Handler { }, }); } + const a = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/downloadProblemFiles`) + .send({ + problemId: +pid, + type: 'AdditionalFile', + filenameList: result.body.additionalFiles.map((node) => node.filename), + }); + const aurls = {}; + if (a.body.error) return; + for (const t of a.body.downloadInfo) aurls[t.filename] = t.downloadUrl; + for (const f of result.body.additionalFiles) { + const p = new PassThrough(); + superagent.get(aurls[f.filename]).pipe(p); + // eslint-disable-next-line no-await-in-loop + await storage.put(`problem/${domainId}/${docId}/additional_file/${f.filename}`, p); + } })().catch(logger.error); this.response.body = { pid: docId }; this.response.redirect = this.url('problem_settings', { pid: docId }); @@ -170,8 +184,8 @@ class ProblemImportSYZOJHandler extends Handler { }); } const docId = await problem.add( - domainId, targetPid || `${host}_${pid}`, p.title, content, this.user._id, - p.tags || [], [], null, hidden, + domainId, targetPid, p.title, content, this.user._id, + p.tags || [], [], hidden, ); const r = download(`${url}testdata/download`); const file = path.resolve(os.tmpdir(), 'hydro', `import_${domainId}_${docId}.zip`); @@ -185,7 +199,7 @@ class ProblemImportSYZOJHandler extends Handler { const entries = zip.getEntries(); for (const entry of entries) { // eslint-disable-next-line no-await-in-loop - await storage.put(`problem/${domainId}/${docId}/testdata/${entry.entryName}`, entry.getData()); + await problem.addTestdata(domainId, docId, entry.entryName, entry.getData()); } await problem.edit(domainId, docId, { config: { diff --git a/packages/hydrooj/src/handler/problem.ts b/packages/hydrooj/src/handler/problem.ts index cb3fd74b1..b5b536414 100644 --- a/packages/hydrooj/src/handler/problem.ts +++ b/packages/hydrooj/src/handler/problem.ts @@ -192,36 +192,6 @@ export class ProblemDetailHandler extends ProblemHandler { async get(...args: any[]) { } // eslint-disable-line - @param('pid', Types.UnsignedInt) - @param('dest', Types.String) - @param('hidden', Types.Boolean) - async postCopy(domainId: string, pid: number, destDomainId: string, hidden = false) { - const udoc = await user.getById(destDomainId, this.user._id); - if (!udoc.hasPerm(PERM.PERM_CREATE_PROBLEM)) { - throw new PermissionError(PERM.PERM_CREATE_PROBLEM); - } - if (!this.pdoc.data) { - // Copy Without Data - pid = await problem.add( - destDomainId, this.pdoc.pid, this.pdoc.title, - this.pdoc.content, this.user._id, this.pdoc.tag, - this.pdoc.category, null, hidden, - ); - } else if (this.pdoc.data instanceof ObjectID) { - // With data - pid = await problem.add( - destDomainId, this.pdoc.pid, this.pdoc.title, - this.pdoc.content, this.user._id, this.pdoc.tag, - this.pdoc.category, { domainId, pid }, hidden, - ); - } else { - // TODO better message - // Data should only be copied once. - throw new BadRequestError('Cannot copy this problem.'); - } - this.response.redirect = this.url('problem_settings', { domainId: destDomainId, pid }); - } - @param('pid', Types.UnsignedInt) async postRejudge(domainId: string, pid: number) { this.checkPerm(PERM.PERM_REJUDGE_PROBLEM); @@ -497,10 +467,19 @@ export class ProblemFilesHandler extends ProblemDetailHandler { const zip = new AdmZip(this.request.files.file.path); const entries = zip.getEntries(); for (const entry of entries) { - // eslint-disable-next-line no-await-in-loop - await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${entry.name}`, entry.getData()); + if (type === 'testdata') { + // eslint-disable-next-line no-await-in-loop + await problem.addTestdata(domainId, this.pdoc.docId, entry.name, entry.getData()); + } else { + // eslint-disable-next-line no-await-in-loop + await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/${entry.name}`, entry.getData()); + } } - } else await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${filename}`, this.request.files.file.path); + } else if (type === 'testdata') { + await problem.addTestdata(domainId, this.pdoc.docId, filename, this.request.files.file.path); + } else { + await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/${filename}`, this.request.files.file.path); + } this.back(); } @@ -509,7 +488,11 @@ export class ProblemFilesHandler extends ProblemDetailHandler { async postDeleteFiles(domainId: string, files: string[], type = 'testdata') { if (this.pdoc.owner !== this.user._id) this.checkPerm(PERM.PERM_EDIT_PROBLEM); else this.checkPerm(PERM.PERM_EDIT_PROBLEM_SELF); - await storage.del(files.map((file) => `problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${file}`)); + if (type === 'testdata') { + await problem.delTestdata(domainId, this.pdoc.docId, files); + } else { + await storage.del(files.map((file) => `problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${file}`)); + } this.back(); } } @@ -660,7 +643,7 @@ export class ProblemCreateHandler extends Handler { } catch { /* Ignore */ } const docId = await problem.add( domainId, pid, title, content, - this.user._id, [], [], null, hidden, + this.user._id, [], [], hidden, ); this.response.body = { pid: docId }; this.response.redirect = this.url('problem_settings', { pid: docId }); @@ -687,7 +670,8 @@ export class ProblemImportHandler extends Handler { await storage.put(`problem/${domainId}/${pid}/${entry.name}`, entry.getData()); } } - await problem.edit(domainId, pid, { html: pdoc.html }); + const data = await storage.list(`problem/${domainId}/${pid}/testdata/`, true); + await problem.edit(domainId, pid, { html: pdoc.html, data }); this.response.redirect = this.url('problem_detail', { pid }); } } diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 2fca1aff2..03dff4309 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -93,15 +93,11 @@ export interface User extends Dictionary { export type Udict = NumericDictionary; -export interface ProblemDataSource { - host?: string, // Empty for local - domainId: string, - pid: number, +export interface ProblemData { + name: string, + size: number, } -// ObjectID for built-in files, ProblemDataSource for other source (RemoteJudge, etc.) -export type ProblemData = ObjectID | ProblemDataSource; - export interface TestCaseConfig { input: string, output: string, @@ -196,7 +192,7 @@ declare module './model/problem' { nAccept: number, tag: string[], category: string[], - data?: ProblemData, + data: ProblemData[], hidden: boolean, config: ProblemConfig, acMsg?: string, diff --git a/packages/hydrooj/src/model/document.ts b/packages/hydrooj/src/model/document.ts index 18f45de54..f4528eed3 100644 --- a/packages/hydrooj/src/model/document.ts +++ b/packages/hydrooj/src/model/document.ts @@ -168,20 +168,31 @@ export function count( export async function push( domainId: string, docType: K, docId: DocType[K]['docId'], - key: keyof DocType[K], content: string, owner: number, args: DocType[K][T][0] = {}, + key: ArrayKeys, value: DocType[K][0], +): Promise<[DocType[K], ObjectID]> +export async function push( + domainId: string, docType: K, docId: DocType[K]['docId'], + key: keyof DocType[K], content: string, owner: number, args?: DocType[K][T][0], +): Promise<[DocType[K], ObjectID]> +export async function push( + domainId: string, docType: number, docId: DocID, key: string, + arg0: any, arg1?: any, arg2?: any, ) { - const _id = new ObjectID(); + const _id = arg2?._id || arg0?._id || new ObjectID(); + const v = arg1 + ? { _id, ...arg2, content: arg0, owner: arg1 } + : { _id, ...arg0 }; const doc = await coll.findOneAndUpdate( { domainId, docType, docId }, - { $push: { [key]: { ...args, content, owner, _id } } }, + { $push: { [key]: v } }, { returnOriginal: false }, ); return [doc.value, _id]; } -export async function pull( +export async function pull>( domainId: string, docType: K, docId: DocType[K]['docId'], - setKey: ArrayKeys, contents: string[], + setKey: T, contents: FilterQuery, ): Promise { const res = await coll.findOneAndUpdate( { domainId, docType, docId }, diff --git a/packages/hydrooj/src/model/problem.ts b/packages/hydrooj/src/model/problem.ts index 9d3c8088a..1f68f8811 100644 --- a/packages/hydrooj/src/model/problem.ts +++ b/packages/hydrooj/src/model/problem.ts @@ -1,12 +1,14 @@ +import type { Readable } from 'stream'; import { ObjectID, FilterQuery } from 'mongodb'; import { Dictionary } from 'lodash'; import { STATUS } from './builtin'; import * as document from './document'; import * as domain from './domain'; -import { ProblemStatusDoc, ProblemDataSource, Pdict } from '../interface'; +import { ProblemStatusDoc, Pdict } from '../interface'; import { Content } from '../loader'; -import { NumberKeys, Projection } from '../typeutils'; +import { ArrayKeys, NumberKeys, Projection } from '../typeutils'; import { ProblemNotFoundError } from '../error'; +import storage from '../service/storage'; export interface Pdoc { _id: ObjectID @@ -35,7 +37,7 @@ export namespace Pdoc { nAccept: 0, tag: [], category: [], - data: null, + data: [], hidden: true, config: {}, acMsg: '', @@ -77,11 +79,11 @@ export const PROJECTION_PUBLIC: Pdoc.Field[] = [ export async function add( domainId: string, pid: string = null, title: string, content: Content, owner: number, - tag: string[] = [], category: string[] = [], data: ProblemDataSource = null, hidden = false, + tag: string[] = [], category: string[] = [], hidden = false, ) { const pidCounter = await domain.inc(domainId, 'pidCounter', 1); const args: Partial = { - title, data, category, tag, hidden, nSubmit: 0, nAccept: 0, + title, category, tag, hidden, nSubmit: 0, nAccept: 0, }; if (pid) args.pid = pid; return await document.add(domainId, content, owner, document.TYPE_PROBLEM, pidCounter, null, null, args); @@ -118,6 +120,14 @@ export function edit(domainId: string, _id: number, $set: Partial): Promis return document.set(domainId, document.TYPE_PROBLEM, _id, $set, delpid ? { pid: '' } : undefined); } +export function push>(domainId: string, _id: number, key: ArrayKeys, value: Pdoc[T][0]) { + return document.push(domainId, document.TYPE_PROBLEM, _id, key, value); +} + +export function pull>(domainId: string, pid: number, key: ArrayKeys, values: Pdoc[T][0][]) { + return document.pull(domainId, document.TYPE_PROBLEM, pid, key, values.map((_id) => ({ _id }))); +} + export function inc(domainId: string, _id: number, field: NumberKeys, n: number): Promise { return document.inc(domainId, document.TYPE_PROBLEM, _id, field, n); } @@ -126,6 +136,18 @@ export function count(domainId: string, query: FilterQuery) { return document.count(domainId, document.TYPE_PROBLEM, query); } +export async function addTestdata(domainId: string, pid: number, name: string, f: Readable | Buffer | string) { + await storage.put(`problem/${domainId}/${pid}/testdata/${name}`, f); + const meta = await storage.getMeta(`problem/${domainId}/${pid}/testdata/${name}`); + return await push(domainId, pid, 'data', meta); +} + +export async function delTestdata(domainId: string, pid: number, name: string | string[]) { + const names = name instanceof Array ? name : [name]; + await storage.del(names.map((t) => `problem/${domainId}/${pid}/testdata/${t}`)); + await pull(domainId, pid, 'data', names); +} + export async function random(domainId: string, query: FilterQuery): Promise { const cursor = document.getMulti(domainId, document.TYPE_PROBLEM, query); const pcount = await cursor.count(); @@ -201,7 +223,11 @@ global.Hydro.model.problem = { get, edit, count, + push, + pull, random, + addTestdata, + delTestdata, getMulti, getList, getListStatus, diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index cf61a985a..331cc3a36 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -132,6 +132,19 @@ const scripts: UpgradeScript[] = [ logger.success('Files copied successfully. You can now remove collection `file` `fs.files` `fs.chunks` in the database.'); return true; }, + async function _4_5() { + const domains = await domain.getMulti().project({ _id: 1 }).toArray(); + for (const d of domains) { + const bulk = document.coll.initializeUnorderedBulkOp(); + const pdocs = await document.getMulti(d._id, document.TYPE_PROBLEM).project({ domainId: 1, docId: 1 }).toArray(); + for (const pdoc of pdocs) { + const data = await storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`, true); + bulk.find({ _id: pdoc._id }).updateOne({ $set: { data } }); + } + if (bulk.length) await bulk.execute(); + } + return true; + }, ]; export = scripts; diff --git a/packages/import-qduoj/handler.ts b/packages/import-qduoj/handler.ts index 33a483dbc..1624aa33b 100644 --- a/packages/import-qduoj/handler.ts +++ b/packages/import-qduoj/handler.ts @@ -100,7 +100,8 @@ class ImportQduojHandler extends Handler { `problem/${domainId}/${pid}/testdata/config.yaml`, Buffer.from(yaml.safeDump(config)), ); - await problem.edit(domainId, pid, { html: true }); + const data = await storage.list(`problem/${domainId}/${pid}/testdata/`, true); + await problem.edit(domainId, pid, { html: true, data }); } } finally { await fs.remove(tmp); diff --git a/packages/migrate-hustoj/script.ts b/packages/migrate-hustoj/script.ts index 2df338480..81763cbe5 100644 --- a/packages/migrate-hustoj/script.ts +++ b/packages/migrate-hustoj/script.ts @@ -96,7 +96,7 @@ export async function run({ hint: pdoc.hint, source: pdoc.source, }, 'html'), - 1, pdoc.source.split(' '), undefined, undefined, pdoc.defunct === 'Y', + 1, pdoc.source.split(' '), undefined, pdoc.defunct === 'Y', ); const [cdoc] = await query(`SELECT * FROM 'privilege' WHERE rightstr = 'p${pdoc.problem_id}'`); await problem.edit('system', pid, {