diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index a43e9a285..7360dc9a3 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "2.16.1", + "version": "2.17.0", "bin": "bin/hydrooj.js", "main": "dist/loader.js", "typings": "dist/loader.d.ts", diff --git a/packages/hydrooj/src/handler/import.syzoj.ts b/packages/hydrooj/src/handler/import.syzoj.ts index 2d6278e93..bba3c1815 100644 --- a/packages/hydrooj/src/handler/import.syzoj.ts +++ b/packages/hydrooj/src/handler/import.syzoj.ts @@ -15,7 +15,6 @@ import { PERM } from '../model/builtin'; import { Route, Handler, Types, post, } from '../service/server'; -import storage from '../service/storage'; import { isPid } from '../lib/validator'; import download from '../lib/download'; @@ -208,7 +207,7 @@ class ProblemImportSYZOJHandler extends Handler { const p = new PassThrough(); superagent.get(aurls[f.filename]).pipe(p); // eslint-disable-next-line no-await-in-loop - await storage.put(`problem/${domainId}/${docId}/additional_file/${f.filename}`, p); + await problem.addAdditionalFile(domainId, docId, f.filename, p); } }; if (wait) await syncFiles(); diff --git a/packages/hydrooj/src/handler/misc.ts b/packages/hydrooj/src/handler/misc.ts index d34be57de..3a900b1a3 100644 --- a/packages/hydrooj/src/handler/misc.ts +++ b/packages/hydrooj/src/handler/misc.ts @@ -1,57 +1,14 @@ /* eslint-disable camelcase */ -import fs from 'fs'; import { BadRequestError } from '../error'; import { PRIV } from '../model/builtin'; -import * as system from '../model/system'; import * as user from '../model/user'; import db from '../service/db'; import { Route, Handler, Types, param, } from '../service/server'; -import storage from '../service/storage'; const coll = db.collection('status'); -class FileUploadHandler extends Handler { - async getQuota() { - let quota = system.get('user.quota'); - if (this.user.hasPriv(PRIV.PRIV_UNLIMITED_QUOTA)) { - quota = 2 ** 63 - 1; - } - return quota; - } - - async get() { - this.response.template = 'fs_upload.html'; - this.response.body = { fdoc: null, usage: this.user.usage, quota: await this.getQuota() }; - } - - @param('title', Types.String) - @param('redirect', Types.String, true) - async post(domainId: string, title: string, redirect?: string) { - if (!this.request.files.file) throw new BadRequestError(); - const quota = await this.getQuota(); - const lfdoc = await fs.promises.stat(this.request.files.file.path); - const ufid = `user/${this.user._id}/${title}`; - const udoc = await user.inc(this.user._id, 'usage', lfdoc.size); - try { - await storage.put(ufid, this.request.files.file.path); - } catch (e) { - await user.inc(this.user._id, 'usage', -lfdoc.size); - throw e; - } - this.response.template = 'fs_upload.html'; - if (redirect) { - this.response.redirect = redirect.includes('?') - ? `${redirect}&ufid=${ufid}` - : `${redirect}?ufid=${ufid}`; - } - this.response.body = { - ufid, usage: udoc.usage, quota, - }; - } -} - class StatusHandler extends Handler { async get() { const stats = await coll.find().sort({ type: 1, updateAt: -1 }).toArray(); @@ -125,7 +82,6 @@ class SwitchLanguageHandler extends Handler { } export async function apply() { - Route('file_upload', '/upload', FileUploadHandler, PRIV.PRIV_CREATE_FILE); Route('status', '/status', StatusHandler); Route('status_update', '/status/update', StatusUpdateHandler); Route('check_in', '/checkin', CheckInHandler, PRIV.PRIV_USER_PROFILE); diff --git a/packages/hydrooj/src/handler/problem.ts b/packages/hydrooj/src/handler/problem.ts index 01e9fa295..c4f7ade6a 100644 --- a/packages/hydrooj/src/handler/problem.ts +++ b/packages/hydrooj/src/handler/problem.ts @@ -225,7 +225,7 @@ export class ProblemExportHandler extends ProblemDetailHandler { const pdoc = pick(this.pdoc, ['pid', 'acMsg', 'content', 'config', 'title', 'html', 'tag', 'category']); const zip = new AdmZip(); if (hasPerm) { - const files = await storage.list(`problem/${this.domainId}/${this.pdoc.docId}/`, true); + const files = await storage.list(`problem/${this.domainId}/${this.pdoc.docId}/`); for (const file of files) { // eslint-disable-next-line no-await-in-loop const buf = await streamToBuffer(await storage.get(`${file.prefix}${file.name}`)); @@ -427,16 +427,8 @@ export class ProblemFilesHandler extends ProblemDetailHandler { @param('additional_file', Types.Boolean) async get(domainId: string, getTestdata = true, getAdditionalFile = true) { const canReadData = this.user._id === this.pdoc.owner || this.user.hasPerm(PERM.PERM_READ_PROBLEM_DATA); - const [testdata, additional_file] = await Promise.all([ - getTestdata && canReadData - ? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/testdata/`, true) - : [], - getAdditionalFile - ? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/`, true) - : [], - ]); - this.response.body.testdata = testdata; - this.response.body.additional_file = additional_file; + this.response.body.testdata = (getTestdata && canReadData) ? this.pdoc.data : []; + this.response.body.additional_file = (getAdditionalFile ? this.pdoc.additional_file : []); this.response.template = 'problem_files.html'; } @@ -474,13 +466,13 @@ export class ProblemFilesHandler extends ProblemDetailHandler { await problem.addTestdata(domainId, this.pdoc.docId, entry.name, entry.getData()); } else { // eslint-disable-next-line no-await-in-loop - await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/${entry.name}`, entry.getData()); + await problem.addAdditionalFile(domainId, this.pdoc.docId, entry.name, entry.getData()); } } } else if (type === 'testdata') { await problem.addTestdata(domainId, this.pdoc.docId, filename, this.request.files.file.path); } else { - await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/${filename}`, this.request.files.file.path); + await problem.addAdditionalFile(domainId, this.pdoc.docId, filename, this.request.files.file.path); } this.back(); } @@ -676,13 +668,15 @@ export class ProblemImportHandler extends Handler { const pid = await problem.add(domainId, pdoc.pid, pdoc.title, pdoc.content, this.user._id, pdoc.tags, pdoc.category); const entries = zip.getEntries(); for (const entry of entries) { - if (entry.name.startsWith('testdata/') || entry.name.startsWith('additional_file')) { + if (entry.name.startsWith('testdata/')) { // eslint-disable-next-line no-await-in-loop - await storage.put(`problem/${domainId}/${pid}/${entry.name}`, entry.getData()); + await problem.addTestdata(domainId, pid, entry.name.split('testdata/')[1], entry.getData()); + } else if (entry.name.startsWith('additional_file/')) { + // eslint-disable-next-line no-await-in-loop + await problem.addAdditionalFile(domainId, pid, entry.name.split('testdata/')[1], entry.getData()); } } - const data = await storage.list(`problem/${domainId}/${pid}/testdata/`, true); - await problem.edit(domainId, pid, { html: pdoc.html, data }); + await problem.edit(domainId, pid, { html: pdoc.html }); this.response.redirect = this.url('problem_detail', { pid }); } } diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 2a31eae61..1ec85365b 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -95,6 +95,8 @@ export type Udict = NumericDictionary; export interface FileInfo { name: string, size: number, + etag: string, + lastModified: Date, } export interface TestCaseConfig { diff --git a/packages/hydrooj/src/model/builtin.ts b/packages/hydrooj/src/model/builtin.ts index 6f4c3c495..f1c126415 100644 --- a/packages/hydrooj/src/model/builtin.ts +++ b/packages/hydrooj/src/model/builtin.ts @@ -232,14 +232,27 @@ export const PRIV = { PRIV_REJUDGE: 1 << 13, PRIV_VIEW_USER_SECRET: 1 << 14, PRIV_VIEW_JUDGE_STATISTICS: 1 << 15, - PRIV_CREATE_FILE: 1 << 16, - PRIV_UNLIMITED_QUOTA: 1 << 17, - PRIV_DELETE_FILE: 1 << 18, - PRIV_DELETE_FILE_SELF: 1 << 19, - PRIV_ALL: -1, + PRIV_ALL: -1, PRIV_DEFAULT: 0, PRIV_NEVER: 1 << 20, + + /** + * @deprecated + */ + PRIV_CREATE_FILE: 1 << 16, + /** + * @deprecated + */ + PRIV_UNLIMITED_QUOTA: 1 << 17, + /** + * @deprecated + */ + PRIV_DELETE_FILE: 1 << 18, + /** + * @deprecated + */ + PRIV_DELETE_FILE_SELF: 1 << 19, }; PRIV.PRIV_DEFAULT = PRIV.PRIV_USER_PROFILE diff --git a/packages/hydrooj/src/model/problem.ts b/packages/hydrooj/src/model/problem.ts index 71b1e116b..f83590b10 100644 --- a/packages/hydrooj/src/model/problem.ts +++ b/packages/hydrooj/src/model/problem.ts @@ -141,7 +141,7 @@ export function del(domainId: string, docId: number) { return Promise.all([ document.deleteOne(domainId, document.TYPE_PROBLEM, docId), document.deleteMultiStatus(domainId, document.TYPE_PROBLEM, { docId }), - storage.list(`problem/${domainId}/${docId}/`, true).then((items) => storage.del(items.map((item) => item.prefix + item.name))), + storage.list(`problem/${domainId}/${docId}/`).then((items) => storage.del(items.map((item) => item.prefix + item.name))), bus.parallel('problem/delete', domainId, docId), ]); } @@ -149,7 +149,9 @@ export function del(domainId: string, docId: number) { export async function addTestdata(domainId: string, pid: number, name: string, f: Readable | Buffer | string) { await storage.put(`problem/${domainId}/${pid}/testdata/${name}`, f); const meta = await storage.getMeta(`problem/${domainId}/${pid}/testdata/${name}`); - return await push(domainId, pid, 'data', meta); + return await push(domainId, pid, 'data', { + _id: name, name, size: meta.size, lastModified: meta.lastModified, etag: meta.etag, + }); } export async function delTestdata(domainId: string, pid: number, name: string | string[]) { @@ -161,7 +163,9 @@ export async function delTestdata(domainId: string, pid: number, name: string | export async function addAdditionalFile(domainId: string, pid: number, name: string, f: Readable | Buffer | string) { await storage.put(`problem/${domainId}/${pid}/additional_file/${name}`, f); const meta = await storage.getMeta(`problem/${domainId}/${pid}/additional_file/${name}`); - return await push(domainId, pid, 'additional_file', meta); + return await push(domainId, pid, 'additional_file', { + _id: name, name, size: meta.size, lastModified: meta.lastModified, etag: meta.etag, + }); } export async function delAdditionalFile(domainId: string, pid: number, name: string | string[]) { diff --git a/packages/hydrooj/src/model/record.ts b/packages/hydrooj/src/model/record.ts index 5d7168ade..5c93298da 100644 --- a/packages/hydrooj/src/model/record.ts +++ b/packages/hydrooj/src/model/record.ts @@ -5,13 +5,13 @@ import { Dictionary } from 'lodash'; import moment from 'moment'; import { STATUS } from './builtin'; import * as task from './task'; +import * as problem from './problem'; import { Rdoc, TestCase, RunConfig, ContestInfo, ProblemConfig, } from '../interface'; import { Time } from '../utils'; import * as bus from '../service/bus'; import db from '../service/db'; -import storage from '../service/storage'; export const coll: Collection = db.collection('record'); @@ -75,7 +75,8 @@ export async function judge(domainId: string, rid: ObjectID, priority = 1) { const config: RunConfig | ProblemConfig = rdoc.config || {}; let data = []; if (rdoc.pid) { - data = await storage.list(`problem/${domainId}/${rdoc.pid}/testdata/`, true); + const pdoc = await problem.get(domainId, rdoc.pid); + data = pdoc.data; } delete rdoc._id; await task.add({ diff --git a/packages/hydrooj/src/service/storage.ts b/packages/hydrooj/src/service/storage.ts index e730460fc..b29df421d 100644 --- a/packages/hydrooj/src/service/storage.ts +++ b/packages/hydrooj/src/service/storage.ts @@ -129,7 +129,7 @@ class StorageService { return await this.client.removeObjects(this.opts.bucket, target); } - async list(target: string, recursive = false) { + async list(target: string, recursive = true) { const stream = this.client.listObjects(this.opts.bucket, target, recursive); return await new Promise((resolve, reject) => { const results: BucketItem[] = []; diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index 8aecc61bf..01e2c246e 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -13,7 +13,6 @@ import gridfs from './service/gridfs'; import storage from './service/storage'; import db from './service/db'; import * as problem from './model/problem'; -import * as user from './model/user'; import * as domain from './model/domain'; import * as document from './model/document'; import * as system from './model/system'; @@ -26,9 +25,9 @@ async function iterateAllDomain(cb: (ddoc: DomainDoc, current?: number, total?: for (const i in ddocs) await cb(ddocs[i], +i, ddocs.length); } -async function iterateAllProblem(cb: (pdoc: problem.Pdoc, current?: number, total?: number) => Promise) { +async function iterateAllProblem(fields: problem.Pdoc.Field[], cb: (pdoc: problem.Pdoc, current?: number, total?: number) => Promise) { await iterateAllDomain(async (d) => { - const pdocs = await problem.getMulti(d._id, {}).toArray(); + const pdocs = await problem.getMulti(d._id, {}, fields).toArray(); for (const i in pdocs) await cb(pdocs[i], +i, pdocs.length); }); } @@ -91,21 +90,16 @@ const scripts: UpgradeScript[] = [ try { const [file, current] = await Promise.all([ streamToBuffer(gridfs.openDownloadStream(pdoc.data)), - storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`, true), + storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`), ]); const zip = new AdmZip(file); const entries = zip.getEntries(); if (entries.map((entry) => entry.entryName).sort().join('?') !== current.map((f) => f.name).sort().join('?')) { await storage.del(current.map((entry) => entry.prefix + entry.name)); const queue = new Queue({ concurrency: 5 }); - const tasks = []; - for (const entry of entries) { - tasks.push(queue.add(() => storage.put( - `problem/${pdoc.domainId}/${pdoc.docId}/testdata/${entry.entryName}`, - entry.getData(), - ))); - } - await Promise.all(tasks); + await Promise.all(entries.map( + (entry) => queue.add(() => problem.addTestdata(pdoc.domainId, pdoc.docId, entry.entryName, entry.getData())), + )); } } catch (e) { if (e.toString().includes('FileNotFound')) { @@ -142,15 +136,21 @@ const scripts: UpgradeScript[] = [ return true; }, async function _6_7() { + return true; // invalid + }, + async function _7_8() { const _FRESH_INSTALL_IGNORE = 1; - await iterateAllProblem(async (pdoc) => { + await iterateAllProblem(['docId', 'domainId', 'config'], async (pdoc) => { + logger.info('%s/%s', pdoc.domainId, pdoc.docId); + const [data, additional_file] = await Promise.all([ + storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`), + storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/additional_files/`), + ]); + await problem.edit(pdoc.domainId, pdoc.docId, { data, additional_file }); if (!pdoc.config) return; - const dst = `problem/${pdoc.domainId}/${pdoc.pid}/testdata/config.yaml`; - await storage.get(dst) - .catch(() => { - const cfg = yaml.dump(pdoc.config); - return problem.addTestdata(pdoc.domainId, pdoc.docId, 'config.yaml', Buffer.from(cfg)); - }); + if (data.map((d) => d.name).includes('config.yaml')) return; + const cfg = yaml.dump(pdoc.config); + await problem.addTestdata(pdoc.domainId, pdoc.docId, 'config.yaml', Buffer.from(cfg)); }); return true; }, diff --git a/packages/import-qduoj/handler.ts b/packages/import-qduoj/handler.ts index 87befb793..ac2c58e24 100644 --- a/packages/import-qduoj/handler.ts +++ b/packages/import-qduoj/handler.ts @@ -6,7 +6,6 @@ import AdmZip from 'adm-zip'; import yaml from 'js-yaml'; import type { ContentNode, ProblemConfig } from 'hydrooj'; import { Route, Handler } from 'hydrooj/dist/service/server'; -import storage from 'hydrooj/dist/service/storage'; import { BadRequestError, ValidationError } from 'hydrooj/dist/error'; import { ProblemAdd } from 'hydrooj/dist/lib/ui'; import * as problem from 'hydrooj/dist/model/problem'; diff --git a/packages/import-qduoj/package.json b/packages/import-qduoj/package.json index 7be195479..d3381ac08 100644 --- a/packages/import-qduoj/package.json +++ b/packages/import-qduoj/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/import-qduoj", - "version": "1.1.3", + "version": "1.1.4", "description": "Import QDUOJ problem export", "main": "package.json", "repository": "https://github.com/hydro-dev/Hydro.git",