core: files: 添加文件列表缓存

packages/hydrooj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "hydrooj",
-    "version": "2.16.1",
+    "version": "2.17.0",
     "bin": "bin/hydrooj.js",
     "main": "dist/loader.js",
     "typings": "dist/loader.d.ts",

packages/hydrooj/src/handler/import.syzoj.ts

@@ -15,7 +15,6 @@ import { PERM } from '../model/builtin';
 import {
     Route, Handler, Types, post,
 } from '../service/server';
-import storage from '../service/storage';
 import { isPid } from '../lib/validator';
 import download from '../lib/download';
 
@@ -208,7 +207,7 @@ class ProblemImportSYZOJHandler extends Handler {
                 const p = new PassThrough();
                 superagent.get(aurls[f.filename]).pipe(p);
                 // eslint-disable-next-line no-await-in-loop
-                await storage.put(`problem/${domainId}/${docId}/additional_file/${f.filename}`, p);
+                await problem.addAdditionalFile(domainId, docId, f.filename, p);
             }
         };
         if (wait) await syncFiles();

packages/hydrooj/src/handler/misc.ts

@@ -1,57 +1,14 @@
 /* eslint-disable camelcase */
-import fs from 'fs';
 import { BadRequestError } from '../error';
 import { PRIV } from '../model/builtin';
-import * as system from '../model/system';
 import * as user from '../model/user';
 import db from '../service/db';
 import {
     Route, Handler, Types, param,
 } from '../service/server';
-import storage from '../service/storage';
 
 const coll = db.collection('status');
 
-class FileUploadHandler extends Handler {
-    async getQuota() {
-        let quota = system.get('user.quota');
-        if (this.user.hasPriv(PRIV.PRIV_UNLIMITED_QUOTA)) {
-            quota = 2 ** 63 - 1;
-        }
-        return quota;
-    }
-
-    async get() {
-        this.response.template = 'fs_upload.html';
-        this.response.body = { fdoc: null, usage: this.user.usage, quota: await this.getQuota() };
-    }
-
-    @param('title', Types.String)
-    @param('redirect', Types.String, true)
-    async post(domainId: string, title: string, redirect?: string) {
-        if (!this.request.files.file) throw new BadRequestError();
-        const quota = await this.getQuota();
-        const lfdoc = await fs.promises.stat(this.request.files.file.path);
-        const ufid = `user/${this.user._id}/${title}`;
-        const udoc = await user.inc(this.user._id, 'usage', lfdoc.size);
-        try {
-            await storage.put(ufid, this.request.files.file.path);
-        } catch (e) {
-            await user.inc(this.user._id, 'usage', -lfdoc.size);
-            throw e;
-        }
-        this.response.template = 'fs_upload.html';
-        if (redirect) {
-            this.response.redirect = redirect.includes('?')
-                ? `${redirect}&ufid=${ufid}`
-                : `${redirect}?ufid=${ufid}`;
-        }
-        this.response.body = {
-            ufid, usage: udoc.usage, quota,
-        };
-    }
-}
-
 class StatusHandler extends Handler {
     async get() {
         const stats = await coll.find().sort({ type: 1, updateAt: -1 }).toArray();
@@ -125,7 +82,6 @@ class SwitchLanguageHandler extends Handler {
 }
 
 export async function apply() {
-    Route('file_upload', '/upload', FileUploadHandler, PRIV.PRIV_CREATE_FILE);
     Route('status', '/status', StatusHandler);
     Route('status_update', '/status/update', StatusUpdateHandler);
     Route('check_in', '/checkin', CheckInHandler, PRIV.PRIV_USER_PROFILE);

packages/hydrooj/src/handler/problem.ts

@@ -225,7 +225,7 @@ export class ProblemExportHandler extends ProblemDetailHandler {
         const pdoc = pick(this.pdoc, ['pid', 'acMsg', 'content', 'config', 'title', 'html', 'tag', 'category']);
         const zip = new AdmZip();
         if (hasPerm) {
-            const files = await storage.list(`problem/${this.domainId}/${this.pdoc.docId}/`, true);
+            const files = await storage.list(`problem/${this.domainId}/${this.pdoc.docId}/`);
             for (const file of files) {
                 // eslint-disable-next-line no-await-in-loop
                 const buf = await streamToBuffer(await storage.get(`${file.prefix}${file.name}`));
@@ -427,16 +427,8 @@ export class ProblemFilesHandler extends ProblemDetailHandler {
     @param('additional_file', Types.Boolean)
     async get(domainId: string, getTestdata = true, getAdditionalFile = true) {
         const canReadData = this.user._id === this.pdoc.owner || this.user.hasPerm(PERM.PERM_READ_PROBLEM_DATA);
-        const [testdata, additional_file] = await Promise.all([
+        this.response.body.testdata = (getTestdata && canReadData) ? this.pdoc.data : [];
-            getTestdata && canReadData
+        this.response.body.additional_file = (getAdditionalFile ? this.pdoc.additional_file : []);
-                ? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/testdata/`, true)
-                : [],
-            getAdditionalFile
-                ? storage.list(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/`, true)
-                : [],
-        ]);
-        this.response.body.testdata = testdata;
-        this.response.body.additional_file = additional_file;
         this.response.template = 'problem_files.html';
     }
 
@@ -474,13 +466,13 @@ export class ProblemFilesHandler extends ProblemDetailHandler {
                     await problem.addTestdata(domainId, this.pdoc.docId, entry.name, entry.getData());
                 } else {
                     // eslint-disable-next-line no-await-in-loop
-                    await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/${entry.name}`, entry.getData());
+                    await problem.addAdditionalFile(domainId, this.pdoc.docId, entry.name, entry.getData());
                 }
             }
         } else if (type === 'testdata') {
             await problem.addTestdata(domainId, this.pdoc.docId, filename, this.request.files.file.path);
         } else {
-            await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/${filename}`, this.request.files.file.path);
+            await problem.addAdditionalFile(domainId, this.pdoc.docId, filename, this.request.files.file.path);
         }
         this.back();
     }
@@ -676,13 +668,15 @@ export class ProblemImportHandler extends Handler {
         const pid = await problem.add(domainId, pdoc.pid, pdoc.title, pdoc.content, this.user._id, pdoc.tags, pdoc.category);
         const entries = zip.getEntries();
         for (const entry of entries) {
-            if (entry.name.startsWith('testdata/') || entry.name.startsWith('additional_file')) {
+            if (entry.name.startsWith('testdata/')) {
                 // eslint-disable-next-line no-await-in-loop
-                await storage.put(`problem/${domainId}/${pid}/${entry.name}`, entry.getData());
+                await problem.addTestdata(domainId, pid, entry.name.split('testdata/')[1], entry.getData());
+            } else if (entry.name.startsWith('additional_file/')) {
+                // eslint-disable-next-line no-await-in-loop
+                await problem.addAdditionalFile(domainId, pid, entry.name.split('testdata/')[1], entry.getData());
             }
         }
-        const data = await storage.list(`problem/${domainId}/${pid}/testdata/`, true);
+        await problem.edit(domainId, pid, { html: pdoc.html });
-        await problem.edit(domainId, pid, { html: pdoc.html, data });
         this.response.redirect = this.url('problem_detail', { pid });
     }
 }

packages/hydrooj/src/interface.ts

@@ -95,6 +95,8 @@ export type Udict = NumericDictionary<User>;
 export interface FileInfo {
     name: string,
     size: number,
+    etag: string,
+    lastModified: Date,
 }
 
 export interface TestCaseConfig {

packages/hydrooj/src/model/builtin.ts

@@ -232,14 +232,27 @@ export const PRIV = {
     PRIV_REJUDGE: 1 << 13,
     PRIV_VIEW_USER_SECRET: 1 << 14,
     PRIV_VIEW_JUDGE_STATISTICS: 1 << 15,
-    PRIV_CREATE_FILE: 1 << 16,
-    PRIV_UNLIMITED_QUOTA: 1 << 17,
-    PRIV_DELETE_FILE: 1 << 18,
-    PRIV_DELETE_FILE_SELF: 1 << 19,
-    PRIV_ALL: -1,
 
+    PRIV_ALL: -1,
     PRIV_DEFAULT: 0,
     PRIV_NEVER: 1 << 20,
+
+    /**
+     * @deprecated
+     */
+    PRIV_CREATE_FILE: 1 << 16,
+    /**
+     * @deprecated
+     */
+    PRIV_UNLIMITED_QUOTA: 1 << 17,
+    /**
+     * @deprecated
+     */
+    PRIV_DELETE_FILE: 1 << 18,
+    /**
+     * @deprecated
+     */
+    PRIV_DELETE_FILE_SELF: 1 << 19,
 };
 
 PRIV.PRIV_DEFAULT = PRIV.PRIV_USER_PROFILE

packages/hydrooj/src/model/problem.ts

@@ -141,7 +141,7 @@ export function del(domainId: string, docId: number) {
     return Promise.all([
         document.deleteOne(domainId, document.TYPE_PROBLEM, docId),
         document.deleteMultiStatus(domainId, document.TYPE_PROBLEM, { docId }),
-        storage.list(`problem/${domainId}/${docId}/`, true).then((items) => storage.del(items.map((item) => item.prefix + item.name))),
+        storage.list(`problem/${domainId}/${docId}/`).then((items) => storage.del(items.map((item) => item.prefix + item.name))),
         bus.parallel('problem/delete', domainId, docId),
     ]);
 }
@@ -149,7 +149,9 @@ export function del(domainId: string, docId: number) {
 export async function addTestdata(domainId: string, pid: number, name: string, f: Readable | Buffer | string) {
     await storage.put(`problem/${domainId}/${pid}/testdata/${name}`, f);
     const meta = await storage.getMeta(`problem/${domainId}/${pid}/testdata/${name}`);
-    return await push(domainId, pid, 'data', meta);
+    return await push(domainId, pid, 'data', {
+        _id: name, name, size: meta.size, lastModified: meta.lastModified, etag: meta.etag,
+    });
 }
 
 export async function delTestdata(domainId: string, pid: number, name: string | string[]) {
@@ -161,7 +163,9 @@ export async function delTestdata(domainId: string, pid: number, name: string |
 export async function addAdditionalFile(domainId: string, pid: number, name: string, f: Readable | Buffer | string) {
     await storage.put(`problem/${domainId}/${pid}/additional_file/${name}`, f);
     const meta = await storage.getMeta(`problem/${domainId}/${pid}/additional_file/${name}`);
-    return await push(domainId, pid, 'additional_file', meta);
+    return await push(domainId, pid, 'additional_file', {
+        _id: name, name, size: meta.size, lastModified: meta.lastModified, etag: meta.etag,
+    });
 }
 
 export async function delAdditionalFile(domainId: string, pid: number, name: string | string[]) {

packages/hydrooj/src/model/record.ts

@@ -5,13 +5,13 @@ import { Dictionary } from 'lodash';
 import moment from 'moment';
 import { STATUS } from './builtin';
 import * as task from './task';
+import * as problem from './problem';
 import {
     Rdoc, TestCase, RunConfig, ContestInfo, ProblemConfig,
 } from '../interface';
 import { Time } from '../utils';
 import * as bus from '../service/bus';
 import db from '../service/db';
-import storage from '../service/storage';
 
 export const coll: Collection<Rdoc> = db.collection('record');
 
@@ -75,7 +75,8 @@ export async function judge(domainId: string, rid: ObjectID, priority = 1) {
     const config: RunConfig | ProblemConfig = rdoc.config || {};
     let data = [];
     if (rdoc.pid) {
-        data = await storage.list(`problem/${domainId}/${rdoc.pid}/testdata/`, true);
+        const pdoc = await problem.get(domainId, rdoc.pid);
+        data = pdoc.data;
     }
     delete rdoc._id;
     await task.add({

packages/hydrooj/src/service/storage.ts

@@ -129,7 +129,7 @@ class StorageService {
         return await this.client.removeObjects(this.opts.bucket, target);
     }
 
-    async list(target: string, recursive = false) {
+    async list(target: string, recursive = true) {
         const stream = this.client.listObjects(this.opts.bucket, target, recursive);
         return await new Promise<BucketItem[]>((resolve, reject) => {
             const results: BucketItem[] = [];

packages/hydrooj/src/upgrade.ts

@@ -13,7 +13,6 @@ import gridfs from './service/gridfs';
 import storage from './service/storage';
 import db from './service/db';
 import * as problem from './model/problem';
-import * as user from './model/user';
 import * as domain from './model/domain';
 import * as document from './model/document';
 import * as system from './model/system';
@@ -26,9 +25,9 @@ async function iterateAllDomain(cb: (ddoc: DomainDoc, current?: number, total?:
     for (const i in ddocs) await cb(ddocs[i], +i, ddocs.length);
 }
 
-async function iterateAllProblem(cb: (pdoc: problem.Pdoc, current?: number, total?: number) => Promise<any>) {
+async function iterateAllProblem(fields: problem.Pdoc.Field[], cb: (pdoc: problem.Pdoc, current?: number, total?: number) => Promise<any>) {
     await iterateAllDomain(async (d) => {
-        const pdocs = await problem.getMulti(d._id, {}).toArray();
+        const pdocs = await problem.getMulti(d._id, {}, fields).toArray();
         for (const i in pdocs) await cb(pdocs[i], +i, pdocs.length);
     });
 }
@@ -91,21 +90,16 @@ const scripts: UpgradeScript[] = [
                     try {
                         const [file, current] = await Promise.all([
                             streamToBuffer(gridfs.openDownloadStream(pdoc.data)),
-                            storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`, true),
+                            storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`),
                         ]);
                         const zip = new AdmZip(file);
                         const entries = zip.getEntries();
                         if (entries.map((entry) => entry.entryName).sort().join('?') !== current.map((f) => f.name).sort().join('?')) {
                             await storage.del(current.map((entry) => entry.prefix + entry.name));
                             const queue = new Queue({ concurrency: 5 });
-                            const tasks = [];
+                            await Promise.all(entries.map(
-                            for (const entry of entries) {
+                                (entry) => queue.add(() => problem.addTestdata(pdoc.domainId, pdoc.docId, entry.entryName, entry.getData())),
-                                tasks.push(queue.add(() => storage.put(
+                            ));
-                                    `problem/${pdoc.domainId}/${pdoc.docId}/testdata/${entry.entryName}`,
-                                    entry.getData(),
-                                )));
-                            }
-                            await Promise.all(tasks);
                         }
                     } catch (e) {
                         if (e.toString().includes('FileNotFound')) {
@@ -142,15 +136,21 @@ const scripts: UpgradeScript[] = [
         return true;
     },
     async function _6_7() {
+        return true; // invalid
+    },
+    async function _7_8() {
         const _FRESH_INSTALL_IGNORE = 1;
-        await iterateAllProblem(async (pdoc) => {
+        await iterateAllProblem(['docId', 'domainId', 'config'], async (pdoc) => {
+            logger.info('%s/%s', pdoc.domainId, pdoc.docId);
+            const [data, additional_file] = await Promise.all([
+                storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`),
+                storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/additional_files/`),
+            ]);
+            await problem.edit(pdoc.domainId, pdoc.docId, { data, additional_file });
             if (!pdoc.config) return;
-            const dst = `problem/${pdoc.domainId}/${pdoc.pid}/testdata/config.yaml`;
+            if (data.map((d) => d.name).includes('config.yaml')) return;
-            await storage.get(dst)
-                .catch(() => {
             const cfg = yaml.dump(pdoc.config);
-                    return problem.addTestdata(pdoc.domainId, pdoc.docId, 'config.yaml', Buffer.from(cfg));
+            await problem.addTestdata(pdoc.domainId, pdoc.docId, 'config.yaml', Buffer.from(cfg));
-                });
         });
         return true;
     },

packages/import-qduoj/handler.ts

@@ -6,7 +6,6 @@ import AdmZip from 'adm-zip';
 import yaml from 'js-yaml';
 import type { ContentNode, ProblemConfig } from 'hydrooj';
 import { Route, Handler } from 'hydrooj/dist/service/server';
-import storage from 'hydrooj/dist/service/storage';
 import { BadRequestError, ValidationError } from 'hydrooj/dist/error';
 import { ProblemAdd } from 'hydrooj/dist/lib/ui';
 import * as problem from 'hydrooj/dist/model/problem';

packages/import-qduoj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hydrooj/import-qduoj",
-    "version": "1.1.3",
+    "version": "1.1.4",
     "description": "Import QDUOJ problem export",
     "main": "package.json",
     "repository": "https://github.com/hydro-dev/Hydro.git",