core: remove problem_copy

packages/fps-importer/handler.ts

@@ -8,7 +8,6 @@ import { LocalProblemConfig } from 'hydrooj';
 import {
     Route, Handler, param, Types,
 } from 'hydrooj/dist/service/server';
-import * as storage from 'hydrooj/dist/service/storage';
 import { buildContent } from 'hydrooj/dist/lib/content';
 import { ProblemAdd } from 'hydrooj/dist/lib/ui';
 import * as solution from 'hydrooj/dist/model/solution';
@@ -39,16 +38,16 @@ class FpsProblemImportHandler extends Handler {
             const title = decodeHTML(p.title.join(' '));
             const tags = filter(p.source, (i: string) => i.trim());
             const pid = await problem.add(domainId, null, title, content, this.user._id, tags, []);
-            await storage.put(`problem/${domainId}/${pid}/testdata/config.yaml`, Buffer.from(`time: ${config.time}\nmemory: ${config.memory}`));
+            await problem.addTestdata(domainId, pid, 'config.yaml', Buffer.from(`time: ${config.time}\nmemory: ${config.memory}`));
             if (p.test_output) {
                 for (let i = 0; i < p.test_input.length; i++) {
-                    await storage.put(`problem/${domainId}/${pid}/testdata/${i + 1}.in`, Buffer.from(p.test_input[i]));
+                    await problem.addTestdata(domainId, pid, `${i + 1}.in`, Buffer.from(p.test_input[i]));
-                    await storage.put(`problem/${domainId}/${pid}/testdata/${i + 1}.out`, Buffer.from(p.test_output[i]));
+                    await problem.addTestdata(domainId, pid, `${i + 1}.out`, Buffer.from(p.test_output[i]));
                 }
             } else if (p.test_input) {
                 for (let i = 0; i < p.test_input.length / 2; i++) {
-                    await storage.put(`problem/${domainId}/${pid}/testdata/${i + 1}.in`, Buffer.from(p.test_input[2 * i]));
+                    await problem.addTestdata(domainId, pid, `${i + 1}.in`, Buffer.from(p.test_input[2 * i]));
-                    await storage.put(`problem/${domainId}/${pid}/testdata/${i + 1}.out`, Buffer.from(p.test_input[2 * i + 1]));
+                    await problem.addTestdata(domainId, pid, `${i + 1}.out`, Buffer.from(p.test_input[2 * i + 1]));
                 }
             }
             await problem.edit(domainId, pid, { html: true });

packages/hydrooj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "hydrooj",
-    "version": "2.15.3",
+    "version": "2.15.4",
     "bin": "bin/hydrooj.js",
     "main": "dist/loader.js",
     "typings": "dist/loader.d.ts",

packages/hydrooj/src/handler/import.syzoj.ts

@@ -75,8 +75,8 @@ class ProblemImportSYZOJHandler extends Handler {
                 ...result.body.localizedContentsOfAllLocales,
             ][0].title;
             const docId = await problem.add(
-                domainId, targetPid || `${host}_${pid}`, title, content, this.user._id,
+                domainId, targetPid, title, content, this.user._id,
-                tags || [], [], null, hidden,
+                tags || [], [], hidden,
             );
             (async () => {
                 const judge = result.body.judgeInfo;
@@ -93,9 +93,8 @@ class ProblemImportSYZOJHandler extends Handler {
                     const p = new PassThrough();
                     superagent.get(urls[f.filename]).pipe(p);
                     // eslint-disable-next-line no-await-in-loop
-                    await storage.put(`problem/${domainId}/${docId}/testdata/${f.filename}`, p);
+                    await problem.addTestdata(domainId, docId, f.filename, p);
                 }
-                // TODO additional_file
                 if (judge) {
                     await problem.edit(domainId, docId, {
                         config: {
@@ -105,6 +104,21 @@ class ProblemImportSYZOJHandler extends Handler {
                         },
                     });
                 }
+                const a = await superagent.post(`${protocol}://${host === 'loj.ac' ? 'api.loj.ac.cn' : host}/api/problem/downloadProblemFiles`)
+                    .send({
+                        problemId: +pid,
+                        type: 'AdditionalFile',
+                        filenameList: result.body.additionalFiles.map((node) => node.filename),
+                    });
+                const aurls = {};
+                if (a.body.error) return;
+                for (const t of a.body.downloadInfo) aurls[t.filename] = t.downloadUrl;
+                for (const f of result.body.additionalFiles) {
+                    const p = new PassThrough();
+                    superagent.get(aurls[f.filename]).pipe(p);
+                    // eslint-disable-next-line no-await-in-loop
+                    await storage.put(`problem/${domainId}/${docId}/additional_file/${f.filename}`, p);
+                }
             })().catch(logger.error);
             this.response.body = { pid: docId };
             this.response.redirect = this.url('problem_settings', { pid: docId });
@@ -170,8 +184,8 @@ class ProblemImportSYZOJHandler extends Handler {
                 });
             }
             const docId = await problem.add(
-                domainId, targetPid || `${host}_${pid}`, p.title, content, this.user._id,
+                domainId, targetPid, p.title, content, this.user._id,
-                p.tags || [], [], null, hidden,
+                p.tags || [], [], hidden,
             );
             const r = download(`${url}testdata/download`);
             const file = path.resolve(os.tmpdir(), 'hydro', `import_${domainId}_${docId}.zip`);
@@ -185,7 +199,7 @@ class ProblemImportSYZOJHandler extends Handler {
             const entries = zip.getEntries();
             for (const entry of entries) {
                 // eslint-disable-next-line no-await-in-loop
-                await storage.put(`problem/${domainId}/${docId}/testdata/${entry.entryName}`, entry.getData());
+                await problem.addTestdata(domainId, docId, entry.entryName, entry.getData());
             }
             await problem.edit(domainId, docId, {
                 config: {

packages/hydrooj/src/handler/problem.ts

@@ -192,36 +192,6 @@ export class ProblemDetailHandler extends ProblemHandler {
 
     async get(...args: any[]) { } // eslint-disable-line
 
-    @param('pid', Types.UnsignedInt)
-    @param('dest', Types.String)
-    @param('hidden', Types.Boolean)
-    async postCopy(domainId: string, pid: number, destDomainId: string, hidden = false) {
-        const udoc = await user.getById(destDomainId, this.user._id);
-        if (!udoc.hasPerm(PERM.PERM_CREATE_PROBLEM)) {
-            throw new PermissionError(PERM.PERM_CREATE_PROBLEM);
-        }
-        if (!this.pdoc.data) {
-            // Copy Without Data
-            pid = await problem.add(
-                destDomainId, this.pdoc.pid, this.pdoc.title,
-                this.pdoc.content, this.user._id, this.pdoc.tag,
-                this.pdoc.category, null, hidden,
-            );
-        } else if (this.pdoc.data instanceof ObjectID) {
-            // With data
-            pid = await problem.add(
-                destDomainId, this.pdoc.pid, this.pdoc.title,
-                this.pdoc.content, this.user._id, this.pdoc.tag,
-                this.pdoc.category, { domainId, pid }, hidden,
-            );
-        } else {
-            // TODO better message
-            // Data should only be copied once.
-            throw new BadRequestError('Cannot copy this problem.');
-        }
-        this.response.redirect = this.url('problem_settings', { domainId: destDomainId, pid });
-    }
-
     @param('pid', Types.UnsignedInt)
     async postRejudge(domainId: string, pid: number) {
         this.checkPerm(PERM.PERM_REJUDGE_PROBLEM);
@@ -497,10 +467,19 @@ export class ProblemFilesHandler extends ProblemDetailHandler {
             const zip = new AdmZip(this.request.files.file.path);
             const entries = zip.getEntries();
             for (const entry of entries) {
-                // eslint-disable-next-line no-await-in-loop
+                if (type === 'testdata') {
-                await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${entry.name}`, entry.getData());
+                    // eslint-disable-next-line no-await-in-loop
+                    await problem.addTestdata(domainId, this.pdoc.docId, entry.name, entry.getData());
+                } else {
+                    // eslint-disable-next-line no-await-in-loop
+                    await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/${entry.name}`, entry.getData());
+                }
             }
-        } else await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${filename}`, this.request.files.file.path);
+        } else if (type === 'testdata') {
+            await problem.addTestdata(domainId, this.pdoc.docId, filename, this.request.files.file.path);
+        } else {
+            await storage.put(`problem/${this.pdoc.domainId}/${this.pdoc.docId}/additional_file/${filename}`, this.request.files.file.path);
+        }
         this.back();
     }
 
@@ -509,7 +488,11 @@ export class ProblemFilesHandler extends ProblemDetailHandler {
     async postDeleteFiles(domainId: string, files: string[], type = 'testdata') {
         if (this.pdoc.owner !== this.user._id) this.checkPerm(PERM.PERM_EDIT_PROBLEM);
         else this.checkPerm(PERM.PERM_EDIT_PROBLEM_SELF);
-        await storage.del(files.map((file) => `problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${file}`));
+        if (type === 'testdata') {
+            await problem.delTestdata(domainId, this.pdoc.docId, files);
+        } else {
+            await storage.del(files.map((file) => `problem/${this.pdoc.domainId}/${this.pdoc.docId}/${type}/${file}`));
+        }
         this.back();
     }
 }
@@ -660,7 +643,7 @@ export class ProblemCreateHandler extends Handler {
         } catch { /* Ignore */ }
         const docId = await problem.add(
             domainId, pid, title, content,
-            this.user._id, [], [], null, hidden,
+            this.user._id, [], [], hidden,
         );
         this.response.body = { pid: docId };
         this.response.redirect = this.url('problem_settings', { pid: docId });
@@ -687,7 +670,8 @@ export class ProblemImportHandler extends Handler {
                 await storage.put(`problem/${domainId}/${pid}/${entry.name}`, entry.getData());
             }
         }
-        await problem.edit(domainId, pid, { html: pdoc.html });
+        const data = await storage.list(`problem/${domainId}/${pid}/testdata/`, true);
+        await problem.edit(domainId, pid, { html: pdoc.html, data });
         this.response.redirect = this.url('problem_detail', { pid });
     }
 }

packages/hydrooj/src/interface.ts

@@ -93,15 +93,11 @@ export interface User extends Dictionary<any> {
 
 export type Udict = NumericDictionary<User>;
 
-export interface ProblemDataSource {
+export interface ProblemData {
-    host?: string, // Empty for local
+    name: string,
-    domainId: string,
+    size: number,
-    pid: number,
 }
 
-// ObjectID for built-in files, ProblemDataSource for other source (RemoteJudge, etc.)
-export type ProblemData = ObjectID | ProblemDataSource;
-
 export interface TestCaseConfig {
     input: string,
     output: string,
@@ -196,7 +192,7 @@ declare module './model/problem' {
         nAccept: number,
         tag: string[],
         category: string[],
-        data?: ProblemData,
+        data: ProblemData[],
         hidden: boolean,
         config: ProblemConfig,
         acMsg?: string,

packages/hydrooj/src/model/document.ts

@@ -168,20 +168,31 @@ export function count<K extends keyof DocType>(
 
 export async function push<K extends keyof DocType, T extends keyof DocType[K]>(
     domainId: string, docType: K, docId: DocType[K]['docId'],
-    key: keyof DocType[K], content: string, owner: number, args: DocType[K][T][0] = {},
+    key: ArrayKeys<DocType[K]>, value: DocType[K][0],
+): Promise<[DocType[K], ObjectID]>
+export async function push<K extends keyof DocType, T extends keyof DocType[K]>(
+    domainId: string, docType: K, docId: DocType[K]['docId'],
+    key: keyof DocType[K], content: string, owner: number, args?: DocType[K][T][0],
+): Promise<[DocType[K], ObjectID]>
+export async function push(
+    domainId: string, docType: number, docId: DocID, key: string,
+    arg0: any, arg1?: any, arg2?: any,
 ) {
-    const _id = new ObjectID();
+    const _id = arg2?._id || arg0?._id || new ObjectID();
+    const v = arg1
+        ? { _id, ...arg2, content: arg0, owner: arg1 }
+        : { _id, ...arg0 };
     const doc = await coll.findOneAndUpdate(
         { domainId, docType, docId },
-        { $push: { [key]: { ...args, content, owner, _id } } },
+        { $push: { [key]: v } },
         { returnOriginal: false },
     );
     return [doc.value, _id];
 }
 
-export async function pull<K extends keyof DocType>(
+export async function pull<K extends keyof DocType, T extends ArrayKeys<DocType[K]>>(
     domainId: string, docType: K, docId: DocType[K]['docId'],
-    setKey: ArrayKeys<DocType[K]>, contents: string[],
+    setKey: T, contents: FilterQuery<DocType[K][T][0]>,
 ): Promise<DocType[K]> {
     const res = await coll.findOneAndUpdate(
         { domainId, docType, docId },

packages/hydrooj/src/model/problem.ts

@@ -1,12 +1,14 @@
+import type { Readable } from 'stream';
 import { ObjectID, FilterQuery } from 'mongodb';
 import { Dictionary } from 'lodash';
 import { STATUS } from './builtin';
 import * as document from './document';
 import * as domain from './domain';
-import { ProblemStatusDoc, ProblemDataSource, Pdict } from '../interface';
+import { ProblemStatusDoc, Pdict } from '../interface';
 import { Content } from '../loader';
-import { NumberKeys, Projection } from '../typeutils';
+import { ArrayKeys, NumberKeys, Projection } from '../typeutils';
 import { ProblemNotFoundError } from '../error';
+import storage from '../service/storage';
 
 export interface Pdoc {
     _id: ObjectID
@@ -35,7 +37,7 @@ export namespace Pdoc {
         nAccept: 0,
         tag: [],
         category: [],
-        data: null,
+        data: [],
         hidden: true,
         config: {},
         acMsg: '',
@@ -77,11 +79,11 @@ export const PROJECTION_PUBLIC: Pdoc.Field[] = [
 
 export async function add(
     domainId: string, pid: string = null, title: string, content: Content, owner: number,
-    tag: string[] = [], category: string[] = [], data: ProblemDataSource = null, hidden = false,
+    tag: string[] = [], category: string[] = [], hidden = false,
 ) {
     const pidCounter = await domain.inc(domainId, 'pidCounter', 1);
     const args: Partial<Pdoc> = {
-        title, data, category, tag, hidden, nSubmit: 0, nAccept: 0,
+        title, category, tag, hidden, nSubmit: 0, nAccept: 0,
     };
     if (pid) args.pid = pid;
     return await document.add(domainId, content, owner, document.TYPE_PROBLEM, pidCounter, null, null, args);
@@ -118,6 +120,14 @@ export function edit(domainId: string, _id: number, $set: Partial<Pdoc>): Promis
     return document.set(domainId, document.TYPE_PROBLEM, _id, $set, delpid ? { pid: '' } : undefined);
 }
 
+export function push<T extends ArrayKeys<Pdoc>>(domainId: string, _id: number, key: ArrayKeys<Pdoc>, value: Pdoc[T][0]) {
+    return document.push(domainId, document.TYPE_PROBLEM, _id, key, value);
+}
+
+export function pull<T extends ArrayKeys<Pdoc>>(domainId: string, pid: number, key: ArrayKeys<Pdoc>, values: Pdoc[T][0][]) {
+    return document.pull(domainId, document.TYPE_PROBLEM, pid, key, values.map((_id) => ({ _id })));
+}
+
 export function inc(domainId: string, _id: number, field: NumberKeys<Pdoc>, n: number): Promise<Pdoc> {
     return document.inc(domainId, document.TYPE_PROBLEM, _id, field, n);
 }
@@ -126,6 +136,18 @@ export function count(domainId: string, query: FilterQuery<Pdoc>) {
     return document.count(domainId, document.TYPE_PROBLEM, query);
 }
 
+export async function addTestdata(domainId: string, pid: number, name: string, f: Readable | Buffer | string) {
+    await storage.put(`problem/${domainId}/${pid}/testdata/${name}`, f);
+    const meta = await storage.getMeta(`problem/${domainId}/${pid}/testdata/${name}`);
+    return await push(domainId, pid, 'data', meta);
+}
+
+export async function delTestdata(domainId: string, pid: number, name: string | string[]) {
+    const names = name instanceof Array ? name : [name];
+    await storage.del(names.map((t) => `problem/${domainId}/${pid}/testdata/${t}`));
+    await pull(domainId, pid, 'data', names);
+}
+
 export async function random(domainId: string, query: FilterQuery<Pdoc>): Promise<string | null> {
     const cursor = document.getMulti(domainId, document.TYPE_PROBLEM, query);
     const pcount = await cursor.count();
@@ -201,7 +223,11 @@ global.Hydro.model.problem = {
     get,
     edit,
     count,
+    push,
+    pull,
     random,
+    addTestdata,
+    delTestdata,
     getMulti,
     getList,
     getListStatus,

packages/hydrooj/src/upgrade.ts

@@ -132,6 +132,19 @@ const scripts: UpgradeScript[] = [
         logger.success('Files copied successfully. You can now remove collection `file` `fs.files` `fs.chunks` in the database.');
         return true;
     },
+    async function _4_5() {
+        const domains = await domain.getMulti().project({ _id: 1 }).toArray();
+        for (const d of domains) {
+            const bulk = document.coll.initializeUnorderedBulkOp();
+            const pdocs = await document.getMulti(d._id, document.TYPE_PROBLEM).project({ domainId: 1, docId: 1 }).toArray();
+            for (const pdoc of pdocs) {
+                const data = await storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`, true);
+                bulk.find({ _id: pdoc._id }).updateOne({ $set: { data } });
+            }
+            if (bulk.length) await bulk.execute();
+        }
+        return true;
+    },
 ];
 
 export = scripts;

packages/import-qduoj/handler.ts

@@ -100,7 +100,8 @@ class ImportQduojHandler extends Handler {
                     `problem/${domainId}/${pid}/testdata/config.yaml`,
                     Buffer.from(yaml.safeDump(config)),
                 );
-                await problem.edit(domainId, pid, { html: true });
+                const data = await storage.list(`problem/${domainId}/${pid}/testdata/`, true);
+                await problem.edit(domainId, pid, { html: true, data });
             }
         } finally {
             await fs.remove(tmp);

packages/migrate-hustoj/script.ts

@@ -96,7 +96,7 @@ export async function run({
                     hint: pdoc.hint,
                     source: pdoc.source,
                 }, 'html'),
-                1, pdoc.source.split(' '), undefined, undefined, pdoc.defunct === 'Y',
+                1, pdoc.source.split(' '), undefined, pdoc.defunct === 'Y',
             );
             const [cdoc] = await query(`SELECT * FROM 'privilege' WHERE rightstr = 'p${pdoc.problem_id}'`);
             await problem.edit('system', pid, {