core: script: add sync file list

packages/hydrooj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "hydrooj",
-    "version": "2.20.50",
+    "version": "2.20.51",
     "bin": "bin/hydrooj.js",
     "main": "dist/loader.js",
     "typings": "dist/loader.d.ts",

packages/hydrooj/src/entry/common.ts

@@ -30,7 +30,7 @@ export const builtinHandler = [
 ];
 
 export const builtinScript = [
-    'rating', 'problemStat', 'blacklist', 'deleteUser',
+    'rating', 'problemStat', 'blacklist', 'deleteUser', 'updateFilelist',
 ];
 
 function getFiles(folder: string, base = ''): string[] {

packages/hydrooj/src/script/updateFilelist.ts

@@ -0,0 +1,33 @@
+import storage from '../service/storage';
+import { iterateAllProblem, iterateAllProblemInDomain } from '../upgrade';
+
+export const description = 'Sync problem filelist from s3 service';
+
+export async function run({ domainId }, report: Function) {
+    const cb = async (pdoc) => {
+        report({ message: `${pdoc.domainId}/${pdoc.docId}` });
+        const [data, additional_file] = await Promise.all([
+            storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`),
+            storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/additional_files/`),
+        ]) as any;
+        for (let i = 0; i < data.length; i++) {
+            data[i]._id = data[i].name;
+        }
+        for (let i = 0; i < additional_file.length; i++) {
+            additional_file[i]._id = additional_file[i].name;
+        }
+        return { data, additional_file };
+    };
+    if (domainId) await iterateAllProblemInDomain(domainId, [], cb);
+    else await iterateAllProblem([], cb);
+    return true;
+}
+
+export const validate = {
+    $or: [
+        { domainId: 'string' },
+        { domainId: 'undefined' },
+    ],
+};
+
+global.Hydro.script.updateFilelist = { run, description, validate };

packages/hydrooj/src/upgrade.ts

@@ -20,7 +20,7 @@ import difficultyAlgorithm from './lib/difficulty';
 const logger = new Logger('upgrade');
 type UpgradeScript = () => Promise<boolean | void>;
 
-async function iterateAllDomain(cb: (ddoc: DomainDoc, current?: number, total?: number) => Promise<any>) {
+export async function iterateAllDomain(cb: (ddoc: DomainDoc, current?: number, total?: number) => Promise<any>) {
     const ddocs = await domain.getMulti().project({ _id: 1, owner: 1 }).toArray();
     for (const i in ddocs) await cb(ddocs[i], +i, ddocs.length);
 }
@@ -29,22 +29,30 @@ interface PartialPdoc extends problem.Pdoc {
     [key: string]: any,
 }
 
-async function iterateAllProblem(
+export async function iterateAllProblemInDomain(
+    domainId: string,
     fields: (problem.Pdoc.Field | string)[],
     cb: (pdoc: PartialPdoc, current?: number, total?: number) => Promise<any>,
 ) {
     if (!fields.includes('domainId')) fields.push('domainId');
     if (!fields.includes('docId')) fields.push('docId');
+    const cursor = problem.getMulti(domainId, {}, fields as any);
+    const total = await problem.getMulti(domainId, {}).count();
+    let i = 0;
+    while (await cursor.hasNext()) {
+        const doc = await cursor.next();
+        i++;
+        const res = await cb(doc, i, total);
+        if (res) await problem.edit(doc.domainId, doc.docId, res);
+    }
+}
+
+export async function iterateAllProblem(
+    fields: (problem.Pdoc.Field | string)[],
+    cb: (pdoc: PartialPdoc, current?: number, total?: number) => Promise<any>,
+) {
     await iterateAllDomain(async (d) => {
-        const cursor = problem.getMulti(d._id, {}, fields as any);
+        await iterateAllProblemInDomain(d._id, fields, cb);
-        const total = await problem.getMulti(d._id, {}).count();
-        let i = 0;
-        while (await cursor.hasNext()) {
-            const doc = await cursor.next();
-            i++;
-            const res = await cb(doc, i, total);
-            if (res) await problem.edit(doc.domainId, doc.docId, res);
-        }
     });
 }
 
@@ -245,6 +253,20 @@ const scripts: UpgradeScript[] = [
         });
         return true;
     },
+    async function _16_18() {
+        await iterateAllProblem(['title', 'pid'], async (pdoc) => {
+            if (pdoc.domainId !== 'bzoj') return;
+            if (pdoc.docId > 100000) return;
+            console.log(pdoc.title, pdoc.docId, pdoc.pid);
+            await problem.edit(pdoc.domainId, pdoc.docId, { docId: pdoc.docId + 100000 });
+        });
+        await iterateAllProblem(['pid', 'title'], async (pdoc) => {
+            if (pdoc.domainId !== 'bzoj') return;
+            console.log(pdoc.title, pdoc.docId, pdoc.pid);
+            if (pdoc.pid.startsWith('P')) pdoc.pid = pdoc.pid.split('P')[1];
+            await problem.edit(pdoc.domainId, pdoc.docId, { docId: +pdoc.pid });
+        });
+    },
 ];
 
-export = scripts;
+export default scripts;