core: schemastery validator

packages/elastic/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hydrooj/elastic-search",
-    "version": "1.0.5",
+    "version": "1.0.6",
     "main": "package.json",
     "repository": "https://github.com/hydro-dev/Hydro",
     "author": "undefined <i@undefined.moe>",

packages/elastic/script.ts

@@ -1,5 +1,6 @@
 import { Client } from '@elastic/elasticsearch';
 import { omit } from 'lodash';
+import { addScript, Schema } from 'hydrooj';
 import DomainModel from 'hydrooj/src/model/domain';
 import ProblemModel from 'hydrooj/src/model/problem';
 import * as system from 'hydrooj/src/model/system';
@@ -61,9 +62,7 @@ global.Hydro.lib.problemSearch = async (domainId, q, opts) => {
     };
 };
 
-export const description = 'Elastic problem search re-index';
+async function run({ domainId }, report) {
-
-export async function run({ domainId }, report) {
     try {
         if (domainId) await client.deleteByQuery({ index: 'problem', query: { match: { domainId } } });
         else await client.deleteByQuery({ index: 'problem', query: { match_all: {} } });
@@ -86,11 +85,8 @@ export async function run({ domainId }, report) {
     return true;
 }
 
-export const validate = {
+addScript('ensureElasticSearch', 'Elastic problem search re-index')
-    $or: [
+    .args(Schema.object({
-        { domainId: 'string' },
+        domainId: Schema.string(),
-        { domainId: 'undefined' },
+    }))
-    ],
+    .action(run);
-};
-
-global.Hydro.script.ensureElasticSearch = { run, description, validate };

packages/hydrooj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "hydrooj",
-    "version": "3.14.23",
+    "version": "3.15.0",
     "bin": "bin/hydrooj.js",
     "main": "src/loader",
     "module": "src/loader",

packages/hydrooj/src/handler/manage.ts

@@ -105,12 +105,17 @@ class SystemScriptHandler extends SystemHandler {
     @param('args', Types.Content, true)
     async post(domainId: string, id: string, raw = '{}') {
         if (!global.Hydro.script[id]) throw new ValidationError('id');
-        const args = JSON.parse(raw);
+        let args = JSON.parse(raw);
+        if (typeof global.Hydro.script[id].validate === 'function') {
+            args = global.Hydro.script[id].validate(args);
+        } else {
+            logger.warn('You are using the legacy script validation API, which will be dropped in the future.');
             validate(global.Hydro.script[id].validate, args);
+        }
         const rid = await record.add(domainId, -1, this.user._id, '-', id, false, raw);
         const report = (data) => judge.next({ domainId, rid, ...data });
         report({ message: `Running script: ${id} `, status: STATUS.STATUS_JUDGING });
-        const start = new Date().getTime();
+        const start = Date.now();
         // Maybe async?
         global.Hydro.script[id].run(args, report)
             .then((ret: any) => {

packages/hydrooj/src/loader.ts

@@ -3,6 +3,7 @@
 /* eslint-disable no-eval */
 import './init';
 import './interface';
+import Schema from 'schemastery';
 import path from 'path';
 import fs from 'fs-extra';
 import './utils';
@@ -10,7 +11,10 @@ import cac from 'cac';
 import { Logger } from './logger';
 import './ui';
 
+// This is the main entry. So let's re-export some modules.
 export * from './interface';
+export { Schema, Logger };
+
 const argv = cac().parse();
 if (argv.options.debug) {
     console.log('Debug mode enabled');
@@ -53,6 +57,19 @@ export function addon(addonPath: string, prepend = false) {
     } else logger.error(`Addon not found: ${addonPath}`);
 }
 
+export function addScript(name: string, description: string) {
+    if (global.Hydro.script[name]) throw new Error(`duplicate script ${name} registered.`);
+    return {
+        args: <K extends Schema>(validate: K) => ({
+            action: (run: (args: ReturnType<K>, report: any) => boolean | Promise<boolean>) => {
+                global.Hydro.script[name] = {
+                    description, validate, run,
+                };
+            },
+        }),
+    };
+}
+
 export async function load() {
     addon(path.resolve(__dirname, '..'), true);
     Error.stackTraceLimit = 50;

packages/hydrooj/src/script/blacklist.ts

@@ -1,13 +1,13 @@
 /* eslint-disable @typescript-eslint/no-use-before-define */
 import { ObjectID } from 'mongodb';
+import Schema from 'schemastery';
+import { addScript } from '../loader';
 import blacklist from '../model/blacklist';
 import * as discussion from '../model/discussion';
 import * as document from '../model/document';
-import user from '../model/user';
+import UserModel from '../model/user';
 import db from '../service/db';
 
-export const description = 'Add blacklist by ip, uid';
-
 async function _address(
     ip: string,
     bset: Set<string>, uset: Set<number>, dset: Set<ObjectID>,
@@ -47,7 +47,7 @@ async function _user(
 ) {
     if (uset.has(uid)) return;
     uset.add(uid);
-    const udoc = await user.getById('system', uid);
+    const udoc = await UserModel.getById('system', uid);
     if (!udoc) return;
     report({ message: `user ${udoc._id} ${udoc.uname}` });
     await _address(udoc._loginip, bset, uset, dset, dryrun, report);
@@ -58,13 +58,22 @@ async function _user(
         tasks.push(_discussion(ddoc.domainId, ddoc.docId, bset, uset, dset, dryrun, report));
     }
     await Promise.all(tasks);
-    if (!dryrun) await user.ban(uid);
+    if (!dryrun) await UserModel.ban(uid);
 }
 
-export async function run({
+addScript('blacklist', 'Add blacklist by ip, uid')
-    // eslint-disable-next-line @typescript-eslint/no-shadow
+    .args(Schema.object({
+        address: Schema.string(),
+        discuss: Schema.object({
+            domainId: Schema.string(),
+            did: Schema.string(),
+        }),
+        user: Schema.number(),
+        dryrun: Schema.boolean(),
+    }))
+    .action(async ({
         address = null, discuss = null, user = null, dryrun = true,
-}, report) {
+    }, report) => {
         if (address) await _address(address, new Set(), new Set(), new Set(), dryrun, report);
         if (discuss) {
             await _discussion(
@@ -73,13 +82,5 @@ export async function run({
             );
         }
         if (user) await _user(user, new Set(), new Set(), new Set(), dryrun, report);
-}
+        return true;
-
+    });
-export const validate = {
-    address: 'string?',
-    discuss: 'string?',
-    user: 'number?',
-    dryrun: 'boolean?',
-};
-
-global.Hydro.script.blacklist = { run, description, validate };

packages/hydrooj/src/script/checkUpdate.ts

@@ -1,11 +1,11 @@
 /* eslint-disable no-await-in-loop */
+import Schema from 'schemastery';
 import * as semver from 'semver';
 import superagent from 'superagent';
+import { addScript } from '../loader';
 import MessageModel from '../model/message';
 import * as system from '../model/system';
 
-export const description = 'Daily update check';
-
 async function getRemoteVersion(id: string) {
     try {
         const res = await superagent.get(`https://registry.yarnpkg.com/${id}`);
@@ -38,6 +38,4 @@ export async function run(_: {}, report: Function) {
     return true;
 }
 
-export const validate = {};
+addScript('checkUpdate', 'Daily update check').args(Schema.any()).action(run);
-
-global.Hydro.script.checkUpdate = { run, description, validate };

packages/hydrooj/src/script/deleteUser.ts

@@ -1,25 +1,23 @@
+import Schema from 'schemastery';
+import { addScript } from '../loader';
 import { PRIV } from '../model/builtin';
 import user from '../model/user';
 import db from '../service/db';
 
-export const description = 'Delete a user';
-
 const collDocument = db.collection('document');
 const collDomainUser = db.collection('domain.user');
 const collRecord = db.collection('record');
 const collMessage = db.collection('message');
 
-export async function run({ uid }) {
+addScript('deleteUser', 'Delete a user')
+    .args(Schema.object({
+        uid: Schema.number(),
+    }))
+    .action(async ({ uid }) => {
         await collDocument.deleteMany({ owner: uid });
         await collRecord.deleteMany({ uid });
         await collDomainUser.deleteMany({ uid });
         await collMessage.deleteMany({ $or: [{ from: uid }, { to: uid }] });
         await user.setPriv(uid, PRIV.PRIV_NONE);
-    return uid;
+        return true;
-}
+    });
-
-export const validate = {
-    uid: 'number',
-};
-
-global.Hydro.script.deleteUser = { run, description, validate };

packages/hydrooj/src/script/problemStat.ts

@@ -1,12 +1,12 @@
 /* eslint-disable no-constant-condition */
 /* eslint-disable no-await-in-loop */
 import { ObjectID } from 'mongodb';
+import Schema from 'schemastery';
+import { addScript } from '../loader';
 import { STATUS } from '../model/builtin';
 import * as document from '../model/document';
 import db from '../service/db';
 
-export const description = 'Recalculates nSubmit and nAccept in problem status.';
-
 const sumStatus = (status) => ({ $sum: { $cond: [{ $eq: ['$status', status] }, 1, 0] } });
 const $match = { contest: { $ne: new ObjectID('000000000000000000000000') } };
 
@@ -144,17 +144,15 @@ export async function pdoc(report) {
     if (bulk.length) await bulk.execute();
 }
 
-export async function run(arg, report) {
+addScript('problemStat', 'Recalculates nSubmit and nAccept in problem status.')
+    .args(Schema.object({
+        udoc: Schema.boolean(),
+        pdoc: Schema.boolean(),
+        psdoc: Schema.boolean(),
+    }))
+    .action(async (arg, report) => {
         if (arg.pdoc === undefined || arg.pdoc) await pdoc(report);
         if (arg.udoc === undefined || arg.udoc) await udoc(report);
         if (arg.psdoc === undefined || arg.psdoc) await psdoc(report);
         return true;
-}
+    });
-
-export const validate = {
-    udoc: 'boolean?',
-    pdoc: 'boolean?',
-    psdoc: 'boolean?',
-};
-
-global.Hydro.script.problemStat = { run, description, validate };

packages/hydrooj/src/script/rating.ts

@@ -2,9 +2,11 @@
 /* eslint-disable no-await-in-loop */
 import { NumericDictionary, unionWith } from 'lodash';
 import { FilterQuery } from 'mongodb';
+import Schema from 'schemastery';
 import { Tdoc, Udoc } from '../interface';
 import difficultyAlgorithm from '../lib/difficulty';
 import rating from '../lib/rating';
+import { addScript } from '../loader';
 import { PRIV, STATUS } from '../model/builtin';
 import * as contest from '../model/contest';
 import domain from '../model/domain';
@@ -182,8 +184,8 @@ export async function run({ domainId }, report: Function) {
     return true;
 }
 
-export const validate = {
+addScript('rp', 'Calculate rp of a domain, or all domains')
-    domainId: 'string?',
+    .args(Schema.object({
-};
+        domainId: Schema.string(),
-
+    }))
-global.Hydro.script.rp = { run, description, validate };
+    .action(run);

packages/hydrooj/src/script/storageUsage.ts

@@ -1,10 +1,10 @@
 /* eslint-disable no-await-in-loop */
+import Schema from 'schemastery';
+import { addScript } from '../loader';
 import { STATUS } from '../model/builtin';
 import storage from '../model/storage';
 
-export const description = 'Calculate storage usage';
+export async function run(_, report) {
-
-export async function run(_: {}, report: Function) {
     const cursor = storage.coll.find({ path: { $regex: /^problem\//i } }).sort({ path: 1 });
     const count = await cursor.count();
     report({ message: `Total ${count} files` });
@@ -34,8 +34,9 @@ export async function run(_: {}, report: Function) {
         }
         memory += doc.size || 0;
     }
+    return true;
 }
 
-export const validate = {};
+addScript('storageUsage', 'Calculate storage usage')
-
+    .args(Schema.any())
-global.Hydro.script.storageUsage = { run, description, validate };
+    .action(run);

packages/meilisearch/script.ts

@@ -1,6 +1,7 @@
 /* eslint-disable no-await-in-loop */
 import { omit } from 'lodash';
 import { Index, MeiliSearch } from 'meilisearch';
+import { addScript, Schema } from 'hydrooj';
 import DomainModel from 'hydrooj/src/model/domain';
 import ProblemModel from 'hydrooj/src/model/problem';
 import * as system from 'hydrooj/src/model/system';
@@ -50,15 +51,6 @@ global.Hydro.lib.problemSearch = async (domainId, q, opts) => {
     };
 };
 
-export const description = 'Meilisearch reindex';
-
-export const validate = {
-    $or: [
-        { domainId: 'string' },
-        { domainId: 'undefined' },
-    ],
-};
-
 async function waitForTask(id: number) {
     // eslint-disable-next-line no-constant-condition
     while (true) {
@@ -109,4 +101,8 @@ export async function run(_, report) {
     return true;
 }
 
-global.Hydro.script.ensureMeiliSearch = { run, description, validate };
+addScript('ensureMeiliSearch', 'Meilisearch reindex')
+    .args(Schema.object({
+        domainId: Schema.string(),
+    }))
+    .action(run);

packages/migrate/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hydrooj/migrate",
-    "version": "0.1.2",
+    "version": "0.1.3",
     "main": "package.json",
     "repository": "git@github.com:hydro-dev/Hydro.git",
     "author": "undefined <i@undefined.moe>",

packages/migrate/scripts/hustoj.ts

@@ -6,6 +6,7 @@ import mysql from 'mysql';
 import TurndownService from 'turndown';
 import { STATUS } from '@hydrooj/utils/lib/status';
 import { noop, Time } from '@hydrooj/utils/lib/utils';
+import { addScript, Schema } from 'hydrooj';
 import { NotFoundError } from 'hydrooj/src/error';
 import { postJudge } from 'hydrooj/src/handler/judge';
 import { RecordDoc } from 'hydrooj/src/interface';
@@ -310,16 +311,14 @@ export async function run({
     return true;
 }
 
-export const description = 'migrate from hustoj';
+addScript('migrateHustoj', 'mirate from hustoj')
-export const validate = {
+    .args(Schema.object({
-    host: 'string',
+        host: Schema.string().required(),
-    port: 'number',
+        port: Schema.number().required(),
-    name: 'string',
+        name: Schema.string().required(),
-    username: 'string',
+        username: Schema.string().required(),
-    password: 'string',
+        password: Schema.string().required(),
-    domainId: 'string',
+        domainId: Schema.string().required(),
-    contestType: 'string',
+        contestType: Schema.string().required(),
-    dataDir: 'string',
+        dataDir: Schema.string().required(),
-};
+    }));
-
-global.Hydro.script.migrateHustoj = { run, description, validate };

packages/migrate/scripts/vijos.ts

@@ -2,7 +2,8 @@
 /* eslint-disable no-await-in-loop */
 import mongodb, { Cursor, Db } from 'mongodb';
 import {
-    DiscussionTailReplyDoc, MessageDoc, RecordDoc, TestCase, TrainingNode,
+    addScript, DiscussionTailReplyDoc, MessageDoc,
+    RecordDoc, Schema, TestCase, TrainingNode,
 } from 'hydrooj';
 
 const dst = global.Hydro.service.db;
@@ -506,9 +507,11 @@ export async function run({
     return true;
 }
 
-export const description = 'migrate from vijos';
+addScript('migrateVijos', 'mirate from vijos')
-export const validate = {
+    .args(Schema.object({
-    host: 'string', port: 'number', name: 'string', username: 'string', password: 'string',
+        host: Schema.string().required(),
-};
+        port: Schema.number().required(),
-
+        name: Schema.string().required(),
-global.Hydro.script.migrateVijos = { run, description, validate };
+        username: Schema.string().required(),
+        password: Schema.string().required(),
+    }));

packages/sonic/package.json

@@ -1,9 +1,8 @@
 {
     "name": "@hydrooj/sonic",
-    "version": "1.2.6",
+    "version": "1.2.7",
     "description": "Sonic search service",
-    "main": "service.js",
+    "main": "service.ts",
-    "typings": "service.d.ts",
     "preferUnplugged": true,
     "repository": "https://github.com/hydro-dev/Hydro.git",
     "author": "undefined <i@undefined.moe>",

packages/sonic/script.ts

@@ -1,10 +1,9 @@
+import { addScript, Schema } from 'hydrooj';
 import DomainModel from 'hydrooj/src/model/domain';
 import { iterateAllProblem, iterateAllProblemInDomain } from 'hydrooj/src/pipelineUtils';
 import sonic from './service';
 
-export const description = 'Sonic problem search re-index';
+async function run({ domainId }, report) {
-
-export async function run({ domainId }, report) {
     if (domainId) await sonic.flushb('problem', domainId);
     else await sonic.flushc('problem');
     let i = 0;
@@ -29,11 +28,8 @@ export async function run({ domainId }, report) {
     return true;
 }
 
-export const validate = {
+addScript('ensureSonicSearch', 'Sonic problem search re-index')
-    $or: [
+    .args(Schema.object({
-        { domainId: 'string' },
+        domainId: Schema.string(),
-        { domainId: 'undefined' },
+    }))
-    ],
+    .action(run);
-};
-
-global.Hydro.script.ensureSonicSearch = { run, description, validate };