core&ui&elastic: support elastic search and allow search result pagination

.eslintrc.yaml

@@ -1,5 +1,6 @@
 root: true
 extends:
+  - airbnb
   - airbnb-typescript/base
 env:
   jquery: true
@@ -16,6 +17,8 @@ ignorePatterns:
   - public/
   - '*.spec.ts'
 rules:
+  function-call-argument-newline: 0
+  react/function-component-definition: 0
   '@typescript-eslint/dot-notation': 0
   '@typescript-eslint/no-implied-eval': 0
   '@typescript-eslint/no-throw-literal': 0

packages/elastic/package.json

@@ -0,0 +1,11 @@
+{
+    "name": "@hydrooj/elastic-search",
+    "version": "1.0.0",
+    "main": "package.json",
+    "repository": "https://github.com/hydro-dev/Hydro",
+    "author": "undefined <i@undefined.moe>",
+    "license": "AGPL-3.0-or-later",
+    "dependencies": {
+        "@elastic/elasticsearch": "^8.1.0"
+    }
+}

packages/elastic/script.ts

@@ -0,0 +1,95 @@
+import { Client } from '@elastic/elasticsearch';
+import { omit } from 'lodash';
+import DomainModel from 'hydrooj/src/model/domain';
+import ProblemModel from 'hydrooj/src/model/problem';
+import * as system from 'hydrooj/src/model/system';
+import { iterateAllProblem, iterateAllProblemInDomain } from 'hydrooj/src/pipelineUtils';
+import * as bus from 'hydrooj/src/service/bus';
+
+const client = new Client({ node: system.get('elasticsearch.url') || 'http://192.168.1.82:9200' });
+
+const indexOmit = ['_id', 'docType', 'data', 'additional_file', 'config', 'stats', 'assign'];
+
+bus.on('problem/add', async (doc, docId) => {
+    await client.index({
+        index: 'problem',
+        id: `${doc.domainId}/${docId}`,
+        document: omit(doc, indexOmit),
+    });
+});
+bus.on('problem/edit', async (pdoc) => {
+    await client.index({
+        index: 'problem',
+        id: `${pdoc.domainId}/${pdoc.docId}`,
+        document: omit(pdoc, indexOmit),
+    });
+});
+bus.on('problem/del', async (domainId, docId) => {
+    await client.delete({
+        index: 'problem',
+        id: `${domainId}/${docId}`,
+    });
+});
+
+global.Hydro.lib.problemSearch = async (domainId, q, opts) => {
+    const size = opts?.limit || system.get('pagination.problem');
+    const from = opts?.skip || 0;
+    const union = await DomainModel.getUnion(domainId);
+    const domainIds = [domainId, ...(union?.union || [])];
+    const res = await client.search({
+        index: 'problem',
+        size,
+        from,
+        query: {
+            simple_query_string: {
+                query: q,
+                fields: ['tag^5', 'title^3', 'content'],
+            },
+        },
+        post_filter: {
+            bool: {
+                minimum_should_match: 1,
+                should: domainIds.map((i) => ({ match: { domainId: i } })),
+            },
+        },
+    });
+    return {
+        countRelation: typeof res.hits.total === 'number' ? 'eq' : res.hits.total.relation,
+        total: typeof res.hits.total === 'number' ? res.hits.total : res.hits.total.value,
+        hits: res.hits.hits.map((i) => i._id),
+    };
+};
+
+export const description = 'Elastic problem search re-index';
+
+export async function run({ domainId }, report) {
+    try {
+        if (domainId) await client.deleteByQuery({ index: 'problem', query: { match: { domainId } } });
+        else await client.deleteByQuery({ index: 'problem', query: { match_all: {} } });
+    } catch (e) {
+        if (!e.message.includes('index_not_found_exception')) throw e;
+    }
+    let i = 0;
+    const cb = async (pdoc) => {
+        i++;
+        if (!(i % 1000)) report({ message: `${i} problems indexed` });
+        await client.index({
+            index: 'problem',
+            id: `${pdoc.domainId}/${pdoc.docId}`,
+            document: omit(pdoc, indexOmit),
+        });
+    };
+    if (domainId) await iterateAllProblemInDomain(domainId, ProblemModel.PROJECTION_PUBLIC, cb);
+    else await iterateAllProblem(ProblemModel.PROJECTION_PUBLIC, cb);
+    await client.indices.refresh({ index: 'problem' });
+    return true;
+}
+
+export const validate = {
+    $or: [
+        { domainId: 'string' },
+        { domainId: 'undefined' },
+    ],
+};
+
+global.Hydro.script.ensureElasticSearch = { run, description, validate };

packages/hydrojudge/src/hosts/index.ts

@@ -1,2 +1,3 @@
 export { default as vj4 } from './vj4';
+// eslint-disable-next-line no-restricted-exports
 export { default as hydro, default } from './hydro';

packages/hydrojudge/src/log.ts

@@ -68,6 +68,7 @@ export class Logger {
     public stream: NodeJS.WritableStream = process.stderr;
 
     constructor(public name: string, private showDiff = false) {
+        // eslint-disable-next-line no-constructor-return
         if (name in instances) return instances[name];
         let hash = 0;
         for (let i = 0; i < name.length; i++) {

packages/hydrooj/bin/commands.ts

@@ -0,0 +1,129 @@
+import child from 'child_process';
+import os from 'os';
+import path from 'path';
+import fs from 'fs-extra';
+import arg from '@hydrooj/utils/lib/arg';
+
+const argv = arg();
+
+const exec = (...args) => {
+    console.log('Executing: ', args[0], args[1].join(' '));
+    const res = child.spawnSync(...args);
+    if (res.error) throw res.error;
+    if (res.status) throw new Error(`Error: Exited with code ${res.status}`);
+    return res;
+};
+
+function buildUrl(opts) {
+    let mongourl = `${opts.protocol || 'mongodb'}://`;
+    if (opts.username) mongourl += `${opts.username}:${opts.password}@`;
+    mongourl += `${opts.host}:${opts.port}/${opts.name}`;
+    if (opts.url) mongourl = opts.url;
+    return mongourl;
+}
+
+const hydroPath = path.resolve(os.homedir(), '.hydro');
+fs.ensureDirSync(hydroPath);
+const addonPath = path.resolve(hydroPath, 'addon.json');
+if (!fs.existsSync(addonPath)) fs.writeFileSync(addonPath, '[]');
+let addons = JSON.parse(fs.readFileSync(addonPath).toString());
+
+if (argv._[0] === 'db') {
+    const dbConfig = fs.readFileSync(path.resolve(hydroPath, 'config.json'), 'utf-8');
+    const url = buildUrl(JSON.parse(dbConfig));
+    child.spawn('mongo', [url], { stdio: 'inherit' });
+    process.exit(0);
+}
+
+if (argv._[0] === 'backup') {
+    const dbConfig = fs.readFileSync(path.resolve(hydroPath, 'config.json'), 'utf-8');
+    const url = buildUrl(JSON.parse(dbConfig));
+    const dir = `${os.tmpdir()}/${Math.random().toString(36).substring(2)}`;
+    exec('mongodump', [url, `--out=${dir}/dump`], { stdio: 'inherit' });
+    const env = `${os.homedir()}/.hydro/env`;
+    if (fs.existsSync(env)) fs.copySync(env, `${dir}/env`);
+    const target = `${process.cwd()}/backup-${new Date().toISOString().replace(':', '-').split(':')[0]}.zip`;
+    exec('zip', ['-r', target, 'dump'], { cwd: dir, stdio: 'inherit' });
+    if (!argv.dbOnly) {
+        exec('zip', ['-r', target, 'file'], { cwd: '/data', stdio: 'inherit' });
+    }
+    exec('rm', ['-rf', dir]);
+    console.log(`Database backup saved at ${target}`);
+    process.exit(0);
+}
+
+if (argv._[0] === 'restore') {
+    const dbConfig = fs.readFileSync(path.resolve(hydroPath, 'config.json'), 'utf-8');
+    const url = buildUrl(JSON.parse(dbConfig));
+    const dir = `${os.tmpdir()}/${Math.random().toString(36).substring(2)}`;
+    if (!fs.existsSync(argv._[1])) {
+        console.error('Cannot find file');
+        process.exit(1);
+    }
+    exec('unzip', [argv._[1], '-d', dir], { stdio: 'inherit' });
+    exec('mongorestore', [`--uri=${url}`, `--dir=${dir}/dump/${JSON.parse(dbConfig).name}`, '--drop'], { stdio: 'inherit' });
+    if (fs.existsSync(`${dir}/file`)) {
+        exec('rm', ['-rf', '/data/file/*'], { stdio: 'inherit' });
+        exec('bash', ['-c', `mv ${dir}/file/* /data/file`], { stdio: 'inherit' });
+    }
+    if (fs.existsSync(`${dir}/env`)) {
+        fs.copySync(`${dir}/env`, `${os.homedir()}/.hydro/env`, { overwrite: true });
+    }
+    fs.removeSync(dir);
+    console.log('Successfully restored.');
+    process.exit(0);
+}
+
+if (!addons.includes('@hydrooj/ui-default')) {
+    try {
+        const ui = argv.ui as string || '@hydrooj/ui-default';
+        require.resolve(ui);
+        addons.push(ui);
+    } catch (e) {
+        console.error('Please also install @hydrooj/ui-default');
+    }
+}
+
+if (argv._[0] && argv._[0] !== 'cli') {
+    const operation = argv._[0];
+    const arg1 = argv._[1];
+    const arg2 = argv._[2];
+    if (operation === 'addon') {
+        if (arg1 === 'create') {
+            fs.mkdirSync('/root/addon');
+            child.execSync('yarn init -y', { cwd: '/root/addon' });
+            fs.mkdirSync('/root/addon/templates');
+            fs.mkdirSync('/root/addon/locales');
+            fs.mkdirSync('/root/addon/public');
+            addons.push('/root/addon');
+        } else if (arg1 === 'add') {
+            for (let i = 0; i < addons.length; i++) {
+                if (addons[i] === arg2) {
+                    addons.splice(i, 1);
+                    break;
+                }
+            }
+            addons.push(arg2);
+        } else if (arg1 === 'remove') {
+            for (let i = 0; i < addons.length; i++) {
+                if (addons[i] === arg2) {
+                    addons.splice(i, 1);
+                    break;
+                }
+            }
+        }
+        addons = Array.from(new Set(addons));
+        console.log('Current Addons: ', addons);
+        fs.writeFileSync(addonPath, JSON.stringify(addons, null, 2));
+        process.exit(0);
+    }
+    console.error('Unknown command: ', argv._[0]);
+} else {
+    const hydro = require('../src/loader');
+    addons = Array.from(new Set(addons));
+    for (const addon of addons) hydro.addon(addon);
+    (argv._[0] === 'cli' ? hydro.loadCli : hydro.load)().catch((e) => {
+        console.error(e);
+        process.exit(1);
+    });
+}

packages/hydrooj/package.json

@@ -1,6 +1,6 @@
 {
     "name": "hydrooj",
-    "version": "3.9.7",
+    "version": "3.10.0",
     "bin": "bin/hydrooj.js",
     "main": "src/loader",
     "module": "src/loader",

packages/hydrooj/src/entry/cli.ts

@@ -7,9 +7,7 @@ import options from '../options';
 import * as bus from '../service/bus';
 import db from '../service/db';
 import {
-    builtinLib, builtinModel, builtinScript,
-    lib, model, script,
-    service,
+    lib, model, script, service,
 } from './common';
 
 const argv = cac().parse();
@@ -92,14 +90,14 @@ export async function load() {
     await db.start(opts);
     const storage = require('../service/storage');
     await storage.start();
-    for (const i of builtinLib) require(`../lib/${i}`);
+    require('../lib/index');
     await lib(pending, fail);
     const systemModel = require('../model/system');
     await systemModel.runConfig();
     await service(pending, fail);
-    for (const i of builtinModel) require(`../model/${i}`);
+    require('../model/index');
     await model(pending, fail);
-    for (const i of builtinScript) require(`../script/${i}`);
+    require('../script/index');
     await script(pending, fail, []);
     await bus.parallel('app/started');
     await cli();

packages/hydrooj/src/entry/common.ts

@@ -1,3 +1,4 @@
+/* eslint-disable import/no-dynamic-require */
 /* eslint-disable no-await-in-loop */
 /* eslint-disable no-eval */
 import os from 'os';
@@ -10,31 +11,6 @@ import * as bus from '../service/bus';
 
 const logger = new Logger('common');
 
-export const builtinLib = [
-    'jwt', 'download', 'i18n', 'mail', 'useragent',
-    'crypto', 'misc', 'paginate', 'hash.hydro', 'rank',
-    'validator', 'ui', 'testdataConfig', 'difficulty', 'content',
-    'avatar',
-];
-
-export const builtinModel = [
-    'builtin', 'document', 'domain', 'blacklist', 'opcount',
-    'setting', 'token', 'user', 'storage', 'problem',
-    'record', 'contest', 'message', 'solution', 'training',
-    'discussion', 'system', 'oplog', 'blog',
-];
-
-export const builtinHandler = [
-    'home', 'problem', 'record', 'judge', 'user',
-    'contest', 'training', 'discussion', 'manage', 'import',
-    'misc', 'homework', 'domain', 'status', 'api', 'blog',
-];
-
-export const builtinScript = [
-    'rating', 'problemStat', 'blacklist', 'deleteUser', 'storageUsage',
-    'checkUpdate',
-];
-
 function getFiles(folder: string, base = ''): string[] {
     const files = [];
     const f = fs.readdirSync(folder);
@@ -53,7 +29,7 @@ export async function handler(pending: string[], fail: string[]) {
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 logger.info('Handler init: %s', i);
-                eval('require')(p);
+                require(p);
             } catch (e) {
                 fail.push(i);
                 logger.error('Handler Load Fail: %s', i);
@@ -136,7 +112,7 @@ export async function template(pending: string[], fail: string[]) {
             try {
                 const files = getFiles(p);
                 for (const file of files) {
-                    if (file.endsWith('.tsx')) global.Hydro.ui.template[file] = eval('require')(path.resolve(p, file));
+                    if (file.endsWith('.tsx')) global.Hydro.ui.template[file] = require(path.resolve(p, file));
                     global.Hydro.ui.template[file] = await fs.readFile(path.resolve(p, file), 'utf-8');
                 }
                 logger.info('Template init: %s', i);
@@ -157,7 +133,7 @@ export async function model(pending: string[], fail: string[]) {
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 logger.info('Model init: %s', i);
-                eval('require')(p);
+                require(p);
             } catch (e) {
                 fail.push(i);
                 logger.error('Model Load Fail: %s', i);
@@ -175,7 +151,7 @@ export async function lib(pending: string[], fail: string[]) {
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 logger.info('Lib init: %s', i);
-                eval('require')(p);
+                require(p);
             } catch (e) {
                 fail.push(i);
                 logger.error('Lib Load Fail: %s', i);
@@ -193,7 +169,7 @@ export async function service(pending: string[], fail: string[]) {
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 logger.info('Service init: %s', i);
-                eval('require')(p);
+                require(p);
             } catch (e) {
                 fail.push(i);
                 logger.error('Service Load Fail: %s', i);
@@ -215,7 +191,7 @@ export async function script(pending: string[], fail: string[], active: string[]
         if (await fs.pathExists(p) && !fail.includes(i)) {
             try {
                 logger.info('Script init: %s', i);
-                eval('require')(p);
+                require(p);
             } catch (e) {
                 fail.push(i);
                 logger.error('Script Load Fail: %s', i);

packages/hydrooj/src/entry/worker.ts

@@ -9,8 +9,7 @@ import options from '../options';
 import * as bus from '../service/bus';
 import db from '../service/db';
 import {
-    builtinHandler, builtinLib, builtinModel,
-    builtinScript, handler, lib, locale, model, script, service, setting, template,
+    handler, lib, locale, model, script, service, setting, template,
 } from './common';
 
 const logger = new Logger('worker');
@@ -45,15 +44,7 @@ export async function load() {
     const storage = require('../service/storage');
     await storage.start();
     if (detail) logger.info('finish: storage.connect');
-    for (const i of builtinLib) {
-        let t;
-        try {
-            t = require.resolve(`../lib/${i}`);
-        } catch (e) {
-            t = require.resolve(`@hydrooj/utils/lib/${i}`);
-        }
-        require(t);
-    }
+    require('../lib/index');
     if (detail) logger.info('finish: lib.builtin');
     await lib(pending, fail);
     if (detail) logger.info('finish: lib.extra');
@@ -64,9 +55,9 @@ export async function load() {
     if (detail) logger.info('finish: server');
     await service(pending, fail);
     if (detail) logger.info('finish: service.extra');
-    for (const i of builtinModel) require(`../model/${i}`);
+    require('../model/index');
     if (detail) logger.info('finish: model.builtin');
-    for (const i of builtinHandler) require(`../handler/${i}`);
+    require('../handler/index');
     if (detail) logger.info('finish: handler.builtin');
     await model(pending, fail);
     if (detail) logger.info('finish: model.extra');
@@ -79,7 +70,7 @@ export async function load() {
     if (detail) logger.info('finish: handler.apply');
     const notfound = require('../handler/notfound');
     await notfound.apply();
-    for (const i of builtinScript) require(`../script/${i}`);
+    require('../script/index');
     if (detail) logger.info('finish: script.builtin');
     await script(pending, fail, active);
     if (detail) logger.info('finish: script.extra');

packages/hydrooj/src/handler/index.ts

@@ -0,0 +1,16 @@
+import './home';
+import './problem';
+import './record';
+import './judge';
+import './user';
+import './contest';
+import './training';
+import './discussion';
+import './manage';
+import './import';
+import './misc';
+import './homework';
+import './domain';
+import './status';
+import './api';
+import './blog';

packages/hydrooj/src/handler/problem.ts

@@ -114,15 +114,15 @@ export class ProblemHandler extends Handler {
     async cleanup() {
         if (this.response.template === 'problem_main.html' && this.request.json) {
             const {
-                page, pcount, ppcount, pdocs, psdict, category,
+                page, pcount, pcountRelation, ppcount, pdocs, psdict, category, qs,
             } = this.response.body;
             this.response.body = {
                 title: this.renderTitle(this.translate('problem_main')),
                 fragments: (await Promise.all([
                     this.renderHTML('partials/problem_list.html', {
-                        page, ppcount, pcount, pdocs, psdict,
+                        page, ppcount, pcount, pdocs, psdict, qs,
                     }),
-                    this.renderHTML('partials/problem_stat.html', { pcount }),
+                    this.renderHTML('partials/problem_stat.html', { pcount, pcountRelation }),
                     this.renderHTML('partials/problem_lucky.html', { category }),
                 ])).map((i) => ({ html: i })),
                 raw: {
@@ -145,21 +145,25 @@ export class ProblemMainHandler extends ProblemHandler {
         const search = global.Hydro.lib.problemSearch;
         let sort: string[];
         let fail = false;
+        let pcountRelation = 'eq';
         if (category.length) query.$and = category.map((tag) => ({ tag }));
         if (q) category.push(q);
         if (category.length) this.extraTitleContent = category.join(',');
+        let total = 0;
         if (q) {
             if (search) {
-                const result = await search(domainId, q);
-                if (!result.length) fail = true;
+                const result = await search(domainId, q, { skip: (page - 1) * system.get('pagination.problem') });
+                total = result.total;
+                pcountRelation = result.countRelation;
+                if (!result.hits.length) fail = true;
                 if (!query.$and) query.$and = [];
                 query.$and.push({
-                    $or: result.map((i) => {
+                    $or: result.hits.map((i) => {
                         const [did, docId] = i.split('/');
                         return { domainId: did, docId: +docId };
                     }),
                 });
-                sort = result;
+                sort = result.hits;
             } else query.$text = { $search: q };
         }
         await bus.serial('problem/list', query, this);
@@ -168,11 +172,15 @@ export class ProblemMainHandler extends ProblemHandler {
             ? [[], 0, 0]
             : await problem.list(
                 domainId, query,
-                page, system.get('pagination.problem'),
+                sort.length ? 1 : page, system.get('pagination.problem'),
                 undefined, this.user._id,
             );
+        if (total) {
+            pcount = total;
+            ppcount = Math.ceil(total / system.get('pagination.problem'));
+        }
         if (sort) pdocs = pdocs.sort((a, b) => sort.indexOf(`${a.domainId}/${a.docId}`) - sort.indexOf(`${b.domainId}/${b.docId}`));
-        if (q) {
+        if (q && page === 1) {
             const pdoc = await problem.get(domainId, +q || q, problem.PROJECTION_LIST);
             if (pdoc && problem.canViewBy(pdoc, this.user)) {
                 const count = pdocs.length;
@@ -192,7 +200,14 @@ export class ProblemMainHandler extends ProblemHandler {
             );
         }
         this.response.body = {
-            page, pcount, ppcount, pdocs, psdict, category: category.join(','),
+            page,
+            pcount,
+            ppcount,
+            pcountRelation,
+            pdocs,
+            psdict,
+            category: category.join(','),
+            qs: q ? `q=${encodeURIComponent(q)}` : '',
         };
     }
 
@@ -829,8 +844,8 @@ export class ProblemPrefixListHandler extends Handler {
         const search = global.Hydro.lib.problemSearch;
         if (pdocs.length < 20) {
             if (search) {
-                const result = await search(domainId, prefix, 20 - pdocs.length);
-                const docs = await problem.getMulti(domainId, { docId: { $in: result } }).toArray();
+                const result = await search(domainId, prefix, { limit: 20 - pdocs.length });
+                const docs = await problem.getMulti(domainId, { docId: { $in: result.hits.map((i) => +i.split('/')[1]) } }).toArray();
                 pdocs.push(...docs);
             }
         }

packages/hydrooj/src/interface.ts

@@ -103,6 +103,13 @@ export interface GDoc {
     uids: number[];
 }
 
+export interface UserPreferenceDoc {
+    _id: ObjectID;
+    filename: string;
+    uid: number;
+    content: string;
+}
+
 export type ownerInfo = { owner: number, maintainer?: number[] };
 
 export type User = import('./model/user').User;
@@ -579,6 +586,7 @@ export interface Collections {
     'document.status': any;
     'problem': ProblemDoc;
     'user': Udoc;
+    'user.preference': UserPreferenceDoc;
     'vuser': VUdoc;
     'user.group': GDoc;
     'check': any;
@@ -631,11 +639,23 @@ export interface Service {
     storage: typeof import('./service/storage'),
 }
 
-interface GeoIP {
+export interface GeoIP {
     provider: string,
     lookup: (ip: string, locale?: string) => any,
 }
 
+export interface ProblemSearchResponse {
+    hits: string[];
+    total: number;
+    countRelation: 'eq' | 'gte';
+}
+export interface ProblemSearchOptions {
+    limit?: number;
+    skip?: number;
+}
+
+export type ProblemSearch = (domainId: string, q: string, options?: ProblemSearchOptions) => Promise<ProblemSearchResponse>;
+
 export interface Lib extends Record<string, any> {
     download: typeof import('./lib/download'),
     difficulty: typeof import('./lib/difficulty'),
@@ -655,6 +675,7 @@ export interface Lib extends Record<string, any> {
     validator: typeof import('./lib/validator'),
     template?: any,
     geoip?: GeoIP,
+    problemSearch: ProblemSearch;
 }
 
 export interface UI {

packages/hydrooj/src/lib/index.ts

@@ -0,0 +1,16 @@
+import './jwt';
+import './download';
+import './i18n';
+import './mail';
+import './useragent';
+import './crypto';
+import './misc';
+import './paginate';
+import './hash.hydro';
+import './rank';
+import './validator';
+import './ui';
+import './testdataConfig';
+import './difficulty';
+import './content';
+import './avatar';

packages/hydrooj/src/logger.ts

@@ -39,6 +39,7 @@ export class Logger {
     }
 
     constructor(public name: string) {
+        // eslint-disable-next-line no-constructor-return
         if (name in instances) return instances[name];
         let hash = 0;
         for (let i = 0; i < name.length; i++) {

packages/hydrooj/src/model/contest.ts

@@ -125,7 +125,9 @@ const acm = buildContestRule({
         const data = await document.collStatus.aggregate([
             {
                 $match: {
-                    domainId: tdoc.domainId, docType: document.TYPE_CONTEST, docId: tdoc.docId,
+                    domainId: tdoc.domainId,
+                    docType: document.TYPE_CONTEST,
+                    docId: tdoc.docId,
                     accept: { $gte: 1 },
                     detail: { $elemMatch: { status: STATUS.STATUS_ACCEPTED } },
                 },
@@ -230,9 +232,10 @@ const oi = buildContestRule({
         const psdict = {};
         const first = {};
         for (const pid of tdoc.pids) {
-            // eslint-disable-next-line @typescript-eslint/no-use-before-define
+            // eslint-disable-next-line @typescript-eslint/no-use-before-define, no-await-in-loop
             const [data] = await getMultiStatus(tdoc.domainId, {
-                docType: document.TYPE_CONTEST, docId: tdoc.docId,
+                docType: document.TYPE_CONTEST,
+                docId: tdoc.docId,
                 [`detail.${pid}.status`]: STATUS.STATUS_ACCEPTED,
             }).sort({ [`detail.${pid}.rid`]: 1 }).limit(1).toArray();
             first[pid] = data ? data.detail[pid].rid.generationTime : new ObjectID().generationTime;

packages/hydrooj/src/model/index.ts

@@ -0,0 +1,19 @@
+import './builtin';
+import './document';
+import './domain';
+import './blacklist';
+import './opcount';
+import './setting';
+import './token';
+import './user';
+import './storage';
+import './problem';
+import './record';
+import './contest';
+import './message';
+import './solution';
+import './training';
+import './discussion';
+import './system';
+import './oplog';
+import './blog';

packages/hydrooj/src/model/task.ts

@@ -72,7 +72,7 @@ class WorkerService implements BaseService {
                 const start = Date.now();
                 await Promise.race([
                     this.handlers[doc.subType](doc),
-                    new Promise((resolve) => setTimeout(resolve, 300000)),
+                    sleep(300000),
                 ]);
                 const spent = Date.now() - start;
                 if (spent > 500) logger.warn('Slow worker task (%d ms): %s', spent, doc);

packages/hydrooj/src/script/index.ts

@@ -0,0 +1,6 @@
+import './rating';
+import './problemStat';
+import './blacklist';
+import './deleteUser';
+import './storageUsage';
+import './checkUpdate';

packages/hydrooj/src/service/__mocks__/db.ts

@@ -1,43 +0,0 @@
-import { Collection, Db, MongoClient } from 'mongodb';
-import { BaseService, Collections } from '../../interface';
-import * as bus from '../bus';
-
-interface MongoConfig {
-    protocol?: string,
-    username?: string,
-    password?: string,
-    host?: string,
-    port?: string,
-    name?: string,
-    url?: string,
-    prefix?: string,
-}
-
-class MongoService implements BaseService {
-    public client: MongoClient;
-    public db: Db;
-    public started = false;
-    private opts: MongoConfig;
-
-    async start(opts: MongoConfig) {
-        this.opts = opts;
-        this.client = await MongoClient.connect(global.__MONGO_URI__, { useNewUrlParser: true, useUnifiedTopology: true });
-        this.db = this.client.db(global.__MONGO_DB_NAME_);
-        await bus.parallel('database/connect', this.db);
-        this.started = true;
-    }
-
-    public collection<K extends keyof Collections>(c: K): Collection<Collections[K]> {
-        if (this.opts.prefix) return this.db.collection(`${this.opts.prefix}.${c}`);
-        return this.db.collection(c);
-    }
-
-    public async stop() {
-        await this.client.close();
-        await this.client2.close();
-    }
-}
-
-const service = new MongoService();
-global.Hydro.service.db = service as any;
-export = service;

packages/login-with-osu/README.md

@@ -1 +0,0 @@
-# Login-With-Osu

packages/login-with-osu/lib.ts

@@ -1,73 +0,0 @@
-import 'hydrooj';
-
-import * as superagent from 'superagent';
-
-declare module 'hydrooj' {
-    interface SystemKeys {
-        'login-with-osu.id': string,
-        'login-with-osu.secret': string,
-    }
-    interface Lib {
-        oauth_osu: typeof import('./lib'),
-    }
-}
-
-const BASE_URL = 'https://osu.ppy.sh/';
-
-async function get() {
-    const { system, token } = global.Hydro.model;
-    const [[appid, url], [state]] = await Promise.all([
-        system.getMany([
-            'login-with-osu.id',
-            'server.url',
-        ]),
-        token.add(token.TYPE_OAUTH, 600, { redirect: this.request.referer }),
-    ]);
-    this.response.redirect = `${BASE_URL}oauth/authorize?client_id=${appid}&state=${state}&redirect_uri=${url}oauth/osu/callback&response_type=code`;
-}
-
-async function callback({ state, code }) {
-    const { system, token } = global.Hydro.model;
-    const { UserFacingError } = global.Hydro.error;
-    const [[appid, secret, url], s] = await Promise.all([
-        system.getMany([
-            'login-with-osu.id',
-            'login-with-osu.secret',
-            'server.url',
-        ]),
-        token.get(state, token.TYPE_OAUTH),
-    ]);
-    const res = await superagent.post(`${BASE_URL}oauth/token`)
-        .send({
-            client_id: appid,
-            client_secret: secret,
-            code,
-            grant_type: 'authorization_code',
-            redirect_uri: `${url}oauth/github/callback`,
-        })
-        .set('accept', 'application/json');
-    if (res.body.error) {
-        throw new UserFacingError(
-            res.body.error, res.body.error_description, res.body.error_uri,
-        );
-    }
-    const t = res.body.access_token;
-    const userInfo = await superagent.get(`${BASE_URL}api/v2/me`)
-        .set('User-Agent', 'Hydro-OAuth')
-        .set('Authorization', `Bearer ${t}`);
-    const ret = {
-        _id: `${userInfo.body.id}@osu.local`,
-        email: `${userInfo.body.id}@osu.local`,
-        bio: '',
-        uname: [userInfo.body.username],
-    };
-    this.response.redirect = s.redirect;
-    await token.del(s._id, token.TYPE_OAUTH);
-    return ret;
-}
-
-global.Hydro.lib.oauth_osu = {
-    text: 'Login with Osu',
-    callback,
-    get,
-};

packages/login-with-osu/locales/zh.yaml

@@ -1 +0,0 @@
-Login with Osu: 使用 Osu 登录

packages/login-with-osu/package.json

@@ -1,18 +0,0 @@
-{
-    "name": "@hydrooj/login-with-osu",
-    "version": "0.1.0",
-    "main": "package.json",
-    "repository": "git@github.com:hydro-dev/Hydro.git",
-    "author": "undefined <i@undefined.moe>",
-    "license": "AGPL-3.0-or-later",
-    "preferUnplugged": true,
-    "devDependencies": {
-        "@types/superagent": "^4.1.15"
-    },
-    "peerDependencies": {
-        "hydrooj": "*"
-    },
-    "dependencies": {
-        "superagent": "^7.1.1"
-    }
-}

packages/login-with-osu/setting.yaml

@@ -1,15 +0,0 @@
-id:
-    type: text
-    category: system
-    desc: Osu OAuth AppID
-    name: id
-secret:
-    type: text
-    category: system
-    desc: Osu OAuth Secret
-    name: secret
-proxy:
-    type: text
-    category: system
-    desc: Osu OAuth Proxy
-    name: proxy

packages/migrate-hustoj/script.ts

@@ -56,7 +56,7 @@ const nameMap = {
 export async function run({
     host = 'localhost', port = 3306, name = 'jol',
     username, password, domainId, contestType = 'oi',
-    dataDir,
+    dataDir, rerun = false,
 }, report: Function) {
     const src = mysql.createConnection({
         host,
@@ -65,7 +65,9 @@ export async function run({
         password,
         database: name,
     });
-    await new Promise((resolve, reject) => src.connect((err) => (err ? reject(err) : resolve(null))));
+    await new Promise((resolve, reject) => {
+        src.connect((err) => (err ? reject(err) : resolve(null)));
+    });
     const query = (q: string | mysql.Query) => new Promise<[values: any[], fields: mysql.FieldInfo[]]>((res, rej) => {
         src.query(q, (err, val, fields) => {
             if (err) rej(err);
@@ -151,23 +153,29 @@ export async function run({
     for (let pageId = 0; pageId < pageCount; pageId++) {
         const [pdocs] = await query(`SELECT * FROM \`problem\` LIMIT ${pageId * step}, ${step}`);
         for (const pdoc of pdocs) {
-            const pid = await problem.add(
-                domainId, `P${pdoc.problem_id}`,
-                pdoc.title, buildContent({
-                    description: pdoc.description,
-                    input: pdoc.input,
-                    output: pdoc.output,
-                    samples: [[pdoc.sample_input.trim(), pdoc.sample_output.trim()]],
-                    hint: pdoc.hint,
-                    source: pdoc.source,
-                }, 'html'),
-                1, pdoc.source.split(' ').map((i) => i.trim()).filter((i) => i), pdoc.defunct === 'Y',
-            );
-            pidMap[pdoc.problem_id] = pid;
+            if (rerun) {
+                const opdoc = await problem.get(domainId, `P${pdoc.problem_id}`);
+                if (opdoc) pidMap[pdoc.problem_id] = opdoc.docId;
+            }
+            if (!pidMap[pdoc.problem_id]) {
+                const pid = await problem.add(
+                    domainId, `P${pdoc.problem_id}`,
+                    pdoc.title, buildContent({
+                        description: pdoc.description,
+                        input: pdoc.input,
+                        output: pdoc.output,
+                        samples: [[pdoc.sample_input.trim(), pdoc.sample_output.trim()]],
+                        hint: pdoc.hint,
+                        source: pdoc.source,
+                    }, 'html'),
+                    1, pdoc.source.split(' ').map((i) => i.trim()).filter((i) => i), pdoc.defunct === 'Y',
+                );
+                pidMap[pdoc.problem_id] = pid;
+            }
             const [cdoc] = await query(`SELECT * FROM \`privilege\` WHERE \`rightstr\` = 'p${pdoc.problem_id}'`);
             const maintainer = [];
             for (let i = 1; i < cdoc.length; i++) maintainer.push(uidMap[cdoc[i].user_id]);
-            await problem.edit(domainId, pid, {
+            await problem.edit(domainId, pidMap[pdoc.problem_id], {
                 nAccept: 0,
                 nSubmit: pdoc.submit,
                 config: `time: ${pdoc.time_limit}s\nmemory: ${pdoc.memory_limit}m`,
@@ -264,6 +272,7 @@ export async function run({
 
     src.end();
 
+    if (!dataDir) return true;
     if (dataDir.endsWith('/')) dataDir = dataDir.slice(0, -1);
     const files = await fs.readdir(dataDir, { withFileTypes: true });
     for (const file of files) {

packages/sonic/model.ts

@@ -46,8 +46,13 @@ bus.on('problem/del', async (domainId, docId) => {
     await Promise.all(tasks);
 });
 
-global.Hydro.lib.problemSearch = async (domainId: string, query: string, limit = system.get('pagination.problem')) => {
+global.Hydro.lib.problemSearch = async (domainId, query, opts) => {
+    const limit = opts?.limit || system.get('pagination.problem');
     const ids = await sonic.query('problem', `${domainId}@title`, query, { limit });
     if (limit - ids.length > 0) ids.push(...await sonic.query('problem', `${domainId}@content`, query, { limit: limit - ids.length }));
-    return ids;
+    return {
+        countRelation: ids.length >= limit ? 'gte' : 'eq',
+        total: ids.length,
+        hits: ids,
+    };
 };

packages/sonic/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hydrooj/sonic",
-    "version": "1.2.4",
+    "version": "1.2.5",
     "description": "Sonic search service",
     "main": "service.js",
     "typings": "service.d.ts",

packages/ui-default/locales/zh.yaml

@@ -527,6 +527,7 @@ Or, with automatically filled invitation code: 或者，这是可以自动填写
 Ordered List: 有序列表
 Original Score: 原始分数
 OS Info: 系统信息
+Over {0} problems: 超过 {0} 道题目
 Owner: 所有者
 page.problem_detail.sidebar.show_category: 点击显示
 page.training_detail.invalid_when_not_enrolled: 未参加训练计划时您不能查看题目详情。

packages/ui-default/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hydrooj/ui-default",
-  "version": "4.34.12",
+  "version": "4.34.13",
   "author": "undefined <i@undefined.moe>",
   "license": "AGPL-3.0",
   "main": "hydro.js",

packages/ui-default/templates/partials/problem_list.html

@@ -6,7 +6,7 @@
 {% if not pdocs.length %}
   {{ nothing.render('Sorry, there is no problem in the problem set') }}
 {% else %}
-  {{ paginator.render(page, ppcount, position='top') }}
+  {{ paginator.render(page, ppcount, position='top', add_qs=qs) }}
   <table class="data-table hide-problem-tag">
     <colgroup>
       <col class="col--checkbox display-mode-hide">
@@ -73,6 +73,6 @@
     {%- endfor -%}
     </tbody>
   </table>
-  {{ paginator.render(page, ppcount) }}
+  {{ paginator.render(page, ppcount, add_qs=qs) }}
 {% endif %}
 </div>

packages/ui-default/templates/partials/problem_stat.html

@@ -1,3 +1,3 @@
 <div data-fragment-id="problem_stat">
-  <p>{{ _('{0} problems').format(pcount) }}</p>
+  <p>{{ _('{0} problems' if pcountRelation == 'eq' else 'Over {0} problems').format(pcount) }}</p>
 </div>

packages/vjudge/src/providers/uoj.ts

@@ -252,7 +252,8 @@ export default class UOJProvider implements IBasicProvider {
                 }
             }
             if (document.querySelector('tbody').innerHTML.includes('Judging')) continue;
-            const score = +summary.children[3]?.children[0].innerHTML || 0;
+            // eslint-disable-next-line no-unsafe-optional-chaining
+            const score = +summary.children[3]?.children[0]?.innerHTML || 0;
             const status = score === 100 ? STATUS.STATUS_ACCEPTED : STATUS.STATUS_WRONG_ANSWER;
             return await end({
                 status,