core&ui&elastic: support elastic search and allow search result pagination

packages/hydrooj/src/entry/cli.ts

@@ -7,9 +7,7 @@ import options from '../options';
 import * as bus from '../service/bus';
 import db from '../service/db';
 import {
-    builtinLib, builtinModel, builtinScript,
-    lib, model, script,
-    service,
+    lib, model, script, service,
 } from './common';
 
 const argv = cac().parse();
@@ -92,14 +90,14 @@ export async function load() {
     await db.start(opts);
     const storage = require('../service/storage');
     await storage.start();
-    for (const i of builtinLib) require(`../lib/${i}`);
+    require('../lib/index');
     await lib(pending, fail);
     const systemModel = require('../model/system');
     await systemModel.runConfig();
     await service(pending, fail);
-    for (const i of builtinModel) require(`../model/${i}`);
+    require('../model/index');
     await model(pending, fail);
-    for (const i of builtinScript) require(`../script/${i}`);
+    require('../script/index');
     await script(pending, fail, []);
     await bus.parallel('app/started');
     await cli();

packages/hydrooj/src/entry/common.ts

@@ -1,3 +1,4 @@
+/* eslint-disable import/no-dynamic-require */
 /* eslint-disable no-await-in-loop */
 /* eslint-disable no-eval */
 import os from 'os';
@@ -10,31 +11,6 @@ import * as bus from '../service/bus';
 
 const logger = new Logger('common');
 
-export const builtinLib = [
-    'jwt', 'download', 'i18n', 'mail', 'useragent',
-    'crypto', 'misc', 'paginate', 'hash.hydro', 'rank',
-    'validator', 'ui', 'testdataConfig', 'difficulty', 'content',
-    'avatar',
-];
-
-export const builtinModel = [
-    'builtin', 'document', 'domain', 'blacklist', 'opcount',
-    'setting', 'token', 'user', 'storage', 'problem',
-    'record', 'contest', 'message', 'solution', 'training',
-    'discussion', 'system', 'oplog', 'blog',
-];
-
-export const builtinHandler = [
-    'home', 'problem', 'record', 'judge', 'user',
-    'contest', 'training', 'discussion', 'manage', 'import',
-    'misc', 'homework', 'domain', 'status', 'api', 'blog',
-];
-
-export const builtinScript = [
-    'rating', 'problemStat', 'blacklist', 'deleteUser', 'storageUsage',
-    'checkUpdate',
-];
-
 function getFiles(folder: string, base = ''): string[] {
     const files = [];
     const f = fs.readdirSync(folder);
@@ -53,7 +29,7 @@ export async function handler(pending: string[], fail: string[]) {
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 logger.info('Handler init: %s', i);
-                eval('require')(p);
+                require(p);
             } catch (e) {
                 fail.push(i);
                 logger.error('Handler Load Fail: %s', i);
@@ -136,7 +112,7 @@ export async function template(pending: string[], fail: string[]) {
             try {
                 const files = getFiles(p);
                 for (const file of files) {
-                    if (file.endsWith('.tsx')) global.Hydro.ui.template[file] = eval('require')(path.resolve(p, file));
+                    if (file.endsWith('.tsx')) global.Hydro.ui.template[file] = require(path.resolve(p, file));
                     global.Hydro.ui.template[file] = await fs.readFile(path.resolve(p, file), 'utf-8');
                 }
                 logger.info('Template init: %s', i);
@@ -157,7 +133,7 @@ export async function model(pending: string[], fail: string[]) {
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 logger.info('Model init: %s', i);
-                eval('require')(p);
+                require(p);
             } catch (e) {
                 fail.push(i);
                 logger.error('Model Load Fail: %s', i);
@@ -175,7 +151,7 @@ export async function lib(pending: string[], fail: string[]) {
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 logger.info('Lib init: %s', i);
-                eval('require')(p);
+                require(p);
             } catch (e) {
                 fail.push(i);
                 logger.error('Lib Load Fail: %s', i);
@@ -193,7 +169,7 @@ export async function service(pending: string[], fail: string[]) {
         if (fs.existsSync(p) && !fail.includes(i)) {
             try {
                 logger.info('Service init: %s', i);
-                eval('require')(p);
+                require(p);
             } catch (e) {
                 fail.push(i);
                 logger.error('Service Load Fail: %s', i);
@@ -215,7 +191,7 @@ export async function script(pending: string[], fail: string[], active: string[]
         if (await fs.pathExists(p) && !fail.includes(i)) {
             try {
                 logger.info('Script init: %s', i);
-                eval('require')(p);
+                require(p);
             } catch (e) {
                 fail.push(i);
                 logger.error('Script Load Fail: %s', i);

packages/hydrooj/src/entry/worker.ts

@@ -9,8 +9,7 @@ import options from '../options';
 import * as bus from '../service/bus';
 import db from '../service/db';
 import {
-    builtinHandler, builtinLib, builtinModel,
-    builtinScript, handler, lib, locale, model, script, service, setting, template,
+    handler, lib, locale, model, script, service, setting, template,
 } from './common';
 
 const logger = new Logger('worker');
@@ -45,15 +44,7 @@ export async function load() {
     const storage = require('../service/storage');
     await storage.start();
     if (detail) logger.info('finish: storage.connect');
-    for (const i of builtinLib) {
-        let t;
-        try {
-            t = require.resolve(`../lib/${i}`);
-        } catch (e) {
-            t = require.resolve(`@hydrooj/utils/lib/${i}`);
-        }
-        require(t);
-    }
+    require('../lib/index');
     if (detail) logger.info('finish: lib.builtin');
     await lib(pending, fail);
     if (detail) logger.info('finish: lib.extra');
@@ -64,9 +55,9 @@ export async function load() {
     if (detail) logger.info('finish: server');
     await service(pending, fail);
     if (detail) logger.info('finish: service.extra');
-    for (const i of builtinModel) require(`../model/${i}`);
+    require('../model/index');
     if (detail) logger.info('finish: model.builtin');
-    for (const i of builtinHandler) require(`../handler/${i}`);
+    require('../handler/index');
     if (detail) logger.info('finish: handler.builtin');
     await model(pending, fail);
     if (detail) logger.info('finish: model.extra');
@@ -79,7 +70,7 @@ export async function load() {
     if (detail) logger.info('finish: handler.apply');
     const notfound = require('../handler/notfound');
     await notfound.apply();
-    for (const i of builtinScript) require(`../script/${i}`);
+    require('../script/index');
     if (detail) logger.info('finish: script.builtin');
     await script(pending, fail, active);
     if (detail) logger.info('finish: script.extra');

packages/hydrooj/src/handler/index.ts

@@ -0,0 +1,16 @@
+import './home';
+import './problem';
+import './record';
+import './judge';
+import './user';
+import './contest';
+import './training';
+import './discussion';
+import './manage';
+import './import';
+import './misc';
+import './homework';
+import './domain';
+import './status';
+import './api';
+import './blog';

packages/hydrooj/src/handler/problem.ts

@@ -114,15 +114,15 @@ export class ProblemHandler extends Handler {
     async cleanup() {
         if (this.response.template === 'problem_main.html' && this.request.json) {
             const {
-                page, pcount, ppcount, pdocs, psdict, category,
+                page, pcount, pcountRelation, ppcount, pdocs, psdict, category, qs,
             } = this.response.body;
             this.response.body = {
                 title: this.renderTitle(this.translate('problem_main')),
                 fragments: (await Promise.all([
                     this.renderHTML('partials/problem_list.html', {
-                        page, ppcount, pcount, pdocs, psdict,
+                        page, ppcount, pcount, pdocs, psdict, qs,
                     }),
-                    this.renderHTML('partials/problem_stat.html', { pcount }),
+                    this.renderHTML('partials/problem_stat.html', { pcount, pcountRelation }),
                     this.renderHTML('partials/problem_lucky.html', { category }),
                 ])).map((i) => ({ html: i })),
                 raw: {
@@ -145,21 +145,25 @@ export class ProblemMainHandler extends ProblemHandler {
         const search = global.Hydro.lib.problemSearch;
         let sort: string[];
         let fail = false;
+        let pcountRelation = 'eq';
         if (category.length) query.$and = category.map((tag) => ({ tag }));
         if (q) category.push(q);
         if (category.length) this.extraTitleContent = category.join(',');
+        let total = 0;
         if (q) {
             if (search) {
-                const result = await search(domainId, q);
-                if (!result.length) fail = true;
+                const result = await search(domainId, q, { skip: (page - 1) * system.get('pagination.problem') });
+                total = result.total;
+                pcountRelation = result.countRelation;
+                if (!result.hits.length) fail = true;
                 if (!query.$and) query.$and = [];
                 query.$and.push({
-                    $or: result.map((i) => {
+                    $or: result.hits.map((i) => {
                         const [did, docId] = i.split('/');
                         return { domainId: did, docId: +docId };
                     }),
                 });
-                sort = result;
+                sort = result.hits;
             } else query.$text = { $search: q };
         }
         await bus.serial('problem/list', query, this);
@@ -168,11 +172,15 @@ export class ProblemMainHandler extends ProblemHandler {
             ? [[], 0, 0]
             : await problem.list(
                 domainId, query,
-                page, system.get('pagination.problem'),
+                sort.length ? 1 : page, system.get('pagination.problem'),
                 undefined, this.user._id,
             );
+        if (total) {
+            pcount = total;
+            ppcount = Math.ceil(total / system.get('pagination.problem'));
+        }
         if (sort) pdocs = pdocs.sort((a, b) => sort.indexOf(`${a.domainId}/${a.docId}`) - sort.indexOf(`${b.domainId}/${b.docId}`));
-        if (q) {
+        if (q && page === 1) {
             const pdoc = await problem.get(domainId, +q || q, problem.PROJECTION_LIST);
             if (pdoc && problem.canViewBy(pdoc, this.user)) {
                 const count = pdocs.length;
@@ -192,7 +200,14 @@ export class ProblemMainHandler extends ProblemHandler {
             );
         }
         this.response.body = {
-            page, pcount, ppcount, pdocs, psdict, category: category.join(','),
+            page,
+            pcount,
+            ppcount,
+            pcountRelation,
+            pdocs,
+            psdict,
+            category: category.join(','),
+            qs: q ? `q=${encodeURIComponent(q)}` : '',
         };
     }
 
@@ -829,8 +844,8 @@ export class ProblemPrefixListHandler extends Handler {
         const search = global.Hydro.lib.problemSearch;
         if (pdocs.length < 20) {
             if (search) {
-                const result = await search(domainId, prefix, 20 - pdocs.length);
-                const docs = await problem.getMulti(domainId, { docId: { $in: result } }).toArray();
+                const result = await search(domainId, prefix, { limit: 20 - pdocs.length });
+                const docs = await problem.getMulti(domainId, { docId: { $in: result.hits.map((i) => +i.split('/')[1]) } }).toArray();
                 pdocs.push(...docs);
             }
         }

packages/hydrooj/src/interface.ts

@@ -103,6 +103,13 @@ export interface GDoc {
     uids: number[];
 }
 
+export interface UserPreferenceDoc {
+    _id: ObjectID;
+    filename: string;
+    uid: number;
+    content: string;
+}
+
 export type ownerInfo = { owner: number, maintainer?: number[] };
 
 export type User = import('./model/user').User;
@@ -579,6 +586,7 @@ export interface Collections {
     'document.status': any;
     'problem': ProblemDoc;
     'user': Udoc;
+    'user.preference': UserPreferenceDoc;
     'vuser': VUdoc;
     'user.group': GDoc;
     'check': any;
@@ -631,11 +639,23 @@ export interface Service {
     storage: typeof import('./service/storage'),
 }
 
-interface GeoIP {
+export interface GeoIP {
     provider: string,
     lookup: (ip: string, locale?: string) => any,
 }
 
+export interface ProblemSearchResponse {
+    hits: string[];
+    total: number;
+    countRelation: 'eq' | 'gte';
+}
+export interface ProblemSearchOptions {
+    limit?: number;
+    skip?: number;
+}
+
+export type ProblemSearch = (domainId: string, q: string, options?: ProblemSearchOptions) => Promise<ProblemSearchResponse>;
+
 export interface Lib extends Record<string, any> {
     download: typeof import('./lib/download'),
     difficulty: typeof import('./lib/difficulty'),
@@ -655,6 +675,7 @@ export interface Lib extends Record<string, any> {
     validator: typeof import('./lib/validator'),
     template?: any,
     geoip?: GeoIP,
+    problemSearch: ProblemSearch;
 }
 
 export interface UI {

packages/hydrooj/src/lib/index.ts

@@ -0,0 +1,16 @@
+import './jwt';
+import './download';
+import './i18n';
+import './mail';
+import './useragent';
+import './crypto';
+import './misc';
+import './paginate';
+import './hash.hydro';
+import './rank';
+import './validator';
+import './ui';
+import './testdataConfig';
+import './difficulty';
+import './content';
+import './avatar';

packages/hydrooj/src/logger.ts

@@ -39,6 +39,7 @@ export class Logger {
     }
 
     constructor(public name: string) {
+        // eslint-disable-next-line no-constructor-return
         if (name in instances) return instances[name];
         let hash = 0;
         for (let i = 0; i < name.length; i++) {

packages/hydrooj/src/model/contest.ts

@@ -125,7 +125,9 @@ const acm = buildContestRule({
         const data = await document.collStatus.aggregate([
             {
                 $match: {
-                    domainId: tdoc.domainId, docType: document.TYPE_CONTEST, docId: tdoc.docId,
+                    domainId: tdoc.domainId,
+                    docType: document.TYPE_CONTEST,
+                    docId: tdoc.docId,
                     accept: { $gte: 1 },
                     detail: { $elemMatch: { status: STATUS.STATUS_ACCEPTED } },
                 },
@@ -230,9 +232,10 @@ const oi = buildContestRule({
         const psdict = {};
         const first = {};
         for (const pid of tdoc.pids) {
-            // eslint-disable-next-line @typescript-eslint/no-use-before-define
+            // eslint-disable-next-line @typescript-eslint/no-use-before-define, no-await-in-loop
             const [data] = await getMultiStatus(tdoc.domainId, {
-                docType: document.TYPE_CONTEST, docId: tdoc.docId,
+                docType: document.TYPE_CONTEST,
+                docId: tdoc.docId,
                 [`detail.${pid}.status`]: STATUS.STATUS_ACCEPTED,
             }).sort({ [`detail.${pid}.rid`]: 1 }).limit(1).toArray();
             first[pid] = data ? data.detail[pid].rid.generationTime : new ObjectID().generationTime;

packages/hydrooj/src/model/index.ts

@@ -0,0 +1,19 @@
+import './builtin';
+import './document';
+import './domain';
+import './blacklist';
+import './opcount';
+import './setting';
+import './token';
+import './user';
+import './storage';
+import './problem';
+import './record';
+import './contest';
+import './message';
+import './solution';
+import './training';
+import './discussion';
+import './system';
+import './oplog';
+import './blog';

packages/hydrooj/src/model/task.ts

@@ -72,7 +72,7 @@ class WorkerService implements BaseService {
                 const start = Date.now();
                 await Promise.race([
                     this.handlers[doc.subType](doc),
-                    new Promise((resolve) => setTimeout(resolve, 300000)),
+                    sleep(300000),
                 ]);
                 const spent = Date.now() - start;
                 if (spent > 500) logger.warn('Slow worker task (%d ms): %s', spent, doc);

packages/hydrooj/src/script/index.ts

@@ -0,0 +1,6 @@
+import './rating';
+import './problemStat';
+import './blacklist';
+import './deleteUser';
+import './storageUsage';
+import './checkUpdate';

packages/hydrooj/src/service/__mocks__/db.ts

@@ -1,43 +0,0 @@
-import { Collection, Db, MongoClient } from 'mongodb';
-import { BaseService, Collections } from '../../interface';
-import * as bus from '../bus';
-
-interface MongoConfig {
-    protocol?: string,
-    username?: string,
-    password?: string,
-    host?: string,
-    port?: string,
-    name?: string,
-    url?: string,
-    prefix?: string,
-}
-
-class MongoService implements BaseService {
-    public client: MongoClient;
-    public db: Db;
-    public started = false;
-    private opts: MongoConfig;
-
-    async start(opts: MongoConfig) {
-        this.opts = opts;
-        this.client = await MongoClient.connect(global.__MONGO_URI__, { useNewUrlParser: true, useUnifiedTopology: true });
-        this.db = this.client.db(global.__MONGO_DB_NAME_);
-        await bus.parallel('database/connect', this.db);
-        this.started = true;
-    }
-
-    public collection<K extends keyof Collections>(c: K): Collection<Collections[K]> {
-        if (this.opts.prefix) return this.db.collection(`${this.opts.prefix}.${c}`);
-        return this.db.collection(c);
-    }
-
-    public async stop() {
-        await this.client.close();
-        await this.client2.close();
-    }
-}
-
-const service = new MongoService();
-global.Hydro.service.db = service as any;
-export = service;