Merge pull request #887 from hydro-dev/remove-union

framework/framework/validator.ts

@@ -82,7 +82,7 @@ export const Types: Types = {
     UidOrName: saslprepString(/^(.{3,31}|[\u4e00-\u9fa5]{2}|-?[0-9]+)$/),
     Username: saslprepString(/^(.{3,31}|[\u4e00-\u9fa5]{2})$/),
     Password: basicString(/^.{6,255}$/),
-    ProblemId: saslprepString(/^[a-zA-Z0-9]+$/i, () => true, (s) => (Number.isSafeInteger(+s) ? +s : s)),
+    ProblemId: saslprepString(/^([a-zA-Z0-9]{1,10}-)?[a-zA-Z0-9]+$/i, () => true, (s) => (Number.isSafeInteger(+s) ? +s : s)),
     Email: saslprepString(/^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)+$/i),
     DomainId: saslprepString(/^[a-zA-Z][a-zA-Z0-9_]{3,31}$/),
     Role: saslprepString(/^[_0-9A-Za-z\u4e00-\u9fa5]{1,31}$/i),

packages/elastic/index.ts

@@ -1,6 +1,6 @@
 import { Client } from '@elastic/elasticsearch';
 import {
-    _, Context, DomainModel, iterateAllProblem, iterateAllProblemInDomain,
+    _, Context, iterateAllProblem, iterateAllProblemInDomain,
     ProblemDoc, ProblemModel, Schema, SystemModel,
 } from 'hydrooj';
 
@@ -19,8 +19,6 @@ global.Hydro.lib.problemSearch = async (domainId, q, opts) => {
     const allowedSize = SystemModel.get('elasic-search.indexSize') || 10000;
     const size = opts?.limit || SystemModel.get('pagination.problem');
     const from = Math.min(allowedSize - size, opts?.skip || 0);
-    const union = await DomainModel.get(domainId);
-    const domainIds = [domainId, ...(union?.union || [])];
     const res = await client.search({
         index: 'problem',
         size,
@@ -34,7 +32,7 @@ global.Hydro.lib.problemSearch = async (domainId, q, opts) => {
         post_filter: {
             bool: {
                 minimum_should_match: 1,
-                should: domainIds.map((i) => ({ match: { domainId: i } })),
+                should: [{ match: { domainId } }],
             },
         },
     });

packages/hydrooj/src/handler/problem.ts

@@ -142,10 +142,9 @@ export class ProblemMainHandler extends Handler {
         // eslint-disable-next-line prefer-const
         let [pdocs, ppcount, pcount] = this.queryContext.fail
             ? [[], 0, 0]
-            : await problem.list(
+            : await this.paginate(
-                domainId, query, sort.length ? 1 : page, limit,
+                problem.getMulti(domainId, query, quick ? ['title', 'pid', 'domainId', 'docId'] : undefined).sort({ sort: 1, docId: 1 }),
-                quick ? ['title', 'pid', 'domainId', 'docId'] : undefined,
+                sort.length ? 1 : page, limit,
-                this.user._id,
             );
         if (total) {
             pcount = total;

packages/hydrooj/src/handler/user.ts

@@ -358,25 +358,22 @@ class UserDetailHandler extends Handler {
     async get(domainId: string, uid: number) {
         if (uid === 0) throw new UserNotFoundError(0);
         const isSelfProfile = this.user._id === uid;
-        const [udoc, sdoc, union] = await Promise.all([
+        const [udoc, sdoc] = await Promise.all([
             user.getById(domainId, uid),
             token.getMostRecentSessionByUid(uid, ['createAt', 'updateAt']),
-            domain.get(domainId),
         ]);
         if (!udoc) throw new UserNotFoundError(uid);
         const pdocs: ProblemDoc[] = [];
         const acInfo: Record<string, number> = {};
         const canViewHidden = this.user.hasPerm(PERM.PERM_VIEW_PROBLEM_HIDDEN) || this.user._id;
         if (this.user.hasPerm(PERM.PERM_VIEW_PROBLEM)) {
-            await Promise.all([domainId, ...(union?.union || [])].map(async (did) => {
+            const psdocs = await problem.getMultiStatus(domainId, { uid, status: STATUS.STATUS_ACCEPTED }).toArray();
-                const psdocs = await problem.getMultiStatus(did, { uid, status: STATUS.STATUS_ACCEPTED }).toArray();
+            pdocs.push(...Object.values(
-                pdocs.push(...Object.values(
+                await problem.getList(
-                    await problem.getList(
+                    domainId, psdocs.map((i) => i.docId), canViewHidden,
-                        did, psdocs.map((i) => i.docId), canViewHidden,
+                    false, problem.PROJECTION_LIST, true,
-                        false, problem.PROJECTION_LIST, true,
+                ),
-                    ),
+            ));
-                ));
-            }));
         }
         for (const pdoc of pdocs) {
             for (const tag of pdoc.tag) {

packages/hydrooj/src/model/problem.ts

@@ -16,6 +16,7 @@ import type {
 } from '../interface';
 import { parseConfig } from '../lib/testdataConfig';
 import * as bus from '../service/bus';
+import db from '../service/db';
 import {
     ArrayKeys, MaybeArray, NumberKeys, Projection,
 } from '../typeutils';
@@ -27,14 +28,15 @@ import RecordModel from './record';
 import SolutionModel from './solution';
 import storage from './storage';
 import * as SystemModel from './system';
-import user from './user';
 
 export interface ProblemDoc extends Document { }
 export type Field = keyof ProblemDoc;
 
 const logger = new Logger('problem');
-function sortable(source: string) {
+function sortable(source: string, namespaces: Record<string, string>) {
-    return source.replace(/(\d+)/g, (str) => (str.length >= 6 ? str : ('0'.repeat(6 - str.length) + str)));
+    const [namespace, pid] = source.includes('-') ? source.split('-') : ['default', source];
+    return ((namespaces ? `${namespaces[namespace]}-` : '') + pid)
+        .replace(/(\d+)/g, (str) => (str.length >= 6 ? str : ('0'.repeat(6 - str.length) + str)));
 }
 
 function findOverrideContent(dir: string, base: string) {
@@ -149,8 +151,9 @@ export class ProblemModel {
         content: string, owner: number, tag: string[] = [],
         meta: ProblemCreateOptions = {},
     ) {
+        const ddoc = await DomainModel.get(domainId);
         const args: Partial<ProblemDoc> = {
-            title, tag, hidden: meta.hidden || false, nSubmit: 0, nAccept: 0, sort: sortable(pid || `P${docId}`),
+            title, tag, hidden: meta.hidden || false, nSubmit: 0, nAccept: 0, sort: sortable(pid || `P${docId}`, ddoc?.namespaces),
         };
         if (pid) args.pid = pid;
         if (meta.difficulty) args.difficulty = meta.difficulty;
@@ -171,9 +174,10 @@ export class ProblemModel {
         rawConfig = false,
     ): Promise<ProblemDoc | null> {
         if (Number.isSafeInteger(+pid)) pid = +pid;
+        const ddoc = await DomainModel.get(domainId);
         const res = typeof pid === 'number'
             ? await document.get(domainId, document.TYPE_PROBLEM, pid, projection)
-            : (await document.getMulti(domainId, document.TYPE_PROBLEM, { sort: sortable(pid), pid })
+            : (await document.getMulti(domainId, document.TYPE_PROBLEM, { sort: sortable(pid, ddoc?.namespaces), pid })
                 .project(buildProjection(projection)).limit(1).toArray())[0];
         if (!res) return null;
         try {
@@ -188,33 +192,16 @@ export class ProblemModel {
         return document.getMulti(domainId, document.TYPE_PROBLEM, query, projection).sort({ sort: 1 });
     }
 
+    /** @deprecated */
     static async list(
         domainId: string, query: Filter<ProblemDoc>,
         page: number, pageSize: number,
-        projection = ProblemModel.PROJECTION_LIST, uid?: number,
+        projection = ProblemModel.PROJECTION_LIST,
     ): Promise<[ProblemDoc[], number, number]> {
-        const union = await DomainModel.get(domainId);
+        return await db.paginate(
-        const domainIds = [domainId, ...(union.union || [])];
+            document.getMulti(domainId, document.TYPE_PROBLEM, query, projection).sort({ sort: 1, docId: 1 }),
-        let count = 0;
+            page, pageSize,
-        const pdocs = [];
+        );
-        for (const id of domainIds) {
-            // TODO enhance performance
-            if (typeof uid === 'number') {
-                // eslint-disable-next-line no-await-in-loop
-                const udoc = await user.getById(id, uid);
-                if (!udoc.hasPerm(PERM.PERM_VIEW_PROBLEM)) continue;
-            }
-            // eslint-disable-next-line no-await-in-loop
-            const ccount = await document.count(id, document.TYPE_PROBLEM, query);
-            if (pdocs.length < pageSize && (page - 1) * pageSize - count <= ccount) {
-                // eslint-disable-next-line no-await-in-loop
-                pdocs.push(...await document.getMulti(id, document.TYPE_PROBLEM, query, projection)
-                    .sort({ sort: 1, docId: 1 })
-                    .skip(Math.max((page - 1) * pageSize - count, 0)).limit(pageSize - pdocs.length).toArray());
-            }
-            count += ccount;
-        }
-        return [pdocs, Math.ceil(count / pageSize), count];
     }
 
     static getStatus(domainId: string, docId: number, uid: number) {
@@ -227,11 +214,12 @@ export class ProblemModel {
 
     static async edit(domainId: string, _id: number, $set: Partial<ProblemDoc>): Promise<ProblemDoc> {
         const delpid = $set.pid === '';
+        const ddoc = await DomainModel.get(domainId);
         if (delpid) {
             delete $set.pid;
-            $set.sort = sortable(`P${_id}`);
+            $set.sort = sortable(`P${_id}`, ddoc.namespaces);
         } else if ($set.pid) {
-            $set.sort = sortable($set.pid);
+            $set.sort = sortable($set.pid, ddoc.namespaces);
         }
         await bus.parallel('problem/before-edit', $set);
         const result = await document.set(domainId, document.TYPE_PROBLEM, _id, $set, delpid ? { pid: '' } : undefined);

packages/hydrooj/src/model/storage.ts

@@ -123,6 +123,19 @@ export class StorageModel {
         return await storage.signDownloadLink(res.value?.link || res.value?._id || target, filename, noExpire, useAlternativeEndpointFor);
     }
 
+    static async move(src: string, dst: string) {
+        const res = await StorageModel.coll.findOneAndUpdate(
+            { path: src, autoDelete: null },
+            { $set: { path: dst } },
+        );
+        return !!res.value;
+    }
+
+    static async exists(path: string) {
+        const value = await StorageModel.coll.findOne({ path, autoDelete: null });
+        return !!value;
+    }
+
     static async copy(src: string, dst: string) {
         const { value } = await StorageModel.coll.findOneAndUpdate(
             { path: src, autoDelete: null },

packages/hydrooj/src/script/rating.ts

@@ -136,14 +136,12 @@ export async function calcLevel(domainId: string, report: Function) {
 }
 
 async function runInDomain(domainId: string, report: Function) {
-    const info = await domain.get(domainId);
-    const domainIds = [domainId, ...(info.union || [])];
     const results: Record<keyof typeof RpTypes, ND> = {};
     const udict = Counter();
     await db.collection('domain.user').updateMany({ domainId }, { $set: { rpInfo: {} } });
     for (const type in RpTypes) {
         results[type] = new Proxy({}, { get: (self, key) => self[key] || RpTypes[type].base });
-        await RpTypes[type].run(domainIds, results[type], report);
+        await RpTypes[type].run([domainId], results[type], report);
         const bulk = db.collection('domain.user').initializeUnorderedBulkOp();
         for (const uid in results[type]) {
             const udoc = await UserModel.getById(domainId, +uid);

packages/hydrooj/src/service/server.ts

@@ -32,6 +32,7 @@ declare module '@hydrooj/framework' {
         domain: DomainDoc;
 
         paginate<T>(cursor: FindCursor<T>, page: number, key: string): Promise<[docs: T[], numPages: number, count: number]>;
+        paginate<T>(cursor: FindCursor<T>, page: number, limit: number): Promise<[docs: T[], numPages: number, count: number]>;
         progress(message: string, params: any[]): void;
         limitRate(op: string, periodSecs: number, maxOperations: number, defaultKey?: string): Promise<void>;
         renderTitle(str: string): string;
@@ -237,8 +238,8 @@ export async function apply(ctx: Context) {
                     : str.toString().translate(...this.context.acceptsLanguages(), system.get('server.language'));
                 return res;
             },
-            paginate<T>(cursor: FindCursor<T>, page: number, key: string) {
+            paginate<T>(cursor: FindCursor<T>, page: number, key: string | number) {
-                return db.paginate(cursor, page, this.ctx.setting.get(`pagination.${key}`) || 20);
+                return db.paginate(cursor, page, typeof key === 'number' ? key : (this.ctx.setting.get(`pagination.${key}`) || 20));
             },
             checkPerm(...args: bigint[]) {
                 if (!this.user.hasPerm(...args)) {

packages/sonic/index.ts

@@ -1,11 +1,12 @@
 import {
-    Context, DomainModel, iterateAllProblem, iterateAllProblemInDomain,
+    Context, iterateAllProblem, iterateAllProblemInDomain,
     Logger, ProblemModel, Schema, SystemModel,
 } from 'hydrooj';
 import { SonicService } from './service';
 
 const logger = new Logger('sonic');
 
+export const inject = { 'sonic': { required: false } };
 export function apply(ctx: Context) {
     ctx.plugin(SonicService);
 
@@ -37,18 +38,14 @@ export function apply(ctx: Context) {
         const cb = async (pdoc) => {
             i++;
             if (!(i % 1000)) report({ message: `${i} problems indexed` });
-            const union = await DomainModel.getMulti({ union: pdoc.domainId }).toArray();
+            await Promise.all([
-            const tasks = [];
+                pdoc.title && ctx.sonic.push(
-            for (const did of [pdoc.domainId, ...union.map((j) => j._id)]) {
+                    'problem', `${pdoc.domainid}@title`, `${pdoc.domainId}/${pdoc.docId}`,
-                tasks.push(
+                    `${pdoc.pid || ''} ${pdoc.title} ${pdoc.tag.join(' ')}`,
-                    pdoc.title && ctx.sonic.push(
+                ),
-                        'problem', `${did}@title`, `${pdoc.domainId}/${pdoc.docId}`,
+                pdoc.content.toString()
-                        `${pdoc.pid || ''} ${pdoc.title} ${pdoc.tag.join(' ')}`,
+                && ctx.sonic.push('problem', `${pdoc.domainId}@content`, `${pdoc.domainId}/${pdoc.docId}`, pdoc.content.toString()),
-                    ),
+            ]).catch((e) => console.log(`${pdoc.domainId}/${pdoc.docId}`, e));
-                    pdoc.content.toString() && ctx.sonic.push('problem', `${did}@content`, `${pdoc.domainId}/${pdoc.docId}`, pdoc.content.toString()),
-                );
-            }
-            await Promise.all(tasks).catch((e) => console.log(`${pdoc.domainId}/${pdoc.docId}`, e));
         };
         if (domainId) await iterateAllProblemInDomain(domainId, ['title', 'content'], cb);
         else await iterateAllProblem(['title', 'content', 'tag'], cb);
@@ -56,43 +53,28 @@ export function apply(ctx: Context) {
     }
 
     ctx.on('problem/add', async (doc, docId) => {
-        const union = await DomainModel.getMulti({ union: doc.domainId }).toArray();
+        Promise.all([
-        const tasks = [];
+            ctx.sonic.push('problem', `${doc.domainId}@title`, `${doc.domainId}/${docId}`, `${doc.pid || ''} ${doc.title} ${doc.tag?.join(' ')}`),
-        for (const domainId of [doc.domainId, ...union.map((i) => i._id)]) {
+            ctx.sonic.push('problem', `${doc.domainId}@content`, `${doc.domainId}/${docId}`, doc.content.toString()),
-            tasks.push(
+        ]).catch((e) => logger.error(e));
-                ctx.sonic.push('problem', `${domainId}@title`, `${doc.domainId}/${docId}`, `${doc.pid || ''} ${doc.title} ${doc.tag?.join(' ')}`),
-                ctx.sonic.push('problem', `${domainId}@content`, `${doc.domainId}/${docId}`, doc.content.toString()),
-            );
-        }
-        Promise.all(tasks).catch((e) => logger.error(e));
     });
 
     ctx.on('problem/edit', async (pdoc) => {
-        const union = await DomainModel.getMulti({ union: pdoc.domainId }).toArray();
-        const tasks = [];
         const id = `${pdoc.domainId}/${pdoc.docId}`;
-        for (const domainId of [pdoc.domainId, ...union.map((i) => i._id)]) {
+        Promise.all([
-            tasks.push(
+            ctx.sonic.flusho('problem', `${pdoc.domainId}@title`, id)
-                ctx.sonic.flusho('problem', `${domainId}@title`, id)
+                .then(() => ctx.sonic.push('problem', `${pdoc.domainId}@title`, id, `${pdoc.pid || ''} ${pdoc.title} ${pdoc.tag?.join(' ')}`)),
-                    .then(() => ctx.sonic.push('problem', `${domainId}@title`, id, `${pdoc.pid || ''} ${pdoc.title} ${pdoc.tag?.join(' ')}`)),
+            ctx.sonic.flusho('problem', `${pdoc.domainId}@content`, id)
-                ctx.sonic.flusho('problem', `${domainId}@content`, id)
+                .then(() => ctx.sonic.push('problem', `${pdoc.domainId}@content`, id, pdoc.content.toString())),
-                    .then(() => ctx.sonic.push('problem', `${domainId}@content`, id, pdoc.content.toString())),
+        ]).catch((e) => logger.error(e));
-            );
-        }
-        Promise.all(tasks).catch((e) => logger.error(e));
     });
 
     ctx.on('problem/del', async (domainId, docId) => {
-        const union = await DomainModel.getMulti({ union: domainId }).toArray();
-        const tasks = [];
         const id = `${domainId}/${docId}`;
-        for (const domain of [domainId, ...union.map((i) => i._id)]) {
+        await Promise.all([
-            tasks.push(
+            ctx.sonic.flusho('problem', `${domainId}@title`, id),
-                ctx.sonic.flusho('problem', `${domain}@title`, id),
+            ctx.sonic.flusho('problem', `${domainId}@content`, id),
-                ctx.sonic.flusho('problem', `${domain}@content`, id),
+        ]);
-            );
-        }
-        await Promise.all(tasks);
     });
 
     ctx.addScript(

packages/ui-default/templates/components/problem.html

@@ -1,17 +1,15 @@
-{% macro render_problem_title(pdoc, tdoc=none, show_tags=true, show_invisible_flag=true, invalid=false, inline=false, small=false, alphabetic=false) %}
+{% macro render_problem_title(pdoc, tdoc=none, show_tags=true, show_invisible_flag=true, invalid=false, inline=false, show_pid=true, small=false, alphabetic=false) %}
 {%- if not invalid -%}
   {% set _linkArgs = { pid:pdoc.pid|default(pdoc.docId) } %}
   {% if tdoc %}{{ set(_linkArgs, 'query', {tid:tdoc.docId}) }}{% endif %}
-  {% if pdoc.domainId !== handler.args.domainId %}{{ set(_linkArgs, 'domainId', pdoc.domainId) }}{% endif %}
   <a href="{{ url('problem_detail', _linkArgs) }}"{% if small %} data-tooltip="{{ pdoc.title }}"{% endif %}>
 {%- endif -%}
-{%- if pdoc.domainId !== handler.args.domainId -%}<b>{{ pdoc.domainId }}#</b>{%- endif -%}
+{% if show_pid %}
-{%- if tdoc and alphabetic -%}
+<b>{%- if tdoc and alphabetic -%}{{ String.fromCharCode(65+tdoc.pids.indexOf(pdoc.docId)) }}
-  <b>{{ String.fromCharCode(65 + tdoc.pids.indexOf(pdoc.docId)) if tdoc.pids.includes(pdoc.docId) else '#' }}</b>
+{%- elif pdoc.pid and pdoc.pid.includes('-') -%}{{ pdoc.pid.split('-').join('#') }}
-{%- else -%}
+{%- else -%}{{ pdoc.pid|default(pdoc.docId) }}{%- endif -%}</b>
-  <b>{{ pdoc.pid or ('P'+pdoc.docId if pdoc.domainId === handler.args.domainId else pdoc.docId) }}</b>
+{% endif %}
-{%- endif -%}
+{%- if not small -%}{% if show_pid %}&nbsp;&nbsp;{% endif %}{{ pdoc.title }}{%- endif -%}
-{%- if not small -%}&nbsp;&nbsp;{{ pdoc.title }}{%- endif -%}
 {%- if not invalid -%}
   </a>
 {%- endif -%}

packages/ui-default/templates/partials/homepage/recent_problems.html

@@ -8,7 +8,7 @@
     <div class="section__body typo no-media col--problem-name">
       {%- for pdoc in pdocs -%}
         <p>
-          {{ problem.render_problem_title(pdoc, show_tags=false, show_invisible_flag=false) }}
+          {{ problem.render_problem_title(pdoc, show_tags=false, show_invisible_flag=false, show_pid=false) }}
           <span style="float: right">{{ datetimeSpan(pdoc._id)|safe }}</span>
         </p>
       {%- endfor -%}

packages/ui-default/templates/problem_detail.html

@@ -45,7 +45,13 @@
               </button>
             </form>
           {% endif %}
-          #{{ String.fromCharCode(65+tdoc.pids.indexOf(pdoc.docId)) if tdoc else pdoc.pid|default(pdoc.docId) }}. {{ pdoc.title }}
+          {%- if tdoc -%}
+            {{ String.fromCharCode(65+tdoc.pids.indexOf(pdoc.docId)) }}
+          {%- elif pdoc.pid and pdoc.pid.includes('-') -%}
+            {{ pdoc.pid.split('-').join('#') }}
+          {%- else -%}
+            #{{ pdoc.pid|default(pdoc.docId) }}
+          {%- endif -%}. {{ pdoc.title }}
         </h1>
         {% if tdoc and tdoc.pids|length <= 26 %}
           <div class="section__tools contest-problems">