vjudge: Codeforces::Gym

packages/vjudge/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hydrooj/vjudge",
-    "version": "1.5.0-next.3",
+    "version": "1.5.0-next.4",
     "description": "Submit problems to remote oj",
     "main": "package.json",
     "repository": "https://github.com/hydro-dev/Hydro.git",

packages/vjudge/src/interface.ts

@@ -8,6 +8,7 @@ export interface RemoteAccount {
     proxy?: string;
     query?: string;
     frozen?: string;
+    problemLists?: string[];
 }
 declare module 'hydrooj/src/interface' {
     interface Collections {
@@ -21,7 +22,7 @@ declare module 'hydrooj/src/interface' {
 }
 export interface IBasicProvider {
     ensureLogin(): Promise<boolean | string>;
-    getProblem(id: string): Promise<{
+    getProblem(id: string, meta: Record<string, any>): Promise<{
         title: string;
         data: Record<string, any>;
         files: Record<string, any>;
@@ -29,7 +30,8 @@ export interface IBasicProvider {
         content: string;
         difficulty?: number;
     }>;
-    listProblem(page: number, resync: boolean): Promise<string[]>;
+    entryProblemLists?: string[];
+    listProblem(page: number, resync: boolean, listId: string): Promise<string[]>;
     submitProblem(id: string, lang: string, code: string, info: any, next: any, end: any): Promise<string>;
     waitForSubmission(id: string, next: any, end: any): Promise<void>;
 }

packages/vjudge/src/model.ts

@@ -19,14 +19,25 @@ const syncing = {};
 
 class Service {
     api: IBasicProvider;
+    problemLists: Set<string>;
+    listUpdated = false;
 
     constructor(public Provider: BasicProvider, public account: RemoteAccount) {
         this.api = new Provider(account, async (data) => {
             await coll.updateOne({ _id: account._id }, { $set: data });
         });
+        this.problemLists = Set.union(this.api.entryProblemLists || ['main'], this.account.problemLists || []);
         this.main().catch((e) => logger.error(`Error occured in ${account.type}/${account.handle}`, e));
     }
 
+    async addProblemList(name: string) {
+        if (this.problemLists.has(name)) return;
+        this.problemLists.add(name);
+        this.listUpdated = true;
+        logger.info(`Discovered new problem list: ${name}`);
+        await coll.updateOne({ _id: this.account._id }, { $push: { problemLists: name } });
+    }
+
     async judge(task) {
         const next = (payload) => Judge.next({ ...payload, rid: task.rid });
         const end = (payload) => Judge.end({ ...payload, rid: task.rid });
@@ -42,33 +53,39 @@ class Service {
         }
     }
 
-    async sync(domainId: string, resync = false) {
+    async sync(domainId: string, resync = false, list: string) {
         let page = 1;
-        let pids = await this.api.listProblem(page, resync);
+        let pids = await this.api.listProblem(page, resync, list);
         while (pids.length) {
             logger.info(`${domainId}: Syncing page ${page}`);
-            for (const pid of pids) {
+            for (const id of pids) {
+                if (id.startsWith('LIST::')) {
+                    await this.addProblemList(id.split('::')[1]);
+                    continue;
+                }
+                const [pid, metastr = '{}'] = id.split('#');
+                const meta = JSON.parse(metastr);
                 if (await ProblemModel.get(domainId, pid) || syncing[`${domainId}/${pid}`]) continue;
                 syncing[`${domainId}/${pid}`] = true;
                 try {
-                    const res = await this.api.getProblem(pid);
+                    const res = await this.api.getProblem(pid, meta);
                     if (!res) continue;
-                    const id = await ProblemModel.add(domainId, pid, res.title, res.content, 1, res.tag, false);
+                    const docId = await ProblemModel.add(domainId, pid, res.title, res.content, 1, res.tag, false);
-                    if (res.difficulty) await ProblemModel.edit(domainId, id, { difficulty: res.difficulty });
+                    if (res.difficulty) await ProblemModel.edit(domainId, docId, { difficulty: res.difficulty });
                     for (const key in res.files) {
-                        await ProblemModel.addAdditionalFile(domainId, id, key, res.files[key]);
+                        await ProblemModel.addAdditionalFile(domainId, docId, key, res.files[key]);
                     }
                     for (const key in res.data) {
-                        await ProblemModel.addTestdata(domainId, id, key, res.data[key]);
+                        await ProblemModel.addTestdata(domainId, docId, key, res.data[key]);
                     }
-                    logger.info(`${domainId}: problem ${id} sync done`);
+                    logger.info(`${domainId}: problem ${docId}(${pid}) sync done`);
                 } finally {
                     delete syncing[`${domainId}/${pid}`];
                 }
                 await sleep(5000);
             }
             page++;
-            pids = await this.api.listProblem(page, resync);
+            pids = await this.api.listProblem(page, resync, list);
         }
     }
 
@@ -88,11 +105,22 @@ class Service {
         setInterval(() => this.login(), 1 * 3600 * 1000);
         TaskModel.consume({ type: 'remotejudge', subType: this.account.type }, this.judge.bind(this), false);
         const ddocs = await DomainModel.getMulti({ mount: this.account.type }).toArray();
-        for (const ddoc of ddocs) {
+        do {
-            if (!ddoc.syncDone) await this.sync(ddoc._id, false);
+            this.listUpdated = false;
-            else await this.sync(ddoc._id, true);
+            for (const listName of this.problemLists) {
-            await DomainModel.edit(ddoc._id, { syncDone: true });
+                for (const ddoc of ddocs) {
-        }
+                    if (ddoc.syncDone === true) {
+                        await DomainModel.edit(ddoc._id, { syncDone: { main: true } });
+                        ddoc.syncDone = { main: true };
+                    }
+                    if (!ddoc.syncDone?.[listName]) await this.sync(ddoc._id, false, listName);
+                    else await this.sync(ddoc._id, true, listName);
+                    await DomainModel.edit(ddoc._id, { [`syncDone.${listName}`]: true });
+                    ddoc.syncDone ||= {};
+                    ddoc.syncDone[listName] = true;
+                }
+            }
+        } while (this.listUpdated);
     }
 }
 

packages/vjudge/src/providers/codeforces.ts

@@ -1,6 +1,7 @@
 /* eslint-disable no-await-in-loop */
 import { PassThrough } from 'stream';
 import findChrome from 'chrome-finder';
+import yaml from 'js-yaml';
 import { JSDOM } from 'jsdom';
 import type { Browser, Page } from 'puppeteer';
 import puppeteer from 'puppeteer-extra';
@@ -224,29 +225,40 @@ export default class CodeforcesProvider implements IBasicProvider {
         return false;
     }
 
-    async getPdfProblem(id: string) {
+    async getPdfProblem(id: string, meta: Record<string, any>) {
-        const [, contestId, problemId] = id.startsWith('P921')
+        const [, , contestId, problemId] = id.startsWith('P921')
             ? ['', '921', '01']
-            : /^P(\d+)([A-Z][0-9]*)$/.exec(id);
+            : /^(P|GYM)(\d+)([A-Z][0-9]*)$/.exec(id);
         const file = new PassThrough();
-        this.get(`/problemset/problem/${contestId}/${problemId}`).pipe(file);
+        this.get(id.startsWith('GYM')
+            ? `/gym/${contestId}/problem/${problemId}`
+            : `/problemset/problem/${contestId}/${problemId}`).pipe(file);
         return {
-            title: '__NO_TITLE__',
+            title: meta.title || '__NO_TITLE__',
-            data: { 'config.yaml': Buffer.from(`type: remote_judge\nsubType: codeforces\ntarget: ${id}`) },
+            data: {
+                'config.yaml': Buffer.from(yaml.dump({
+                    ...meta,
+                    type: 'remote_judge',
+                    subType: 'codeforces',
+                    target: id,
+                })),
+            },
             files: { 'problem.pdf': file },
             tag: [],
-            content: '@[pdf](file://problem.pdf)',
+            content: '@[pdf](file://problem.pdf?noDisposition=1)',
         };
     }
 
-    async getProblem(id: string) {
+    async getProblem(id: string, meta: Record<string, any>) {
         logger.info(id);
         if (id === 'P936E') return null; // Problem Missing
-        const [, contestId, problemId] = id.startsWith('P921')
+        const [, , contestId, problemId] = id.startsWith('P921')
             ? ['', '921', '01']
-            : /^P(\d+)([A-Z][0-9]*)$/.exec(id);
+            : /^(P|GYM)(\d+)([A-Z][0-9]*)$/.exec(id);
-        const res = await this.get(`/problemset/problem/${contestId}/${problemId}`);
+        const res = await this.get(id.startsWith('GYM')
-        if (!res.text) return await this.getPdfProblem(id);
+            ? `/gym/${contestId}/problem/${problemId}`
+            : `/problemset/problem/${contestId}/${problemId}`);
+        if (!res.text) return await this.getPdfProblem(id, meta);
         const $dom = new JSDOM(res.text.replace(/\$\$\$/g, '$'));
         const tag = Array.from($dom.window.document.querySelectorAll('.tag-box')).map((i) => i.textContent.trim());
         const text = $dom.window.document.querySelector('.problem-statement').innerHTML;
@@ -309,13 +321,38 @@ export default class CodeforcesProvider implements IBasicProvider {
         };
     }
 
-    async listProblem(page: number, resync = false) {
+    // TL;DR; add `gym` to this list to enable codeforces gym
+    entryProblemLists = ['main'];
+    async listProblem(page: number, resync = false, listName: string) {
         if (resync && page > 1) return [];
-        const res = await this.get(`/problemset/page/${page}`);
+        if (resync && listName.startsWith('GYM')) return [];
-        const $dom = new JSDOM(res.text);
+        if (listName.startsWith('GYM') && page > 1) return [];
-        const index = $dom.window.document.querySelector('.page-index.active').getAttribute('pageindex');
+        const res = await this.get(listName === 'main'
-        if (index !== page.toString()) return [];
+            ? `/problemset/page/${page}`
-        return Array.from($dom.window.document.querySelectorAll('.id>a')).map((i) => `P${i.innerHTML.trim()}`);
+            : listName === 'gym'
+                ? `/gyms/page/${page}`
+                : `/gym/${listName.split('GYM')[1]}`,
+        );
+        const { window: { document } } = new JSDOM(res.text);
+        if (['gym', 'main'].includes(listName)) {
+            const index = document.querySelector('.page-index.active').getAttribute('pageindex');
+            if (index !== page.toString()) return [];
+        }
+        if (listName === 'main') {
+            return Array.from(document.querySelectorAll('.id>a')).map((i) => `P${i.innerHTML.trim()}`);
+        }
+        if (listName === 'gym') {
+            return Array.from(document.querySelectorAll('[data-contestId]')).map((i) => `LIST::GYM${i.getAttribute('data-contestId')}`);
+        }
+        return Array.from(document.querySelectorAll('.id a')).map((i) => {
+            const detail = i.parentElement.parentElement.children[1].children[0];
+            return `${listName}${i.textContent.trim()}#${JSON.stringify({
+                title: detail.children[0].children[0].textContent.trim(),
+                time: `${/(\d+m?) *s/.exec(detail.children[1].childNodes[2].textContent)[1]}s`,
+                memory: `${/(\d+) *MB/.exec(detail.children[1].childNodes[2].textContent)[1]}m`,
+                filename: /(\w+)\.in/.exec(detail.children[1].textContent)?.[1],
+            })}`;
+        });
     }
 
     async submitProblem(id: string, lang: string, code: string, info) {
@@ -326,25 +363,35 @@ export default class CodeforcesProvider implements IBasicProvider {
             if (typeof comment === 'string') code = `${comment} ${msg}\n${code}`;
             else if (comment instanceof Array) code = `${comment[0]} ${msg} ${comment[1]}\n${code}`;
         }
-        const [csrf, ftaa, bfaa] = await this.getCsrfToken('/problemset/submit');
+        const contestId = /(\d+)/.exec(id)?.[1];
+        const [csrf, ftaa, bfaa] = await this.getCsrfToken((+contestId) < 100000
+            ? '/problemset/submit'
+            : `/gym/${contestId}/submit`);
         // TODO check submit time to ensure submission
-        await this.post(`/problemset/submit?csrf_token=${csrf}`).send({
+        await this.post(`/${(+contestId) < 100000 ? 'problemset' : `gym/${contestId}`}/submit?csrf_token=${csrf}`).send({
             csrf_token: csrf,
             action: 'submitSolutionFormSubmitted',
             programTypeId,
-            submittedProblemCode: id.split('P')[1],
             source: code,
             tabsize: 4,
             sourceFile: '',
             ftaa,
             bfaa,
             _tta: this.tta(this.getCookie('39ce7')),
-            sourceCodeConfirmed: true,
+            ...(+contestId < 100000)
+                ? {
+                    submittedProblemCode: id.split('P')[1],
+                    sourceCodeConfirmed: true,
+                } : {
+                    submittedProblemIndex: /([A-Z])$/.exec(id)?.[1],
+                },
         });
-        const { text: status } = await this.get('/problemset/status?my=on');
+        const { text: status } = await this.get((+contestId) < 100000
-        const $dom = new JSDOM(status);
+            ? '/problemset/status?my=on'
-        this.csrf = $dom.window.document.querySelector('meta[name="X-Csrf-Token"]').getAttribute('content');
+            : `/gym/${contestId}/my`);
-        return $dom.window.document.querySelector('[data-submission-id]').getAttribute('data-submission-id');
+        const { window: { document } } = new JSDOM(status);
+        this.csrf = document.querySelector('meta[name="X-Csrf-Token"]').getAttribute('content');
+        return document.querySelector('[data-submission-id]').getAttribute('data-submission-id');
     }
 
     async waitForSubmission(id: string, next, end) {