From 6f791903d4f36b1987879bdda96062b1957e92a4 Mon Sep 17 00:00:00 2001 From: undefined Date: Fri, 8 Jul 2022 09:28:29 +0800 Subject: [PATCH] vjudge: Codeforces::Gym --- packages/vjudge/package.json | 2 +- packages/vjudge/src/interface.ts | 6 +- packages/vjudge/src/model.ts | 58 ++++++++---- packages/vjudge/src/providers/codeforces.ts | 99 +++++++++++++++------ 4 files changed, 121 insertions(+), 44 deletions(-) diff --git a/packages/vjudge/package.json b/packages/vjudge/package.json index 0ffc5b872..469f24fe3 100644 --- a/packages/vjudge/package.json +++ b/packages/vjudge/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/vjudge", - "version": "1.5.0-next.3", + "version": "1.5.0-next.4", "description": "Submit problems to remote oj", "main": "package.json", "repository": "https://github.com/hydro-dev/Hydro.git", diff --git a/packages/vjudge/src/interface.ts b/packages/vjudge/src/interface.ts index 48288a437..4c8c944f4 100644 --- a/packages/vjudge/src/interface.ts +++ b/packages/vjudge/src/interface.ts @@ -8,6 +8,7 @@ export interface RemoteAccount { proxy?: string; query?: string; frozen?: string; + problemLists?: string[]; } declare module 'hydrooj/src/interface' { interface Collections { @@ -21,7 +22,7 @@ declare module 'hydrooj/src/interface' { } export interface IBasicProvider { ensureLogin(): Promise; - getProblem(id: string): Promise<{ + getProblem(id: string, meta: Record): Promise<{ title: string; data: Record; files: Record; @@ -29,7 +30,8 @@ export interface IBasicProvider { content: string; difficulty?: number; }>; - listProblem(page: number, resync: boolean): Promise; + entryProblemLists?: string[]; + listProblem(page: number, resync: boolean, listId: string): Promise; submitProblem(id: string, lang: string, code: string, info: any, next: any, end: any): Promise; waitForSubmission(id: string, next: any, end: any): Promise; } diff --git a/packages/vjudge/src/model.ts b/packages/vjudge/src/model.ts index f7af2ea97..5dbb568a5 100644 --- a/packages/vjudge/src/model.ts +++ b/packages/vjudge/src/model.ts @@ -19,14 +19,25 @@ const syncing = {}; class Service { api: IBasicProvider; + problemLists: Set; + listUpdated = false; constructor(public Provider: BasicProvider, public account: RemoteAccount) { this.api = new Provider(account, async (data) => { await coll.updateOne({ _id: account._id }, { $set: data }); }); + this.problemLists = Set.union(this.api.entryProblemLists || ['main'], this.account.problemLists || []); this.main().catch((e) => logger.error(`Error occured in ${account.type}/${account.handle}`, e)); } + async addProblemList(name: string) { + if (this.problemLists.has(name)) return; + this.problemLists.add(name); + this.listUpdated = true; + logger.info(`Discovered new problem list: ${name}`); + await coll.updateOne({ _id: this.account._id }, { $push: { problemLists: name } }); + } + async judge(task) { const next = (payload) => Judge.next({ ...payload, rid: task.rid }); const end = (payload) => Judge.end({ ...payload, rid: task.rid }); @@ -42,33 +53,39 @@ class Service { } } - async sync(domainId: string, resync = false) { + async sync(domainId: string, resync = false, list: string) { let page = 1; - let pids = await this.api.listProblem(page, resync); + let pids = await this.api.listProblem(page, resync, list); while (pids.length) { logger.info(`${domainId}: Syncing page ${page}`); - for (const pid of pids) { + for (const id of pids) { + if (id.startsWith('LIST::')) { + await this.addProblemList(id.split('::')[1]); + continue; + } + const [pid, metastr = '{}'] = id.split('#'); + const meta = JSON.parse(metastr); if (await ProblemModel.get(domainId, pid) || syncing[`${domainId}/${pid}`]) continue; syncing[`${domainId}/${pid}`] = true; try { - const res = await this.api.getProblem(pid); + const res = await this.api.getProblem(pid, meta); if (!res) continue; - const id = await ProblemModel.add(domainId, pid, res.title, res.content, 1, res.tag, false); - if (res.difficulty) await ProblemModel.edit(domainId, id, { difficulty: res.difficulty }); + const docId = await ProblemModel.add(domainId, pid, res.title, res.content, 1, res.tag, false); + if (res.difficulty) await ProblemModel.edit(domainId, docId, { difficulty: res.difficulty }); for (const key in res.files) { - await ProblemModel.addAdditionalFile(domainId, id, key, res.files[key]); + await ProblemModel.addAdditionalFile(domainId, docId, key, res.files[key]); } for (const key in res.data) { - await ProblemModel.addTestdata(domainId, id, key, res.data[key]); + await ProblemModel.addTestdata(domainId, docId, key, res.data[key]); } - logger.info(`${domainId}: problem ${id} sync done`); + logger.info(`${domainId}: problem ${docId}(${pid}) sync done`); } finally { delete syncing[`${domainId}/${pid}`]; } await sleep(5000); } page++; - pids = await this.api.listProblem(page, resync); + pids = await this.api.listProblem(page, resync, list); } } @@ -88,11 +105,22 @@ class Service { setInterval(() => this.login(), 1 * 3600 * 1000); TaskModel.consume({ type: 'remotejudge', subType: this.account.type }, this.judge.bind(this), false); const ddocs = await DomainModel.getMulti({ mount: this.account.type }).toArray(); - for (const ddoc of ddocs) { - if (!ddoc.syncDone) await this.sync(ddoc._id, false); - else await this.sync(ddoc._id, true); - await DomainModel.edit(ddoc._id, { syncDone: true }); - } + do { + this.listUpdated = false; + for (const listName of this.problemLists) { + for (const ddoc of ddocs) { + if (ddoc.syncDone === true) { + await DomainModel.edit(ddoc._id, { syncDone: { main: true } }); + ddoc.syncDone = { main: true }; + } + if (!ddoc.syncDone?.[listName]) await this.sync(ddoc._id, false, listName); + else await this.sync(ddoc._id, true, listName); + await DomainModel.edit(ddoc._id, { [`syncDone.${listName}`]: true }); + ddoc.syncDone ||= {}; + ddoc.syncDone[listName] = true; + } + } + } while (this.listUpdated); } } diff --git a/packages/vjudge/src/providers/codeforces.ts b/packages/vjudge/src/providers/codeforces.ts index 913ac8461..aeb763fb8 100644 --- a/packages/vjudge/src/providers/codeforces.ts +++ b/packages/vjudge/src/providers/codeforces.ts @@ -1,6 +1,7 @@ /* eslint-disable no-await-in-loop */ import { PassThrough } from 'stream'; import findChrome from 'chrome-finder'; +import yaml from 'js-yaml'; import { JSDOM } from 'jsdom'; import type { Browser, Page } from 'puppeteer'; import puppeteer from 'puppeteer-extra'; @@ -224,29 +225,40 @@ export default class CodeforcesProvider implements IBasicProvider { return false; } - async getPdfProblem(id: string) { - const [, contestId, problemId] = id.startsWith('P921') + async getPdfProblem(id: string, meta: Record) { + const [, , contestId, problemId] = id.startsWith('P921') ? ['', '921', '01'] - : /^P(\d+)([A-Z][0-9]*)$/.exec(id); + : /^(P|GYM)(\d+)([A-Z][0-9]*)$/.exec(id); const file = new PassThrough(); - this.get(`/problemset/problem/${contestId}/${problemId}`).pipe(file); + this.get(id.startsWith('GYM') + ? `/gym/${contestId}/problem/${problemId}` + : `/problemset/problem/${contestId}/${problemId}`).pipe(file); return { - title: '__NO_TITLE__', - data: { 'config.yaml': Buffer.from(`type: remote_judge\nsubType: codeforces\ntarget: ${id}`) }, + title: meta.title || '__NO_TITLE__', + data: { + 'config.yaml': Buffer.from(yaml.dump({ + ...meta, + type: 'remote_judge', + subType: 'codeforces', + target: id, + })), + }, files: { 'problem.pdf': file }, tag: [], - content: '@[pdf](file://problem.pdf)', + content: '@[pdf](file://problem.pdf?noDisposition=1)', }; } - async getProblem(id: string) { + async getProblem(id: string, meta: Record) { logger.info(id); if (id === 'P936E') return null; // Problem Missing - const [, contestId, problemId] = id.startsWith('P921') + const [, , contestId, problemId] = id.startsWith('P921') ? ['', '921', '01'] - : /^P(\d+)([A-Z][0-9]*)$/.exec(id); - const res = await this.get(`/problemset/problem/${contestId}/${problemId}`); - if (!res.text) return await this.getPdfProblem(id); + : /^(P|GYM)(\d+)([A-Z][0-9]*)$/.exec(id); + const res = await this.get(id.startsWith('GYM') + ? `/gym/${contestId}/problem/${problemId}` + : `/problemset/problem/${contestId}/${problemId}`); + if (!res.text) return await this.getPdfProblem(id, meta); const $dom = new JSDOM(res.text.replace(/\$\$\$/g, '$')); const tag = Array.from($dom.window.document.querySelectorAll('.tag-box')).map((i) => i.textContent.trim()); const text = $dom.window.document.querySelector('.problem-statement').innerHTML; @@ -309,13 +321,38 @@ export default class CodeforcesProvider implements IBasicProvider { }; } - async listProblem(page: number, resync = false) { + // TL;DR; add `gym` to this list to enable codeforces gym + entryProblemLists = ['main']; + async listProblem(page: number, resync = false, listName: string) { if (resync && page > 1) return []; - const res = await this.get(`/problemset/page/${page}`); - const $dom = new JSDOM(res.text); - const index = $dom.window.document.querySelector('.page-index.active').getAttribute('pageindex'); - if (index !== page.toString()) return []; - return Array.from($dom.window.document.querySelectorAll('.id>a')).map((i) => `P${i.innerHTML.trim()}`); + if (resync && listName.startsWith('GYM')) return []; + if (listName.startsWith('GYM') && page > 1) return []; + const res = await this.get(listName === 'main' + ? `/problemset/page/${page}` + : listName === 'gym' + ? `/gyms/page/${page}` + : `/gym/${listName.split('GYM')[1]}`, + ); + const { window: { document } } = new JSDOM(res.text); + if (['gym', 'main'].includes(listName)) { + const index = document.querySelector('.page-index.active').getAttribute('pageindex'); + if (index !== page.toString()) return []; + } + if (listName === 'main') { + return Array.from(document.querySelectorAll('.id>a')).map((i) => `P${i.innerHTML.trim()}`); + } + if (listName === 'gym') { + return Array.from(document.querySelectorAll('[data-contestId]')).map((i) => `LIST::GYM${i.getAttribute('data-contestId')}`); + } + return Array.from(document.querySelectorAll('.id a')).map((i) => { + const detail = i.parentElement.parentElement.children[1].children[0]; + return `${listName}${i.textContent.trim()}#${JSON.stringify({ + title: detail.children[0].children[0].textContent.trim(), + time: `${/(\d+m?) *s/.exec(detail.children[1].childNodes[2].textContent)[1]}s`, + memory: `${/(\d+) *MB/.exec(detail.children[1].childNodes[2].textContent)[1]}m`, + filename: /(\w+)\.in/.exec(detail.children[1].textContent)?.[1], + })}`; + }); } async submitProblem(id: string, lang: string, code: string, info) { @@ -326,25 +363,35 @@ export default class CodeforcesProvider implements IBasicProvider { if (typeof comment === 'string') code = `${comment} ${msg}\n${code}`; else if (comment instanceof Array) code = `${comment[0]} ${msg} ${comment[1]}\n${code}`; } - const [csrf, ftaa, bfaa] = await this.getCsrfToken('/problemset/submit'); + const contestId = /(\d+)/.exec(id)?.[1]; + const [csrf, ftaa, bfaa] = await this.getCsrfToken((+contestId) < 100000 + ? '/problemset/submit' + : `/gym/${contestId}/submit`); // TODO check submit time to ensure submission - await this.post(`/problemset/submit?csrf_token=${csrf}`).send({ + await this.post(`/${(+contestId) < 100000 ? 'problemset' : `gym/${contestId}`}/submit?csrf_token=${csrf}`).send({ csrf_token: csrf, action: 'submitSolutionFormSubmitted', programTypeId, - submittedProblemCode: id.split('P')[1], source: code, tabsize: 4, sourceFile: '', ftaa, bfaa, _tta: this.tta(this.getCookie('39ce7')), - sourceCodeConfirmed: true, + ...(+contestId < 100000) + ? { + submittedProblemCode: id.split('P')[1], + sourceCodeConfirmed: true, + } : { + submittedProblemIndex: /([A-Z])$/.exec(id)?.[1], + }, }); - const { text: status } = await this.get('/problemset/status?my=on'); - const $dom = new JSDOM(status); - this.csrf = $dom.window.document.querySelector('meta[name="X-Csrf-Token"]').getAttribute('content'); - return $dom.window.document.querySelector('[data-submission-id]').getAttribute('data-submission-id'); + const { text: status } = await this.get((+contestId) < 100000 + ? '/problemset/status?my=on' + : `/gym/${contestId}/my`); + const { window: { document } } = new JSDOM(status); + this.csrf = document.querySelector('meta[name="X-Csrf-Token"]').getAttribute('content'); + return document.querySelector('[data-submission-id]').getAttribute('data-submission-id'); } async waitForSubmission(id: string, next, end) {