From 939584ec5ef146efc74222713441636ca3eb745a Mon Sep 17 00:00:00 2001 From: undefined Date: Mon, 21 Jun 2021 13:39:16 +0800 Subject: [PATCH] vjudge: codeforces --- packages/vjudge/package.json | 18 ++ packages/vjudge/src/interface.ts | 34 +++ packages/vjudge/src/model.ts | 75 +++++++ packages/vjudge/src/providers/codeforces.ts | 231 ++++++++++++++++++++ packages/vjudge/src/providers/index.ts | 3 + 5 files changed, 361 insertions(+) create mode 100644 packages/vjudge/package.json create mode 100644 packages/vjudge/src/interface.ts create mode 100644 packages/vjudge/src/model.ts create mode 100644 packages/vjudge/src/providers/codeforces.ts create mode 100644 packages/vjudge/src/providers/index.ts diff --git a/packages/vjudge/package.json b/packages/vjudge/package.json new file mode 100644 index 000000000..3d5a3e680 --- /dev/null +++ b/packages/vjudge/package.json @@ -0,0 +1,18 @@ +{ + "name": "@hydrooj/vjudge", + "version": "1.0.0", + "description": "Submit problems to remote oj", + "main": "package.json", + "repository": "https://github.com/hydro-dev/Hydro.git", + "author": "undefined", + "license": "AGPL-3.0-or-later", + "preferUnplugged": true, + "dependencies": { + "@hydrooj/utils": "workspace:packages/utils", + "jsdom": "^16.6.0", + "superagent": "^6.1.0" + }, + "devDependencies": { + "@types/jsdom": "^16.2.11" + } +} diff --git a/packages/vjudge/src/interface.ts b/packages/vjudge/src/interface.ts new file mode 100644 index 000000000..7492e2d32 --- /dev/null +++ b/packages/vjudge/src/interface.ts @@ -0,0 +1,34 @@ +export interface RemoteAccount { + _id: string; + type: string; + cookie?: string[]; + handle: string; + password: string; +} +declare module 'hydrooj/dist/interface' { + interface Collections { + vjudge: RemoteAccount; + } + + interface DomainDoc { + mount: string; + mountInfo: any; + } +} +export interface IBasicProvider { + ensureLogin(): Promise; + getProblem(id: string): Promise<{ + title: string; + data: Record; + files: Record; + tag: string[]; + content: string; + }>; + listProblem(page: number): Promise; + submitProblem(id: string, lang: string, code: string): Promise; + waitForSubmission(id: string, next: any, end: any): Promise; +} + +export interface BasicProvider { + new(account: RemoteAccount, save: (data: any) => Promise): IBasicProvider +} diff --git a/packages/vjudge/src/model.ts b/packages/vjudge/src/model.ts new file mode 100644 index 000000000..a38ee45cc --- /dev/null +++ b/packages/vjudge/src/model.ts @@ -0,0 +1,75 @@ +/* eslint-disable no-await-in-loop */ +import { Logger } from 'hydrooj/dist/logger'; +import db from 'hydrooj/dist/service/db'; +import * as bus from 'hydrooj/dist/service/bus'; +import * as Judge from 'hydrooj/dist/handler/judge'; +import TaskModel from 'hydrooj/dist/model/task'; +import DomainModel from 'hydrooj/dist/model/domain'; +import ProblemModel from 'hydrooj/dist/model/problem'; +import { sleep } from '@hydrooj/utils/lib/utils'; +import providers from './providers/index'; +import { BasicProvider, IBasicProvider, RemoteAccount } from './interface'; + +const coll = db.collection('vjudge'); +const Pool = {}; +const logger = new Logger('vjudge'); + +class Service { + api: IBasicProvider; + + constructor(public Provider: BasicProvider, public account: RemoteAccount) { + this.api = new Provider(account, async (data) => { + await coll.updateOne({ _id: account._id }, { $set: data }); + }); + this.main(); + } + + async judge(task) { + const rid = await this.api.submitProblem(task.target, task.lang, task.code); + const next = (payload) => Judge.next({ ...payload, rid: task.rid }); + const end = (payload) => Judge.end({ ...payload, rid: task.rid }); + await this.api.waitForSubmission(rid, next, end); + } + + async sync(domainId: string) { + let page = 1; + let pids = await this.api.listProblem(page); + while (pids.length) { + logger.info(`${domainId}: Syncing page ${page}`); + for (const pid of pids) { + if (await ProblemModel.get(domainId, pid)) continue; + const res = await this.api.getProblem(pid); + const id = await ProblemModel.add(domainId, pid, res.title, res.content, 1, res.tag, false); + for (const key in res.files) { + await ProblemModel.addAdditionalFile(domainId, id, key, res.files[key]); + } + for (const key in res.data) { + await ProblemModel.addTestdata(domainId, id, key, res.data[key]); + } + logger.info(`${domainId}: problem ${id} sync done`); + await sleep(5000); + } + page++; + pids = await this.api.listProblem(page); + } + } + + async main() { + const login = await this.api.ensureLogin(); + if (login) logger.info(`${this.account.type}/${this.account.handle}: logged in`); + else return; + TaskModel.consume({ type: 'remotejudge', subType: this.account.type }, this.judge.bind(this)); + const ddocs = await DomainModel.getMulti({ mount: this.account.type }).toArray(); + for (const ddoc of ddocs) await this.sync(ddoc._id); + } +} + +async function loadAccounts() { + const accounts = await coll.find().toArray(); + for (const account of accounts) { + if (!providers[account.type]) continue; + Pool[`${account.type}/${account.handle}`] = new Service(providers[account.type], account); + } +} + +bus.on('app/started', loadAccounts); diff --git a/packages/vjudge/src/providers/codeforces.ts b/packages/vjudge/src/providers/codeforces.ts new file mode 100644 index 000000000..cb1ba02e7 --- /dev/null +++ b/packages/vjudge/src/providers/codeforces.ts @@ -0,0 +1,231 @@ +/* eslint-disable no-await-in-loop */ +import superagent from 'superagent'; +import { JSDOM } from 'jsdom'; +import { STATUS } from '@hydrooj/utils/lib/status'; +import { sleep } from '@hydrooj/utils/lib/utils'; +import { buildContent } from 'hydrooj/dist/lib/content'; +import { Logger } from 'hydrooj/dist/logger'; +import { PassThrough } from 'stream'; +import { IBasicProvider, RemoteAccount } from '../interface'; + +const logger = new Logger('codeforces'); + +const VERDICT = { + RUNTIME_ERROR: STATUS.STATUS_RUNTIME_ERROR, + WRONG_ANSWER: STATUS.STATUS_WRONG_ANSWER, + OK: STATUS.STATUS_ACCEPTED, + TIME_LIMIT_EXCEEDED: STATUS.STATUS_TIME_LIMIT_EXCEEDED, + MEMORY_LIMIT_EXCEEDED: STATUS.STATUS_MEMORY_LIMIT_EXCEEDED, + Accepted: STATUS.STATUS_ACCEPTED, + 'Wrong answer': STATUS.STATUS_WRONG_ANSWER, + 'Runtime error': STATUS.STATUS_RUNTIME_ERROR, + 'Time limit exceeded': STATUS.STATUS_TIME_LIMIT_EXCEEDED, + 'Memory limit exceeded': STATUS.STATUS_MEMORY_LIMIT_EXCEEDED, +}; + +export default class CodeforcesProvider implements IBasicProvider { + constructor(public account: RemoteAccount, private save: (data: any) => Promise) { + if (account.cookie) this.cookie = account.cookie; + } + + cookie: string[] = []; + // @ts-ignore + ftaa = String.random(18, 'abcdef1234567890'); + csrf: string; + + get(url: string) { + logger.debug('get', url); + return superagent.get(`https://codeforces.com${url}`).set('Cookie', this.cookie); + } + + post(url: string) { + logger.debug('post', url, this.cookie); + return superagent.post(`https://codeforces.com${url}`).type('form').set('Cookie', this.cookie); + } + + tta(_39ce7: string) { + let _tta = 0; + for (let c = 0; c < _39ce7.length; c++) { + _tta = (_tta + (c + 1) * (c + 2) * _39ce7.charCodeAt(c)) % 1009; + if (c % 3 === 0) _tta++; + if (c % 2 === 0) _tta *= 2; + if (c > 0) _tta -= Math.floor(_39ce7.charCodeAt(Math.floor(c / 2)) / 2) * (_tta % 5); + _tta = ((_tta % 1009) + 1009) % 1009; + } + return _tta; + } + + async getCsrfToken(url: string) { + const { text: html } = await this.get(url); + const $dom = new JSDOM(html); + return $dom.window.document.querySelector('meta[name="X-Csrf-Token"]').getAttribute('content'); + } + + get loggedIn() { + return this.get('/').then(({ text: html }) => { + const $dom = new JSDOM(html); + if (!$dom.window.document.querySelectorAll('a[href="/enter?back=%2F"]').length) return true; + return false; + }); + } + + async ensureLogin() { + if (await this.loggedIn) return true; + logger.info('retry login'); + const csrf_token = await this.getCsrfToken('/enter'); + const res = await this.post('/enter').send({ + csrf_token, + action: 'enter', + ftaa: '', + bfaa: '', + handleOrEmail: this.account.handle, + password: this.account.password, + remember: 'on', + }); + const cookie = res.header['set-cookie']; + if (cookie) { + await this.save({ cookie }); + this.cookie = cookie; + } + if (await this.loggedIn) return true; + return false; + } + + async getProblem(id: string) { + logger.info(id); + const [, contestId, problemId] = /^P(\d+)([A-Z][0-9]?)$/.exec(id); + const res = await this.get(`/problemset/problem/${contestId}/${problemId}`); + const $dom = new JSDOM(res.text.replace(/\$\$\$/g, '$')); + const tag = Array.from($dom.window.document.querySelectorAll('.tag-box')).map((i) => i.textContent.trim()); + const text = $dom.window.document.querySelector('.problem-statement').innerHTML; + const { window: { document } } = new JSDOM(text); + const files = {}; + document.querySelectorAll('img[src]').forEach((ele) => { + const src = ele.getAttribute('src'); + if (!src.startsWith('http')) return; + const file = new PassThrough(); + superagent.get(src).pipe(file); + const fid = String.random(8); + files[`${fid}.png`] = file; + ele.setAttribute('src', `%file%://${fid}.png`); + }); + const title = document.querySelector('.title').innerHTML; + const time = parseInt(document.querySelector('.time-limit').innerHTML.substr(53, 2), 10); + const memory = parseInt(document.querySelector('.memory-limit').innerHTML.substr(55, 4), 10); + document.body.firstChild.remove(); + document.querySelector('.input-specification')?.firstChild.remove(); + document.querySelector('.output-specification')?.firstChild.remove(); + document.querySelector('.note')?.firstChild.remove(); + const input = document.querySelector('.input-specification')?.innerHTML.trim(); + const output = document.querySelector('.output-specification')?.innerHTML.trim(); + const inputs = Array.from(document.querySelectorAll('.input>pre')).map((i) => i.innerHTML.trim()); + const outputs = Array.from(document.querySelectorAll('.output>pre')).map((i) => i.innerHTML.trim()); + const note = document.querySelector('.note')?.innerHTML.trim(); + document.querySelector('.note')?.remove(); + document.querySelector('.sample-tests')?.remove(); + document.querySelectorAll('.section-title').forEach((ele) => { + const e = document.createElement('h2'); + e.innerHTML = ele.innerHTML; + ele.replaceWith(e); + }); + const description = document.body.innerHTML.trim(); + return { + title, + data: { + 'config.yaml': Buffer.from(`time: ${time}s\nmemory: ${memory}m\ntype: remote_judge\nsubType: codeforces\ntarget: ${id}`), + }, + files, + tag, + content: buildContent([ + { + type: 'Text', sectionTitle: 'Description', subType: 'markdown', text: description, + }, + ...input ? [{ + type: 'Text', sectionTitle: 'Input', subType: 'markdown', text: input, + }] : [], + ...output ? [{ + type: 'Text', sectionTitle: 'Output', subType: 'markdown', text: output, + }] : [], + ...inputs.map((_, i) => ({ + type: 'Sample', payload: [inputs[i], outputs[i]], text: '', sectionTitle: 'Samples', + }) as any), + ...note ? [{ + type: 'Text', sectionTitle: 'Note', subType: 'markdown', text: note, + }] : [], + ]), + }; + } + + async listProblem(page: number) { + const res = await this.get(`/problemset/page/${page}`); + const $dom = new JSDOM(res.text); + const index = $dom.window.document.querySelector('.page-index.active').getAttribute('pageindex'); + if (index !== page.toString()) return []; + return Array.from($dom.window.document.querySelectorAll('.id>a')).map((i) => `P${i.innerHTML.trim()}`); + } + + async submitProblem(id: string, lang: string, code: string) { + const programTypeId = lang.includes('codeforces.') ? lang.split('codeforces.')[1] : '42'; + const [, contestId, submittedProblemIndex] = /^P(\d+)([A-Z][0-9]?)$/.exec(id); + const csrf_token = await this.getCsrfToken('/problemset/submit'); + // TODO check submit time to ensure submission + await this.post(`/problemset/submit?csrf_token=${csrf_token}`).send({ + csrf_token, + contestId, + action: 'submitSolutionFormSubmitted', + programTypeId, + submittedProblemIndex, + source: code, + tabsize: 4, + sourceFile: '', + ftaa: '', + bfaa: 'f1b3f18c715565b589b7823cda7448ce', + _tta: 594, + sourceCodeConfirmed: true, + }); + const { text: info } = await this.get('/problemset/status?my=on'); + const $dom = new JSDOM(info); + this.csrf = $dom.window.document.querySelector('meta[name="X-Csrf-Token"]').getAttribute('content'); + return $dom.window.document.querySelector('[data-submission-id]').getAttribute('data-submission-id'); + } + + async waitForSubmission(id: string, next, end) { + let i = 1; + // eslint-disable-next-line no-constant-condition + while (true) { + await sleep(3000); + const { body } = await this.post('/data/submitSource').send({ + csrf_token: this.csrf, + submissionId: id, + }); + if (body.compilationError === 'true') { + await next({ compilerText: body['checkerStdoutAndStderr#1'] }); + return await end({ + status: STATUS.STATUS_COMPILE_ERROR, score: 0, time: 0, memory: 0, + }); + } + const time = Math.sum(Object.keys(body).filter((k) => k.startsWith('timeConsumed#')).map((k) => +body[k])); + const memory = Math.max(...Object.keys(body).filter((k) => k.startsWith('memoryConsumed#')).map((k) => +body[k])) / 1024; + for (; i <= +body.testCount; i++) { + const status = VERDICT[body[`verdict#${i}`]] || STATUS.STATUS_WRONG_ANSWER; + await next({ + status: STATUS.STATUS_JUDGING, + case: { + status, + time: +body[`timeConsumed#${i}`], + memory: +body[`memoryConsumed#${i}`] / 1024, + message: body[`checkerStdoutAndStderr#${i}`], + }, + }); + } + if (body.waiting === 'true') continue; + const status = VERDICT[Object.keys(VERDICT).filter((k) => body.verdict.includes(k))[0]]; + return await end({ + status, + score: status === STATUS.STATUS_ACCEPTED ? 100 : 0, + time, + memory, + }); + } + } +} diff --git a/packages/vjudge/src/providers/index.ts b/packages/vjudge/src/providers/index.ts new file mode 100644 index 000000000..8734aabbd --- /dev/null +++ b/packages/vjudge/src/providers/index.ts @@ -0,0 +1,3 @@ +import codeforces from './codeforces'; + +export = { codeforces };