vjudge: codeforces

packages/vjudge/package.json

@@ -0,0 +1,18 @@
+{
+    "name": "@hydrooj/vjudge",
+    "version": "1.0.0",
+    "description": "Submit problems to remote oj",
+    "main": "package.json",
+    "repository": "https://github.com/hydro-dev/Hydro.git",
+    "author": "undefined",
+    "license": "AGPL-3.0-or-later",
+    "preferUnplugged": true,
+    "dependencies": {
+        "@hydrooj/utils": "workspace:packages/utils",
+        "jsdom": "^16.6.0",
+        "superagent": "^6.1.0"
+    },
+    "devDependencies": {
+        "@types/jsdom": "^16.2.11"
+    }
+}

packages/vjudge/src/interface.ts

@@ -0,0 +1,34 @@
+export interface RemoteAccount {
+    _id: string;
+    type: string;
+    cookie?: string[];
+    handle: string;
+    password: string;
+}
+declare module 'hydrooj/dist/interface' {
+    interface Collections {
+        vjudge: RemoteAccount;
+    }
+
+    interface DomainDoc {
+        mount: string;
+        mountInfo: any;
+    }
+}
+export interface IBasicProvider {
+    ensureLogin(): Promise<boolean>;
+    getProblem(id: string): Promise<{
+        title: string;
+        data: Record<string, any>;
+        files: Record<string, any>;
+        tag: string[];
+        content: string;
+    }>;
+    listProblem(page: number): Promise<string[]>;
+    submitProblem(id: string, lang: string, code: string): Promise<string>;
+    waitForSubmission(id: string, next: any, end: any): Promise<void>;
+}
+
+export interface BasicProvider {
+    new(account: RemoteAccount, save: (data: any) => Promise<void>): IBasicProvider
+}

packages/vjudge/src/model.ts

@@ -0,0 +1,75 @@
+/* eslint-disable no-await-in-loop */
+import { Logger } from 'hydrooj/dist/logger';
+import db from 'hydrooj/dist/service/db';
+import * as bus from 'hydrooj/dist/service/bus';
+import * as Judge from 'hydrooj/dist/handler/judge';
+import TaskModel from 'hydrooj/dist/model/task';
+import DomainModel from 'hydrooj/dist/model/domain';
+import ProblemModel from 'hydrooj/dist/model/problem';
+import { sleep } from '@hydrooj/utils/lib/utils';
+import providers from './providers/index';
+import { BasicProvider, IBasicProvider, RemoteAccount } from './interface';
+
+const coll = db.collection('vjudge');
+const Pool = {};
+const logger = new Logger('vjudge');
+
+class Service {
+    api: IBasicProvider;
+
+    constructor(public Provider: BasicProvider, public account: RemoteAccount) {
+        this.api = new Provider(account, async (data) => {
+            await coll.updateOne({ _id: account._id }, { $set: data });
+        });
+        this.main();
+    }
+
+    async judge(task) {
+        const rid = await this.api.submitProblem(task.target, task.lang, task.code);
+        const next = (payload) => Judge.next({ ...payload, rid: task.rid });
+        const end = (payload) => Judge.end({ ...payload, rid: task.rid });
+        await this.api.waitForSubmission(rid, next, end);
+    }
+
+    async sync(domainId: string) {
+        let page = 1;
+        let pids = await this.api.listProblem(page);
+        while (pids.length) {
+            logger.info(`${domainId}: Syncing page ${page}`);
+            for (const pid of pids) {
+                if (await ProblemModel.get(domainId, pid)) continue;
+                const res = await this.api.getProblem(pid);
+                const id = await ProblemModel.add(domainId, pid, res.title, res.content, 1, res.tag, false);
+                for (const key in res.files) {
+                    await ProblemModel.addAdditionalFile(domainId, id, key, res.files[key]);
+                }
+                for (const key in res.data) {
+                    await ProblemModel.addTestdata(domainId, id, key, res.data[key]);
+                }
+                logger.info(`${domainId}: problem ${id} sync done`);
+                await sleep(5000);
+            }
+            page++;
+            pids = await this.api.listProblem(page);
+        }
+    }
+
+    async main() {
+        const login = await this.api.ensureLogin();
+        if (login) logger.info(`${this.account.type}/${this.account.handle}: logged in`);
+        else return;
+        TaskModel.consume({ type: 'remotejudge', subType: this.account.type }, this.judge.bind(this));
+        const ddocs = await DomainModel.getMulti({ mount: this.account.type }).toArray();
+        for (const ddoc of ddocs) await this.sync(ddoc._id);
+    }
+}
+
+async function loadAccounts() {
+    const accounts = await coll.find().toArray();
+    for (const account of accounts) {
+        if (!providers[account.type]) continue;
+        Pool[`${account.type}/${account.handle}`] = new Service(providers[account.type], account);
+    }
+}
+
+bus.on('app/started', loadAccounts);

packages/vjudge/src/providers/codeforces.ts

@@ -0,0 +1,231 @@
+/* eslint-disable no-await-in-loop */
+import superagent from 'superagent';
+import { JSDOM } from 'jsdom';
+import { STATUS } from '@hydrooj/utils/lib/status';
+import { sleep } from '@hydrooj/utils/lib/utils';
+import { buildContent } from 'hydrooj/dist/lib/content';
+import { Logger } from 'hydrooj/dist/logger';
+import { PassThrough } from 'stream';
+import { IBasicProvider, RemoteAccount } from '../interface';
+
+const logger = new Logger('codeforces');
+
+const VERDICT = {
+    RUNTIME_ERROR: STATUS.STATUS_RUNTIME_ERROR,
+    WRONG_ANSWER: STATUS.STATUS_WRONG_ANSWER,
+    OK: STATUS.STATUS_ACCEPTED,
+    TIME_LIMIT_EXCEEDED: STATUS.STATUS_TIME_LIMIT_EXCEEDED,
+    MEMORY_LIMIT_EXCEEDED: STATUS.STATUS_MEMORY_LIMIT_EXCEEDED,
+    Accepted: STATUS.STATUS_ACCEPTED,
+    'Wrong answer': STATUS.STATUS_WRONG_ANSWER,
+    'Runtime error': STATUS.STATUS_RUNTIME_ERROR,
+    'Time limit exceeded': STATUS.STATUS_TIME_LIMIT_EXCEEDED,
+    'Memory limit exceeded': STATUS.STATUS_MEMORY_LIMIT_EXCEEDED,
+};
+
+export default class CodeforcesProvider implements IBasicProvider {
+    constructor(public account: RemoteAccount, private save: (data: any) => Promise<void>) {
+        if (account.cookie) this.cookie = account.cookie;
+    }
+
+    cookie: string[] = [];
+    // @ts-ignore
+    ftaa = String.random(18, 'abcdef1234567890');
+    csrf: string;
+
+    get(url: string) {
+        logger.debug('get', url);
+        return superagent.get(`https://codeforces.com${url}`).set('Cookie', this.cookie);
+    }
+
+    post(url: string) {
+        logger.debug('post', url, this.cookie);
+        return superagent.post(`https://codeforces.com${url}`).type('form').set('Cookie', this.cookie);
+    }
+
+    tta(_39ce7: string) {
+        let _tta = 0;
+        for (let c = 0; c < _39ce7.length; c++) {
+            _tta = (_tta + (c + 1) * (c + 2) * _39ce7.charCodeAt(c)) % 1009;
+            if (c % 3 === 0) _tta++;
+            if (c % 2 === 0) _tta *= 2;
+            if (c > 0) _tta -= Math.floor(_39ce7.charCodeAt(Math.floor(c / 2)) / 2) * (_tta % 5);
+            _tta = ((_tta % 1009) + 1009) % 1009;
+        }
+        return _tta;
+    }
+
+    async getCsrfToken(url: string) {
+        const { text: html } = await this.get(url);
+        const $dom = new JSDOM(html);
+        return $dom.window.document.querySelector('meta[name="X-Csrf-Token"]').getAttribute('content');
+    }
+
+    get loggedIn() {
+        return this.get('/').then(({ text: html }) => {
+            const $dom = new JSDOM(html);
+            if (!$dom.window.document.querySelectorAll('a[href="/enter?back=%2F"]').length) return true;
+            return false;
+        });
+    }
+
+    async ensureLogin() {
+        if (await this.loggedIn) return true;
+        logger.info('retry login');
+        const csrf_token = await this.getCsrfToken('/enter');
+        const res = await this.post('/enter').send({
+            csrf_token,
+            action: 'enter',
+            ftaa: '',
+            bfaa: '',
+            handleOrEmail: this.account.handle,
+            password: this.account.password,
+            remember: 'on',
+        });
+        const cookie = res.header['set-cookie'];
+        if (cookie) {
+            await this.save({ cookie });
+            this.cookie = cookie;
+        }
+        if (await this.loggedIn) return true;
+        return false;
+    }
+
+    async getProblem(id: string) {
+        logger.info(id);
+        const [, contestId, problemId] = /^P(\d+)([A-Z][0-9]?)$/.exec(id);
+        const res = await this.get(`/problemset/problem/${contestId}/${problemId}`);
+        const $dom = new JSDOM(res.text.replace(/\$\$\$/g, '$'));
+        const tag = Array.from($dom.window.document.querySelectorAll('.tag-box')).map((i) => i.textContent.trim());
+        const text = $dom.window.document.querySelector('.problem-statement').innerHTML;
+        const { window: { document } } = new JSDOM(text);
+        const files = {};
+        document.querySelectorAll('img[src]').forEach((ele) => {
+            const src = ele.getAttribute('src');
+            if (!src.startsWith('http')) return;
+            const file = new PassThrough();
+            superagent.get(src).pipe(file);
+            const fid = String.random(8);
+            files[`${fid}.png`] = file;
+            ele.setAttribute('src', `%file%://${fid}.png`);
+        });
+        const title = document.querySelector('.title').innerHTML;
+        const time = parseInt(document.querySelector('.time-limit').innerHTML.substr(53, 2), 10);
+        const memory = parseInt(document.querySelector('.memory-limit').innerHTML.substr(55, 4), 10);
+        document.body.firstChild.remove();
+        document.querySelector('.input-specification')?.firstChild.remove();
+        document.querySelector('.output-specification')?.firstChild.remove();
+        document.querySelector('.note')?.firstChild.remove();
+        const input = document.querySelector('.input-specification')?.innerHTML.trim();
+        const output = document.querySelector('.output-specification')?.innerHTML.trim();
+        const inputs = Array.from(document.querySelectorAll('.input>pre')).map((i) => i.innerHTML.trim());
+        const outputs = Array.from(document.querySelectorAll('.output>pre')).map((i) => i.innerHTML.trim());
+        const note = document.querySelector('.note')?.innerHTML.trim();
+        document.querySelector('.note')?.remove();
+        document.querySelector('.sample-tests')?.remove();
+        document.querySelectorAll('.section-title').forEach((ele) => {
+            const e = document.createElement('h2');
+            e.innerHTML = ele.innerHTML;
+            ele.replaceWith(e);
+        });
+        const description = document.body.innerHTML.trim();
+        return {
+            title,
+            data: {
+                'config.yaml': Buffer.from(`time: ${time}s\nmemory: ${memory}m\ntype: remote_judge\nsubType: codeforces\ntarget: ${id}`),
+            },
+            files,
+            tag,
+            content: buildContent([
+                {
+                    type: 'Text', sectionTitle: 'Description', subType: 'markdown', text: description,
+                },
+                ...input ? [{
+                    type: 'Text', sectionTitle: 'Input', subType: 'markdown', text: input,
+                }] : [],
+                ...output ? [{
+                    type: 'Text', sectionTitle: 'Output', subType: 'markdown', text: output,
+                }] : [],
+                ...inputs.map((_, i) => ({
+                    type: 'Sample', payload: [inputs[i], outputs[i]], text: '', sectionTitle: 'Samples',
+                }) as any),
+                ...note ? [{
+                    type: 'Text', sectionTitle: 'Note', subType: 'markdown', text: note,
+                }] : [],
+            ]),
+        };
+    }
+
+    async listProblem(page: number) {
+        const res = await this.get(`/problemset/page/${page}`);
+        const $dom = new JSDOM(res.text);
+        const index = $dom.window.document.querySelector('.page-index.active').getAttribute('pageindex');
+        if (index !== page.toString()) return [];
+        return Array.from($dom.window.document.querySelectorAll('.id>a')).map((i) => `P${i.innerHTML.trim()}`);
+    }
+
+    async submitProblem(id: string, lang: string, code: string) {
+        const programTypeId = lang.includes('codeforces.') ? lang.split('codeforces.')[1] : '42';
+        const [, contestId, submittedProblemIndex] = /^P(\d+)([A-Z][0-9]?)$/.exec(id);
+        const csrf_token = await this.getCsrfToken('/problemset/submit');
+        // TODO check submit time to ensure submission
+        await this.post(`/problemset/submit?csrf_token=${csrf_token}`).send({
+            csrf_token,
+            contestId,
+            action: 'submitSolutionFormSubmitted',
+            programTypeId,
+            submittedProblemIndex,
+            source: code,
+            tabsize: 4,
+            sourceFile: '',
+            ftaa: '',
+            bfaa: 'f1b3f18c715565b589b7823cda7448ce',
+            _tta: 594,
+            sourceCodeConfirmed: true,
+        });
+        const { text: info } = await this.get('/problemset/status?my=on');
+        const $dom = new JSDOM(info);
+        this.csrf = $dom.window.document.querySelector('meta[name="X-Csrf-Token"]').getAttribute('content');
+        return $dom.window.document.querySelector('[data-submission-id]').getAttribute('data-submission-id');
+    }
+
+    async waitForSubmission(id: string, next, end) {
+        let i = 1;
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            await sleep(3000);
+            const { body } = await this.post('/data/submitSource').send({
+                csrf_token: this.csrf,
+                submissionId: id,
+            });
+            if (body.compilationError === 'true') {
+                await next({ compilerText: body['checkerStdoutAndStderr#1'] });
+                return await end({
+                    status: STATUS.STATUS_COMPILE_ERROR, score: 0, time: 0, memory: 0,
+                });
+            }
+            const time = Math.sum(Object.keys(body).filter((k) => k.startsWith('timeConsumed#')).map((k) => +body[k]));
+            const memory = Math.max(...Object.keys(body).filter((k) => k.startsWith('memoryConsumed#')).map((k) => +body[k])) / 1024;
+            for (; i <= +body.testCount; i++) {
+                const status = VERDICT[body[`verdict#${i}`]] || STATUS.STATUS_WRONG_ANSWER;
+                await next({
+                    status: STATUS.STATUS_JUDGING,
+                    case: {
+                        status,
+                        time: +body[`timeConsumed#${i}`],
+                        memory: +body[`memoryConsumed#${i}`] / 1024,
+                        message: body[`checkerStdoutAndStderr#${i}`],
+                    },
+                });
+            }
+            if (body.waiting === 'true') continue;
+            const status = VERDICT[Object.keys(VERDICT).filter((k) => body.verdict.includes(k))[0]];
+            return await end({
+                status,
+                score: status === STATUS.STATUS_ACCEPTED ? 100 : 0,
+                time,
+                memory,
+            });
+        }
+    }
+}

packages/vjudge/src/providers/index.ts

@@ -0,0 +1,3 @@
+import codeforces from './codeforces';
+
+export = { codeforces };