vjudge: use BasicFetcher (#496)

This commit is contained in:
undefined authored and GitHub committed 2023-01-23 16:40:33 +08:00
1 parent eff22e1506
commit 3d3a5480e3
10 files changed
+118 -218

No files matched your search

+1 -3
View File
@@ -2,7 +2,7 @@
"name": "@hydrooj/vjudge",
"version": "1.7.3",
"description": "Submit problems to remote oj",
"main": "package.json",
"main": "src/model.ts",
"repository": "https://github.com/hydro-dev/Hydro.git",
"author": "undefined",
"license": "AGPL-3.0-or-later",
@@ -10,12 +10,10 @@
"dependencies": {
"@hydrooj/utils": "workspace:*",
"jsdom": "^21.0.0",
"superagent": "^8.0.6",
"superagent-proxy": "^3.0.0"
},
"devDependencies": {
"@types/jsdom": "^20.0.1",
"@types/superagent": "^4.1.16",
"@types/superagent-proxy": "^3.0.0"
}
}
+50
View File
@@ -0,0 +1,50 @@
import { DOMWindow, JSDOM } from 'jsdom';
import proxy from 'superagent-proxy';
import { Logger } from '@hydrooj/utils';
import { superagent } from 'hydrooj';
import { RemoteAccount } from './interface';
proxy(superagent);
interface FetchOptions {
headers?: { [key: string]: string };
post?: Omit<FetchOptions, 'get' | 'post'>
get?: Omit<FetchOptions, 'get' | 'post'>
}
export class BasicFetcher {
cookie: string[] = [];
constructor(
public account: RemoteAccount, private defaultEndpoint: string,
private formType: 'form' | 'json', public logger: Logger,
public fetchOptions: FetchOptions = {},
) {
if (account.cookie) this.cookie = account.cookie;
}
get(url: string) {
this.logger.debug('get', url);
url = new URL(url, this.account.endpoint || this.defaultEndpoint).toString();
let req = superagent.get(url).set('Cookie', this.cookie);
if (this.fetchOptions.headers) req = req.set(this.fetchOptions.headers);
if (this.fetchOptions.get?.headers) req = req.set(this.fetchOptions.get.headers);
return this.account.proxy ? req.proxy(this.account.proxy) : req;
}
async html(url: string) {
const { text: html } = await this.get(url);
const $dom = new JSDOM(html);
$dom.window.html = html;
return $dom.window as DOMWindow & { html: string };
}
post(url: string) {
this.logger.debug('post', url, this.cookie);
url = new URL(url, this.account.endpoint || this.defaultEndpoint).toString();
let req = superagent.post(url).set('Cookie', this.cookie).type(this.formType);
if (this.fetchOptions.headers) req = req.set(this.fetchOptions.headers);
if (this.fetchOptions.post?.headers) req = req.set(this.fetchOptions.post.headers);
return this.account.proxy ? req.proxy(this.account.proxy) : req;
}
}
+7 -19
View File
@@ -5,7 +5,6 @@ import {
ProblemModel, RecordModel, Service, sleep, STATUS, TaskModel, Time,
} from 'hydrooj';
import { BasicProvider, IBasicProvider, RemoteAccount } from './interface';
import { getDifficulty } from './providers/codeforces';
import providers from './providers/index';
const coll = db.collection('vjudge');
@@ -45,7 +44,10 @@ class AccountService {
await next({ status: STATUS.STATUS_JUDGING, message: `ID = ${rid}` });
await this.api.waitForSubmission(rid, next, end);
} catch (e) {
if (process.env.DEV) logger.error(e);
if (process.env.DEV) {
logger.error(e);
if (e.response) console.error(e.response);
}
end({ status: STATUS.STATUS_SYSTEM_ERROR, message: e.message });
}
}
@@ -123,24 +125,11 @@ class AccountService {
}
declare module 'hydrooj' {
interface Model {
vjudge: VJudgeModel;
}
interface Context {
vjudge: VJudgeService;
}
}
class VJudgeModel {
static async fixCodeforcesDifficulty(domainId = 'codeforces') {
const pdocs = await ProblemModel.getMulti(domainId, {}).toArray();
for (const pdoc of pdocs) {
await ProblemModel.edit(domainId, pdoc.docId, { difficulty: getDifficulty(pdoc.tag) });
}
return true;
}
}
class VJudgeService extends Service {
constructor(ctx) {
super(ctx, 'vjudge', false);
@@ -153,8 +142,9 @@ class VJudgeService extends Service {
this.accounts = await coll.find().toArray();
}
addProvider(type: string, provider: BasicProvider) {
if (this.providers[type]) throw new Error(`duplicate provider ${type}`);
addProvider(type: string, provider: BasicProvider, override = false) {
if (process.env.VJUDGE_DEBUG && process.env.VJUDGE_DEBUG !== type) return;
if (!override && this.providers[type]) throw new Error(`duplicate provider ${type}`);
this.providers[type] = provider;
for (const account of this.accounts.filter((a) => a.type === type)) {
if (account.enableOn && !account.enableOn.includes(os.hostname())) continue;
@@ -166,8 +156,6 @@ class VJudgeService extends Service {
}
}
global.Hydro.model.vjudge = VJudgeModel;
Context.service('vjudge', VJudgeService);
export const name = 'vjudge';
export async function apply(ctx: Context) {
+6 -29
View File
@@ -2,15 +2,13 @@
import { PassThrough } from 'stream';
import yaml from 'js-yaml';
import { JSDOM } from 'jsdom';
import superagent from 'superagent';
import proxy from 'superagent-proxy';
import {
buildContent, Logger, SettingModel, sleep, STATUS,
} from 'hydrooj';
import { BasicFetcher } from '../fetch';
import { IBasicProvider, RemoteAccount } from '../interface';
import { normalize, VERDICT } from '../verdict';
proxy(superagent);
const logger = new Logger('remote/codeforces');
function parseProblemId(id: string) {
@@ -61,31 +59,13 @@ ${lines.map((i) => i.innerHTML).join('\n').trim()}
\n`;
};
export default class CodeforcesProvider implements IBasicProvider {
export default class CodeforcesProvider extends BasicFetcher implements IBasicProvider {
constructor(public account: RemoteAccount, private save: (data: any) => Promise<void>) {
if (account.cookie) this.cookie = account.cookie;
this.account.endpoint ||= 'https://codeforces.com';
super(account, 'https://codeforces.com', 'form', logger);
}
cookie: string[] = [];
csrf: string;
get(url: string) {
logger.debug('get', url);
if (!url.includes('//')) url = `${this.account.endpoint}${url}`;
const req = superagent.get(url).set('Cookie', this.cookie);
if (this.account.proxy) return req.proxy(this.account.proxy);
return req;
}
post(url: string) {
logger.debug('post', url, this.cookie);
if (!url.includes('//')) url = `${this.account.endpoint}${url}`;
const req = superagent.post(url).type('form').set('Cookie', this.cookie);
if (this.account.proxy) return req.proxy(this.account.proxy);
return req;
}
getCookie(target: string) {
return this.cookie.find((i) => i.startsWith(`${target}=`))?.split('=')[1]?.split(';')[0];
}
@@ -103,8 +83,7 @@ export default class CodeforcesProvider implements IBasicProvider {
}
async getCsrfToken(url: string) {
const { text: html } = await this.get(url);
const { window: { document } } = new JSDOM(html);
const { document, html } = await this.html(url);
if (document.body.children.length < 2 && html.length < 512) {
throw new Error(document.body.textContent);
}
@@ -261,13 +240,12 @@ export default class CodeforcesProvider implements IBasicProvider {
if (resync && page > 1) return [];
if (resync && listName.startsWith('GYM')) return [];
if (listName.startsWith('GYM') && page > 1) return [];
const res = await this.get(listName === 'main'
const { document } = await this.html(listName === 'main'
? `/problemset/page/${page}`
: listName === 'gym'
? `/gyms/page/${page}`
: `/gym/${listName.split('GYM')[1]}`,
);
const { window: { document } } = new JSDOM(res.text);
if (['gym', 'main'].includes(listName)) {
const index = document.querySelector('.page-index.active').getAttribute('pageindex');
if (index !== page.toString()) return [];
@@ -330,10 +308,9 @@ export default class CodeforcesProvider implements IBasicProvider {
end({ status: STATUS.STATUS_COMPILE_ERROR, message });
return null;
}
const { text: status } = await this.get(type !== 'GYM'
const { document } = await this.html(type !== 'GYM'
? '/problemset/status?my=on'
: `/gym/${contestId}/my`);
const { window: { document } } = new JSDOM(status);
this.csrf = document.querySelector('meta[name="X-Csrf-Token"]').getAttribute('content');
return document.querySelector('[data-submission-id]').getAttribute('data-submission-id');
}
+6 -28
View File
@@ -1,12 +1,10 @@
/* eslint-disable no-await-in-loop */
import { PassThrough } from 'stream';
import { JSDOM } from 'jsdom';
import * as superagent from 'superagent';
import proxy from 'superagent-proxy';
import { Logger, sleep, STATUS } from 'hydrooj';
import { BasicFetcher } from '../fetch';
import { IBasicProvider, RemoteAccount } from '../interface';
proxy(superagent as any);
const logger = new Logger('remote/csgoj');
const statusDict = {
0: STATUS.STATUS_COMPILING,
@@ -53,35 +51,15 @@ csgoj.3:
const userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:104.0) Gecko/20100101 Firefox/104.0';
export default class CSGOJProvider implements IBasicProvider {
export default class CSGOJProvider extends BasicFetcher implements IBasicProvider {
constructor(public account: RemoteAccount, private save: (data: any) => Promise<void>) {
super(account, 'https://cpc.csgrandeur.cn', 'form', logger, {
headers: { 'User-Agent': userAgent },
post: { headers: { 'X-Requested-With': 'XMLHttpRequest' } },
});
if (account.cookie) this.cookie = account.cookie;
}
cookie: string[] = [];
get(url: string) {
logger.debug('get', url);
if (!url.includes('//')) url = `${this.account.endpoint || 'https://cpc.csgrandeur.cn'}${url}`;
const req = superagent.get(url)
.set('Cookie', this.cookie)
.set('User-Agent', userAgent);
if (this.account.proxy) return req.proxy(this.account.proxy);
return req;
}
post(url: string) {
logger.debug('post', url, this.cookie);
if (!url.includes('//')) url = `${this.account.endpoint || 'https://cpc.csgrandeur.cn'}${url}`;
const req = superagent.post(url)
.set('Cookie', this.cookie)
.set('X-Requested-With', 'XMLHttpRequest')
.set('User-Agent', userAgent)
.type('form');
if (this.account.proxy) return req.proxy(this.account.proxy);
return req;
}
get loggedIn() {
return this.get('/').then(({ text: html }) => !html
.includes('<form id="login_form" class="form-signin" method="post" action="/csgoj/user/login_ajax">'));
+18 -20
View File
@@ -1,7 +1,7 @@
import { defaultsDeep } from 'lodash';
import {
SettingModel, sleep, STATUS, superagent,
_, Logger, SettingModel, sleep, STATUS,
} from 'hydrooj';
import { BasicFetcher } from '../fetch';
import { IBasicProvider, RemoteAccount } from '../interface';
import { VERDICT } from '../verdict';
@@ -11,6 +11,7 @@ import { VERDICT } from '../verdict';
/* eslint-disable no-await-in-loop */
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36';
const logger = new Logger('vjudge/hustoj');
interface HustOJRemoteConfig {
login?: {
@@ -83,33 +84,30 @@ function isProcessing(t: string) {
return [STATUS.STATUS_WAITING, STATUS.STATUS_COMPILING, STATUS.STATUS_JUDGING].includes(VERDICT[t]);
}
export class HUSTOJ implements IBasicProvider {
cookie: string[] = [];
config = { ...defaultConfig };
export class HUSTOJ extends BasicFetcher implements IBasicProvider {
config: HustOJRemoteConfig;
state = {
username: '',
pid: '',
};
constructor(public account: RemoteAccount, private save: (data: any) => Promise<void>) {
defaultsDeep(this.config, account);
const config = _.defaultsDeep({ ...defaultConfig }, account);
super(account, '', 'form', logger, {
post: {
headers: {
Referer: config.server,
'User-Agent': UA,
Accept: 'application/json',
},
},
});
this.config = config;
this.updateConfig();
}
updateConfig() { }
get(url: string) {
return superagent.get(url).set('Cookie', this.cookie);
}
post(url: string, data: Record<string, any>) {
return superagent.post(url)
.set('Cookie', this.cookie).send(data).type('form')
.set('Referer', this.config.server)
.set('User-Agent', UA)
.set('Accept', 'application/json');
}
getProblem() {
return null;
}
@@ -120,7 +118,7 @@ export class HUSTOJ implements IBasicProvider {
async login(username: string, password: string) {
this.cookie = [];
const res = await this.post(this.config.login[0], {
const res = await this.post(this.config.login[0]).send({
[this.config.login.usernameField]: username,
[this.config.login.passwordField]: password,
...this.config.login.extra,
@@ -143,7 +141,7 @@ export class HUSTOJ implements IBasicProvider {
if (typeof comment === 'string') code = `${comment} ${msg}\n${code}`;
else if (comment instanceof Array) code = `${comment[0]} ${msg} ${comment[1]}\n${code}`;
}
const res = await this.post(this.config.submit.endpoint, {
const res = await this.post(this.config.submit.endpoint).send({
[this.config.submit.idField]: id,
[this.config.submit.langField]: langId,
[this.config.submit.codeField]: code,
+17 -44
View File
@@ -1,13 +1,10 @@
/* eslint-disable no-await-in-loop */
import { JSDOM } from 'jsdom';
import superagent from 'superagent';
import proxy from 'superagent-proxy';
import {
_, Logger, SettingModel, sleep, STATUS,
} from 'hydrooj';
import { BasicFetcher } from '../fetch';
import { IBasicProvider, RemoteAccount } from '../interface';
proxy(superagent);
const logger = new Logger('remote/luogu');
const STATUS_MAP = [
@@ -33,43 +30,24 @@ const UA = [
`Vjudge/${global.Hydro.version.vjudge}`,
].join(' ');
export default class LuoguProvider implements IBasicProvider {
export default class LuoguProvider extends BasicFetcher implements IBasicProvider {
constructor(public account: RemoteAccount, private save: (data: any) => Promise<void>) {
if (account.cookie) this.cookie = account.cookie;
super(account, 'https://www.luogu.com.cn', 'json', logger, {
headers: { 'User-Agent': UA },
post: {
headers: {
'x-requested-with': 'XMLHttpRequest',
origin: 'https://www.luogu.com.cn',
},
},
});
setInterval(() => this.getCsrfToken('/user/setting'), 5 * 60 * 1000);
}
cookie: string[] = [];
csrf: string;
get(url: string) {
logger.debug('get', url);
if (!url.includes('//')) url = `${this.account.endpoint || 'https://www.luogu.com.cn'}${url}`;
const req = superagent.get(url)
.set('Cookie', this.cookie)
.set('User-Agent', UA);
if (this.account.proxy) return req.proxy(this.account.proxy);
return req;
}
post(url: string) {
logger.debug('post', url, this.cookie);
if (!url.includes('//')) url = `${this.account.endpoint || 'https://www.luogu.com.cn'}${url}`;
const req = superagent.post(url)
.set('Cookie', this.cookie)
.set('x-csrf-token', this.csrf)
.set('User-Agent', UA)
.set('x-requested-with', 'XMLHttpRequest')
.set('origin', 'https://www.luogu.com.cn');
if (this.account.proxy) return req.proxy(this.account.proxy);
return req;
}
async getCsrfToken(url: string) {
const { text: html } = await this.get(url);
const $dom = new JSDOM(html);
this.csrf = $dom.window.document.querySelector('meta[name="csrf-token"]').getAttribute('content');
logger.info('csrf-token=', this.csrf);
const csrf = (await this.html(url)).document.querySelector('meta[name="csrf-token"]').getAttribute('content');
logger.info('csrf-token=', csrf);
// this.fetchOptions.post.headers['x-csrf-token'] = csrf;
}
get loggedIn() {
@@ -86,14 +64,8 @@ export default class LuoguProvider implements IBasicProvider {
return false;
}
async getProblem(id: string) {
return {
title: id,
data: {},
files: {},
tag: [],
content: '',
};
async getProblem() {
return null;
}
async listProblem() {
@@ -109,6 +81,7 @@ export default class LuoguProvider implements IBasicProvider {
}
if (!lang.startsWith('luogu.')) {
end({ status: STATUS.STATUS_COMPILE_ERROR, message: `Language not supported: ${lang}` });
return null;
}
if (comment) {
const msg = `Hydro submission #${info.rid}@${new Date().toLocaleString()}`;
+3 -23
View File
@@ -1,15 +1,13 @@
/* eslint-disable no-await-in-loop */
import { PassThrough } from 'stream';
import { JSDOM } from 'jsdom';
import * as superagent from 'superagent';
import proxy from 'superagent-proxy';
import {
htmlEncode, Logger, parseMemoryMB, parseTimeMS, SettingModel, sleep, STATUS,
} from 'hydrooj';
import { BasicFetcher } from '../fetch';
import { IBasicProvider, RemoteAccount } from '../interface';
import { VERDICT } from '../verdict';
proxy(superagent as any);
const logger = new Logger('remote/poj');
/* langs
@@ -59,27 +57,9 @@ const langs = {
'zh-CN': 'zh',
};
export default class POJProvider implements IBasicProvider {
export default class POJProvider extends BasicFetcher implements IBasicProvider {
constructor(public account: RemoteAccount, private save: (data: any) => Promise<void>) {
if (account.cookie) this.cookie = account.cookie;
}
cookie: string[] = [];
get(url: string) {
logger.debug('get', url);
if (!url.startsWith('http')) url = new URL(url, this.account.endpoint || 'http://poj.org').toString();
const req = superagent.get(url).set('Cookie', this.cookie);
if (this.account.proxy) return req.proxy(this.account.proxy);
return req;
}
post(url: string) {
logger.debug('post', url, this.cookie);
if (!url.includes('//')) url = `${this.account.endpoint || 'http://poj.org'}${url}`;
const req = superagent.post(url).set('Cookie', this.cookie).type('form');
if (this.account.proxy) return req.proxy(this.account.proxy);
return req;
super(account, 'http://poj.org', 'form', logger);
}
async getCsrfToken(url: string) {
+7 -30
View File
@@ -1,14 +1,12 @@
/* eslint-disable no-await-in-loop */
import { PassThrough } from 'stream';
import { JSDOM } from 'jsdom';
import superagent from 'superagent';
import proxy from 'superagent-proxy';
import {
Logger, parseMemoryMB, SettingModel, sleep, STATUS,
} from 'hydrooj';
import { BasicFetcher } from '../fetch';
import { IBasicProvider, RemoteAccount } from '../interface';
proxy(superagent);
const logger = new Logger('spoj');
const VERDICT = {
@@ -19,27 +17,9 @@ const VERDICT = {
15: STATUS.STATUS_ACCEPTED,
};
export default class SPOJProvider implements IBasicProvider {
export default class SPOJProvider extends BasicFetcher implements IBasicProvider {
constructor(public account: RemoteAccount, private save: (data: any) => Promise<void>) {
if (account.cookie) this.cookie = account.cookie;
}
cookie: string[] = [];
get(url: string) {
logger.debug('get', url);
if (!url.includes('//')) url = `${this.account.endpoint || 'https://www.spoj.com'}${url}`;
const req = superagent.get(url).set('Cookie', this.cookie);
if (this.account.proxy) return req.proxy(this.account.proxy);
return req;
}
post(url: string) {
logger.debug('post', url, this.cookie);
if (!url.includes('//')) url = `${this.account.endpoint || 'https://www.spoj.com'}${url}`;
const req = superagent.post(url).type('form').set('Cookie', this.cookie);
if (this.account.proxy) return req.proxy(this.account.proxy);
return req;
super(account, 'https://www.spoj.com', 'form', logger);
}
get loggedIn() {
@@ -70,8 +50,7 @@ export default class SPOJProvider implements IBasicProvider {
async getProblem(id: string) {
logger.info(id);
const { text } = await this.get(`/problems/${id}/`);
const { window: { document } } = new JSDOM(text);
const { document } = await this.html(`/problems/${id}/`);
const files = {};
document.querySelector('#problem-body').querySelectorAll('img[src]').forEach((ele) => {
const src = ele.getAttribute('src');
@@ -83,9 +62,8 @@ export default class SPOJProvider implements IBasicProvider {
ele.setAttribute('src', `file://${fid}.png`);
});
const meta = document.querySelector('#problem-meta').children[1];
const { text: submit } = await this.get(`/submit/${id}/`);
const $dom = new JSDOM(submit);
const langs = Array.from($dom.window.document.querySelector('#lang').querySelectorAll('option'))
const window = await this.html(`/submit/${id}/`);
const langs = Array.from(window.document.querySelector('#lang').querySelectorAll('option'))
.map((i) => `spoj.${i.getAttribute('value')}`);
let time = meta.children[2].children[1].innerHTML.trim().toLowerCase();
if (time.includes('-')) time = time.split('-')[1];
@@ -109,8 +87,7 @@ langs: ${JSON.stringify(langs)}`),
async listProblem(page: number, resync = false) {
if (resync) return [];
const res = await this.get(`/problems/classical/sort=0,start=${page * 50 - 50}`);
const { window: { document } } = new JSDOM(res.text);
const { document } = await this.html(`/problems/classical/sort=0,start=${page * 50 - 50}`);
const index = document.querySelector('ul.pagination').querySelector('li.active').children[0].innerHTML.trim();
if (index !== page.toString()) return [];
return Array.from(document.querySelectorAll('td[align=left]')).map((i) => i.children[0].getAttribute('href').split('/problems/')[1]);
+3 -22
View File
@@ -1,15 +1,13 @@
/* eslint-disable no-await-in-loop */
import { PassThrough } from 'stream';
import { JSDOM } from 'jsdom';
import * as superagent from 'superagent';
import proxy from 'superagent-proxy';
import {
Logger, parseMemoryMB, parseTimeMS, SettingModel, sleep, STATUS,
} from 'hydrooj';
import { BasicFetcher } from '../fetch';
import { IBasicProvider, RemoteAccount } from '../interface';
import { VERDICT } from '../verdict';
proxy(superagent as any);
const logger = new Logger('remote/uoj');
const MAPPING = {
一: 1,
@@ -24,30 +22,13 @@ const MAPPING = {
十: 10,
};
export default class UOJProvider implements IBasicProvider {
export default class UOJProvider extends BasicFetcher implements IBasicProvider {
constructor(public account: RemoteAccount, private save: (data: any) => Promise<void>) {
if (account.cookie) this.cookie = account.cookie;
super(account, 'https://uoj.ac', 'form', logger);
}
cookie: string[] = [];
csrf: string;
get(url: string) {
logger.debug('get', url);
if (!url.includes('//')) url = `${this.account.endpoint || 'https://uoj.ac'}${url}`;
const req = superagent.get(url).set('Cookie', this.cookie);
if (this.account.proxy) return req.proxy(this.account.proxy);
return req;
}
post(url: string) {
logger.debug('post', url, this.cookie);
if (!url.includes('//')) url = `${this.account.endpoint || 'https://uoj.ac'}${url}`;
const req = superagent.post(url).set('Cookie', this.cookie).type('form');
if (this.account.proxy) return req.proxy(this.account.proxy);
return req;
}
async getCsrfToken(url: string) {
const { text: html, header } = await this.get(url);
if (header['set-cookie']) {