From e8573d70611b6fe5fd25e183b826a9bfee937180 Mon Sep 17 00:00:00 2001 From: undefined Date: Sun, 26 Oct 2025 20:27:46 +0800 Subject: [PATCH] vjudge: handle problem resync & fix uoj --- packages/hydrooj/src/service/storage.ts | 2 +- packages/vjudge/src/index.ts | 27 +++++++++++---------- packages/vjudge/src/interface.ts | 4 +-- packages/vjudge/src/providers/codeforces.ts | 8 +++--- packages/vjudge/src/providers/csgoj.ts | 3 +-- packages/vjudge/src/providers/hduoj.ts | 3 +-- packages/vjudge/src/providers/poj.ts | 3 +-- packages/vjudge/src/providers/spoj.ts | 3 +-- packages/vjudge/src/providers/uoj.ts | 10 +++++--- packages/vjudge/src/providers/yacs.ts | 3 +-- 10 files changed, 34 insertions(+), 32 deletions(-) diff --git a/packages/hydrooj/src/service/storage.ts b/packages/hydrooj/src/service/storage.ts index 3e0fc59ac..118f58a20 100644 --- a/packages/hydrooj/src/service/storage.ts +++ b/packages/hydrooj/src/service/storage.ts @@ -325,7 +325,7 @@ class LocalStorageService { async get(target: string, path?: string) { target = resolve(this.dir, convertPath(target)); - if (!existsSync(target)) throw new Error('File not found'); + if (!existsSync(target)) throw new Error(`File not found: ${target}`); if (path) await copyFile(target, path); return createReadStream(target); } diff --git a/packages/vjudge/src/index.ts b/packages/vjudge/src/index.ts index 8895864bf..536485fbd 100644 --- a/packages/vjudge/src/index.ts +++ b/packages/vjudge/src/index.ts @@ -80,12 +80,13 @@ class AccountService { } } - async sync(target: string, resync = false, list: string) { - let page = 1; - let pids = await this.api.listProblem(page, resync, list); + async sync(target: string, fromPage = 0, list: string) { + let page = Math.max(fromPage, 1); + let pids: string[] = []; const [domainId, namespaceId] = target.split('.'); - while (pids.length) { - logger.info(`${domainId}: Syncing page ${page}`); + do { + pids = await this.api.listProblem(page, fromPage, list); + logger.info(`${target}: Syncing page ${page}`); for (const id of pids) { if (id.startsWith('LIST::')) { await this.addProblemList(id.split('::')[1]); @@ -110,16 +111,16 @@ class AccountService { await ProblemModel.addTestdata(domainId, docId, key, res.data[key]); } if (res.solution) await SolutionModel.add(domainId, docId, 1, res.solution); - logger.info(`${domainId}: problem ${docId}(${pid}) sync done -> ${targetPid}(${docId})`); + logger.info(`${domainId}: problem ${pid} sync done -> ${targetPid}(${docId})`); } finally { delete syncing[`${domainId}/${pid}`]; } await sleep(5000); } page++; - if (this.stopped) return; - pids = await this.api.listProblem(page, resync, list); - } + if (this.stopped) return page - 1; + } while (pids.length); + return page - 2; } async login() { @@ -141,11 +142,11 @@ class AccountService { this.listUpdated = false; for (const listName of this.problemLists) { for (const mount of mounts) { - if (!mount.syncDone?.[listName]) await this.sync(mount._id, false, listName); - else await this.sync(mount._id, true, listName); - await collMount.updateOne({ _id: mount._id }, { $set: { [`syncDone.${listName}`]: true } }); + const from = typeof mount.syncDone?.[listName] === 'number' ? mount.syncDone?.[listName] : 0; + const page = await this.sync(mount._id, from, listName); + await collMount.updateOne({ _id: mount._id }, { $set: { [`syncDone.${listName}`]: page } }); mount.syncDone ||= {}; - mount.syncDone[listName] = true; + mount.syncDone[listName] = page; } } } while (this.listUpdated); diff --git a/packages/vjudge/src/interface.ts b/packages/vjudge/src/interface.ts index ea0f7adc2..c6af639a4 100644 --- a/packages/vjudge/src/interface.ts +++ b/packages/vjudge/src/interface.ts @@ -17,7 +17,7 @@ export interface RemoteAccount { export interface VjudgeMount { _id: string; // domainId, or `${domainId}.${namespace}` mount: string; - syncDone: Record; + syncDone: Record; } declare module 'hydrooj' { interface Collections { @@ -38,7 +38,7 @@ export interface IBasicProvider { solution?: string; }>; entryProblemLists?: string[]; - listProblem: (page: number, resync: boolean, listId: string) => Promise; + listProblem: (page: number, resyncFrom: number, listId: string) => Promise; submitProblem: (id: string, lang: string, code: string, info: any, next: NextFunction, end: NextFunction) => Promise; waitForSubmission: (id: string, next: NextFunction, end: NextFunction) => Promise; checkStatus?: (onCheckFunc: boolean) => Promise; diff --git a/packages/vjudge/src/providers/codeforces.ts b/packages/vjudge/src/providers/codeforces.ts index 9923568b5..02b85e592 100644 --- a/packages/vjudge/src/providers/codeforces.ts +++ b/packages/vjudge/src/providers/codeforces.ts @@ -230,10 +230,12 @@ export default class CodeforcesProvider extends BasicFetcher implements IBasicPr // TL;DR; add `gym` to this list to enable codeforces gym entryProblemLists = ['main']; - async listProblem(page: number, resync = false, listName: string) { - if (resync && page > 1) return []; - if (resync && listName.startsWith('GYM')) return []; + async listProblem(page: number, resyncFrom: number, listName: string) { if (listName.startsWith('GYM') && page > 1) return []; + if (resyncFrom) { + if (page > resyncFrom || listName.startsWith('GYM')) return []; + page = 1; + } const { document } = await this.html(listName === 'main' ? `/problemset/page/${page}` : listName === 'gym' diff --git a/packages/vjudge/src/providers/csgoj.ts b/packages/vjudge/src/providers/csgoj.ts index 7992253ad..2530e202c 100644 --- a/packages/vjudge/src/providers/csgoj.ts +++ b/packages/vjudge/src/providers/csgoj.ts @@ -123,8 +123,7 @@ export default class CSGOJProvider extends BasicFetcher implements IBasicProvide }; } - async listProblem(page: number, resync = false) { - if (resync && page > 1) return []; + async listProblem(page: number) { const offset = (page - 1) * 100; const result = await this .get(`/csgoj/problemset/problemset_ajax?search=&sort=problem_id&order=asc&offset=${offset}&limit=100`) diff --git a/packages/vjudge/src/providers/hduoj.ts b/packages/vjudge/src/providers/hduoj.ts index 0aacc3402..ab0d8d867 100644 --- a/packages/vjudge/src/providers/hduoj.ts +++ b/packages/vjudge/src/providers/hduoj.ts @@ -130,8 +130,7 @@ export default class HDUOJProvider extends BasicFetcher implements IBasicProvide }; } - async listProblem(page: number, resync = false) { - if (resync && page > 1) return []; + async listProblem(page: number) { const { text } = await this.get(`/listproblem.php?vol=${page}`); const $dom = new JSDOM(text); const ProblemTable = $dom.window.document.querySelector('.table_text'); diff --git a/packages/vjudge/src/providers/poj.ts b/packages/vjudge/src/providers/poj.ts index 35ef180a6..9c9f658e5 100644 --- a/packages/vjudge/src/providers/poj.ts +++ b/packages/vjudge/src/providers/poj.ts @@ -161,8 +161,7 @@ export default class POJProvider extends BasicFetcher implements IBasicProvider }; } - async listProblem(page: number, resync = false) { - if (resync && page > 1) return []; + async listProblem(page: number) { const { text } = await this.get(`/problemlist?volume=${page}`); const $dom = new JSDOM(text); return Array.from($dom.window.document.querySelectorAll('.a>tbody>tr[align="center"]')) diff --git a/packages/vjudge/src/providers/spoj.ts b/packages/vjudge/src/providers/spoj.ts index bf6baa606..f7bc57ffb 100644 --- a/packages/vjudge/src/providers/spoj.ts +++ b/packages/vjudge/src/providers/spoj.ts @@ -81,8 +81,7 @@ langs: ${JSON.stringify(langs)}`), }; } - async listProblem(page: number, resync = false) { - if (resync) return []; + async listProblem(page: number) { const { document } = await this.html(`/problems/classical/sort=0,start=${page * 50 - 50}`); const index = document.querySelector('ul.pagination').querySelector('li.active').children[0].innerHTML.trim(); if (index !== page.toString()) return []; diff --git a/packages/vjudge/src/providers/uoj.ts b/packages/vjudge/src/providers/uoj.ts index 8809948b4..2398272c9 100644 --- a/packages/vjudge/src/providers/uoj.ts +++ b/packages/vjudge/src/providers/uoj.ts @@ -24,7 +24,12 @@ const MAPPING = { export default class UOJProvider extends BasicFetcher implements IBasicProvider { constructor(public account: RemoteAccount, private save: (data: any) => Promise) { - super(account, 'https://uoj.ac', 'form', logger); + // UOJ WAF start blocking vjudge User-Agent + super(account, 'https://uoj.ac', 'form', logger, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + }, + }); } csrf: string; @@ -131,8 +136,7 @@ export default class UOJProvider extends BasicFetcher implements IBasicProvider }; } - async listProblem(page: number, resync = false) { - if (resync && page > 1) return []; + async listProblem(page: number) { const { text } = await this.get(`/problems?page=${page}`); const $dom = new JSDOM(text); const index = $dom.window.document.querySelector('ul.pagination>li.active>a').innerHTML; diff --git a/packages/vjudge/src/providers/yacs.ts b/packages/vjudge/src/providers/yacs.ts index 5768fdbf4..69ab8f8f6 100644 --- a/packages/vjudge/src/providers/yacs.ts +++ b/packages/vjudge/src/providers/yacs.ts @@ -84,8 +84,7 @@ export default class YACSProvider extends BasicFetcher implements IBasicProvider }; } - async listProblem(page: number, resync = false) { - if (resync && page > 1) return []; + async listProblem(page: number) { const data = await this.getData(`/problem?pi=${page}`); return data.problemList .filter((problem) => problem.contest.status === '榜单已公布')