vjudge: handle problem resync & fix uoj

packages/hydrooj/src/service/storage.ts

@@ -325,7 +325,7 @@ class LocalStorageService {
 
     async get(target: string, path?: string) {
         target = resolve(this.dir, convertPath(target));
-        if (!existsSync(target)) throw new Error('File not found');
+        if (!existsSync(target)) throw new Error(`File not found: ${target}`);
         if (path) await copyFile(target, path);
         return createReadStream(target);
     }

packages/vjudge/src/index.ts

@@ -80,12 +80,13 @@ class AccountService {
         }
     }
 
-    async sync(target: string, resync = false, list: string) {
+    async sync(target: string, fromPage = 0, list: string) {
-        let page = 1;
+        let page = Math.max(fromPage, 1);
-        let pids = await this.api.listProblem(page, resync, list);
+        let pids: string[] = [];
         const [domainId, namespaceId] = target.split('.');
-        while (pids.length) {
+        do {
-            logger.info(`${domainId}: Syncing page ${page}`);
+            pids = await this.api.listProblem(page, fromPage, list);
+            logger.info(`${target}: Syncing page ${page}`);
             for (const id of pids) {
                 if (id.startsWith('LIST::')) {
                     await this.addProblemList(id.split('::')[1]);
@@ -110,16 +111,16 @@ class AccountService {
                         await ProblemModel.addTestdata(domainId, docId, key, res.data[key]);
                     }
                     if (res.solution) await SolutionModel.add(domainId, docId, 1, res.solution);
-                    logger.info(`${domainId}: problem ${docId}(${pid}) sync done -> ${targetPid}(${docId})`);
+                    logger.info(`${domainId}: problem ${pid} sync done -> ${targetPid}(${docId})`);
                 } finally {
                     delete syncing[`${domainId}/${pid}`];
                 }
                 await sleep(5000);
             }
             page++;
-            if (this.stopped) return;
+            if (this.stopped) return page - 1;
-            pids = await this.api.listProblem(page, resync, list);
+        } while (pids.length);
-        }
+        return page - 2;
     }
 
     async login() {
@@ -141,11 +142,11 @@ class AccountService {
                 this.listUpdated = false;
                 for (const listName of this.problemLists) {
                     for (const mount of mounts) {
-                        if (!mount.syncDone?.[listName]) await this.sync(mount._id, false, listName);
+                        const from = typeof mount.syncDone?.[listName] === 'number' ? mount.syncDone?.[listName] : 0;
-                        else await this.sync(mount._id, true, listName);
+                        const page = await this.sync(mount._id, from, listName);
-                        await collMount.updateOne({ _id: mount._id }, { $set: { [`syncDone.${listName}`]: true } });
+                        await collMount.updateOne({ _id: mount._id }, { $set: { [`syncDone.${listName}`]: page } });
                         mount.syncDone ||= {};
-                        mount.syncDone[listName] = true;
+                        mount.syncDone[listName] = page;
                     }
                 }
             } while (this.listUpdated);

packages/vjudge/src/interface.ts

@@ -17,7 +17,7 @@ export interface RemoteAccount {
 export interface VjudgeMount {
     _id: string; //  domainId, or `${domainId}.${namespace}`
     mount: string;
-    syncDone: Record<string, boolean>;
+    syncDone: Record<string, number>;
 }
 declare module 'hydrooj' {
     interface Collections {
@@ -38,7 +38,7 @@ export interface IBasicProvider {
         solution?: string;
     }>;
     entryProblemLists?: string[];
-    listProblem: (page: number, resync: boolean, listId: string) => Promise<string[]>;
+    listProblem: (page: number, resyncFrom: number, listId: string) => Promise<string[]>;
     submitProblem: (id: string, lang: string, code: string, info: any, next: NextFunction, end: NextFunction) => Promise<string | void>;
     waitForSubmission: (id: string, next: NextFunction, end: NextFunction) => Promise<void>;
     checkStatus?: (onCheckFunc: boolean) => Promise<void>;

packages/vjudge/src/providers/codeforces.ts

@@ -230,10 +230,12 @@ export default class CodeforcesProvider extends BasicFetcher implements IBasicPr
 
     // TL;DR; add `gym` to this list to enable codeforces gym
     entryProblemLists = ['main'];
-    async listProblem(page: number, resync = false, listName: string) {
+    async listProblem(page: number, resyncFrom: number, listName: string) {
-        if (resync && page > 1) return [];
-        if (resync && listName.startsWith('GYM')) return [];
         if (listName.startsWith('GYM') && page > 1) return [];
+        if (resyncFrom) {
+            if (page > resyncFrom || listName.startsWith('GYM')) return [];
+            page = 1;
+        }
         const { document } = await this.html(listName === 'main'
             ? `/problemset/page/${page}`
             : listName === 'gym'

packages/vjudge/src/providers/csgoj.ts

@@ -123,8 +123,7 @@ export default class CSGOJProvider extends BasicFetcher implements IBasicProvide
         };
     }
 
-    async listProblem(page: number, resync = false) {
+    async listProblem(page: number) {
-        if (resync && page > 1) return [];
         const offset = (page - 1) * 100;
         const result = await this
             .get(`/csgoj/problemset/problemset_ajax?search=&sort=problem_id&order=asc&offset=${offset}&limit=100`)

packages/vjudge/src/providers/hduoj.ts

@@ -130,8 +130,7 @@ export default class HDUOJProvider extends BasicFetcher implements IBasicProvide
         };
     }
 
-    async listProblem(page: number, resync = false) {
+    async listProblem(page: number) {
-        if (resync && page > 1) return [];
         const { text } = await this.get(`/listproblem.php?vol=${page}`);
         const $dom = new JSDOM(text);
         const ProblemTable = $dom.window.document.querySelector('.table_text');

packages/vjudge/src/providers/poj.ts

@@ -161,8 +161,7 @@ export default class POJProvider extends BasicFetcher implements IBasicProvider
         };
     }
 
-    async listProblem(page: number, resync = false) {
+    async listProblem(page: number) {
-        if (resync && page > 1) return [];
         const { text } = await this.get(`/problemlist?volume=${page}`);
         const $dom = new JSDOM(text);
         return Array.from($dom.window.document.querySelectorAll('.a>tbody>tr[align="center"]'))

packages/vjudge/src/providers/spoj.ts

@@ -81,8 +81,7 @@ langs: ${JSON.stringify(langs)}`),
         };
     }
 
-    async listProblem(page: number, resync = false) {
+    async listProblem(page: number) {
-        if (resync) return [];
         const { document } = await this.html(`/problems/classical/sort=0,start=${page * 50 - 50}`);
         const index = document.querySelector('ul.pagination').querySelector('li.active').children[0].innerHTML.trim();
         if (index !== page.toString()) return [];

packages/vjudge/src/providers/uoj.ts

@@ -24,7 +24,12 @@ const MAPPING = {
 
 export default class UOJProvider extends BasicFetcher implements IBasicProvider {
     constructor(public account: RemoteAccount, private save: (data: any) => Promise<void>) {
-        super(account, 'https://uoj.ac', 'form', logger);
+        // UOJ WAF start blocking vjudge User-Agent
+        super(account, 'https://uoj.ac', 'form', logger, {
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            },
+        });
     }
 
     csrf: string;
@@ -131,8 +136,7 @@ export default class UOJProvider extends BasicFetcher implements IBasicProvider
         };
     }
 
-    async listProblem(page: number, resync = false) {
+    async listProblem(page: number) {
-        if (resync && page > 1) return [];
         const { text } = await this.get(`/problems?page=${page}`);
         const $dom = new JSDOM(text);
         const index = $dom.window.document.querySelector('ul.pagination>li.active>a').innerHTML;

packages/vjudge/src/providers/yacs.ts

@@ -84,8 +84,7 @@ export default class YACSProvider extends BasicFetcher implements IBasicProvider
         };
     }
 
-    async listProblem(page: number, resync = false) {
+    async listProblem(page: number) {
-        if (resync && page > 1) return [];
         const data = await this.getData(`/problem?pi=${page}`);
         return data.problemList
             .filter((problem) => problem.contest.status === '榜单已公布')