judge: change testdata download logic

packages/hydrojudge/src/hosts/hydro.ts

@@ -1,7 +1,6 @@
 /* eslint-disable no-await-in-loop */
 import path from 'path';
 import axios from 'axios';
-import AdmZip from 'adm-zip';
 import fs from 'fs-extra';
 import WebSocket from 'ws';
 import { argv } from 'yargs';
@@ -206,45 +205,41 @@ export default class Hydro {
         setInterval(() => { this.axios.get('judge/noop'); }, 30000000);
     }
 
-    async problemDataVersion(domainId: string, pid: string) {
-        await this.ensureLogin();
-        const res = await this.axios.get(`d/${domainId}/p/${pid}`);
-        const data = res.data.pdoc?.data;
-        if (!data) throw new FormatError(`没有找到测试数据 ${domainId}/${pid}`);
-        return data;
-    }
-
-    async problemData(domainId: string, pid: string, savePath: string, retry = 3, next?) {
-        log.info(`Getting problem data: ${this.config.host}/${domainId}/${pid}`);
-        await this.ensureLogin();
-        if (next) next({ judge_text: '正在同步测试数据，请稍后' });
-        const tmpFilePath = path.resolve(CACHE_DIR, `download_${this.config.host}_${domainId}_${pid}`);
+    async cacheOpen(domainId: string, pid: string, files: any[], next?) {
+        const domainDir = path.join(CACHE_DIR, this.config.host, domainId);
+        const filePath = path.join(domainDir, pid); await fs.ensureDir(filePath);
+        if (!files.length) throw new SystemError('Problem data not found.');
+        let etags: Record<string, string>;
         try {
-            const t = await this.axios.get(`${this.config.server_url}d/${domainId}/p/${pid}/data`);
-            const res = await this.axios.get(t.data.url, { responseType: 'stream' });
-            const w = fs.createWriteStream(tmpFilePath);
-            res.data.pipe(w);
-            await new Promise((resolve, reject) => {
-                w.on('finish', resolve);
-                w.on('error', reject);
-            });
-            fs.ensureDirSync(path.dirname(savePath));
-            const zip = new AdmZip(tmpFilePath);
-            const entries = zip.getEntries();
-            if (entries.length > 512) throw new FormatError('Too many files');
-            await new Promise((resolve, reject) => {
-                zip.extractAllToAsync(savePath, true, (e) => {
-                    if (e) reject(e);
-                    else resolve();
-                });
-            });
-            await fs.unlink(tmpFilePath);
-            await this.processData(savePath).catch(noop);
-        } catch (e) {
-            if (retry) await this.problemData(domainId, pid, savePath, retry - 1);
-            else throw e;
+            etags = JSON.parse(fs.readFileSync(path.join(filePath, 'etags')).toString());
+        } catch (e) { /* ignore */ }
+        const version = {};
+        const filenames = [];
+        for (const file of files) {
+            version[file.name] = file.etag;
+            if (etags[file.name] !== file.etag) filenames.push(file.name);
         }
-        return savePath;
+        if (filenames.length) {
+            log.info(`Getting problem data: ${this.config.host}/${domainId}/${pid}`);
+            if (next) next({ judge_text: '正在同步测试数据，请稍后' });
+            await this.ensureLogin();
+            const res = await this.axios.post(`/d/${domainId}/p/${pid}/files`, {
+                operation: 'get_links',
+                files: filenames,
+            });
+            for (const name in res.urls) {
+                const f = await this.axios.get(res.urls[name], { responseType: 'stream' });
+                const w = fs.createWriteStream(path.join(filePath, name));
+                f.data.pipe(w);
+                await new Promise((resolve, reject) => {
+                    w.on('finish', resolve);
+                    w.on('error', reject);
+                });
+            }
+            fs.writeFileSync(path.join(filePath, 'etags'), JSON.stringify(version));
+            await this.processData(filePath).catch(noop);
+        }
+        return filePath;
     }
 
     async consume(queue: Queue<any>) {
@@ -338,28 +333,6 @@ export default class Hydro {
         }
     }
 
-    async cacheOpen(domainId: string, pid: string, next) {
-        const domainDir = path.join(CACHE_DIR, this.config.host, domainId);
-        const filePath = path.join(domainDir, pid);
-        const version = await this.problemDataVersion(domainId, pid);
-        if (fs.existsSync(filePath)) {
-            let ver;
-            try {
-                ver = fs.readFileSync(path.join(filePath, 'version')).toString();
-            } catch (e) { /* ignore */ }
-            if (version === ver) {
-                log.debug('Cache found at %s', filePath);
-                return filePath;
-            }
-            fs.removeSync(filePath);
-        }
-        log.debug('Downloading testdata to %s', filePath);
-        fs.ensureDirSync(domainDir);
-        await this.problemData(domainId, pid, filePath, 3, next);
-        fs.writeFileSync(path.join(filePath, 'version'), version);
-        return filePath;
-    }
-
     async retry(queue: Queue<any>) {
         this.consume(queue).catch(() => {
             setTimeout(() => {

packages/hydrojudge/src/service.ts

@@ -6,7 +6,6 @@ import { argv } from 'yargs';
 import { ObjectID } from 'bson';
 import fs from 'fs-extra';
 import { homedir, tmpdir } from 'os';
-import AdmZip from 'adm-zip';
 import { noop } from 'lodash';
 import { Logger } from 'hydrooj/dist/logger';
 import * as monitor from 'hydrooj/dist/service/monitor';
@@ -30,7 +29,8 @@ async function postInit() {
     const judge = require('./judge');
     const sysinfo = require('./sysinfo');
 
-    const { problem, file, task } = global.Hydro.model;
+    const { task } = global.Hydro.model;
+    const { storage } = global.Hydro.service;
     const _judge = global.Hydro.handler.judge as any;
 
     const info = await sysinfo.get();
@@ -62,51 +62,23 @@ async function postInit() {
         }
     }
 
-    async function problemData(domainId: string, pid: string, savePath: string) {
-        const tmpFilePath = path.resolve(homedir(), '.cache', 'hydro', 'judge', `download_${domainId}_${pid}`);
-        const pdoc = await problem.get(domainId, pid);
-        // FIXME doesnt work on copied problems
-        // @ts-ignore
-        const data = await file.get(pdoc.data);
-        if (!data) throw new SystemError('Problem data not found.');
-        const w = fs.createWriteStream(tmpFilePath);
-        data.pipe(w);
-        await new Promise((resolve, reject) => {
-            w.on('finish', resolve);
-            w.on('error', reject);
-        });
-        fs.ensureDirSync(path.dirname(savePath));
-        const zip = new AdmZip(tmpFilePath);
-        const entries = zip.getEntries();
-        if (entries.length > 512) throw new FormatError('Too many files.');
-        await new Promise((resolve, reject) => {
-            zip.extractAllToAsync(savePath, true, (e) => {
-                if (e) reject(e);
-                else resolve();
-            });
-        });
-        await fs.unlink(tmpFilePath);
-        await processData(savePath).catch(noop);
-        return savePath;
-    }
-
-    async function cacheOpen(domainId: string, pid: string, version: string) {
+    async function cacheOpen(domainId: string, pid: string, files: any[]) {
         const filePath = path.join(homedir(), '.cache', 'hydro', 'judge', domainId, pid);
-        if (fs.existsSync(filePath)) {
-            let ver: string;
-            try {
-                ver = fs.readFileSync(path.join(filePath, 'version')).toString();
-            } catch (e) { /* ignore */ }
-            if (version === ver) {
-                logger.debug('Cache found at %s', filePath);
-                return filePath;
+        await fs.ensureDir(filePath);
+        if (!files.length) throw new SystemError('Problem data not found.');
+        let etags: Record<string, string>;
+        try {
+            etags = JSON.parse(fs.readFileSync(path.join(filePath, 'etags')).toString());
+        } catch (e) { /* ignore */ }
+        const version = {};
+        for (const file of files) {
+            version[file.name] = file.etag;
+            if (etags[file.name] !== file.etag) {
+                await storage.get(`${file.prefix}${file.name}`, path.join(filePath, file.name));
             }
-            await fs.remove(filePath);
         }
-        logger.debug('Downloading testdata to %s', filePath);
-        fs.ensureDirSync(filePath);
-        await problemData(domainId, pid, filePath);
-        fs.writeFileSync(path.join(filePath, 'version'), version);
+        fs.writeFileSync(path.join(filePath, 'etags'), JSON.stringify(version));
+        await processData(filePath).catch(noop);
         return filePath;
     }
 
@@ -181,7 +153,7 @@ async function postInit() {
 
         code: string;
 
-        data: string;
+        data: any[];
 
         config: any;
 

packages/hydrooj/src/model/record.ts

@@ -9,6 +9,7 @@ import {
     Rdoc, TestCase, RunConfig, ContestInfo, ProblemConfig,
 } from '../interface';
 import db from '../service/db';
+import storage from '../service/storage';
 
 export const coll: Collection<Rdoc> = db.collection('record');
 
@@ -57,11 +58,11 @@ export async function get(domainId: string, _id: ObjectID): Promise<Rdoc | null>
 export async function judge(domainId: string, rid: ObjectID, priority = 1) {
     const rdoc = await get(domainId, rid);
     let config: RunConfig | ProblemConfig = rdoc.config;
-    let data;
+    let data = [];
     if (rdoc.pid) {
         const pdoc = await problem.get(domainId, rdoc.pid);
+        data = await storage.list(`problem/${domainId}/${rdoc.pid}/testdata/`);
         if (!config) config = pdoc?.config || {};
-        data = pdoc?.data;
     }
     delete rdoc._id;
     await task.add({