diff --git a/packages/hydrojudge/src/hosts/hydro.ts b/packages/hydrojudge/src/hosts/hydro.ts index 92267d710..33ca540fa 100644 --- a/packages/hydrojudge/src/hosts/hydro.ts +++ b/packages/hydrojudge/src/hosts/hydro.ts @@ -1,7 +1,6 @@ /* eslint-disable no-await-in-loop */ import path from 'path'; import axios from 'axios'; -import AdmZip from 'adm-zip'; import fs from 'fs-extra'; import WebSocket from 'ws'; import { argv } from 'yargs'; @@ -206,45 +205,41 @@ export default class Hydro { setInterval(() => { this.axios.get('judge/noop'); }, 30000000); } - async problemDataVersion(domainId: string, pid: string) { - await this.ensureLogin(); - const res = await this.axios.get(`d/${domainId}/p/${pid}`); - const data = res.data.pdoc?.data; - if (!data) throw new FormatError(`没有找到测试数据 ${domainId}/${pid}`); - return data; - } - - async problemData(domainId: string, pid: string, savePath: string, retry = 3, next?) { - log.info(`Getting problem data: ${this.config.host}/${domainId}/${pid}`); - await this.ensureLogin(); - if (next) next({ judge_text: '正在同步测试数据，请稍后' }); - const tmpFilePath = path.resolve(CACHE_DIR, `download_${this.config.host}_${domainId}_${pid}`); + async cacheOpen(domainId: string, pid: string, files: any[], next?) { + const domainDir = path.join(CACHE_DIR, this.config.host, domainId); + const filePath = path.join(domainDir, pid); await fs.ensureDir(filePath); + if (!files.length) throw new SystemError('Problem data not found.'); + let etags: Record; try { - const t = await this.axios.get(`${this.config.server_url}d/${domainId}/p/${pid}/data`); - const res = await this.axios.get(t.data.url, { responseType: 'stream' }); - const w = fs.createWriteStream(tmpFilePath); - res.data.pipe(w); - await new Promise((resolve, reject) => { - w.on('finish', resolve); - w.on('error', reject); - }); - fs.ensureDirSync(path.dirname(savePath)); - const zip = new AdmZip(tmpFilePath); - const entries = zip.getEntries(); - if (entries.length > 512) throw new FormatError('Too many files'); - await new Promise((resolve, reject) => { - zip.extractAllToAsync(savePath, true, (e) => { - if (e) reject(e); - else resolve(); - }); - }); - await fs.unlink(tmpFilePath); - await this.processData(savePath).catch(noop); - } catch (e) { - if (retry) await this.problemData(domainId, pid, savePath, retry - 1); - else throw e; + etags = JSON.parse(fs.readFileSync(path.join(filePath, 'etags')).toString()); + } catch (e) { /* ignore */ } + const version = {}; + const filenames = []; + for (const file of files) { + version[file.name] = file.etag; + if (etags[file.name] !== file.etag) filenames.push(file.name); } - return savePath; + if (filenames.length) { + log.info(`Getting problem data: ${this.config.host}/${domainId}/${pid}`); + if (next) next({ judge_text: '正在同步测试数据，请稍后' }); + await this.ensureLogin(); + const res = await this.axios.post(`/d/${domainId}/p/${pid}/files`, { + operation: 'get_links', + files: filenames, + }); + for (const name in res.urls) { + const f = await this.axios.get(res.urls[name], { responseType: 'stream' }); + const w = fs.createWriteStream(path.join(filePath, name)); + f.data.pipe(w); + await new Promise((resolve, reject) => { + w.on('finish', resolve); + w.on('error', reject); + }); + } + fs.writeFileSync(path.join(filePath, 'etags'), JSON.stringify(version)); + await this.processData(filePath).catch(noop); + } + return filePath; } async consume(queue: Queue) { @@ -338,28 +333,6 @@ export default class Hydro { } } - async cacheOpen(domainId: string, pid: string, next) { - const domainDir = path.join(CACHE_DIR, this.config.host, domainId); - const filePath = path.join(domainDir, pid); - const version = await this.problemDataVersion(domainId, pid); - if (fs.existsSync(filePath)) { - let ver; - try { - ver = fs.readFileSync(path.join(filePath, 'version')).toString(); - } catch (e) { /* ignore */ } - if (version === ver) { - log.debug('Cache found at %s', filePath); - return filePath; - } - fs.removeSync(filePath); - } - log.debug('Downloading testdata to %s', filePath); - fs.ensureDirSync(domainDir); - await this.problemData(domainId, pid, filePath, 3, next); - fs.writeFileSync(path.join(filePath, 'version'), version); - return filePath; - } - async retry(queue: Queue) { this.consume(queue).catch(() => { setTimeout(() => { diff --git a/packages/hydrojudge/src/service.ts b/packages/hydrojudge/src/service.ts index 56e6e17d0..4bc73349b 100644 --- a/packages/hydrojudge/src/service.ts +++ b/packages/hydrojudge/src/service.ts @@ -6,7 +6,6 @@ import { argv } from 'yargs'; import { ObjectID } from 'bson'; import fs from 'fs-extra'; import { homedir, tmpdir } from 'os'; -import AdmZip from 'adm-zip'; import { noop } from 'lodash'; import { Logger } from 'hydrooj/dist/logger'; import * as monitor from 'hydrooj/dist/service/monitor'; @@ -30,7 +29,8 @@ async function postInit() { const judge = require('./judge'); const sysinfo = require('./sysinfo'); - const { problem, file, task } = global.Hydro.model; + const { task } = global.Hydro.model; + const { storage } = global.Hydro.service; const _judge = global.Hydro.handler.judge as any; const info = await sysinfo.get(); @@ -62,51 +62,23 @@ async function postInit() { } } - async function problemData(domainId: string, pid: string, savePath: string) { - const tmpFilePath = path.resolve(homedir(), '.cache', 'hydro', 'judge', `download_${domainId}_${pid}`); - const pdoc = await problem.get(domainId, pid); - // FIXME doesnt work on copied problems - // @ts-ignore - const data = await file.get(pdoc.data); - if (!data) throw new SystemError('Problem data not found.'); - const w = fs.createWriteStream(tmpFilePath); - data.pipe(w); - await new Promise((resolve, reject) => { - w.on('finish', resolve); - w.on('error', reject); - }); - fs.ensureDirSync(path.dirname(savePath)); - const zip = new AdmZip(tmpFilePath); - const entries = zip.getEntries(); - if (entries.length > 512) throw new FormatError('Too many files.'); - await new Promise((resolve, reject) => { - zip.extractAllToAsync(savePath, true, (e) => { - if (e) reject(e); - else resolve(); - }); - }); - await fs.unlink(tmpFilePath); - await processData(savePath).catch(noop); - return savePath; - } - - async function cacheOpen(domainId: string, pid: string, version: string) { + async function cacheOpen(domainId: string, pid: string, files: any[]) { const filePath = path.join(homedir(), '.cache', 'hydro', 'judge', domainId, pid); - if (fs.existsSync(filePath)) { - let ver: string; - try { - ver = fs.readFileSync(path.join(filePath, 'version')).toString(); - } catch (e) { /* ignore */ } - if (version === ver) { - logger.debug('Cache found at %s', filePath); - return filePath; + await fs.ensureDir(filePath); + if (!files.length) throw new SystemError('Problem data not found.'); + let etags: Record; + try { + etags = JSON.parse(fs.readFileSync(path.join(filePath, 'etags')).toString()); + } catch (e) { /* ignore */ } + const version = {}; + for (const file of files) { + version[file.name] = file.etag; + if (etags[file.name] !== file.etag) { + await storage.get(`${file.prefix}${file.name}`, path.join(filePath, file.name)); } - await fs.remove(filePath); } - logger.debug('Downloading testdata to %s', filePath); - fs.ensureDirSync(filePath); - await problemData(domainId, pid, filePath); - fs.writeFileSync(path.join(filePath, 'version'), version); + fs.writeFileSync(path.join(filePath, 'etags'), JSON.stringify(version)); + await processData(filePath).catch(noop); return filePath; } @@ -181,7 +153,7 @@ async function postInit() { code: string; - data: string; + data: any[]; config: any; diff --git a/packages/hydrooj/src/model/record.ts b/packages/hydrooj/src/model/record.ts index 07e8f0097..74cbaaa14 100644 --- a/packages/hydrooj/src/model/record.ts +++ b/packages/hydrooj/src/model/record.ts @@ -9,6 +9,7 @@ import { Rdoc, TestCase, RunConfig, ContestInfo, ProblemConfig, } from '../interface'; import db from '../service/db'; +import storage from '../service/storage'; export const coll: Collection = db.collection('record'); @@ -57,11 +58,11 @@ export async function get(domainId: string, _id: ObjectID): Promise export async function judge(domainId: string, rid: ObjectID, priority = 1) { const rdoc = await get(domainId, rid); let config: RunConfig | ProblemConfig = rdoc.config; - let data; + let data = []; if (rdoc.pid) { const pdoc = await problem.get(domainId, rdoc.pid); + data = await storage.list(`problem/${domainId}/${rdoc.pid}/testdata/`); if (!config) config = pdoc?.config || {}; - data = pdoc?.data; } delete rdoc._id; await task.add({