diff --git a/packages/hydrojudge/src/compile.ts b/packages/hydrojudge/src/compile.ts index 72512169b..8623277c3 100644 --- a/packages/hydrojudge/src/compile.ts +++ b/packages/hydrojudge/src/compile.ts @@ -11,17 +11,18 @@ export default async function compile( ): Promise { const target = lang.target || 'foo'; const execute = copyIn['execute.sh'] ? '/bin/bash execute.sh' : lang.execute; - if (lang.compile) { + const command = copyIn['compile.sh'] ? '/bin/bash compile.sh' : lang.compile; + if (command) { const { status, stdout, stderr, fileIds, } = await runQueued( - copyIn['compile.sh'] ? '/bin/bash compile.sh' : lang.compile, + command, { copyIn: { ...copyIn, [lang.code_file]: code }, copyOutCached: [target], env: { HYDRO_LANG: lang.key }, time: lang.compile_time_limit || 10000, - memory: lang.compile_memory_limit || 256, + memory: lang.compile_memory_limit || 512, }, `compile[${lang.key}]`, 3, @@ -36,6 +37,7 @@ export default async function compile( execute, copyIn: { ...copyIn, [target]: { fileId: fileIds[target] } }, clean: () => del(fileIds[target]), + _cacheable: target, }; } return { diff --git a/packages/hydrojudge/src/hosts/hydro.ts b/packages/hydrojudge/src/hosts/hydro.ts index 335675842..541efe4b2 100644 --- a/packages/hydrojudge/src/hosts/hydro.ts +++ b/packages/hydrojudge/src/hosts/hydro.ts @@ -51,47 +51,6 @@ export default class Hydro implements Session { setInterval(() => { this.get(''); }, 30000000); // Cookie refresh only } - async cacheOpen(source: string, files: any[], next?) { - await Lock.acquire(`${this.config.host}/${source}`); - try { - return await this._cacheOpen(source, files, next); - } catch (e) { - log.warn('CacheOpen Fail: %s %o %o', source, files, e); - throw e; - } finally { - Lock.release(`${this.config.host}/${source}`); - } - } - - async _cacheOpen(source: string, files: any[], next?) { - const filePath = path.join(getConfig('cache_dir'), this.config.host, source); - await fs.ensureDir(filePath); - if (!files?.length) throw new FormatError('Problem data not found.'); - let etags: Record = {}; - try { - etags = JSON.parse(await fs.readFile(path.join(filePath, 'etags'), 'utf-8')); - } catch (e) { /* ignore */ } - const version = {}; - const filenames = []; - const allFiles = new Set(); - for (const file of files) { - allFiles.add(file.name); - version[file.name] = file.etag + file.lastModified; - if (etags[file.name] !== file.etag + file.lastModified) filenames.push(file.name); - } - for (const name in etags) { - if (!allFiles.has(name) && fs.existsSync(path.join(filePath, name))) await fs.remove(path.join(filePath, name)); - } - if (filenames.length) { - log.info(`Getting problem data: ${this.config.host}/${source}`); - next?.({ message: 'Syncing testdata, please wait...' }); - await this.fetchFile(source, Object.fromEntries(files.map((i) => [i.name, path.join(filePath, i.name)]))); - await fs.writeFile(path.join(filePath, 'etags'), JSON.stringify(version)); - } - await fs.writeFile(path.join(filePath, 'lastUsage'), new Date().getTime().toString()); - return filePath; - } - async fetchFile(namespace: T, files: Record): Promise { if (!namespace) { // record-related resource (code) const name = Object.keys(files)[0].split('#')[0]; diff --git a/packages/hydrojudge/src/interface.ts b/packages/hydrojudge/src/interface.ts index 12f266c7a..35f7e6d2c 100644 --- a/packages/hydrojudge/src/interface.ts +++ b/packages/hydrojudge/src/interface.ts @@ -8,6 +8,7 @@ export interface Execute { execute: string; clean: () => Promise; copyIn: Record; + _cacheable?: string; } export type NextFunction = (body: Partial) => Promise | void; diff --git a/packages/hydrojudge/src/sandbox.ts b/packages/hydrojudge/src/sandbox.ts index 0591ada45..c9128804b 100644 --- a/packages/hydrojudge/src/sandbox.ts +++ b/packages/hydrojudge/src/sandbox.ts @@ -186,8 +186,8 @@ export async function del(fileId: string) { await client.deleteFile(fileId); } -export async function get(fileId: string) { - return await client.getFile(fileId); +export async function get(fileId: string, dest?: string) { + return await client.getFile(fileId, dest); } const queue = new PQueue({ concurrency: getConfig('concurrency') || getConfig('parallelism') }); diff --git a/packages/hydrojudge/src/task.ts b/packages/hydrojudge/src/task.ts index e99d73ae6..ad7b90e73 100644 --- a/packages/hydrojudge/src/task.ts +++ b/packages/hydrojudge/src/task.ts @@ -13,7 +13,9 @@ import { } from './interface'; import judge from './judge'; import { Logger } from './log'; -import { CopyIn, CopyInFile, runQueued } from './sandbox'; +import { + CopyIn, CopyInFile, get, PreparedFile, runQueued, +} from './sandbox'; import { compilerText, Lock, md5 } from './utils'; const logger = new Logger('judge'); @@ -39,6 +41,7 @@ export class JudgeTask { end: (data: Partial) => void; env: Record; callbackCache?: TestCase[]; + compileCache: Record> = {}; constructor(public session: Session, public request: JudgeRequest) { this.stat.receive = new Date(); @@ -98,33 +101,43 @@ export class JudgeTask { async cacheOpen(source: string, files: FileInfo[]) { // Backward compatibility for vj4 if ((this.session as any).cacheOpen) return (this.session as any).cacheOpen(source, files); - const lockKey = `${this.session.config?.host || 'local'}/${source}`; - await Lock.acquire(lockKey); - const filePath = join(getConfig('cache_dir'), lockKey); - await fs.ensureDir(filePath); - if (!files?.length) throw new FormatError('Problem data not found.'); - let etags: Record = {}; + const host = this.session.config?.host || 'local'; + const filePath = join(getConfig('cache_dir'), host, source); + await Lock.acquire(filePath); try { - etags = JSON.parse(await fs.readFile(join(filePath, 'etags'), 'utf-8')); - } catch (e) { /* ignore */ } - const version = {}; - const filenames = []; - const allFiles = new Set(); - for (const file of files) { - allFiles.add(file.name); - version[file.name] = file.etag + file.lastModified; - if (etags[file.name] !== file.etag + file.lastModified) filenames.push(file.name); + await fs.ensureDir(filePath); + if (!files?.length) throw new FormatError('Problem data not found.'); + let etags: Record = {}; + try { + etags = JSON.parse(await fs.readFile(join(filePath, 'etags'), 'utf-8')); + } catch (e) { /* ignore */ } + this.compileCache = etags['*cache'] as any || {}; + delete etags['*cache']; + const version = {}; + const filenames = []; + const allFiles = new Set(); + for (const file of files) { + allFiles.add(file.name); + version[file.name] = file.etag + file.lastModified; + if (etags[file.name] !== file.etag + file.lastModified) filenames.push(file.name); + } + const allFilesToRemove = Object.keys(etags).filter((name) => !allFiles.has(name) && fs.existsSync(join(filePath, name))); + await Promise.all(allFilesToRemove.map((name) => fs.remove(join(filePath, name)))); + if (filenames.length) { + logger.info(`Getting problem data: ${this.session?.config.host || 'local'}/${source}`); + this.next({ message: 'Syncing testdata, please wait...' }); + await this.session.fetchFile(source, Object.fromEntries(files.map((i) => [i.name, join(filePath, i.name)]))); + await fs.writeFile(join(filePath, 'etags'), JSON.stringify(version)); + this.compileCache = {}; + } + await fs.writeFile(join(filePath, 'lastUsage'), Date.now().toString()); + return filePath; + } catch (e) { + logger.warn('CacheOpen Fail: %s %o %o', source, files, e); + throw e; + } finally { + Lock.release(filePath); } - const allFilesToRemove = Object.keys(etags).filter((name) => !allFiles.has(name) && fs.existsSync(join(filePath, name))); - await Promise.all(allFilesToRemove.map((name) => fs.remove(join(filePath, name)))); - if (filenames.length) { - logger.info(`Getting problem data: ${this.session?.config.host || 'local'}/${source}`); - this.next({ message: 'Syncing testdata, please wait...' }); - await this.session.fetchFile(source, Object.fromEntries(files.map((i) => [i.name, join(filePath, i.name)]))); - await fs.writeFile(join(filePath, 'etags'), JSON.stringify(version)); - } - await fs.writeFile(join(filePath, 'lastUsage'), new Date().getTime().toString()); - return filePath; } async doSubmission() { @@ -172,11 +185,16 @@ export class JudgeTask { async compileLocalFile( type: 'interactor' | 'validator' | 'checker' | 'generator' | 'manager' | 'std', - source: CompilableSource, - checkerType?: string, - ) { + source: CompilableSource, checkerType?: string, + ): Promise { if (type === 'checker' && ['default', 'strict'].includes(checkerType)) return { execute: '', copyIn: {}, clean: () => Promise.resolve(null) }; if (type === 'checker' && !checkers[checkerType]) throw new FormatError('Unknown checker type {0}.', [checkerType]); + if (this.compileCache?.[type]) { + return { + ...this.compileCache[type], + clean: () => Promise.resolve(null), + }; + } const withTestlib = type !== 'std' && (type !== 'checker' || checkerType === 'testlib'); const extra = type === 'std' ? this.config.user_extra_files : this.config.judge_extra_files; const copyIn = { @@ -199,8 +217,22 @@ export class JudgeTask { } else lang = this.session.getLang(langId); if (!lang) throw new FormatError(`Unknown ${type} language.`); // TODO cache compiled binary - const result = await compile(lang, { src: file }, copyIn, this.next); + const result = await compile(lang, { src: file }, copyIn); this.clean.push(result.clean); + if (!result._cacheable) return result; + await Lock.acquire(this.folder); + try { + const loc = join(this.folder, `_${type}.cache`); + this.compileCache[type] = { + execute: result.execute, + copyIn: { ...result.copyIn, [result._cacheable]: { src: loc } }, + }; + await get((result.copyIn[result._cacheable] as PreparedFile).fileId, loc); + const currEtag = await fs.readFile(join(this.folder, 'etags'), 'utf-8'); + await fs.writeFile(join(this.folder, 'etags'), JSON.stringify({ ...JSON.parse(currEtag), '*cache': this.compileCache })); + } finally { + Lock.release(this.folder); + } return result; }