cache compiled binaries

packages/hydrojudge/src/compile.ts

@@ -11,17 +11,18 @@ export default async function compile(
 ): Promise<Execute> {
     const target = lang.target || 'foo';
     const execute = copyIn['execute.sh'] ? '/bin/bash execute.sh' : lang.execute;
-    if (lang.compile) {
+    const command = copyIn['compile.sh'] ? '/bin/bash compile.sh' : lang.compile;
+    if (command) {
         const {
             status, stdout, stderr, fileIds,
         } = await runQueued(
-            copyIn['compile.sh'] ? '/bin/bash compile.sh' : lang.compile,
+            command,
             {
                 copyIn: { ...copyIn, [lang.code_file]: code },
                 copyOutCached: [target],
                 env: { HYDRO_LANG: lang.key },
                 time: lang.compile_time_limit || 10000,
-                memory: lang.compile_memory_limit || 256,
+                memory: lang.compile_memory_limit || 512,
             },
             `compile[${lang.key}]`,
             3,
@@ -36,6 +37,7 @@ export default async function compile(
             execute,
             copyIn: { ...copyIn, [target]: { fileId: fileIds[target] } },
             clean: () => del(fileIds[target]),
+            _cacheable: target,
         };
     }
     return {

packages/hydrojudge/src/hosts/hydro.ts

@@ -51,47 +51,6 @@ export default class Hydro implements Session {
         setInterval(() => { this.get(''); }, 30000000); // Cookie refresh only
     }
 
-    async cacheOpen(source: string, files: any[], next?) {
-        await Lock.acquire(`${this.config.host}/${source}`);
-        try {
-            return await this._cacheOpen(source, files, next);
-        } catch (e) {
-            log.warn('CacheOpen Fail: %s %o %o', source, files, e);
-            throw e;
-        } finally {
-            Lock.release(`${this.config.host}/${source}`);
-        }
-    }
-
-    async _cacheOpen(source: string, files: any[], next?) {
-        const filePath = path.join(getConfig('cache_dir'), this.config.host, source);
-        await fs.ensureDir(filePath);
-        if (!files?.length) throw new FormatError('Problem data not found.');
-        let etags: Record<string, string> = {};
-        try {
-            etags = JSON.parse(await fs.readFile(path.join(filePath, 'etags'), 'utf-8'));
-        } catch (e) { /* ignore */ }
-        const version = {};
-        const filenames = [];
-        const allFiles = new Set<string>();
-        for (const file of files) {
-            allFiles.add(file.name);
-            version[file.name] = file.etag + file.lastModified;
-            if (etags[file.name] !== file.etag + file.lastModified) filenames.push(file.name);
-        }
-        for (const name in etags) {
-            if (!allFiles.has(name) && fs.existsSync(path.join(filePath, name))) await fs.remove(path.join(filePath, name));
-        }
-        if (filenames.length) {
-            log.info(`Getting problem data: ${this.config.host}/${source}`);
-            next?.({ message: 'Syncing testdata, please wait...' });
-            await this.fetchFile(source, Object.fromEntries(files.map((i) => [i.name, path.join(filePath, i.name)])));
-            await fs.writeFile(path.join(filePath, 'etags'), JSON.stringify(version));
-        }
-        await fs.writeFile(path.join(filePath, 'lastUsage'), new Date().getTime().toString());
-        return filePath;
-    }
-
     async fetchFile<T extends string | null>(namespace: T, files: Record<string, string>): Promise<T extends null ? string : null> {
         if (!namespace) { // record-related resource (code)
             const name = Object.keys(files)[0].split('#')[0];

packages/hydrojudge/src/interface.ts

@@ -8,6 +8,7 @@ export interface Execute {
     execute: string;
     clean: () => Promise<any>;
     copyIn: Record<string, CopyInFile>;
+    _cacheable?: string;
 }
 
 export type NextFunction = (body: Partial<JudgeResultBody>) => Promise<void> | void;

packages/hydrojudge/src/sandbox.ts

@@ -186,8 +186,8 @@ export async function del(fileId: string) {
     await client.deleteFile(fileId);
 }
 
-export async function get(fileId: string) {
+export async function get(fileId: string, dest?: string) {
-    return await client.getFile(fileId);
+    return await client.getFile(fileId, dest);
 }
 
 const queue = new PQueue({ concurrency: getConfig('concurrency') || getConfig('parallelism') });

packages/hydrojudge/src/task.ts

@@ -13,7 +13,9 @@ import {
 } from './interface';
 import judge from './judge';
 import { Logger } from './log';
-import { CopyIn, CopyInFile, runQueued } from './sandbox';
+import {
+    CopyIn, CopyInFile, get, PreparedFile, runQueued,
+} from './sandbox';
 import { compilerText, Lock, md5 } from './utils';
 
 const logger = new Logger('judge');
@@ -39,6 +41,7 @@ export class JudgeTask {
     end: (data: Partial<JudgeResultBody>) => void;
     env: Record<string, string>;
     callbackCache?: TestCase[];
+    compileCache: Record<string, Pick<Execute, 'execute' | 'copyIn'>> = {};
 
     constructor(public session: Session, public request: JudgeRequest) {
         this.stat.receive = new Date();
@@ -98,33 +101,43 @@ export class JudgeTask {
     async cacheOpen(source: string, files: FileInfo[]) {
         // Backward compatibility for vj4
         if ((this.session as any).cacheOpen) return (this.session as any).cacheOpen(source, files);
-        const lockKey = `${this.session.config?.host || 'local'}/${source}`;
+        const host = this.session.config?.host || 'local';
-        await Lock.acquire(lockKey);
+        const filePath = join(getConfig('cache_dir'), host, source);
-        const filePath = join(getConfig('cache_dir'), lockKey);
+        await Lock.acquire(filePath);
-        await fs.ensureDir(filePath);
-        if (!files?.length) throw new FormatError('Problem data not found.');
-        let etags: Record<string, string> = {};
         try {
-            etags = JSON.parse(await fs.readFile(join(filePath, 'etags'), 'utf-8'));
+            await fs.ensureDir(filePath);
-        } catch (e) { /* ignore */ }
+            if (!files?.length) throw new FormatError('Problem data not found.');
-        const version = {};
+            let etags: Record<string, string> = {};
-        const filenames = [];
+            try {
-        const allFiles = new Set<string>();
+                etags = JSON.parse(await fs.readFile(join(filePath, 'etags'), 'utf-8'));
-        for (const file of files) {
+            } catch (e) { /* ignore */ }
-            allFiles.add(file.name);
+            this.compileCache = etags['*cache'] as any || {};
-            version[file.name] = file.etag + file.lastModified;
+            delete etags['*cache'];
-            if (etags[file.name] !== file.etag + file.lastModified) filenames.push(file.name);
+            const version = {};
+            const filenames = [];
+            const allFiles = new Set<string>();
+            for (const file of files) {
+                allFiles.add(file.name);
+                version[file.name] = file.etag + file.lastModified;
+                if (etags[file.name] !== file.etag + file.lastModified) filenames.push(file.name);
+            }
+            const allFilesToRemove = Object.keys(etags).filter((name) => !allFiles.has(name) && fs.existsSync(join(filePath, name)));
+            await Promise.all(allFilesToRemove.map((name) => fs.remove(join(filePath, name))));
+            if (filenames.length) {
+                logger.info(`Getting problem data: ${this.session?.config.host || 'local'}/${source}`);
+                this.next({ message: 'Syncing testdata, please wait...' });
+                await this.session.fetchFile(source, Object.fromEntries(files.map((i) => [i.name, join(filePath, i.name)])));
+                await fs.writeFile(join(filePath, 'etags'), JSON.stringify(version));
+                this.compileCache = {};
+            }
+            await fs.writeFile(join(filePath, 'lastUsage'), Date.now().toString());
+            return filePath;
+        } catch (e) {
+            logger.warn('CacheOpen Fail: %s %o %o', source, files, e);
+            throw e;
+        } finally {
+            Lock.release(filePath);
         }
-        const allFilesToRemove = Object.keys(etags).filter((name) => !allFiles.has(name) && fs.existsSync(join(filePath, name)));
-        await Promise.all(allFilesToRemove.map((name) => fs.remove(join(filePath, name))));
-        if (filenames.length) {
-            logger.info(`Getting problem data: ${this.session?.config.host || 'local'}/${source}`);
-            this.next({ message: 'Syncing testdata, please wait...' });
-            await this.session.fetchFile(source, Object.fromEntries(files.map((i) => [i.name, join(filePath, i.name)])));
-            await fs.writeFile(join(filePath, 'etags'), JSON.stringify(version));
-        }
-        await fs.writeFile(join(filePath, 'lastUsage'), new Date().getTime().toString());
-        return filePath;
     }
 
     async doSubmission() {
@@ -172,11 +185,16 @@ export class JudgeTask {
 
     async compileLocalFile(
         type: 'interactor' | 'validator' | 'checker' | 'generator' | 'manager' | 'std',
-        source: CompilableSource,
+        source: CompilableSource, checkerType?: string,
-        checkerType?: string,
+    ): Promise<Execute> {
-    ) {
         if (type === 'checker' && ['default', 'strict'].includes(checkerType)) return { execute: '', copyIn: {}, clean: () => Promise.resolve(null) };
         if (type === 'checker' && !checkers[checkerType]) throw new FormatError('Unknown checker type {0}.', [checkerType]);
+        if (this.compileCache?.[type]) {
+            return {
+                ...this.compileCache[type],
+                clean: () => Promise.resolve(null),
+            };
+        }
         const withTestlib = type !== 'std' && (type !== 'checker' || checkerType === 'testlib');
         const extra = type === 'std' ? this.config.user_extra_files : this.config.judge_extra_files;
         const copyIn = {
@@ -199,8 +217,22 @@ export class JudgeTask {
         } else lang = this.session.getLang(langId);
         if (!lang) throw new FormatError(`Unknown ${type} language.`);
         // TODO cache compiled binary
-        const result = await compile(lang, { src: file }, copyIn, this.next);
+        const result = await compile(lang, { src: file }, copyIn);
         this.clean.push(result.clean);
+        if (!result._cacheable) return result;
+        await Lock.acquire(this.folder);
+        try {
+            const loc = join(this.folder, `_${type}.cache`);
+            this.compileCache[type] = {
+                execute: result.execute,
+                copyIn: { ...result.copyIn, [result._cacheable]: { src: loc } },
+            };
+            await get((result.copyIn[result._cacheable] as PreparedFile).fileId, loc);
+            const currEtag = await fs.readFile(join(this.folder, 'etags'), 'utf-8');
+            await fs.writeFile(join(this.folder, 'etags'), JSON.stringify({ ...JSON.parse(currEtag), '*cache': this.compileCache }));
+        } finally {
+            Lock.release(this.folder);
+        }
         return result;
     }