Weishen/Hydro
2
0 Fork 0
mirror of https://gitcode.com/gh_mirrors/hy/Hydro.git synced 2026-10-17 09:36:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

cache compiled binaries

Browse Source
This commit is contained in:
undefined committed 2025-02-09 15:17:10 +00:00
1 parent 5c9cecc197
commit be82131210
5 files changed
+70 -76
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/hydrojudge/src/compile.ts
+5 -3
View File
@@ -11,17 +11,18 @@ export default async function compile(
): Promise<Execute> {
const target = lang.target || 'foo';
const execute = copyIn['execute.sh'] ? '/bin/bash execute.sh' : lang.execute;
if (lang.compile) {
const command = copyIn['compile.sh'] ? '/bin/bash compile.sh' : lang.compile;
if (command) {
const {
status, stdout, stderr, fileIds,
} = await runQueued(
copyIn['compile.sh'] ? '/bin/bash compile.sh' : lang.compile,
command,
{
copyIn: { ...copyIn, [lang.code_file]: code },
copyOutCached: [target],
env: { HYDRO_LANG: lang.key },
time: lang.compile_time_limit || 10000,
memory: lang.compile_memory_limit || 256,
memory: lang.compile_memory_limit || 512,
},
`compile[${lang.key}]`,
3,
@@ -36,6 +37,7 @@ export default async function compile(
execute,
copyIn: { ...copyIn, [target]: { fileId: fileIds[target] } },
clean: () => del(fileIds[target]),
_cacheable: target,
};
}
return {
packages/hydrojudge/src/hosts/hydro.ts
-41
View File
@@ -51,47 +51,6 @@ export default class Hydro implements Session {
setInterval(() => { this.get(''); }, 30000000); // Cookie refresh only
}
async cacheOpen(source: string, files: any[], next?) {
await Lock.acquire(`${this.config.host}/${source}`);
try {
return await this._cacheOpen(source, files, next);
} catch (e) {
log.warn('CacheOpen Fail: %s %o %o', source, files, e);
throw e;
} finally {
Lock.release(`${this.config.host}/${source}`);
}
}
async _cacheOpen(source: string, files: any[], next?) {
const filePath = path.join(getConfig('cache_dir'), this.config.host, source);
await fs.ensureDir(filePath);
if (!files?.length) throw new FormatError('Problem data not found.');
let etags: Record<string, string> = {};
try {
etags = JSON.parse(await fs.readFile(path.join(filePath, 'etags'), 'utf-8'));
} catch (e) { /* ignore */ }
const version = {};
const filenames = [];
const allFiles = new Set<string>();
for (const file of files) {
allFiles.add(file.name);
version[file.name] = file.etag + file.lastModified;
if (etags[file.name] !== file.etag + file.lastModified) filenames.push(file.name);
}
for (const name in etags) {
if (!allFiles.has(name) && fs.existsSync(path.join(filePath, name))) await fs.remove(path.join(filePath, name));
}
if (filenames.length) {
log.info(`Getting problem data: ${this.config.host}/${source}`);
next?.({ message: 'Syncing testdata, please wait...' });
await this.fetchFile(source, Object.fromEntries(files.map((i) => [i.name, path.join(filePath, i.name)])));
await fs.writeFile(path.join(filePath, 'etags'), JSON.stringify(version));
}
await fs.writeFile(path.join(filePath, 'lastUsage'), new Date().getTime().toString());
return filePath;
}
async fetchFile<T extends string | null>(namespace: T, files: Record<string, string>): Promise<T extends null ? string : null> {
if (!namespace) { // record-related resource (code)
const name = Object.keys(files)[0].split('#')[0];
packages/hydrojudge/src/interface.ts
+1
View File
@@ -8,6 +8,7 @@ export interface Execute {
execute: string;
clean: () => Promise<any>;
copyIn: Record<string, CopyInFile>;
_cacheable?: string;
}
export type NextFunction = (body: Partial<JudgeResultBody>) => Promise<void> | void;
packages/hydrojudge/src/sandbox.ts
+2 -2
View File
@@ -186,8 +186,8 @@ export async function del(fileId: string) {
await client.deleteFile(fileId);
}
export async function get(fileId: string) {
return await client.getFile(fileId);
export async function get(fileId: string, dest?: string) {
return await client.getFile(fileId, dest);
}
const queue = new PQueue({ concurrency: getConfig('concurrency') || getConfig('parallelism') });
packages/hydrojudge/src/task.ts
+62 -30
View File
@@ -13,7 +13,9 @@ import {
} from './interface';
import judge from './judge';
import { Logger } from './log';
import { CopyIn, CopyInFile, runQueued } from './sandbox';
import {
CopyIn, CopyInFile, get, PreparedFile, runQueued,
} from './sandbox';
import { compilerText, Lock, md5 } from './utils';
const logger = new Logger('judge');
@@ -39,6 +41,7 @@ export class JudgeTask {
end: (data: Partial<JudgeResultBody>) => void;
env: Record<string, string>;
callbackCache?: TestCase[];
compileCache: Record<string, Pick<Execute, 'execute' | 'copyIn'>> = {};
constructor(public session: Session, public request: JudgeRequest) {
this.stat.receive = new Date();
@@ -98,33 +101,43 @@ export class JudgeTask {
async cacheOpen(source: string, files: FileInfo[]) {
// Backward compatibility for vj4
if ((this.session as any).cacheOpen) return (this.session as any).cacheOpen(source, files);
const lockKey = `${this.session.config?.host || 'local'}/${source}`;
await Lock.acquire(lockKey);
const filePath = join(getConfig('cache_dir'), lockKey);
await fs.ensureDir(filePath);
if (!files?.length) throw new FormatError('Problem data not found.');
let etags: Record<string, string> = {};
const host = this.session.config?.host || 'local';
const filePath = join(getConfig('cache_dir'), host, source);
await Lock.acquire(filePath);
try {
etags = JSON.parse(await fs.readFile(join(filePath, 'etags'), 'utf-8'));
} catch (e) { /* ignore */ }
const version = {};
const filenames = [];
const allFiles = new Set<string>();
for (const file of files) {
allFiles.add(file.name);
version[file.name] = file.etag + file.lastModified;
if (etags[file.name] !== file.etag + file.lastModified) filenames.push(file.name);
await fs.ensureDir(filePath);
if (!files?.length) throw new FormatError('Problem data not found.');
let etags: Record<string, string> = {};
try {
etags = JSON.parse(await fs.readFile(join(filePath, 'etags'), 'utf-8'));
} catch (e) { /* ignore */ }
this.compileCache = etags['*cache'] as any || {};
delete etags['*cache'];
const version = {};
const filenames = [];
const allFiles = new Set<string>();
for (const file of files) {
allFiles.add(file.name);
version[file.name] = file.etag + file.lastModified;
if (etags[file.name] !== file.etag + file.lastModified) filenames.push(file.name);
}
const allFilesToRemove = Object.keys(etags).filter((name) => !allFiles.has(name) && fs.existsSync(join(filePath, name)));
await Promise.all(allFilesToRemove.map((name) => fs.remove(join(filePath, name))));
if (filenames.length) {
logger.info(`Getting problem data: ${this.session?.config.host || 'local'}/${source}`);
this.next({ message: 'Syncing testdata, please wait...' });
await this.session.fetchFile(source, Object.fromEntries(files.map((i) => [i.name, join(filePath, i.name)])));
await fs.writeFile(join(filePath, 'etags'), JSON.stringify(version));
this.compileCache = {};
}
await fs.writeFile(join(filePath, 'lastUsage'), Date.now().toString());
return filePath;
} catch (e) {
logger.warn('CacheOpen Fail: %s %o %o', source, files, e);
throw e;
} finally {
Lock.release(filePath);
}
const allFilesToRemove = Object.keys(etags).filter((name) => !allFiles.has(name) && fs.existsSync(join(filePath, name)));
await Promise.all(allFilesToRemove.map((name) => fs.remove(join(filePath, name))));
if (filenames.length) {
logger.info(`Getting problem data: ${this.session?.config.host || 'local'}/${source}`);
this.next({ message: 'Syncing testdata, please wait...' });
await this.session.fetchFile(source, Object.fromEntries(files.map((i) => [i.name, join(filePath, i.name)])));
await fs.writeFile(join(filePath, 'etags'), JSON.stringify(version));
}
await fs.writeFile(join(filePath, 'lastUsage'), new Date().getTime().toString());
return filePath;
}
async doSubmission() {
@@ -172,11 +185,16 @@ export class JudgeTask {
async compileLocalFile(
type: 'interactor' | 'validator' | 'checker' | 'generator' | 'manager' | 'std',
source: CompilableSource,
checkerType?: string,
) {
source: CompilableSource, checkerType?: string,
): Promise<Execute> {
if (type === 'checker' && ['default', 'strict'].includes(checkerType)) return { execute: '', copyIn: {}, clean: () => Promise.resolve(null) };
if (type === 'checker' && !checkers[checkerType]) throw new FormatError('Unknown checker type {0}.', [checkerType]);
if (this.compileCache?.[type]) {
return {
...this.compileCache[type],
clean: () => Promise.resolve(null),
};
}
const withTestlib = type !== 'std' && (type !== 'checker' || checkerType === 'testlib');
const extra = type === 'std' ? this.config.user_extra_files : this.config.judge_extra_files;
const copyIn = {
@@ -199,8 +217,22 @@ export class JudgeTask {
} else lang = this.session.getLang(langId);
if (!lang) throw new FormatError(`Unknown ${type} language.`);
// TODO cache compiled binary
const result = await compile(lang, { src: file }, copyIn, this.next);
const result = await compile(lang, { src: file }, copyIn);
this.clean.push(result.clean);
if (!result._cacheable) return result;
await Lock.acquire(this.folder);
try {
const loc = join(this.folder, `_${type}.cache`);
this.compileCache[type] = {
execute: result.execute,
copyIn: { ...result.copyIn, [result._cacheable]: { src: loc } },
};
await get((result.copyIn[result._cacheable] as PreparedFile).fileId, loc);
const currEtag = await fs.readFile(join(this.folder, 'etags'), 'utf-8');
await fs.writeFile(join(this.folder, 'etags'), JSON.stringify({ ...JSON.parse(currEtag), '*cache': this.compileCache }));
} finally {
Lock.release(this.folder);
}
return result;
}