core: config: refactor (allow schemastery)

packages/hydrooj/src/service/db.ts

@@ -4,6 +4,7 @@ import {
 } from 'mongodb';
 import mongoUri from 'mongodb-uri';
 import { Time } from '@hydrooj/utils';
+import { Context, Service } from '../context';
 import { ValidationError } from '../error';
 import { Logger } from '../logger';
 import { load } from '../options';
@@ -25,12 +26,28 @@ interface MongoConfig {
     collectionMap?: Record<string, string>,
 }
 
-class MongoService {
+declare module '../context' {
+    interface Context {
+        db: MongoService;
+    }
+}
+
+export class MongoService extends Service {
     public client: MongoClient;
     public db: Db;
-    private opts: MongoConfig;
 
-    static buildUrl(opts: MongoConfig) {
+    constructor(ctx: Context, private config: MongoConfig = {}) {
+        super(ctx, 'db');
+    }
+
+    static async getUrl() {
+        if (process.env.CI) {
+            const { MongoMemoryServer } = require('mongodb-memory-server');
+            const mongod = await MongoMemoryServer.create();
+            return mongod.getUri();
+        }
+        const opts = load();
+        if (!opts) return null;
         let mongourl = `${opts.protocol || 'mongodb'}://`;
         if (opts.username) mongourl += `${opts.username}:${encodeURIComponent(opts.password)}@`;
         mongourl += `${opts.host}:${opts.port}/${opts.name}`;
@@ -38,25 +55,18 @@ class MongoService {
         return mongourl;
     }
 
-    async start() {
-        const opts = load() || {};
-        let mongourl = MongoService.buildUrl(opts);
+    async [Service.setup]() {
+        const mongourl = await MongoService.getUrl();
         const url = mongoUri.parse(mongourl);
-        if (process.env.CI) {
-            const { MongoMemoryServer } = require('mongodb-memory-server');
-            const mongod = await MongoMemoryServer.create();
-            mongourl = mongod.getUri();
-        }
-        this.opts = opts;
         this.client = await MongoClient.connect(mongourl);
         this.db = this.client.db(url.database || 'hydro');
         await bus.parallel('database/connect', this.db);
-        setInterval(() => this.fixExpireAfter(), Time.hour);
+        this.ctx.setInterval(() => this.fixExpireAfter(), Time.hour);
     }
 
     public collection<K extends keyof Collections>(c: K) {
-        let coll = this.opts.prefix ? `${this.opts.prefix}.${c}` : c;
-        if (this.opts.collectionMap?.[coll]) coll = this.opts.collectionMap[coll];
+        let coll = this.config.prefix ? `${this.config.prefix}.${c}` : c;
+        if (this.config.collectionMap?.[coll]) coll = this.config.collectionMap[coll];
         return this.db.collection<Collections[K]>(coll);
     }
 
@@ -173,7 +183,11 @@ class MongoService {
     }
 }
 
-const service = new MongoService();
-global.Hydro.service.db = service;
-export default service;
-export const collection = service.collection.bind(service);
+/** @deprecated use ctx.db instead */
+const db = new Proxy({} as MongoService, {
+    get(target, prop) {
+        return app.get('db')?.[prop];
+    },
+});
+
+export default db;

packages/hydrooj/src/service/server.ts

@@ -1,7 +1,6 @@
 import { resolve } from 'path';
 import cac from 'cac';
 import fs from 'fs-extra';
-import proxy from 'koa-proxies';
 import cache from 'koa-static-cache';
 import { type FindCursor, ObjectId } from 'mongodb';
 import {
@@ -18,7 +17,6 @@ import { PERM, PRIV } from '../model/builtin';
 import * as opcount from '../model/opcount';
 import * as OplogModel from '../model/oplog';
 import * as system from '../model/system';
-import { builtinConfig } from '../settings';
 import db from './db';
 import baseLayer from './layers/base';
 import domainLayer from './layers/domain';
@@ -93,26 +91,6 @@ export async function apply(ctx: Context) {
     });
     if (process.env.HYDRO_CLI) return;
     ctx.inject(['server'], ({ server, on }) => {
-        let endpoint = builtinConfig.file.endPoint;
-        if (builtinConfig.file.type === 's3' && !builtinConfig.file.pathStyle) {
-            try {
-                const parsed = new URL(builtinConfig.file.endPoint);
-                parsed.hostname = `${builtinConfig.file.bucket}.${parsed.hostname}`;
-                endpoint = parsed.toString();
-            } catch (e) {
-                logger.warn('Failed to parse file endpoint');
-            }
-        }
-        const proxyMiddleware = proxy('/fs', {
-            target: endpoint,
-            changeOrigin: true,
-            rewrite: (p) => p.replace('/fs', ''),
-        });
-        server.addCaptureRoute('/fs/', async (c, next) => {
-            if (c.request.search.toLowerCase().includes('x-amz-credential')) return await proxyMiddleware(c, next);
-            c.request.path = c.path = c.path.split('/fs')[1];
-            return await next();
-        });
         server.addHandlerLayer('init', async (c, next) => {
             const init = Date.now();
             try {

packages/hydrooj/src/service/storage.ts

@@ -12,9 +12,12 @@ import {
     copyFile, createReadStream, createWriteStream, ensureDir,
     existsSync, remove, stat, writeFile,
 } from 'fs-extra';
+import proxy from 'koa-proxies';
 import { lookup } from 'mime-types';
+import { nanoid } from 'nanoid';
+import Schema from 'schemastery';
+import { Context } from '../context';
 import { Logger } from '../logger';
-import { builtinConfig } from '../settings';
 import { MaybeArray } from '../typeutils';
 import { md5, streamToBuffer } from '../utils';
 
@@ -57,6 +60,43 @@ const convertPath = (p: string) => {
     return p;
 };
 
+const defaultPath = process.env.CI ? '/tmp/file'
+    : process.env.DEFAULT_STORE_PATH || '/data/file/hydro';
+const FileSetting = Schema.intersect([
+    Schema.object({
+        type: Schema.union([
+            Schema.const('file').description('local file provider'),
+            Schema.const('s3').description('s3 provider'),
+        ] as const).description('provider type'),
+        endPointForUser: Schema.string().default('/fs/'),
+        endPointForJudge: Schema.string().default('/fs/'),
+    }).description('setting_file'),
+    Schema.union([
+        Schema.object({
+            type: Schema.const('file').required(),
+            path: Schema.string().default(defaultPath).description('Storage path'),
+            secret: Schema.string().description('Download file sign secret').default(nanoid()),
+        }),
+        Schema.object({
+            type: Schema.const('s3').required(),
+            endPoint: Schema.string(),
+            accessKey: Schema.string().description('access key'),
+            secretKey: Schema.string().description('secret key').role('secret'),
+            bucket: Schema.string().default('hydro'),
+            region: Schema.string().default('us-east-1'),
+            pathStyle: Schema.boolean().default(true),
+        }),
+    ] as const),
+] as const).default({
+    type: 'file',
+    path: defaultPath,
+    endPointForUser: '/fs/',
+    endPointForJudge: '/fs/',
+    secret: nanoid(),
+});
+
+export const Config = FileSetting;
+
 class RemoteStorageService {
     public client: S3Client;
     public error = '';
@@ -67,9 +107,12 @@ class RemoteStorageService {
         judge: null,
     };
 
+    constructor(private config: ReturnType<typeof FileSetting>) {
+    }
+
     async start() {
         try {
-            logger.info('Starting storage service with endpoint:', builtinConfig.file.endPoint);
+            logger.info('Starting storage service with endpoint:', this.config.endPoint);
             const {
                 endPoint,
                 accessKey,
@@ -79,7 +122,7 @@ class RemoteStorageService {
                 pathStyle,
                 endPointForUser,
                 endPointForJudge,
-            } = builtinConfig.file;
+            } = this.config;
             this.bucket = bucket;
             const base = {
                 region,
@@ -261,13 +304,16 @@ class LocalStorageService {
     opts: null;
     private replaceWithAlternativeUrlFor: Record<'user' | 'judge', (originalUrl: string) => string>;
 
+    constructor(private config: ReturnType<typeof FileSetting>) {
+    }
+
     async start() {
-        logger.debug('Loading local storage service with path:', builtinConfig.file.path);
-        await ensureDir(builtinConfig.file.path);
-        this.dir = builtinConfig.file.path;
+        logger.debug('Loading local storage service with path:', this.config.path);
+        await ensureDir(this.config.path);
+        this.dir = this.config.path;
         this.replaceWithAlternativeUrlFor = {
-            user: parseAlternativeEndpointUrl(builtinConfig.file.endPointForUser),
-            judge: parseAlternativeEndpointUrl(builtinConfig.file.endPointForJudge),
+            user: parseAlternativeEndpointUrl(this.config.endPointForUser),
+            judge: parseAlternativeEndpointUrl(this.config.endPointForJudge),
         };
     }
 
@@ -314,7 +360,7 @@ class LocalStorageService {
         if (filename) url.searchParams.set('filename', filename);
         const expire = (Date.now() + (noExpire ? 7 * 24 * 3600 : 600) * 1000).toString();
         url.searchParams.set('expire', expire);
-        url.searchParams.set('secret', md5(`${target}/${expire}/${builtinConfig.file.secret}`));
+        url.searchParams.set('secret', md5(`${target}/${expire}/${this.config.secret}`));
         if (useAlternativeEndpointFor) return this.replaceWithAlternativeUrlFor[useAlternativeEndpointFor](url.toString());
         return `/${url.toString().split('localhost/')[1]}`;
     }
@@ -333,16 +379,47 @@ class LocalStorageService {
     }
 }
 
-let service; // eslint-disable-line import/no-mutable-exports
+let service;
 
-export async function loadStorageService() {
-    service = builtinConfig.file.type === 's3' ? new RemoteStorageService() : new LocalStorageService();
-    global.Hydro.service.storage = service;
+export async function apply(ctx: Context, config: ReturnType<typeof FileSetting>) {
+    if (config.type === 's3') {
+        service = new RemoteStorageService(config);
+    } else {
+        service = new LocalStorageService(config);
+    }
     await service.start();
+    ctx.inject(['server'], ({ server }) => {
+        let endpoint = config.endPoint;
+        if (config.type === 's3' && !config.pathStyle) {
+            try {
+                const parsed = new URL(config.endPoint);
+                parsed.hostname = `${config.bucket}.${parsed.hostname}`;
+                endpoint = parsed.toString();
+            } catch (e) {
+                logger.warn('Failed to parse file endpoint');
+            }
+        }
+        const proxyMiddleware = proxy('/fs', {
+            target: endpoint,
+            changeOrigin: true,
+            rewrite: (p) => p.replace('/fs', ''),
+        });
+        server.addCaptureRoute('/fs/', async (c, next) => {
+            if (c.request.search.toLowerCase().includes('x-amz-credential')) {
+                c.nolog = true;
+                return await proxyMiddleware(c, next);
+            }
+            c.request.path = c.path = c.path.split('/fs')[1];
+            return await next();
+        });
+    });
+    ctx.set('storage', service);
 }
 
-export default new Proxy({}, {
+/** @deprecated use ctx.storage instead */
+const serviceProxy = new Proxy({}, {
     get(self, key) {
         return service[key];
     },
 }) as RemoteStorageService | LocalStorageService;
+export default serviceProxy;