core, ui, judge: rewrite http & ws layer (#352)

packages/hydrooj/src/service/layers/base.ts

@@ -0,0 +1,88 @@
+import { PassThrough } from 'stream';
+import { cloneDeep } from 'lodash';
+import * as system from 'hydrooj/src/model/system';
+import token from 'hydrooj/src/model/token';
+import type { HydroRequest, HydroResponse } from 'hydrooj/src/service/server';
+
+export interface UiContextBase {
+    cdn_prefix: string;
+    url_prefix: string;
+}
+export const UiContextBase: UiContextBase = {
+    cdn_prefix: '/',
+    url_prefix: '/',
+};
+
+export default async (ctx, next) => {
+    // Base Layer
+    const { domainId, domainInfo } = ctx;
+    const isWebsocket = ctx.request.headers.upgrade === 'websocket';
+    const [xff, xhost] = system.getMany(['server.xff', 'server.xhost']);
+    const ip = ctx.request.headers[xff?.toLowerCase() || ''] as string || ctx.request.ip;
+    const host = ctx.request.headers[xhost?.toLowerCase() || ''] as string || ctx.request.host;
+    const request: HydroRequest = {
+        method: ctx.request.method.toLowerCase(),
+        host,
+        hostname: ctx.request.hostname,
+        ip,
+        headers: ctx.request.headers,
+        cookies: ctx.cookies,
+        body: ctx.request.body,
+        files: ctx.request.files as any,
+        query: ctx.query,
+        path: ctx.path,
+        params: ctx.params,
+        referer: ctx.request.headers.referer || '',
+        json: (ctx.request.headers.accept || '').includes('application/json'),
+        websocket: isWebsocket,
+    };
+    const response: HydroResponse = {
+        body: {},
+        type: '',
+        status: null,
+        template: null,
+        redirect: null,
+        attachment: (name, streamOrBuffer) => {
+            ctx.attachment(name);
+            if (streamOrBuffer instanceof Buffer) {
+                response.body = null;
+                ctx.body = streamOrBuffer;
+            } else {
+                response.body = null;
+                ctx.body = streamOrBuffer.pipe(new PassThrough());
+            }
+        },
+        addHeader: (name: string, value: string) => ctx.set(name, value),
+        disposition: null,
+    };
+    const args = {
+        domainId, ...ctx.params, ...ctx.query, ...ctx.request.body, __start: Date.now(),
+    };
+    const UiContext: any = cloneDeep(UiContextBase);
+    if (!process.env.DEV) UiContext.cdn_prefix = system.get('server.cdn');
+    UiContext.domainId = domainId;
+    UiContext.domain = domainInfo;
+    ctx.HydroContext = {
+        request, response, UiContext, domain: domainInfo, user: null, args,
+    };
+    await next();
+    const ua = request.headers['user-agent'] || '';
+    if (!ctx.session.uid && system.get('server.ignoreUA').replace(/\r/g, '').split('\n').filter((i) => i && ua.includes(i)).length) return;
+    const expireSeconds = ctx.session.save
+        ? system.get('session.saved_expire_seconds')
+        : system.get('session.unsaved_expire_seconds');
+    Object.assign(ctx.session, { updateIp: ip, updateUa: ua });
+    if (ctx.session._id) {
+        await token.update(ctx.session._id, token.TYPE_SESSION, expireSeconds, ctx.session);
+    } else {
+        Object.assign(ctx.session, { createIp: ip, createUa: ua, createHost: host });
+        [ctx.session._id] = await token.add(token.TYPE_SESSION, expireSeconds, ctx.session);
+    }
+    if (!request.websocket) {
+        ctx.cookies.set('sid', ctx.session._id, {
+            expires: new Date(Date.now() + expireSeconds * 1000),
+            secure: !!system.get('session.secure'),
+            httpOnly: false,
+        });
+    }
+};

packages/hydrooj/src/service/layers/domain.ts

@@ -0,0 +1,32 @@
+import BlackListModel from 'hydrooj/src/model/blacklist';
+import DomainModel from 'hydrooj/src/model/domain';
+import * as system from 'hydrooj/src/model/system';
+import token from 'hydrooj/src/model/token';
+
+export default async (ctx, next) => {
+    const forceDomain = /^\/d\/([^/]+)\//.exec(ctx.request.path);
+    ctx.path = ctx.request.path = ctx.request.path.replace(/^\/d\/[^/]+\//, '/');
+    const [xff, xhost] = system.getMany(['server.xff', 'server.xhost']);
+    const ip = ctx.request.headers[xff?.toLowerCase() || ''] as string || ctx.request.ip;
+    const host = ctx.request.headers[xhost?.toLowerCase() || ''] as string || ctx.request.host;
+    const header = ctx.request.headers.authorization;
+    const sid = header
+        ? header.split(' ')[1] // Accept bearer token
+        : ctx.cookies.get('sid') || ctx.query.sid; // FIXME maybe a better way for shared conn?
+    let domainId = forceDomain?.[1] || 'system';
+    const [absoluteDomain, inferDomain, bdoc, session] = await Promise.all([
+        DomainModel.get(domainId),
+        forceDomain ? Promise.resolve() : DomainModel.getByHost(host),
+        BlackListModel.get(`ip::${ip}`),
+        token.get(sid, token.TYPE_SESSION),
+    ]);
+    if (bdoc) {
+        ctx.body = 'blacklisted'; // Just return 404 if blacklisted
+        return;
+    }
+    if (inferDomain) domainId = inferDomain._id;
+    ctx.domainId = domainId;
+    ctx.domainInfo = inferDomain || absoluteDomain;
+    ctx.session = session || { uid: 0 };
+    await next();
+};

packages/hydrooj/src/service/layers/renderer.ts

@@ -0,0 +1,72 @@
+import { ObjectID } from 'mongodb';
+import avatar from 'hydrooj/src/lib/avatar';
+import * as system from 'hydrooj/src/model/system';
+import type { KoaContext } from '../server';
+
+function serializer(k: string, v: any) {
+    if (k.startsWith('_') && k !== '_id') return undefined;
+    if (typeof v === 'bigint') return `BigInt::${v.toString()}`;
+    return v;
+}
+
+export default (router, logger) => async (ctx: KoaContext, next) => {
+    const { request, response } = ctx.HydroContext;
+    ctx.renderHTML = (templateName, args) => {
+        const UserContext: any = {
+            ...(ctx.HydroContext.user || {}),
+            avatar: avatar(ctx.HydroContext.user?.avatar || '', 128),
+            viewLang: ctx.translate('__id'),
+        };
+        const engine = global.Hydro.lib.template?.render || (() => JSON.stringify(args, serializer));
+        return engine(templateName, {
+            handler: ctx.handler,
+            UserContext,
+            url: ctx.getUrl,
+            _: ctx.translate,
+            ...args,
+        });
+    };
+    ctx.render = async (name, args) => {
+        response.body = await ctx.renderHTML(name, args);
+        response.type = 'text/html';
+    };
+    ctx.getUrl = (name: string, kwargs: any = {}) => {
+        let res = '#';
+        const args: any = {};
+        const query: any = {};
+        for (const key in kwargs) {
+            if (kwargs[key] instanceof ObjectID) args[key] = kwargs[key].toHexString();
+            else args[key] = kwargs[key].toString().replace(/\//g, '%2F');
+        }
+        for (const key in kwargs.query || {}) {
+            if (query[key] instanceof ObjectID) query[key] = kwargs.query[key].toHexString();
+            else query[key] = encodeURIComponent(kwargs.query[key].toString());
+        }
+        try {
+            const { anchor } = args;
+            let withDomainId = args.domainId || false;
+            const domainId = ctx.HydroContext.args.domainId;
+            const host = ctx.HydroContext.domain?.host;
+            if (domainId !== 'system' && (
+                !request.host
+                || (host instanceof Array
+                    ? (!host.includes(request.host))
+                    : request.host !== host)
+            )) withDomainId = domainId;
+            res = router.url(name, args, { query }).toString();
+            if (anchor) res = `${res}#${anchor}`;
+            if (withDomainId) res = `/d/${withDomainId}${res}`;
+        } catch (e) {
+            logger.warn(e.message);
+            logger.info('%s %o', name, args);
+            if (!e.message.includes('Expected') || !e.message.includes('to match')) logger.info('%s', e.stack);
+        }
+        return res;
+    };
+    ctx.translate = (str: string) => {
+        if (!str) return '';
+        const lang = ctx.HydroContext.user?.viewLang || ctx.session?.viewLang;
+        return str.toString().translate(lang, ...ctx.acceptsLanguages(), system.get('server.language'));
+    };
+    await next();
+};

packages/hydrooj/src/service/layers/response.ts

@@ -0,0 +1,76 @@
+import { errorMessage } from '@hydrooj/utils/lib/utils';
+import { SystemError, UserFacingError } from 'hydrooj/src/error';
+import type { KoaContext } from '../server';
+
+function serializer(k: string, v: any) {
+    if (k.startsWith('_') && k !== '_id') return undefined;
+    if (typeof v === 'bigint') return `BigInt::${v.toString()}`;
+    return v;
+}
+
+export default (logger) => async (ctx: KoaContext, next) => {
+    const {
+        request, response, UiContext, user, args,
+    } = ctx.HydroContext;
+    try {
+        await next();
+        if (response.redirect) {
+            response.body ||= {};
+            response.body.url = response.redirect;
+        }
+        if (!response.type) {
+            if (
+                request.json || response.redirect
+                || request.query.noTemplate || !response.template) {
+                // Send raw data
+                try {
+                    if (typeof response.body === 'object') {
+                        response.body.UiContext = UiContext;
+                        response.body.UserContext = user;
+                    }
+                    response.body = JSON.stringify(response.body, serializer);
+                } catch (e) {
+                    response.body = new SystemError('Serialize failure', e.message);
+                }
+                response.type = 'application/json';
+            } else if (response.template) {
+                const s = response.template.split('.');
+                let templateName = `${s[0]}.${args.domainId}.${s[1]}`;
+                if (!global.Hydro.ui.template[templateName]) templateName = response.template;
+                await ctx.render(templateName, response.body || {});
+            }
+        }
+        if (response.disposition) ctx.set('Content-Disposition', response.disposition);
+        if (response.etag) {
+            ctx.set('ETag', response.etag);
+            ctx.set('Cache-Control', 'public');
+        }
+    } catch (err) {
+        const error = errorMessage(err);
+        response.status = error instanceof UserFacingError ? error.code : 500;
+        if (request.json) response.body = { error };
+        else {
+            try {
+                await ctx.render(error instanceof UserFacingError ? 'error.html' : 'bsod.html', { error });
+            } catch (e) {
+                logger.error(e);
+                // this.response.body.error = {};
+            }
+        }
+    } finally {
+        if (response.etag && request.headers['if-none-match'] === response.etag) {
+            ctx.response.status = 304;
+        } else if (response.redirect && !request.json) {
+            ctx.response.type = 'application/octet-stream';
+            ctx.response.status = 302;
+            ctx.redirect(response.redirect);
+        } else if (response.body) {
+            ctx.body = response.body;
+            ctx.response.status = response.status || 200;
+            ctx.response.type = response.type
+                || (request.json
+                    ? 'application/json'
+                    : ctx.response.type);
+        }
+    }
+};

packages/hydrooj/src/service/layers/user.ts

@@ -0,0 +1,26 @@
+import { CsrfTokenError, NotFoundError } from 'hydrooj/src/error';
+import avatar from 'hydrooj/src/lib/avatar';
+import { PERM } from 'hydrooj/src/model/builtin';
+import UserModel from 'hydrooj/src/model/user';
+import type { KoaContext } from '../server';
+
+export default async (ctx: KoaContext, next) => {
+    // User Layer
+    const { request, args, domain } = ctx.HydroContext;
+    const domainId = domain ? args.domainId : 'system';
+    let user = await UserModel.getById(domainId, ctx.session.uid, ctx.session.scope);
+    if (!user) {
+        ctx.session.uid = 0;
+        ctx.session.scope = PERM.PERM_ALL.toString();
+        user = await UserModel.getById(domainId, ctx.session.uid, ctx.session.scope);
+    }
+    if (user._id === 0) delete user.viewLang;
+    user.avatarUrl = avatar(user.avatar, 128);
+    ctx.HydroContext.user = user;
+    if (!domain) throw new NotFoundError(args.domainId);
+    if (request.method === 'post' && request.headers.referer) {
+        const host = new URL(request.headers.referer).host;
+        if (host !== request.host) throw new CsrfTokenError(host);
+    }
+    await next();
+};