diff --git a/packages/prom-client/index.ts b/packages/prom-client/index.ts index 60e7cfd2b..0eee59696 100644 --- a/packages/prom-client/index.ts +++ b/packages/prom-client/index.ts @@ -46,15 +46,29 @@ export function apply(ctx: Context) { if (process.env.HYDRO_CLI) return; const registry = createRegistry(ctx); ctx.on('metrics', (id, metrics) => { instances[id] = metrics; }); + let pushError = ''; ctx.setInterval(async () => { - const [gateway, name, pass] = SystemModel.getMany(['prom-client.gateway', 'prom-client.name', 'prom-client.password']); - if (gateway) { - const prefix = gateway.endsWith('/') ? gateway : `${gateway}/`; - const endpoint = `${prefix}metrics/job/hydro-web/instance/${encodeURIComponent(hostname())}:${process.env.NODE_APP_INSTANCE}`; - let req = superagent.post(endpoint); - if (name) req = req.set('Authorization', `Basic ${Buffer.from(`${name}:${pass}`).toString('base64')}`); - await req.send(await registry.metrics()); - } else ctx.broadcast('metrics', `${hostname()}/${process.env.NODE_APP_INSTANCE}`, await registry.getMetricsAsJSON()); + try { + const [gateway, name, pass] = SystemModel.getMany(['prom-client.gateway', 'prom-client.name', 'prom-client.password']); + if (gateway) { + const prefix = gateway.endsWith('/') ? gateway : `${gateway}/`; + const endpoint = `${prefix}metrics/job/hydro-web/instance/${encodeURIComponent(hostname())}:${process.env.NODE_APP_INSTANCE}`; + let req = superagent.post(endpoint); + if (name) req = req.auth(name, pass, { type: 'basic' }); + await req.send(await registry.metrics()); + } else ctx.broadcast('metrics', `${hostname()}/${process.env.NODE_APP_INSTANCE}`, await registry.getMetricsAsJSON()); + pushError = ''; + } catch (e) { + pushError = e.message; + } }, 5000 * (+SystemModel.get('prom-client.collect_rate') || 1)); + ctx.inject(['check'], (c) => { + const gateway = SystemModel.get('prom-client.gateway'); + c.check.addChecker('prom-client', async (_, log, warn, error) => { + if (pushError) error(`Prometheus push error: ${pushError}`); + else if (gateway) log(`Prometheus pushed to gateway: ${gateway}`); + else log('Prometheus metrics server running.'); + }); + }); ctx.Route('metrics', '/metrics', MetricsHandler); }