diff --git a/src/shadowbox/infrastructure/prometheus_scraper.ts b/src/shadowbox/infrastructure/prometheus_scraper.ts index 8ff6bf8c2..6c54a1d6b 100644 --- a/src/shadowbox/infrastructure/prometheus_scraper.ts +++ b/src/shadowbox/infrastructure/prometheus_scraper.ts @@ -69,10 +69,14 @@ export class PrometheusClient { } } -export async function runPrometheusScraper( - args: string[], configFilename: string, configJson: {}, - prometheusEndpoint: string): Promise { - mkdirp.sync(path.dirname(configFilename)); +export async function startPrometheus( + configFilename: string, configJson: {}, processArgs: string[], endpoint: string) { + await writePrometheusConfigToDisk(configFilename, configJson); + await spawnPrometheusSubprocess(processArgs, endpoint); +} + +async function writePrometheusConfigToDisk(configFilename: string, configJson: {}) { + await mkdirp.sync(path.dirname(configFilename)); const ymlTxt = jsyaml.safeDump(configJson, {'sortKeys': true}); // Write the file asynchronously to prevent blocking the node thread. await new Promise((resolve, reject) => { @@ -84,29 +88,32 @@ export async function runPrometheusScraper( } }); }); - const commandArguments = ['--config.file', configFilename]; - commandArguments.push(...args); - const runProcess = child_process.spawn('/root/shadowbox/bin/prometheus', commandArguments); +} + +async function spawnPrometheusSubprocess( + processArgs: string[], prometheusEndpoint: string): Promise { + logging.info(`Starting Prometheus with args [${processArgs}]`); + const runProcess = child_process.spawn('/root/shadowbox/bin/prometheus', processArgs); runProcess.on('error', (error) => { - logging.error(`Error spawning prometheus: ${error}`); + logging.error(`Error spawning Prometheus: ${error}`); }); - // TODO(fortuna): Add restart logic. runProcess.on('exit', (code, signal) => { - logging.info(`prometheus has exited with error. Code: ${code}, Signal: ${signal}`); + logging.error(`Prometheus has exited with error. Code: ${code}, Signal: ${signal}`); + logging.error('Restarting Prometheus...'); + spawnPrometheusSubprocess(processArgs, prometheusEndpoint); }); // TODO(fortuna): Consider saving the output and expose it through the manager service. runProcess.stdout.pipe(process.stdout); runProcess.stderr.pipe(process.stderr); await waitForPrometheusReady(`${prometheusEndpoint}/api/v1/status/flags`); + logging.info('Prometheus is ready!'); return runProcess; } async function waitForPrometheusReady(prometheusEndpoint: string) { - logging.debug('Waiting for Prometheus to be ready...'); while (!(await isHttpEndpointHealthy(prometheusEndpoint))) { await new Promise((resolve) => setTimeout(resolve, 1000)); } - logging.debug('Prometheus is ready'); } function isHttpEndpointHealthy(endpoint: string): Promise { diff --git a/src/shadowbox/server/main.ts b/src/shadowbox/server/main.ts index fa661b2c8..58cdb44fc 100644 --- a/src/shadowbox/server/main.ts +++ b/src/shadowbox/server/main.ts @@ -24,7 +24,7 @@ import {RealClock} from '../infrastructure/clock'; import {PortProvider} from '../infrastructure/get_port'; import * as json_config from '../infrastructure/json_config'; import * as logging from '../infrastructure/logging'; -import {PrometheusClient, runPrometheusScraper} from '../infrastructure/prometheus_scraper'; +import {PrometheusClient, startPrometheus} from '../infrastructure/prometheus_scraper'; import {RolloutTracker} from '../infrastructure/rollout'; import {AccessKeyId} from '../model/access_key'; @@ -146,15 +146,18 @@ async function main() { if (isReplayProtectionEnabled) { shadowsocksServer.enableReplayProtection(); } + + // Start Prometheus subprocess and wait for it to be up and running. + const prometheusConfigFilename = getPersistentFilename('prometheus/config.yml'); + const prometheusTsdbFilename = getPersistentFilename('prometheus/data'); const prometheusEndpoint = `http://${prometheusLocation}`; - // Wait for Prometheus to be up and running. - await runPrometheusScraper( - [ - '--storage.tsdb.retention', '31d', '--storage.tsdb.path', - getPersistentFilename('prometheus/data'), '--web.listen-address', prometheusLocation, - '--log.level', verbose ? 'debug' : 'info' - ], - getPersistentFilename('prometheus/config.yml'), prometheusConfigJson, prometheusEndpoint); + const prometheusArgs = [ + '--config.file', prometheusConfigFilename, '--storage.tsdb.retention.time', '31d', + '--storage.tsdb.path', prometheusTsdbFilename, '--web.listen-address', prometheusLocation, + '--log.level', verbose ? 'debug' : 'info' + ]; + await startPrometheus( + prometheusConfigFilename, prometheusConfigJson, prometheusArgs, prometheusEndpoint); const prometheusClient = new PrometheusClient(prometheusEndpoint); if (!serverConfig.data().portForNewAccessKeys) {