Geneh/queue count

.vscode/launch.json

@@ -85,7 +85,7 @@
         "NODE_ENV": "localhost",
         "CONFIGURATION_ENVIRONMENT": "localhost"
       },
-      "console": "internalConsole",
+      "console": "integratedTerminal",
       "sourceMaps": false,
       "outFiles": []
     },
@@ -103,7 +103,7 @@
         "CRAWLER_SERVICE_URL": "http://localhost:3000",
         "DEBUG_ALLOW_HTTP": "1"
       },
-      "console": "internalConsole"
+      "console": "integratedTerminal"
     }
   ]
 }

dashboard/business/queueInfoPoller.js

@@ -16,7 +16,7 @@ class QueueInfoPoller {
     this.deadletterQueueName = 'deadletter';
   }
 
-  initialize(config) {
+  initialize(config, insights) {
     this.crawlerClient = new CrawlerClient(config.dashboard.crawler.url, config.dashboard.crawler.apiToken);
     this.queueNames = config.dashboard.queuing.queueNames;
     this.queueNames.forEach(queueName => {
@@ -25,6 +25,7 @@ class QueueInfoPoller {
     this.stats.data[this.deadletterQueueName] = [];
     this.pollingFrequencySec = config.dashboard.queuing.pollingFrequencySec;
     this.maxNumberOfDataPoints = Math.floor(maxSec / this.pollingFrequencySec);
+    this.insights = insights;
   }
 
   getQueuesActiveMessageCountsData(sec = maxSec) {
@@ -59,6 +60,9 @@ class QueueInfoPoller {
         self.crawlerClient.getInfo(`${queueName}`).then(info => {
           self.infos[queueName] = info;
           self.stats.data[queueName].push(parseInt(info.count));
+          if (queueName === 'normal') {
+            this.insights.trackMetric({ name: 'NormalQueueCount', value: info.count });
+          }
         }).catch(() => {
           self.stats.data[queueName].push(0);
         }).finally(() => {

dashboard/routes/index.js

@@ -40,7 +40,8 @@ router.init = (app, callback) => {
   const crawlerToken = config.dashboard.crawler.apiToken;
   crawlerClient = new CrawlerClient(crawlerUrl, crawlerToken);
 
-  queueInfoPoller.initialize(config);
+  const insights = app.get('providers').insights;
+  queueInfoPoller.initialize(config, insights);
   queueInfoPoller.startCollectingData();
   const redisClient = app.get('providers').redisClient;
   messageRates.initialize(config, redisClient);
@@ -52,23 +53,19 @@ router.get('/', (request, response) => {
 });
 
 router.get('/config', wrap(function* (request, response) {
-  request.insights.trackEvent('crawlerConfigGetStart');
   const config = yield crawlerClient.getConfiguration();
   config.crawler.url = crawlerClient.url;
   queueInfoPoller.crawlerName = config.crawler.name;
   response.json(config);
-  request.insights.trackEvent('crawlerConfigGetComplete');
 }));
 
 router.patch('/config', wrap(function* (request, response) {
-  request.insights.trackEvent('crawlerConfigPatchStart');
+  request.insights.trackTrace({ message: 'CrawlerConfigPatch' });
   yield crawlerClient.configureCrawler(request.body);
   response.json({ success: true });
-  request.insights.trackEvent('crawlerConfigPatchComplete');
 }));
 
 router.get('/deadletters', wrap(function* (request, response) {
-  request.insights.trackEvent('dashboardListDeadlettersStart');
   let deadletters = yield crawlerClient.listDeadletters();
   deadletters = deadletters.map(letter => {
     return {
@@ -80,18 +77,14 @@ router.get('/deadletters', wrap(function* (request, response) {
     };
   });
   response.json(deadletters);
-  request.insights.trackEvent('dashboardListDeadlettersComplete');
 }));
 
 router.get('/deadletters/:urn', wrap(function* (request, response) {
-  request.insights.trackEvent('dashboardGetDeadletterStart');
   const deadletters = yield crawlerClient.getDeadletter(request.params.urn);
   response.json(deadletters);
-  request.insights.trackEvent('dashboardGetDeadletterComplete');
 }));
 
 router.post('/deadletters', expressJoi.joiValidate(deadlettersSchema), wrap(function* (request, response) {
-  request.insights.trackEvent('dashboardPostDeadletterStart');
   const action = request.query.action;
   const requeueQueueName = request.query.queue || 'soon';
   const urns = request.body.urns;
@@ -112,47 +105,34 @@ router.post('/deadletters', expressJoi.joiValidate(deadlettersSchema), wrap(func
     respMessage.warnings = errors;
   }
   response.json(respMessage);
-  request.insights.trackEvent('dashboardPostDeadletterComplete');
 }));
 
 router.get('/requests/:queue', expressJoi.joiValidate(requestsSchema), wrap(function* (request, response) {
-  request.insights.trackEvent('dashboardGetRequestsStart');
   const requests = yield crawlerClient.getRequests(request.params.queue, parseInt(request.query.count, 10));
   response.json(requests);
-  request.insights.trackEvent('dashboardGetRequestsComplete');
 }));
 
 router.delete('/requests/:queue', expressJoi.joiValidate(requestsSchema), wrap(function* (request, response) {
-  request.insights.trackEvent('dashboardDeleteRequestsStart');
   const requests = yield crawlerClient.deleteRequests(request.params.queue, parseInt(request.query.count, 10));
   response.json(requests);
-  request.insights.trackEvent('dashboardDeleteRequestsComplete');
 }));
 
 router.post('/requests/:queue', wrap(function* (request, response) {
-  request.insights.trackEvent('dashboardQueueRequestStart');
   yield crawlerClient.queueRequests(request.body, request.params.queue || 'normal');
   response.sendStatus(201);
-  request.insights.trackEvent('dashboardQueueRequestComplete');
 }));
 
 router.put('/queue/:name', expressJoi.joiValidate(queueSchema), wrap(function* (request, response) {
-  request.insights.trackEvent('dashboardFlushQueueStart');
   yield crawlerClient.flushQueue(request.params.name);
   response.sendStatus(200);
-  request.insights.trackEvent('dashboardFlushQueueComplete');
 }));
 
 router.get('/queuesData', (request, response) => {
-  request.insights.trackEvent('queuesDataGetStart');
   response.json(queueInfoPoller.getQueuesActiveMessageCountsData(request.query.sec));
-  request.insights.trackEvent('queuesDataGetComplete');
 });
 
 router.get('/messageRatesData', (request, response) => {
-  request.insights.trackEvent('messageRatesDataStart');
   return messageRates.getMessageRatesData(request.query.sec).then(stats => {
-    request.insights.trackEvent('messageRatesDataComplete');
     return response.json(stats);
   }).catch(() => {
     return response.status(500).end();

middleware/appInsights.js

@@ -3,26 +3,29 @@
 
 'use strict';
 
-const insights = require('../lib/insights');
+const colors = require('colors');
+const debug = require('debug')('appinsights');
 
 module.exports = function initializeAppInsights(app, config) {
-  let client = undefined;
   const key = config && config.telemetry && config.telemetry.applicationInsightsKey ? config.telemetry.applicationInsightsKey : null;
+  let client = {};
   if (key) {
     const appInsights = require('applicationinsights');
-    const instance = appInsights.setup(key);
-    client = instance.getClient(key);
-    instance.start();
+    appInsights.setup(key)
+      .setAutoCollectPerformance(false)
+      .start();
+    client = appInsights.defaultClient;
+  } else {
+    client.trackEvent = (msg) => debug(colors.yellow(msg.name) + ' ' + JSON.stringify(msg.properties));
+    client.trackException = (msg) => debug(colors.red(msg.exception) + JSON.stringify(msg.properties));
+    client.trackMetric = (msg) => debug(colors.blue(msg.name) + ' ' + msg.value);
+    client.trackTrace = (msg) => debug(colors.green(msg.message));
   }
 
   app.use((req, res, next) => {
-    // Provide application insight event tracking with correlation ID
-    const extraProperties = {
-      correlationId: req.correlationId,
-    };
-    req.insights = insights(extraProperties, client);
+    req.insights = client;
     next();
   });
 
-  return insights({}, client);
+  return client;
 };

middleware/errorHandler.js

@@ -43,9 +43,13 @@ module.exports = function (err, req, res, next) {
   if (req && req.app && req.app.settings && req.app.settings.runtimeConfig) {
     config = req.app.settings.runtimeConfig;
     if (config.logging.errors && err.status !== 403 && err.skipLog !== true) {
-      req.insights.trackException(err, {
-        url: req.scrubbedUrl || req.originalUrl || req.url,
-        statusCode: errorStatus,
+      req.insights.trackException({
+        exception: err,
+        properties: {
+          url: req.scrubbedUrl || req.originalUrl || req.url,
+          statusCode: errorStatus,
+          correlationId: req.correlationId,
+        }
       });
     }
   }

middleware/initialize.js

@@ -33,7 +33,10 @@ module.exports = function init(app, express, rootdir, config, configurationError
         };
       };
       if (appInsightsClient) {
-        appInsightsClient.trackException(error, { info: 'App crashed because of an initialization error.' });
+        appInsightsClient.trackException({
+          exception: error,
+          properties: { info: 'App crashed because of an initialization error.' }
+        });
         appInsightsClient.sendPendingData(crash(error));
       } else {
         crash(error)();
@@ -65,21 +68,21 @@ module.exports = function init(app, express, rootdir, config, configurationError
   app.set('redisHelper', redisHelper);
   providers.redis = redisHelper;
   redisClient.on('connect', () => {
-    providers.insights.trackEvent('CrawlerDashboardRedisConnect');
+    providers.insights.trackTrace({ message: 'CrawlerDashboardRedisConnect' });
     if (redisFirstCallback) {
       const cb = redisFirstCallback;
       redisFirstCallback = null;
       cb();
     }
   });
   redisClient.on('error', error => {
-    providers.insights.trackException(error, { name: 'CrawlerDashboardRedisError' });
+    providers.insights.trackException({ exception: error, properties: { name: 'CrawlerDashboardRedisError' } });
   });
   redisClient.on('reconnecting', properties => {
-    providers.insights.trackEvent('CrawlerDashboardRedisReconnecting', properties);
+    providers.insights.trackEvent({ name: 'CrawlerDashboardRedisReconnecting', properties });
   });
   redisClient.on('end', () => {
-    providers.insights.trackEvent('CrawlerDashboardRedisEnd');
+    providers.insights.trackTrace({ message: 'CrawlerDashboardRedisEnd' });
   });
   async.parallel([
     cb => {