move log functions to utils

src/routes/index.js

@@ -3,7 +3,7 @@ const RateLimit = require('koa2-ratelimit').RateLimit;
 const router = require('@koa/router')();
 const { listApps, describeApp, reloadApp, restartApp, stopApp } = require('../providers/pm2/api')
 const { bytesToSize, timeSince } = require('../providers/pm2/ux.helper')
-const { readLogs } = require('../utils/logs')
+const  { readLogs } = require('../utils/readLogs.util')
 const { getCurrentGitBranch, getCurrentGitCommit } = require('../utils/git.util')
 const { getEnvFileContent } = require('../utils/env.util')
 const isAuthenticated = require('../middlewares/auth')

src/utils/readLogs.util.js

@@ -0,0 +1,96 @@
+
+var fs = require('fs');
+var es = require('event-stream');
+
+const DEFAULT_LINES_PER_PAGE = 50;
+const DEFAULT_PAGE_NUMBER = 1;
+const DEFAULT_SORT_ORDER = 'desc';
+
+function countFileLines(file_path){
+  return new Promise((resolve, reject) => {
+  let lineCount = 0;
+  fs.createReadStream(file_path)
+    .on("data", (buffer) => {
+      let idx = -1;
+      lineCount--; // Because the loop will run once for idx=-1
+      do {
+        idx = buffer.indexOf(10, idx+1);
+        lineCount++;
+      } while (idx !== -1);
+    }).on("end", () => {
+      resolve(lineCount);
+    }).on("error", reject);
+  });
+};
+
+const readLogs = (params) => {
+  return new Promise(async (resolve, reject) => {
+    let { file_path, sort_order=DEFAULT_SORT_ORDER, page_number=DEFAULT_PAGE_NUMBER, lines_per_page=DEFAULT_LINES_PER_PAGE} = params;
+    page_number = parseInt(page_number)
+    if(page_number < 1){
+      throw new Error('Page number should be greater or equal to 1')
+    }
+    if(lines_per_page < 1){
+      throw new Error('Lines per page should be greater or equal to 1')
+    }
+    const total_lines = await countFileLines(file_path) + 1
+    let line_start = 1;
+    let line_end = 1;
+    if(sort_order == 'asc'){
+      line_start = Math.min(total_lines, ((page_number - 1) * lines_per_page) + 1)
+      line_end = Math.min(total_lines, (line_start + lines_per_page) - 1)
+    }
+    else{
+      line_start = Math.max(1, total_lines - (page_number * lines_per_page) + 1)
+      if(line_start === 1){
+        line_end = total_lines % lines_per_page;
+      }
+      else{
+        line_end = Math.min(total_lines, (line_start + lines_per_page) - 1)
+      }
+    }
+    let line_current = 1;
+    let lines_data = []
+    const stream = fs
+    .createReadStream(file_path)
+    .pipe(es.split())
+    .pipe(
+      es.mapSync(function(line) {
+        if(line_current >= line_start){
+          lines_data.push(`${line_current} : ${line}`)
+        }
+        if(line_current === line_end){
+          stream.end();
+        }
+        line_current++;
+      })
+      .on('error', reject)
+      .on('end', function() {
+        let page_order_data = {}
+        if(sort_order == 'asc'){
+          page_order_data.page_up = (page_number - 1)
+          page_order_data.page_down = page_number + 1
+        }
+        else{
+          page_order_data.page_up = page_number + 1
+          page_order_data.page_down = (page_number - 1)
+        }
+        resolve({
+          lines: lines_data,
+          ...page_order_data,
+          page_current: page_number,
+          total_pages: Math.ceil(total_lines/lines_per_page)
+        }) 
+      }),
+    ); 
+  })
+}
+
+async function test(){
+  console.time('Performance')
+  const logs = await readLogs({file_path: '/home/ubuntu/.pm2/logs/admin-service-out.log', page_number: 200, lines_per_page: 100, sort_order:'desc'})
+  console.timeEnd('Performance')
+  console.log('Output Lines : ',logs.lines.length)
+}
+
+module.exports = { readLogs }

src/utils/readLogsFuzzy.util.js

@@ -0,0 +1,56 @@
+
+const fs = require('fs');
+const es = require('event-stream');
+
+const DEFAULT_LINES_PER_PAGE = 30;
+const DEFAULT_PAGE_NUMBER = 1;
+const DEFAULT_SORT_ORDER = 'desc';
+
+
+var readLogsFuzzy = function (params) {
+  let { filePath, sort_order=DEFAULT_SORT_ORDER, page_number=DEFAULT_PAGE_NUMBER, lines_per_page=DEFAULT_LINES_PER_PAGE} = params;
+  return new Promise((resolve, reject) => {
+    const fileSize =  fs.statSync(filePath).size
+    console.log('Filesize : ', fileSize)
+    let chunk = '';
+    let options = {};
+    let line_cursor = 0;
+    if(sort_order === 'asc'){
+      line_cursor = ((page_number - 1) * lines_per_page) + 1
+      options.start = Math.min(fileSize, line_cursor * 200);
+      options.end = Math.min(fileSize, options.start + (lines_per_page * 200));
+    }
+    else{
+      line_cursor = ((page_number - 1) * lines_per_page) + 1
+      options.start = Math.max(0, fileSize - (line_cursor * 200));
+      options.end = Math.min(fileSize, options.start + (lines_per_page * 200));
+    }
+    console.log('Sort Order : ', sort_order)
+    console.log('Line Cursor : ', line_cursor)
+    console.log('Read Stream Options : ',options)
+    const fd = fs.createReadStream(filePath, options);
+    fd.on('data', function(data) { chunk += data.toString(); });
+    fd.on('error', reject)
+    fd.on('end', function() {
+      chunk = chunk.split('\n')
+      const lines_limit = Math.min(chunk.length, lines_per_page);
+      if(sort_order === 'asc'){
+        chunk = chunk.slice(0,(lines_limit+1));
+      }
+      else{
+        chunk = chunk.slice(-(lines_limit+1));
+      }
+      chunk.pop();
+      resolve(chunk);
+    });
+  })
+};
+
+async function test(){
+  console.time('Performance Fuzzy')
+  const logs = await readLogsFuzzy({filePath: '/home/ubuntu/.pm2/logs/admin-service-out.log', page_number: 100, lines_per_page: 100, sort_order:'asc'})
+  console.timeEnd('Performance Fuzzy')
+  console.log('Output Lines : ',logs.length)
+}
+
+module.exports =  { readLogsFuzzy }

src/utils/searchLogs.util.js

@@ -0,0 +1,38 @@
+var fs = require('fs');
+var es = require('event-stream');
+
+const MAX_RESULTS = 30;
+
+function findTerm(line, term) {
+    return line.indexOf(term) !== -1
+  }
+
+const searchLogs = async (filePath, term, max_results = MAX_RESULTS) => {
+    let line_current = 0;
+    let lines_data = []
+    const stream = fs
+    .createReadStream(filePath)
+    .pipe(es.split())
+    .pipe(
+      es.mapSync(function(line) {
+        line_current++;
+        if(findTerm(line, term)){
+          lines_data.push({line_number: line_current, line: line})
+        }
+        if(lines_data.length === max_results){
+          stream.end();
+        }
+      })
+      .on('error', function(err) {
+        console.log('Error while reading file.', err);
+      })
+      .on('end', function() {
+        console.log(lines_data)
+      }),
+    ); 
+}
+
+function test(){
+  // searchLogs('/home/ubuntu/.pm2/logs/admin-service-out.log', 'is_deleted: false')
+}
+module.exports  = { searchLogs }