diff --git a/CHANGELOG.md b/CHANGELOG.md index 1731ab9..9928fd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # check-http-status changelog +## 1.3.0 - Oct 15, 2021 + +### Enhancement + +- Allow Sitemap(s) and particular URL(s) together +- Limit Promise to max. 10 concurrent requests +- Show progress of the HTTP Status list + ## 1.2.0 - May 21, 2021 ### Enhancement diff --git a/README.md b/README.md index 99c16a7..7d5c544 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,15 @@ [![Downloads][downloads-image]][npm-url] [![Build Status][travis-image]][travis-url] +Easily check status codes, response headers, and redirect chains in `Node.js` +similar as done from the [httpstatus](https://httpstatus.io/) website. + +When the site is on VPN so this is where it plays an important role. You can +simply connect your system/machine with VPN and run this package locally so it +can check the status of your VPN connected URL(s). + +It can also check the website that are secured with HTTP Authentication. + ## Install Via `npm` @@ -18,14 +27,19 @@ Via Yarn yarn add check-http-status --dev ``` -## Usage +## Examples -### Sitemap Example +### Status Code of the Sitemap(s) URL(s) ```node const checkHttpStatus = require('check-http-status'); checkHttpStatus({ + 'sitemaps': [ + 'https://www.trunkcode.com/page-sitemap.xml', + 'https://www.trunkcode.com/post-sitemap.xml' + ], + 'skip200': true, // Do not report the URLs having HTTP code 200. 'export': { 'format': 'xlsx', 'location': '/Users/trunkcode/Desktop/', @@ -37,22 +51,25 @@ checkHttpStatus({ }, 'headers': { 'Accept': 'text/html', - }, - }, - 'sitemap': [ - 'https://www.trunkcode.com/page-sitemap.xml', - 'https://www.trunkcode.com/post-sitemap.xml' - ], - 'skip200': true, // Do not report the URLs having HTTP code 200. + } + } }); ``` -### URLs Example +### Status Code of the particular URL(s) ```node const checkHttpStatus = require('check-http-status'); checkHttpStatus({ + 'urls': [ + 'http://trunkcode.com/', + 'https://example.com/', + 'https://example1234.com/', + 'https://www.trunkcode.com/', + 'https://www.trunkcode.com/test/' + ], + 'skip200': true, // Do not report the URLs having HTTP code 200. 'export': { 'format': 'xlsx', 'location': '/Users/trunkcode/Desktop/', @@ -64,19 +81,59 @@ checkHttpStatus({ }, 'headers': { 'Accept': 'text/html', - }, - }, - 'skip200': true, // Do not report the URLs having HTTP code 200. - 'urls': [ + } + } +}); +``` + +### Status Code of the Sitemap(s) URL(s) with particular URL(s) + +```node +const checkHttpStatus = require('check-http-status'); + +checkHttpStatus({ + 'sitemaps': [ + 'https://www.trunkcode.com/page-sitemap.xml', + 'https://www.trunkcode.com/post-sitemap.xml' + ], + 'urls': [ 'http://trunkcode.com/', 'https://example.com/', 'https://example1234.com/', 'https://www.trunkcode.com/', 'https://www.trunkcode.com/test/' - ] + ], + 'skip200': true, // Do not report the URLs having HTTP code 200. + 'export': { + 'format': 'xlsx', + 'location': '/Users/trunkcode/Desktop/', + }, + 'options': { + 'auth': { + 'password': 'Testing1234', + 'username': 'trunkcode' + }, + 'headers': { + 'Accept': 'text/html', + } + } }); ``` +## Parameters + +| Attributes | Type | Required | Default | Description | +|:----------:|:-------:|:--------:|:-------:|:------------------------------------------------------------------------------------------------:| +| sitemaps | Array | Yes | | Sitemap(s) URL(s) where the Actual site URL(s) needs to be fetched for checking the HTTP Status. | +| urls | Array | Yes | | URL(s) for which HTTP Status needs to be checked. | +| skip200 | Boolean | No | `false` | Whether to list the HTTP status `200` URL(s) or not. | +| export | Object | No | `{}` | Whether to export the status report or not. By default it logs the report on the screen. | +| options | Object | No | `{}` | Define options like HTTP Auth credentials if the site is locked or headers etc. | + +**NOTE:** `sitemaps` or `urls` is required. You can define both parameters as +well to fetch URL(s) from sitemap and the URL(s) that are not listed in the +ssitemap, you can provide them separately. + [npm-image]: https://img.shields.io/npm/v/check-http-status.svg [npm-url]: https://www.npmjs.com/package/check-http-status [downloads-image]: https://img.shields.io/npm/dt/check-http-status.svg diff --git a/index.js b/index.js index a7b65df..6aab47d 100644 --- a/index.js +++ b/index.js @@ -1,7 +1,7 @@ 'use strict'; const consoleColumns = require('./lib/console-columns'); -const fetchAllSitemaps = require('./lib/fetch-from-sitemap'); +const fetchAllSitemaps = require('./lib/fetch-from-sitemaps'); const fs = require('fs'); const generateExcel = require('./lib/generate-excel'); const httpList = require('./lib/http-list'); @@ -11,67 +11,69 @@ const httpList = require('./lib/http-list'); * config. */ async function checkHttpStatus(config) { - const allowedExportTypes = [ - 'csv', - 'xlsx', - ]; - var skip200 = false; - var urlsList = []; + const allowedExportTypes = [ + 'csv', + 'xlsx', + ]; + var skip200 = false; + var urlsList = []; - if (config.skip200) { - skip200 = true; - } + if (config.skip200) { + skip200 = true; + } - if (!config) { - console.error('\x1b[31m%s\x1b[0m', 'Error: Missing required Parameters.'); - process.exit(); - } else if (!config.options) { - config.options = {}; - } + if (!config) { + console.error('\x1b[31m%s\x1b[0m', 'Error: Missing required Parameters.'); + process.exit(); + } else if (!config.options) { + config.options = {}; + } - if (config.sitemap) { - urlsList = await fetchAllSitemaps(config.sitemap); - } else if (config.urls && Array.isArray(config.urls)) { - urlsList = config.urls; - } + if (config.sitemaps) { + urlsList = await fetchAllSitemaps(config.sitemaps); + } - if (urlsList.length === 0) { - console.error('\x1b[31m%s\x1b[0m', 'Error: No URL(s) found.'); - process.exit(); - } else if (config.export && !config.export.location) { - console.error('\x1b[31m%s\x1b[0m', 'Error: Missing export location.'); - process.exit(); - } else if (config.export && !fs.existsSync(config.export.location)) { - console.error('\x1b[31m%s\x1b[0m', 'Error: Export Location is undefined.'); - process.exit(); - } + if (config.urls && Array.isArray(config.urls)) { + urlsList = urlsList.concat(config.urls); + } - const httpStatusList = await httpList(urlsList, config.options, skip200); + if (urlsList.length === 0) { + console.error('\x1b[31m%s\x1b[0m', 'Error: No URL(s) found.'); + process.exit(); + } else if (config.export && !config.export.location) { + console.error('\x1b[31m%s\x1b[0m', 'Error: Missing export location.'); + process.exit(); + } else if (config.export && !fs.existsSync(config.export.location)) { + console.error('\x1b[31m%s\x1b[0m', 'Error: Export Location is undefined.'); + process.exit(); + } - if (config.export && !config.export.format) { - config.export.format = 'xlsx'; - } + const httpStatusList = await httpList(urlsList, config.options, skip200); - if (skip200 && httpStatusList.length === 0) { - // Add empty line - console.log(); - console.log('\x1b[32m%s\x1b[0m', 'All the URLs are 200 and there is nothing to worry about!'); - } else if (config.export && allowedExportTypes.includes(config.export.format)) { - const urlLength = Math.max(...urlsList.map((el) => el.length)); - const rowLength = { - 'errorMessage': 50, - 'requestedUrl': urlLength, - 'url': urlLength + 20 - }; + if (config.export && !config.export.format) { + config.export.format = 'xlsx'; + } - generateExcel(httpStatusList, rowLength, config.export); - } else { - consoleColumns(httpStatusList); - } + if (skip200 && httpStatusList.length === 0) { + // Add empty line + console.log(); + console.log('\x1b[32m%s\x1b[0m', 'All the URLs are 200 and there is nothing to worry about!'); + } else if (config.export && allowedExportTypes.includes(config.export.format)) { + const urlLength = Math.max(...urlsList.map((el) => el.length)); + const rowLength = { + 'errorMessage': 50, + 'requestedUrl': urlLength, + 'url': urlLength + 20 + }; - // Add empty line - console.log(); - console.log('\x1b[32m%s\x1b[0m', 'HTTP Status check completed!'); + generateExcel(httpStatusList, rowLength, config.export); + } else { + consoleColumns(httpStatusList); + } + + // Add empty line + console.log(); + console.log('\x1b[32m%s\x1b[0m', 'HTTP Status check completed!'); } module.exports = checkHttpStatus; diff --git a/lib/check-status-code.js b/lib/check-status-code.js index ec2ccd0..591b156 100644 --- a/lib/check-status-code.js +++ b/lib/check-status-code.js @@ -5,6 +5,12 @@ const axios = require('axios'); /** * Check HTTP Status as per settings and return status code and other * information for further process. + * + * @param string urlToCheck URL that needs to be requested. + * @param object axiosOptions Axios Options. + * @param bool redirect Whether the URL is crawling or the redirected URL is crawling. + * + * @returns array Response of checked URL. */ function axiosRequest(urlToCheck, axiosOptions, redirect) { var httpStatus = [ @@ -81,6 +87,12 @@ function axiosRequest(urlToCheck, axiosOptions, redirect) { /** * Call main function to generate array with all the required information * and await until that all are not completed. + * + * @param string urlToCheck URL that needs to be checked. + * @param object options Axios options. + * @param bool skip200 Whether to log URLs that returns 200 status code or not. + * + * @returns array List of URLs with their status code and other information. */ async function checkStatusCode(urlToCheck, options, skip200) { const statusList = []; diff --git a/lib/console-columns.js b/lib/console-columns.js index cc2f7eb..d172faa 100644 --- a/lib/console-columns.js +++ b/lib/console-columns.js @@ -4,6 +4,9 @@ const columnify = require('columnify'); /** * List down all the URLs with the status code in console. + * + * @param array statusList List of URLs with their status code and other + * information to be logged on console. */ function terminalColumns(statusList) { const data = []; diff --git a/lib/fetch-from-sitemap.js b/lib/fetch-from-sitemaps.js similarity index 84% rename from lib/fetch-from-sitemap.js rename to lib/fetch-from-sitemaps.js index b655069..cec890d 100644 --- a/lib/fetch-from-sitemap.js +++ b/lib/fetch-from-sitemaps.js @@ -5,6 +5,10 @@ const xml2js = require('xml2js'); /** * Fetch Sitemap URL and dumb all the URLs from the XML to a variable. + * + * @param string sitemapUrl Sitemap URL from which list of URLs needs to be fetched. + * + * @returns array List of URLs fetched from the sitemap. */ function fetchFromSitemap(sitemapUrl) { var errorMessage = 'Error: ' + sitemapUrl + ' returns with status code '; @@ -42,6 +46,13 @@ function fetchFromSitemap(sitemapUrl) { }); } +/** + * Fetch URLs from the sitemap one-by-one. + * + * @param array sitemapUrls List of Sitemap URL(s). + * + * @returns array List of URLs fetched from all the sitemap(s). + */ async function fetchAllSitemaps(sitemapUrls) { const sitemapUrlsLists = []; const urlsList = []; diff --git a/lib/generate-excel.js b/lib/generate-excel.js index e916363..4642feb 100644 --- a/lib/generate-excel.js +++ b/lib/generate-excel.js @@ -4,6 +4,10 @@ const excelJS = require('exceljs'); /** * Generate filename with location to save the exported file. + * + * @param object exportDetails Pass export information like format type, location etc. + * + * @returns string File Location with filename. */ function exportFileName(exportDetails) { const currentTimestamp = Math.floor(Date.now() / 1000); @@ -16,6 +20,10 @@ function exportFileName(exportDetails) { /** * List down all the URLs with the status code in .xlsx format. + * + * @param array statusList List of the URLs needs to bee added in excel file. + * @param object rowLength Length that needs to be set for each column. + * @param object exportDetails Pass export information like format type, location etc. */ function generateExcel(statusList, rowLength, exportDetails) { const location = exportFileName(exportDetails); diff --git a/lib/http-list.js b/lib/http-list.js index 1ca18cb..80143c5 100644 --- a/lib/http-list.js +++ b/lib/http-list.js @@ -2,24 +2,57 @@ const checkStatusCode = require('./check-status-code'); +/** + * Show Progress of HTTP Status. + * + * @param int progressStatus Currently Processing URL(s). + * @param int totalList Total URL(s) needs to be processed. + */ +function printProgress(progressList, totalList) { + process.stdout.clearLine(); + process.stdout.cursorTo(0); + process.stdout.write('Processing ' + progressList + ' of ' + totalList + ' ... '); +} + /** * List down all the URLs with the status code. + * + * @param array urlsList List of URLs. + * @param object configOptions config options for Axios like HTTP Auth etc. + * @param bool skip200 Whether to log URLs that returns 200 status code or not. + * + * @returns array Result of each requested with its status code etc. */ async function httpList(urlsList, configOptions, skip200) { - const httpStatus = []; - var combineResults = []; + const httpStatus = []; + const totalUrls = urlsList.length; + + var combineResults = []; + var currentProgress = 0; + var maxLimit = 0; + + + printProgress(currentProgress, totalUrls); + for (const checkUrl of urlsList) { + httpStatus.push(checkStatusCode(checkUrl, configOptions, skip200)); + currentProgress += 1; + maxLimit += 1; - for (const checkUrl of urlsList) { - httpStatus.push(checkStatusCode(checkUrl, configOptions, skip200)); - } + printProgress(currentProgress, totalUrls); - const httpResults = await Promise.all(httpStatus); + // Resolve previous promises before pushing more. + if (maxLimit === 10) { + await Promise.all(httpStatus); + maxLimit = 0; + } + } - for (const status of httpResults) { - combineResults = combineResults.concat(status); - } + const httpResults = await Promise.all(httpStatus); + for (const status of httpResults) { + combineResults = combineResults.concat(status); + } - return combineResults; + return combineResults; } module.exports = httpList; diff --git a/package.json b/package.json index 46b8120..84f8ba3 100644 --- a/package.json +++ b/package.json @@ -1,44 +1,46 @@ { - "name": "check-http-status", - "description": "Check HTTP Status and create a list for all the HTTP status code.", - "version": "1.2.0", - "main": "index.js", - "files": [ - "LICENSE", - "index.js", - "test/runkitExample.js", - "lib/*" - ], - "scripts": { - "lint": "eslint .", - "test:sitemap": "node test/sitemap.js", - "test:urls": "node test/urls.js" - }, - "repository": { - "type": "git", - "url": "https://github.com/trunkcode/check-http-status.git" - }, - "keywords": [ - "http status", - "link checker", - "status", - "redirects", - "301", - "404" - ], - "author": "TrunkCode (https://www.trunkcode.com/)", - "license": "MIT", - "bugs": { - "url": "https://github.com/trunkcode/check-http-status/issues" - }, - "homepage": "https://github.com/trunkcode/check-http-status", - "dependencies": { - "axios": "^0.21.1", - "columnify": "^1.5.4", - "exceljs": "^4.2.0", - "xml2js": "^0.4.23" - }, - "devDependencies": { - "eslint": "^7.16.0" - } + "name": "check-http-status", + "description": "Check HTTP Status and create a list for all the HTTP status code.", + "version": "1.3.0", + "main": "index.js", + "files": [ + "LICENSE", + "index.js", + "test/runkitExample.js", + "lib/*" + ], + "scripts": { + "lint": "eslint .", + "test:sitemaps": "node test/sitemaps.js", + "test:sitemaps-urls": "node test/sitemaps-urls.js", + "test:urls": "node test/urls.js" + }, + "repository": { + "type": "git", + "url": "https://github.com/trunkcode/check-http-status.git" + }, + "keywords": [ + "http status", + "redirect chain", + "link checker", + "status", + "redirects", + "301", + "404" + ], + "author": "TrunkCode (https://www.trunkcode.com/)", + "license": "MIT", + "bugs": { + "url": "https://github.com/trunkcode/check-http-status/issues" + }, + "homepage": "https://github.com/trunkcode/check-http-status", + "dependencies": { + "axios": "^0.23.0", + "columnify": "^1.5.4", + "exceljs": "^4.3.0", + "xml2js": "^0.4.23" + }, + "devDependencies": { + "eslint": "^8.0.1" + } } diff --git a/test/runkitExample.js b/test/runkitExample.js index c24e4ed..59d5eff 100644 --- a/test/runkitExample.js +++ b/test/runkitExample.js @@ -1,23 +1,21 @@ 'use strict'; -const checkHttpStatus = require('../index'); +const checkHttpStatus = require('check-http-status'); checkHttpStatus({ - 'options': { - 'auth': { - 'password': 'Testing1234', - 'username': 'trunkcode' - }, - 'headers': { - 'Accept': 'text/html', - }, - }, - 'skip200': true, - 'urls': [ - 'https://trunkcode.com/', - 'https://example.com/', - 'https://example1234.com/', - 'https://www.trunkcode.com/', - 'https://www.trunkcode.com/test/' - ] + 'options': { + 'headers': { + 'Accept': 'text/html', + } + }, + 'sitemaps': [ + 'https://www.trunkcode.com/page-sitemap.xml', + 'https://www.trunkcode.com/post-sitemap.xml' + ], + 'skip200': true, + 'urls': [ + 'https://example.com/', + 'https://example1234.com/', + 'https://www.trunkcode.com/test/' + ] }); diff --git a/test/sitemap.js b/test/sitemap.js deleted file mode 100644 index d54234e..0000000 --- a/test/sitemap.js +++ /dev/null @@ -1,24 +0,0 @@ -'use strict'; - -const checkHttpStatus = require('../index'); - -checkHttpStatus({ - 'export': { - 'format': 'xlsx', - 'location': '/Users/trunkcode/Desktop/', - }, - 'options': { - 'auth': { - 'password': 'Testing1234', - 'username': 'trunkcode' - }, - 'headers': { - 'Accept': 'text/html', - }, - }, - 'sitemap': [ - 'https://www.trunkcode.com/page-sitemap.xml', - 'https://www.trunkcode.com/post-sitemap.xml' - ], - 'skip200': true, -}); diff --git a/test/sitemaps-urls.js b/test/sitemaps-urls.js new file mode 100644 index 0000000..6712050 --- /dev/null +++ b/test/sitemaps-urls.js @@ -0,0 +1,25 @@ +'use strict'; + +const checkHttpStatus = require('../index'); + +checkHttpStatus({ + 'export': { + 'format': 'xlsx', + 'location': '/Users/trunkcode/Desktop/', + }, + 'options': { + 'headers': { + 'Accept': 'text/html', + } + }, + 'sitemaps': [ + 'https://www.trunkcode.com/page-sitemap.xml', + 'https://www.trunkcode.com/post-sitemap.xml' + ], + 'skip200': true, + 'urls': [ + 'https://example.com/', + 'https://example1234.com/', + 'https://www.trunkcode.com/test/' + ] +}); diff --git a/test/sitemaps.js b/test/sitemaps.js new file mode 100644 index 0000000..570e085 --- /dev/null +++ b/test/sitemaps.js @@ -0,0 +1,20 @@ +'use strict'; + +const checkHttpStatus = require('../index'); + +checkHttpStatus({ + 'export': { + 'format': 'xlsx', + 'location': '/Users/trunkcode/Desktop/', + }, + 'options': { + 'headers': { + 'Accept': 'text/html', + } + }, + 'sitemaps': [ + 'https://www.trunkcode.com/page-sitemap.xml', + 'https://www.trunkcode.com/post-sitemap.xml' + ], + 'skip200': true, +}); diff --git a/test/urls.js b/test/urls.js index c1ee48b..4f42b9e 100644 --- a/test/urls.js +++ b/test/urls.js @@ -3,25 +3,21 @@ const checkHttpStatus = require('../index'); checkHttpStatus({ - 'export': { - 'format': 'xlsx', - 'location': '/Users/trunkcode/Desktop/', - }, - 'options': { - 'auth': { - 'password': 'Testing1234', - 'username': 'trunkcode' - }, - 'headers': { - 'Accept': 'text/html', - }, - }, - 'skip200': true, - 'urls': [ - 'https://trunkcode.com/', - 'https://example.com/', - 'https://example1234.com/', - 'https://www.trunkcode.com/', - 'https://www.trunkcode.com/test/' - ] + 'export': { + 'format': 'xlsx', + 'location': '/Users/trunkcode/Desktop/', + }, + 'options': { + 'headers': { + 'Accept': 'text/html', + }, + }, + 'skip200': true, + 'urls': [ + 'https://trunkcode.com/', + 'https://example.com/', + 'https://example1234.com/', + 'https://www.trunkcode.com/', + 'https://www.trunkcode.com/test/' + ] });