From 1593c80897097a63aea308b7edaa8c681292202c Mon Sep 17 00:00:00 2001 From: Brian Date: Tue, 30 Mar 2021 13:42:51 -0500 Subject: [PATCH] Bugfixes --- dist/handler.js | 46 --------- dist/lib/async.js | 218 ---------------------------------------- dist/lib/logger.js | 8 -- dist/lib/spawn.js | 18 ---- dist/lighthouse.js | 143 -------------------------- dist/server.js | 24 ----- scripts/batch-lambda.sh | 7 +- src/handler.ts | 2 +- src/lighthouse.ts | 7 +- tsconfig.json | 4 +- 10 files changed, 12 insertions(+), 465 deletions(-) delete mode 100644 dist/handler.js delete mode 100644 dist/lib/async.js delete mode 100644 dist/lib/logger.js delete mode 100644 dist/lib/spawn.js delete mode 100644 dist/lighthouse.js delete mode 100644 dist/server.js diff --git a/dist/handler.js b/dist/handler.js deleted file mode 100644 index 542930a..0000000 --- a/dist/handler.js +++ /dev/null @@ -1,46 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -const fs = require("fs"); -const path = require("path"); -const logger_1 = require("./lib/logger"); -const lighthouse_1 = require("./lighthouse"); -const isProd = process.env.NODE_ENV === 'production'; -const packageJson = JSON.parse(fs.readFileSync(path.join(__dirname, '../package.json'), 'utf-8')); -async function handler(event, context) { - logger_1.default.info(`Received event on version ${packageJson.version}: ${JSON.stringify(event)}`); - let body; - let statusCode = '200'; - const headers = { - 'Content-Type': 'application/json', - }; - const { url, type } = event.queryStringParameters; - try { - if (!url) - throw new Error('url query param is required.'); - switch (event.httpMethod) { - case 'GET': - body = await lighthouse_1.default(url, type); - break; - default: - throw new Error(`Unsupported method "${event.httpMethod}"`); - } - } - catch (err) { - if (!isProd) - throw err; - else - logger_1.default.error(err); - statusCode = '400'; - body = err.message; - } - // finally { - // body = JSON.stringify(body, null, 2); - // } - return { - statusCode, - body, - headers, - }; -} -exports.default = handler; -; diff --git a/dist/lib/async.js b/dist/lib/async.js deleted file mode 100644 index 16af94f..0000000 --- a/dist/lib/async.js +++ /dev/null @@ -1,218 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.waitForTruthy = exports.waitFor = exports.delay = void 0; -const debugLog = (message) => null; // console.debug; -/** - * Return a promise that resolves after ms milliseconds - * - * Is basically the same as Rambdax's delay - * - * Can be used in async functions to wait for stuff. - * - * For example, - * while(checkIfTrue()) await sleep(200); - * - * @param ms: Number of milliseconds to wait - * - **/ -const delay = (ms) => { - return new Promise(function (resolve) { - setTimeout(resolve, ms); - }); -}; -exports.delay = delay; -/** - * Return a promise that resolves when a callback resolves without throwing, or - * on give-up (see options). The final promise resolves to the final result of - * the callback. Will throw an error on give-up by default. - * - * For example, - * const getElement = (selector: string) => document.querySelector(selector) as HTMLDivElement; - * const element = await waitFor( - * () => { - * const e = getElement("#treasure") - * if (!e) throw new waitFor.NotReadyException() - * return e - * }, - * { interval: 50, timeout: 2000 } - * ); - * element.innerHTML = "Harrrrr!"; // type = HTMLDivElement automagically! - * element.notAThing = "boo"; // fails type check! - * - * @param callback: A function to retry until resolves without a NotReadyException (promises work too) - * @param options: Options to control when and how to give-up - * @param options.interval: How often to retry. Will never retry until last is done, though. - * @param options.timeout: How long to wait before giving up - * @param options.maxTries: How many times to retry before giving up - * @param options.throwsErrorOnBail: Whether to throw an error or not on bail (aka give-up) - * - */ -class NotReadyException extends Error { -} -const waitForPass = async (callback, options) => { - const { interval = 100, timeout = 10000, maxTries, throwsErrorOnBail = true, } = options; - const callbackPromise = async () => callback(); // promisify callback - let tries = 0; - const before = Date.now(); - const getElapsed = () => Date.now() - before; - let res; - while (!res) { - const timeBeforeCall = getElapsed(); - if (timeBeforeCall > timeout) { - if (throwsErrorOnBail) - throw new Error(`waitFor: timed out after ${timeout}ms`); - else - return null; - } - await callbackPromise() - .then((r) => (res = r)) - .catch((e) => { - if (!(e instanceof NotReadyException)) - throw e; - }); - const timeAfterCall = getElapsed(); - const callDuration = timeAfterCall - timeBeforeCall; - tries++; - if (maxTries && tries > maxTries) { - if (throwsErrorOnBail) - throw new Error(`waitFor: reached maxTries at ${tries}`); - else - return null; - } - const timeUntilInterval = interval - callDuration; - debugLog(`waitFor: tries=${tries} timeUntilInt=${timeUntilInterval}`); - if (timeUntilInterval > 0) { - debugLog(`waitFor: waiting ${timeUntilInterval}`); - await exports.delay(timeUntilInterval); - } - } - return res; -}; -exports.waitFor = Object.assign(waitForPass, { NotReadyException }); -/** - * Return a promise that resolves when a callback resolves to truthy or on give-up (see options), - * and that promise resolves to the final result of the callback. Will throw an error on give-up - * by default. - * - * For example, - * const getElement = (selector: string) => document.querySelector(selector) as HTMLDivElement; - * const element = await waitForTruthy( - * () => getElement("#treasure"), - * { interval: 50, timeout: 2000 } - * ); - * element.innerHTML = "Harrrrr!"; // type = HTMLDivElement automagically! - * element.notAThing = "boo"; // fails type check! - * - * @param callback: A function to retry until truthy (promises work too) - * @param options: Options to control when and how to give-up - * @param options.interval: How often to retry. Will never retry until last is done, though. - * @param options.timeout: How long to wait before giving up - * @param options.maxTries: How many times to retry before giving up - * @param options.throwsErrorOnBail: Whether to throw an error or not on bail (aka give-up) - * - */ -const waitForTruthy = async (callback, options = {}) => { - const { interval = 100, timeout = 60000, maxTries, throwsErrorOnBail = true } = options; - const callbackPromise = async () => callback(); // promisify callback - let tries = 0; - const before = Date.now(); - const getElapsed = () => Date.now() - before; - let res; - while (!res) { - const timeBeforeCall = getElapsed(); - if (timeBeforeCall > timeout) { - if (throwsErrorOnBail) - throw new Error(`waitFor: timed out after ${timeout}ms`); - else - return null; - } - res = await callbackPromise(); - const timeAfterCall = getElapsed(); - const callDuration = timeAfterCall - timeBeforeCall; - tries++; - if (maxTries && tries > maxTries) { - if (throwsErrorOnBail) - throw new Error(`waitFor: reached maxTries at ${tries}`); - else - return null; - } - const timeUntilInterval = interval - callDuration; - debugLog(`waitFor: tries=${tries} timeUntilInt=${timeUntilInterval}`); - if (timeUntilInterval > 0) { - debugLog(`waitFor: waiting ${timeUntilInterval}`); - await exports.delay(timeUntilInterval); - } - } - return res; -}; -exports.waitForTruthy = waitForTruthy; -// const testWait = async () => { -// const before = Date.now(); -// await delay(100); -// const elapsed = Date.now() - before; -// if (elapsed < 100) console.log(`testWait.elapsed: Failed`); -// else console.log(`testWait.elapsed: Passed with ${elapsed}`); -// }; -// testWait(); -// const testWaitForPass = async () => { -// let tries = 0 -// let before = Date.now() -// const getElapsed = () => Date.now() - before -// -// const callback = async () => { -// console.log(`Tries: ${tries}; Elapsed: ${getElapsed()}`) -// await delay(10) -// if (tries++ < 5) throw 0 -// return "success" -// } -// -// tries = 0 -// before = Date.now() -// await waitForPass(callback, { interval: 0, timeout: 10000 }) -// .then(() => console.log("Success passed")) -// .catch(() => console.log("Success failed")) -// -// tries = 0 -// before = Date.now() -// await waitForPass(callback, { interval: 10, timeout: 10 }) -// .then(() => console.log("Timeout test failed")) -// .catch(() => console.log("Timeout test passed")) -// -// tries = 0 -// before = Date.now() -// await waitForPass(callback, { maxTries: 1 }) -// .then(() => console.log("Max tries test failed")) -// .catch(() => console.log("Max tries test passed")) -// } -// testWaitForPass() -// const testWaitForTruthy = async () => { -// let tries = 0; -// let before = Date.now(); -// const getElapsed = () => Date.now() - before; -// -// const callback = async () => { -// console.log(`Tries: ${tries}; Elapsed: ${getElapsed()}`); -// await delay(10); -// tries++; -// return tries > 5; -// }; -// -// tries = 0; -// before = Date.now(); -// await waitForTruthy(callback, { interval: 0, timeout: 10000 }) -// .then(() => console.log("Success passed")) -// .catch(() => console.log("Success failed")); -// -// tries = 0; -// before = Date.now(); -// await waitForTruthy(callback, { interval: 10, timeout: 10 }) -// .then(() => console.log("Timeout test failed")) -// .catch(() => console.log("Timeout test passed")); -// -// tries = 0; -// before = Date.now(); -// await waitForTruthy(callback, { maxTries: 1 }) -// .then(() => console.log("Max tries test failed")) -// .catch(() => console.log("Max tries test passed")); -// }; -// testWaitForTruthy(); diff --git a/dist/lib/logger.js b/dist/lib/logger.js deleted file mode 100644 index 80503a7..0000000 --- a/dist/lib/logger.js +++ /dev/null @@ -1,8 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -const debugMode = process.env.DEBUG; -exports.default = { - info: console.info, - debug: debugMode ? console.debug : (...p) => null, - error: console.error, -}; diff --git a/dist/lib/spawn.js b/dist/lib/spawn.js deleted file mode 100644 index 7b86027..0000000 --- a/dist/lib/spawn.js +++ /dev/null @@ -1,18 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -const child_process_1 = require("child_process"); -const logger_1 = require("./logger"); -const spawnExt = Object.assign(child_process_1.spawn, { - // child.kill() doesn't always work. Use this if it doesnt. - kill(pid) { - logger_1.default.debug("Killing PID: " + pid); - return new Promise(resolve => { - const killChild = child_process_1.spawn('pkill', ['-P', `${pid}`]); - killChild.on('exit', resolve); - killChild.on('error', (error) => { - logger_1.default.error(`killChild process errored with error ${error}`); - }); - }); - } -}); -exports.default = spawnExt; diff --git a/dist/lighthouse.js b/dist/lighthouse.js deleted file mode 100644 index 73711f1..0000000 --- a/dist/lighthouse.js +++ /dev/null @@ -1,143 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -const child_process_1 = require("child_process"); -const fs = require("fs"); -const lighthouse = require('lighthouse'); -const async_1 = require("./lib/async"); -const logger_1 = require("./lib/logger"); -let runningLock = false; -let jobCount = 0; -let chromeLaunched = 0; -chromeLauncher(); -async function lighthouseRunner(url, type = 'html') { - const reqNo = ++jobCount; - logger_1.default.info(`Queud #${reqNo}:${url}`); - await async_1.waitForTruthy(() => !runningLock, { timeout: 2 * 60 * 1000 }); - logger_1.default.info(`Running #${reqNo}:${url}`); - runningLock = true; - logger_1.default.info('1. Awaiting Chrome'); - await chromeLauncher(); - // Run lighthouse twice and skip the first, to ensure that caches are pumped for the second - const runner = () => lighthouse(url, { logLevel: 'error', output: type, onlyCategories: ['performance'], port: 9223 }); - logger_1.default.info('2. Pumping caches'); - await runner(); - logger_1.default.info('3. Running audit'); - const runnerResult = await runner(); - // `.lhr` is the Lighthouse Result as a JS object - logger_1.default.info(`Result #${reqNo}:${url}: ${runnerResult.lhr.categories.performance.score * 100}`); - runningLock = false; - return runnerResult.report; -} -exports.default = lighthouseRunner; -async function chromeLauncher() { - if (chromeLaunched++) { - await async_1.waitForTruthy(isChromeReady); - await async_1.delay(10000); - return; - } - const chromePath = await findChrome(); - logger_1.default.debug("Launching chrome with path: " + chromePath); - const child = child_process_1.spawn(chromePath, [ - '--ignore-certificate-errors', - '--remote-debugging-port=9223', - '--headless', - '--disable-gpu', - '--no-sandbox', - '--homedir=/tmp', - '--single-process', - '--data-path=/tmp/data-path', - '--disk-cache-dir=/tmp/cache-dir', - '--autoplay-policy=user-gesture-required', - '--user-data-dir=/tmp/chromium', - '--disable-web-security', - '--disable-background-networking', - '--disable-background-timer-throttling', - '--disable-backgrounding-occluded-windows', - '--disable-breakpad', - '--disable-client-side-phishing-detection', - '--disable-component-update', - '--disable-default-apps', - '--disable-dev-shm-usage', - '--disable-domain-reliability', - '--disable-extensions', - '--disable-features=AudioServiceOutOfProcess', - '--disable-hang-monitor', - '--disable-ipc-flooding-protection', - '--disable-notifications', - '--disable-offer-store-unmasked-wallet-cards', - '--disable-popup-blocking', - '--disable-print-preview', - '--disable-prompt-on-repost', - '--disable-renderer-backgrounding', - '--disable-setuid-sandbox', - '--disable-speech-api', - '--disable-sync', - '--disk-cache-size=33554432', - '--hide-scrollbars', - '--ignore-gpu-blocklist', - '--metrics-recording-only', - '--mute-audio', - '--no-default-browser-check', - '--no-first-run', - '--no-pings', - '--no-sandbox', - '--no-zygote', - '--password-store=basic', - '--use-gl=swiftshader', - '--use-mock-keychain', - ], { cwd: require('os').tmpdir(), shell: true }); - child.stdout.on('data', (data) => { - logger_1.default.debug(`stdout: ${data}`); - }); - child.stderr.on('data', (data) => { - logger_1.default.debug(`stderr: ${data}`); - }); - child.on('close', (code) => { - logger_1.default.debug(`child process closed with code ${code}`); - }); - child.on('error', (error) => { - logger_1.default.debug(`child process errored with error ${error}`); - }); - child.on('exit', (code) => { - logger_1.default.debug(`child process exited with code ${code}`); - }); - await async_1.waitForTruthy(isChromeReady, { timeout: 2 * 60 * 1000 }); - await async_1.delay(10000); - return; - function findChrome() { - var _a; - const execPath = (_a = [ - '/usr/bin/chromium-browser', - '/usr/bin/google-chrome', - '/Applications/Chromium.app/Contents/MacOS/Chromium' - ] - .filter(p => fs.existsSync(p))) === null || _a === void 0 ? void 0 : _a[0]; - if (!execPath) - throw new Error('Can\'nt find Chrome'); - return execPath; - } - async function isChromeReady() { - const net = require("net"); - return new Promise((resolve) => { - const client = net.createConnection(9223); - client.once('error', () => { - logger_1.default.debug('Chrome is not ready'); - cleanup(client); - resolve(0); - }); - client.once('connect', () => { - logger_1.default.debug('Chrome is ready'); - cleanup(client); - resolve(1); - }); - }); - function cleanup(client) { - if (client) { - client.removeAllListeners(); - client.end(); - client.destroy(); - client.unref(); - } - } - } -} diff --git a/dist/server.js b/dist/server.js deleted file mode 100644 index 6062941..0000000 --- a/dist/server.js +++ /dev/null @@ -1,24 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -const http = require("http"); -const url = require("url"); -const fs = require("fs"); -const path = require("path"); -const logger_1 = require("./lib/logger"); -const lighthouse_1 = require("./lighthouse"); -const packageJson = JSON.parse(fs.readFileSync(path.join(__dirname, '../package.json'), 'utf-8')); -http.createServer(async function (req, res) { - const { url: qUrl, type } = url.parse(req.url, true).query; - res.setHeader('Content-Type', type === 'json' ? 'application/json' : 'text/html'); - try { - res.write(await lighthouse_1.default(qUrl, type)); - } - catch (e) { - logger_1.default.error(e); - res.statusCode = 400; - res.write("Error: " + e.message); - } - res.end(); -}) - .listen(8080); -logger_1.default.info(`v${packageJson.version} listening on 8080`); diff --git a/scripts/batch-lambda.sh b/scripts/batch-lambda.sh index 1568ff8..1b92018 100755 --- a/scripts/batch-lambda.sh +++ b/scripts/batch-lambda.sh @@ -1,5 +1,8 @@ #!/bin/bash +SET_COUNT=5 # 5 seems like the magic number for not overwhelming the URL +NOW=`date '+%Y.%m.%d-%H:%M:%S'` + URL=$1 if [ -z "$URL" ]; then echo -n "URL: " @@ -14,6 +17,7 @@ if [ -z "$OUT" ]; then echo -n "Output Dir: " read OUT fi +OUT="$OUT/$NOW" # If your target server is faulty, this can help ignore the failures REDO_SERVER_FAILURES=$3 @@ -25,9 +29,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" rm -rf $OUT &> /dev/null -SET_COUNT=5 # 5 seems like the magic number for not overwhelming the URL -NOW=`date '+%Y.%m.%d-%H:%M:%S'` - audit () { local setNo=$1 local jobNo=$2 diff --git a/src/handler.ts b/src/handler.ts index a8f4746..0621bc5 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -10,7 +10,7 @@ const packageJson = JSON.parse(fs.readFileSync(path.join(__dirname, '../package. export default async function handler (event: {httpMethod: string, queryStringParameters: Record}, context: any) { logger.info(`Received event on version ${packageJson.version}: ${JSON.stringify(event)}`) - let body; + let body = ''; let statusCode = '200'; const headers = { 'Content-Type': 'application/json', diff --git a/src/lighthouse.ts b/src/lighthouse.ts index 47d5904..4d195a1 100644 --- a/src/lighthouse.ts +++ b/src/lighthouse.ts @@ -28,13 +28,16 @@ export default async function lighthouseRunner(url: string, type = 'html') { output: type as any, onlyCategories: ['performance'], port: 9223, + extraHeaders: { + ...process.env.HTTPHEADERS && JSON.parse(process.env.HTTPHEADERS) + }, }) // `.lhr` is the Lighthouse Result as a JS object logger.info(`Result #${reqNo}:${url}: ${runnerResult.lhr.categories.performance.score * 100}`) runningLock = false - return runnerResult.report + return runnerResult.report as string } async function chromeLauncher() { @@ -53,7 +56,7 @@ async function chromeLauncher() { '--disable-gpu', '--no-sandbox', '--homedir=/tmp', - // '--single-process', + '--single-process', '--data-path=/tmp/data-path', '--disk-cache-dir=/tmp/cache-dir', '--autoplay-policy=user-gesture-required', diff --git a/tsconfig.json b/tsconfig.json index 78ceb3e..e7f2635 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "module": "commonjs", - "target": "ES2020", + "target": "ES2019", "moduleResolution": "node", "baseUrl": "./", "strict": true, @@ -11,7 +11,7 @@ "allowSyntheticDefaultImports": true, "importsNotUsedAsValues": "error", "allowUnreachableCode": true, - "lib": ["ES2020", "DOM"], + "lib": ["ES2019", "DOM"], "outDir": "dist" }, "include": ["src"],