Skip to content
This repository has been archived by the owner on Jan 31, 2021. It is now read-only.

Commit

Permalink
longer timeouts
Browse files Browse the repository at this point in the history
  • Loading branch information
99littlebugs committed Jan 17, 2021
1 parent 0ddc271 commit 3523d8e
Showing 1 changed file with 4 additions and 4 deletions.
8 changes: 4 additions & 4 deletions scraper.js
Original file line number Diff line number Diff line change
Expand Up @@ -135,9 +135,9 @@ const nyGovSites = [{
try {
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.75');
await page.goto(site.link, {timeout: 10000});
await page.waitForSelector("#pagetitle, h1, #notfound", { timeout: 10000 });
await page.waitForTimeout(1000); // sometimes jquery wasn't loaded
await page.goto(site.link, {timeout: 20000});
await page.waitForSelector("#pagetitle, h1, #notfound", { timeout: 20000 });
await page.waitForTimeout(2000); // sometimes jquery wasn't loaded
const isError = await page.$("h1, #notfound");
if (!isError) {
site.events = await page.evaluate(() => {
Expand Down Expand Up @@ -170,7 +170,7 @@ const nyGovSites = [{
});
}
} catch (err) {
console.error("[" + site.link + "]: " + err); // todo: log oout to node
console.error(site.link + " " + err);
}
})())
}
Expand Down

0 comments on commit 3523d8e

Please sign in to comment.