Skip to content
This repository has been archived by the owner on Jan 31, 2021. It is now read-only.

Commit

Permalink
northwell urgent cares
Browse files Browse the repository at this point in the history
  • Loading branch information
@99littlebugs
99littlebugs committed Jan 19, 2021
1 parent c38823e commit 0cf6f01
Show file tree
Hide file tree
Showing 2 changed files with 136 additions and 35 deletions.
169 changes: 135 additions & 34 deletions scraper.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,54 @@ const nyGovSites = [
state: "NY",
zip: "11420"
}
},
{
name: "Northwell Health-GoHealth Urgent Care | Forest Hills",
link: "https://www.clockwisemd.com/hospitals/803/appointments/schedule_visit",
address: {
name: "Northwell Health-GoHealth Urgent Care",
street: "102-29 Queens Blvd",
city: "Forest Hills",
state: "NY",
zip: "11375"
}
},
{
name: "Northwell Health-GoHealth Urgent Care | Syosset",
link: "https://www.clockwisemd.com/hospitals/781/appointments/schedule_visit",
address: {
name: "Northwell Health-GoHealth Urgent Care",
street: "103 Jackson Ave",
city: "Syossett",
state: "NY",
zip: "11791"
}
},
{
name: "Northwell Health-GoHealth Urgent Care | Hewlett",
link: "https://www.clockwisemd.com/hospitals/3255/appointments/schedule_visit",
address: {
name: "Northwell Health-GoHealth Urgent Care",
street: "1332 Peninsula Blvd",
city: "Hewlett",
state: "NY",
zip: "11557"
}
},
{
name: "Northwell Health-GoHealth Urgent Care | Riverhead",
link: "https://www.clockwisemd.com/hospitals/1866/appointments/schedule_visit",
address: {
name: "Northwell Health-GoHealth Urgent Care",
street: "1842 Old Country Road",
city: "Riverhead",
state: "NY",
zip: "11901"
}
}
];


(async () => {
const browser = await puppeteer.launch();
const promises = [];
Expand All @@ -147,45 +192,19 @@ const nyGovSites = [
site.events = [];
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.75');
await page.exposeFunction("getApproxAppointmentCount", getApproxAppointmentCount);

// let one retry and use default timeouts of 30s
for (let i = 0; i < 2; i++) {
try {
try {
await page.goto(site.link);
await page.waitForSelector("#pagetitle, h1, #notfound");
const isError = await page.$("h1, #notfound");
if (!isError) {
site.events = await page.evaluate(() => {
const events = [];
const eventsWeb = $(".event-type");
for (const event of eventsWeb) {
const isAvailable = $(event).find("button").text() !== "Event Full";
var approxAppointmentCount = getApproxAppointmentCount($(event).find("div div:contains('Appointments Available:'):last").first().text().substring(24));
if (isAvailable && approxAppointmentCount > 0) {
events.push({
date: new Date($(event).find("div div:contains('Date:'):last").first().text().substring(6) + " UTC").toISOString(),
time: $(event).find("div div:contains('Time:'):last").first().text().substring(6),
appointments: approxAppointmentCount,
linkId: $(event).parent().attr("id")
});
}
}
return events;

function getApproxAppointmentCount(appointmentCount) {
const actualAppointmentCount = parseInt(appointmentCount);
const buckets = [0, 5, 10, 25, 50, 100, 150, 200, 250]
for (let i = 1; i < buckets.length; i++) {
if (actualAppointmentCount < buckets[i]) {
return buckets[i - 1];
}
}
return buckets[buckets.length - 1];
}
});
await waitForPageLoad(page);
const isSuccess = await page.$(getSucccessSelectorForSite(site.link));
if (isSuccess) {
site.events = await page.evaluate(getEventsForSite(site.link));
await page.close();
break; // if succesful, don't retry
}
await page.close();
break; // if succesful, don't retry
} catch (err) {
console.error(`try: [${i}] ${site.link} ${err}`);
}
Expand All @@ -199,3 +218,85 @@ const nyGovSites = [
const filename = path.join("data", "ny_state.json");
fs.writeFileSync(path.resolve(filename), data);
})();

async function waitForPageLoad(page) {
if (page.url().startsWith("https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties")) {
return await page.waitForSelector("#pagetitle, h1, #notfound");
}
else if (page.url().startsWith("https://www.clockwisemd.com/hospitals/")) {
// https://github.com/puppeteer/puppeteer/issues/4356
// https://stackoverflow.com/a/61917280
return await Promise.race([
page.waitForSelector("#reason-and-provider-container", { visible: true }).catch(),
page.waitForSelector(".time-selector", { visible: true }).catch(),
]);
}
}

function getSucccessSelectorForSite(siteLink) {
if (siteLink.startsWith("https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties")) {
return "#pagetitle";
}
else if (siteLink.startsWith("https://www.clockwisemd.com/hospitals/")) {
return "#available-times-table div";
}
}

function getEventsForSite(siteLink) {
if (siteLink.startsWith("https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties")) {
return getEventsForNYGovSites;
}
else if (siteLink.startsWith("https://www.clockwisemd.com/hospitals/")) {
return getEventsForNorthwell;
}
}

function getApproxAppointmentCount(appointmentCount) {
const actualAppointmentCount = parseInt(appointmentCount);
const buckets = [0, 5, 10, 25, 50, 100, 150, 200, 250]
for (let i = 1; i < buckets.length; i++) {
if (actualAppointmentCount < buckets[i]) {
return buckets[i - 1];
}
}
return buckets[buckets.length - 1];
}

async function getEventsForNYGovSites() {
const events = [];
const eventsWeb = $(".event-type");
for (const event of eventsWeb) {
const isAvailable = $(event).find("button").text() !== "Event Full";
var approxAppointmentCount = await window.getApproxAppointmentCount($(event).find("div div:contains('Appointments Available:'):last").first().text().substring(24));
if (isAvailable && approxAppointmentCount > 0) {
events.push({
date: new Date($(event).find("div div:contains('Date:'):last").first().text().substring(6) + " UTC").toISOString(),
time: $(event).find("div div:contains('Time:'):last").first().text().substring(6),
appointments: approxAppointmentCount,
linkId: $(event).parent().attr("id")
});
}
}
return events;
}

async function getEventsForNorthwell(){
console.log("in northwell")
const events = [];
for(let i = 0; i < 5; i++) {
const eventsWeb = $(".times-column");
for (const event of eventsWeb) {
const isAvailable = $(event).find(".clockwise-time").length > 0;
var approxAppointmentCount = await window.getApproxAppointmentCount($(event).find(".clockwise-time").length);
if (isAvailable && approxAppointmentCount > 0) {
events.push({
date: new Date($(event).find(".clockwise-time").first().data("value").substring.substring(0, 10)).toISOString(),
time: $(event).find(".clockwise-time").first().text() + " - " + $(event).find(".clockwise-time").last().text(),
appointments: approxAppointmentCount
});
}
}
$("#later-dates").click(); // week by week
}
return events;
}
2 changes: 1 addition & 1 deletion site/index.hbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@
<td>{{date date}}</td>
<td>{{time}}</td>
<td>{{appointments}}</td>
<td><a target="_blank" class="u-pull-right button button-primary" href="{{site.link}}#{{linkId}}">Book</a>
<td><a target="_blank" class="u-pull-right button button-primary" href="{{site.link}}{{#if linkId}}#{{linkId}}{{/if}}">Book</a>
</td>
</tr>
{{else}}
Expand Down

0 comments on commit 0cf6f01

Please sign in to comment.