diff --git a/data/locations.json b/data/locations.json new file mode 100644 index 0000000..19fdedb --- /dev/null +++ b/data/locations.json @@ -0,0 +1,213 @@ +{ + "sites": [ + { + "name": "NYS DOH Hospital / FQHC Scheduling COVID-19 Sutphin Health Center", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50501251", + "address": { + "name": "Sutphin Health Center", + "street": "105-04 Sutphin Boulevard", + "city": "Queens", + "state": "NY", + "zip": "11435" + } + }, + { + "name": "Northwell CoVID-19 Vaccination Program: Nassau County - Lake Success", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50500075", + "address": { + "name": "Lake Success/New Hyde Park", + "street": "1111 Marcus Avenue", + "city": "New Hyde Park", + "state": "NY", + "zip": "11042" + } + }, + { + "name": "NYS DOH Hospital/FQHC Appointment Scheduling COVID-19 at Nassau University Medical Center", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50500121", + "address": { + "name": "Nassau University Medical Center", + "street": "2201 Hempstead Turnpike", + "city": "East Meadow", + "state": "NY", + "zip": "11554" + } + }, + { + "name": "Northwell CoVID-19 Vaccination Program: Nassau County - Elmont", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50501190", + "address": { + "name": "NYRA/Belmont", + "street": "2150 Hempstead Turnpike", + "city": "Elmont", + "state": "NY", + "zip": "11053" + } + }, + { + "name": "FIRST DOSE - NYS DOH Hospital/FQHC Appointment Scheduling COVID-19 at Mount Sinai South Nassau", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50500074", + "address": { + "name": "Mount Sinai South Nassau", + "street": "One Healthy Way", + "city": "Oceanside", + "state": "NY", + "zip": "11598" + } + }, + { + "name": "NYS COVID Vaccine POD - JAVax - Manhattan, N.Y.", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50502320", + "address": { + "name": "JAVax Manhattan, N.Y.", + "street": "429 11th Avenue", + "city": "New York", + "state": "NY", + "zip": "10018" + } + }, + { + "name": "Northwell CoVID-19 Vaccination Program: Manhattan - Lenox Hill Hospital", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50501473", + "address": { + "name": "LHH Einhorn Auditorium", + "street": "131 East 76th Street", + "city": "New York", + "state": "NY", + "zip": "10021" + } + }, + { + "name": "NC Dept. of Health at Nassau Community College CCB", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50500881", + "address": { + "name": "Nassau Community College", + "street": "1 Education Drive", + "city": "Garden City", + "state": "NY", + "zip": "11530" + } + }, + { + "name": "NCDOH at Yes We Can Community Center", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50501806", + "address": { + "name": "Yes We Can Community Center", + "street": "141 Garden Street", + "city": "Westbury", + "state": "NY", + "zip": "11590" + } + }, + { + "name": "NYS COVID Vaccine POD - Jones Beach State Park - Field 3", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50502584", + "address": { + "name": "Jones Beach State Park", + "street": "1901 Ocean Parkway", + "city": "Wantagh", + "state": "NY", + "zip": "11793" + } + }, + { + "name": "NYS COVID Vaccine POD - Westchester County Center", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50502420", + "address": { + "name": "Westchester County Center", + "street": "198 Central Avenue", + "city": "White Plains", + "state": "NY", + "zip": "10607" + } + }, + { + "name": "NYS COVID Vaccine POD - Aqueduct Racetrack", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50503467", + "address": { + "name": "Aqueduct Racetrack - Racing Hall", + "street": "110-00 Rockaway Blvd", + "city": "Jamaica", + "state": "NY", + "zip": "11420" + } + }, + { + "name": "Northwell Health-GoHealth Urgent Care | Forest Hills", + "link": "https://www.clockwisemd.com/hospitals/803/appointments/schedule_visit", + "address": { + "name": "Northwell Health-GoHealth Urgent Care", + "street": "102-29 Queens Blvd", + "city": "Forest Hills", + "state": "NY", + "zip": "11375" + } + }, + { + "name": "Northwell Health-GoHealth Urgent Care | Syosset", + "link": "https://www.clockwisemd.com/hospitals/781/appointments/schedule_visit", + "address": { + "name": "Northwell Health-GoHealth Urgent Care", + "street": "103 Jackson Ave", + "city": "Syossett", + "state": "NY", + "zip": "11791" + } + }, + { + "name": "Northwell Health-GoHealth Urgent Care | Hewlett", + "link": "https://www.clockwisemd.com/hospitals/3255/appointments/schedule_visit", + "address": { + "name": "Northwell Health-GoHealth Urgent Care", + "street": "1332 Peninsula Blvd", + "city": "Hewlett", + "state": "NY", + "zip": "11557" + } + }, + { + "name": "Northwell Health-GoHealth Urgent Care | Riverhead", + "link": "https://www.clockwisemd.com/hospitals/1866/appointments/schedule_visit", + "address": { + "name": "Northwell Health-GoHealth Urgent Care", + "street": "1842 Old Country Road", + "city": "Riverhead", + "state": "NY", + "zip": "11901" + } + }, + { + "name": "01/23 FIRST DOSE Covid-19 Vaccine Westchester County Clinic", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?DateID=10025", + "address": { + "name": "White Plains DO", + "street": "134 Court Street", + "city": "White Plains", + "state": "NY", + "zip": "10601" + } + }, + { + "name": "01/22 FIRST DOSE Covid-19 Vaccine Westchester County Clinic", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?DateID=10021", + "address": { + "name": "White Plains DO", + "street": "134 Court Street", + "city": "White Plains", + "state": "NY", + "zip": "10601" + } + }, + { + "name": "Jan 21-25: Office of Aging (Over 65)", + "link": "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50503908", + "address": { + "name": "RCDOH", + "street": "50 Sanatorium Road", + "city": "Pomona", + "state": "NY", + "zip": "10970" + } + } + ] +} \ No newline at end of file diff --git a/scraper.js b/scraper.js index 3dc95bc..829511f 100644 --- a/scraper.js +++ b/scraper.js @@ -1,275 +1,20 @@ #!/usr/bin/env node -const fs = require("fs"); +const fs = require('fs'); const path = require('path'); const puppeteer = require('puppeteer'); -const nyGovSites = [ - { - name: "NYS DOH Hospital / FQHC Scheduling COVID-19 Sutphin Health Center", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50501251", - address: { - name: "Sutphin Health Center", - street: "105-04 Sutphin Boulevard", - city: "Queens", - state: "NY", - zip: "11435" - } - }, - { - name: "Northwell CoVID-19 Vaccination Program: Nassau County - Lake Success", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50500075", - address: { - name: "Lake Success/New Hyde Park", - street: "1111 Marcus Avenue", - city: "New Hyde Park", - state: "NY", - zip: "11042" - } - }, - { - name: "NYS DOH Hospital/FQHC Appointment Scheduling COVID-19 at Nassau University Medical Center", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50500121", - address: { - name: "Nassau University Medical Center", - street: "2201 Hempstead Turnpike", - city: "East Meadow", - state: "NY", - zip: "11554" - } - }, - { - name: "Northwell CoVID-19 Vaccination Program: Nassau County - Elmont", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50501190", - address: { - name: "NYRA/Belmont", - street: "2150 Hempstead Turnpike", - city: "Elmont", - state: "NY", - zip: "11053" - } - }, - { - name: "FIRST DOSE - NYS DOH Hospital/FQHC Appointment Scheduling COVID-19 at Mount Sinai South Nassau", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50500074", - address: { - name: "Mount Sinai South Nassau", - street: "One Healthy Way", - city: "Oceanside", - state: "NY", - zip: "11598" - }, - }, - { - name: "NYS COVID Vaccine POD - JAVax - Manhattan, N.Y.", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50502320", - address: { - name: "JAVax Manhattan, N.Y.", - street: "429 11th Avenue", - city: "New York", - state: "NY", - zip: "10018" - } - }, - { - name: "Northwell CoVID-19 Vaccination Program: Manhattan - Lenox Hill Hospital", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50501473", - address: { - name: "LHH Einhorn Auditorium", - street: "131 East 76th Street", - city: "New York", - state: "NY", - zip: "10021" - } - }, - { - name: "NC Dept. of Health at Nassau Community College CCB", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50500881", - address: { - name: "Nassau Community College", - street: "1 Education Drive", - city: "Garden City", - state: "NY", - zip: "11530" - } - }, - { - name: "NCDOH at Yes We Can Community Center", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50501806", - address: { - name: "Yes We Can Community Center", - street: "141 Garden Street", - city: "Westbury", - state: "NY", - zip: "11590" - } - }, - { - name: "NYS COVID Vaccine POD - Jones Beach State Park - Field 3", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50502584", - address: { - name: "Jones Beach State Park", - street: "1901 Ocean Parkway", - city: "Wantagh", - state: "NY", - zip: "11793" - } - }, - { - name: "NYS COVID Vaccine POD - Westchester County Center", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50502420", - address: { - name: "Westchester County Center", - street: "198 Central Avenue", - city: "White Plains", - state: "NY", - zip: "10607" - } - }, - { - name: "NYS COVID Vaccine POD - Aqueduct Racetrack", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50503467", - address: { - name: "Aqueduct Racetrack - Racing Hall", - street: "110-00 Rockaway Blvd", - city: "Jamaica", - state: "NY", - zip: "11420" - } - }, - { - name: "Northwell Health-GoHealth Urgent Care | Forest Hills", - link: "https://www.clockwisemd.com/hospitals/803/appointments/schedule_visit", - address: { - name: "Northwell Health-GoHealth Urgent Care", - street: "102-29 Queens Blvd", - city: "Forest Hills", - state: "NY", - zip: "11375" - } - }, - { - name: "Northwell Health-GoHealth Urgent Care | Syosset", - link: "https://www.clockwisemd.com/hospitals/781/appointments/schedule_visit", - address: { - name: "Northwell Health-GoHealth Urgent Care", - street: "103 Jackson Ave", - city: "Syossett", - state: "NY", - zip: "11791" - } - }, - { - name: "Northwell Health-GoHealth Urgent Care | Hewlett", - link: "https://www.clockwisemd.com/hospitals/3255/appointments/schedule_visit", - address: { - name: "Northwell Health-GoHealth Urgent Care", - street: "1332 Peninsula Blvd", - city: "Hewlett", - state: "NY", - zip: "11557" - } - }, - { - name: "Northwell Health-GoHealth Urgent Care | Riverhead", - link: "https://www.clockwisemd.com/hospitals/1866/appointments/schedule_visit", - address: { - name: "Northwell Health-GoHealth Urgent Care", - street: "1842 Old Country Road", - city: "Riverhead", - state: "NY", - zip: "11901" - } - }, - { - name: "Northwell Health-GoHealth Urgent Care | Eltingville", - link: "https://www.clockwisemd.com/hospitals/2356/appointments/schedule_visit", - address: { - name: "Northwell Health-GoHealth Urgent Care", - street: "4316 Amboy Road", - city: "Staten Island", - state: "NY", - zip: "13012" - } - }, - { - name: "Northwell Health-GoHealth Urgent Care | W. 69th", - link: "https://www.clockwisemd.com/hospitals/808/appointments/schedule_visit", - address: { - name: "Northwell Health-GoHealth Urgent Care", - street: "199 Amsterdam Ave", - city: "New York", - state: "NY", - zip: "10023" - } - }, - { - name: "Northwell Health-GoHealth Urgent Care | New Rochelle", - link: "https://www.clockwisemd.com/hospitals/814/appointments/schedule_visit", - address: { - name: "Northwell Health-GoHealth Urgent Care", - street: "77 Quaker Ridge Rd", - city: "New Rochelle", - state: "NY", - zip: "10804" - } - }, - { - name: "Northwell Health-GoHealth Urgent Care | Huntington", - link: "https://www.clockwisemd.com/hospitals/4141/appointments/schedule_visit", - address: { - name: "Northwell Health-GoHealth Urgent Care", - street: "241 East Main St. Suite 1A", - city: "Huntington", - state: "NY", - zip: "11743" - } - }, - { - name: "01/23 FIRST DOSE Covid-19 Vaccine Westchester County Clinic", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?DateID=10025", - address: { - name: "White Plains DO", - street: "134 Court Street", - city: "White Plains", - state: "NY", - zip: "10601" - } - }, - { - name: "01/22 FIRST DOSE Covid-19 Vaccine Westchester County Clinic", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?DateID=10021", - address: { - name: "White Plains DO", - street: "134 Court Street", - city: "White Plains", - state: "NY", - zip: "10601" - } - }, - { - name: "Jan 21-25: Office of Aging (Over 65)", - link: "https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties?OpID=50503908", - address: { - name: "RCDOH", - street: "50 Sanatorium Road", - city: "Pomona", - state: "NY", - zip: "10970" - } - } -]; - +const locations = JSON.parse(fs.readFileSync(path.join('data', 'locations.json')).toString('utf8')); (async () => { const browser = await puppeteer.launch(); const promises = []; - for (const site of nyGovSites) { + for (const site of locations.sites) { promises.push((async () => { site.events = []; const page = await browser.newPage(); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.75'); - await page.exposeFunction("getApproxAppointmentCount", getApproxAppointmentCount); + await page.exposeFunction('getApproxAppointmentCount', getApproxAppointmentCount); // let one retry and use default timeouts of 30s for (let i = 0; i < 2; i++) { @@ -291,39 +36,39 @@ const nyGovSites = [ await Promise.all(promises); await browser.close(); - const data = JSON.stringify({ sites: nyGovSites }, null, 2); - const filename = path.join("data", "ny_state.json"); + const data = JSON.stringify({ sites: locations.sites }, null, 2); + const filename = path.join('data', 'ny_state.json'); fs.writeFileSync(path.resolve(filename), data); })(); async function waitForPageLoad(page) { - if (page.url().startsWith("https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties")) { - return await page.waitForSelector("#pagetitle, h1, #notfound"); + if (page.url().startsWith('https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties')) { + return await page.waitForSelector('#pagetitle, h1, #notfound'); } - else if (page.url().startsWith("https://www.clockwisemd.com/hospitals/")) { + else if (page.url().startsWith('https://www.clockwisemd.com/hospitals/')) { // https://github.com/puppeteer/puppeteer/issues/4356 // https://stackoverflow.com/a/61917280 return await Promise.race([ - page.waitForSelector("#reason-and-provider-container", { visible: true }).catch(), - page.waitForSelector(".time-selector", { visible: true }).catch(), + page.waitForSelector('#reason-and-provider-container', { visible: true }).catch(), + page.waitForSelector('.time-selector', { visible: true }).catch(), ]); } } function getSucccessSelectorForSite(siteLink) { - if (siteLink.startsWith("https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties")) { - return "#pagetitle"; + if (siteLink.startsWith('https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties')) { + return '#pagetitle'; } - else if (siteLink.startsWith("https://www.clockwisemd.com/hospitals/")) { - return "#available-times-table div"; + else if (siteLink.startsWith('https://www.clockwisemd.com/hospitals/')) { + return '#available-times-table div'; } } function getEventsForSite(siteLink) { - if (siteLink.startsWith("https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties")) { + if (siteLink.startsWith('https://apps3.health.ny.gov/doh2/applinks/cdmspr/2/counties')) { return getEventsForNYGovSites; } - else if (siteLink.startsWith("https://www.clockwisemd.com/hospitals/")) { + else if (siteLink.startsWith('https://www.clockwisemd.com/hospitals/')) { return getEventsForNorthwell; } } @@ -341,16 +86,16 @@ function getApproxAppointmentCount(appointmentCount) { async function getEventsForNYGovSites() { const events = []; - const eventsWeb = $(".event-type"); + const eventsWeb = $('.event-type'); for (const event of eventsWeb) { - const isAvailable = $(event).find("button").text() !== "Event Full"; - var approxAppointmentCount = await window.getApproxAppointmentCount($(event).find("div div:contains('Appointments Available:'):last").first().text().substring(24)); + const isAvailable = $(event).find('button').text() !== 'Event Full'; + var approxAppointmentCount = await window.getApproxAppointmentCount($(event).find('div div:contains("Appointments Available:"):last').first().text().substring(24)); if (isAvailable && approxAppointmentCount > 0) { events.push({ - date: new Date($(event).find("div div:contains('Date:'):last").first().text().substring(6) + " UTC").toISOString(), - time: $(event).find("div div:contains('Time:'):last").first().text().substring(6), + date: new Date($(event).find('div div:contains("Date:"):last').first().text().substring(6) + ' UTC').toISOString(), + time: $(event).find('div div:contains("Time:"):last').first().text().substring(6), appointments: approxAppointmentCount, - linkId: $(event).parent().attr("id") + linkId: $(event).parent().attr('id') }); } } @@ -358,22 +103,21 @@ async function getEventsForNYGovSites() { } async function getEventsForNorthwell(){ - console.log("in northwell") const events = []; for(let i = 0; i < 5; i++) { - const eventsWeb = $(".times-column"); + const eventsWeb = $('.times-column'); for (const event of eventsWeb) { - const isAvailable = $(event).find(".clockwise-time").length > 0; - var approxAppointmentCount = await window.getApproxAppointmentCount($(event).find(".clockwise-time").length); + const isAvailable = $(event).find('.clockwise-time').length > 0; + var approxAppointmentCount = await window.getApproxAppointmentCount($(event).find('.clockwise-time').length); if (isAvailable && approxAppointmentCount > 0) { events.push({ - date: new Date($(event).find(".clockwise-time").first().data("value").substring.substring(0, 10)).toISOString(), - time: $(event).find(".clockwise-time").first().text() + " - " + $(event).find(".clockwise-time").last().text(), + date: new Date($(event).find('.clockwise-time').first().data('value').substring.substring(0, 10)).toISOString(), + time: $(event).find('.clockwise-time').first().text() + ' - ' + $(event).find('.clockwise-time').last().text(), appointments: approxAppointmentCount }); } } - $("#later-dates").click(); // week by week + $('#later-dates').click(); // week by week } return events; } \ No newline at end of file