update webcast job

r-spacex · Jun 27, 2020 · f7de5fa · f7de5fa
1 parent 1c0dc81
commit f7de5fa
Show file tree

Hide file tree

Showing 2 changed files with 67 additions and 98 deletions.
diff --git a/jobs/webcast.js b/jobs/webcast.js
@@ -1,117 +1,86 @@
 const got = require('got');
-const cheerio = require('cheerio');
 const fuzz = require('fuzzball');
 const { logger } = require('../middleware/logger');
 
-const SPACEX_WEBCAST = 'https://www.spacex.com/webcast';
+const YOUTUBE_PREFIX = 'https://youtu.be';
 const SPACEX_API = 'https://api.spacexdata.com/v4';
-const KEY = process.env.SPACEX_KEY;
-const HEALTHCHECK = process.env.WEBCAST_HEALTHCHECK;
+const CHANNEL_ID = 'UCtI0Hodo5o5dUb67FeUjDeA';
+const {
+  SPACEX_KEY,
+  YOUTUBE_KEY,
+  WEBCAST_HEALTHCHECK,
+} = process.env;
 
 /**
  * Check for new SpaceX webcast links
  * @return {Promise<void>}
  */
 module.exports = async () => {
   try {
-    const launches = await got.post(`${SPACEX_API}/launches/query`, {
-      json: {
-        query: {
-          upcoming: true,
-        },
-        options: {
-          sort: {
-            flight_number: 'asc',
-          },
-          limit: 1,
-        },
-      },
+    // Check if any upcoming streams on youtube
+    const url = `https://www.googleapis.com/youtube/v3/search?part=snippet&channelId=${CHANNEL_ID}&eventType=upcoming&maxResults=1&type=video&key=${YOUTUBE_KEY}`;
+    const upcomingStreams = await got(url, {
       resolveBodyOnly: true,
       responseType: 'json',
     });
 
-    const launchId = launches.docs[0].id;
-    const missionName = launches.docs[0].name;
-
-    // Get most recent launch youtube link
-    const previousLaunches = await got.post(`${SPACEX_API}/launches/query`, {
-      json: {
-        query: {
-          upcoming: false,
-        },
-        options: {
-          sort: {
-            flight_number: 'desc',
-          },
-          limit: 1,
-        },
-      },
-      resolveBodyOnly: true,
-      responseType: 'json',
-    });
-
-    const prevYoutubeUrl = previousLaunches.docs[0].links.webcast;
-
-    const response = await got(SPACEX_WEBCAST);
-    const $ = cheerio.load(response.body);
-
-    const embedSource = $('#content > div.left_column > font > iframe').attr('src');
-    const embedName = $('#page-title').text();
-
-    const youtubeUrl = embedSource.replace(/https:\/\/www\.youtube\.com\/embed/i, 'https://youtu.be');
-    const youtubeId = youtubeUrl.replace(/https:\/\/youtu\.be\//i, '');
-
-    const update = {
-      'links.video_link': youtubeUrl,
-      'links.youtube_id': youtubeId,
-    };
-
-    const ratio = fuzz.ratio(embedName.replace(/mission/i, ''), missionName.replace(/mission/i, ''));
-
-    // Check if most recent launch matches
-    // Prevents early triggering for extremely similar launch names
-    if (prevYoutubeUrl === youtubeUrl) {
-      logger.info({
-        rawMission: embedName,
-        apiMission: missionName,
-        update,
-        matchRatio: ratio,
-        match: false,
-        matchesRecent: true,
-      });
-      // Might need to play with this ratio, but 50% match should be good enough to
-      // reasonably assume it's the correct mission. Worst case, if it doesn't pick it
-      // up correctly, the data would be entered regardless, this script is purely for convenience
-    } else if (ratio >= 50) {
-      logger.info({
-        rawMission: embedName,
-        apiMission: missionName,
-        update,
-        matchRatio: ratio,
-        match: true,
-        matchesRecent: false,
-      });
-      await got.patch(`${SPACEX_API}/launches/${launchId}`, {
+    if (upcomingStreams?.items?.length === 1) {
+      const launches = await got.post(`${SPACEX_API}/launches/query`, {
         json: {
-          'links.webcast': youtubeUrl,
-          'links.youtube_id': youtubeId,
-        },
-        headers: {
-          'spacex-key': KEY,
+          query: {
+            upcoming: true,
+          },
+          options: {
+            sort: {
+              flight_number: 'asc',
+            },
+            limit: 1,
+          },
         },
+        resolveBodyOnly: true,
+        responseType: 'json',
       });
+      const launchId = launches.docs[0].id;
+      const missionName = launches.docs[0].name;
+      const youtubeTitle = upcomingStreams.items[0].snippet.title;
+      const youtubeId = upcomingStreams.items[0].id.videoId;
+
+      // Fuzzy check video title to make sure it's at least related to the launch
+      const ratio = fuzz.ratio(youtubeTitle, missionName);
+      if (ratio >= 50) {
+        await got.patch(`${SPACEX_API}/launches/${launchId}`, {
+          json: {
+            'links.webcast': `${YOUTUBE_PREFIX}/${youtubeId}`,
+            'links.youtube_id': youtubeId,
+          },
+          headers: {
+            'spacex-key': SPACEX_KEY,
+          },
+        });
+        logger.info({
+          rawMission: youtubeTitle,
+          apiMission: missionName,
+          url: `${YOUTUBE_PREFIX}/${youtubeId}`,
+          matchRatio: ratio,
+          match: true,
+        });
+      } else {
+        logger.info({
+          rawMission: youtubeTitle,
+          apiMission: missionName,
+          url: `${YOUTUBE_PREFIX}/${youtubeId}`,
+          matchRatio: ratio,
+          match: false,
+        });
+      }
     } else {
       logger.info({
-        rawMission: embedName,
-        apiMission: missionName,
-        update,
-        matchRatio: ratio,
         match: false,
-        matchesRecent: false,
       });
     }
-    if (HEALTHCHECK) {
-      await got(HEALTHCHECK);
+
+    if (WEBCAST_HEALTHCHECK) {
+      await got(WEBCAST_HEALTHCHECK);
     }
   } catch (error) {
     console.log(error);

diff --git a/jobs/worker.js b/jobs/worker.js
@@ -1,5 +1,4 @@
 const { CronJob } = require('cron');
-// const webcast = require('./webcast');
 const launches = require('./launches');
 const payloads = require('./payloads');
 const landpads = require('./landpads');
@@ -9,16 +8,11 @@ const cores = require('./cores');
 const roadster = require('./roadster');
 const upcoming = require('./upcoming');
 const starlink = require('./starlink');
-
-// Every 10 minutes
-// const webcastJob = new CronJob('*/10 * * * *', webcast);
+const webcast = require('./webcast');
 
 // Every 10 minutes
 const launchesJob = new CronJob('*/10 * * * *', launches);
 
-// Every hour on :20
-const payloadsJob = new CronJob('20 * * * *', payloads);
-
 // Every 10 minutes
 const landpadsJob = new CronJob('*/10 * * * *', landpads);
 
@@ -37,10 +31,15 @@ const roadsterJob = new CronJob('*/10 * * * *', roadster);
 // Every 10 minutes
 const upcomingJob = new CronJob('*/10 * * * *', upcoming);
 
+// Every hour on :25
+const payloadsJob = new CronJob('25 * * * *', payloads);
+
 // Every hour on :35
 const starlinkJob = new CronJob('35 * * * *', starlink);
 
-// webcastJob.start();
+// Every hour on :45
+const webcastJob = new CronJob('45 * * * *', webcast);
+
 launchesJob.start();
 payloadsJob.start();
 landpadsJob.start();
@@ -50,3 +49,4 @@ coresJob.start();
 roadsterJob.start();
 upcomingJob.start();
 starlinkJob.start();
+webcastJob.start();