From 5106baab49abb7ed960c103fe0e21c3f429d4776 Mon Sep 17 00:00:00 2001 From: "Chris K.Y. FUNG" <8746768+chriskyfung@users.noreply.github.com> Date: Sun, 12 Sep 2021 14:18:11 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E2=99=BB=F0=9F=8F=97Rebuild=20Badge=20Upda?= =?UTF-8?q?te=20Func.=20with=20Class=20DriveApp?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gascript/code.gs | 156 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 121 insertions(+), 35 deletions(-) diff --git a/gascript/code.gs b/gascript/code.gs index d31c0f4..e0b3be9 100644 --- a/gascript/code.gs +++ b/gascript/code.gs @@ -16,14 +16,23 @@ var X_IG_WWW_CLAIM = ''; var isDebug = false; +/** + * Compose the URL and the query string to the Instagram's API endpoint + */ function getQuery(ig_user_id){ return "https://i.instagram.com/api/v1/feed/reels_media/?reel_ids=" + ig_user_id; } +/** + * Fetch data from the Instagram's API with a custom header. + * @param {string} query - The URL and the query string to the API endpoint. + * @returns {Object} The content of an HTTP response encoded as a string. + */ function fetch_ig_stories(query){ var opt3 = { "headers": { "accept": "*/*", + "accept-language": "zh-HK,zh;q=0.9,en-HK;q=0.8,en;q=0.7,ja-JP;q=0.6,ja;q=0.5,en-US;q=0.4,zh-TW;q=0.3", "cache-control": "no-cache", "pragma": "no-cache", "sec-fetch-dest": "empty", @@ -39,21 +48,27 @@ function fetch_ig_stories(query){ "method": "GET", "mode": "cors" }; - var data = UrlFetchApp.fetch(query, opt3).getContentText(); - if (isDebug) { Logger.log(data); } - return data; + var response = UrlFetchApp.fetch(query, opt3).getContentText(); + if (isDebug) { Logger.log(response); } + return response; } -function parseDownloadUrl(html,isTest){ - if (html == '{"reels": {}, "reels_media": [], "status": "ok"}') { +/** + * Resolve the URLs of downloadable media files from a textual HTTP response. + * @param {string} rawdata - The JSON string contains the data retrieved from Instagram's API. + * @param {boolen} isTest - Set true to run in Test mode without changing the log file. + * @returns {string[]} The URLs of downloadable media files. + */ +function parseDownloadUrl(response,isTest){ + if (response == '{"reels":{},"reels_media":[],"status":"ok"}') { return []; }; if (isTest) { - var data = JSON.parse(html); + var data = JSON.parse(response); } else { var fetchContentlog = DocumentApp.openById(fetchContentLog_id); fetchContentlog.clear(); - fetchContentlog.appendParagraph(html); + fetchContentlog.appendParagraph(response); var body = fetchContentlog.getBody().getText(); var data = JSON.parse(body); } @@ -69,10 +84,19 @@ function parseDownloadUrl(html,isTest){ urls[i] = item.image_versions2.candidates[0].url; } } - if (isDebug) { Logger.log(urls); } + //Logger.log(urls); return urls; } +/** + * Main function of this Apps Script. + * Append logs to the specified Goolge Doc file. + * Fetch data from Instagram's API using fetch_ig_stories(). + * Resolve the URLs of downloadable media files from the retrieved data using parseDownloadUrl(). + * Upload the media files to Google Drive if any URLs do not exist in the log file. + * @param {Object} targetIgUser - A JSON object contains the name and id of an Instagram account, e.g. { "name": "john", "id": "1234567890" }. + * @returns {string} The log messages. + */ function __main__(targetIgUser) { // Access the body of the log google doc file @@ -84,12 +108,12 @@ function __main__(targetIgUser) { // Fetch URL var query_url = getQuery(targetIgUser.id); - var html = fetch_ig_stories(query_url); - var urls = parseDownloadUrl(html,false); + var response = fetch_ig_stories(query_url); + var urls = parseDownloadUrl(response,false); var msg = ''; - // For each article element found in the HTML + // For each article element found in the HTML response if (urls != null && urls.length > 0) { for (i in urls) { var url = urls[i]; @@ -108,7 +132,7 @@ function __main__(targetIgUser) { msg = 'File has already been fetched once.' } lastlogfile.getBody().appendParagraph('\n\t' + msg); // log the upload msg to the Doc file - Logger.log('msg'); + Logger.log(msg); } } var logtxt = lastlogfile.getBody().getText(); @@ -118,7 +142,12 @@ function __main__(targetIgUser) { return logtxt; } -function test_fucntion (targetIgUser) { +/** + * Test getting the URLs of media files in the data retrieved from Instagram's API using fetch_ig_stories() and parseDownloadUrl(). + * @param {Object} targetIgUser - A JSON object contains the name and id of an Instagram account, e.g. { "name": "john", "id": "1234567890" }. + * @returns {number} The number of URLs obtained. + */ +function test_fucntion(targetIgUser) { var query_url = getQuery(targetIgUser.id); var html = fetch_ig_stories(query_url); @@ -127,37 +156,47 @@ function test_fucntion (targetIgUser) { return urls.length; } -function test_nasa_ig(){ - return test_fucntion({ "name": "nasa", "id":"528817151"}); +/** Run test_function() with BBC News's instagram account */ +function test_bbcnews_ig(){ + return test_fucntion({ "name": "bbcnews", "id":"16278726"}); } - +/** Run test_function() with CNN's instagram account */ +function test_cnn_ig(){ + return test_fucntion({ "name": "cnn", "id":"217723373"}); +} +/** Run test_function() with Medium's instagram account */ function test_medium_ig(){ return test_fucntion({ "name": "medium", "id":"1112881921"}); } - -function test_bbcnews_ig(){ - return test_fucntion({ "name": "bbcnews", "id":"16278726"}); +/** Run test_function() with NASA's instagram account */ +function test_nasa_ig(){ + return test_fucntion({ "name": "nasa", "id":"528817151"}); } +/** + * Update "test date" and "test status" Badges. + * Send a report email to if no downloadable URLs are obtained from the tested Instagram accounts + * @returns {boolean} + */ function test_pipeline(){ - updateLastTestedBadge(); - if (test_bbcnews_ig() == 0) { - if (test_nasa_ig() == 0) { - if (test_medium_ig() == 0) { - MailApp.sendEmail(crashReportEmail, - "Google Apps Script [AutoFetcher-IG-Stories-to-GDrive] Crash reporter", - "Get none urls from the test accounts.\nPlease verify the service websites and check update on https://bit.ly/2zQLd6p."); - Logger.log('Failed to fetch data from the test accounts!!!') - updateBadgeToFailStatus(); - return false; - }; - }; + setTestDateBadge(); + if (test_bbcnews_ig() == 0 && test_cnn_ig() == 0 && test_medium_ig() == 0 && test_nasa_ig() == 0) { + MailApp.sendEmail(crashReportEmail, + "Google Apps Script [AutoFetcher-IG-Stories-to-GDrive] Crash reporter", + "Get none urls from the test accounts.\nPlease verify the service websites and check update on https://bit.ly/2zQLd6p."); + Logger.log('Failed to fetch data from the test accounts!!!') + setTestStatusBadge(status='failed') + return false; }; Logger.log('Successfully fetch data from the test accounts!') - updateBadgeToSucessStatus(); + setTestStatusBadge(status='passed') return true; } +/** + * DEPRECATED + * Update the SVG file of "Test Status" badge to "Failed" via Drive API v3 + */ function updateBadgeToFailStatus() { if (statusBadge_id != '') { var fileDescr = Drive.Files.get(statusBadge_id).getDescription(); @@ -173,6 +212,10 @@ function updateBadgeToFailStatus() { }; } +/** + * DEPRECATED + * Update the SVG file of "Test Status" badge to "Passed" via Drive API v3 + */ function updateBadgeToSucessStatus() { if (statusBadge_id != '') { var fileDescr = Drive.Files.get(statusBadge_id).getDescription(); @@ -188,6 +231,24 @@ function updateBadgeToSucessStatus() { }; } +/** + * Update the SVG file of "Test Status" badge to "Failed" using Class DriveApp. + * @param {string} status - The arguement to determine the badge color and the text to display in the badge. + * @returns {string|null} The URL that can be used to download the file. Otherwise, returns null. + */ +function setTestStatusBadge(status='failed') { + if (statusBadge_id != '') { + const color = status == 'passed' ? '#4c1' : '#f00'; + const file = DriveApp.getFileById(statusBadge_id); + return DriveApp.getFileByIdAndResourceKey(statusBadge_id, file.getResourceKey()).setContent(`ig fetch test: ${status}ig fetch test${status}`).setDescription(`test-${status}`).getDownloadUrl(); + } + return null; +} + +/** + * DEPRECATED + * Update the SVG file of "Test Date" badge via Drive API v3 + */ function updateLastTestedBadge() { if (lastTestedBadge_id != '') { var formattedDate = Utilities.formatDate(new Date(), "GMT", "MMM dd, YYYY"); @@ -203,13 +264,31 @@ function updateLastTestedBadge() { }; } +/** + * Update the SVG file of "last tested" badge using Class DriveApp + * @returns {string|null} The URL that can be used to download the file. Otherwise, returns null. + */ +function setTestDateBadge() { + if (lastTestedBadge_id != '') { + const formattedDate = Utilities.formatDate(new Date(), "GMT", "MMM dd, YYYY"); + const file = DriveApp.getFileById(lastTestedBadge_id); + return DriveApp.getFileByIdAndResourceKey(lastTestedBadge_id, file.getResourceKey()).setContent(`last tested: ${formattedDate}last tested${formattedDate}`).getDownloadUrl(); + } + return null; +} + +/** + * Handle all HTTP GET requests made to the web app URL. + * @param {Object} e - An event object containing request parameters, including the username and password for simple security check and the id and name of an Instagram account. + * @returns {string} The log messages. + */ function doGet(e) { var usr = ''; var pwd = ''; var target = ''; - // parse query + // parse the username, password, and targeted Instagrm account information try { usr = e.parameter.usr.trim(); pwd = e.parameter.pwd.trim(); @@ -221,17 +300,24 @@ function doGet(e) { var msg = ''; - // execute if correct username and password in the query + /** + * Run __main__() if the request made with valid username, password, target parameters. + * Send the textual log or error message as the HTML response. + */ if (usr == g_username && pwd == g_password && target != '') { msg = __main__(target); msg = ContentService.createTextOutput(msg); } else { msg = ContentService.createTextOutput('Invalid username and password or targetID!'); Logger.log(msg); - }; + }; + return msg; } +/** + * Test doGet() with targeting NASA instagram stories + */ function try_get() { var e = { "parameter": { From f6b200ecdd0c03019f9d4908f365ed8520959c51 Mon Sep 17 00:00:00 2001 From: "Chris K.Y. FUNG" <8746768+chriskyfung@users.noreply.github.com> Date: Sun, 12 Sep 2021 14:28:37 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=9A=AERemove=20Function?= =?UTF-8?q?s=20With=20Drive=20API=20v3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gascript/code.gs | 57 ------------------------------------------------ 1 file changed, 57 deletions(-) diff --git a/gascript/code.gs b/gascript/code.gs index e0b3be9..314002b 100644 --- a/gascript/code.gs +++ b/gascript/code.gs @@ -193,44 +193,6 @@ function test_pipeline(){ return true; } -/** - * DEPRECATED - * Update the SVG file of "Test Status" badge to "Failed" via Drive API v3 - */ -function updateBadgeToFailStatus() { - if (statusBadge_id != '') { - var fileDescr = Drive.Files.get(statusBadge_id).getDescription(); - if (fileDescr != 'test-failed') { - var newBadge = UrlFetchApp.fetch("https://img.shields.io/badge/ig%20fetch%20test-failed-red"); - var badgeBlob = newBadge.getBlob(); - var file_meta = { - mimeType: 'image/svg+xml', - description: 'test-failed' - }; - var updatedFile = Drive.Files.update(file_meta, statusBadge_id,badgeBlob); - }; - }; -} - -/** - * DEPRECATED - * Update the SVG file of "Test Status" badge to "Passed" via Drive API v3 - */ -function updateBadgeToSucessStatus() { - if (statusBadge_id != '') { - var fileDescr = Drive.Files.get(statusBadge_id).getDescription(); - if (fileDescr != 'test-passed') { - var newBadge = UrlFetchApp.fetch("https://img.shields.io/badge/ig%20fetch%20test-passed-brightgreen"); - var badgeBlob = newBadge.getBlob(); - var file_meta = { - mimeType: 'image/svg+xml', - description: 'test-passed' - }; - var updatedFile = Drive.Files.update(file_meta, statusBadge_id,badgeBlob); - }; - }; -} - /** * Update the SVG file of "Test Status" badge to "Failed" using Class DriveApp. * @param {string} status - The arguement to determine the badge color and the text to display in the badge. @@ -245,25 +207,6 @@ function setTestStatusBadge(status='failed') { return null; } -/** - * DEPRECATED - * Update the SVG file of "Test Date" badge via Drive API v3 - */ -function updateLastTestedBadge() { - if (lastTestedBadge_id != '') { - var formattedDate = Utilities.formatDate(new Date(), "GMT", "MMM dd, YYYY"); - var webSafeFormattedDate = formattedDate.replace(/\s/g, '%20').replace(/,/g, '%2C'); - Logger.log(formattedDate) ; - var newBadge = UrlFetchApp.fetch("https://img.shields.io/badge/last%20tested-" + webSafeFormattedDate + "-orange"); - var badgeBlob = newBadge.getBlob(); - var file_meta = { - mimeType: 'image/svg+xml', - description: 'last test badge' - }; - var updatedFile = Drive.Files.update(file_meta, lastTestedBadge_id,badgeBlob); - }; -} - /** * Update the SVG file of "last tested" badge using Class DriveApp * @returns {string|null} The URL that can be used to download the file. Otherwise, returns null. From 03b858627948cac9864cfd67ca6c87f20a029b45 Mon Sep 17 00:00:00 2001 From: "Chris K.Y. FUNG" <8746768+chriskyfung@users.noreply.github.com> Date: Sun, 12 Sep 2021 14:37:22 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=9A=AE=20Remove=20Unused=20Scri?= =?UTF-8?q?pts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gascript/parseHtml.gs | 53 ------------------------------------------- gascript/test.gs | 16 ------------- 2 files changed, 69 deletions(-) delete mode 100644 gascript/parseHtml.gs delete mode 100644 gascript/test.gs diff --git a/gascript/parseHtml.gs b/gascript/parseHtml.gs deleted file mode 100644 index 765b4e6..0000000 --- a/gascript/parseHtml.gs +++ /dev/null @@ -1,53 +0,0 @@ -// References: -// Parsing HTML - Google Apps Script Examples -// https://sites.google.com/site/scriptsexamples/learn-by-example/parsing-html - -// Extract HTML elements by tag name -function getElementsByTagName(element, tagName) { - var data = []; - var descendants = element.getDescendants(); - for(i in descendants) { - var elt = descendants[i].asElement(); - if( elt !=null && elt.getName()== tagName) data.push(elt); - } - return data; -} - -// Extract HTML elements by class name -function getElementsByClassName(element, classToFind) { - var data = []; - var descendants = element.getDescendants(); - descendants.push(element); - for(i in descendants) { - var elt = descendants[i].asElement(); - if(elt != null) { - var classes = elt.getAttribute('class'); - if(classes != null) { - classes = classes.getValue(); - if(classes == classToFind) data.push(elt); - else { - classes = classes.split(' '); - for(j in classes) { - if(classes[j] == classToFind) { - data.push(elt); - break; - } - } - } - } - } - } - return data; -} - -// Extract HTML elements by ID -function getElementById(element, idToFind) { - var descendants = element.getDescendants(); - for(i in descendants) { - var elt = descendants[i].asElement(); - if( elt !=null) { - var id = elt.getAttribute('id'); - if( id !=null && id.getValue()== idToFind) return elt; - } - } -} \ No newline at end of file diff --git a/gascript/test.gs b/gascript/test.gs deleted file mode 100644 index 9f07c61..0000000 --- a/gascript/test.gs +++ /dev/null @@ -1,16 +0,0 @@ -function PicasaGetAlbums() { - var albums = PicasaApp.getAlbums(); - var output = ''; - for(i in albums) { - var title = albums[i].getTitle(); - output += title + '
'; - } -} - -function createAndSendDocument() { - // Create a new Google Doc named 'Hello, world!' - var doc = DocumentApp.create('Hello, world!'); - - // Access the body of the document, then add a paragraph. - doc.getBody().appendParagraph('This document was created by Google Apps Script.'); -} \ No newline at end of file From c7f5e1fb5705bcc3236012836e7be6420b17a6c3 Mon Sep 17 00:00:00 2001 From: "Chris K.Y. FUNG" <8746768+chriskyfung@users.noreply.github.com> Date: Sun, 12 Sep 2021 15:20:02 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E2=99=BB=F0=9F=93=84=20Split=20Code=20to?= =?UTF-8?q?=20Multiple=20Script=20Files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +++ gascript/code.gs | 63 +++++++++--------------------------------------- gascript/test.gs | 62 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 51 deletions(-) create mode 100644 gascript/test.gs diff --git a/README.md b/README.md index bc1a70b..56be5bd 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,10 @@ For Build 2020.06.05, a new function called `test_pipeline()` has been added to ## History +🚧 **AN IMPORTANT UPDATE ON 2021-09-12** 🚧 + +([#17](https://github.com/chriskyfung/AutoFetcher-IG-Stories-to-GDrive/issues/17)) Google Drive Drive will apply a security update on September 13, 2021. Please update your Apps Script code to avoid failing access to Google Drive files. + 🚧 **AN IMPORTANT UPDATE ON 2020-12-09** 🚧 ([#11](https://github.com/chriskyfung/AutoFetcher-IG-Stories-to-GDrive/issues/11)) Instagram changed code around noon, 7 Dec, UTC. Please update to Build 2020.12.09. diff --git a/gascript/code.gs b/gascript/code.gs index 314002b..cc83f0f 100644 --- a/gascript/code.gs +++ b/gascript/code.gs @@ -1,3 +1,15 @@ +/** + * Copyright (c) 2018-2021 + * + * A Google Apps Script for deploying a web application that automatically fetches the latest available IG Stories of a target Instagram user to your Google Drive. + * + * @summary short description for the file + * @author Chris K.Y. Fung + * + * Created at : 2018-01-29 + * Last modified : 2021-09-12 + */ + // User-defined Variables var g_username = 'your username for this app'; var g_password = 'your password for this app'; @@ -142,57 +154,6 @@ function __main__(targetIgUser) { return logtxt; } -/** - * Test getting the URLs of media files in the data retrieved from Instagram's API using fetch_ig_stories() and parseDownloadUrl(). - * @param {Object} targetIgUser - A JSON object contains the name and id of an Instagram account, e.g. { "name": "john", "id": "1234567890" }. - * @returns {number} The number of URLs obtained. - */ -function test_fucntion(targetIgUser) { - var query_url = getQuery(targetIgUser.id); - var html = fetch_ig_stories(query_url); - - var urls = parseDownloadUrl(html,true); - Logger.log("Number of URL(s) from @" + targetIgUser.id + ": " + urls.length); - return urls.length; -} - -/** Run test_function() with BBC News's instagram account */ -function test_bbcnews_ig(){ - return test_fucntion({ "name": "bbcnews", "id":"16278726"}); -} -/** Run test_function() with CNN's instagram account */ -function test_cnn_ig(){ - return test_fucntion({ "name": "cnn", "id":"217723373"}); -} -/** Run test_function() with Medium's instagram account */ -function test_medium_ig(){ - return test_fucntion({ "name": "medium", "id":"1112881921"}); -} -/** Run test_function() with NASA's instagram account */ -function test_nasa_ig(){ - return test_fucntion({ "name": "nasa", "id":"528817151"}); -} - -/** - * Update "test date" and "test status" Badges. - * Send a report email to if no downloadable URLs are obtained from the tested Instagram accounts - * @returns {boolean} - */ -function test_pipeline(){ - setTestDateBadge(); - if (test_bbcnews_ig() == 0 && test_cnn_ig() == 0 && test_medium_ig() == 0 && test_nasa_ig() == 0) { - MailApp.sendEmail(crashReportEmail, - "Google Apps Script [AutoFetcher-IG-Stories-to-GDrive] Crash reporter", - "Get none urls from the test accounts.\nPlease verify the service websites and check update on https://bit.ly/2zQLd6p."); - Logger.log('Failed to fetch data from the test accounts!!!') - setTestStatusBadge(status='failed') - return false; - }; - Logger.log('Successfully fetch data from the test accounts!') - setTestStatusBadge(status='passed') - return true; -} - /** * Update the SVG file of "Test Status" badge to "Failed" using Class DriveApp. * @param {string} status - The arguement to determine the badge color and the text to display in the badge. diff --git a/gascript/test.gs b/gascript/test.gs new file mode 100644 index 0000000..b102f1e --- /dev/null +++ b/gascript/test.gs @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2021 + * + * This file contains the code to test fetching Instagram stories using the Apps Script. + * + * @summary short description for the file + * @author Chris K.Y. Fung + * + * Created at : 2021-09-12 + * Last modified : 2021-09-12 + */ + + /** + * Test getting the URLs of media files in the data retrieved from Instagram's API using fetch_ig_stories() and parseDownloadUrl(). + * @param {Object} targetIgUser - A JSON object contains the name and id of an Instagram account, e.g. { "name": "john", "id": "1234567890" }. + * @returns {number} The number of URLs obtained. + */ +function tryGetStories(targetIgUser) { + var query_url = getQuery(targetIgUser.id); + var html = fetch_ig_stories(query_url); + + var urls = parseDownloadUrl(html,true); + Logger.log("Number of URL(s) from @" + targetIgUser.id + ": " + urls.length); + return urls.length; +} + +/** Run test_function() with BBC News's instagram account */ +function test_bbcnews_ig(){ + return tryGetStories({ "name": "bbcnews", "id":"16278726"}); +} +/** Run test_function() with CNN's instagram account */ +function test_cnn_ig(){ + return tryGetStories({ "name": "cnn", "id":"217723373"}); +} +/** Run test_function() with Medium's instagram account */ +function test_medium_ig(){ + return tryGetStories({ "name": "medium", "id":"1112881921"}); +} +/** Run test_function() with NASA's instagram account */ +function test_nasa_ig(){ + return tryGetStories({ "name": "nasa", "id":"528817151"}); +} + +/** + * Update "test date" and "test status" Badges. + * Send a report email to if no downloadable URLs are obtained from the tested Instagram accounts + * @returns {boolean} + */ +function test_pipeline(){ + setTestDateBadge(); + if (test_bbcnews_ig() == 0 && test_cnn_ig() == 0 && test_medium_ig() == 0 && test_nasa_ig() == 0) { + MailApp.sendEmail(crashReportEmail, + "Google Apps Script [AutoFetcher-IG-Stories-to-GDrive] Crash reporter", + "Get none urls from the test accounts.\nPlease verify the service websites and check update on https://bit.ly/2zQLd6p."); + Logger.log('Failed to fetch data from the test accounts!!!') + setTestStatusBadge(status='failed') + return false; + }; + Logger.log('Successfully fetch data from the test accounts!') + setTestStatusBadge(status='passed') + return true; +}