From 840ff6fdda20dec922dfe3be8e7e3877c0e3d3cd Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Fri, 17 Aug 2018 09:34:17 +0200 Subject: [PATCH 001/442] Update Changelog --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c0128d11..b3cabb475 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,17 @@ -## (2018-08-15) +## (2018-08-17) +## 3.12.1 (2018-08-17) + +* Update Changelog ([718b0c8](https://github.com/microlinkhq/metascraper/commit/718b0c8)) +* Update dependencies ([51c7c52](https://github.com/microlinkhq/metascraper/commit/51c7c52)) +* Use title instead of to-title-case (#103) ([5d25245](https://github.com/microlinkhq/metascraper/commit/5d25245)), closes [#103](https://github.com/microlinkhq/metascraper/issues/103) +* v3.12.1 ([9baa717](https://github.com/microlinkhq/metascraper/commit/9baa717)) + + + ## 3.12.0 (2018-08-15) * Use twdown for twitter videos ([0a74da4](https://github.com/microlinkhq/metascraper/commit/0a74da4)) From eb8778cee0d22ba0ec71603155ff81bac3f8d7e5 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Fri, 17 Aug 2018 12:36:54 +0200 Subject: [PATCH 002/442] Better twitter video extraction (#104) --- .../metascraper-video-provider/package.json | 8 ++-- .../src/get-video-info/index.js | 40 ++++++++++++++++++ .../src/get-video-info/twitter-video-info.js | 42 +++++++++++++++++++ .../src/get-video-info/video-info.js | 6 +++ .../{ => src}/index.js | 38 +++-------------- .../metascraper-video-provider/test/index.js | 3 +- 6 files changed, 98 insertions(+), 39 deletions(-) create mode 100644 packages/metascraper-video-provider/src/get-video-info/index.js create mode 100644 packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js create mode 100644 packages/metascraper-video-provider/src/get-video-info/video-info.js rename packages/metascraper-video-provider/{ => src}/index.js (70%) diff --git a/packages/metascraper-video-provider/package.json b/packages/metascraper-video-provider/package.json index 138c7776c..a9f520195 100644 --- a/packages/metascraper-video-provider/package.json +++ b/packages/metascraper-video-provider/package.json @@ -3,7 +3,7 @@ "description": "Get video property from HTML markup", "homepage": "https://metascraper.js.org", "version": "3.12.1", - "main": "index.js", + "main": "src/index", "author": { "email": "ian@ianstormtaylor.com", "name": "Ian Storm Taylor" @@ -17,13 +17,12 @@ }, "dependencies": { "@metascraper/helpers": "^3.12.1", + "got": "~9.0.0", "json-future": "~2.1.2", "lodash": "~4.17.10", - "twdown": "~1.0.3", "youtube-dl": "~1.12.2" }, "devDependencies": { - "browserless": "latest", "mocha": "latest", "nyc": "latest", "puppeteer": "latest", @@ -35,8 +34,7 @@ "node": ">= 8" }, "files": [ - "index.js", - "scripts" + "src" ], "scripts": { "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" diff --git a/packages/metascraper-video-provider/src/get-video-info/index.js b/packages/metascraper-video-provider/src/get-video-info/index.js new file mode 100644 index 000000000..2cb2a9155 --- /dev/null +++ b/packages/metascraper-video-provider/src/get-video-info/index.js @@ -0,0 +1,40 @@ +'use strict' + +const {isTwitterUrl, getTwitterVideoInfo} = require('./twitter-video-info') +const getVideoInfo = require('./video-info') +const { chain } = require('lodash') + +const getInfo = async url => { + if (!isTwitterUrl(url)) return getVideoInfo(url) + + const [videoInfo, twitterVideos] = await Promise.all([ + getVideoInfo(url), + getTwitterVideoInfo(url) + ]) + + const formats = chain(videoInfo.formats) + .reduce((acc, format, index) => { + const { url } = twitterVideos[index] + const item = {...format, url} + return [...acc, item] + }, []) + .value() + + return {...videoInfo, formats} +} + +// Local cache for successive calls +let cachedVideoInfoUrl +let cachedVideoInfo + +module.exports = async url => { + if (url === cachedVideoInfoUrl) return cachedVideoInfo + cachedVideoInfoUrl = url + + try { + cachedVideoInfo = await getInfo(url) + } catch (err) { + cachedVideoInfo = {} + } + return cachedVideoInfo +} diff --git a/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js b/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js new file mode 100644 index 000000000..5db22f1ad --- /dev/null +++ b/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js @@ -0,0 +1,42 @@ +'use strict' + +const { chain } = require('lodash') +const { URL } = require('url') +const got = require('got') + +// twitter guest web token +// https://github.com/soimort/you-get/blob/da8c982608c9308765e0960e08fc28cccb74b215/src/you_get/extractors/twitter.py#L72 +const TWITTER_BEARER_TOKEN = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA' + +const TWITTER_HOSTNAMES = ['twitter.com', 'mobile.twitter.com'] + +const isTwitterUrl = url => TWITTER_HOSTNAMES.includes(new URL(url).hostname) + +const getTweetId = url => url.split('/').reverse()[0] + +const getGuestToken = async () => { + const { body } = await got.post('https://api.twitter.com/1.1/guest/activate.json', { + headers: { authorization: TWITTER_BEARER_TOKEN }, + json: true + }) + return body.guest_token +} + +const getTwitterVideoInfo = async url => { + const tweetId = getTweetId(url) + const apiUrl = `https://api.twitter.com/2/timeline/conversation/${tweetId}.json?tweet_mode=extended` + const { body } = await got(apiUrl, { + json: true, + headers: { + authorization: TWITTER_BEARER_TOKEN, + 'x-guest-token': await getGuestToken() + } + }) + + return chain(body) + .get(`globalObjects.tweets.${tweetId}.extended_entities.media.0.video_info.variants`) + .orderBy('bitrate', 'asc') + .value() +} + +module.exports = { getTwitterVideoInfo, isTwitterUrl } diff --git a/packages/metascraper-video-provider/src/get-video-info/video-info.js b/packages/metascraper-video-provider/src/get-video-info/video-info.js new file mode 100644 index 000000000..c91e07c95 --- /dev/null +++ b/packages/metascraper-video-provider/src/get-video-info/video-info.js @@ -0,0 +1,6 @@ +'use strict' + +const youtubedl = require('youtube-dl') +const { promisify } = require('util') + +module.exports = promisify(youtubedl.getInfo) diff --git a/packages/metascraper-video-provider/index.js b/packages/metascraper-video-provider/src/index.js similarity index 70% rename from packages/metascraper-video-provider/index.js rename to packages/metascraper-video-provider/src/index.js index c35012c7e..8844cc704 100644 --- a/packages/metascraper-video-provider/index.js +++ b/packages/metascraper-video-provider/src/index.js @@ -2,33 +2,9 @@ const { overEvery, isEmpty, eq, has, round, size, get, chain, find, isString } = require('lodash') const { isUrl, titleize } = require('@metascraper/helpers') -const youtubedl = require('youtube-dl') -const { promisify } = require('util') -const twdown = require('twdown') -const { URL } = require('url') const path = require('path') -const getInfo = promisify(youtubedl.getInfo) - -const TWITTER_HOSTNAMES = ['twitter.com', 'mobile.twitter.com'] - -const isTwitterUrl = url => TWITTER_HOSTNAMES.includes(new URL(url).hostname) - -// Local cache for successive calls -let cachedVideoInfoUrl -let cachedVideoInfo - -const getVideoInfo = async url => { - if (url === cachedVideoInfoUrl) return cachedVideoInfo - cachedVideoInfoUrl = url - - try { - cachedVideoInfo = await getInfo(url) - } catch (err) { - cachedVideoInfo = {} - } - return cachedVideoInfo -} +const getVideoInfo = require('./get-video-info') const isMp4 = video => eq(get(video, 'ext'), 'mp4') || path.extname(get(video, 'url')).startsWith('.mp4') @@ -52,12 +28,10 @@ const getVideoUrl = (videos, filters = []) => { } /** - * Get a URL-like video source. + * Get a URL-like video source */ -const getVideoProvider = getBrowserless => async ({ url }) => { - const formats = !isTwitterUrl(url) - ? (await getVideoInfo(url)).formats - : await twdown({ url, browserless: await getBrowserless() }) +const getVideoProvider = async ({ url }) => { + const { formats } = await getVideoInfo(url) const videoUrl = getVideoUrl(formats, [isMp4, isHttps, hasAudio]) || @@ -97,9 +71,9 @@ const getVideoDate = async ({ url }) => { return timestamp && new Date(timestamp * 1000).toISOString() } -module.exports = ({ getBrowserless }) => { +module.exports = () => { return { - video: getVideoProvider(getBrowserless), + video: getVideoProvider, author: getVideoAuthor, publisher: getVideoPublisher, title: getVideoTitle, diff --git a/packages/metascraper-video-provider/test/index.js b/packages/metascraper-video-provider/test/index.js index d2c9bcdf0..3fd553948 100644 --- a/packages/metascraper-video-provider/test/index.js +++ b/packages/metascraper-video-provider/test/index.js @@ -1,7 +1,6 @@ 'use strict' const { isUrl } = require('@metascraper/helpers') -const browserless = require('browserless')() const { isString } = require('lodash') const snapshot = require('snap-shot') const { promisify } = require('util') @@ -11,7 +10,7 @@ const should = require('should') const fs = require('fs') const metascraper = require('metascraper').load([ - require('metascraper-video-provider')({ getBrowserless: () => browserless }), + require('metascraper-video-provider')(), require('metascraper-author')(), require('metascraper-date')(), require('metascraper-description')(), From 4e2ae4d7babba6658c31bffbd2fd32df9b207bf3 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Fri, 17 Aug 2018 12:44:33 +0200 Subject: [PATCH 003/442] Unify lint-staged --- package.json | 19 ++++++++++++------- packages/metascraper-amazon/package.json | 6 ------ .../metascraper-clearbit-logo/package.json | 6 ------ .../metascraper-logo-favicon/package.json | 6 ------ packages/metascraper-soundcloud/package.json | 6 ------ packages/metascraper-youtube/package.json | 6 ------ 6 files changed, 12 insertions(+), 37 deletions(-) diff --git a/package.json b/package.json index 7b491c40d..994a49b64 100644 --- a/package.json +++ b/package.json @@ -52,13 +52,18 @@ "private": true, "license": "MIT", "lint-staged": { - "src/**/*.{js,css}": [ - "prettier-standard", - "git add" - ], - "package.json": [ - "finepack", - "git add" + "linters": { + "*.js": [ + "prettier-standard", + "git add" + ], + "package.json": [ + "finepack", + "git add" + ] + }, + "ignore": [ + "static/*.{js, css}" ] }, "standard": { diff --git a/packages/metascraper-amazon/package.json b/packages/metascraper-amazon/package.json index 21eb73d3e..655ea7884 100644 --- a/packages/metascraper-amazon/package.json +++ b/packages/metascraper-amazon/package.json @@ -41,12 +41,6 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", - "lint-staged": { - "*.js": [ - "prettier-standard", - "git add" - ] - }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-clearbit-logo/package.json b/packages/metascraper-clearbit-logo/package.json index 7cb87c955..09b41f184 100644 --- a/packages/metascraper-clearbit-logo/package.json +++ b/packages/metascraper-clearbit-logo/package.json @@ -36,12 +36,6 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", - "lint-staged": { - "*.js": [ - "prettier-standard", - "git add" - ] - }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-logo-favicon/package.json b/packages/metascraper-logo-favicon/package.json index 7ccf49946..b71aaf4bf 100644 --- a/packages/metascraper-logo-favicon/package.json +++ b/packages/metascraper-logo-favicon/package.json @@ -38,12 +38,6 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", - "lint-staged": { - "*.js": [ - "prettier-standard", - "git add" - ] - }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-soundcloud/package.json b/packages/metascraper-soundcloud/package.json index 32f58d383..bdb48657f 100644 --- a/packages/metascraper-soundcloud/package.json +++ b/packages/metascraper-soundcloud/package.json @@ -40,12 +40,6 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", - "lint-staged": { - "*.js": [ - "prettier-standard", - "git add" - ] - }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-youtube/package.json b/packages/metascraper-youtube/package.json index 8745e7280..df9ddbe0c 100644 --- a/packages/metascraper-youtube/package.json +++ b/packages/metascraper-youtube/package.json @@ -44,12 +44,6 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", - "lint-staged": { - "*.js": [ - "prettier-standard", - "git add" - ] - }, "standard": { "env": [ "mocha" From 3327bab62f98f0fab272d8b531bd15561bfaadb9 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Fri, 17 Aug 2018 14:25:16 +0200 Subject: [PATCH 004/442] v3.13.0 --- lerna.json | 2 +- packages/metascraper-amazon/package.json | 2 +- packages/metascraper-clearbit-logo/package.json | 2 +- packages/metascraper-logo-favicon/package.json | 2 +- packages/metascraper-soundcloud/package.json | 2 +- packages/metascraper-video-provider/package.json | 2 +- packages/metascraper-youtube/package.json | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lerna.json b/lerna.json index 8695c5a80..01152820f 100644 --- a/lerna.json +++ b/lerna.json @@ -3,5 +3,5 @@ "packages": [ "packages/*" ], - "version": "3.12.1" + "version": "3.13.0" } diff --git a/packages/metascraper-amazon/package.json b/packages/metascraper-amazon/package.json index 655ea7884..61bf605c2 100644 --- a/packages/metascraper-amazon/package.json +++ b/packages/metascraper-amazon/package.json @@ -2,7 +2,7 @@ "name": "metascraper-amazon", "description": "metascraper integration with Amazon", "homepage": "https://documentup.com/microlinkhq/metascraper-amazon", - "version": "3.12.1", + "version": "3.13.0", "main": "index.js", "author": { "email": "hello@microlink.io", diff --git a/packages/metascraper-clearbit-logo/package.json b/packages/metascraper-clearbit-logo/package.json index 09b41f184..6a107f478 100644 --- a/packages/metascraper-clearbit-logo/package.json +++ b/packages/metascraper-clearbit-logo/package.json @@ -2,7 +2,7 @@ "name": "metascraper-clearbit-logo", "description": "metascraper integration with Clearbit Logo API", "homepage": "https://documentup.com/microlinkhq/metascraper-clearbit-logo", - "version": "3.11.12", + "version": "3.13.0", "main": "index.js", "author": { "email": "hello@microlink.io", diff --git a/packages/metascraper-logo-favicon/package.json b/packages/metascraper-logo-favicon/package.json index b71aaf4bf..24877758a 100644 --- a/packages/metascraper-logo-favicon/package.json +++ b/packages/metascraper-logo-favicon/package.json @@ -2,7 +2,7 @@ "name": "metascraper-logo-favicon", "description": "metascraper logo favicon fallback", "homepage": "https://documentup.com/microlinkhq/metascraper-logo-favicon", - "version": "3.11.4", + "version": "3.13.0", "main": "index.js", "author": { "email": "hello@microlink.io", diff --git a/packages/metascraper-soundcloud/package.json b/packages/metascraper-soundcloud/package.json index bdb48657f..ecb56f80a 100644 --- a/packages/metascraper-soundcloud/package.json +++ b/packages/metascraper-soundcloud/package.json @@ -2,7 +2,7 @@ "name": "metascraper-soundcloud", "description": "metascraper integration with SoundCloud", "homepage": "https://documentup.com/microlinkhq/metascraper-soundcloud", - "version": "3.12.1", + "version": "3.13.0", "main": "index.js", "author": { "email": "hello@microlink.io", diff --git a/packages/metascraper-video-provider/package.json b/packages/metascraper-video-provider/package.json index a9f520195..b62629cb9 100644 --- a/packages/metascraper-video-provider/package.json +++ b/packages/metascraper-video-provider/package.json @@ -2,7 +2,7 @@ "name": "metascraper-video-provider", "description": "Get video property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.12.1", + "version": "3.13.0", "main": "src/index", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-youtube/package.json b/packages/metascraper-youtube/package.json index df9ddbe0c..3091f4501 100644 --- a/packages/metascraper-youtube/package.json +++ b/packages/metascraper-youtube/package.json @@ -2,7 +2,7 @@ "name": "metascraper-youtube", "description": "metascraper integration with YouTube", "homepage": "https://documentup.com/microlinkhq/metascraper-youtube", - "version": "3.12.1", + "version": "3.13.0", "main": "index.js", "author": { "email": "hello@microlink.io", From b012514adc27cdac692c6ba9b19d2d3f79fabf61 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Fri, 17 Aug 2018 14:26:03 +0200 Subject: [PATCH 005/442] Update Changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3cabb475..3989bac1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,15 @@ +## 3.13.0 (2018-08-17) + +* Better twitter video extraction (#104) ([eb8778c](https://github.com/microlinkhq/metascraper/commit/eb8778c)), closes [#104](https://github.com/microlinkhq/metascraper/issues/104) +* Unify lint-staged ([4e2ae4d](https://github.com/microlinkhq/metascraper/commit/4e2ae4d)) +* Update Changelog ([840ff6f](https://github.com/microlinkhq/metascraper/commit/840ff6f)) +* v3.13.0 ([3327bab](https://github.com/microlinkhq/metascraper/commit/3327bab)) + + + ## 3.12.1 (2018-08-17) * Update Changelog ([718b0c8](https://github.com/microlinkhq/metascraper/commit/718b0c8)) From f6fd92a2fed5570acab024fd146847d8acce1aa0 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sat, 18 Aug 2018 14:17:24 +0200 Subject: [PATCH 006/442] At least video need to be mp4 format --- .../metascraper-video-provider/src/index.js | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/metascraper-video-provider/src/index.js b/packages/metascraper-video-provider/src/index.js index 8844cc704..19ec9a075 100644 --- a/packages/metascraper-video-provider/src/index.js +++ b/packages/metascraper-video-provider/src/index.js @@ -1,13 +1,25 @@ 'use strict' -const { overEvery, isEmpty, eq, has, round, size, get, chain, find, isString } = require('lodash') +const { + overEvery, + isEmpty, + eq, + has, + round, + size, + get, + chain, + find, + isString +} = require('lodash') const { isUrl, titleize } = require('@metascraper/helpers') const path = require('path') const getVideoInfo = require('./get-video-info') const isMp4 = video => - eq(get(video, 'ext'), 'mp4') || path.extname(get(video, 'url')).startsWith('.mp4') + eq(get(video, 'ext'), 'mp4') || + path.extname(get(video, 'url')).startsWith('.mp4') const isHttp = video => eq(get(video, 'protocol'), 'http') const isHttps = video => eq(get(video, 'protocol'), 'https') const hasAudio = video => has(video, 'abr') @@ -37,8 +49,7 @@ const getVideoProvider = async ({ url }) => { getVideoUrl(formats, [isMp4, isHttps, hasAudio]) || getVideoUrl(formats, [isMp4, isHttp, hasAudio]) || getVideoUrl(formats, [isMp4, isHttps]) || - getVideoUrl(formats, [isMp4]) || - getVideoUrl(formats) + getVideoUrl(formats, [isMp4]) return isUrl(videoUrl) && videoUrl } @@ -61,7 +72,9 @@ const getVideoPublisher = async ({ url }) => { } const getVideoTitle = async ({ url }) => { - const { title: mainTitle, alt_title: secondaryTitle } = await getVideoInfo(url) + const { title: mainTitle, alt_title: secondaryTitle } = await getVideoInfo( + url + ) const title = find([mainTitle, secondaryTitle], isString) return title && titleize(title) } From 8928e9580cbef012a905b67edbcafaf627c0df88 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sat, 18 Aug 2018 14:22:01 +0200 Subject: [PATCH 007/442] v3.13.1 --- lerna.json | 2 +- packages/metascraper-video-provider/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lerna.json b/lerna.json index 01152820f..a1313a93b 100644 --- a/lerna.json +++ b/lerna.json @@ -3,5 +3,5 @@ "packages": [ "packages/*" ], - "version": "3.13.0" + "version": "3.13.1" } diff --git a/packages/metascraper-video-provider/package.json b/packages/metascraper-video-provider/package.json index b62629cb9..e83305322 100644 --- a/packages/metascraper-video-provider/package.json +++ b/packages/metascraper-video-provider/package.json @@ -2,7 +2,7 @@ "name": "metascraper-video-provider", "description": "Get video property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.13.0", + "version": "3.13.1", "main": "src/index", "author": { "email": "ian@ianstormtaylor.com", From 9c5463b1501006a23829b5eb6ffd97ac3e30e330 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sat, 18 Aug 2018 14:22:34 +0200 Subject: [PATCH 008/442] Update Changelog --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3989bac1d..92d078dc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,16 @@ -## (2018-08-17) +## (2018-08-18) +## 3.13.1 (2018-08-18) + +* At least video need to be mp4 format ([f6fd92a](https://github.com/microlinkhq/metascraper/commit/f6fd92a)) +* Update Changelog ([b012514](https://github.com/microlinkhq/metascraper/commit/b012514)) +* v3.13.1 ([8928e95](https://github.com/microlinkhq/metascraper/commit/8928e95)) + + + ## 3.13.0 (2018-08-17) * Better twitter video extraction (#104) ([eb8778c](https://github.com/microlinkhq/metascraper/commit/eb8778c)), closes [#104](https://github.com/microlinkhq/metascraper/issues/104) From 23f641d8e3fd1e053874430fbb606334279c9504 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sun, 19 Aug 2018 00:48:11 +0200 Subject: [PATCH 009/442] Oh twitter --- .../src/get-video-info/twitter-video-info.js | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js b/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js index 5db22f1ad..599d295b4 100644 --- a/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js +++ b/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js @@ -6,7 +6,9 @@ const got = require('got') // twitter guest web token // https://github.com/soimort/you-get/blob/da8c982608c9308765e0960e08fc28cccb74b215/src/you_get/extractors/twitter.py#L72 -const TWITTER_BEARER_TOKEN = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA' +// https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/twitter.py#L235 +const TWITTER_BEARER_TOKEN = + 'Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw' const TWITTER_HOSTNAMES = ['twitter.com', 'mobile.twitter.com'] @@ -14,27 +16,33 @@ const isTwitterUrl = url => TWITTER_HOSTNAMES.includes(new URL(url).hostname) const getTweetId = url => url.split('/').reverse()[0] -const getGuestToken = async () => { - const { body } = await got.post('https://api.twitter.com/1.1/guest/activate.json', { - headers: { authorization: TWITTER_BEARER_TOKEN }, - json: true - }) +const getGuestToken = async url => { + const { body } = await got.post( + 'https://api.twitter.com/1.1/guest/activate.json', + { + headers: { Authorization: TWITTER_BEARER_TOKEN, Referer: url }, + json: true + } + ) return body.guest_token } const getTwitterVideoInfo = async url => { const tweetId = getTweetId(url) + console.log('tweetId', tweetId) const apiUrl = `https://api.twitter.com/2/timeline/conversation/${tweetId}.json?tweet_mode=extended` const { body } = await got(apiUrl, { json: true, headers: { authorization: TWITTER_BEARER_TOKEN, - 'x-guest-token': await getGuestToken() + 'x-guest-token': await getGuestToken(url) } }) return chain(body) - .get(`globalObjects.tweets.${tweetId}.extended_entities.media.0.video_info.variants`) + .get( + `globalObjects.tweets.${tweetId}.extended_entities.media.0.video_info.variants` + ) .orderBy('bitrate', 'asc') .value() } From f4a0b21102004e658db63d94b046619ac591b2b5 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sun, 19 Aug 2018 00:50:00 +0200 Subject: [PATCH 010/442] v3.13.2 --- lerna.json | 2 +- packages/metascraper-video-provider/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lerna.json b/lerna.json index a1313a93b..ee0698bcf 100644 --- a/lerna.json +++ b/lerna.json @@ -3,5 +3,5 @@ "packages": [ "packages/*" ], - "version": "3.13.1" + "version": "3.13.2" } diff --git a/packages/metascraper-video-provider/package.json b/packages/metascraper-video-provider/package.json index e83305322..8016a7e99 100644 --- a/packages/metascraper-video-provider/package.json +++ b/packages/metascraper-video-provider/package.json @@ -2,7 +2,7 @@ "name": "metascraper-video-provider", "description": "Get video property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.13.1", + "version": "3.13.2", "main": "src/index", "author": { "email": "ian@ianstormtaylor.com", From 76d5329df93c58d748835dca6b6f406fe4c153d7 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sun, 19 Aug 2018 00:50:25 +0200 Subject: [PATCH 011/442] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92d078dc8..109e59e0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ +## 3.13.2 (2018-08-18) + +* Oh twitter ([23f641d](https://github.com/microlinkhq/metascraper/commit/23f641d)) +* Update Changelog ([9c5463b](https://github.com/microlinkhq/metascraper/commit/9c5463b)) +* v3.13.2 ([f4a0b21](https://github.com/microlinkhq/metascraper/commit/f4a0b21)) + + + ## 3.13.1 (2018-08-18) * At least video need to be mp4 format ([f6fd92a](https://github.com/microlinkhq/metascraper/commit/f6fd92a)) From 9d3745b86cbc2e3fe005af125bd8c3a25ef099b5 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Wed, 22 Aug 2018 16:29:03 +0200 Subject: [PATCH 012/442] Remove autoload --- README.md | 124 +----------------- packages/metascraper-amazon/package.json | 3 + packages/metascraper-amazon/test/index.js | 27 ++-- packages/metascraper-author/package.json | 5 +- .../metascraper-clearbit-logo/package.json | 9 +- .../metascraper-clearbit-logo/test/index.js | 2 +- packages/metascraper-date/package.json | 5 +- packages/metascraper-description/package.json | 5 +- packages/metascraper-image/package.json | 5 +- packages/metascraper-lang/package.json | 4 +- packages/metascraper-lang/test/index.js | 5 +- .../metascraper-logo-favicon/package.json | 3 + .../metascraper-logo-favicon/test/index.js | 15 +-- packages/metascraper-logo/package.json | 5 +- packages/metascraper-publisher/package.json | 5 +- packages/metascraper-soundcloud/package.json | 3 + packages/metascraper-soundcloud/test/index.js | 5 +- packages/metascraper-title/package.json | 5 +- packages/metascraper-url/package.json | 5 +- .../metascraper-video-provider/package.json | 3 + .../src/get-video-info/twitter-video-info.js | 1 - .../metascraper-video-provider/test/index.js | 2 +- packages/metascraper-video/package.json | 3 + packages/metascraper-video/test/index.js | 21 ++- packages/metascraper-youtube/package.json | 3 + packages/metascraper-youtube/test/index.js | 18 ++- packages/metascraper/package.json | 12 -- packages/metascraper/src/index.js | 23 ++-- packages/metascraper/src/load-rules.js | 54 +------- .../test/integration/anandtech/index.js | 14 +- .../test/integration/arstechnica/index.js | 14 +- .../test/integration/astier/index.js | 14 +- .../test/integration/atlasobscura/index.js | 14 +- .../test/integration/audiense/index.js | 14 +- .../metascraper/test/integration/bbc/index.js | 14 +- .../test/integration/bloomberg/index.js | 14 +- .../test/integration/business-today/index.js | 14 +- .../metascraper/test/integration/cbr/index.js | 14 +- .../metascraper/test/integration/cio/index.js | 14 +- .../test/integration/cloud-pro/index.js | 14 +- .../test/integration/cnet/index.js | 14 +- .../test/integration/computerworld/index.js | 14 +- .../metascraper/test/integration/crn/index.js | 14 +- .../test/integration/economic-times/index.js | 14 +- .../test/integration/entrepreneur/index.js | 14 +- .../test/integration/et-tech/index.js | 14 +- .../test/integration/eweek/index.js | 14 +- .../test/integration/fast-company/index.js | 14 +- .../test/integration/fastersite/index.js | 14 +- .../test/integration/fierce-devops/index.js | 14 +- .../test/integration/financial-times/index.js | 14 +- .../test/integration/forbes/index.js | 14 +- .../test/integration/fortune/index.js | 14 +- .../test/integration/geek-time/index.js | 14 +- .../test/integration/github/index.js | 14 +- .../test/integration/googleblog/index.js | 14 +- .../metascraper/test/integration/inc/index.js | 14 +- .../integration/information-week/index.js | 14 +- .../test/integration/instagram/index.js | 14 +- .../integration/jewish-business-news/index.js | 14 +- .../test/integration/lean-data/index.js | 14 +- .../integration/linkedin-company/index.js | 14 +- .../test/integration/linkedin-pulse/index.js | 14 +- .../integration/los-angeles-times/index.js | 14 +- .../test/integration/mac-rumors/index.js | 14 +- .../test/integration/market-wired/index.js | 14 +- .../test/integration/marketing-land/index.js | 14 +- .../test/integration/mashable/index.js | 14 +- .../test/integration/medium/index.js | 14 +- .../test/integration/motherboard/index.js | 14 +- .../test/integration/nytimes/index.js | 14 +- .../test/integration/pikabu/index.js | 14 +- .../test/integration/pr-newswire/index.js | 14 +- .../test/integration/recode/index.js | 14 +- .../test/integration/reuters/index.js | 14 +- .../san-francisco-chronicle/index.js | 14 +- .../test/integration/segment-academy/index.js | 14 +- .../test/integration/segment/index.js | 14 +- .../test/integration/silicon-angle/index.js | 14 +- .../test/integration/silicon-beat/index.js | 14 +- .../test/integration/silicon-tap/index.js | 14 +- .../silicon-valley-business-journal/index.js | 14 +- .../test/integration/smitten-kitchen/index.js | 14 +- .../test/integration/startup-grind/index.js | 14 +- .../test/integration/techcrunch/index.js | 14 +- .../integration/the-boston-globe/index.js | 14 +- .../test/integration/the-guardian/index.js | 14 +- .../test/integration/the-register/index.js | 14 +- .../test/integration/the-verge/index.js | 14 +- .../test/integration/twitter-gif/index.js | 14 +- .../test/integration/twitter-image/index.js | 14 +- .../test/integration/usa-today/index.js | 14 +- .../test/integration/venture-beat/index.js | 14 +- .../test/integration/vimeo/index.js | 14 +- .../test/integration/washington-post/index.js | 14 +- .../test/integration/wikipedia/index.js | 14 +- .../test/integration/wired/index.js | 14 +- .../metascraper/test/integration/wsj/index.js | 14 +- .../test/integration/xconomy/index.js | 14 +- .../test/integration/yahoo-news/index.js | 14 +- .../test/integration/zdnet/index.js | 14 +- packages/metascraper/test/unit/interface.js | 18 ++- .../test/unit/load-rules/config/index.js | 29 ---- .../load-rules/config/metascraper.config.js | 17 --- .../test/unit/load-rules/pkg/index.js | 29 ---- .../test/unit/load-rules/pkg/package.json | 19 --- .../test/unit/load-rules/rc/.metascraperrc | 15 --- .../test/unit/load-rules/rc/index.js | 29 ---- packages/metascraper/test/unit/merge-rules.js | 46 +++---- 109 files changed, 1109 insertions(+), 481 deletions(-) delete mode 100644 packages/metascraper/test/unit/load-rules/config/index.js delete mode 100644 packages/metascraper/test/unit/load-rules/config/metascraper.config.js delete mode 100644 packages/metascraper/test/unit/load-rules/pkg/index.js delete mode 100644 packages/metascraper/test/unit/load-rules/pkg/package.json delete mode 100644 packages/metascraper/test/unit/load-rules/rc/.metascraperrc delete mode 100644 packages/metascraper/test/unit/load-rules/rc/index.js diff --git a/README.md b/README.md index 548a0dad6..458c388bf 100644 --- a/README.md +++ b/README.md @@ -131,116 +131,12 @@ Rules work as fallback between them: **metascraper** do that until finish all the rule or find the first rule that resolves the value. -## Loading rules - -When you call **metascraper** in your code, a set of [core rules](#core-rules) are loaded by default. - -Although these rules are sufficient for most cases, **metascraper** was designed to be easy to adapt and load more or custom rules set. - -We provide two approach for do that. - -### Configuration file - -This consists in declaring a configuration file that contains the names of the rule sets corresponding to npm packages that **metascraper** will be load automagically. - -The configuration file could be declared via: - -- A `.metascraperrc` file, written in YAML or JSON, with optional extensions: **.yaml**, **.yml**, **.json** and **.js**. -- A `metascraper.config.js` file that exports an object. -- A **metascraper** key in your `package.json` file. - -The configuration file will be resolved starting from the location of the file being formatted, and searching up the file tree until a config file is (or isn't) found. - -The order of rules are loaded are important: Just the first rule that resolve the value will be applied. - -#### Basic Configuration - -Declared an **array** of **rules**, specifying each rule as **string** name of the module to load. - -##### JSON - -```json -// .metascraperrc -{ - "rules": [ - "metascraper-author", - "metascraper-date", - "metascraper-description", - "metascraper-image", - "metascraper-lang", - "metascraper-logo", - "metascraper-publisher", - "metascraper-title", - "metascraper-url" - ] -} -``` - -##### YAML - -```yaml -# .metascraperrc -rules: - - metascraper-author - - metascraper-date - - metascraper-description - - metascraper-image - - metascraper-lang - - metascraper-logo - - metascraper-publisher - - metascraper-title - - metascraper-url -``` - -#### Advanced Configuration - -Additionally, you can pass specific configuration per module using a **object** declaration: - -##### JSON - -```json -// .metascraperrc -{ - "rules": [ - "metascraper-author", - "metascraper-date", - "metascraper-description", - "metascraper-image", - "metascraper-lang", - "metascraper-logo", - {"metascraper-clearbit-logo": { - "format": "jpg" - }}, - "metascraper-publisher", - "metascraper-title", - "metascraper-url" - ] -} -``` - -##### YAML - -```yaml -# .metascraperrc -rules: - - metascraper-author - - metascraper-date - - metascraper-description - - metascraper-image - - metascraper-lang - - metascraper-clearbit-logo: - format: jpg - - metascraper-publisher - - metascraper-title - - metascraper-url -``` - -### Constructor +## Usage -If you need more control, you can load the rules set calling directly the metascraper constructor [`.load`](#metascraperloadrules): +**metascraper** exports a constructors that need to be initalized providing a collection of rules set to load: ```js -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-author')(), require('metascraper-date')(), require('metascraper-description')(), @@ -255,10 +151,10 @@ const metascraper = require('metascraper').load([ Again, the order of rules are loaded are important: Just the first rule that resolve the value will be applied. -Use the first parameter to pass custom options if you need it: +Use the first parameter to pass custom options specific per each rules set: ```js -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-clearbit-logo')({ size: 256, format: 'jpg' @@ -266,14 +162,6 @@ const metascraper = require('metascraper').load([ ]) ``` -Using this way you are not limited to load just npm modules as rules set. For example, you can load a custom file of rules: - -```js -const metascraper = require('metascraper').load([ - require('./my-custom-rules-file')() -]) -``` - ## Rules ?> Can't find a rules set that you want? Let's [open an issue](https://github.com/microlinkhq/metascraper/issues/new) to create it. @@ -368,7 +256,7 @@ Type: `Array` You can pass additional rules on execution time. These rules will be merged with your loaded rules. -### metascraper.load(rules) +### metascraper([rules]) Create a new **metascraper** instance declaring the rules set to be used explicitly. diff --git a/packages/metascraper-amazon/package.json b/packages/metascraper-amazon/package.json index 61bf605c2..bbe51324b 100644 --- a/packages/metascraper-amazon/package.json +++ b/packages/metascraper-amazon/package.json @@ -41,6 +41,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-amazon/test/index.js b/packages/metascraper-amazon/test/index.js index 8c7f412b7..00fcc5168 100644 --- a/packages/metascraper-amazon/test/index.js +++ b/packages/metascraper-amazon/test/index.js @@ -9,7 +9,7 @@ const fs = require('fs') const readFile = promisify(fs.readFile) -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-amazon')(), require('metascraper-author')(), require('metascraper-date')(), @@ -25,8 +25,11 @@ const metascraper = require('metascraper').load([ describe('metascraper-amazon', () => { describe('amazon.co.uk', () => { it('product url', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/amazon-co-uk/product-url.html')) - const url = 'https://www.amazon.co.uk/Vegetable-Perfection-tasty-recipes-shoots/dp/1849757097/ref=asap_bc?ie=UTF8' + const html = await readFile( + resolve(__dirname, 'fixtures/amazon-co-uk/product-url.html') + ) + const url = + 'https://www.amazon.co.uk/Vegetable-Perfection-tasty-recipes-shoots/dp/1849757097/ref=asap_bc?ie=UTF8' const meta = omit(await metascraper({ html, url }), ['date']) snapshot(meta) }) @@ -34,7 +37,9 @@ describe('metascraper-amazon', () => { describe('amazon.com', () => { it('ansi url', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/amazon-com/ansi-url.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/amazon-com/ansi-url.html') + ) const url = 'https://www.amazon.com/gp/product/B0057OC5O8/' const metadata = await metascraper({ html, url }) @@ -44,8 +49,11 @@ describe('metascraper-amazon', () => { }) it('product url', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/amazon-com/product-url.html')) - const url = 'https://www.amazon.com/The-Whole-Truth-Shaw-Book-ebook/dp/B0011UCPM4/ref=pd_zg_rss_ts_b_17_6?ie=UTF8&tag=recomshop-22' + const html = await readFile( + resolve(__dirname, 'fixtures/amazon-com/product-url.html') + ) + const url = + 'https://www.amazon.com/The-Whole-Truth-Shaw-Book-ebook/dp/B0011UCPM4/ref=pd_zg_rss_ts_b_17_6?ie=UTF8&tag=recomshop-22' const metadata = await metascraper({ html, url }) // omit date because it is non deterministic @@ -56,8 +64,11 @@ describe('metascraper-amazon', () => { describe('amazon.es', () => { it('product url', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/amazon-es/product-url.html')) - const url = 'https://www.amazon.es/aspirador-Excellence-Programable-limpieza-Silencioso/dp/B01MUGXRT9' + const html = await readFile( + resolve(__dirname, 'fixtures/amazon-es/product-url.html') + ) + const url = + 'https://www.amazon.es/aspirador-Excellence-Programable-limpieza-Silencioso/dp/B01MUGXRT9' const metadata = await metascraper({ html, url }) // omit date because it is non deterministic diff --git a/packages/metascraper-author/package.json b/packages/metascraper-author/package.json index a28665926..78158f87f 100644 --- a/packages/metascraper-author/package.json +++ b/packages/metascraper-author/package.json @@ -31,5 +31,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-clearbit-logo/package.json b/packages/metascraper-clearbit-logo/package.json index 6a107f478..740f7765f 100644 --- a/packages/metascraper-clearbit-logo/package.json +++ b/packages/metascraper-clearbit-logo/package.json @@ -20,6 +20,9 @@ "clearbit", "metascraper" ], + "dependencies": { + "got": "~9.0.0" + }, "devDependencies": { "mocha": "latest", "nyc": "latest", @@ -36,12 +39,12 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" ] - }, - "dependencies": { - "got": "~9.0.0" } } diff --git a/packages/metascraper-clearbit-logo/test/index.js b/packages/metascraper-clearbit-logo/test/index.js index 6e3f79197..62ea13f69 100644 --- a/packages/metascraper-clearbit-logo/test/index.js +++ b/packages/metascraper-clearbit-logo/test/index.js @@ -2,7 +2,7 @@ const should = require('should') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-author')(), require('metascraper-date')(), require('metascraper-description')(), diff --git a/packages/metascraper-date/package.json b/packages/metascraper-date/package.json index 136062926..1e14d546a 100644 --- a/packages/metascraper-date/package.json +++ b/packages/metascraper-date/package.json @@ -31,5 +31,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-description/package.json b/packages/metascraper-description/package.json index 87c1a655f..22b6ecec6 100644 --- a/packages/metascraper-description/package.json +++ b/packages/metascraper-description/package.json @@ -31,5 +31,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-image/package.json b/packages/metascraper-image/package.json index 9c186db84..5f0f0d95e 100644 --- a/packages/metascraper-image/package.json +++ b/packages/metascraper-image/package.json @@ -30,5 +30,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-lang/package.json b/packages/metascraper-lang/package.json index 8f747ca14..1a2cd2b11 100644 --- a/packages/metascraper-lang/package.json +++ b/packages/metascraper-lang/package.json @@ -19,7 +19,6 @@ "lodash": "~4.17.10" }, "devDependencies": { - "metascraper": "latest", "mocha": "latest", "nyc": "latest", "should": "latest", @@ -36,6 +35,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-lang/test/index.js b/packages/metascraper-lang/test/index.js index 58272ede0..d377d8227 100644 --- a/packages/metascraper-lang/test/index.js +++ b/packages/metascraper-lang/test/index.js @@ -5,7 +5,7 @@ const { promisify } = require('util') const { resolve } = require('path') const fs = require('fs') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-amazon')(), require('metascraper-author')(), require('metascraper-date')(), @@ -23,7 +23,8 @@ const readFile = promisify(fs.readFile) describe('metascraper-lang', () => { it('html lang property', async () => { const html = await readFile(resolve(__dirname, 'fixtures/html-lang.html')) - const url = 'http://www.dwutygodnik.com/artykul/7615-churchill-bohater-naszych-czasow.html' + const url = + 'http://www.dwutygodnik.com/artykul/7615-churchill-bohater-naszych-czasow.html' const metadata = await metascraper({ html, url }) snapshot(metadata) }) diff --git a/packages/metascraper-logo-favicon/package.json b/packages/metascraper-logo-favicon/package.json index 24877758a..19bd4be22 100644 --- a/packages/metascraper-logo-favicon/package.json +++ b/packages/metascraper-logo-favicon/package.json @@ -38,6 +38,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-logo-favicon/test/index.js b/packages/metascraper-logo-favicon/test/index.js index 04f6bdeac..56908f0a9 100644 --- a/packages/metascraper-logo-favicon/test/index.js +++ b/packages/metascraper-logo-favicon/test/index.js @@ -6,7 +6,7 @@ const { resolve } = require('path') const { omit } = require('lodash') const fs = require('fs') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-amazon')(), require('metascraper-author')(), require('metascraper-date')(), @@ -22,12 +22,11 @@ const metascraper = require('metascraper').load([ const readFile = promisify(fs.readFile) describe('metascraper-logo-favicon', () => { - describe('metascraper logo favicon', () => { - it('create an absolute favicon url if the logo is not present', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/input.html')) - const url = 'https://www.amazon.co.uk/Vegetable-Perfection-tasty-recipes-shoots/dp/1849757097/ref=asap_bc?ie=UTF8' - const meta = omit(await metascraper({ html, url }), ['date']) - snapshot(meta) - }) + it('create an absolute favicon url if the logo is not present', async () => { + const html = await readFile(resolve(__dirname, 'fixtures/input.html')) + const url = + 'https://www.amazon.co.uk/Vegetable-Perfection-tasty-recipes-shoots/dp/1849757097/ref=asap_bc?ie=UTF8' + const meta = omit(await metascraper({ html, url }), ['date']) + snapshot(meta) }) }) diff --git a/packages/metascraper-logo/package.json b/packages/metascraper-logo/package.json index 09f30027b..0660812cf 100644 --- a/packages/metascraper-logo/package.json +++ b/packages/metascraper-logo/package.json @@ -31,5 +31,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-publisher/package.json b/packages/metascraper-publisher/package.json index 64d2fff97..af04b3642 100644 --- a/packages/metascraper-publisher/package.json +++ b/packages/metascraper-publisher/package.json @@ -31,5 +31,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-soundcloud/package.json b/packages/metascraper-soundcloud/package.json index ecb56f80a..4f56fe9e5 100644 --- a/packages/metascraper-soundcloud/package.json +++ b/packages/metascraper-soundcloud/package.json @@ -40,6 +40,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-soundcloud/test/index.js b/packages/metascraper-soundcloud/test/index.js index cf713a7d1..b339b09df 100644 --- a/packages/metascraper-soundcloud/test/index.js +++ b/packages/metascraper-soundcloud/test/index.js @@ -5,7 +5,7 @@ const { promisify } = require('util') const { resolve } = require('path') const fs = require('fs') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-soundcloud')(), require('metascraper-author')(), require('metascraper-date')(), @@ -23,7 +23,8 @@ const readFile = promisify(fs.readFile) describe('metascraper-soundcloud', () => { it('song', async () => { const html = await readFile(resolve(__dirname, 'fixtures/song.html')) - const url = 'https://soundcloud.com/beautybrainsp/beauty-brain-swag-bandicoot' + const url = + 'https://soundcloud.com/beautybrainsp/beauty-brain-swag-bandicoot' const metadata = await metascraper({ html, url }) snapshot(metadata) diff --git a/packages/metascraper-title/package.json b/packages/metascraper-title/package.json index 9bd12617f..2f0c9a2ae 100644 --- a/packages/metascraper-title/package.json +++ b/packages/metascraper-title/package.json @@ -31,5 +31,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-url/package.json b/packages/metascraper-url/package.json index 311e38bd2..df227781d 100644 --- a/packages/metascraper-url/package.json +++ b/packages/metascraper-url/package.json @@ -30,5 +30,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-video-provider/package.json b/packages/metascraper-video-provider/package.json index 8016a7e99..314d842ea 100644 --- a/packages/metascraper-video-provider/package.json +++ b/packages/metascraper-video-provider/package.json @@ -40,6 +40,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js b/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js index 599d295b4..48f4d9d8b 100644 --- a/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js +++ b/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js @@ -29,7 +29,6 @@ const getGuestToken = async url => { const getTwitterVideoInfo = async url => { const tweetId = getTweetId(url) - console.log('tweetId', tweetId) const apiUrl = `https://api.twitter.com/2/timeline/conversation/${tweetId}.json?tweet_mode=extended` const { body } = await got(apiUrl, { json: true, diff --git a/packages/metascraper-video-provider/test/index.js b/packages/metascraper-video-provider/test/index.js index 3fd553948..c5f7642f9 100644 --- a/packages/metascraper-video-provider/test/index.js +++ b/packages/metascraper-video-provider/test/index.js @@ -9,7 +9,7 @@ const { omit } = require('lodash') const should = require('should') const fs = require('fs') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-video-provider')(), require('metascraper-author')(), require('metascraper-date')(), diff --git a/packages/metascraper-video/package.json b/packages/metascraper-video/package.json index 753ad3c86..505492dd0 100644 --- a/packages/metascraper-video/package.json +++ b/packages/metascraper-video/package.json @@ -37,6 +37,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-video/test/index.js b/packages/metascraper-video/test/index.js index ff5332442..437dacf43 100644 --- a/packages/metascraper-video/test/index.js +++ b/packages/metascraper-video/test/index.js @@ -6,7 +6,7 @@ const { resolve } = require('path') const { omit } = require('lodash') const fs = require('fs') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-video')(), require('metascraper-author')(), require('metascraper-date')(), @@ -25,14 +25,17 @@ describe('metascraper-video', () => { describe('video', () => { it('video src', async () => { const html = await readFile(resolve(__dirname, 'fixtures/video-src.html')) - const url = 'https://www.theverge.com/2018/1/22/16921092/pentagon-secret-nuclear-bunker-reconstruction-minecraft-cns-miis-model' + const url = + 'https://www.theverge.com/2018/1/22/16921092/pentagon-secret-nuclear-bunker-reconstruction-minecraft-cns-miis-model' const metadata = await metascraper({ html, url }) snapshot(metadata) }) it('source src', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/source-src.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/source-src.html') + ) const url = 'https://9gag.com/gag/aGjVLDK' const metadata = await metascraper({ html, url }) @@ -49,14 +52,18 @@ describe('metascraper-video', () => { describe('specific providers', () => { it('clips.twitch.tv', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/providers/clip.twitch.tv.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/providers/clip.twitch.tv.html') + ) const url = 'https://clips.twitch.tv/AwkwardBoredWaffleItsBoshyTime' const metadata = await metascraper({ html, url }) snapshot(metadata) }) it('play.tv', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/providers/play.tv.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/providers/play.tv.html') + ) const url = 'https://plays.tv/video/5a6f64b1bef69a7fa9/holy-shit' const metadata = await metascraper({ html, url }) snapshot(omit(metadata, ['date'])) @@ -66,7 +73,9 @@ describe('metascraper-video', () => { describe('image', () => { it('src:poster', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/source-poster.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/source-poster.html') + ) const url = 'https://gfycat.com/gifs/detail/timelyhealthyarmadillo' const metadata = await metascraper({ html, url }) snapshot(metadata) diff --git a/packages/metascraper-youtube/package.json b/packages/metascraper-youtube/package.json index 3091f4501..e43d2fa7b 100644 --- a/packages/metascraper-youtube/package.json +++ b/packages/metascraper-youtube/package.json @@ -44,6 +44,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-youtube/test/index.js b/packages/metascraper-youtube/test/index.js index 7b8af3da1..7e79f5d35 100644 --- a/packages/metascraper-youtube/test/index.js +++ b/packages/metascraper-youtube/test/index.js @@ -6,7 +6,7 @@ const { resolve } = require('path') const { omit } = require('lodash') const fs = require('fs') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-youtube')(), require('metascraper-author')(), require('metascraper-date')(), @@ -24,7 +24,9 @@ const readFile = promisify(fs.readFile) describe('metascraper-youtube', () => { describe('urls', () => { it('youtube video', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/youtube-video.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/youtube-video.html') + ) const url = 'https://www.youtube.com/watch?v=hwMkbaS_M_c' const metadata = await metascraper({ html, url }) @@ -32,7 +34,9 @@ describe('metascraper-youtube', () => { }) it('youtube video old', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/youtube-video-old.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/youtube-video-old.html') + ) const url = 'https://www.youtube.com/watch?v=GDRd-BFTYIg' const metadata = await metascraper({ html, url }) @@ -40,7 +44,9 @@ describe('metascraper-youtube', () => { }) it('youtube channel', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/youtube-channel.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/youtube-channel.html') + ) const url = 'https://www.youtube.com/channel/UCzcRQ3vRNr6fJ1A9rqFn7QA' const metadata = omit(await metascraper({ html, url }), ['date']) @@ -50,7 +56,9 @@ describe('metascraper-youtube', () => { describe('image size', () => { it('get the high image size', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/image-size.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/image-size.html') + ) const url = 'https://www.youtube.com/watch?v=rXyKq7izYCQ' const metadata = omit(await metascraper({ html, url }), ['date']) snapshot(metadata) diff --git a/packages/metascraper/package.json b/packages/metascraper/package.json index 22893ff57..b9bf6e1fc 100644 --- a/packages/metascraper/package.json +++ b/packages/metascraper/package.json @@ -58,20 +58,8 @@ "@metascraper/helpers": "^3.12.1", "cheerio": "~1.0.0-rc.2", "cheerio-advanced-selectors": "~2.0.1", - "cosmiconfig": "~5.0.2", "lodash": "~4.17.10", - "metascraper-author": "^3.12.1", - "metascraper-date": "^3.11.4", - "metascraper-description": "^3.12.1", - "metascraper-image": "^3.12.1", - "metascraper-lang": "^3.11.4", - "metascraper-logo": "^3.12.1", - "metascraper-publisher": "^3.11.4", - "metascraper-title": "^3.12.1", - "metascraper-url": "^3.12.1", - "metascraper-video": "^3.12.1", "p-reduce": "~1.0.0", - "resolve-from": "~4.0.0", "sanitize-html": "~1.18.2" }, "devDependencies": { diff --git a/packages/metascraper/src/index.js b/packages/metascraper/src/index.js index 93f0ae04c..2fd7be3e1 100644 --- a/packages/metascraper/src/index.js +++ b/packages/metascraper/src/index.js @@ -1,22 +1,23 @@ 'use strict' const { isUrl } = require('@metascraper/helpers') -const { isEmpty, partial } = require('lodash') +const { isEmpty } = require('lodash') -const {autoload: autoLoadRules, load: loadRules} = require('./load-rules') const mergeRules = require('./merge-rules') +const loadRules = require('./load-rules') const loadHTML = require('./load-html') const getData = require('./get-data') -const create = loader => { - const lazyLoadedRules = loader() - return async ({url, html, rules: inlineRules} = {}) => { - const loadedRules = await lazyLoadedRules +module.exports = rules => { + const lazyloader = loadRules(rules) + return async ({ url, html, rules: inlineRules } = {}) => { + const loadedRules = await lazyloader if (!isUrl(url)) throw new TypeError('You need to provide a valid url.') - if (isEmpty(html)) throw new TypeError('You need to provide a valid HTML markup.') - return getData({ url, htmlDom: loadHTML(html), rules: mergeRules(inlineRules, loadedRules) }) + if (isEmpty(html)) { throw new TypeError('You need to provide a valid HTML markup.') } + return getData({ + url, + htmlDom: loadHTML(html), + rules: mergeRules(inlineRules, loadedRules) + }) } } - -module.exports = create(autoLoadRules) -module.exports.load = rules => create(partial(loadRules, rules)) diff --git a/packages/metascraper/src/load-rules.js b/packages/metascraper/src/load-rules.js index adae33f7e..78b81ddd4 100644 --- a/packages/metascraper/src/load-rules.js +++ b/packages/metascraper/src/load-rules.js @@ -1,32 +1,14 @@ 'use strict' -const cwd = process.env.METASCRAPER_CWD || process.cwd() -const config = require('cosmiconfig')('metascraper').search(cwd) -const resolveFrom = require('resolve-from') +const { concat, findIndex, forEach, chain } = require('lodash') -const {concat, map, findIndex, forEach, chain, isObject, isArray, isString, get} = require('lodash') - -const DEFAULT_RULES = [ - 'metascraper-author', - 'metascraper-date', - 'metascraper-description', - 'metascraper-video', - 'metascraper-image', - 'metascraper-lang', - 'metascraper-logo', - 'metascraper-publisher', - 'metascraper-title', - 'metascraper-url' -] - -const load = rules => +module.exports = rules => chain(rules) // merge rules with same props .reduce((acc, rules) => { forEach(rules, function (rule, propName) { const index = findIndex(acc, item => item[propName]) - if (index !== -1) acc[index][propName] = concat(acc[index][propName], rule) - else acc.push({[propName]: concat(rule)}) + if (index !== -1) { acc[index][propName] = concat(acc[index][propName], rule) } else acc.push({ [propName]: concat(rule) }) }) return acc }, []) @@ -37,33 +19,3 @@ const load = rules => return [key, value] }) .value() - -const autoload = async () => { - const configFile = await config - const rulesConfig = get(configFile, 'config.rules', DEFAULT_RULES) - - const rules = map(rulesConfig, rule => { - let moduleName - let moduleConfig - - if (isString(rule)) { - moduleName = rule - } else if (isArray(rule)) { - moduleName = rule[0] - moduleConfig = rule[1] - } else if (isObject(rule)) { - moduleName = Object.keys(rule)[0] - moduleConfig = rule[moduleName] - } - - const modulePath = resolveFrom(cwd, moduleName) - return require(modulePath)(moduleConfig) - }) - - return load(rules) -} - -module.exports = { - autoload, - load -} diff --git a/packages/metascraper/test/integration/anandtech/index.js b/packages/metascraper/test/integration/anandtech/index.js index 5deebf2e2..050af44eb 100644 --- a/packages/metascraper/test/integration/anandtech/index.js +++ b/packages/metascraper/test/integration/anandtech/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/arstechnica/index.js b/packages/metascraper/test/integration/arstechnica/index.js index f4e0b540c..7deaad67d 100644 --- a/packages/metascraper/test/integration/arstechnica/index.js +++ b/packages/metascraper/test/integration/arstechnica/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/astier/index.js b/packages/metascraper/test/integration/astier/index.js index bba7001b5..283d93c6c 100644 --- a/packages/metascraper/test/integration/astier/index.js +++ b/packages/metascraper/test/integration/astier/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://anisse.astier.eu/awk-driven-iot.html' diff --git a/packages/metascraper/test/integration/atlasobscura/index.js b/packages/metascraper/test/integration/atlasobscura/index.js index 4b607b692..250443d85 100644 --- a/packages/metascraper/test/integration/atlasobscura/index.js +++ b/packages/metascraper/test/integration/atlasobscura/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://www.atlasobscura.com/articles/ikea-bowl-blanda-blank-fire' diff --git a/packages/metascraper/test/integration/audiense/index.js b/packages/metascraper/test/integration/audiense/index.js index 76ff9df83..6fbd742e3 100644 --- a/packages/metascraper/test/integration/audiense/index.js +++ b/packages/metascraper/test/integration/audiense/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/bbc/index.js b/packages/metascraper/test/integration/bbc/index.js index 836d0861c..82c9b346f 100644 --- a/packages/metascraper/test/integration/bbc/index.js +++ b/packages/metascraper/test/integration/bbc/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://www.bbc.com/news/business-40504764' diff --git a/packages/metascraper/test/integration/bloomberg/index.js b/packages/metascraper/test/integration/bloomberg/index.js index 6cf3bc5c5..e8b11d8fd 100644 --- a/packages/metascraper/test/integration/bloomberg/index.js +++ b/packages/metascraper/test/integration/bloomberg/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/business-today/index.js b/packages/metascraper/test/integration/business-today/index.js index 1eb63d5f6..f19c87ca0 100644 --- a/packages/metascraper/test/integration/business-today/index.js +++ b/packages/metascraper/test/integration/business-today/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/cbr/index.js b/packages/metascraper/test/integration/cbr/index.js index 20d413a34..4abc83cd7 100644 --- a/packages/metascraper/test/integration/cbr/index.js +++ b/packages/metascraper/test/integration/cbr/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/cio/index.js b/packages/metascraper/test/integration/cio/index.js index cb1a3142a..6acbbcae3 100644 --- a/packages/metascraper/test/integration/cio/index.js +++ b/packages/metascraper/test/integration/cio/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/cloud-pro/index.js b/packages/metascraper/test/integration/cloud-pro/index.js index bb824b5d0..727e93f60 100644 --- a/packages/metascraper/test/integration/cloud-pro/index.js +++ b/packages/metascraper/test/integration/cloud-pro/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://www.cloudpro.co.uk/go/6024' diff --git a/packages/metascraper/test/integration/cnet/index.js b/packages/metascraper/test/integration/cnet/index.js index b50cd5e6d..c98eb4022 100644 --- a/packages/metascraper/test/integration/cnet/index.js +++ b/packages/metascraper/test/integration/cnet/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/computerworld/index.js b/packages/metascraper/test/integration/computerworld/index.js index fcd23ee70..ab0a38006 100644 --- a/packages/metascraper/test/integration/computerworld/index.js +++ b/packages/metascraper/test/integration/computerworld/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/crn/index.js b/packages/metascraper/test/integration/crn/index.js index 1c99901a9..e4ce01a25 100644 --- a/packages/metascraper/test/integration/crn/index.js +++ b/packages/metascraper/test/integration/crn/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/economic-times/index.js b/packages/metascraper/test/integration/economic-times/index.js index df322cf21..ca7adacb1 100644 --- a/packages/metascraper/test/integration/economic-times/index.js +++ b/packages/metascraper/test/integration/economic-times/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/entrepreneur/index.js b/packages/metascraper/test/integration/entrepreneur/index.js index 5450b5439..7aa335ee7 100644 --- a/packages/metascraper/test/integration/entrepreneur/index.js +++ b/packages/metascraper/test/integration/entrepreneur/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://www.entrepreneur.com/article/275188' diff --git a/packages/metascraper/test/integration/et-tech/index.js b/packages/metascraper/test/integration/et-tech/index.js index d51a2bcff..d400981d4 100644 --- a/packages/metascraper/test/integration/et-tech/index.js +++ b/packages/metascraper/test/integration/et-tech/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/eweek/index.js b/packages/metascraper/test/integration/eweek/index.js index 3c5e2ebd8..9e9c52c9c 100644 --- a/packages/metascraper/test/integration/eweek/index.js +++ b/packages/metascraper/test/integration/eweek/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/fast-company/index.js b/packages/metascraper/test/integration/fast-company/index.js index af8d139df..2b3fb32a3 100644 --- a/packages/metascraper/test/integration/fast-company/index.js +++ b/packages/metascraper/test/integration/fast-company/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/fastersite/index.js b/packages/metascraper/test/integration/fastersite/index.js index 1b930f1d5..fe209bc7d 100644 --- a/packages/metascraper/test/integration/fastersite/index.js +++ b/packages/metascraper/test/integration/fastersite/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://gent.ilcore.com/2012/06/better-timer-for-javascript.html' diff --git a/packages/metascraper/test/integration/fierce-devops/index.js b/packages/metascraper/test/integration/fierce-devops/index.js index a9bedbf4d..5d0b985a0 100644 --- a/packages/metascraper/test/integration/fierce-devops/index.js +++ b/packages/metascraper/test/integration/fierce-devops/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/financial-times/index.js b/packages/metascraper/test/integration/financial-times/index.js index 37fd25dad..556b8c227 100644 --- a/packages/metascraper/test/integration/financial-times/index.js +++ b/packages/metascraper/test/integration/financial-times/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/forbes/index.js b/packages/metascraper/test/integration/forbes/index.js index 98a40ada1..f2d2abdfc 100644 --- a/packages/metascraper/test/integration/forbes/index.js +++ b/packages/metascraper/test/integration/forbes/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/fortune/index.js b/packages/metascraper/test/integration/fortune/index.js index f748c4451..fa231ce74 100644 --- a/packages/metascraper/test/integration/fortune/index.js +++ b/packages/metascraper/test/integration/fortune/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://fortune.com/2015/10/05/hackerrank-recruiting-tool' diff --git a/packages/metascraper/test/integration/geek-time/index.js b/packages/metascraper/test/integration/geek-time/index.js index ace9d2f55..c39ae5355 100644 --- a/packages/metascraper/test/integration/geek-time/index.js +++ b/packages/metascraper/test/integration/geek-time/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/github/index.js b/packages/metascraper/test/integration/github/index.js index 75685af07..85beb4a15 100644 --- a/packages/metascraper/test/integration/github/index.js +++ b/packages/metascraper/test/integration/github/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://github.com/microlinkhq/microlink-core' diff --git a/packages/metascraper/test/integration/googleblog/index.js b/packages/metascraper/test/integration/googleblog/index.js index 2cd995fb7..73e363975 100644 --- a/packages/metascraper/test/integration/googleblog/index.js +++ b/packages/metascraper/test/integration/googleblog/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/inc/index.js b/packages/metascraper/test/integration/inc/index.js index f41b0b59d..8c5b68fd0 100644 --- a/packages/metascraper/test/integration/inc/index.js +++ b/packages/metascraper/test/integration/inc/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://www.inc.com/jeremy-quittner/2016-30-under-30-neurensic.html' diff --git a/packages/metascraper/test/integration/information-week/index.js b/packages/metascraper/test/integration/information-week/index.js index 747ec39e7..16a8426e8 100644 --- a/packages/metascraper/test/integration/information-week/index.js +++ b/packages/metascraper/test/integration/information-week/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/instagram/index.js b/packages/metascraper/test/integration/instagram/index.js index 99643d390..79373744d 100644 --- a/packages/metascraper/test/integration/instagram/index.js +++ b/packages/metascraper/test/integration/instagram/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://www.instagram.com/p/BWUDBntl3_Z' diff --git a/packages/metascraper/test/integration/jewish-business-news/index.js b/packages/metascraper/test/integration/jewish-business-news/index.js index 8e7b8c091..d4bc0ed65 100644 --- a/packages/metascraper/test/integration/jewish-business-news/index.js +++ b/packages/metascraper/test/integration/jewish-business-news/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/lean-data/index.js b/packages/metascraper/test/integration/lean-data/index.js index df7f6c167..d6c3e3598 100644 --- a/packages/metascraper/test/integration/lean-data/index.js +++ b/packages/metascraper/test/integration/lean-data/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/linkedin-company/index.js b/packages/metascraper/test/integration/linkedin-company/index.js index a434fd228..818fa05f1 100644 --- a/packages/metascraper/test/integration/linkedin-company/index.js +++ b/packages/metascraper/test/integration/linkedin-company/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://www.linkedin.com/company/10529043/' diff --git a/packages/metascraper/test/integration/linkedin-pulse/index.js b/packages/metascraper/test/integration/linkedin-pulse/index.js index 08892560e..2e7cb41d1 100644 --- a/packages/metascraper/test/integration/linkedin-pulse/index.js +++ b/packages/metascraper/test/integration/linkedin-pulse/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/los-angeles-times/index.js b/packages/metascraper/test/integration/los-angeles-times/index.js index 9218dd9ce..4fc02b450 100644 --- a/packages/metascraper/test/integration/los-angeles-times/index.js +++ b/packages/metascraper/test/integration/los-angeles-times/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/mac-rumors/index.js b/packages/metascraper/test/integration/mac-rumors/index.js index 5c10ec3b9..3ab0bb560 100644 --- a/packages/metascraper/test/integration/mac-rumors/index.js +++ b/packages/metascraper/test/integration/mac-rumors/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/market-wired/index.js b/packages/metascraper/test/integration/market-wired/index.js index 5dc3cd6f5..df846c962 100644 --- a/packages/metascraper/test/integration/market-wired/index.js +++ b/packages/metascraper/test/integration/market-wired/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/marketing-land/index.js b/packages/metascraper/test/integration/marketing-land/index.js index c0f037381..52eeedfc9 100644 --- a/packages/metascraper/test/integration/marketing-land/index.js +++ b/packages/metascraper/test/integration/marketing-land/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/mashable/index.js b/packages/metascraper/test/integration/mashable/index.js index e298d2fa3..8c6400b21 100644 --- a/packages/metascraper/test/integration/mashable/index.js +++ b/packages/metascraper/test/integration/mashable/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://mashable.com/2015/05/13/analytics-power-up-revenue' diff --git a/packages/metascraper/test/integration/medium/index.js b/packages/metascraper/test/integration/medium/index.js index 02d99b656..c4cd7873f 100644 --- a/packages/metascraper/test/integration/medium/index.js +++ b/packages/metascraper/test/integration/medium/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://medium.com/webpack/webpack-3-official-release-15fd2dd8f07b' diff --git a/packages/metascraper/test/integration/motherboard/index.js b/packages/metascraper/test/integration/motherboard/index.js index 227d27e1c..0a25fdb62 100644 --- a/packages/metascraper/test/integration/motherboard/index.js +++ b/packages/metascraper/test/integration/motherboard/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/nytimes/index.js b/packages/metascraper/test/integration/nytimes/index.js index 26cf3a615..4f93d58cd 100644 --- a/packages/metascraper/test/integration/nytimes/index.js +++ b/packages/metascraper/test/integration/nytimes/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/pikabu/index.js b/packages/metascraper/test/integration/pikabu/index.js index 5edcf709e..0618844d5 100644 --- a/packages/metascraper/test/integration/pikabu/index.js +++ b/packages/metascraper/test/integration/pikabu/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://pikabu.ru' diff --git a/packages/metascraper/test/integration/pr-newswire/index.js b/packages/metascraper/test/integration/pr-newswire/index.js index 578263ec0..c7540e212 100644 --- a/packages/metascraper/test/integration/pr-newswire/index.js +++ b/packages/metascraper/test/integration/pr-newswire/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/recode/index.js b/packages/metascraper/test/integration/recode/index.js index 772076f19..4d6ebaf22 100644 --- a/packages/metascraper/test/integration/recode/index.js +++ b/packages/metascraper/test/integration/recode/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/reuters/index.js b/packages/metascraper/test/integration/reuters/index.js index a64b6064f..85d6c4774 100644 --- a/packages/metascraper/test/integration/reuters/index.js +++ b/packages/metascraper/test/integration/reuters/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/san-francisco-chronicle/index.js b/packages/metascraper/test/integration/san-francisco-chronicle/index.js index 25e1f96a8..a00dbbb95 100644 --- a/packages/metascraper/test/integration/san-francisco-chronicle/index.js +++ b/packages/metascraper/test/integration/san-francisco-chronicle/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/segment-academy/index.js b/packages/metascraper/test/integration/segment-academy/index.js index ef47d9d55..9d2499427 100644 --- a/packages/metascraper/test/integration/segment-academy/index.js +++ b/packages/metascraper/test/integration/segment-academy/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/segment/index.js b/packages/metascraper/test/integration/segment/index.js index 5dae5ff15..44136e6ee 100644 --- a/packages/metascraper/test/integration/segment/index.js +++ b/packages/metascraper/test/integration/segment/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://segment.com/blog/scaling-nsq' diff --git a/packages/metascraper/test/integration/silicon-angle/index.js b/packages/metascraper/test/integration/silicon-angle/index.js index c790b50c3..522064b9d 100644 --- a/packages/metascraper/test/integration/silicon-angle/index.js +++ b/packages/metascraper/test/integration/silicon-angle/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/silicon-beat/index.js b/packages/metascraper/test/integration/silicon-beat/index.js index c4c605f80..e37fed374 100644 --- a/packages/metascraper/test/integration/silicon-beat/index.js +++ b/packages/metascraper/test/integration/silicon-beat/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://www.siliconbeat.com/2016/04/19/time-come-woman' diff --git a/packages/metascraper/test/integration/silicon-tap/index.js b/packages/metascraper/test/integration/silicon-tap/index.js index 417b9d727..8e1049628 100644 --- a/packages/metascraper/test/integration/silicon-tap/index.js +++ b/packages/metascraper/test/integration/silicon-tap/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://www.silicontap.com/story/0053475.html' diff --git a/packages/metascraper/test/integration/silicon-valley-business-journal/index.js b/packages/metascraper/test/integration/silicon-valley-business-journal/index.js index b4a0ed7a4..3fcdd8fbf 100644 --- a/packages/metascraper/test/integration/silicon-valley-business-journal/index.js +++ b/packages/metascraper/test/integration/silicon-valley-business-journal/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/smitten-kitchen/index.js b/packages/metascraper/test/integration/smitten-kitchen/index.js index 9e8b0a254..03f31565a 100644 --- a/packages/metascraper/test/integration/smitten-kitchen/index.js +++ b/packages/metascraper/test/integration/smitten-kitchen/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://smittenkitchen.com/blog/2016/05/cucumber-yogurt-raita-salad' diff --git a/packages/metascraper/test/integration/startup-grind/index.js b/packages/metascraper/test/integration/startup-grind/index.js index 41ad13927..ae9e272c9 100644 --- a/packages/metascraper/test/integration/startup-grind/index.js +++ b/packages/metascraper/test/integration/startup-grind/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/techcrunch/index.js b/packages/metascraper/test/integration/techcrunch/index.js index f1c8b260c..15f31f50b 100644 --- a/packages/metascraper/test/integration/techcrunch/index.js +++ b/packages/metascraper/test/integration/techcrunch/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/the-boston-globe/index.js b/packages/metascraper/test/integration/the-boston-globe/index.js index 3edd5db38..387fd8707 100644 --- a/packages/metascraper/test/integration/the-boston-globe/index.js +++ b/packages/metascraper/test/integration/the-boston-globe/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/the-guardian/index.js b/packages/metascraper/test/integration/the-guardian/index.js index 2cb27145c..b0e5dea8d 100644 --- a/packages/metascraper/test/integration/the-guardian/index.js +++ b/packages/metascraper/test/integration/the-guardian/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/the-register/index.js b/packages/metascraper/test/integration/the-register/index.js index 300ca0f27..3a0d431b9 100644 --- a/packages/metascraper/test/integration/the-register/index.js +++ b/packages/metascraper/test/integration/the-register/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/the-verge/index.js b/packages/metascraper/test/integration/the-verge/index.js index 4f5815a5d..b3463f3c0 100644 --- a/packages/metascraper/test/integration/the-verge/index.js +++ b/packages/metascraper/test/integration/the-verge/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/twitter-gif/index.js b/packages/metascraper/test/integration/twitter-gif/index.js index a9fadc65d..0cb58bb36 100644 --- a/packages/metascraper/test/integration/twitter-gif/index.js +++ b/packages/metascraper/test/integration/twitter-gif/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://twitter.com/Kikobeats/status/880139124791029763' diff --git a/packages/metascraper/test/integration/twitter-image/index.js b/packages/metascraper/test/integration/twitter-image/index.js index 4b3bbc268..e6f8d38a1 100644 --- a/packages/metascraper/test/integration/twitter-image/index.js +++ b/packages/metascraper/test/integration/twitter-image/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://twitter.com/k4rliky/status/934482867480121345' diff --git a/packages/metascraper/test/integration/usa-today/index.js b/packages/metascraper/test/integration/usa-today/index.js index 023b7c1c3..79b928567 100644 --- a/packages/metascraper/test/integration/usa-today/index.js +++ b/packages/metascraper/test/integration/usa-today/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/venture-beat/index.js b/packages/metascraper/test/integration/venture-beat/index.js index e4b217855..2d2152202 100644 --- a/packages/metascraper/test/integration/venture-beat/index.js +++ b/packages/metascraper/test/integration/venture-beat/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/vimeo/index.js b/packages/metascraper/test/integration/vimeo/index.js index 0ef83e251..20d34d789 100644 --- a/packages/metascraper/test/integration/vimeo/index.js +++ b/packages/metascraper/test/integration/vimeo/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://vimeo.com/200104989' diff --git a/packages/metascraper/test/integration/washington-post/index.js b/packages/metascraper/test/integration/washington-post/index.js index 38e3ea983..c4b1f2768 100644 --- a/packages/metascraper/test/integration/washington-post/index.js +++ b/packages/metascraper/test/integration/washington-post/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/wikipedia/index.js b/packages/metascraper/test/integration/wikipedia/index.js index ba04512a1..e0b3cd9c8 100644 --- a/packages/metascraper/test/integration/wikipedia/index.js +++ b/packages/metascraper/test/integration/wikipedia/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://en.wikipedia.org/wiki/Bob_Dylan' diff --git a/packages/metascraper/test/integration/wired/index.js b/packages/metascraper/test/integration/wired/index.js index cf3abf108..7e1ec3037 100644 --- a/packages/metascraper/test/integration/wired/index.js +++ b/packages/metascraper/test/integration/wired/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/wsj/index.js b/packages/metascraper/test/integration/wsj/index.js index 38519b95b..1219b9684 100644 --- a/packages/metascraper/test/integration/wsj/index.js +++ b/packages/metascraper/test/integration/wsj/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/xconomy/index.js b/packages/metascraper/test/integration/xconomy/index.js index 04d265567..e0394c5fd 100644 --- a/packages/metascraper/test/integration/xconomy/index.js +++ b/packages/metascraper/test/integration/xconomy/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/yahoo-news/index.js b/packages/metascraper/test/integration/yahoo-news/index.js index 3eb803140..9f65cfe91 100644 --- a/packages/metascraper/test/integration/yahoo-news/index.js +++ b/packages/metascraper/test/integration/yahoo-news/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/zdnet/index.js b/packages/metascraper/test/integration/zdnet/index.js index b5538c7d5..168c7a955 100644 --- a/packages/metascraper/test/integration/zdnet/index.js +++ b/packages/metascraper/test/integration/zdnet/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/unit/interface.js b/packages/metascraper/test/unit/interface.js index c096e7b01..9c8519371 100644 --- a/packages/metascraper/test/unit/interface.js +++ b/packages/metascraper/test/unit/interface.js @@ -1,9 +1,19 @@ - 'use strict' const should = require('should') -const metascraper = require('../..') +const metascraper = require('../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) it('url is required', async () => { try { @@ -49,9 +59,7 @@ it('load extra rules', async () => { const rules = [ { - foo: [ - () => 'bar' - ] + foo: [() => 'bar'] } ] diff --git a/packages/metascraper/test/unit/load-rules/config/index.js b/packages/metascraper/test/unit/load-rules/config/index.js deleted file mode 100644 index b045669a3..000000000 --- a/packages/metascraper/test/unit/load-rules/config/index.js +++ /dev/null @@ -1,29 +0,0 @@ -'use strict' - -const clearModule = require('clear-module') -const should = require('should') - -let metascraper - -describe('automagically load rules', () => { - before(() => { - clearModule.all() - process.env.METASCRAPER_CWD = __dirname - metascraper = require('../../../..') - }) - - after(() => { - clearModule.all() - delete process.env.METASCRAPER_CWD - }) - - it('based on a configuration file', async () => { - const url = 'https://facebook.com' - const html = '
' - - const meta = await metascraper({ url, html }) - should(meta.logo).be.equal( - 'https://logo.clearbit.com/facebook.com?size=128&format=jpg' - ) - }) -}) diff --git a/packages/metascraper/test/unit/load-rules/config/metascraper.config.js b/packages/metascraper/test/unit/load-rules/config/metascraper.config.js deleted file mode 100644 index a902272ea..000000000 --- a/packages/metascraper/test/unit/load-rules/config/metascraper.config.js +++ /dev/null @@ -1,17 +0,0 @@ -module.exports = { - rules: [ - 'metascraper-author', - 'metascraper-date', - 'metascraper-description', - 'metascraper-image', - 'metascraper-logo', - { - 'metascraper-clearbit-logo': { - format: 'jpg' - } - }, - 'metascraper-publisher', - 'metascraper-title', - 'metascraper-url' - ] -} diff --git a/packages/metascraper/test/unit/load-rules/pkg/index.js b/packages/metascraper/test/unit/load-rules/pkg/index.js deleted file mode 100644 index 62627917b..000000000 --- a/packages/metascraper/test/unit/load-rules/pkg/index.js +++ /dev/null @@ -1,29 +0,0 @@ -'use strict' - -const clearModule = require('clear-module') -const should = require('should') - -let metascraper - -describe('automagically load rules', () => { - before(() => { - clearModule.all() - process.env.METASCRAPER_CWD = __dirname - metascraper = require('../../../..') - }) - - after(() => { - clearModule.all() - delete process.env.METASCRAPER_CWD - }) - - it('based on a package file', async () => { - const url = 'https://facebook.com' - const html = '
' - - const meta = await metascraper({ url, html }) - should(meta.logo).be.equal( - 'https://logo.clearbit.com/facebook.com?size=128&format=jpg' - ) - }) -}) diff --git a/packages/metascraper/test/unit/load-rules/pkg/package.json b/packages/metascraper/test/unit/load-rules/pkg/package.json deleted file mode 100644 index fe3f59811..000000000 --- a/packages/metascraper/test/unit/load-rules/pkg/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "metascraper": { - "rules": [ - "metascraper-author", - "metascraper-date", - "metascraper-description", - "metascraper-image", - "metascraper-logo", - { - "metascraper-clearbit-logo": { - "format": "jpg" - } - }, - "metascraper-publisher", - "metascraper-title", - "metascraper-url" - ] - } -} diff --git a/packages/metascraper/test/unit/load-rules/rc/.metascraperrc b/packages/metascraper/test/unit/load-rules/rc/.metascraperrc deleted file mode 100644 index 5daeffa19..000000000 --- a/packages/metascraper/test/unit/load-rules/rc/.metascraperrc +++ /dev/null @@ -1,15 +0,0 @@ -{ - "rules": [ - "metascraper-author", - "metascraper-date", - "metascraper-description", - "metascraper-image", - "metascraper-logo", - {"metascraper-clearbit-logo": { - "format": "jpg" - }}, - "metascraper-publisher", - "metascraper-title", - "metascraper-url" - ] -} diff --git a/packages/metascraper/test/unit/load-rules/rc/index.js b/packages/metascraper/test/unit/load-rules/rc/index.js deleted file mode 100644 index f2e86df87..000000000 --- a/packages/metascraper/test/unit/load-rules/rc/index.js +++ /dev/null @@ -1,29 +0,0 @@ -'use strict' - -const clearModule = require('clear-module') -const should = require('should') - -let metascraper - -describe('automagically load rules', () => { - before(() => { - clearModule.all() - process.env.METASCRAPER_CWD = __dirname - metascraper = require('../../../..') - }) - - after(() => { - clearModule.all() - delete process.env.METASCRAPER_CWD - }) - - it('based on a rc file', async () => { - const url = 'https://facebook.com' - const html = '
' - - const meta = await metascraper({ url, html }) - should(meta.logo).be.equal( - 'https://logo.clearbit.com/facebook.com?size=128&format=jpg' - ) - }) -}) diff --git a/packages/metascraper/test/unit/merge-rules.js b/packages/metascraper/test/unit/merge-rules.js index a4b0d2a73..c99c763e1 100644 --- a/packages/metascraper/test/unit/merge-rules.js +++ b/packages/metascraper/test/unit/merge-rules.js @@ -2,7 +2,7 @@ const should = require('should') -it('add a new rule from a prop that doesn\'t exist', async () => { +it("add a new rule from a prop that doesn't exist", async () => { const url = 'https://microlink.io' const html = ` @@ -30,14 +30,12 @@ it('add a new rule from a prop that doesn\'t exist', async () => { const rules = [ { - foo: [ - () => 'bar' - ] + foo: [() => 'bar'] } ] - const metascraper = require('../..').load([]) - const meta = await metascraper({url, html, rules}) + const metascraper = require('../..')([]) + const meta = await metascraper({ url, html, rules }) should(meta.foo).be.equal('bar') }) @@ -69,21 +67,17 @@ it('add a new rule for a prop that exists', async () => { const rules = [ { - foo: [ - () => 'bar' - ] + foo: [() => 'bar'] } ] - const metascraper = require('../..').load([{ - foo: [ - () => false, - () => false, - () => false - ] - }]) + const metascraper = require('../..')([ + { + foo: [() => false, () => false, () => false] + } + ]) - const meta = await metascraper({url, html, rules}) + const meta = await metascraper({ url, html, rules }) should(meta.foo).be.equal('bar') }) @@ -115,20 +109,16 @@ it('rules are added from the end', async () => { const rules = [ { - foo: [ - () => 'bar' - ] + foo: [() => 'bar'] } ] - const metascraper = require('../..').load([{ - foo: [ - () => false, - () => false, - () => 'baz' - ] - }]) + const metascraper = require('../..')([ + { + foo: [() => false, () => false, () => 'baz'] + } + ]) - const meta = await metascraper({url, html, rules}) + const meta = await metascraper({ url, html, rules }) should(meta.foo).be.equal('baz') }) From d63fcc80b209c87acce562099f2a5791bcdd8e2a Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Thu, 23 Aug 2018 19:18:47 +0200 Subject: [PATCH 013/442] Remove lazy intializer code is sync --- packages/metascraper/src/index.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/metascraper/src/index.js b/packages/metascraper/src/index.js index 2fd7be3e1..0ca2d84f7 100644 --- a/packages/metascraper/src/index.js +++ b/packages/metascraper/src/index.js @@ -9,11 +9,12 @@ const loadHTML = require('./load-html') const getData = require('./get-data') module.exports = rules => { - const lazyloader = loadRules(rules) + const loadedRules = loadRules(rules) return async ({ url, html, rules: inlineRules } = {}) => { - const loadedRules = await lazyloader if (!isUrl(url)) throw new TypeError('You need to provide a valid url.') - if (isEmpty(html)) { throw new TypeError('You need to provide a valid HTML markup.') } + if (isEmpty(html)) { + throw new TypeError('You need to provide a valid HTML markup.') + } return getData({ url, htmlDom: loadHTML(html), From 7fd6b15397be3025a78279757f0d077ef95dd2d4 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Thu, 23 Aug 2018 19:52:49 +0200 Subject: [PATCH 014/442] Update Docs --- README.md | 97 ++++++++++++++++++++++++++----------------------------- 1 file changed, 46 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 458c388bf..cfa47f191 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,10 @@ - [Usage](#usage) - [Metadata](#metadata) - [How it works](#how-it-works) -- [Customization](#customization) -- [Rules](#rules) +- [Importing Rules](#importing-rules) +- [Rules bundles](#rules-bundles) - [API](#api) -- [Environment Variables](#environment-variables) -- [Comparison](#comparison) +- [Benchmark](#benchmark) - [License](#license) ## Getting Started @@ -48,8 +47,21 @@ Let's extract accurate information from the following article: [![](https://raw.githubusercontent.com/microlinkhq/metascraper/add-comparison/support/screenshot.png)](http://www.bloomberg.com/news/articles/2016-05-24/as-zenefits-stumbles-gusto-goes-head-on-by-selling-insurance) +Then call **metascraper** with the rules bundle you want to apply for extracting content: + ```js -const metascraper = require('metascraper') +const metascraper = require('metascraper')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-image')(), + require('metascraper-logo')(), + require('metascraper-clearbit-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const got = require('got') const targetUrl = 'http://www.bloomberg.com/news/articles/2016-05-24/as-zenefits-stumbles-gusto-goes-head-on-by-selling-insurance' @@ -61,7 +73,8 @@ const targetUrl = 'http://www.bloomberg.com/news/articles/2016-05-24/as-zenefits })() ``` -Where the output will be something like: + +The output will be something like: ```json { @@ -77,7 +90,9 @@ Where the output will be something like: ## Metadata -Here is a list of the metadata that **metascraper** collects by default: +?> Other metadata can be defined using a custom [rule bundle](#rule-bundles). + +Here is an example of the metadata that **metascraper** can collect: - `author` — eg. *Noah Kulwin*
A human-readable representation of the author's name. @@ -111,13 +126,11 @@ Here is a list of the metadata that **metascraper** collects by default: ## How it works -?> Configuration file follow the same approach than projects like Babel or Prettier. - -**metascraper** is built out of rules. +**metascraper** is built out of rules bundles. It was designed to be easy to adapt. You can compose your own transformation pipeline using existing rules or write your own. -Rules are a collection of HTML selectors around a determinate property. When you load the library, implicitly it is loading [core rules](#core-rules). +Rules bundles are a collection of HTML selectors around a determinate property. When you load the library, implicitly it is loading [core rules](#core-rules). Each set of rules load a set of selectors in order to get a determinate value. @@ -131,9 +144,9 @@ Rules work as fallback between them: **metascraper** do that until finish all the rule or find the first rule that resolves the value. -## Usage +## Importing Rules -**metascraper** exports a constructors that need to be initalized providing a collection of rules set to load: +**metascraper** exports a constructor that need to be initialized providing a collection of rules to load: ```js const metascraper = require('metascraper')([ @@ -151,7 +164,7 @@ const metascraper = require('metascraper')([ Again, the order of rules are loaded are important: Just the first rule that resolve the value will be applied. -Use the first parameter to pass custom options specific per each rules set: +Use the first parameter to pass custom options specific per each rules bundle: ```js const metascraper = require('metascraper')([ @@ -162,13 +175,9 @@ const metascraper = require('metascraper')([ ]) ``` -## Rules +## Rules bundles -?> Can't find a rules set that you want? Let's [open an issue](https://github.com/microlinkhq/metascraper/issues/new) to create it. - -### Core rules - -These rules set will be shipped with **metascraper** and loaded by default. +?> Can't find the rules bundle that you want? Let's [open an issue](https://github.com/microlinkhq/metascraper/issues/new) to create it. | Package | Version | Dependencies | |--------|-------|------------| @@ -181,13 +190,6 @@ These rules set will be shipped with **metascraper** and loaded by default. | [`metascraper-publisher`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-publisher) | [![npm](https://img.shields.io/npm/v/metascraper-publisher.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-publisher) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-publisher&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-publisher) | | [`metascraper-title`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-title) | [![npm](https://img.shields.io/npm/v/metascraper-title.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-title) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-title&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-title) | | [`metascraper-url`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-url) | [![npm](https://img.shields.io/npm/v/metascraper-url.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-url) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-url&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-url) | - -### Community rules - -These rule set will not be shipped with **metascraper** by default and need to be specific using a configuration file. - -| Package | Version | Dependencies | -|--------|-------|------------| | [`metascraper-amazon`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-amazon) | [![npm](https://img.shields.io/npm/v/metascraper-amazon.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-amazon) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-amazon&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-amazon) | | [`metascraper-clearbit-logo`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-clearbit-logo) | [![npm](https://img.shields.io/npm/v/metascraper-clearbit-logo.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-clearbit-logo) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-clearbit-logo&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-clearbit-logo) | | [`metascraper-logo-favicon`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-logo-favicon) | [![npm](https://img.shields.io/npm/v/metascraper-logo-favicon.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-logo-favicon) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-logo-favicon&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-logo-favicon) | @@ -197,11 +199,11 @@ These rule set will not be shipped with **metascraper** by default and need to ### Write your own rules -A rule set is the simplest way for extending **metascraper** functionality. +A rule bundle is the simplest way for extending **metascraper** functionality. -A rule set can add one or more properties support. +A rule bundle can add one or more properties support. -The following schema represents the API compromise that a rule set need to follow: +The following schema represents the API compromise that a rule bundle need to follow: ```js 'use strict' @@ -224,12 +226,24 @@ module.exports = opts => { } ``` -We recommend check [core rules packages](/packages) as examples. +We recommend check one of the [rule bundles](#rules-bundles) already implemented to have a better perspective. ## API +### constructor(rules) + +Create a new **metascraper** instance declaring the rules bundle to be used explicitly. + +#### rules + +Type: `Array` + +The collection of rules bundle to be loaded. + ### metascraper(options) +Call the instance for extracting content based on rules bundle provided at the constructor. + #### options ##### html @@ -256,26 +270,7 @@ Type: `Array` You can pass additional rules on execution time. These rules will be merged with your loaded rules. -### metascraper([rules]) - -Create a new **metascraper** instance declaring the rules set to be used explicitly. - -#### rules - -Type: `Array` - -The collection fo rules set to be loaded. - -## Environment Variables - -### METASCRAPER_CWD - -Type: `String`
-Default: `process.cwd()` - -This variable is used to determine where starting search for a configuration object. - -## Comparison +## Benchmark To give you an idea of how accurate **metascraper** is, here is a comparison of similar libraries: From 19aca42d8e1dc0fa06c3417fe25b090d72d7e524 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Thu, 23 Aug 2018 20:08:44 +0200 Subject: [PATCH 015/442] Update dependencies --- packages/metascraper-clearbit-logo/package.json | 2 +- packages/metascraper-video-provider/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/metascraper-clearbit-logo/package.json b/packages/metascraper-clearbit-logo/package.json index 740f7765f..bf1445a61 100644 --- a/packages/metascraper-clearbit-logo/package.json +++ b/packages/metascraper-clearbit-logo/package.json @@ -21,7 +21,7 @@ "metascraper" ], "dependencies": { - "got": "~9.0.0" + "got": "~9.1.0" }, "devDependencies": { "mocha": "latest", diff --git a/packages/metascraper-video-provider/package.json b/packages/metascraper-video-provider/package.json index 314d842ea..41827aace 100644 --- a/packages/metascraper-video-provider/package.json +++ b/packages/metascraper-video-provider/package.json @@ -17,7 +17,7 @@ }, "dependencies": { "@metascraper/helpers": "^3.12.1", - "got": "~9.0.0", + "got": "~9.1.0", "json-future": "~2.1.2", "lodash": "~4.17.10", "youtube-dl": "~1.12.2" From 60b0adb15cf69a6bc0c3dc77ab3fd54c0b790f9f Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Thu, 23 Aug 2018 20:10:11 +0200 Subject: [PATCH 016/442] Remove autoload (#106) * Remove autoload * Remove lazy intializer code is sync * Update Docs --- README.md | 214 ++++-------------- packages/metascraper-amazon/package.json | 3 + packages/metascraper-amazon/test/index.js | 27 ++- packages/metascraper-author/package.json | 5 +- .../metascraper-clearbit-logo/package.json | 9 +- .../metascraper-clearbit-logo/test/index.js | 2 +- packages/metascraper-date/package.json | 5 +- packages/metascraper-description/package.json | 5 +- packages/metascraper-image/package.json | 5 +- packages/metascraper-lang/package.json | 4 +- packages/metascraper-lang/test/index.js | 5 +- .../metascraper-logo-favicon/package.json | 3 + .../metascraper-logo-favicon/test/index.js | 15 +- packages/metascraper-logo/package.json | 5 +- packages/metascraper-publisher/package.json | 5 +- packages/metascraper-soundcloud/package.json | 3 + packages/metascraper-soundcloud/test/index.js | 5 +- packages/metascraper-title/package.json | 5 +- packages/metascraper-url/package.json | 5 +- .../metascraper-video-provider/package.json | 3 + .../src/get-video-info/twitter-video-info.js | 1 - .../metascraper-video-provider/test/index.js | 2 +- packages/metascraper-video/package.json | 3 + packages/metascraper-video/test/index.js | 21 +- packages/metascraper-youtube/package.json | 3 + packages/metascraper-youtube/test/index.js | 18 +- packages/metascraper/package.json | 12 - packages/metascraper/src/index.js | 24 +- packages/metascraper/src/load-rules.js | 54 +---- .../test/integration/anandtech/index.js | 14 +- .../test/integration/arstechnica/index.js | 14 +- .../test/integration/astier/index.js | 14 +- .../test/integration/atlasobscura/index.js | 14 +- .../test/integration/audiense/index.js | 14 +- .../metascraper/test/integration/bbc/index.js | 14 +- .../test/integration/bloomberg/index.js | 14 +- .../test/integration/business-today/index.js | 14 +- .../metascraper/test/integration/cbr/index.js | 14 +- .../metascraper/test/integration/cio/index.js | 14 +- .../test/integration/cloud-pro/index.js | 14 +- .../test/integration/cnet/index.js | 14 +- .../test/integration/computerworld/index.js | 14 +- .../metascraper/test/integration/crn/index.js | 14 +- .../test/integration/economic-times/index.js | 14 +- .../test/integration/entrepreneur/index.js | 14 +- .../test/integration/et-tech/index.js | 14 +- .../test/integration/eweek/index.js | 14 +- .../test/integration/fast-company/index.js | 14 +- .../test/integration/fastersite/index.js | 14 +- .../test/integration/fierce-devops/index.js | 14 +- .../test/integration/financial-times/index.js | 14 +- .../test/integration/forbes/index.js | 14 +- .../test/integration/fortune/index.js | 14 +- .../test/integration/geek-time/index.js | 14 +- .../test/integration/github/index.js | 14 +- .../test/integration/googleblog/index.js | 14 +- .../metascraper/test/integration/inc/index.js | 14 +- .../integration/information-week/index.js | 14 +- .../test/integration/instagram/index.js | 14 +- .../integration/jewish-business-news/index.js | 14 +- .../test/integration/lean-data/index.js | 14 +- .../integration/linkedin-company/index.js | 14 +- .../test/integration/linkedin-pulse/index.js | 14 +- .../integration/los-angeles-times/index.js | 14 +- .../test/integration/mac-rumors/index.js | 14 +- .../test/integration/market-wired/index.js | 14 +- .../test/integration/marketing-land/index.js | 14 +- .../test/integration/mashable/index.js | 14 +- .../test/integration/medium/index.js | 14 +- .../test/integration/motherboard/index.js | 14 +- .../test/integration/nytimes/index.js | 14 +- .../test/integration/pikabu/index.js | 14 +- .../test/integration/pr-newswire/index.js | 14 +- .../test/integration/recode/index.js | 14 +- .../test/integration/reuters/index.js | 14 +- .../san-francisco-chronicle/index.js | 14 +- .../test/integration/segment-academy/index.js | 14 +- .../test/integration/segment/index.js | 14 +- .../test/integration/silicon-angle/index.js | 14 +- .../test/integration/silicon-beat/index.js | 14 +- .../test/integration/silicon-tap/index.js | 14 +- .../silicon-valley-business-journal/index.js | 14 +- .../test/integration/smitten-kitchen/index.js | 14 +- .../test/integration/startup-grind/index.js | 14 +- .../test/integration/techcrunch/index.js | 14 +- .../integration/the-boston-globe/index.js | 14 +- .../test/integration/the-guardian/index.js | 14 +- .../test/integration/the-register/index.js | 14 +- .../test/integration/the-verge/index.js | 14 +- .../test/integration/twitter-gif/index.js | 14 +- .../test/integration/twitter-image/index.js | 14 +- .../test/integration/usa-today/index.js | 14 +- .../test/integration/venture-beat/index.js | 14 +- .../test/integration/vimeo/index.js | 14 +- .../test/integration/washington-post/index.js | 14 +- .../test/integration/wikipedia/index.js | 14 +- .../test/integration/wired/index.js | 14 +- .../metascraper/test/integration/wsj/index.js | 14 +- .../test/integration/xconomy/index.js | 14 +- .../test/integration/yahoo-news/index.js | 14 +- .../test/integration/zdnet/index.js | 14 +- packages/metascraper/test/unit/interface.js | 18 +- .../test/unit/load-rules/config/index.js | 29 --- .../load-rules/config/metascraper.config.js | 17 -- .../test/unit/load-rules/pkg/index.js | 29 --- .../test/unit/load-rules/pkg/package.json | 19 -- .../test/unit/load-rules/rc/.metascraperrc | 15 -- .../test/unit/load-rules/rc/index.js | 29 --- packages/metascraper/test/unit/merge-rules.js | 46 ++-- 109 files changed, 1153 insertions(+), 528 deletions(-) delete mode 100644 packages/metascraper/test/unit/load-rules/config/index.js delete mode 100644 packages/metascraper/test/unit/load-rules/config/metascraper.config.js delete mode 100644 packages/metascraper/test/unit/load-rules/pkg/index.js delete mode 100644 packages/metascraper/test/unit/load-rules/pkg/package.json delete mode 100644 packages/metascraper/test/unit/load-rules/rc/.metascraperrc delete mode 100644 packages/metascraper/test/unit/load-rules/rc/index.js diff --git a/README.md b/README.md index 548a0dad6..d3591ad90 100644 --- a/README.md +++ b/README.md @@ -14,16 +14,16 @@ ## Table of Contents +- [Table of Contents](#table-of-contents) - [Getting Started](#getting-started) - [Installation](#installation) - [Usage](#usage) - [Metadata](#metadata) - [How it works](#how-it-works) -- [Customization](#customization) -- [Rules](#rules) +- [Importing Rules](#importing-rules) +- [Rules bundles](#rules-bundles) - [API](#api) -- [Environment Variables](#environment-variables) -- [Comparison](#comparison) +- [Benchmark](#benchmark) - [License](#license) ## Getting Started @@ -48,8 +48,21 @@ Let's extract accurate information from the following article: [![](https://raw.githubusercontent.com/microlinkhq/metascraper/add-comparison/support/screenshot.png)](http://www.bloomberg.com/news/articles/2016-05-24/as-zenefits-stumbles-gusto-goes-head-on-by-selling-insurance) +Then call **metascraper** with the rules bundle you want to apply for extracting content: + ```js -const metascraper = require('metascraper') +const metascraper = require('metascraper')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-image')(), + require('metascraper-logo')(), + require('metascraper-clearbit-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const got = require('got') const targetUrl = 'http://www.bloomberg.com/news/articles/2016-05-24/as-zenefits-stumbles-gusto-goes-head-on-by-selling-insurance' @@ -61,7 +74,8 @@ const targetUrl = 'http://www.bloomberg.com/news/articles/2016-05-24/as-zenefits })() ``` -Where the output will be something like: + +The output will be something like: ```json { @@ -77,7 +91,9 @@ Where the output will be something like: ## Metadata -Here is a list of the metadata that **metascraper** collects by default: +?> Other metadata can be defined using a custom [rule bundle](#rule-bundles). + +Here is an example of the metadata that **metascraper** can collect: - `author` — eg. *Noah Kulwin*
A human-readable representation of the author's name. @@ -111,13 +127,11 @@ Here is a list of the metadata that **metascraper** collects by default: ## How it works -?> Configuration file follow the same approach than projects like Babel or Prettier. - -**metascraper** is built out of rules. +**metascraper** is built out of rules bundles. It was designed to be easy to adapt. You can compose your own transformation pipeline using existing rules or write your own. -Rules are a collection of HTML selectors around a determinate property. When you load the library, implicitly it is loading [core rules](#core-rules). +Rules bundles are a collection of HTML selectors around a determinate property. When you load the library, implicitly it is loading [core rules](#core-rules). Each set of rules load a set of selectors in order to get a determinate value. @@ -131,116 +145,12 @@ Rules work as fallback between them: **metascraper** do that until finish all the rule or find the first rule that resolves the value. -## Loading rules - -When you call **metascraper** in your code, a set of [core rules](#core-rules) are loaded by default. - -Although these rules are sufficient for most cases, **metascraper** was designed to be easy to adapt and load more or custom rules set. - -We provide two approach for do that. - -### Configuration file - -This consists in declaring a configuration file that contains the names of the rule sets corresponding to npm packages that **metascraper** will be load automagically. - -The configuration file could be declared via: - -- A `.metascraperrc` file, written in YAML or JSON, with optional extensions: **.yaml**, **.yml**, **.json** and **.js**. -- A `metascraper.config.js` file that exports an object. -- A **metascraper** key in your `package.json` file. - -The configuration file will be resolved starting from the location of the file being formatted, and searching up the file tree until a config file is (or isn't) found. - -The order of rules are loaded are important: Just the first rule that resolve the value will be applied. - -#### Basic Configuration - -Declared an **array** of **rules**, specifying each rule as **string** name of the module to load. - -##### JSON - -```json -// .metascraperrc -{ - "rules": [ - "metascraper-author", - "metascraper-date", - "metascraper-description", - "metascraper-image", - "metascraper-lang", - "metascraper-logo", - "metascraper-publisher", - "metascraper-title", - "metascraper-url" - ] -} -``` - -##### YAML - -```yaml -# .metascraperrc -rules: - - metascraper-author - - metascraper-date - - metascraper-description - - metascraper-image - - metascraper-lang - - metascraper-logo - - metascraper-publisher - - metascraper-title - - metascraper-url -``` - -#### Advanced Configuration - -Additionally, you can pass specific configuration per module using a **object** declaration: - -##### JSON - -```json -// .metascraperrc -{ - "rules": [ - "metascraper-author", - "metascraper-date", - "metascraper-description", - "metascraper-image", - "metascraper-lang", - "metascraper-logo", - {"metascraper-clearbit-logo": { - "format": "jpg" - }}, - "metascraper-publisher", - "metascraper-title", - "metascraper-url" - ] -} -``` - -##### YAML - -```yaml -# .metascraperrc -rules: - - metascraper-author - - metascraper-date - - metascraper-description - - metascraper-image - - metascraper-lang - - metascraper-clearbit-logo: - format: jpg - - metascraper-publisher - - metascraper-title - - metascraper-url -``` - -### Constructor +## Importing Rules -If you need more control, you can load the rules set calling directly the metascraper constructor [`.load`](#metascraperloadrules): +**metascraper** exports a constructor that need to be initialized providing a collection of rules to load: ```js -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-author')(), require('metascraper-date')(), require('metascraper-description')(), @@ -255,10 +165,10 @@ const metascraper = require('metascraper').load([ Again, the order of rules are loaded are important: Just the first rule that resolve the value will be applied. -Use the first parameter to pass custom options if you need it: +Use the first parameter to pass custom options specific per each rules bundle: ```js -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-clearbit-logo')({ size: 256, format: 'jpg' @@ -266,21 +176,9 @@ const metascraper = require('metascraper').load([ ]) ``` -Using this way you are not limited to load just npm modules as rules set. For example, you can load a custom file of rules: +## Rules bundles -```js -const metascraper = require('metascraper').load([ - require('./my-custom-rules-file')() -]) -``` - -## Rules - -?> Can't find a rules set that you want? Let's [open an issue](https://github.com/microlinkhq/metascraper/issues/new) to create it. - -### Core rules - -These rules set will be shipped with **metascraper** and loaded by default. +?> Can't find the rules bundle that you want? Let's [open an issue](https://github.com/microlinkhq/metascraper/issues/new) to create it. | Package | Version | Dependencies | |--------|-------|------------| @@ -293,13 +191,6 @@ These rules set will be shipped with **metascraper** and loaded by default. | [`metascraper-publisher`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-publisher) | [![npm](https://img.shields.io/npm/v/metascraper-publisher.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-publisher) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-publisher&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-publisher) | | [`metascraper-title`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-title) | [![npm](https://img.shields.io/npm/v/metascraper-title.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-title) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-title&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-title) | | [`metascraper-url`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-url) | [![npm](https://img.shields.io/npm/v/metascraper-url.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-url) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-url&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-url) | - -### Community rules - -These rule set will not be shipped with **metascraper** by default and need to be specific using a configuration file. - -| Package | Version | Dependencies | -|--------|-------|------------| | [`metascraper-amazon`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-amazon) | [![npm](https://img.shields.io/npm/v/metascraper-amazon.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-amazon) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-amazon&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-amazon) | | [`metascraper-clearbit-logo`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-clearbit-logo) | [![npm](https://img.shields.io/npm/v/metascraper-clearbit-logo.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-clearbit-logo) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-clearbit-logo&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-clearbit-logo) | | [`metascraper-logo-favicon`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-logo-favicon) | [![npm](https://img.shields.io/npm/v/metascraper-logo-favicon.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-logo-favicon) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-logo-favicon&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-logo-favicon) | @@ -309,11 +200,11 @@ These rule set will not be shipped with **metascraper** by default and need to ### Write your own rules -A rule set is the simplest way for extending **metascraper** functionality. +A rule bundle is the simplest way for extending **metascraper** functionality. -A rule set can add one or more properties support. +A rule bundle can add one or more properties support. -The following schema represents the API compromise that a rule set need to follow: +The following schema represents the API compromise that a rule bundle need to follow: ```js 'use strict' @@ -336,12 +227,24 @@ module.exports = opts => { } ``` -We recommend check [core rules packages](/packages) as examples. +We recommend check one of the [rule bundles](#rules-bundles) already implemented to have a better perspective. ## API +### constructor(rules) + +Create a new **metascraper** instance declaring the rules bundle to be used explicitly. + +#### rules + +Type: `Array` + +The collection of rules bundle to be loaded. + ### metascraper(options) +Call the instance for extracting content based on rules bundle provided at the constructor. + #### options ##### html @@ -368,26 +271,7 @@ Type: `Array` You can pass additional rules on execution time. These rules will be merged with your loaded rules. -### metascraper.load(rules) - -Create a new **metascraper** instance declaring the rules set to be used explicitly. - -#### rules - -Type: `Array` - -The collection fo rules set to be loaded. - -## Environment Variables - -### METASCRAPER_CWD - -Type: `String`
-Default: `process.cwd()` - -This variable is used to determine where starting search for a configuration object. - -## Comparison +## Benchmark To give you an idea of how accurate **metascraper** is, here is a comparison of similar libraries: diff --git a/packages/metascraper-amazon/package.json b/packages/metascraper-amazon/package.json index 61bf605c2..bbe51324b 100644 --- a/packages/metascraper-amazon/package.json +++ b/packages/metascraper-amazon/package.json @@ -41,6 +41,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-amazon/test/index.js b/packages/metascraper-amazon/test/index.js index 8c7f412b7..00fcc5168 100644 --- a/packages/metascraper-amazon/test/index.js +++ b/packages/metascraper-amazon/test/index.js @@ -9,7 +9,7 @@ const fs = require('fs') const readFile = promisify(fs.readFile) -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-amazon')(), require('metascraper-author')(), require('metascraper-date')(), @@ -25,8 +25,11 @@ const metascraper = require('metascraper').load([ describe('metascraper-amazon', () => { describe('amazon.co.uk', () => { it('product url', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/amazon-co-uk/product-url.html')) - const url = 'https://www.amazon.co.uk/Vegetable-Perfection-tasty-recipes-shoots/dp/1849757097/ref=asap_bc?ie=UTF8' + const html = await readFile( + resolve(__dirname, 'fixtures/amazon-co-uk/product-url.html') + ) + const url = + 'https://www.amazon.co.uk/Vegetable-Perfection-tasty-recipes-shoots/dp/1849757097/ref=asap_bc?ie=UTF8' const meta = omit(await metascraper({ html, url }), ['date']) snapshot(meta) }) @@ -34,7 +37,9 @@ describe('metascraper-amazon', () => { describe('amazon.com', () => { it('ansi url', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/amazon-com/ansi-url.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/amazon-com/ansi-url.html') + ) const url = 'https://www.amazon.com/gp/product/B0057OC5O8/' const metadata = await metascraper({ html, url }) @@ -44,8 +49,11 @@ describe('metascraper-amazon', () => { }) it('product url', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/amazon-com/product-url.html')) - const url = 'https://www.amazon.com/The-Whole-Truth-Shaw-Book-ebook/dp/B0011UCPM4/ref=pd_zg_rss_ts_b_17_6?ie=UTF8&tag=recomshop-22' + const html = await readFile( + resolve(__dirname, 'fixtures/amazon-com/product-url.html') + ) + const url = + 'https://www.amazon.com/The-Whole-Truth-Shaw-Book-ebook/dp/B0011UCPM4/ref=pd_zg_rss_ts_b_17_6?ie=UTF8&tag=recomshop-22' const metadata = await metascraper({ html, url }) // omit date because it is non deterministic @@ -56,8 +64,11 @@ describe('metascraper-amazon', () => { describe('amazon.es', () => { it('product url', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/amazon-es/product-url.html')) - const url = 'https://www.amazon.es/aspirador-Excellence-Programable-limpieza-Silencioso/dp/B01MUGXRT9' + const html = await readFile( + resolve(__dirname, 'fixtures/amazon-es/product-url.html') + ) + const url = + 'https://www.amazon.es/aspirador-Excellence-Programable-limpieza-Silencioso/dp/B01MUGXRT9' const metadata = await metascraper({ html, url }) // omit date because it is non deterministic diff --git a/packages/metascraper-author/package.json b/packages/metascraper-author/package.json index a28665926..78158f87f 100644 --- a/packages/metascraper-author/package.json +++ b/packages/metascraper-author/package.json @@ -31,5 +31,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-clearbit-logo/package.json b/packages/metascraper-clearbit-logo/package.json index 6a107f478..740f7765f 100644 --- a/packages/metascraper-clearbit-logo/package.json +++ b/packages/metascraper-clearbit-logo/package.json @@ -20,6 +20,9 @@ "clearbit", "metascraper" ], + "dependencies": { + "got": "~9.0.0" + }, "devDependencies": { "mocha": "latest", "nyc": "latest", @@ -36,12 +39,12 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" ] - }, - "dependencies": { - "got": "~9.0.0" } } diff --git a/packages/metascraper-clearbit-logo/test/index.js b/packages/metascraper-clearbit-logo/test/index.js index 6e3f79197..62ea13f69 100644 --- a/packages/metascraper-clearbit-logo/test/index.js +++ b/packages/metascraper-clearbit-logo/test/index.js @@ -2,7 +2,7 @@ const should = require('should') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-author')(), require('metascraper-date')(), require('metascraper-description')(), diff --git a/packages/metascraper-date/package.json b/packages/metascraper-date/package.json index 136062926..1e14d546a 100644 --- a/packages/metascraper-date/package.json +++ b/packages/metascraper-date/package.json @@ -31,5 +31,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-description/package.json b/packages/metascraper-description/package.json index 87c1a655f..22b6ecec6 100644 --- a/packages/metascraper-description/package.json +++ b/packages/metascraper-description/package.json @@ -31,5 +31,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-image/package.json b/packages/metascraper-image/package.json index 9c186db84..5f0f0d95e 100644 --- a/packages/metascraper-image/package.json +++ b/packages/metascraper-image/package.json @@ -30,5 +30,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-lang/package.json b/packages/metascraper-lang/package.json index 8f747ca14..1a2cd2b11 100644 --- a/packages/metascraper-lang/package.json +++ b/packages/metascraper-lang/package.json @@ -19,7 +19,6 @@ "lodash": "~4.17.10" }, "devDependencies": { - "metascraper": "latest", "mocha": "latest", "nyc": "latest", "should": "latest", @@ -36,6 +35,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-lang/test/index.js b/packages/metascraper-lang/test/index.js index 58272ede0..d377d8227 100644 --- a/packages/metascraper-lang/test/index.js +++ b/packages/metascraper-lang/test/index.js @@ -5,7 +5,7 @@ const { promisify } = require('util') const { resolve } = require('path') const fs = require('fs') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-amazon')(), require('metascraper-author')(), require('metascraper-date')(), @@ -23,7 +23,8 @@ const readFile = promisify(fs.readFile) describe('metascraper-lang', () => { it('html lang property', async () => { const html = await readFile(resolve(__dirname, 'fixtures/html-lang.html')) - const url = 'http://www.dwutygodnik.com/artykul/7615-churchill-bohater-naszych-czasow.html' + const url = + 'http://www.dwutygodnik.com/artykul/7615-churchill-bohater-naszych-czasow.html' const metadata = await metascraper({ html, url }) snapshot(metadata) }) diff --git a/packages/metascraper-logo-favicon/package.json b/packages/metascraper-logo-favicon/package.json index 24877758a..19bd4be22 100644 --- a/packages/metascraper-logo-favicon/package.json +++ b/packages/metascraper-logo-favicon/package.json @@ -38,6 +38,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-logo-favicon/test/index.js b/packages/metascraper-logo-favicon/test/index.js index 04f6bdeac..56908f0a9 100644 --- a/packages/metascraper-logo-favicon/test/index.js +++ b/packages/metascraper-logo-favicon/test/index.js @@ -6,7 +6,7 @@ const { resolve } = require('path') const { omit } = require('lodash') const fs = require('fs') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-amazon')(), require('metascraper-author')(), require('metascraper-date')(), @@ -22,12 +22,11 @@ const metascraper = require('metascraper').load([ const readFile = promisify(fs.readFile) describe('metascraper-logo-favicon', () => { - describe('metascraper logo favicon', () => { - it('create an absolute favicon url if the logo is not present', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/input.html')) - const url = 'https://www.amazon.co.uk/Vegetable-Perfection-tasty-recipes-shoots/dp/1849757097/ref=asap_bc?ie=UTF8' - const meta = omit(await metascraper({ html, url }), ['date']) - snapshot(meta) - }) + it('create an absolute favicon url if the logo is not present', async () => { + const html = await readFile(resolve(__dirname, 'fixtures/input.html')) + const url = + 'https://www.amazon.co.uk/Vegetable-Perfection-tasty-recipes-shoots/dp/1849757097/ref=asap_bc?ie=UTF8' + const meta = omit(await metascraper({ html, url }), ['date']) + snapshot(meta) }) }) diff --git a/packages/metascraper-logo/package.json b/packages/metascraper-logo/package.json index 09f30027b..0660812cf 100644 --- a/packages/metascraper-logo/package.json +++ b/packages/metascraper-logo/package.json @@ -31,5 +31,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-publisher/package.json b/packages/metascraper-publisher/package.json index 64d2fff97..af04b3642 100644 --- a/packages/metascraper-publisher/package.json +++ b/packages/metascraper-publisher/package.json @@ -31,5 +31,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-soundcloud/package.json b/packages/metascraper-soundcloud/package.json index ecb56f80a..4f56fe9e5 100644 --- a/packages/metascraper-soundcloud/package.json +++ b/packages/metascraper-soundcloud/package.json @@ -40,6 +40,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-soundcloud/test/index.js b/packages/metascraper-soundcloud/test/index.js index cf713a7d1..b339b09df 100644 --- a/packages/metascraper-soundcloud/test/index.js +++ b/packages/metascraper-soundcloud/test/index.js @@ -5,7 +5,7 @@ const { promisify } = require('util') const { resolve } = require('path') const fs = require('fs') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-soundcloud')(), require('metascraper-author')(), require('metascraper-date')(), @@ -23,7 +23,8 @@ const readFile = promisify(fs.readFile) describe('metascraper-soundcloud', () => { it('song', async () => { const html = await readFile(resolve(__dirname, 'fixtures/song.html')) - const url = 'https://soundcloud.com/beautybrainsp/beauty-brain-swag-bandicoot' + const url = + 'https://soundcloud.com/beautybrainsp/beauty-brain-swag-bandicoot' const metadata = await metascraper({ html, url }) snapshot(metadata) diff --git a/packages/metascraper-title/package.json b/packages/metascraper-title/package.json index 9bd12617f..2f0c9a2ae 100644 --- a/packages/metascraper-title/package.json +++ b/packages/metascraper-title/package.json @@ -31,5 +31,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-url/package.json b/packages/metascraper-url/package.json index 311e38bd2..df227781d 100644 --- a/packages/metascraper-url/package.json +++ b/packages/metascraper-url/package.json @@ -30,5 +30,8 @@ "scripts": { "test": "exit 0" }, - "license": "MIT" + "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + } } diff --git a/packages/metascraper-video-provider/package.json b/packages/metascraper-video-provider/package.json index 8016a7e99..314d842ea 100644 --- a/packages/metascraper-video-provider/package.json +++ b/packages/metascraper-video-provider/package.json @@ -40,6 +40,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js b/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js index 599d295b4..48f4d9d8b 100644 --- a/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js +++ b/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js @@ -29,7 +29,6 @@ const getGuestToken = async url => { const getTwitterVideoInfo = async url => { const tweetId = getTweetId(url) - console.log('tweetId', tweetId) const apiUrl = `https://api.twitter.com/2/timeline/conversation/${tweetId}.json?tweet_mode=extended` const { body } = await got(apiUrl, { json: true, diff --git a/packages/metascraper-video-provider/test/index.js b/packages/metascraper-video-provider/test/index.js index 3fd553948..c5f7642f9 100644 --- a/packages/metascraper-video-provider/test/index.js +++ b/packages/metascraper-video-provider/test/index.js @@ -9,7 +9,7 @@ const { omit } = require('lodash') const should = require('should') const fs = require('fs') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-video-provider')(), require('metascraper-author')(), require('metascraper-date')(), diff --git a/packages/metascraper-video/package.json b/packages/metascraper-video/package.json index 753ad3c86..505492dd0 100644 --- a/packages/metascraper-video/package.json +++ b/packages/metascraper-video/package.json @@ -37,6 +37,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-video/test/index.js b/packages/metascraper-video/test/index.js index ff5332442..437dacf43 100644 --- a/packages/metascraper-video/test/index.js +++ b/packages/metascraper-video/test/index.js @@ -6,7 +6,7 @@ const { resolve } = require('path') const { omit } = require('lodash') const fs = require('fs') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-video')(), require('metascraper-author')(), require('metascraper-date')(), @@ -25,14 +25,17 @@ describe('metascraper-video', () => { describe('video', () => { it('video src', async () => { const html = await readFile(resolve(__dirname, 'fixtures/video-src.html')) - const url = 'https://www.theverge.com/2018/1/22/16921092/pentagon-secret-nuclear-bunker-reconstruction-minecraft-cns-miis-model' + const url = + 'https://www.theverge.com/2018/1/22/16921092/pentagon-secret-nuclear-bunker-reconstruction-minecraft-cns-miis-model' const metadata = await metascraper({ html, url }) snapshot(metadata) }) it('source src', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/source-src.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/source-src.html') + ) const url = 'https://9gag.com/gag/aGjVLDK' const metadata = await metascraper({ html, url }) @@ -49,14 +52,18 @@ describe('metascraper-video', () => { describe('specific providers', () => { it('clips.twitch.tv', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/providers/clip.twitch.tv.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/providers/clip.twitch.tv.html') + ) const url = 'https://clips.twitch.tv/AwkwardBoredWaffleItsBoshyTime' const metadata = await metascraper({ html, url }) snapshot(metadata) }) it('play.tv', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/providers/play.tv.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/providers/play.tv.html') + ) const url = 'https://plays.tv/video/5a6f64b1bef69a7fa9/holy-shit' const metadata = await metascraper({ html, url }) snapshot(omit(metadata, ['date'])) @@ -66,7 +73,9 @@ describe('metascraper-video', () => { describe('image', () => { it('src:poster', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/source-poster.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/source-poster.html') + ) const url = 'https://gfycat.com/gifs/detail/timelyhealthyarmadillo' const metadata = await metascraper({ html, url }) snapshot(metadata) diff --git a/packages/metascraper-youtube/package.json b/packages/metascraper-youtube/package.json index 3091f4501..e43d2fa7b 100644 --- a/packages/metascraper-youtube/package.json +++ b/packages/metascraper-youtube/package.json @@ -44,6 +44,9 @@ "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", + "peerDependencies": { + "metascraper": "^3" + }, "standard": { "env": [ "mocha" diff --git a/packages/metascraper-youtube/test/index.js b/packages/metascraper-youtube/test/index.js index 7b8af3da1..7e79f5d35 100644 --- a/packages/metascraper-youtube/test/index.js +++ b/packages/metascraper-youtube/test/index.js @@ -6,7 +6,7 @@ const { resolve } = require('path') const { omit } = require('lodash') const fs = require('fs') -const metascraper = require('metascraper').load([ +const metascraper = require('metascraper')([ require('metascraper-youtube')(), require('metascraper-author')(), require('metascraper-date')(), @@ -24,7 +24,9 @@ const readFile = promisify(fs.readFile) describe('metascraper-youtube', () => { describe('urls', () => { it('youtube video', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/youtube-video.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/youtube-video.html') + ) const url = 'https://www.youtube.com/watch?v=hwMkbaS_M_c' const metadata = await metascraper({ html, url }) @@ -32,7 +34,9 @@ describe('metascraper-youtube', () => { }) it('youtube video old', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/youtube-video-old.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/youtube-video-old.html') + ) const url = 'https://www.youtube.com/watch?v=GDRd-BFTYIg' const metadata = await metascraper({ html, url }) @@ -40,7 +44,9 @@ describe('metascraper-youtube', () => { }) it('youtube channel', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/youtube-channel.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/youtube-channel.html') + ) const url = 'https://www.youtube.com/channel/UCzcRQ3vRNr6fJ1A9rqFn7QA' const metadata = omit(await metascraper({ html, url }), ['date']) @@ -50,7 +56,9 @@ describe('metascraper-youtube', () => { describe('image size', () => { it('get the high image size', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/image-size.html')) + const html = await readFile( + resolve(__dirname, 'fixtures/image-size.html') + ) const url = 'https://www.youtube.com/watch?v=rXyKq7izYCQ' const metadata = omit(await metascraper({ html, url }), ['date']) snapshot(metadata) diff --git a/packages/metascraper/package.json b/packages/metascraper/package.json index 22893ff57..b9bf6e1fc 100644 --- a/packages/metascraper/package.json +++ b/packages/metascraper/package.json @@ -58,20 +58,8 @@ "@metascraper/helpers": "^3.12.1", "cheerio": "~1.0.0-rc.2", "cheerio-advanced-selectors": "~2.0.1", - "cosmiconfig": "~5.0.2", "lodash": "~4.17.10", - "metascraper-author": "^3.12.1", - "metascraper-date": "^3.11.4", - "metascraper-description": "^3.12.1", - "metascraper-image": "^3.12.1", - "metascraper-lang": "^3.11.4", - "metascraper-logo": "^3.12.1", - "metascraper-publisher": "^3.11.4", - "metascraper-title": "^3.12.1", - "metascraper-url": "^3.12.1", - "metascraper-video": "^3.12.1", "p-reduce": "~1.0.0", - "resolve-from": "~4.0.0", "sanitize-html": "~1.18.2" }, "devDependencies": { diff --git a/packages/metascraper/src/index.js b/packages/metascraper/src/index.js index 93f0ae04c..0ca2d84f7 100644 --- a/packages/metascraper/src/index.js +++ b/packages/metascraper/src/index.js @@ -1,22 +1,24 @@ 'use strict' const { isUrl } = require('@metascraper/helpers') -const { isEmpty, partial } = require('lodash') +const { isEmpty } = require('lodash') -const {autoload: autoLoadRules, load: loadRules} = require('./load-rules') const mergeRules = require('./merge-rules') +const loadRules = require('./load-rules') const loadHTML = require('./load-html') const getData = require('./get-data') -const create = loader => { - const lazyLoadedRules = loader() - return async ({url, html, rules: inlineRules} = {}) => { - const loadedRules = await lazyLoadedRules +module.exports = rules => { + const loadedRules = loadRules(rules) + return async ({ url, html, rules: inlineRules } = {}) => { if (!isUrl(url)) throw new TypeError('You need to provide a valid url.') - if (isEmpty(html)) throw new TypeError('You need to provide a valid HTML markup.') - return getData({ url, htmlDom: loadHTML(html), rules: mergeRules(inlineRules, loadedRules) }) + if (isEmpty(html)) { + throw new TypeError('You need to provide a valid HTML markup.') + } + return getData({ + url, + htmlDom: loadHTML(html), + rules: mergeRules(inlineRules, loadedRules) + }) } } - -module.exports = create(autoLoadRules) -module.exports.load = rules => create(partial(loadRules, rules)) diff --git a/packages/metascraper/src/load-rules.js b/packages/metascraper/src/load-rules.js index adae33f7e..78b81ddd4 100644 --- a/packages/metascraper/src/load-rules.js +++ b/packages/metascraper/src/load-rules.js @@ -1,32 +1,14 @@ 'use strict' -const cwd = process.env.METASCRAPER_CWD || process.cwd() -const config = require('cosmiconfig')('metascraper').search(cwd) -const resolveFrom = require('resolve-from') +const { concat, findIndex, forEach, chain } = require('lodash') -const {concat, map, findIndex, forEach, chain, isObject, isArray, isString, get} = require('lodash') - -const DEFAULT_RULES = [ - 'metascraper-author', - 'metascraper-date', - 'metascraper-description', - 'metascraper-video', - 'metascraper-image', - 'metascraper-lang', - 'metascraper-logo', - 'metascraper-publisher', - 'metascraper-title', - 'metascraper-url' -] - -const load = rules => +module.exports = rules => chain(rules) // merge rules with same props .reduce((acc, rules) => { forEach(rules, function (rule, propName) { const index = findIndex(acc, item => item[propName]) - if (index !== -1) acc[index][propName] = concat(acc[index][propName], rule) - else acc.push({[propName]: concat(rule)}) + if (index !== -1) { acc[index][propName] = concat(acc[index][propName], rule) } else acc.push({ [propName]: concat(rule) }) }) return acc }, []) @@ -37,33 +19,3 @@ const load = rules => return [key, value] }) .value() - -const autoload = async () => { - const configFile = await config - const rulesConfig = get(configFile, 'config.rules', DEFAULT_RULES) - - const rules = map(rulesConfig, rule => { - let moduleName - let moduleConfig - - if (isString(rule)) { - moduleName = rule - } else if (isArray(rule)) { - moduleName = rule[0] - moduleConfig = rule[1] - } else if (isObject(rule)) { - moduleName = Object.keys(rule)[0] - moduleConfig = rule[moduleName] - } - - const modulePath = resolveFrom(cwd, moduleName) - return require(modulePath)(moduleConfig) - }) - - return load(rules) -} - -module.exports = { - autoload, - load -} diff --git a/packages/metascraper/test/integration/anandtech/index.js b/packages/metascraper/test/integration/anandtech/index.js index 5deebf2e2..050af44eb 100644 --- a/packages/metascraper/test/integration/anandtech/index.js +++ b/packages/metascraper/test/integration/anandtech/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/arstechnica/index.js b/packages/metascraper/test/integration/arstechnica/index.js index f4e0b540c..7deaad67d 100644 --- a/packages/metascraper/test/integration/arstechnica/index.js +++ b/packages/metascraper/test/integration/arstechnica/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/astier/index.js b/packages/metascraper/test/integration/astier/index.js index bba7001b5..283d93c6c 100644 --- a/packages/metascraper/test/integration/astier/index.js +++ b/packages/metascraper/test/integration/astier/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://anisse.astier.eu/awk-driven-iot.html' diff --git a/packages/metascraper/test/integration/atlasobscura/index.js b/packages/metascraper/test/integration/atlasobscura/index.js index 4b607b692..250443d85 100644 --- a/packages/metascraper/test/integration/atlasobscura/index.js +++ b/packages/metascraper/test/integration/atlasobscura/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://www.atlasobscura.com/articles/ikea-bowl-blanda-blank-fire' diff --git a/packages/metascraper/test/integration/audiense/index.js b/packages/metascraper/test/integration/audiense/index.js index 76ff9df83..6fbd742e3 100644 --- a/packages/metascraper/test/integration/audiense/index.js +++ b/packages/metascraper/test/integration/audiense/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/bbc/index.js b/packages/metascraper/test/integration/bbc/index.js index 836d0861c..82c9b346f 100644 --- a/packages/metascraper/test/integration/bbc/index.js +++ b/packages/metascraper/test/integration/bbc/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://www.bbc.com/news/business-40504764' diff --git a/packages/metascraper/test/integration/bloomberg/index.js b/packages/metascraper/test/integration/bloomberg/index.js index 6cf3bc5c5..e8b11d8fd 100644 --- a/packages/metascraper/test/integration/bloomberg/index.js +++ b/packages/metascraper/test/integration/bloomberg/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/business-today/index.js b/packages/metascraper/test/integration/business-today/index.js index 1eb63d5f6..f19c87ca0 100644 --- a/packages/metascraper/test/integration/business-today/index.js +++ b/packages/metascraper/test/integration/business-today/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/cbr/index.js b/packages/metascraper/test/integration/cbr/index.js index 20d413a34..4abc83cd7 100644 --- a/packages/metascraper/test/integration/cbr/index.js +++ b/packages/metascraper/test/integration/cbr/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/cio/index.js b/packages/metascraper/test/integration/cio/index.js index cb1a3142a..6acbbcae3 100644 --- a/packages/metascraper/test/integration/cio/index.js +++ b/packages/metascraper/test/integration/cio/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/cloud-pro/index.js b/packages/metascraper/test/integration/cloud-pro/index.js index bb824b5d0..727e93f60 100644 --- a/packages/metascraper/test/integration/cloud-pro/index.js +++ b/packages/metascraper/test/integration/cloud-pro/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://www.cloudpro.co.uk/go/6024' diff --git a/packages/metascraper/test/integration/cnet/index.js b/packages/metascraper/test/integration/cnet/index.js index b50cd5e6d..c98eb4022 100644 --- a/packages/metascraper/test/integration/cnet/index.js +++ b/packages/metascraper/test/integration/cnet/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/computerworld/index.js b/packages/metascraper/test/integration/computerworld/index.js index fcd23ee70..ab0a38006 100644 --- a/packages/metascraper/test/integration/computerworld/index.js +++ b/packages/metascraper/test/integration/computerworld/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/crn/index.js b/packages/metascraper/test/integration/crn/index.js index 1c99901a9..e4ce01a25 100644 --- a/packages/metascraper/test/integration/crn/index.js +++ b/packages/metascraper/test/integration/crn/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/economic-times/index.js b/packages/metascraper/test/integration/economic-times/index.js index df322cf21..ca7adacb1 100644 --- a/packages/metascraper/test/integration/economic-times/index.js +++ b/packages/metascraper/test/integration/economic-times/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/entrepreneur/index.js b/packages/metascraper/test/integration/entrepreneur/index.js index 5450b5439..7aa335ee7 100644 --- a/packages/metascraper/test/integration/entrepreneur/index.js +++ b/packages/metascraper/test/integration/entrepreneur/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://www.entrepreneur.com/article/275188' diff --git a/packages/metascraper/test/integration/et-tech/index.js b/packages/metascraper/test/integration/et-tech/index.js index d51a2bcff..d400981d4 100644 --- a/packages/metascraper/test/integration/et-tech/index.js +++ b/packages/metascraper/test/integration/et-tech/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/eweek/index.js b/packages/metascraper/test/integration/eweek/index.js index 3c5e2ebd8..9e9c52c9c 100644 --- a/packages/metascraper/test/integration/eweek/index.js +++ b/packages/metascraper/test/integration/eweek/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/fast-company/index.js b/packages/metascraper/test/integration/fast-company/index.js index af8d139df..2b3fb32a3 100644 --- a/packages/metascraper/test/integration/fast-company/index.js +++ b/packages/metascraper/test/integration/fast-company/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/fastersite/index.js b/packages/metascraper/test/integration/fastersite/index.js index 1b930f1d5..fe209bc7d 100644 --- a/packages/metascraper/test/integration/fastersite/index.js +++ b/packages/metascraper/test/integration/fastersite/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://gent.ilcore.com/2012/06/better-timer-for-javascript.html' diff --git a/packages/metascraper/test/integration/fierce-devops/index.js b/packages/metascraper/test/integration/fierce-devops/index.js index a9bedbf4d..5d0b985a0 100644 --- a/packages/metascraper/test/integration/fierce-devops/index.js +++ b/packages/metascraper/test/integration/fierce-devops/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/financial-times/index.js b/packages/metascraper/test/integration/financial-times/index.js index 37fd25dad..556b8c227 100644 --- a/packages/metascraper/test/integration/financial-times/index.js +++ b/packages/metascraper/test/integration/financial-times/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/forbes/index.js b/packages/metascraper/test/integration/forbes/index.js index 98a40ada1..f2d2abdfc 100644 --- a/packages/metascraper/test/integration/forbes/index.js +++ b/packages/metascraper/test/integration/forbes/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/fortune/index.js b/packages/metascraper/test/integration/fortune/index.js index f748c4451..fa231ce74 100644 --- a/packages/metascraper/test/integration/fortune/index.js +++ b/packages/metascraper/test/integration/fortune/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://fortune.com/2015/10/05/hackerrank-recruiting-tool' diff --git a/packages/metascraper/test/integration/geek-time/index.js b/packages/metascraper/test/integration/geek-time/index.js index ace9d2f55..c39ae5355 100644 --- a/packages/metascraper/test/integration/geek-time/index.js +++ b/packages/metascraper/test/integration/geek-time/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/github/index.js b/packages/metascraper/test/integration/github/index.js index 75685af07..85beb4a15 100644 --- a/packages/metascraper/test/integration/github/index.js +++ b/packages/metascraper/test/integration/github/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://github.com/microlinkhq/microlink-core' diff --git a/packages/metascraper/test/integration/googleblog/index.js b/packages/metascraper/test/integration/googleblog/index.js index 2cd995fb7..73e363975 100644 --- a/packages/metascraper/test/integration/googleblog/index.js +++ b/packages/metascraper/test/integration/googleblog/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/inc/index.js b/packages/metascraper/test/integration/inc/index.js index f41b0b59d..8c5b68fd0 100644 --- a/packages/metascraper/test/integration/inc/index.js +++ b/packages/metascraper/test/integration/inc/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://www.inc.com/jeremy-quittner/2016-30-under-30-neurensic.html' diff --git a/packages/metascraper/test/integration/information-week/index.js b/packages/metascraper/test/integration/information-week/index.js index 747ec39e7..16a8426e8 100644 --- a/packages/metascraper/test/integration/information-week/index.js +++ b/packages/metascraper/test/integration/information-week/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/instagram/index.js b/packages/metascraper/test/integration/instagram/index.js index 99643d390..79373744d 100644 --- a/packages/metascraper/test/integration/instagram/index.js +++ b/packages/metascraper/test/integration/instagram/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://www.instagram.com/p/BWUDBntl3_Z' diff --git a/packages/metascraper/test/integration/jewish-business-news/index.js b/packages/metascraper/test/integration/jewish-business-news/index.js index 8e7b8c091..d4bc0ed65 100644 --- a/packages/metascraper/test/integration/jewish-business-news/index.js +++ b/packages/metascraper/test/integration/jewish-business-news/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/lean-data/index.js b/packages/metascraper/test/integration/lean-data/index.js index df7f6c167..d6c3e3598 100644 --- a/packages/metascraper/test/integration/lean-data/index.js +++ b/packages/metascraper/test/integration/lean-data/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/linkedin-company/index.js b/packages/metascraper/test/integration/linkedin-company/index.js index a434fd228..818fa05f1 100644 --- a/packages/metascraper/test/integration/linkedin-company/index.js +++ b/packages/metascraper/test/integration/linkedin-company/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://www.linkedin.com/company/10529043/' diff --git a/packages/metascraper/test/integration/linkedin-pulse/index.js b/packages/metascraper/test/integration/linkedin-pulse/index.js index 08892560e..2e7cb41d1 100644 --- a/packages/metascraper/test/integration/linkedin-pulse/index.js +++ b/packages/metascraper/test/integration/linkedin-pulse/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/los-angeles-times/index.js b/packages/metascraper/test/integration/los-angeles-times/index.js index 9218dd9ce..4fc02b450 100644 --- a/packages/metascraper/test/integration/los-angeles-times/index.js +++ b/packages/metascraper/test/integration/los-angeles-times/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/mac-rumors/index.js b/packages/metascraper/test/integration/mac-rumors/index.js index 5c10ec3b9..3ab0bb560 100644 --- a/packages/metascraper/test/integration/mac-rumors/index.js +++ b/packages/metascraper/test/integration/mac-rumors/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/market-wired/index.js b/packages/metascraper/test/integration/market-wired/index.js index 5dc3cd6f5..df846c962 100644 --- a/packages/metascraper/test/integration/market-wired/index.js +++ b/packages/metascraper/test/integration/market-wired/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/marketing-land/index.js b/packages/metascraper/test/integration/marketing-land/index.js index c0f037381..52eeedfc9 100644 --- a/packages/metascraper/test/integration/marketing-land/index.js +++ b/packages/metascraper/test/integration/marketing-land/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/mashable/index.js b/packages/metascraper/test/integration/mashable/index.js index e298d2fa3..8c6400b21 100644 --- a/packages/metascraper/test/integration/mashable/index.js +++ b/packages/metascraper/test/integration/mashable/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://mashable.com/2015/05/13/analytics-power-up-revenue' diff --git a/packages/metascraper/test/integration/medium/index.js b/packages/metascraper/test/integration/medium/index.js index 02d99b656..c4cd7873f 100644 --- a/packages/metascraper/test/integration/medium/index.js +++ b/packages/metascraper/test/integration/medium/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://medium.com/webpack/webpack-3-official-release-15fd2dd8f07b' diff --git a/packages/metascraper/test/integration/motherboard/index.js b/packages/metascraper/test/integration/motherboard/index.js index 227d27e1c..0a25fdb62 100644 --- a/packages/metascraper/test/integration/motherboard/index.js +++ b/packages/metascraper/test/integration/motherboard/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/nytimes/index.js b/packages/metascraper/test/integration/nytimes/index.js index 26cf3a615..4f93d58cd 100644 --- a/packages/metascraper/test/integration/nytimes/index.js +++ b/packages/metascraper/test/integration/nytimes/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/pikabu/index.js b/packages/metascraper/test/integration/pikabu/index.js index 5edcf709e..0618844d5 100644 --- a/packages/metascraper/test/integration/pikabu/index.js +++ b/packages/metascraper/test/integration/pikabu/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://pikabu.ru' diff --git a/packages/metascraper/test/integration/pr-newswire/index.js b/packages/metascraper/test/integration/pr-newswire/index.js index 578263ec0..c7540e212 100644 --- a/packages/metascraper/test/integration/pr-newswire/index.js +++ b/packages/metascraper/test/integration/pr-newswire/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/recode/index.js b/packages/metascraper/test/integration/recode/index.js index 772076f19..4d6ebaf22 100644 --- a/packages/metascraper/test/integration/recode/index.js +++ b/packages/metascraper/test/integration/recode/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/reuters/index.js b/packages/metascraper/test/integration/reuters/index.js index a64b6064f..85d6c4774 100644 --- a/packages/metascraper/test/integration/reuters/index.js +++ b/packages/metascraper/test/integration/reuters/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/san-francisco-chronicle/index.js b/packages/metascraper/test/integration/san-francisco-chronicle/index.js index 25e1f96a8..a00dbbb95 100644 --- a/packages/metascraper/test/integration/san-francisco-chronicle/index.js +++ b/packages/metascraper/test/integration/san-francisco-chronicle/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/segment-academy/index.js b/packages/metascraper/test/integration/segment-academy/index.js index ef47d9d55..9d2499427 100644 --- a/packages/metascraper/test/integration/segment-academy/index.js +++ b/packages/metascraper/test/integration/segment-academy/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/segment/index.js b/packages/metascraper/test/integration/segment/index.js index 5dae5ff15..44136e6ee 100644 --- a/packages/metascraper/test/integration/segment/index.js +++ b/packages/metascraper/test/integration/segment/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://segment.com/blog/scaling-nsq' diff --git a/packages/metascraper/test/integration/silicon-angle/index.js b/packages/metascraper/test/integration/silicon-angle/index.js index c790b50c3..522064b9d 100644 --- a/packages/metascraper/test/integration/silicon-angle/index.js +++ b/packages/metascraper/test/integration/silicon-angle/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/silicon-beat/index.js b/packages/metascraper/test/integration/silicon-beat/index.js index c4c605f80..e37fed374 100644 --- a/packages/metascraper/test/integration/silicon-beat/index.js +++ b/packages/metascraper/test/integration/silicon-beat/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://www.siliconbeat.com/2016/04/19/time-come-woman' diff --git a/packages/metascraper/test/integration/silicon-tap/index.js b/packages/metascraper/test/integration/silicon-tap/index.js index 417b9d727..8e1049628 100644 --- a/packages/metascraper/test/integration/silicon-tap/index.js +++ b/packages/metascraper/test/integration/silicon-tap/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://www.silicontap.com/story/0053475.html' diff --git a/packages/metascraper/test/integration/silicon-valley-business-journal/index.js b/packages/metascraper/test/integration/silicon-valley-business-journal/index.js index b4a0ed7a4..3fcdd8fbf 100644 --- a/packages/metascraper/test/integration/silicon-valley-business-journal/index.js +++ b/packages/metascraper/test/integration/silicon-valley-business-journal/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/smitten-kitchen/index.js b/packages/metascraper/test/integration/smitten-kitchen/index.js index 9e8b0a254..03f31565a 100644 --- a/packages/metascraper/test/integration/smitten-kitchen/index.js +++ b/packages/metascraper/test/integration/smitten-kitchen/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'http://smittenkitchen.com/blog/2016/05/cucumber-yogurt-raita-salad' diff --git a/packages/metascraper/test/integration/startup-grind/index.js b/packages/metascraper/test/integration/startup-grind/index.js index 41ad13927..ae9e272c9 100644 --- a/packages/metascraper/test/integration/startup-grind/index.js +++ b/packages/metascraper/test/integration/startup-grind/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/techcrunch/index.js b/packages/metascraper/test/integration/techcrunch/index.js index f1c8b260c..15f31f50b 100644 --- a/packages/metascraper/test/integration/techcrunch/index.js +++ b/packages/metascraper/test/integration/techcrunch/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/the-boston-globe/index.js b/packages/metascraper/test/integration/the-boston-globe/index.js index 3edd5db38..387fd8707 100644 --- a/packages/metascraper/test/integration/the-boston-globe/index.js +++ b/packages/metascraper/test/integration/the-boston-globe/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/the-guardian/index.js b/packages/metascraper/test/integration/the-guardian/index.js index 2cb27145c..b0e5dea8d 100644 --- a/packages/metascraper/test/integration/the-guardian/index.js +++ b/packages/metascraper/test/integration/the-guardian/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/the-register/index.js b/packages/metascraper/test/integration/the-register/index.js index 300ca0f27..3a0d431b9 100644 --- a/packages/metascraper/test/integration/the-register/index.js +++ b/packages/metascraper/test/integration/the-register/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/the-verge/index.js b/packages/metascraper/test/integration/the-verge/index.js index 4f5815a5d..b3463f3c0 100644 --- a/packages/metascraper/test/integration/the-verge/index.js +++ b/packages/metascraper/test/integration/the-verge/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/twitter-gif/index.js b/packages/metascraper/test/integration/twitter-gif/index.js index a9fadc65d..0cb58bb36 100644 --- a/packages/metascraper/test/integration/twitter-gif/index.js +++ b/packages/metascraper/test/integration/twitter-gif/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://twitter.com/Kikobeats/status/880139124791029763' diff --git a/packages/metascraper/test/integration/twitter-image/index.js b/packages/metascraper/test/integration/twitter-image/index.js index 4b3bbc268..e6f8d38a1 100644 --- a/packages/metascraper/test/integration/twitter-image/index.js +++ b/packages/metascraper/test/integration/twitter-image/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://twitter.com/k4rliky/status/934482867480121345' diff --git a/packages/metascraper/test/integration/usa-today/index.js b/packages/metascraper/test/integration/usa-today/index.js index 023b7c1c3..79b928567 100644 --- a/packages/metascraper/test/integration/usa-today/index.js +++ b/packages/metascraper/test/integration/usa-today/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/venture-beat/index.js b/packages/metascraper/test/integration/venture-beat/index.js index e4b217855..2d2152202 100644 --- a/packages/metascraper/test/integration/venture-beat/index.js +++ b/packages/metascraper/test/integration/venture-beat/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/vimeo/index.js b/packages/metascraper/test/integration/vimeo/index.js index 0ef83e251..20d34d789 100644 --- a/packages/metascraper/test/integration/vimeo/index.js +++ b/packages/metascraper/test/integration/vimeo/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://vimeo.com/200104989' diff --git a/packages/metascraper/test/integration/washington-post/index.js b/packages/metascraper/test/integration/washington-post/index.js index 38e3ea983..c4b1f2768 100644 --- a/packages/metascraper/test/integration/washington-post/index.js +++ b/packages/metascraper/test/integration/washington-post/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/wikipedia/index.js b/packages/metascraper/test/integration/wikipedia/index.js index ba04512a1..e0b3cd9c8 100644 --- a/packages/metascraper/test/integration/wikipedia/index.js +++ b/packages/metascraper/test/integration/wikipedia/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = 'https://en.wikipedia.org/wiki/Bob_Dylan' diff --git a/packages/metascraper/test/integration/wired/index.js b/packages/metascraper/test/integration/wired/index.js index cf3abf108..7e1ec3037 100644 --- a/packages/metascraper/test/integration/wired/index.js +++ b/packages/metascraper/test/integration/wired/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/wsj/index.js b/packages/metascraper/test/integration/wsj/index.js index 38519b95b..1219b9684 100644 --- a/packages/metascraper/test/integration/wsj/index.js +++ b/packages/metascraper/test/integration/wsj/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/xconomy/index.js b/packages/metascraper/test/integration/xconomy/index.js index 04d265567..e0394c5fd 100644 --- a/packages/metascraper/test/integration/xconomy/index.js +++ b/packages/metascraper/test/integration/xconomy/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/yahoo-news/index.js b/packages/metascraper/test/integration/yahoo-news/index.js index 3eb803140..9f65cfe91 100644 --- a/packages/metascraper/test/integration/yahoo-news/index.js +++ b/packages/metascraper/test/integration/yahoo-news/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/integration/zdnet/index.js b/packages/metascraper/test/integration/zdnet/index.js index b5538c7d5..168c7a955 100644 --- a/packages/metascraper/test/integration/zdnet/index.js +++ b/packages/metascraper/test/integration/zdnet/index.js @@ -6,7 +6,19 @@ const { resolve } = require('path') const fs = require('fs') -const metascraper = require('../../..') +const metascraper = require('../../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) + const readFile = promisify(fs.readFile) const url = diff --git a/packages/metascraper/test/unit/interface.js b/packages/metascraper/test/unit/interface.js index c096e7b01..9c8519371 100644 --- a/packages/metascraper/test/unit/interface.js +++ b/packages/metascraper/test/unit/interface.js @@ -1,9 +1,19 @@ - 'use strict' const should = require('should') -const metascraper = require('../..') +const metascraper = require('../..')([ + require('metascraper-author')(), + require('metascraper-date')(), + require('metascraper-description')(), + require('metascraper-video')(), + require('metascraper-image')(), + require('metascraper-lang')(), + require('metascraper-logo')(), + require('metascraper-publisher')(), + require('metascraper-title')(), + require('metascraper-url')() +]) it('url is required', async () => { try { @@ -49,9 +59,7 @@ it('load extra rules', async () => { const rules = [ { - foo: [ - () => 'bar' - ] + foo: [() => 'bar'] } ] diff --git a/packages/metascraper/test/unit/load-rules/config/index.js b/packages/metascraper/test/unit/load-rules/config/index.js deleted file mode 100644 index b045669a3..000000000 --- a/packages/metascraper/test/unit/load-rules/config/index.js +++ /dev/null @@ -1,29 +0,0 @@ -'use strict' - -const clearModule = require('clear-module') -const should = require('should') - -let metascraper - -describe('automagically load rules', () => { - before(() => { - clearModule.all() - process.env.METASCRAPER_CWD = __dirname - metascraper = require('../../../..') - }) - - after(() => { - clearModule.all() - delete process.env.METASCRAPER_CWD - }) - - it('based on a configuration file', async () => { - const url = 'https://facebook.com' - const html = '
' - - const meta = await metascraper({ url, html }) - should(meta.logo).be.equal( - 'https://logo.clearbit.com/facebook.com?size=128&format=jpg' - ) - }) -}) diff --git a/packages/metascraper/test/unit/load-rules/config/metascraper.config.js b/packages/metascraper/test/unit/load-rules/config/metascraper.config.js deleted file mode 100644 index a902272ea..000000000 --- a/packages/metascraper/test/unit/load-rules/config/metascraper.config.js +++ /dev/null @@ -1,17 +0,0 @@ -module.exports = { - rules: [ - 'metascraper-author', - 'metascraper-date', - 'metascraper-description', - 'metascraper-image', - 'metascraper-logo', - { - 'metascraper-clearbit-logo': { - format: 'jpg' - } - }, - 'metascraper-publisher', - 'metascraper-title', - 'metascraper-url' - ] -} diff --git a/packages/metascraper/test/unit/load-rules/pkg/index.js b/packages/metascraper/test/unit/load-rules/pkg/index.js deleted file mode 100644 index 62627917b..000000000 --- a/packages/metascraper/test/unit/load-rules/pkg/index.js +++ /dev/null @@ -1,29 +0,0 @@ -'use strict' - -const clearModule = require('clear-module') -const should = require('should') - -let metascraper - -describe('automagically load rules', () => { - before(() => { - clearModule.all() - process.env.METASCRAPER_CWD = __dirname - metascraper = require('../../../..') - }) - - after(() => { - clearModule.all() - delete process.env.METASCRAPER_CWD - }) - - it('based on a package file', async () => { - const url = 'https://facebook.com' - const html = '
' - - const meta = await metascraper({ url, html }) - should(meta.logo).be.equal( - 'https://logo.clearbit.com/facebook.com?size=128&format=jpg' - ) - }) -}) diff --git a/packages/metascraper/test/unit/load-rules/pkg/package.json b/packages/metascraper/test/unit/load-rules/pkg/package.json deleted file mode 100644 index fe3f59811..000000000 --- a/packages/metascraper/test/unit/load-rules/pkg/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "metascraper": { - "rules": [ - "metascraper-author", - "metascraper-date", - "metascraper-description", - "metascraper-image", - "metascraper-logo", - { - "metascraper-clearbit-logo": { - "format": "jpg" - } - }, - "metascraper-publisher", - "metascraper-title", - "metascraper-url" - ] - } -} diff --git a/packages/metascraper/test/unit/load-rules/rc/.metascraperrc b/packages/metascraper/test/unit/load-rules/rc/.metascraperrc deleted file mode 100644 index 5daeffa19..000000000 --- a/packages/metascraper/test/unit/load-rules/rc/.metascraperrc +++ /dev/null @@ -1,15 +0,0 @@ -{ - "rules": [ - "metascraper-author", - "metascraper-date", - "metascraper-description", - "metascraper-image", - "metascraper-logo", - {"metascraper-clearbit-logo": { - "format": "jpg" - }}, - "metascraper-publisher", - "metascraper-title", - "metascraper-url" - ] -} diff --git a/packages/metascraper/test/unit/load-rules/rc/index.js b/packages/metascraper/test/unit/load-rules/rc/index.js deleted file mode 100644 index f2e86df87..000000000 --- a/packages/metascraper/test/unit/load-rules/rc/index.js +++ /dev/null @@ -1,29 +0,0 @@ -'use strict' - -const clearModule = require('clear-module') -const should = require('should') - -let metascraper - -describe('automagically load rules', () => { - before(() => { - clearModule.all() - process.env.METASCRAPER_CWD = __dirname - metascraper = require('../../../..') - }) - - after(() => { - clearModule.all() - delete process.env.METASCRAPER_CWD - }) - - it('based on a rc file', async () => { - const url = 'https://facebook.com' - const html = '
' - - const meta = await metascraper({ url, html }) - should(meta.logo).be.equal( - 'https://logo.clearbit.com/facebook.com?size=128&format=jpg' - ) - }) -}) diff --git a/packages/metascraper/test/unit/merge-rules.js b/packages/metascraper/test/unit/merge-rules.js index a4b0d2a73..c99c763e1 100644 --- a/packages/metascraper/test/unit/merge-rules.js +++ b/packages/metascraper/test/unit/merge-rules.js @@ -2,7 +2,7 @@ const should = require('should') -it('add a new rule from a prop that doesn\'t exist', async () => { +it("add a new rule from a prop that doesn't exist", async () => { const url = 'https://microlink.io' const html = ` @@ -30,14 +30,12 @@ it('add a new rule from a prop that doesn\'t exist', async () => { const rules = [ { - foo: [ - () => 'bar' - ] + foo: [() => 'bar'] } ] - const metascraper = require('../..').load([]) - const meta = await metascraper({url, html, rules}) + const metascraper = require('../..')([]) + const meta = await metascraper({ url, html, rules }) should(meta.foo).be.equal('bar') }) @@ -69,21 +67,17 @@ it('add a new rule for a prop that exists', async () => { const rules = [ { - foo: [ - () => 'bar' - ] + foo: [() => 'bar'] } ] - const metascraper = require('../..').load([{ - foo: [ - () => false, - () => false, - () => false - ] - }]) + const metascraper = require('../..')([ + { + foo: [() => false, () => false, () => false] + } + ]) - const meta = await metascraper({url, html, rules}) + const meta = await metascraper({ url, html, rules }) should(meta.foo).be.equal('bar') }) @@ -115,20 +109,16 @@ it('rules are added from the end', async () => { const rules = [ { - foo: [ - () => 'bar' - ] + foo: [() => 'bar'] } ] - const metascraper = require('../..').load([{ - foo: [ - () => false, - () => false, - () => 'baz' - ] - }]) + const metascraper = require('../..')([ + { + foo: [() => false, () => false, () => 'baz'] + } + ]) - const meta = await metascraper({url, html, rules}) + const meta = await metascraper({ url, html, rules }) should(meta.foo).be.equal('baz') }) From 7901fb66d88513ffba49c109d873d3262b7d9aa7 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Fri, 24 Aug 2018 10:19:49 +0200 Subject: [PATCH 017/442] v4.0.0 --- lerna.json | 2 +- packages/metascraper-amazon/package.json | 4 ++-- packages/metascraper-author/package.json | 4 ++-- packages/metascraper-clearbit-logo/package.json | 2 +- packages/metascraper-date/package.json | 2 +- packages/metascraper-description/package.json | 4 ++-- packages/metascraper-helpers/package.json | 2 +- packages/metascraper-image/package.json | 4 ++-- packages/metascraper-lang/package.json | 2 +- packages/metascraper-logo-favicon/package.json | 2 +- packages/metascraper-logo/package.json | 4 ++-- packages/metascraper-publisher/package.json | 2 +- packages/metascraper-soundcloud/package.json | 4 ++-- packages/metascraper-title/package.json | 4 ++-- packages/metascraper-url/package.json | 4 ++-- packages/metascraper-video-provider/package.json | 4 ++-- packages/metascraper-video/package.json | 4 ++-- packages/metascraper-youtube/package.json | 4 ++-- packages/metascraper/package.json | 4 ++-- 19 files changed, 31 insertions(+), 31 deletions(-) diff --git a/lerna.json b/lerna.json index ee0698bcf..c0fa6a2d8 100644 --- a/lerna.json +++ b/lerna.json @@ -3,5 +3,5 @@ "packages": [ "packages/*" ], - "version": "3.13.2" + "version": "4.0.0" } diff --git a/packages/metascraper-amazon/package.json b/packages/metascraper-amazon/package.json index bbe51324b..aa8d27e59 100644 --- a/packages/metascraper-amazon/package.json +++ b/packages/metascraper-amazon/package.json @@ -2,7 +2,7 @@ "name": "metascraper-amazon", "description": "metascraper integration with Amazon", "homepage": "https://documentup.com/microlinkhq/metascraper-amazon", - "version": "3.13.0", + "version": "4.0.0", "main": "index.js", "author": { "email": "hello@microlink.io", @@ -21,7 +21,7 @@ "metascraper" ], "dependencies": { - "@metascraper/helpers": "^3.12.1" + "@metascraper/helpers": "^4.0.0" }, "devDependencies": { "lodash": "latest", diff --git a/packages/metascraper-author/package.json b/packages/metascraper-author/package.json index 78158f87f..30f3b3c2d 100644 --- a/packages/metascraper-author/package.json +++ b/packages/metascraper-author/package.json @@ -2,7 +2,7 @@ "name": "metascraper-author", "description": "Get author property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.12.1", + "version": "4.0.0", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^3.12.1", + "@metascraper/helpers": "^4.0.0", "lodash": "~4.17.10" }, "devDependencies": { diff --git a/packages/metascraper-clearbit-logo/package.json b/packages/metascraper-clearbit-logo/package.json index bf1445a61..ca669dbad 100644 --- a/packages/metascraper-clearbit-logo/package.json +++ b/packages/metascraper-clearbit-logo/package.json @@ -2,7 +2,7 @@ "name": "metascraper-clearbit-logo", "description": "metascraper integration with Clearbit Logo API", "homepage": "https://documentup.com/microlinkhq/metascraper-clearbit-logo", - "version": "3.13.0", + "version": "4.0.0", "main": "index.js", "author": { "email": "hello@microlink.io", diff --git a/packages/metascraper-date/package.json b/packages/metascraper-date/package.json index 1e14d546a..307f039ad 100644 --- a/packages/metascraper-date/package.json +++ b/packages/metascraper-date/package.json @@ -2,7 +2,7 @@ "name": "metascraper-date", "description": "Get date property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.11.4", + "version": "4.0.0", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-description/package.json b/packages/metascraper-description/package.json index 22b6ecec6..d5292318a 100644 --- a/packages/metascraper-description/package.json +++ b/packages/metascraper-description/package.json @@ -2,7 +2,7 @@ "name": "metascraper-description", "description": "Get description property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.12.1", + "version": "4.0.0", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^3.12.1", + "@metascraper/helpers": "^4.0.0", "lodash": "~4.17.10" }, "devDependencies": { diff --git a/packages/metascraper-helpers/package.json b/packages/metascraper-helpers/package.json index fdd1577ca..a2a31c1c5 100644 --- a/packages/metascraper-helpers/package.json +++ b/packages/metascraper-helpers/package.json @@ -2,7 +2,7 @@ "name": "@metascraper/helpers", "description": "Collection of helper functions used by metascraper", "homepage": "https://metascraper.js.org", - "version": "3.12.1", + "version": "4.0.0", "main": "src/index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-image/package.json b/packages/metascraper-image/package.json index 5f0f0d95e..234b4bc87 100644 --- a/packages/metascraper-image/package.json +++ b/packages/metascraper-image/package.json @@ -2,7 +2,7 @@ "name": "metascraper-image", "description": "Get image property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.12.1", + "version": "4.0.0", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^3.12.1" + "@metascraper/helpers": "^4.0.0" }, "devDependencies": { "standard": "latest" diff --git a/packages/metascraper-lang/package.json b/packages/metascraper-lang/package.json index 1a2cd2b11..2dc0f5e81 100644 --- a/packages/metascraper-lang/package.json +++ b/packages/metascraper-lang/package.json @@ -2,7 +2,7 @@ "name": "metascraper-lang", "description": "Get lang property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.11.4", + "version": "4.0.0", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-logo-favicon/package.json b/packages/metascraper-logo-favicon/package.json index 19bd4be22..539639477 100644 --- a/packages/metascraper-logo-favicon/package.json +++ b/packages/metascraper-logo-favicon/package.json @@ -2,7 +2,7 @@ "name": "metascraper-logo-favicon", "description": "metascraper logo favicon fallback", "homepage": "https://documentup.com/microlinkhq/metascraper-logo-favicon", - "version": "3.13.0", + "version": "4.0.0", "main": "index.js", "author": { "email": "hello@microlink.io", diff --git a/packages/metascraper-logo/package.json b/packages/metascraper-logo/package.json index 0660812cf..dff8ff4cf 100644 --- a/packages/metascraper-logo/package.json +++ b/packages/metascraper-logo/package.json @@ -2,7 +2,7 @@ "name": "metascraper-logo", "description": "Get logo property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.12.1", + "version": "4.0.0", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^3.12.1", + "@metascraper/helpers": "^4.0.0", "lodash": "~4.17.10" }, "devDependencies": { diff --git a/packages/metascraper-publisher/package.json b/packages/metascraper-publisher/package.json index af04b3642..1078938a5 100644 --- a/packages/metascraper-publisher/package.json +++ b/packages/metascraper-publisher/package.json @@ -2,7 +2,7 @@ "name": "metascraper-publisher", "description": "Get publisher property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.11.4", + "version": "4.0.0", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-soundcloud/package.json b/packages/metascraper-soundcloud/package.json index 4f56fe9e5..2f93bd737 100644 --- a/packages/metascraper-soundcloud/package.json +++ b/packages/metascraper-soundcloud/package.json @@ -2,7 +2,7 @@ "name": "metascraper-soundcloud", "description": "metascraper integration with SoundCloud", "homepage": "https://documentup.com/microlinkhq/metascraper-soundcloud", - "version": "3.13.0", + "version": "4.0.0", "main": "index.js", "author": { "email": "hello@microlink.io", @@ -21,7 +21,7 @@ "metascraper" ], "dependencies": { - "@metascraper/helpers": "^3.12.1" + "@metascraper/helpers": "^4.0.0" }, "devDependencies": { "mocha": "latest", diff --git a/packages/metascraper-title/package.json b/packages/metascraper-title/package.json index 2f0c9a2ae..a7f2dbc96 100644 --- a/packages/metascraper-title/package.json +++ b/packages/metascraper-title/package.json @@ -2,7 +2,7 @@ "name": "metascraper-title", "description": "Get title property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.12.1", + "version": "4.0.0", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^3.12.1", + "@metascraper/helpers": "^4.0.0", "lodash": "~4.17.10" }, "devDependencies": { diff --git a/packages/metascraper-url/package.json b/packages/metascraper-url/package.json index df227781d..5a494293b 100644 --- a/packages/metascraper-url/package.json +++ b/packages/metascraper-url/package.json @@ -2,7 +2,7 @@ "name": "metascraper-url", "description": "Get url property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.12.1", + "version": "4.0.0", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^3.12.1" + "@metascraper/helpers": "^4.0.0" }, "devDependencies": { "standard": "latest" diff --git a/packages/metascraper-video-provider/package.json b/packages/metascraper-video-provider/package.json index 41827aace..85aa702a7 100644 --- a/packages/metascraper-video-provider/package.json +++ b/packages/metascraper-video-provider/package.json @@ -2,7 +2,7 @@ "name": "metascraper-video-provider", "description": "Get video property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.13.2", + "version": "4.0.0", "main": "src/index", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^3.12.1", + "@metascraper/helpers": "^4.0.0", "got": "~9.1.0", "json-future": "~2.1.2", "lodash": "~4.17.10", diff --git a/packages/metascraper-video/package.json b/packages/metascraper-video/package.json index 505492dd0..5a303aead 100644 --- a/packages/metascraper-video/package.json +++ b/packages/metascraper-video/package.json @@ -2,7 +2,7 @@ "name": "metascraper-video", "description": "Get video property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "3.12.1", + "version": "4.0.0", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^3.12.1", + "@metascraper/helpers": "^4.0.0", "video-extensions": "~1.1.0" }, "devDependencies": { diff --git a/packages/metascraper-youtube/package.json b/packages/metascraper-youtube/package.json index e43d2fa7b..55d01a201 100644 --- a/packages/metascraper-youtube/package.json +++ b/packages/metascraper-youtube/package.json @@ -2,7 +2,7 @@ "name": "metascraper-youtube", "description": "metascraper integration with YouTube", "homepage": "https://documentup.com/microlinkhq/metascraper-youtube", - "version": "3.13.0", + "version": "4.0.0", "main": "index.js", "author": { "email": "hello@microlink.io", @@ -21,7 +21,7 @@ "metascraper" ], "dependencies": { - "@metascraper/helpers": "^3.12.1", + "@metascraper/helpers": "^4.0.0", "get-video-id": "~3.1.0", "is-reachable": "~2.4.0", "lodash": "~4.17.10", diff --git a/packages/metascraper/package.json b/packages/metascraper/package.json index b9bf6e1fc..c14039d9c 100644 --- a/packages/metascraper/package.json +++ b/packages/metascraper/package.json @@ -2,7 +2,7 @@ "name": "metascraper", "description": "A library to easily scrape metadata from an article on the web using Open Graph metadata, regular HTML metadata, and series of fallbacks.", "homepage": "https://metascraper.js.org", - "version": "3.12.1", + "version": "4.0.0", "main": "src/index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -55,7 +55,7 @@ "website" ], "dependencies": { - "@metascraper/helpers": "^3.12.1", + "@metascraper/helpers": "^4.0.0", "cheerio": "~1.0.0-rc.2", "cheerio-advanced-selectors": "~2.0.1", "lodash": "~4.17.10", From 2a3dc73e22e7b4e7603561ea55f8453fe966d115 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Fri, 24 Aug 2018 10:20:54 +0200 Subject: [PATCH 018/442] Update Changelog --- CHANGELOG.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 109e59e0f..733b590bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,20 @@ -## (2018-08-18) +## (2018-08-24) +## 4.0.0 (2018-08-24) + +* Remove autoload ([9d3745b](https://github.com/microlinkhq/metascraper/commit/9d3745b)) +* Remove autoload (#106) ([60b0adb](https://github.com/microlinkhq/metascraper/commit/60b0adb)), closes [#106](https://github.com/microlinkhq/metascraper/issues/106) +* Remove lazy intializer ([d63fcc8](https://github.com/microlinkhq/metascraper/commit/d63fcc8)) +* Update Changelog ([76d5329](https://github.com/microlinkhq/metascraper/commit/76d5329)) +* Update dependencies ([19aca42](https://github.com/microlinkhq/metascraper/commit/19aca42)) +* Update Docs ([7fd6b15](https://github.com/microlinkhq/metascraper/commit/7fd6b15)) +* v4.0.0 ([7901fb6](https://github.com/microlinkhq/metascraper/commit/7901fb6)) + + + ## 3.13.2 (2018-08-18) * Oh twitter ([23f641d](https://github.com/microlinkhq/metascraper/commit/23f641d)) From 58c26f0cb1f633a81247ceca4ae897338cccc02b Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sat, 25 Aug 2018 11:30:00 +0200 Subject: [PATCH 019/442] Add tests --- packages/metascraper-helpers/package.json | 10 +++++++++- packages/metascraper-helpers/test/index.js | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 packages/metascraper-helpers/test/index.js diff --git a/packages/metascraper-helpers/package.json b/packages/metascraper-helpers/package.json index a2a31c1c5..5c33e7075 100644 --- a/packages/metascraper-helpers/package.json +++ b/packages/metascraper-helpers/package.json @@ -25,6 +25,9 @@ "url-regex": "~4.1.1" }, "devDependencies": { + "mocha": "latest", + "nyc": "latest", + "should": "latest", "standard": "latest" }, "engines": { @@ -34,7 +37,12 @@ "index.js" ], "scripts": { - "test": "exit 0" + "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" + }, + "standard": { + "env": [ + "mocha" + ] }, "license": "MIT" } diff --git a/packages/metascraper-helpers/test/index.js b/packages/metascraper-helpers/test/index.js new file mode 100644 index 000000000..4f930d154 --- /dev/null +++ b/packages/metascraper-helpers/test/index.js @@ -0,0 +1,22 @@ +'use strict' + +const should = require('should') + +const { getAbsoluteUrl } = require('../') + +describe('metascraper-helpers', () => { + it('getAbsoluteUrl', () => { + should(getAbsoluteUrl('https://kikobeats.com/', 'blog')).be.equal( + 'https://kikobeats.com/blog' + ) + should(getAbsoluteUrl('https://kikobeats.com', '/blog')).be.equal( + 'https://kikobeats.com/blog' + ) + should(getAbsoluteUrl('https://kikobeats.com/', '/blog')).be.equal( + 'https://kikobeats.com/blog' + ) + should(getAbsoluteUrl('http://kikobeats.com/', '/blog')).be.equal( + 'http://kikobeats.com/blog' + ) + }) +}) From f35ca69b4410ac7282c9aba046e8cc4338e9237b Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sat, 25 Aug 2018 22:32:47 +0200 Subject: [PATCH 020/442] Update ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 76c543dbe..fe8b259a0 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -9,6 +9,8 @@ Describe your issue here. ### Steps to reproduce +> **Note**: You can reproduce the code using [interactive Node.js shell by Runkit](https://npm.runkit.com/metascraper). + Tell us how to reproduce this issue. ### Expected behaviour From 840ac8b74a988393c04b4871603704f0295e5c66 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Mon, 27 Aug 2018 00:44:13 +0200 Subject: [PATCH 021/442] Improve publisher detection based on title (#109) --- packages/metascraper-publisher/index.js | 2 +- packages/metascraper-publisher/package.json | 10 +++++- packages/metascraper-publisher/test/index.js | 32 +++++++++++++++++++ .../__snapshots__/index.js.snap-shot | 4 +-- 4 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 packages/metascraper-publisher/test/index.js diff --git a/packages/metascraper-publisher/index.js b/packages/metascraper-publisher/index.js index d0f5c8fba..5f032955f 100644 --- a/packages/metascraper-publisher/index.js +++ b/packages/metascraper-publisher/index.js @@ -4,7 +4,7 @@ const { isString } = require('lodash') const condenseWhitespace = require('condense-whitespace') const REGEX_RSS = /^(.*?)\s[-|]\satom$/i -const REGEX_TITLE = /^.*?\|\s+(.*)$/ +const REGEX_TITLE = /^.*?[-|]\s+(.*)$/ const validator = value => isString(value) && condenseWhitespace(value) diff --git a/packages/metascraper-publisher/package.json b/packages/metascraper-publisher/package.json index 1078938a5..31d7c83af 100644 --- a/packages/metascraper-publisher/package.json +++ b/packages/metascraper-publisher/package.json @@ -20,6 +20,9 @@ "lodash": "~4.17.10" }, "devDependencies": { + "mocha": "latest", + "nyc": "latest", + "should": "latest", "standard": "latest" }, "engines": { @@ -29,10 +32,15 @@ "index.js" ], "scripts": { - "test": "exit 0" + "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test" }, "license": "MIT", "peerDependencies": { "metascraper": "^3" + }, + "standard": { + "env": [ + "mocha" + ] } } diff --git a/packages/metascraper-publisher/test/index.js b/packages/metascraper-publisher/test/index.js new file mode 100644 index 000000000..49316c449 --- /dev/null +++ b/packages/metascraper-publisher/test/index.js @@ -0,0 +1,32 @@ +'use strict' + +const should = require('should') + +const metascraper = require('metascraper')([require('..')()]) + +const getHtml = title => ` + + + + + + + ${title} + + + + +` + +describe('metascraper-publisher', () => { + describe('from title', async () => { + ;['Murcia | Wikipedia', 'Murcia - Wikipedia', '| Wikipedia'].forEach( + title => + it(`${title} → Wikipedia`, async () => { + const url = 'https://en.wikipedia.org/wiki/Murcia' + const { publisher } = await metascraper({ html: getHtml(title), url }) + should(publisher).be.equal('Wikipedia') + }) + ) + }) +}) diff --git a/packages/metascraper/__snapshots__/index.js.snap-shot b/packages/metascraper/__snapshots__/index.js.snap-shot index a64fdc120..5303c95b7 100644 --- a/packages/metascraper/__snapshots__/index.js.snap-shot +++ b/packages/metascraper/__snapshots__/index.js.snap-shot @@ -32,7 +32,7 @@ exports['astier 1'] = { "video": null, "lang": "en", "logo": "https://anisse.astier.eu/theme/img/favicon.png", - "publisher": null, + "publisher": "awk driven IoT", "title": "Linux Engineer’s random thoughts - awk driven IoT", "url": "https://anisse.astier.eu/awk-driven-iot.html" } @@ -976,7 +976,7 @@ exports['wikipedia 1'] = { "video": "https://upload.wikimedia.org/wikipedia/en/e/e7/Bob_Dylan_-_Blowin'_in_the_Wind.ogg", "lang": "en", "logo": "https://en.wikipedia.org/static/apple-touch/wikipedia.png", - "publisher": null, + "publisher": "Wikipedia", "title": "Bob Dylan - Wikipedia", "url": "https://en.wikipedia.org/wiki/Bob_Dylan" } From 94c028192e57eff0d44319f52495de4d8c5f3137 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Mon, 27 Aug 2018 00:46:03 +0200 Subject: [PATCH 022/442] v4.0.1 --- lerna.json | 2 +- packages/metascraper-amazon/package.json | 4 ++-- packages/metascraper-author/package.json | 4 ++-- packages/metascraper-description/package.json | 4 ++-- packages/metascraper-helpers/package.json | 2 +- packages/metascraper-image/package.json | 4 ++-- packages/metascraper-logo/package.json | 4 ++-- packages/metascraper-publisher/package.json | 2 +- packages/metascraper-soundcloud/package.json | 4 ++-- packages/metascraper-title/package.json | 4 ++-- packages/metascraper-url/package.json | 4 ++-- packages/metascraper-video-provider/package.json | 4 ++-- packages/metascraper-video/package.json | 4 ++-- packages/metascraper-youtube/package.json | 4 ++-- packages/metascraper/package.json | 4 ++-- 15 files changed, 27 insertions(+), 27 deletions(-) diff --git a/lerna.json b/lerna.json index c0fa6a2d8..d9857ae55 100644 --- a/lerna.json +++ b/lerna.json @@ -3,5 +3,5 @@ "packages": [ "packages/*" ], - "version": "4.0.0" + "version": "4.0.1" } diff --git a/packages/metascraper-amazon/package.json b/packages/metascraper-amazon/package.json index aa8d27e59..bcc3f6682 100644 --- a/packages/metascraper-amazon/package.json +++ b/packages/metascraper-amazon/package.json @@ -2,7 +2,7 @@ "name": "metascraper-amazon", "description": "metascraper integration with Amazon", "homepage": "https://documentup.com/microlinkhq/metascraper-amazon", - "version": "4.0.0", + "version": "4.0.1", "main": "index.js", "author": { "email": "hello@microlink.io", @@ -21,7 +21,7 @@ "metascraper" ], "dependencies": { - "@metascraper/helpers": "^4.0.0" + "@metascraper/helpers": "^4.0.1" }, "devDependencies": { "lodash": "latest", diff --git a/packages/metascraper-author/package.json b/packages/metascraper-author/package.json index 30f3b3c2d..805fb35ec 100644 --- a/packages/metascraper-author/package.json +++ b/packages/metascraper-author/package.json @@ -2,7 +2,7 @@ "name": "metascraper-author", "description": "Get author property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.0", + "version": "4.0.1", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^4.0.0", + "@metascraper/helpers": "^4.0.1", "lodash": "~4.17.10" }, "devDependencies": { diff --git a/packages/metascraper-description/package.json b/packages/metascraper-description/package.json index d5292318a..0764dcf19 100644 --- a/packages/metascraper-description/package.json +++ b/packages/metascraper-description/package.json @@ -2,7 +2,7 @@ "name": "metascraper-description", "description": "Get description property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.0", + "version": "4.0.1", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^4.0.0", + "@metascraper/helpers": "^4.0.1", "lodash": "~4.17.10" }, "devDependencies": { diff --git a/packages/metascraper-helpers/package.json b/packages/metascraper-helpers/package.json index 5c33e7075..b858a489f 100644 --- a/packages/metascraper-helpers/package.json +++ b/packages/metascraper-helpers/package.json @@ -2,7 +2,7 @@ "name": "@metascraper/helpers", "description": "Collection of helper functions used by metascraper", "homepage": "https://metascraper.js.org", - "version": "4.0.0", + "version": "4.0.1", "main": "src/index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-image/package.json b/packages/metascraper-image/package.json index 234b4bc87..88a8612aa 100644 --- a/packages/metascraper-image/package.json +++ b/packages/metascraper-image/package.json @@ -2,7 +2,7 @@ "name": "metascraper-image", "description": "Get image property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.0", + "version": "4.0.1", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^4.0.0" + "@metascraper/helpers": "^4.0.1" }, "devDependencies": { "standard": "latest" diff --git a/packages/metascraper-logo/package.json b/packages/metascraper-logo/package.json index dff8ff4cf..a9ccbeea2 100644 --- a/packages/metascraper-logo/package.json +++ b/packages/metascraper-logo/package.json @@ -2,7 +2,7 @@ "name": "metascraper-logo", "description": "Get logo property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.0", + "version": "4.0.1", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^4.0.0", + "@metascraper/helpers": "^4.0.1", "lodash": "~4.17.10" }, "devDependencies": { diff --git a/packages/metascraper-publisher/package.json b/packages/metascraper-publisher/package.json index 31d7c83af..f4a699ce0 100644 --- a/packages/metascraper-publisher/package.json +++ b/packages/metascraper-publisher/package.json @@ -2,7 +2,7 @@ "name": "metascraper-publisher", "description": "Get publisher property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.0", + "version": "4.0.1", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-soundcloud/package.json b/packages/metascraper-soundcloud/package.json index 2f93bd737..20b623cad 100644 --- a/packages/metascraper-soundcloud/package.json +++ b/packages/metascraper-soundcloud/package.json @@ -2,7 +2,7 @@ "name": "metascraper-soundcloud", "description": "metascraper integration with SoundCloud", "homepage": "https://documentup.com/microlinkhq/metascraper-soundcloud", - "version": "4.0.0", + "version": "4.0.1", "main": "index.js", "author": { "email": "hello@microlink.io", @@ -21,7 +21,7 @@ "metascraper" ], "dependencies": { - "@metascraper/helpers": "^4.0.0" + "@metascraper/helpers": "^4.0.1" }, "devDependencies": { "mocha": "latest", diff --git a/packages/metascraper-title/package.json b/packages/metascraper-title/package.json index a7f2dbc96..2053e7f86 100644 --- a/packages/metascraper-title/package.json +++ b/packages/metascraper-title/package.json @@ -2,7 +2,7 @@ "name": "metascraper-title", "description": "Get title property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.0", + "version": "4.0.1", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^4.0.0", + "@metascraper/helpers": "^4.0.1", "lodash": "~4.17.10" }, "devDependencies": { diff --git a/packages/metascraper-url/package.json b/packages/metascraper-url/package.json index 5a494293b..1d7841fc1 100644 --- a/packages/metascraper-url/package.json +++ b/packages/metascraper-url/package.json @@ -2,7 +2,7 @@ "name": "metascraper-url", "description": "Get url property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.0", + "version": "4.0.1", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^4.0.0" + "@metascraper/helpers": "^4.0.1" }, "devDependencies": { "standard": "latest" diff --git a/packages/metascraper-video-provider/package.json b/packages/metascraper-video-provider/package.json index 85aa702a7..1df2a58da 100644 --- a/packages/metascraper-video-provider/package.json +++ b/packages/metascraper-video-provider/package.json @@ -2,7 +2,7 @@ "name": "metascraper-video-provider", "description": "Get video property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.0", + "version": "4.0.1", "main": "src/index", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^4.0.0", + "@metascraper/helpers": "^4.0.1", "got": "~9.1.0", "json-future": "~2.1.2", "lodash": "~4.17.10", diff --git a/packages/metascraper-video/package.json b/packages/metascraper-video/package.json index 5a303aead..3d95cf8be 100644 --- a/packages/metascraper-video/package.json +++ b/packages/metascraper-video/package.json @@ -2,7 +2,7 @@ "name": "metascraper-video", "description": "Get video property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.0", + "version": "4.0.1", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^4.0.0", + "@metascraper/helpers": "^4.0.1", "video-extensions": "~1.1.0" }, "devDependencies": { diff --git a/packages/metascraper-youtube/package.json b/packages/metascraper-youtube/package.json index 55d01a201..cf0efef57 100644 --- a/packages/metascraper-youtube/package.json +++ b/packages/metascraper-youtube/package.json @@ -2,7 +2,7 @@ "name": "metascraper-youtube", "description": "metascraper integration with YouTube", "homepage": "https://documentup.com/microlinkhq/metascraper-youtube", - "version": "4.0.0", + "version": "4.0.1", "main": "index.js", "author": { "email": "hello@microlink.io", @@ -21,7 +21,7 @@ "metascraper" ], "dependencies": { - "@metascraper/helpers": "^4.0.0", + "@metascraper/helpers": "^4.0.1", "get-video-id": "~3.1.0", "is-reachable": "~2.4.0", "lodash": "~4.17.10", diff --git a/packages/metascraper/package.json b/packages/metascraper/package.json index c14039d9c..3410c78a4 100644 --- a/packages/metascraper/package.json +++ b/packages/metascraper/package.json @@ -2,7 +2,7 @@ "name": "metascraper", "description": "A library to easily scrape metadata from an article on the web using Open Graph metadata, regular HTML metadata, and series of fallbacks.", "homepage": "https://metascraper.js.org", - "version": "4.0.0", + "version": "4.0.1", "main": "src/index.js", "author": { "email": "ian@ianstormtaylor.com", @@ -55,7 +55,7 @@ "website" ], "dependencies": { - "@metascraper/helpers": "^4.0.0", + "@metascraper/helpers": "^4.0.1", "cheerio": "~1.0.0-rc.2", "cheerio-advanced-selectors": "~2.0.1", "lodash": "~4.17.10", From 321a2abc02fbfda9ceb37ca1f34646df721f7e39 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Mon, 27 Aug 2018 00:47:07 +0200 Subject: [PATCH 023/442] Update Changelog --- CHANGELOG.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 733b590bf..be5b93b43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,18 @@ -## (2018-08-24) +## (2018-08-26) +## 4.0.1 (2018-08-26) + +* Add tests ([58c26f0](https://github.com/microlinkhq/metascraper/commit/58c26f0)) +* Improve publisher detection based on title (#109) ([840ac8b](https://github.com/microlinkhq/metascraper/commit/840ac8b)), closes [#109](https://github.com/microlinkhq/metascraper/issues/109) +* Update Changelog ([2a3dc73](https://github.com/microlinkhq/metascraper/commit/2a3dc73)) +* Update ISSUE_TEMPLATE.md ([f35ca69](https://github.com/microlinkhq/metascraper/commit/f35ca69)) +* v4.0.1 ([94c0281](https://github.com/microlinkhq/metascraper/commit/94c0281)) + + + ## 4.0.0 (2018-08-24) * Remove autoload ([9d3745b](https://github.com/microlinkhq/metascraper/commit/9d3745b)) From a83729905b148b8c3a34a184b8fd38de77fc23be Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Mon, 27 Aug 2018 09:55:46 +0200 Subject: [PATCH 024/442] Update peer dependency to metascraper 4 --- packages/metascraper-amazon/package.json | 2 +- packages/metascraper-author/package.json | 2 +- packages/metascraper-clearbit-logo/package.json | 2 +- packages/metascraper-date/package.json | 2 +- packages/metascraper-description/package.json | 2 +- packages/metascraper-image/package.json | 2 +- packages/metascraper-lang/package.json | 2 +- packages/metascraper-logo-favicon/package.json | 2 +- packages/metascraper-logo/package.json | 2 +- packages/metascraper-publisher/package.json | 2 +- packages/metascraper-soundcloud/package.json | 2 +- packages/metascraper-title/package.json | 2 +- packages/metascraper-url/package.json | 2 +- packages/metascraper-video-provider/package.json | 2 +- packages/metascraper-video/package.json | 2 +- packages/metascraper-youtube/package.json | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/metascraper-amazon/package.json b/packages/metascraper-amazon/package.json index bcc3f6682..5703a7954 100644 --- a/packages/metascraper-amazon/package.json +++ b/packages/metascraper-amazon/package.json @@ -42,7 +42,7 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" }, "standard": { "env": [ diff --git a/packages/metascraper-author/package.json b/packages/metascraper-author/package.json index 805fb35ec..804c301a9 100644 --- a/packages/metascraper-author/package.json +++ b/packages/metascraper-author/package.json @@ -33,6 +33,6 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" } } diff --git a/packages/metascraper-clearbit-logo/package.json b/packages/metascraper-clearbit-logo/package.json index ca669dbad..bb0c52a26 100644 --- a/packages/metascraper-clearbit-logo/package.json +++ b/packages/metascraper-clearbit-logo/package.json @@ -40,7 +40,7 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" }, "standard": { "env": [ diff --git a/packages/metascraper-date/package.json b/packages/metascraper-date/package.json index 307f039ad..8a6ffb7ec 100644 --- a/packages/metascraper-date/package.json +++ b/packages/metascraper-date/package.json @@ -33,6 +33,6 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" } } diff --git a/packages/metascraper-description/package.json b/packages/metascraper-description/package.json index 0764dcf19..dc1e9058a 100644 --- a/packages/metascraper-description/package.json +++ b/packages/metascraper-description/package.json @@ -33,6 +33,6 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" } } diff --git a/packages/metascraper-image/package.json b/packages/metascraper-image/package.json index 88a8612aa..61c2ee681 100644 --- a/packages/metascraper-image/package.json +++ b/packages/metascraper-image/package.json @@ -32,6 +32,6 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" } } diff --git a/packages/metascraper-lang/package.json b/packages/metascraper-lang/package.json index 2dc0f5e81..8e092f481 100644 --- a/packages/metascraper-lang/package.json +++ b/packages/metascraper-lang/package.json @@ -36,7 +36,7 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" }, "standard": { "env": [ diff --git a/packages/metascraper-logo-favicon/package.json b/packages/metascraper-logo-favicon/package.json index 539639477..fb96d7c8e 100644 --- a/packages/metascraper-logo-favicon/package.json +++ b/packages/metascraper-logo-favicon/package.json @@ -39,7 +39,7 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" }, "standard": { "env": [ diff --git a/packages/metascraper-logo/package.json b/packages/metascraper-logo/package.json index a9ccbeea2..afd50845d 100644 --- a/packages/metascraper-logo/package.json +++ b/packages/metascraper-logo/package.json @@ -33,6 +33,6 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" } } diff --git a/packages/metascraper-publisher/package.json b/packages/metascraper-publisher/package.json index f4a699ce0..15600d09c 100644 --- a/packages/metascraper-publisher/package.json +++ b/packages/metascraper-publisher/package.json @@ -36,7 +36,7 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" }, "standard": { "env": [ diff --git a/packages/metascraper-soundcloud/package.json b/packages/metascraper-soundcloud/package.json index 20b623cad..4157e7d2a 100644 --- a/packages/metascraper-soundcloud/package.json +++ b/packages/metascraper-soundcloud/package.json @@ -41,7 +41,7 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" }, "standard": { "env": [ diff --git a/packages/metascraper-title/package.json b/packages/metascraper-title/package.json index 2053e7f86..9ec92d7a4 100644 --- a/packages/metascraper-title/package.json +++ b/packages/metascraper-title/package.json @@ -33,6 +33,6 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" } } diff --git a/packages/metascraper-url/package.json b/packages/metascraper-url/package.json index 1d7841fc1..0cfd5f66e 100644 --- a/packages/metascraper-url/package.json +++ b/packages/metascraper-url/package.json @@ -32,6 +32,6 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" } } diff --git a/packages/metascraper-video-provider/package.json b/packages/metascraper-video-provider/package.json index 1df2a58da..61c4ae3bc 100644 --- a/packages/metascraper-video-provider/package.json +++ b/packages/metascraper-video-provider/package.json @@ -41,7 +41,7 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" }, "standard": { "env": [ diff --git a/packages/metascraper-video/package.json b/packages/metascraper-video/package.json index 3d95cf8be..cae4ffc3d 100644 --- a/packages/metascraper-video/package.json +++ b/packages/metascraper-video/package.json @@ -38,7 +38,7 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" }, "standard": { "env": [ diff --git a/packages/metascraper-youtube/package.json b/packages/metascraper-youtube/package.json index cf0efef57..5c4ee80f8 100644 --- a/packages/metascraper-youtube/package.json +++ b/packages/metascraper-youtube/package.json @@ -45,7 +45,7 @@ }, "license": "MIT", "peerDependencies": { - "metascraper": "^3" + "metascraper": "^4" }, "standard": { "env": [ From a287d23604639a01b60faeed5673d732dda7c126 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Mon, 27 Aug 2018 09:56:45 +0200 Subject: [PATCH 025/442] v4.0.2 --- lerna.json | 2 +- packages/metascraper-amazon/package.json | 2 +- packages/metascraper-author/package.json | 2 +- packages/metascraper-clearbit-logo/package.json | 2 +- packages/metascraper-date/package.json | 2 +- packages/metascraper-description/package.json | 2 +- packages/metascraper-image/package.json | 2 +- packages/metascraper-lang/package.json | 2 +- packages/metascraper-logo-favicon/package.json | 2 +- packages/metascraper-logo/package.json | 2 +- packages/metascraper-publisher/package.json | 2 +- packages/metascraper-soundcloud/package.json | 2 +- packages/metascraper-title/package.json | 2 +- packages/metascraper-url/package.json | 2 +- packages/metascraper-video-provider/package.json | 2 +- packages/metascraper-video/package.json | 2 +- packages/metascraper-youtube/package.json | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lerna.json b/lerna.json index d9857ae55..5faea7a97 100644 --- a/lerna.json +++ b/lerna.json @@ -3,5 +3,5 @@ "packages": [ "packages/*" ], - "version": "4.0.1" + "version": "4.0.2" } diff --git a/packages/metascraper-amazon/package.json b/packages/metascraper-amazon/package.json index 5703a7954..efa629d50 100644 --- a/packages/metascraper-amazon/package.json +++ b/packages/metascraper-amazon/package.json @@ -2,7 +2,7 @@ "name": "metascraper-amazon", "description": "metascraper integration with Amazon", "homepage": "https://documentup.com/microlinkhq/metascraper-amazon", - "version": "4.0.1", + "version": "4.0.2", "main": "index.js", "author": { "email": "hello@microlink.io", diff --git a/packages/metascraper-author/package.json b/packages/metascraper-author/package.json index 804c301a9..6f6d35136 100644 --- a/packages/metascraper-author/package.json +++ b/packages/metascraper-author/package.json @@ -2,7 +2,7 @@ "name": "metascraper-author", "description": "Get author property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.1", + "version": "4.0.2", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-clearbit-logo/package.json b/packages/metascraper-clearbit-logo/package.json index bb0c52a26..2cd4ec945 100644 --- a/packages/metascraper-clearbit-logo/package.json +++ b/packages/metascraper-clearbit-logo/package.json @@ -2,7 +2,7 @@ "name": "metascraper-clearbit-logo", "description": "metascraper integration with Clearbit Logo API", "homepage": "https://documentup.com/microlinkhq/metascraper-clearbit-logo", - "version": "4.0.0", + "version": "4.0.2", "main": "index.js", "author": { "email": "hello@microlink.io", diff --git a/packages/metascraper-date/package.json b/packages/metascraper-date/package.json index 8a6ffb7ec..04df1e766 100644 --- a/packages/metascraper-date/package.json +++ b/packages/metascraper-date/package.json @@ -2,7 +2,7 @@ "name": "metascraper-date", "description": "Get date property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.0", + "version": "4.0.2", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-description/package.json b/packages/metascraper-description/package.json index dc1e9058a..be3e925f2 100644 --- a/packages/metascraper-description/package.json +++ b/packages/metascraper-description/package.json @@ -2,7 +2,7 @@ "name": "metascraper-description", "description": "Get description property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.1", + "version": "4.0.2", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-image/package.json b/packages/metascraper-image/package.json index 61c2ee681..729f82d69 100644 --- a/packages/metascraper-image/package.json +++ b/packages/metascraper-image/package.json @@ -2,7 +2,7 @@ "name": "metascraper-image", "description": "Get image property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.1", + "version": "4.0.2", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-lang/package.json b/packages/metascraper-lang/package.json index 8e092f481..c5e1bd95d 100644 --- a/packages/metascraper-lang/package.json +++ b/packages/metascraper-lang/package.json @@ -2,7 +2,7 @@ "name": "metascraper-lang", "description": "Get lang property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.0", + "version": "4.0.2", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-logo-favicon/package.json b/packages/metascraper-logo-favicon/package.json index fb96d7c8e..dc4376149 100644 --- a/packages/metascraper-logo-favicon/package.json +++ b/packages/metascraper-logo-favicon/package.json @@ -2,7 +2,7 @@ "name": "metascraper-logo-favicon", "description": "metascraper logo favicon fallback", "homepage": "https://documentup.com/microlinkhq/metascraper-logo-favicon", - "version": "4.0.0", + "version": "4.0.2", "main": "index.js", "author": { "email": "hello@microlink.io", diff --git a/packages/metascraper-logo/package.json b/packages/metascraper-logo/package.json index afd50845d..8c8a160dd 100644 --- a/packages/metascraper-logo/package.json +++ b/packages/metascraper-logo/package.json @@ -2,7 +2,7 @@ "name": "metascraper-logo", "description": "Get logo property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.1", + "version": "4.0.2", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-publisher/package.json b/packages/metascraper-publisher/package.json index 15600d09c..aa9054b98 100644 --- a/packages/metascraper-publisher/package.json +++ b/packages/metascraper-publisher/package.json @@ -2,7 +2,7 @@ "name": "metascraper-publisher", "description": "Get publisher property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.1", + "version": "4.0.2", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-soundcloud/package.json b/packages/metascraper-soundcloud/package.json index 4157e7d2a..52699c905 100644 --- a/packages/metascraper-soundcloud/package.json +++ b/packages/metascraper-soundcloud/package.json @@ -2,7 +2,7 @@ "name": "metascraper-soundcloud", "description": "metascraper integration with SoundCloud", "homepage": "https://documentup.com/microlinkhq/metascraper-soundcloud", - "version": "4.0.1", + "version": "4.0.2", "main": "index.js", "author": { "email": "hello@microlink.io", diff --git a/packages/metascraper-title/package.json b/packages/metascraper-title/package.json index 9ec92d7a4..6b61bc9cf 100644 --- a/packages/metascraper-title/package.json +++ b/packages/metascraper-title/package.json @@ -2,7 +2,7 @@ "name": "metascraper-title", "description": "Get title property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.1", + "version": "4.0.2", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-url/package.json b/packages/metascraper-url/package.json index 0cfd5f66e..99d2b49f6 100644 --- a/packages/metascraper-url/package.json +++ b/packages/metascraper-url/package.json @@ -2,7 +2,7 @@ "name": "metascraper-url", "description": "Get url property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.1", + "version": "4.0.2", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-video-provider/package.json b/packages/metascraper-video-provider/package.json index 61c4ae3bc..7c28c7799 100644 --- a/packages/metascraper-video-provider/package.json +++ b/packages/metascraper-video-provider/package.json @@ -2,7 +2,7 @@ "name": "metascraper-video-provider", "description": "Get video property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.1", + "version": "4.0.2", "main": "src/index", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-video/package.json b/packages/metascraper-video/package.json index cae4ffc3d..091bd7caf 100644 --- a/packages/metascraper-video/package.json +++ b/packages/metascraper-video/package.json @@ -2,7 +2,7 @@ "name": "metascraper-video", "description": "Get video property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.1", + "version": "4.0.2", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-youtube/package.json b/packages/metascraper-youtube/package.json index 5c4ee80f8..a42cb1dce 100644 --- a/packages/metascraper-youtube/package.json +++ b/packages/metascraper-youtube/package.json @@ -2,7 +2,7 @@ "name": "metascraper-youtube", "description": "metascraper integration with YouTube", "homepage": "https://documentup.com/microlinkhq/metascraper-youtube", - "version": "4.0.1", + "version": "4.0.2", "main": "index.js", "author": { "email": "hello@microlink.io", From 8766f136cc397a5ce2e56721390db555b73c0490 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Mon, 27 Aug 2018 09:57:39 +0200 Subject: [PATCH 026/442] Update Changelog --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be5b93b43..3f0a879ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,16 @@ -## (2018-08-26) +## (2018-08-27) +## 4.0.2 (2018-08-27) + +* Update Changelog ([321a2ab](https://github.com/microlinkhq/metascraper/commit/321a2ab)) +* Update peer dependency to metascraper 4 ([a837299](https://github.com/microlinkhq/metascraper/commit/a837299)) +* v4.0.2 ([a287d23](https://github.com/microlinkhq/metascraper/commit/a287d23)) + + + ## 4.0.1 (2018-08-26) * Add tests ([58c26f0](https://github.com/microlinkhq/metascraper/commit/58c26f0)) From e49a85c67c08513de6a9ad9c5f0053917cffe25d Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Tue, 28 Aug 2018 00:55:12 +0200 Subject: [PATCH 027/442] Add condition to stop --- .../__snapshots__/index.js.snap-shot | 2 +- packages/metascraper-publisher/index.js | 5 ++++- packages/metascraper-publisher/test/index.js | 20 +++++++++++-------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/metascraper-lang/__snapshots__/index.js.snap-shot b/packages/metascraper-lang/__snapshots__/index.js.snap-shot index 8344fb705..05e5c77e9 100644 --- a/packages/metascraper-lang/__snapshots__/index.js.snap-shot +++ b/packages/metascraper-lang/__snapshots__/index.js.snap-shot @@ -2,7 +2,7 @@ exports['html lang property 1'] = { "lang": "pl", "author": "Jakub Majmurek", "title": "Churchill, bohater naszych czasów / Film / dwutygodnik.com", - "publisher": "Film | Dwutygodnik | Dwutygodnik", + "publisher": "Dwutygodnik", "image": "http://www.dwutygodnik.com/public/media/article/image_full/7615.png", "date": "2018-01-01T12:00:00.000Z", "description": "Wysyp filmów o Churchillu w pobrexitowej Brytanii, wydaje się czymś zrozumiałym. Uosabia on ostatni moment prawdziwej wielkości Zjednoczonego Królestwa – wspomnienie tej historycznej chwili pozwala oswoić traumy i lęki", diff --git a/packages/metascraper-publisher/index.js b/packages/metascraper-publisher/index.js index 5f032955f..c1989fcb1 100644 --- a/packages/metascraper-publisher/index.js +++ b/packages/metascraper-publisher/index.js @@ -22,7 +22,10 @@ const wrap = rule => ({ htmlDom }) => { const getFromTitle = (text, regex) => { const matches = regex.exec(text) - return matches ? matches[1] : false + if (!matches) return false + let result = matches[1] + while (regex.test(result)) result = regex.exec(result)[1] + return result } /** diff --git a/packages/metascraper-publisher/test/index.js b/packages/metascraper-publisher/test/index.js index 49316c449..40d127ba0 100644 --- a/packages/metascraper-publisher/test/index.js +++ b/packages/metascraper-publisher/test/index.js @@ -1,7 +1,6 @@ 'use strict' const should = require('should') - const metascraper = require('metascraper')([require('..')()]) const getHtml = title => ` @@ -20,13 +19,18 @@ const getHtml = title => ` describe('metascraper-publisher', () => { describe('from title', async () => { - ;['Murcia | Wikipedia', 'Murcia - Wikipedia', '| Wikipedia'].forEach( - title => - it(`${title} → Wikipedia`, async () => { - const url = 'https://en.wikipedia.org/wiki/Murcia' - const { publisher } = await metascraper({ html: getHtml(title), url }) - should(publisher).be.equal('Wikipedia') - }) + ;[ + 'Murcia | Wikipedia', + 'Murcia - Wikipedia', + '| Wikipedia', + 'San Antonio Spurs guard Manu Ginobili... - San Antonio Spurs | Wikipedia', + 'San Antonio Spurs guard Manu Ginobili... | San Antonio Spurs - Wikipedia' + ].forEach(title => + it(`${title} → Wikipedia`, async () => { + const url = 'https://en.wikipedia.org/wiki/Murcia' + const { publisher } = await metascraper({ html: getHtml(title), url }) + should(publisher).be.equal('Wikipedia') + }) ) }) }) From a2027c9687154fe1874651e94be921474988f2d6 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Tue, 28 Aug 2018 09:46:51 +0200 Subject: [PATCH 028/442] v4.0.3 --- lerna.json | 2 +- packages/metascraper-lang/package.json | 2 +- packages/metascraper-publisher/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lerna.json b/lerna.json index 5faea7a97..0888609a4 100644 --- a/lerna.json +++ b/lerna.json @@ -3,5 +3,5 @@ "packages": [ "packages/*" ], - "version": "4.0.2" + "version": "4.0.3" } diff --git a/packages/metascraper-lang/package.json b/packages/metascraper-lang/package.json index c5e1bd95d..d14db8789 100644 --- a/packages/metascraper-lang/package.json +++ b/packages/metascraper-lang/package.json @@ -2,7 +2,7 @@ "name": "metascraper-lang", "description": "Get lang property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.2", + "version": "4.0.3", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", diff --git a/packages/metascraper-publisher/package.json b/packages/metascraper-publisher/package.json index aa9054b98..e189793ba 100644 --- a/packages/metascraper-publisher/package.json +++ b/packages/metascraper-publisher/package.json @@ -2,7 +2,7 @@ "name": "metascraper-publisher", "description": "Get publisher property from HTML markup", "homepage": "https://metascraper.js.org", - "version": "4.0.2", + "version": "4.0.3", "main": "index.js", "author": { "email": "ian@ianstormtaylor.com", From 4ddd2093003b10b618f0e0e8495c78dbc3d24ea4 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Tue, 28 Aug 2018 09:47:26 +0200 Subject: [PATCH 029/442] Update Changelog --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f0a879ff..155dac40a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,16 @@ -## (2018-08-27) +## (2018-08-28) +## 4.0.3 (2018-08-28) + +* Add condition to stop ([e49a85c](https://github.com/microlinkhq/metascraper/commit/e49a85c)) +* Update Changelog ([8766f13](https://github.com/microlinkhq/metascraper/commit/8766f13)) +* v4.0.3 ([a2027c9](https://github.com/microlinkhq/metascraper/commit/a2027c9)) + + + ## 4.0.2 (2018-08-27) * Update Changelog ([321a2ab](https://github.com/microlinkhq/metascraper/commit/321a2ab)) From 77080db0f3445877386a722a67e1da9aed57e1c1 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Wed, 29 Aug 2018 09:20:16 +0200 Subject: [PATCH 030/442] =?UTF-8?q?Rename=20video-provider=20=E2=86=92=20m?= =?UTF-8?q?edia-provider?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- .../.npmrc | 0 .../README.md | 10 +++++----- .../__snapshots__/index.js.snap-shot | 0 .../package.json | 2 +- .../src/get-video-info/index.js | 6 +++--- .../src/get-video-info/twitter-video-info.js | 0 .../src/get-video-info/video-info.js | 0 .../src/index.js | 0 .../test/fixtures/facebook.html | 0 .../test/fixtures/output.json | 0 .../test/fixtures/twitter.html | 0 .../test/fixtures/vimeo.html | 0 .../test/fixtures/youtube.html | 0 .../test/index.js | 6 +++--- .../test/mocha.opts | 0 16 files changed, 13 insertions(+), 13 deletions(-) rename packages/{metascraper-video-provider => metascraper-media-provider}/.npmrc (100%) rename packages/{metascraper-video-provider => metascraper-media-provider}/README.md (68%) rename packages/{metascraper-video-provider => metascraper-media-provider}/__snapshots__/index.js.snap-shot (100%) rename packages/{metascraper-video-provider => metascraper-media-provider}/package.json (96%) rename packages/{metascraper-video-provider => metascraper-media-provider}/src/get-video-info/index.js (84%) rename packages/{metascraper-video-provider => metascraper-media-provider}/src/get-video-info/twitter-video-info.js (100%) rename packages/{metascraper-video-provider => metascraper-media-provider}/src/get-video-info/video-info.js (100%) rename packages/{metascraper-video-provider => metascraper-media-provider}/src/index.js (100%) rename packages/{metascraper-video-provider => metascraper-media-provider}/test/fixtures/facebook.html (100%) rename packages/{metascraper-video-provider => metascraper-media-provider}/test/fixtures/output.json (100%) rename packages/{metascraper-video-provider => metascraper-media-provider}/test/fixtures/twitter.html (100%) rename packages/{metascraper-video-provider => metascraper-media-provider}/test/fixtures/vimeo.html (100%) rename packages/{metascraper-video-provider => metascraper-media-provider}/test/fixtures/youtube.html (100%) rename packages/{metascraper-video-provider => metascraper-media-provider}/test/index.js (94%) rename packages/{metascraper-video-provider => metascraper-media-provider}/test/mocha.opts (100%) diff --git a/README.md b/README.md index d3591ad90..436a186f4 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ const metascraper = require('metascraper')([ | [`metascraper-logo-favicon`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-logo-favicon) | [![npm](https://img.shields.io/npm/v/metascraper-logo-favicon.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-logo-favicon) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-logo-favicon&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-logo-favicon) | | [`metascraper-soundcloud`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-soundcloud) | [![npm](https://img.shields.io/npm/v/metascraper-soundcloud.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-soundcloud) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-soundcloud&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-soundcloud) | | [`metascraper-youtube`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-youtube) | [![npm](https://img.shields.io/npm/v/metascraper-youtube.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-youtube) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-youtube&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-youtube) | -| [`metascraper-video-provider`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-video-provider) | [![npm](https://img.shields.io/npm/v/metascraper-video-provider.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-video-provider) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-video-provider&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-video-provider) | +| [`metascraper-media-provider`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-media-provider) | [![npm](https://img.shields.io/npm/v/metascraper-media-provider.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-media-provider) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-media-provider&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-media-provider) | ### Write your own rules diff --git a/packages/metascraper-video-provider/.npmrc b/packages/metascraper-media-provider/.npmrc similarity index 100% rename from packages/metascraper-video-provider/.npmrc rename to packages/metascraper-media-provider/.npmrc diff --git a/packages/metascraper-video-provider/README.md b/packages/metascraper-media-provider/README.md similarity index 68% rename from packages/metascraper-video-provider/README.md rename to packages/metascraper-media-provider/README.md index 04ae629a9..f75a2610f 100644 --- a/packages/metascraper-video-provider/README.md +++ b/packages/metascraper-media-provider/README.md @@ -1,19 +1,19 @@ -# metascraper-video-provider +# metascraper-media-provider -[![npm](https://img.shields.io/npm/v/metascraper-video-provider.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-video-provider) -[![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-video-provider&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-video-provider) +[![npm](https://img.shields.io/npm/v/metascraper-media-provider.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-media-provider) +[![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-media-provider&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-media-provider) > Get specific video provider url (Facebook/Twitter/Vimeo/etc). ## Install ```bash -$ npm install metascraper-video-provider --save +$ npm install metascraper-media-provider --save ``` ## License -**metascraper-video-provider** © [microlink.io](https://microlink.io), Released under the [MIT](https://github.com/microlinkhq//blob/master/LICENSE.md) License.
+**metascraper-media-provider** © [microlink.io](https://microlink.io), Released under the [MIT](https://github.com/microlinkhq//blob/master/LICENSE.md) License.
Authored and maintained by microlink.io with help from [contributors](https://github.com/microlinkhq//contributors). > [microlink.io](https://microlink.io) · GitHub [@microlink.io](https://github.com/microlinkhq) · Twitter [@microlinkhq](https://twitter.com/microlinkhq) diff --git a/packages/metascraper-video-provider/__snapshots__/index.js.snap-shot b/packages/metascraper-media-provider/__snapshots__/index.js.snap-shot similarity index 100% rename from packages/metascraper-video-provider/__snapshots__/index.js.snap-shot rename to packages/metascraper-media-provider/__snapshots__/index.js.snap-shot diff --git a/packages/metascraper-video-provider/package.json b/packages/metascraper-media-provider/package.json similarity index 96% rename from packages/metascraper-video-provider/package.json rename to packages/metascraper-media-provider/package.json index 7c28c7799..dc3d42555 100644 --- a/packages/metascraper-video-provider/package.json +++ b/packages/metascraper-media-provider/package.json @@ -1,5 +1,5 @@ { - "name": "metascraper-video-provider", + "name": "metascraper-media-provider", "description": "Get video property from HTML markup", "homepage": "https://metascraper.js.org", "version": "4.0.2", diff --git a/packages/metascraper-video-provider/src/get-video-info/index.js b/packages/metascraper-media-provider/src/get-video-info/index.js similarity index 84% rename from packages/metascraper-video-provider/src/get-video-info/index.js rename to packages/metascraper-media-provider/src/get-video-info/index.js index 2cb2a9155..ef7612028 100644 --- a/packages/metascraper-video-provider/src/get-video-info/index.js +++ b/packages/metascraper-media-provider/src/get-video-info/index.js @@ -1,6 +1,6 @@ 'use strict' -const {isTwitterUrl, getTwitterVideoInfo} = require('./twitter-video-info') +const { isTwitterUrl, getTwitterVideoInfo } = require('./twitter-video-info') const getVideoInfo = require('./video-info') const { chain } = require('lodash') @@ -15,12 +15,12 @@ const getInfo = async url => { const formats = chain(videoInfo.formats) .reduce((acc, format, index) => { const { url } = twitterVideos[index] - const item = {...format, url} + const item = { ...format, url } return [...acc, item] }, []) .value() - return {...videoInfo, formats} + return { ...videoInfo, formats } } // Local cache for successive calls diff --git a/packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js b/packages/metascraper-media-provider/src/get-video-info/twitter-video-info.js similarity index 100% rename from packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js rename to packages/metascraper-media-provider/src/get-video-info/twitter-video-info.js diff --git a/packages/metascraper-video-provider/src/get-video-info/video-info.js b/packages/metascraper-media-provider/src/get-video-info/video-info.js similarity index 100% rename from packages/metascraper-video-provider/src/get-video-info/video-info.js rename to packages/metascraper-media-provider/src/get-video-info/video-info.js diff --git a/packages/metascraper-video-provider/src/index.js b/packages/metascraper-media-provider/src/index.js similarity index 100% rename from packages/metascraper-video-provider/src/index.js rename to packages/metascraper-media-provider/src/index.js diff --git a/packages/metascraper-video-provider/test/fixtures/facebook.html b/packages/metascraper-media-provider/test/fixtures/facebook.html similarity index 100% rename from packages/metascraper-video-provider/test/fixtures/facebook.html rename to packages/metascraper-media-provider/test/fixtures/facebook.html diff --git a/packages/metascraper-video-provider/test/fixtures/output.json b/packages/metascraper-media-provider/test/fixtures/output.json similarity index 100% rename from packages/metascraper-video-provider/test/fixtures/output.json rename to packages/metascraper-media-provider/test/fixtures/output.json diff --git a/packages/metascraper-video-provider/test/fixtures/twitter.html b/packages/metascraper-media-provider/test/fixtures/twitter.html similarity index 100% rename from packages/metascraper-video-provider/test/fixtures/twitter.html rename to packages/metascraper-media-provider/test/fixtures/twitter.html diff --git a/packages/metascraper-video-provider/test/fixtures/vimeo.html b/packages/metascraper-media-provider/test/fixtures/vimeo.html similarity index 100% rename from packages/metascraper-video-provider/test/fixtures/vimeo.html rename to packages/metascraper-media-provider/test/fixtures/vimeo.html diff --git a/packages/metascraper-video-provider/test/fixtures/youtube.html b/packages/metascraper-media-provider/test/fixtures/youtube.html similarity index 100% rename from packages/metascraper-video-provider/test/fixtures/youtube.html rename to packages/metascraper-media-provider/test/fixtures/youtube.html diff --git a/packages/metascraper-video-provider/test/index.js b/packages/metascraper-media-provider/test/index.js similarity index 94% rename from packages/metascraper-video-provider/test/index.js rename to packages/metascraper-media-provider/test/index.js index c5f7642f9..24107d88b 100644 --- a/packages/metascraper-video-provider/test/index.js +++ b/packages/metascraper-media-provider/test/index.js @@ -10,7 +10,7 @@ const should = require('should') const fs = require('fs') const metascraper = require('metascraper')([ - require('metascraper-video-provider')(), + require('metascraper-media-provider')(), require('metascraper-author')(), require('metascraper-date')(), require('metascraper-description')(), @@ -24,11 +24,11 @@ const metascraper = require('metascraper')([ const readFile = promisify(fs.readFile) -const { getVideoUrl, isMp4 } = require('metascraper-video-provider') +const { getVideoUrl, isMp4 } = require('metascraper-media-provider') const output = require('./fixtures/output.json') -describe('metascraper-video-provider', () => { +describe('metascraper-media-provider', () => { describe('.getVideoUrl', () => { it('isMp4', () => { const videoUrl = getVideoUrl(output.formats, [isMp4]) diff --git a/packages/metascraper-video-provider/test/mocha.opts b/packages/metascraper-media-provider/test/mocha.opts similarity index 100% rename from packages/metascraper-video-provider/test/mocha.opts rename to packages/metascraper-media-provider/test/mocha.opts From 9ffcd6770da4f266c6673a5e77f44fb3604581b4 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Wed, 29 Aug 2018 09:50:19 +0200 Subject: [PATCH 031/442] Return a collection of urls --- .../__snapshots__/index.js.snap-shot | 11 + .../metascraper-media-provider/src/index.js | 26 +- .../test/fixtures/output.json | 1 - .../test/fixtures/youtube-dl.json | 706 ++++++++++++++++++ .../metascraper-media-provider/test/index.js | 12 +- 5 files changed, 731 insertions(+), 25 deletions(-) delete mode 100644 packages/metascraper-media-provider/test/fixtures/output.json create mode 100644 packages/metascraper-media-provider/test/fixtures/youtube-dl.json diff --git a/packages/metascraper-media-provider/__snapshots__/index.js.snap-shot b/packages/metascraper-media-provider/__snapshots__/index.js.snap-shot index acee2c2a0..d4e68481d 100644 --- a/packages/metascraper-media-provider/__snapshots__/index.js.snap-shot +++ b/packages/metascraper-media-provider/__snapshots__/index.js.snap-shot @@ -42,3 +42,14 @@ exports['vimeo 1'] = { "url": "https://vimeo.com/188175573" } +exports['filter by mp4 1'] = [ + "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=160&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184853954309&fvip=4&source=youtube&clen=1232192&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=B71A314816CBE56289EC15C8A1C6BF7EE74C169D.164175CF39A4BDFD2DA2DE6147BD09545AAA01DE&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=133&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184853956556&fvip=4&source=youtube&clen=2122314&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=E3307C18153DDB5E7AACAEAB9942F8B002012050.173DE890AE005232725CB8ADA7A50B7CF25692F0&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=134&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854055225&fvip=4&source=youtube&clen=5015666&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=3DEFAB448DC70677960CAF56B11FB769592FC440.5CD5B13E517668562CCE3F430CF16734E3DD63CA&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=135&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854155850&fvip=4&source=youtube&clen=9629112&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=B4B7A543DEAEEF890B8E6D2DCD83F8F07633DA9F.503058111885647E7E14FA29CA781D0AC0EACA6D&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=136&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854254121&fvip=4&source=youtube&clen=18536895&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=15992BA735625E478060A1DFD9C01D00C0702BE4.C887CC9578BFCBE0ABF4FFCA47139C40B07436E3&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=298&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184834230729&fvip=4&source=youtube&clen=30831030&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=3ED7C0216A6CBFD79647E8BD4E9463A955EB2E8D.4B933FF684BCA75D1E10C2771A101A646875F4BF&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=18&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534185137343773&fvip=4&source=youtube&clen=14141551&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&ratebypass=yes&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=250354E58D1DBCA03CBD4DC9E3FB244C5E1152A0.952F5690B6C36A18ADEA9DDA89ED78148CAFA2A4&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&dur=283.910&ei=KhRyW4nkJIvkV7yXvugN", + "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=22&mm=31%2C29&c=WEB&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mime=video%2Fmp4&ip=95.17.206.54&pl=20&signature=26B8FCB51DC7903132039080C3888D83EC65E0F1.07B5FDE4D99FF3C3354D4D164FBC3CA838F6DEFF&expire=1534224522&initcwndbps=928750&mv=m&mt=1534202758&ms=au%2Crdu&ipbits=0&lmt=1534185082305521&requiressl=yes&sparams=dur%2Cei%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&source=youtube&ratebypass=yes&dur=283.910&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&key=yt6&ei=KhRyW4nkJIvkV7yXvugN&fvip=4" +] + diff --git a/packages/metascraper-media-provider/src/index.js b/packages/metascraper-media-provider/src/index.js index 19ec9a075..a57cfe772 100644 --- a/packages/metascraper-media-provider/src/index.js +++ b/packages/metascraper-media-provider/src/index.js @@ -5,8 +5,6 @@ const { isEmpty, eq, has, - round, - size, get, chain, find, @@ -24,32 +22,24 @@ const isHttp = video => eq(get(video, 'protocol'), 'http') const isHttps = video => eq(get(video, 'protocol'), 'https') const hasAudio = video => has(video, 'abr') -/** - * Get a Video source quality enough good - * compatible to be consumed for the browser. - */ -const getVideoUrl = (videos, filters = []) => { +const getVideoUrls = (videos, filters = []) => { const urls = chain(videos) .filter(overEvery(filters)) + .orderBy('tbr', 'asc') .map('url') .value() - if (isEmpty(urls)) return false - const index = round(size(urls) / 2) - 1 - return get(urls, index) + return isEmpty(urls) ? false : urls } -/** - * Get a URL-like video source - */ const getVideoProvider = async ({ url }) => { const { formats } = await getVideoInfo(url) const videoUrl = - getVideoUrl(formats, [isMp4, isHttps, hasAudio]) || - getVideoUrl(formats, [isMp4, isHttp, hasAudio]) || - getVideoUrl(formats, [isMp4, isHttps]) || - getVideoUrl(formats, [isMp4]) + getVideoUrls(formats, [isMp4, isHttps, hasAudio]) || + getVideoUrls(formats, [isMp4, isHttp, hasAudio]) || + getVideoUrls(formats, [isMp4, isHttps]) || + getVideoUrls(formats, [isMp4]) return isUrl(videoUrl) && videoUrl } @@ -94,7 +84,7 @@ module.exports = () => { } } -module.exports.getVideoUrl = getVideoUrl +module.exports.getVideoUrls = getVideoUrls module.exports.isMp4 = isMp4 module.exports.isHttp = isHttp module.exports.isHttps = isHttps diff --git a/packages/metascraper-media-provider/test/fixtures/output.json b/packages/metascraper-media-provider/test/fixtures/output.json deleted file mode 100644 index 1f4f2fc24..000000000 --- a/packages/metascraper-media-provider/test/fixtures/output.json +++ /dev/null @@ -1 +0,0 @@ -{"upload_date": "20180813", "creator": null, "series": null, "chapters": null, "height": 720, "like_count": 1225, "duration": 284, "id": "2hZDd4Yfhyw", "requested_formats": [{"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "tbr": 2696.517, "protocol": "https", "format": "298 - 1280x720 (720p60)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=298&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184834230729&fvip=4&source=youtube&clen=30831030&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=3ED7C0216A6CBFD79647E8BD4E9463A955EB2E8D.4B933FF684BCA75D1E10C2771A101A646875F4BF&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "avc1.4d4020", "format_note": "720p60", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "downloader_options": {"http_chunk_size": 10485760}, "width": 1280, "ext": "mp4", "filesize": 30831030, "fps": 60, "format_id": "298", "height": 720, "quality": -1, "acodec": "none"}, {"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "tbr": 127.913, "container": "m4a_dash", "format": "140 - audio only (DASH audio)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=140&mime=audio%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184829638266&fvip=4&source=youtube&clen=4509873&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=A906D1491BFCC44A63F6F2E65D4F54E6A212D341.97A0F255237A73DA79E95D77055B067644E714E8&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.910&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "none", "format_note": "DASH audio", "abr": 128, "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "downloader_options": {"http_chunk_size": 10485760}, "ext": "m4a", "filesize": 4509873, "protocol": "https", "format_id": "140", "quality": -1, "acodec": "mp4a.40.2"}], "view_count": 94812, "playlist": null, "title": "Ice Cube gives his take on whether LeBron James can achieve greatness in LA | First Take | ESPN", "format": "298 - 1280x720 (720p60)+140 - audio only (DASH audio)", "ext": "mp4", "playlist_index": null, "dislike_count": 66, "average_rating": 4.79550743103, "abr": 128, "uploader_url": "http://www.youtube.com/user/ESPN", "subtitles": {}, "fps": 60, "stretched_ratio": null, "season_number": null, "annotations": null, "webpage_url_basename": "watch", "acodec": "mp4a.40.2", "display_id": "2hZDd4Yfhyw", "automatic_captions": {}, "description": "Ice Cube says that if LeBron James can win 4 ring in 4 years then he can be considered for the greatest in Los Angeles Lakers history, but he will be respected no matter what he does.\n\n\u2714 Subscribe to ESPN on YouTube: http://es.pn/SUBSCRIBEtoYOUTUBE\n\u2714 Subscribe to ESPN FC on YouTube: http://bit.ly/SUBSCRIBEtoESPNFC\n\u2714 Subscribe to NBA on ESPN on YouTube: http://bit.ly/SUBSCRIBEtoNBAonESPN\n\u2714 Watch ESPN on YouTube TV: http://es.pn/YouTubeTV\n\nESPN on Social Media:\n\u25ba Follow on Twitter: http://www.twitter.com/espn\n\u25ba Like on Facebook: http://www.facebook.com/espn\n\u25ba Follow on Instagram: http://www.instagram.com/espn\n\nVisit ESPN on YouTube to get up-to-the-minute sports news coverage, scores, highlights and commentary for NFL, NHL, MLB, NBA, College Football, NCAA Basketball, soccer and more.\n\nMore on ESPN.com: http://www.espn.com", "tags": ["ice cube first take lebron james", "ice cube first take espn", "ice cube", "ice cube first take", "ice cube lebron james espn", "ice cube on first take today", "ice cube on first take", "ice cube lebron james", "first take ice cube", "ice cube lebron james los angeles", "ice cube on lebron james", "ice cube lebron james lakers", "ice cube on lebron to lakers", "first take", "stephen a smith", "max kellerman", "first take espn", "espn first take", "first take stephen a smith", "espn"], "track": null, "requested_subtitles": null, "start_time": null, "uploader": "ESPN", "format_id": "298+140", "episode_number": null, "uploader_id": "ESPN", "categories": ["Sports"], "thumbnails": [{"url": "https://i.ytimg.com/vi/2hZDd4Yfhyw/hqdefault.jpg", "id": "0"}], "license": null, "alt_title": null, "extractor_key": "Youtube", "vcodec": "avc1.4d4020", "artist": null, "thumbnail": "https://i.ytimg.com/vi/2hZDd4Yfhyw/hqdefault.jpg", "vbr": null, "is_live": null, "extractor": "youtube", "end_time": null, "webpage_url": "https://www.youtube.com/watch?v=2hZDd4Yfhyw", "formats": [{"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "format_note": "DASH audio", "protocol": "https", "format": "249 - audio only (DASH audio)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=249&mime=audio%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185555220562&fvip=4&source=youtube&clen=1767940&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=491AFF87944CC9FD25B5D21C589F90377C3FCB3E.E05E599007F8D098CC04FAA293BAE8755110A4EB&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.861&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "none", "tbr": 52.905, "abr": 50, "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "downloader_options": {"http_chunk_size": 10485760}, "ext": "webm", "filesize": 1767940, "format_id": "249", "quality": -1, "acodec": "opus"}, {"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "format_note": "DASH audio", "protocol": "https", "format": "250 - audio only (DASH audio)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=250&mime=audio%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185554470802&fvip=4&source=youtube&clen=2075656&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=26CBC1F3175DCD572EB2DE5551DF47DE6F38E953.2CD5A527DABFE81155C7F04C5FDE2F78C82C79C7&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.861&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "none", "tbr": 67.232, "abr": 70, "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "downloader_options": {"http_chunk_size": 10485760}, "ext": "webm", "filesize": 2075656, "format_id": "250", "quality": -1, "acodec": "opus"}, {"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "format_note": "DASH audio", "protocol": "https", "format": "171 - audio only (DASH audio)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=171&mime=audio%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185555965278&fvip=4&source=youtube&clen=3312137&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=210B8EFBF6E1B837217E38F9A99C7377B489F24D.02B40582854DD3E3D2BE16CCB330B6B78F52F536&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.847&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "none", "tbr": 103.254, "abr": 128, "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "downloader_options": {"http_chunk_size": 10485760}, "ext": "webm", "filesize": 3312137, "format_id": "171", "quality": -1, "acodec": "vorbis"}, {"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "format_note": "DASH audio", "protocol": "https", "format": "251 - audio only (DASH audio)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=251&mime=audio%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185555925305&fvip=4&source=youtube&clen=3710023&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=BBA4D9E803869BDFA4CA85AC3975A4E74F654DFC.C8B8D633CDAB9FE61DF583E8DBB9A2FF981C855B&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.861&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "none", "tbr": 121.803, "abr": 160, "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "downloader_options": {"http_chunk_size": 10485760}, "ext": "webm", "filesize": 3710023, "format_id": "251", "quality": -1, "acodec": "opus"}, {"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "tbr": 127.913, "container": "m4a_dash", "format": "140 - audio only (DASH audio)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=140&mime=audio%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184829638266&fvip=4&source=youtube&clen=4509873&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=A906D1491BFCC44A63F6F2E65D4F54E6A212D341.97A0F255237A73DA79E95D77055B067644E714E8&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.910&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "none", "format_note": "DASH audio", "abr": 128, "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "downloader_options": {"http_chunk_size": 10485760}, "ext": "m4a", "filesize": 4509873, "protocol": "https", "format_id": "140", "quality": -1, "acodec": "mp4a.40.2"}, {"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "tbr": 96.238, "container": "webm", "format": "278 - 256x144 (144p)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=278&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220177397&fvip=4&source=youtube&clen=3047257&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=31AEE690FC015CBF5606A0D9102FFC96B9BCA24B.2FF11959EEB4DE6777B07B938CE8057B30DD9648&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "vp9", "format_note": "144p", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "downloader_options": {"http_chunk_size": 10485760}, "width": 256, "ext": "webm", "filesize": 3047257, "fps": 30, "protocol": "https", "format_id": "278", "height": 144, "quality": -1, "acodec": "none"}, {"format_note": "144p", "protocol": "https", "format": "160 - 256x144 (144p)", "tbr": 104.641, "height": 144, "downloader_options": {"http_chunk_size": 10485760}, "format_id": "160", "quality": -1, "http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=160&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184853954309&fvip=4&source=youtube&clen=1232192&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=B71A314816CBE56289EC15C8A1C6BF7EE74C169D.164175CF39A4BDFD2DA2DE6147BD09545AAA01DE&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "avc1.4d400c", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 256, "ext": "mp4", "filesize": 1232192, "fps": 30, "acodec": "none"}, {"format_note": "240p", "protocol": "https", "format": "133 - 426x240 (240p)", "tbr": 191.392, "height": 240, "downloader_options": {"http_chunk_size": 10485760}, "format_id": "133", "quality": -1, "http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=133&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184853956556&fvip=4&source=youtube&clen=2122314&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=E3307C18153DDB5E7AACAEAB9942F8B002012050.173DE890AE005232725CB8ADA7A50B7CF25692F0&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "avc1.4d4015", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 426, "ext": "mp4", "filesize": 2122314, "fps": 30, "acodec": "none"}, {"format_note": "240p", "protocol": "https", "format": "242 - 426x240 (240p)", "tbr": 222.403, "height": 240, "downloader_options": {"http_chunk_size": 10485760}, "format_id": "242", "quality": -1, "http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=242&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220160992&fvip=4&source=youtube&clen=3850378&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=ADE29866D0E224F4FA327931066956085EDB8A01.473453C889918A9BB6E6B336A627D4D7B3F33824&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "vp9", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 426, "ext": "webm", "filesize": 3850378, "fps": 30, "acodec": "none"}, {"format_note": "360p", "protocol": "https", "format": "243 - 640x360 (360p)", "tbr": 401.458, "height": 360, "downloader_options": {"http_chunk_size": 10485760}, "format_id": "243", "quality": -1, "http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=243&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220165648&fvip=4&source=youtube&clen=7618198&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=C5BE6688672F7461261A4254C66466434E6B9DD5.383565CD710DFB4CAAA0A895D3C4C58BD1685374&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "vp9", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 640, "ext": "webm", "filesize": 7618198, "fps": 30, "acodec": "none"}, {"format_note": "360p", "protocol": "https", "format": "134 - 640x360 (360p)", "tbr": 471.132, "height": 360, "downloader_options": {"http_chunk_size": 10485760}, "format_id": "134", "quality": -1, "http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=134&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854055225&fvip=4&source=youtube&clen=5015666&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=3DEFAB448DC70677960CAF56B11FB769592FC440.5CD5B13E517668562CCE3F430CF16734E3DD63CA&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "avc1.4d401e", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 640, "ext": "mp4", "filesize": 5015666, "fps": 30, "acodec": "none"}, {"format_note": "480p", "protocol": "https", "format": "244 - 854x480 (480p)", "tbr": 757.845, "height": 480, "downloader_options": {"http_chunk_size": 10485760}, "format_id": "244", "quality": -1, "http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=244&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220165309&fvip=4&source=youtube&clen=13504592&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=8BE1DAB5B67AEAEDDEC26141F4EB2323434677F9.40D8005C9A361EA5C6FA16BBE4EE5C03D692FD8E&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "vp9", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 854, "ext": "webm", "filesize": 13504592, "fps": 30, "acodec": "none"}, {"format_note": "480p", "protocol": "https", "format": "135 - 854x480 (480p)", "tbr": 900.881, "height": 480, "downloader_options": {"http_chunk_size": 10485760}, "format_id": "135", "quality": -1, "http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=135&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854155850&fvip=4&source=youtube&clen=9629112&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=B4B7A543DEAEEF890B8E6D2DCD83F8F07633DA9F.503058111885647E7E14FA29CA781D0AC0EACA6D&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "avc1.4d401f", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 854, "ext": "mp4", "filesize": 9629112, "fps": 30, "acodec": "none"}, {"format_note": "720p", "protocol": "https", "format": "247 - 1280x720 (720p)", "tbr": 1463.626, "height": 720, "downloader_options": {"http_chunk_size": 10485760}, "format_id": "247", "quality": -1, "http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=247&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220174836&fvip=4&source=youtube&clen=29547867&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=4EB17F5834B600075C7EB219E5876609A1FFC932.75C777BD0FC651CF34137FF53964B03149CA5C49&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "vp9", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 1280, "ext": "webm", "filesize": 29547867, "fps": 30, "acodec": "none"}, {"format_note": "720p", "protocol": "https", "format": "136 - 1280x720 (720p)", "tbr": 1592.796, "height": 720, "downloader_options": {"http_chunk_size": 10485760}, "format_id": "136", "quality": -1, "http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=136&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854254121&fvip=4&source=youtube&clen=18536895&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=15992BA735625E478060A1DFD9C01D00C0702BE4.C887CC9578BFCBE0ABF4FFCA47139C40B07436E3&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "avc1.4d401f", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 1280, "ext": "mp4", "filesize": 18536895, "fps": 30, "acodec": "none"}, {"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "tbr": 2586.548, "protocol": "https", "format": "302 - 1280x720 (720p60)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=302&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185013746111&fvip=4&source=youtube&clen=41986806&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=46D6B16D847DEF9A654E8E15E805A488D348A2AF.3D6F1BA63D1AE2CA7736A59F18CCCF80C9003854&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.849&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "vp9", "format_note": "720p60", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "downloader_options": {"http_chunk_size": 10485760}, "width": 1280, "ext": "webm", "filesize": 41986806, "fps": 60, "format_id": "302", "height": 720, "quality": -1, "acodec": "none"}, {"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "tbr": 2696.517, "protocol": "https", "format": "298 - 1280x720 (720p60)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=298&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184834230729&fvip=4&source=youtube&clen=30831030&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=3ED7C0216A6CBFD79647E8BD4E9463A955EB2E8D.4B933FF684BCA75D1E10C2771A101A646875F4BF&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "avc1.4d4020", "format_note": "720p60", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "downloader_options": {"http_chunk_size": 10485760}, "width": 1280, "ext": "mp4", "filesize": 30831030, "fps": 60, "format_id": "298", "height": 720, "quality": -1, "acodec": "none"}, {"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "protocol": "https", "format": "17 - 176x144 (small)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=17&mime=video%2F3gpp&pl=20&expire=1534224522&mv=m&lmt=1534184587965484&fvip=4&source=youtube&clen=2893543&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=8035AFAEEB9BE2736E041BB5B907F28F881CB218.CAA02DA4FA239160EE8B3F091884A7EFC53B93EC&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.933&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "mp4v.20.3", "format_note": "small", "ext": "3gp", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 176, "abr": 24, "filesize": 2893543, "format_id": "17", "height": 144, "resolution": "176x144", "acodec": "mp4a.40.2"}, {"protocol": "https", "format": "36 - 320x180 (small)", "format_note": "small", "height": 180, "format_id": "36", "http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=36&mime=video%2F3gpp&pl=20&expire=1534224522&mv=m&lmt=1534184588565234&fvip=4&source=youtube&clen=7869391&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=12F4A55FD6D6D2484F0FD0BF775EEA23D3E97639.D1B15C0BBCDAD61CBDD72D5695B2FE3B8C4668D8&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.933&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", "vcodec": "mp4v.20.3", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 320, "ext": "3gp", "filesize": 7869391, "resolution": "320x180", "acodec": "mp4a.40.2"}, {"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "protocol": "https", "format": "18 - 640x360 (medium)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=18&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534185137343773&fvip=4&source=youtube&clen=14141551&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&ratebypass=yes&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=250354E58D1DBCA03CBD4DC9E3FB244C5E1152A0.952F5690B6C36A18ADEA9DDA89ED78148CAFA2A4&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&dur=283.910&ei=KhRyW4nkJIvkV7yXvugN", "quality": 1, "vcodec": "avc1.42001E", "format_note": "medium", "ext": "mp4", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 640, "abr": 96, "filesize": 14141551, "format_id": "18", "height": 360, "resolution": "640x360", "acodec": "mp4a.40.2"}, {"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "protocol": "https", "format": "43 - 640x360 (medium)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=43&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185610458623&fvip=4&source=youtube&clen=22258178&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&ratebypass=yes&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=DA20EA5896E4F5E5B2F2B026BE63DF56AC4D5CF1.77668799BD2EB81CEBD00EA8F6191F3419947357&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&dur=0.000&ei=KhRyW4nkJIvkV7yXvugN", "quality": 1, "vcodec": "vp8.0", "format_note": "medium", "ext": "webm", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 640, "abr": 128, "filesize": 22258178, "format_id": "43", "height": 360, "resolution": "640x360", "acodec": "vorbis"}, {"http_headers": {"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept-Language": "en-us,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)"}, "protocol": "https", "format": "22 - 1280x720 (hd720)", "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=22&mm=31%2C29&c=WEB&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mime=video%2Fmp4&ip=95.17.206.54&pl=20&signature=26B8FCB51DC7903132039080C3888D83EC65E0F1.07B5FDE4D99FF3C3354D4D164FBC3CA838F6DEFF&expire=1534224522&initcwndbps=928750&mv=m&mt=1534202758&ms=au%2Crdu&ipbits=0&lmt=1534185082305521&requiressl=yes&sparams=dur%2Cei%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&source=youtube&ratebypass=yes&dur=283.910&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&key=yt6&ei=KhRyW4nkJIvkV7yXvugN&fvip=4", "quality": 2, "vcodec": "avc1.64001F", "format_note": "hd720", "ext": "mp4", "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", "width": 1280, "abr": 192, "format_id": "22", "height": 720, "resolution": "1280x720", "acodec": "mp4a.40.2"}], "resolution": null, "width": 1280, "age_limit": 0} diff --git a/packages/metascraper-media-provider/test/fixtures/youtube-dl.json b/packages/metascraper-media-provider/test/fixtures/youtube-dl.json new file mode 100644 index 000000000..118d0afd4 --- /dev/null +++ b/packages/metascraper-media-provider/test/fixtures/youtube-dl.json @@ -0,0 +1,706 @@ +{ + "upload_date": "20180813", + "creator": null, + "series": null, + "chapters": null, + "height": 720, + "like_count": 1225, + "duration": 284, + "id": "2hZDd4Yfhyw", + "requested_formats": [ + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "tbr": 2696.517, + "protocol": "https", + "format": "298 - 1280x720 (720p60)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=298&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184834230729&fvip=4&source=youtube&clen=30831030&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=3ED7C0216A6CBFD79647E8BD4E9463A955EB2E8D.4B933FF684BCA75D1E10C2771A101A646875F4BF&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "avc1.4d4020", + "format_note": "720p60", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "width": 1280, + "ext": "mp4", + "filesize": 30831030, + "fps": 60, + "format_id": "298", + "height": 720, + "quality": -1, + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "tbr": 127.913, + "container": "m4a_dash", + "format": "140 - audio only (DASH audio)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=140&mime=audio%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184829638266&fvip=4&source=youtube&clen=4509873&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=A906D1491BFCC44A63F6F2E65D4F54E6A212D341.97A0F255237A73DA79E95D77055B067644E714E8&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.910&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "none", + "format_note": "DASH audio", + "abr": 128, + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "ext": "m4a", + "filesize": 4509873, + "protocol": "https", + "format_id": "140", + "quality": -1, + "acodec": "mp4a.40.2" + } + ], + "view_count": 94812, + "playlist": null, + "title": "Ice Cube gives his take on whether LeBron James can achieve greatness in LA | First Take | ESPN", + "format": "298 - 1280x720 (720p60)+140 - audio only (DASH audio)", + "ext": "mp4", + "playlist_index": null, + "dislike_count": 66, + "average_rating": 4.79550743103, + "abr": 128, + "uploader_url": "http://www.youtube.com/user/ESPN", + "subtitles": {}, + "fps": 60, + "stretched_ratio": null, + "season_number": null, + "annotations": null, + "webpage_url_basename": "watch", + "acodec": "mp4a.40.2", + "display_id": "2hZDd4Yfhyw", + "automatic_captions": {}, + "description": "Ice Cube says that if LeBron James can win 4 ring in 4 years then he can be considered for the greatest in Los Angeles Lakers history, but he will be respected no matter what he does.\n\n✔ Subscribe to ESPN on YouTube: http://es.pn/SUBSCRIBEtoYOUTUBE\n✔ Subscribe to ESPN FC on YouTube: http://bit.ly/SUBSCRIBEtoESPNFC\n✔ Subscribe to NBA on ESPN on YouTube: http://bit.ly/SUBSCRIBEtoNBAonESPN\n✔ Watch ESPN on YouTube TV: http://es.pn/YouTubeTV\n\nESPN on Social Media:\n► Follow on Twitter: http://www.twitter.com/espn\n► Like on Facebook: http://www.facebook.com/espn\n► Follow on Instagram: http://www.instagram.com/espn\n\nVisit ESPN on YouTube to get up-to-the-minute sports news coverage, scores, highlights and commentary for NFL, NHL, MLB, NBA, College Football, NCAA Basketball, soccer and more.\n\nMore on ESPN.com: http://www.espn.com", + "tags": [ + "ice cube first take lebron james", + "ice cube first take espn", + "ice cube", + "ice cube first take", + "ice cube lebron james espn", + "ice cube on first take today", + "ice cube on first take", + "ice cube lebron james", + "first take ice cube", + "ice cube lebron james los angeles", + "ice cube on lebron james", + "ice cube lebron james lakers", + "ice cube on lebron to lakers", + "first take", + "stephen a smith", + "max kellerman", + "first take espn", + "espn first take", + "first take stephen a smith", + "espn" + ], + "track": null, + "requested_subtitles": null, + "start_time": null, + "uploader": "ESPN", + "format_id": "298+140", + "episode_number": null, + "uploader_id": "ESPN", + "categories": [ + "Sports" + ], + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/2hZDd4Yfhyw/hqdefault.jpg", + "id": "0" + } + ], + "license": null, + "alt_title": null, + "extractor_key": "Youtube", + "vcodec": "avc1.4d4020", + "artist": null, + "thumbnail": "https://i.ytimg.com/vi/2hZDd4Yfhyw/hqdefault.jpg", + "vbr": null, + "is_live": null, + "extractor": "youtube", + "end_time": null, + "webpage_url": "https://www.youtube.com/watch?v=2hZDd4Yfhyw", + "formats": [ + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "format_note": "DASH audio", + "protocol": "https", + "format": "249 - audio only (DASH audio)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=249&mime=audio%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185555220562&fvip=4&source=youtube&clen=1767940&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=491AFF87944CC9FD25B5D21C589F90377C3FCB3E.E05E599007F8D098CC04FAA293BAE8755110A4EB&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.861&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "none", + "tbr": 52.905, + "abr": 50, + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "ext": "webm", + "filesize": 1767940, + "format_id": "249", + "quality": -1, + "acodec": "opus" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "format_note": "DASH audio", + "protocol": "https", + "format": "250 - audio only (DASH audio)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=250&mime=audio%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185554470802&fvip=4&source=youtube&clen=2075656&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=26CBC1F3175DCD572EB2DE5551DF47DE6F38E953.2CD5A527DABFE81155C7F04C5FDE2F78C82C79C7&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.861&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "none", + "tbr": 67.232, + "abr": 70, + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "ext": "webm", + "filesize": 2075656, + "format_id": "250", + "quality": -1, + "acodec": "opus" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "format_note": "DASH audio", + "protocol": "https", + "format": "171 - audio only (DASH audio)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=171&mime=audio%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185555965278&fvip=4&source=youtube&clen=3312137&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=210B8EFBF6E1B837217E38F9A99C7377B489F24D.02B40582854DD3E3D2BE16CCB330B6B78F52F536&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.847&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "none", + "tbr": 103.254, + "abr": 128, + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "ext": "webm", + "filesize": 3312137, + "format_id": "171", + "quality": -1, + "acodec": "vorbis" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "format_note": "DASH audio", + "protocol": "https", + "format": "251 - audio only (DASH audio)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=251&mime=audio%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185555925305&fvip=4&source=youtube&clen=3710023&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=BBA4D9E803869BDFA4CA85AC3975A4E74F654DFC.C8B8D633CDAB9FE61DF583E8DBB9A2FF981C855B&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.861&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "none", + "tbr": 121.803, + "abr": 160, + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "ext": "webm", + "filesize": 3710023, + "format_id": "251", + "quality": -1, + "acodec": "opus" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "tbr": 127.913, + "container": "m4a_dash", + "format": "140 - audio only (DASH audio)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=140&mime=audio%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184829638266&fvip=4&source=youtube&clen=4509873&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=A906D1491BFCC44A63F6F2E65D4F54E6A212D341.97A0F255237A73DA79E95D77055B067644E714E8&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.910&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "none", + "format_note": "DASH audio", + "abr": 128, + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "ext": "m4a", + "filesize": 4509873, + "protocol": "https", + "format_id": "140", + "quality": -1, + "acodec": "mp4a.40.2" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "tbr": 96.238, + "container": "webm", + "format": "278 - 256x144 (144p)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=278&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220177397&fvip=4&source=youtube&clen=3047257&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=31AEE690FC015CBF5606A0D9102FFC96B9BCA24B.2FF11959EEB4DE6777B07B938CE8057B30DD9648&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "vp9", + "format_note": "144p", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "width": 256, + "ext": "webm", + "filesize": 3047257, + "fps": 30, + "protocol": "https", + "format_id": "278", + "height": 144, + "quality": -1, + "acodec": "none" + }, + { + "format_note": "144p", + "protocol": "https", + "format": "160 - 256x144 (144p)", + "tbr": 104.641, + "height": 144, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "160", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=160&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184853954309&fvip=4&source=youtube&clen=1232192&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=B71A314816CBE56289EC15C8A1C6BF7EE74C169D.164175CF39A4BDFD2DA2DE6147BD09545AAA01DE&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "avc1.4d400c", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 256, + "ext": "mp4", + "filesize": 1232192, + "fps": 30, + "acodec": "none" + }, + { + "format_note": "240p", + "protocol": "https", + "format": "133 - 426x240 (240p)", + "tbr": 191.392, + "height": 240, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "133", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=133&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184853956556&fvip=4&source=youtube&clen=2122314&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=E3307C18153DDB5E7AACAEAB9942F8B002012050.173DE890AE005232725CB8ADA7A50B7CF25692F0&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "avc1.4d4015", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 426, + "ext": "mp4", + "filesize": 2122314, + "fps": 30, + "acodec": "none" + }, + { + "format_note": "240p", + "protocol": "https", + "format": "242 - 426x240 (240p)", + "tbr": 222.403, + "height": 240, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "242", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=242&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220160992&fvip=4&source=youtube&clen=3850378&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=ADE29866D0E224F4FA327931066956085EDB8A01.473453C889918A9BB6E6B336A627D4D7B3F33824&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "vp9", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 426, + "ext": "webm", + "filesize": 3850378, + "fps": 30, + "acodec": "none" + }, + { + "format_note": "360p", + "protocol": "https", + "format": "243 - 640x360 (360p)", + "tbr": 401.458, + "height": 360, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "243", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=243&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220165648&fvip=4&source=youtube&clen=7618198&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=C5BE6688672F7461261A4254C66466434E6B9DD5.383565CD710DFB4CAAA0A895D3C4C58BD1685374&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "vp9", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 640, + "ext": "webm", + "filesize": 7618198, + "fps": 30, + "acodec": "none" + }, + { + "format_note": "360p", + "protocol": "https", + "format": "134 - 640x360 (360p)", + "tbr": 471.132, + "height": 360, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "134", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=134&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854055225&fvip=4&source=youtube&clen=5015666&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=3DEFAB448DC70677960CAF56B11FB769592FC440.5CD5B13E517668562CCE3F430CF16734E3DD63CA&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "avc1.4d401e", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 640, + "ext": "mp4", + "filesize": 5015666, + "fps": 30, + "acodec": "none" + }, + { + "format_note": "480p", + "protocol": "https", + "format": "244 - 854x480 (480p)", + "tbr": 757.845, + "height": 480, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "244", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=244&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220165309&fvip=4&source=youtube&clen=13504592&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=8BE1DAB5B67AEAEDDEC26141F4EB2323434677F9.40D8005C9A361EA5C6FA16BBE4EE5C03D692FD8E&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "vp9", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 854, + "ext": "webm", + "filesize": 13504592, + "fps": 30, + "acodec": "none" + }, + { + "format_note": "480p", + "protocol": "https", + "format": "135 - 854x480 (480p)", + "tbr": 900.881, + "height": 480, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "135", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=135&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854155850&fvip=4&source=youtube&clen=9629112&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=B4B7A543DEAEEF890B8E6D2DCD83F8F07633DA9F.503058111885647E7E14FA29CA781D0AC0EACA6D&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "avc1.4d401f", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 854, + "ext": "mp4", + "filesize": 9629112, + "fps": 30, + "acodec": "none" + }, + { + "format_note": "720p", + "protocol": "https", + "format": "247 - 1280x720 (720p)", + "tbr": 1463.626, + "height": 720, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "247", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=247&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220174836&fvip=4&source=youtube&clen=29547867&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=4EB17F5834B600075C7EB219E5876609A1FFC932.75C777BD0FC651CF34137FF53964B03149CA5C49&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "vp9", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 1280, + "ext": "webm", + "filesize": 29547867, + "fps": 30, + "acodec": "none" + }, + { + "format_note": "720p", + "protocol": "https", + "format": "136 - 1280x720 (720p)", + "tbr": 1592.796, + "height": 720, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "136", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=136&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854254121&fvip=4&source=youtube&clen=18536895&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=15992BA735625E478060A1DFD9C01D00C0702BE4.C887CC9578BFCBE0ABF4FFCA47139C40B07436E3&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "avc1.4d401f", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 1280, + "ext": "mp4", + "filesize": 18536895, + "fps": 30, + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "tbr": 2586.548, + "protocol": "https", + "format": "302 - 1280x720 (720p60)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=302&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185013746111&fvip=4&source=youtube&clen=41986806&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=46D6B16D847DEF9A654E8E15E805A488D348A2AF.3D6F1BA63D1AE2CA7736A59F18CCCF80C9003854&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.849&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "vp9", + "format_note": "720p60", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "width": 1280, + "ext": "webm", + "filesize": 41986806, + "fps": 60, + "format_id": "302", + "height": 720, + "quality": -1, + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "tbr": 2696.517, + "protocol": "https", + "format": "298 - 1280x720 (720p60)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=298&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184834230729&fvip=4&source=youtube&clen=30831030&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=3ED7C0216A6CBFD79647E8BD4E9463A955EB2E8D.4B933FF684BCA75D1E10C2771A101A646875F4BF&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "avc1.4d4020", + "format_note": "720p60", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "width": 1280, + "ext": "mp4", + "filesize": 30831030, + "fps": 60, + "format_id": "298", + "height": 720, + "quality": -1, + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "protocol": "https", + "format": "17 - 176x144 (small)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=17&mime=video%2F3gpp&pl=20&expire=1534224522&mv=m&lmt=1534184587965484&fvip=4&source=youtube&clen=2893543&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=8035AFAEEB9BE2736E041BB5B907F28F881CB218.CAA02DA4FA239160EE8B3F091884A7EFC53B93EC&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.933&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "mp4v.20.3", + "format_note": "small", + "ext": "3gp", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 176, + "abr": 24, + "filesize": 2893543, + "format_id": "17", + "height": 144, + "resolution": "176x144", + "acodec": "mp4a.40.2" + }, + { + "protocol": "https", + "format": "36 - 320x180 (small)", + "format_note": "small", + "height": 180, + "format_id": "36", + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=36&mime=video%2F3gpp&pl=20&expire=1534224522&mv=m&lmt=1534184588565234&fvip=4&source=youtube&clen=7869391&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=12F4A55FD6D6D2484F0FD0BF775EEA23D3E97639.D1B15C0BBCDAD61CBDD72D5695B2FE3B8C4668D8&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.933&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", + "vcodec": "mp4v.20.3", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 320, + "ext": "3gp", + "filesize": 7869391, + "resolution": "320x180", + "acodec": "mp4a.40.2" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "protocol": "https", + "format": "18 - 640x360 (medium)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=18&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534185137343773&fvip=4&source=youtube&clen=14141551&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&ratebypass=yes&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=250354E58D1DBCA03CBD4DC9E3FB244C5E1152A0.952F5690B6C36A18ADEA9DDA89ED78148CAFA2A4&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&dur=283.910&ei=KhRyW4nkJIvkV7yXvugN", + "quality": 1, + "vcodec": "avc1.42001E", + "format_note": "medium", + "ext": "mp4", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 640, + "abr": 96, + "filesize": 14141551, + "format_id": "18", + "height": 360, + "resolution": "640x360", + "acodec": "mp4a.40.2" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "protocol": "https", + "format": "43 - 640x360 (medium)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=43&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185610458623&fvip=4&source=youtube&clen=22258178&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&ratebypass=yes&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=DA20EA5896E4F5E5B2F2B026BE63DF56AC4D5CF1.77668799BD2EB81CEBD00EA8F6191F3419947357&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&dur=0.000&ei=KhRyW4nkJIvkV7yXvugN", + "quality": 1, + "vcodec": "vp8.0", + "format_note": "medium", + "ext": "webm", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 640, + "abr": 128, + "filesize": 22258178, + "format_id": "43", + "height": 360, + "resolution": "640x360", + "acodec": "vorbis" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" + }, + "protocol": "https", + "format": "22 - 1280x720 (hd720)", + "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=22&mm=31%2C29&c=WEB&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mime=video%2Fmp4&ip=95.17.206.54&pl=20&signature=26B8FCB51DC7903132039080C3888D83EC65E0F1.07B5FDE4D99FF3C3354D4D164FBC3CA838F6DEFF&expire=1534224522&initcwndbps=928750&mv=m&mt=1534202758&ms=au%2Crdu&ipbits=0&lmt=1534185082305521&requiressl=yes&sparams=dur%2Cei%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&source=youtube&ratebypass=yes&dur=283.910&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&key=yt6&ei=KhRyW4nkJIvkV7yXvugN&fvip=4", + "quality": 2, + "vcodec": "avc1.64001F", + "format_note": "hd720", + "ext": "mp4", + "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", + "width": 1280, + "abr": 192, + "format_id": "22", + "height": 720, + "resolution": "1280x720", + "acodec": "mp4a.40.2" + } + ], + "resolution": null, + "width": 1280, + "age_limit": 0 +} diff --git a/packages/metascraper-media-provider/test/index.js b/packages/metascraper-media-provider/test/index.js index 24107d88b..c4071cf8d 100644 --- a/packages/metascraper-media-provider/test/index.js +++ b/packages/metascraper-media-provider/test/index.js @@ -24,15 +24,15 @@ const metascraper = require('metascraper')([ const readFile = promisify(fs.readFile) -const { getVideoUrl, isMp4 } = require('metascraper-media-provider') +const { getVideoUrls, isMp4 } = require('metascraper-media-provider') -const output = require('./fixtures/output.json') +const output = require('./fixtures/youtube-dl.json') describe('metascraper-media-provider', () => { - describe('.getVideoUrl', () => { - it('isMp4', () => { - const videoUrl = getVideoUrl(output.formats, [isMp4]) - should(videoUrl).be.an.String() + describe('.getVideoUrls', () => { + it('filter by mp4', () => { + const videoUrls = getVideoUrls(output.formats, [isMp4]) + snapshot(videoUrls) }) }) describe('provider', () => { From 9f09429554f9868ae27006f04a15a1a2d5cd714f Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Wed, 29 Aug 2018 10:48:36 +0200 Subject: [PATCH 032/442] Add audio support --- .../metascraper-media-provider/src/index.js | 98 ++++++++++--------- .../metascraper-media-provider/test/index.js | 9 +- 2 files changed, 57 insertions(+), 50 deletions(-) diff --git a/packages/metascraper-media-provider/src/index.js b/packages/metascraper-media-provider/src/index.js index a57cfe772..730a80c42 100644 --- a/packages/metascraper-media-provider/src/index.js +++ b/packages/metascraper-media-provider/src/index.js @@ -8,6 +8,7 @@ const { get, chain, find, + negate, isString } = require('lodash') const { isUrl, titleize } = require('@metascraper/helpers') @@ -15,40 +16,53 @@ const path = require('path') const getVideoInfo = require('./get-video-info') -const isMp4 = video => - eq(get(video, 'ext'), 'mp4') || - path.extname(get(video, 'url')).startsWith('.mp4') -const isHttp = video => eq(get(video, 'protocol'), 'http') -const isHttps = video => eq(get(video, 'protocol'), 'https') -const hasAudio = video => has(video, 'abr') +const isMIME = ext => item => + eq(get(item, 'ext'), ext) || + path.extname(get(item, 'url')).startsWith(`.${ext}`) -const getVideoUrls = (videos, filters = []) => { +const isMp4 = isMIME('mp4') +const isMp3 = isMIME('mp3') +const isM4a = isMIME('m4a') +const isAac = isMIME('aac') + +const isHttp = item => eq(get(item, 'protocol'), 'http') +const isHttps = item => eq(get(item, 'protocol'), 'https') +const hasAudio = item => has(item, 'abr') +const hasVideo = item => has(item, 'tbr') + +const getFormatUrls = orderBy => (videos, filters = []) => { const urls = chain(videos) .filter(overEvery(filters)) - .orderBy('tbr', 'asc') + .orderBy(orderBy, 'asc') .map('url') + .filter(isUrl) .value() return isEmpty(urls) ? false : urls } -const getVideoProvider = async ({ url }) => { - const { formats } = await getVideoInfo(url) +const getVideoUrls = getFormatUrls('tbr') +const getAudioUrls = getFormatUrls('abr') - const videoUrl = - getVideoUrls(formats, [isMp4, isHttps, hasAudio]) || - getVideoUrls(formats, [isMp4, isHttp, hasAudio]) || - getVideoUrls(formats, [isMp4, isHttps]) || - getVideoUrls(formats, [isMp4]) +const getVideo = ({ formats }) => + getVideoUrls(formats, [hasAudio, isMp4, isHttps]) || + getVideoUrls(formats, [hasAudio, isMp4, isHttp]) || + getVideoUrls(formats, [isMp4, isHttps]) || + getVideoUrls(formats, [isMp4, isHttp]) || + getVideoUrls(formats, [isMp4]) - return isUrl(videoUrl) && videoUrl -} +const getAudio = ({ formats }) => + getAudioUrls(formats, [negate(hasVideo), isMp3, isHttps]) || + getAudioUrls(formats, [negate(hasVideo), isAac, isHttps]) || + getAudioUrls(formats, [negate(hasVideo), isM4a, isHttps]) || + getAudioUrls(formats, [negate(hasVideo), isMp3, isHttp]) || + getAudioUrls(formats, [negate(hasVideo), isAac, isHttp]) || + getAudioUrls(formats, [negate(hasVideo), isM4a, isHttp]) || + getAudioUrls(formats, [negate(hasVideo), isMp3]) || + getAudioUrls(formats, [negate(hasVideo), isAac]) || + getAudioUrls(formats, [negate(hasVideo), isM4a]) -/** - * Get the Author of the video. - */ -const getVideoAuthor = async ({ url }) => { - const { uploader, creator, uploader_id: uploaderId } = await getVideoInfo(url) +const getAuthor = ({ uploader, creator, uploader_id: uploaderId }) => { const author = find( [creator, uploader, uploaderId], str => isString(str) && !isUrl(str, { relative: false }) @@ -56,36 +70,32 @@ const getVideoAuthor = async ({ url }) => { return author && titleize(author, { removeBy: true }) } -const getVideoPublisher = async ({ url }) => { - const { extractor_key: extractorKey } = await getVideoInfo(url) - return isString(extractorKey) && extractorKey -} +const getPublisher = ({ extractor_key: extractorKey }) => + isString(extractorKey) && extractorKey -const getVideoTitle = async ({ url }) => { - const { title: mainTitle, alt_title: secondaryTitle } = await getVideoInfo( - url - ) +const getTitle = ({ title: mainTitle, alt_title: secondaryTitle }) => { const title = find([mainTitle, secondaryTitle], isString) return title && titleize(title) } -const getVideoDate = async ({ url }) => { - const { timestamp } = await getVideoInfo(url) - return timestamp && new Date(timestamp * 1000).toISOString() -} +const getDate = ({ timestamp }) => + timestamp && new Date(timestamp * 1000).toISOString() module.exports = () => { return { - video: getVideoProvider, - author: getVideoAuthor, - publisher: getVideoPublisher, - title: getVideoTitle, - date: getVideoDate + video: async ({ url }) => getVideo(await getVideoInfo(url)), + audio: async ({ url }) => getAudio(await getVideoInfo(url)), + author: async ({ url }) => getAuthor(await getVideoInfo(url)), + publisher: async ({ url }) => getPublisher(await getVideoInfo(url)), + title: async ({ url }) => getTitle(await getVideoInfo(url)), + date: async ({ url }) => getDate(await getVideoInfo(url)) } } -module.exports.getVideoUrls = getVideoUrls -module.exports.isMp4 = isMp4 -module.exports.isHttp = isHttp -module.exports.isHttps = isHttps -module.exports.hasAudio = hasAudio +module.exports.getFormatUrls = getFormatUrls +module.exports.getVideo = getVideo +module.exports.getAudio = getAudio +module.exports.getAuthor = getAuthor +module.exports.getPublisher = getPublisher +module.exports.getTitle = getTitle +module.exports.getDate = getDate diff --git a/packages/metascraper-media-provider/test/index.js b/packages/metascraper-media-provider/test/index.js index c4071cf8d..91e15a474 100644 --- a/packages/metascraper-media-provider/test/index.js +++ b/packages/metascraper-media-provider/test/index.js @@ -24,16 +24,13 @@ const metascraper = require('metascraper')([ const readFile = promisify(fs.readFile) -const { getVideoUrls, isMp4 } = require('metascraper-media-provider') +const { getVideo } = require('metascraper-media-provider') const output = require('./fixtures/youtube-dl.json') describe('metascraper-media-provider', () => { - describe('.getVideoUrls', () => { - it('filter by mp4', () => { - const videoUrls = getVideoUrls(output.formats, [isMp4]) - snapshot(videoUrls) - }) + it('.getVideo', () => { + snapshot(getVideo(output.formats)) }) describe('provider', () => { it('vimeo', async () => { From 137b2f4039f44d953bd7437cbb75f244de51c662 Mon Sep 17 00:00:00 2001 From: Nikola Ristic Date: Fri, 31 Aug 2018 11:13:34 +0200 Subject: [PATCH 033/442] Add `greyscale` option to Clearbit logo API (#110) --- packages/metascraper-clearbit-logo/README.md | 7 +++++++ packages/metascraper-clearbit-logo/index.js | 9 ++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/metascraper-clearbit-logo/README.md b/packages/metascraper-clearbit-logo/README.md index 7bc74541d..081002aeb 100644 --- a/packages/metascraper-clearbit-logo/README.md +++ b/packages/metascraper-clearbit-logo/README.md @@ -31,6 +31,13 @@ Default: `png` Image format, either "png" or "jpg". +##### greyscale + +Type: `boolean`
+Default: `false` + +Desaturates image if set to `true`. + ## License **metascraper-clearbit-logo** © [microlink.io](https://microlink.io), Released under the [MIT](https://github.com/microlinkhq/metascraper-clearbit-logo/blob/master/LICENSE.md) License.
diff --git a/packages/metascraper-clearbit-logo/index.js b/packages/metascraper-clearbit-logo/index.js index ce63b44ef..b4bb2367a 100644 --- a/packages/metascraper-clearbit-logo/index.js +++ b/packages/metascraper-clearbit-logo/index.js @@ -5,18 +5,21 @@ const got = require('got') const DEFAULTS = { size: '128', - format: 'png' + format: 'png', + greyscale: false } const ENDPOINT = 'https://logo.clearbit.com' module.exports = opts => { opts = Object.assign({}, DEFAULTS, opts) - const { size, format } = opts + const { size, format, greyscale } = opts const clearbitLogo = async ({ url }) => { const { hostname } = new URL(url) - const logoUrl = `${ENDPOINT}/${hostname}?size=${size}&format=${format}` + const logoUrl = `${ENDPOINT}/${hostname}?size=${size}&format=${format}${ + greyscale ? '&greyscale=true' : '' + }` try { await got.head(logoUrl) From a807b70f6969af8466923a3a2c05daf87ff8362b Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sat, 1 Sep 2018 12:53:25 +0200 Subject: [PATCH 034/442] Move helpers method into helpers package --- packages/metascraper-amazon/index.js | 43 +- packages/metascraper-amazon/test/index.js | 6 +- packages/metascraper-author/index.js | 31 +- packages/metascraper-date/index.js | 25 +- packages/metascraper-date/package.json | 3 +- packages/metascraper-description/index.js | 20 +- packages/metascraper-description/package.json | 3 +- packages/metascraper-helpers/index.js | 116 +- packages/metascraper-helpers/package.json | 7 +- packages/metascraper-helpers/test/index.js | 12 +- packages/metascraper-image/index.js | 12 +- packages/metascraper-lang/index.js | 8 +- packages/metascraper-lang/package.json | 2 +- packages/metascraper-lang/test/index.js | 2 +- .../metascraper-logo-favicon/test/index.js | 2 +- packages/metascraper-logo/index.js | 13 +- .../__snapshots__/index.js.snap-shot | 115 +- .../metascraper-media-provider/package.json | 2 - .../src/get-video-info/index.js | 60 +- .../src/get-video-info/video-info.js | 7 +- .../metascraper-media-provider/src/index.js | 129 +- .../test/fixtures/twitter.json | 115 ++ .../test/fixtures/vimeo.json | 1166 +++++++++++++++++ .../test/fixtures/youtube-dl.json | 706 ---------- .../test/fixtures/youtube.json | 654 +++++++++ .../metascraper-media-provider/test/index.js | 47 +- packages/metascraper-publisher/index.js | 9 +- packages/metascraper-publisher/package.json | 3 +- packages/metascraper-soundcloud/index.js | 12 +- packages/metascraper-title/index.js | 11 +- packages/metascraper-url/index.js | 8 +- packages/metascraper-video/index.js | 28 +- packages/metascraper-video/package.json | 3 +- packages/metascraper-video/test/index.js | 2 +- packages/metascraper-youtube/index.js | 22 +- packages/metascraper-youtube/package.json | 1 - 36 files changed, 2351 insertions(+), 1054 deletions(-) create mode 100644 packages/metascraper-media-provider/test/fixtures/twitter.json create mode 100644 packages/metascraper-media-provider/test/fixtures/vimeo.json delete mode 100644 packages/metascraper-media-provider/test/fixtures/youtube-dl.json create mode 100644 packages/metascraper-media-provider/test/fixtures/youtube.json diff --git a/packages/metascraper-amazon/index.js b/packages/metascraper-amazon/index.js index d83aaef1f..a56a047af 100644 --- a/packages/metascraper-amazon/index.js +++ b/packages/metascraper-amazon/index.js @@ -1,6 +1,6 @@ 'use strict' -const { getUrl, getValue, titleize, isUrl } = require('@metascraper/helpers') +const { url: urlFn, $filter, titleize } = require('@metascraper/helpers') const { URL } = require('url') const { chain } = require('lodash') @@ -8,29 +8,28 @@ const REGEX_AMAZON_URL = /https?:\/\/(.*amazon\..*\/.*|.*amzn\..*\/.*|.*a\.co\/. const isAmazonUrl = url => REGEX_AMAZON_URL.test(url) const SUFFIX_LANGUAGES = { - 'ca': 'en', - 'cn': 'zh', + ca: 'en', + cn: 'zh', 'co.jp': 'ja', 'co.uk': 'en', 'com.mx': 'es', - 'com': 'en', - 'de': 'de', - 'es': 'es', - 'fr': 'fr', - 'in': 'en', - 'it': 'it' + com: 'en', + de: 'de', + es: 'es', + fr: 'fr', + in: 'en', + it: 'it' } -const getSuffix = host => chain(host) - .replace('www.', '') - .split('.') - .tail() - .join('.') - .value() +const getSuffix = host => + chain(host) + .replace('www.', '') + .split('.') + .tail() + .join('.') + .value() -const getDomainLanguage = url => ( - SUFFIX_LANGUAGES[getSuffix(new URL(url).host)] -) +const getDomainLanguage = url => SUFFIX_LANGUAGES[getSuffix(new URL(url).host)] const createWrap = fn => rule => ({ htmlDom, url }) => { const value = isAmazonUrl(url) && rule(htmlDom) @@ -38,10 +37,12 @@ const createWrap = fn => rule => ({ htmlDom, url }) => { } const wrap = createWrap((url, value) => value) -const wrapUrl = createWrap((url, value) => isUrl(value) && getUrl(url, value)) +const wrapUrl = createWrap((url, value) => urlFn(value, { url })) module.exports = () => ({ - lang: [({ htmlDom: $, meta, url }) => isAmazonUrl(url) && getDomainLanguage(url)], + lang: [ + ({ htmlDom: $, meta, url }) => isAmazonUrl(url) && getDomainLanguage(url) + ], author: [ wrap($ => titleize($('.contributorNameID').text())), wrap($ => titleize($('#bylineInfo').text())), @@ -50,7 +51,7 @@ module.exports = () => ({ title: [ wrap($ => titleize($('#productTitle').text())), wrap($ => titleize($('#btAsinTitle').text())), - wrap($ => titleize(getValue($, $('h1.a-size-large')))), + wrap($ => titleize($filter($, $('h1.a-size-large')))), wrap($ => titleize($('#item_name').text())) ], publisher: [wrap($ => 'Amazon')], diff --git a/packages/metascraper-amazon/test/index.js b/packages/metascraper-amazon/test/index.js index 00fcc5168..1279a5cf8 100644 --- a/packages/metascraper-amazon/test/index.js +++ b/packages/metascraper-amazon/test/index.js @@ -10,7 +10,7 @@ const fs = require('fs') const readFile = promisify(fs.readFile) const metascraper = require('metascraper')([ - require('metascraper-amazon')(), + require('..')(), require('metascraper-author')(), require('metascraper-date')(), require('metascraper-description')(), @@ -30,8 +30,8 @@ describe('metascraper-amazon', () => { ) const url = 'https://www.amazon.co.uk/Vegetable-Perfection-tasty-recipes-shoots/dp/1849757097/ref=asap_bc?ie=UTF8' - const meta = omit(await metascraper({ html, url }), ['date']) - snapshot(meta) + const metadata = omit(await metascraper({ html, url }), ['date']) + snapshot(metadata) }) }) diff --git a/packages/metascraper-author/index.js b/packages/metascraper-author/index.js index c33a47354..2809e2cf2 100644 --- a/packages/metascraper-author/index.js +++ b/packages/metascraper-author/index.js @@ -1,16 +1,9 @@ 'use strict' -const { getValue, isUrl, titleize } = require('@metascraper/helpers') -const { isString } = require('lodash') +const { $filter, author } = require('@metascraper/helpers') const REGEX_STRICT = /^\S+\s+\S+/ -const validator = value => ( - isString(value) && - !isUrl(value, {relative: false}) && - titleize(value, {removeBy: true}) -) - /** * Wrap a rule with validation and formatting logic. * @@ -20,7 +13,7 @@ const validator = value => ( const wrap = rule => ({ htmlDom }) => { const value = rule(htmlDom) - return validator(value) + return author(value) } /** @@ -44,16 +37,14 @@ module.exports = () => ({ wrap($ => $('meta[name="author"]').attr('content')), wrap($ => $('meta[property="author"]').attr('content')), wrap($ => $('meta[property="article:author"]').attr('content')), - wrap($ => getValue($, $('[itemprop*="author"] [itemprop="name"]'))), - wrap($ => getValue($, $('[itemprop*="author"]'))), - wrap($ => getValue($, $('[rel="author"]'))), - strict(wrap($ => getValue($, $('a[class*="author"]')))), - strict(wrap($ => getValue($, $('[class*="author"] a')))), - strict(wrap($ => getValue($, $('a[href*="/author/"]')))), - wrap($ => getValue($, $('a[class*="screenname"]'))), - strict(wrap($ => getValue($, $('[class*="author"]')))), - strict(wrap($ => getValue($, $('[class*="byline"]')))) + wrap($ => $filter($, $('[itemprop*="author"] [itemprop="name"]'))), + wrap($ => $filter($, $('[itemprop*="author"]'))), + wrap($ => $filter($, $('[rel="author"]'))), + strict(wrap($ => $filter($, $('a[class*="author"]')))), + strict(wrap($ => $filter($, $('[class*="author"] a')))), + strict(wrap($ => $filter($, $('a[href*="/author/"]')))), + wrap($ => $filter($, $('a[class*="screenname"]'))), + strict(wrap($ => $filter($, $('[class*="author"]')))), + strict(wrap($ => $filter($, $('[class*="byline"]')))) ] }) - -module.exports.validator = validator diff --git a/packages/metascraper-date/index.js b/packages/metascraper-date/index.js index 3a504b112..820d19730 100644 --- a/packages/metascraper-date/index.js +++ b/packages/metascraper-date/index.js @@ -1,25 +1,6 @@ 'use strict' -const chrono = require('chrono-node') -const isIso = require('isostring') - -const validator = value => { - if (!value) return false - - // remove whitespace for easier parsing - value = value.trim() - - // convert isodates to restringify, because sometimes they are truncated - if (isIso(value)) return new Date(value).toISOString() - - // try to parse with the built-in date parser - const native = new Date(value) - if (!isNaN(native.getTime())) return native.toISOString() - - // try to parse a complex date string - const parsed = chrono.parseDate(value) - if (parsed) return parsed.toISOString() -} +const { date } = require('@metascraper/helpers') /** * Wrap a rule with validation and formatting logic. @@ -30,7 +11,7 @@ const validator = value => { const wrap = rule => ({ htmlDom }) => { const value = rule(htmlDom) - return validator(value) + return date(value) } /** @@ -68,5 +49,3 @@ module.exports = () => ({ wrap($ => $('[class*="time"]').text()) ] }) - -module.exports.validator = validator diff --git a/packages/metascraper-date/package.json b/packages/metascraper-date/package.json index 04df1e766..04aa43e91 100644 --- a/packages/metascraper-date/package.json +++ b/packages/metascraper-date/package.json @@ -16,8 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "chrono-node": "~1.3.5", - "isostring": "0.0.1" + "@metascraper/helpers": "^4.0.1" }, "devDependencies": { "standard": "latest" diff --git a/packages/metascraper-description/index.js b/packages/metascraper-description/index.js index 140cf3970..d68d8c75d 100644 --- a/packages/metascraper-description/index.js +++ b/packages/metascraper-description/index.js @@ -1,16 +1,6 @@ 'use strict' -const { getValue, titleize } = require('@metascraper/helpers') -const { isString } = require('lodash') - -const REGEX_LOCATION = /^[A-Z\s]+\s+[-—–]\s+/ - -const removeLocation = value => value.replace(REGEX_LOCATION, '') - -const validator = value => ( - isString(value) && - titleize(removeLocation(value), { capitalize: false }) -) +const { $filter, description } = require('@metascraper/helpers') /** * Wrap a rule with validation and formatting logic. @@ -21,7 +11,7 @@ const validator = value => ( const wrap = rule => ({ htmlDom }) => { const value = rule(htmlDom) - return validator(value) + return description(value) } /** @@ -35,9 +25,7 @@ module.exports = () => ({ wrap($ => $('meta[name="description"]').attr('content')), wrap($ => $('meta[itemprop="description"]').attr('content')), wrap($ => $('#description').text()), - wrap($ => getValue($, $('[class*="content"] > p'))), - wrap($ => getValue($, $('[class*="content"] p'))) + wrap($ => $filter($, $('[class*="content"] > p'))), + wrap($ => $filter($, $('[class*="content"] p'))) ] }) - -module.exports.validator = validator diff --git a/packages/metascraper-description/package.json b/packages/metascraper-description/package.json index be3e925f2..06a28c953 100644 --- a/packages/metascraper-description/package.json +++ b/packages/metascraper-description/package.json @@ -16,8 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^4.0.1", - "lodash": "~4.17.10" + "@metascraper/helpers": "^4.0.1" }, "devDependencies": { "standard": "latest" diff --git a/packages/metascraper-helpers/index.js b/packages/metascraper-helpers/index.js index 808d2a07b..39fb96f56 100644 --- a/packages/metascraper-helpers/index.js +++ b/packages/metascraper-helpers/index.js @@ -1,34 +1,48 @@ 'use strict' +const { toLower, replace, includes, isString, trim, flow, isEmpty } = require('lodash') const condenseWhitespace = require('condense-whitespace') -const { trim, flow, isEmpty } = require('lodash') +const videoExtensions = require('video-extensions') +const audioExtensions = require('audio-extensions') const isRelativeUrl = require('is-relative-url') +const fileExtension = require('file-extension') const { resolve: resolveUrl } = require('url') -const sanetizeUrl = require('normalize-url') +const _normalizeUrl = require('normalize-url') const smartquotes = require('smartquotes') -const toTitle = require('title') +const chrono = require('chrono-node') const urlRegex = require('url-regex') +const isIso = require('isostring') +const toTitle = require('title') +const { URL } = require('url') const REGEX_BY = /^[\s\n]*by|@[\s\n]*/i -const urlTest = (url, {relative = true}) => relative +const REGEX_LOCATION = /^[A-Z\s]+\s+[-—–]\s+/ + +const removeLocation = value => replace(value, REGEX_LOCATION, '') + +const urlTest = (url, { relative = true }) => relative ? isRelativeUrl(url) || urlRegex().test(url) : urlRegex().test(url) const isUrl = (url, opts = {}) => !isEmpty(url) && urlTest(url, opts) -const normalizeUrl = (url, opts) => sanetizeUrl(url, { - normalizeHttp: false, - stripWWW: false, - sortQueryParameters: false, - removeTrailingSlash: false, - ...opts -}) +const absoluteUrl = (baseUrl, relativePath = '') => ( + resolveUrl(baseUrl, relativePath) +) -const getAbsoluteUrl = (baseUrl, relativePath = '') => resolveUrl(baseUrl, relativePath) +const sanetizeUrl = (url, opts) => ( + _normalizeUrl(url, { + normalizeHttp: false, + stripWWW: false, + sortQueryParameters: false, + removeTrailingSlash: false, + ...opts + }) +) -const getUrl = (baseUrl, relativePath, opts) => ( - normalizeUrl(getAbsoluteUrl(baseUrl, relativePath), opts) +const normalizeUrl = (baseUrl, relativePath, opts) => ( + sanetizeUrl(absoluteUrl(baseUrl, relativePath), opts) ) const removeByPrefix = flow([ @@ -47,16 +61,80 @@ const titleize = (src, { capitalize = false, removeBy = false } = {}) => { const defaultFn = el => el.text().trim() -const getValue = ($, collection, fn = defaultFn) => { +const $filter = ($, collection, fn = defaultFn) => { const el = collection.filter((i, el) => fn($(el))).first() return fn(el) } +const isAuthor = (str, opts = { relative: false }) => ( + isString(str) && !isUrl(str, opts) +) + +const getAuthor = (str, opts = { removeBy: true }) => titleize(str, opts) + +const protocol = url => { + const { protocol = '' } = new URL(url) + return protocol.replace(':', '') +} + +const createUrlExtensionValidator = collection => url => ( + isUrl(url) && includes(collection, extension(url)) +) + +const isVideoUrl = createUrlExtensionValidator(videoExtensions) + +const isAudioUrl = createUrlExtensionValidator(audioExtensions) + +const extension = url => fileExtension(url).split('?')[0] + +const description = value => isString(value) && getDescription(value) + +const getDescription = value => titleize(removeLocation(value), { capitalize: false }) + +const publisher = value => isString(value) && condenseWhitespace(value) + +const author = value => isAuthor(value) && getAuthor(value) + +const url = (value, { url }) => isUrl(value) && normalizeUrl(url, value) + +const date = value => { + if (!value) return false + + // remove whitespace for easier parsing + value = value.trim() + + // convert isodates to restringify, because sometimes they are truncated + if (isIso(value)) return new Date(value).toISOString() + + // try to parse with the built-in date parser + const native = new Date(value) + if (!isNaN(native.getTime())) return native.toISOString() + + // try to parse a complex date string + const parsed = chrono.parseDate(value) + if (parsed) return parsed.toISOString() +} + +const lang = value => isString(value) && toLower(value.substring(0, 2)) + +const title = value => isString(value) && titleize(value) + module.exports = { + author, + title, + lang, + url, + description, + date, + $filter, titleize, - getUrl, - isUrl, + absoluteUrl, + sanetizeUrl, + extension, + protocol, + publisher, normalizeUrl, - getAbsoluteUrl, - getValue + isUrl, + isVideoUrl, + isAudioUrl } diff --git a/packages/metascraper-helpers/package.json b/packages/metascraper-helpers/package.json index b858a489f..44524754f 100644 --- a/packages/metascraper-helpers/package.json +++ b/packages/metascraper-helpers/package.json @@ -16,13 +16,18 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { + "audio-extensions": "0.0.0", + "chrono-node": "~1.3.5", "condense-whitespace": "~1.0.0", + "file-extension": "~4.0.5", "is-relative-url": "~2.0.0", + "isostring": "0.0.1", "lodash": "~4.17.10", "normalize-url": "~3.2.0", "smartquotes": "~2.3.1", "title": "~3.3.2", - "url-regex": "~4.1.1" + "url-regex": "~4.1.1", + "video-extensions": "~1.1.0" }, "devDependencies": { "mocha": "latest", diff --git a/packages/metascraper-helpers/test/index.js b/packages/metascraper-helpers/test/index.js index 4f930d154..1ed005d95 100644 --- a/packages/metascraper-helpers/test/index.js +++ b/packages/metascraper-helpers/test/index.js @@ -2,20 +2,20 @@ const should = require('should') -const { getAbsoluteUrl } = require('../') +const { absoluteUrl } = require('../') describe('metascraper-helpers', () => { - it('getAbsoluteUrl', () => { - should(getAbsoluteUrl('https://kikobeats.com/', 'blog')).be.equal( + it('absoluteUrl', () => { + should(absoluteUrl('https://kikobeats.com/', 'blog')).be.equal( 'https://kikobeats.com/blog' ) - should(getAbsoluteUrl('https://kikobeats.com', '/blog')).be.equal( + should(absoluteUrl('https://kikobeats.com', '/blog')).be.equal( 'https://kikobeats.com/blog' ) - should(getAbsoluteUrl('https://kikobeats.com/', '/blog')).be.equal( + should(absoluteUrl('https://kikobeats.com/', '/blog')).be.equal( 'https://kikobeats.com/blog' ) - should(getAbsoluteUrl('http://kikobeats.com/', '/blog')).be.equal( + should(absoluteUrl('http://kikobeats.com/', '/blog')).be.equal( 'http://kikobeats.com/blog' ) }) diff --git a/packages/metascraper-image/index.js b/packages/metascraper-image/index.js index 880e296f7..932e5b94e 100644 --- a/packages/metascraper-image/index.js +++ b/packages/metascraper-image/index.js @@ -1,8 +1,6 @@ 'use strict' -const { getValue, getUrl, isUrl } = require('@metascraper/helpers') - -const validator = (value, url) => isUrl(value) && getUrl(url, value) +const { $filter, url: urlFn } = require('@metascraper/helpers') /** * Wrap a rule with validation and formatting logic. @@ -13,7 +11,7 @@ const validator = (value, url) => isUrl(value) && getUrl(url, value) const wrap = rule => ({ htmlDom, url }) => { const value = rule(htmlDom) - return validator(value, url) + return urlFn(value, { url }) } const getSrc = el => el.attr('src') @@ -29,11 +27,9 @@ module.exports = () => ({ wrap($ => $('meta[name="twitter:image:src"]').attr('content')), wrap($ => $('meta[name="twitter:image"]').attr('content')), wrap($ => $('meta[itemprop="image"]').attr('content')), - wrap($ => getValue($, $('article img[src]'), getSrc)), - wrap($ => getValue($, $('#content img[src]'), getSrc)), + wrap($ => $filter($, $('article img[src]'), getSrc)), + wrap($ => $filter($, $('#content img[src]'), getSrc)), wrap($ => $('img[alt*="author"]').attr('src')), wrap($ => $('img[src]').attr('src')) ] }) - -module.exports.validator = validator diff --git a/packages/metascraper-lang/index.js b/packages/metascraper-lang/index.js index 41e159dee..50b146606 100644 --- a/packages/metascraper-lang/index.js +++ b/packages/metascraper-lang/index.js @@ -1,12 +1,10 @@ 'use strict' -const { isString, toLower } = require('lodash') - -const validator = value => isString(value) && toLower(value.substring(0, 2)) +const { lang } = require('@metascraper/helpers') const wrap = rule => ({ htmlDom }) => { const value = rule(htmlDom) - return validator(value) + return lang(value) } module.exports = () => ({ @@ -15,5 +13,3 @@ module.exports = () => ({ wrap($ => $('html').attr('lang')) ] }) - -module.exports.validator = validator diff --git a/packages/metascraper-lang/package.json b/packages/metascraper-lang/package.json index d14db8789..fe98c6902 100644 --- a/packages/metascraper-lang/package.json +++ b/packages/metascraper-lang/package.json @@ -16,7 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "lodash": "~4.17.10" + "@metascraper/helpers": "^4.0.1" }, "devDependencies": { "mocha": "latest", diff --git a/packages/metascraper-lang/test/index.js b/packages/metascraper-lang/test/index.js index d377d8227..a9d31eecb 100644 --- a/packages/metascraper-lang/test/index.js +++ b/packages/metascraper-lang/test/index.js @@ -11,7 +11,7 @@ const metascraper = require('metascraper')([ require('metascraper-date')(), require('metascraper-description')(), require('metascraper-image')(), - require('metascraper-lang')(), + require('..')(), require('metascraper-logo')(), require('metascraper-publisher')(), require('metascraper-title')(), diff --git a/packages/metascraper-logo-favicon/test/index.js b/packages/metascraper-logo-favicon/test/index.js index 56908f0a9..e23e4b2cb 100644 --- a/packages/metascraper-logo-favicon/test/index.js +++ b/packages/metascraper-logo-favicon/test/index.js @@ -13,7 +13,7 @@ const metascraper = require('metascraper')([ require('metascraper-description')(), require('metascraper-image')(), require('metascraper-logo')(), - require('metascraper-logo-favicon')(), + require('..')(), require('metascraper-publisher')(), require('metascraper-title')(), require('metascraper-url')() diff --git a/packages/metascraper-logo/index.js b/packages/metascraper-logo/index.js index 854c9eaa2..74b9462ed 100644 --- a/packages/metascraper-logo/index.js +++ b/packages/metascraper-logo/index.js @@ -1,7 +1,7 @@ 'use strict' const { flow, chain, first, concat, toNumber, split } = require('lodash') -const { getUrl, isUrl } = require('@metascraper/helpers') +const { url: urlFn } = require('@metascraper/helpers') const getSize = flow([str => split(str, 'x'), first, toNumber]) @@ -31,8 +31,6 @@ const sizeSelectors = [ { tag: 'link[rel="shortcut icon"]', attr: 'href' } ] -const validator = (value, url) => isUrl(value) && getUrl(url, value) - /** * Wrap a rule with validation and formatting logic. * @@ -42,7 +40,7 @@ const validator = (value, url) => isUrl(value) && getUrl(url, value) const wrap = rule => ({ htmlDom, url }) => { const value = rule(htmlDom) - return validator(value, url) + return urlFn(value, { url }) } /** @@ -56,10 +54,11 @@ module.exports = () => ({ wrap($ => $('img[itemprop="logo"]').attr('src')), wrap($ => { const sizes = getSizes($, sizeSelectors) - const size = chain(sizes).first().get('link').value() + const size = chain(sizes) + .first() + .get('link') + .value() return size }) ] }) - -module.exports.validator = validator diff --git a/packages/metascraper-media-provider/__snapshots__/index.js.snap-shot b/packages/metascraper-media-provider/__snapshots__/index.js.snap-shot index d4e68481d..a2a652fb3 100644 --- a/packages/metascraper-media-provider/__snapshots__/index.js.snap-shot +++ b/packages/metascraper-media-provider/__snapshots__/index.js.snap-shot @@ -1,55 +1,108 @@ -exports['twitter 1'] = { +exports['twitter 1'] = [ + "https://video.twimg.com/amplify_video/943561675927519232/vid/240x240/mijiQdCq-p9FaO8H.mp4", + "https://video.twimg.com/amplify_video/943561675927519232/vid/480x480/qURzB_XtWBE-dvRa.mp4", + "https://video.twimg.com/amplify_video/943561675927519232/vid/720x720/h1uN7biCI-Fbzm9D.mp4" +] + +exports['vimeo 1'] = [ + "https://gcs-vimeo.akamaized.net/exp=1535800514~acl=%2A%2F823603785.mp4%2A~hmac=c1d7138053fb25092b67c26ddcbd2fc8087da208a30f47a74025e62f844ed1ba/vimeo-prod-skyfire-std-us/01/2635/7/188175573/823603785.mp4", + "https://gcs-vimeo.akamaized.net/exp=1535800514~acl=%2A%2F823603780.mp4%2A~hmac=f4c67d41844557065d91bed1f1e24ac1061252f8a37ddad76af75b84f8798429/vimeo-prod-skyfire-std-us/01/2635/7/188175573/823603780.mp4", + "https://gcs-vimeo.akamaized.net/exp=1535800514~acl=%2A%2F823603783.mp4%2A~hmac=69a7fc2bb5ccbb888ad958f68b8f6e7b480cd635249a8f6e06fa7bd5dc600a3f/vimeo-prod-skyfire-std-us/01/2635/7/188175573/823603783.mp4", + "https://gcs-vimeo.akamaized.net/exp=1535800514~acl=%2A%2F823603782.mp4%2A~hmac=1ff98e8349d222bbb11b2e7e3105729a578f83c632a34e252db61034b2b05d6b/vimeo-prod-skyfire-std-us/01/2635/7/188175573/823603782.mp4" +] + +exports['youtube 1'] = [ + "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fmp4&gir=yes&ipbits=0&lmt=1507952889801167&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=160&mt=1535797130&dur=141.440&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=503873&signature=5CCB9E814DC47000AE95012B9BAC5E638CF7B335.AEBC98B5759C5A7B2BFCD057539543D7FF3F0682&ratebypass=yes", + "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fmp4&gir=yes&ipbits=0&lmt=1507952889799661&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=133&mt=1535797130&dur=141.440&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=916551&signature=CB679D12966286AEDBC316AF52CB1502D31252C6.28D58615691F9E540559B860B5719E7A55C90BC0&ratebypass=yes", + "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fmp4&gir=yes&ipbits=0&lmt=1507952889802693&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=134&mt=1535797130&dur=141.440&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=2207390&signature=90CC91716540A24CFC117B94831E34CBF2E11519.1253E700A2076681949A8077EF0A0CC13EE0A70A&ratebypass=yes", + "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fmp4&gir=yes&ipbits=0&lmt=1507952889999957&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=135&mt=1535797130&dur=141.440&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=4380005&signature=20C47063DBB07F696EA3ADDB14B9127CB5C5CB3A.B722ED0B37D06C014D5747F1481559B0D0B36D66&ratebypass=yes", + "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fmp4&gir=yes&ipbits=0&lmt=1507952890403137&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=136&mt=1535797130&dur=141.440&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=8054202&signature=2B28B5EC1C1EE32C0BE79537EE448B1BF0441A57.09D9F53E9F626EB8DB06508DCA3AA28B4C89D183&ratebypass=yes", + "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fmp4&gir=yes&ipbits=0&lmt=1507952977382584&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=137&mt=1535797130&dur=141.440&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=14903843&signature=16960EBA740BA2C11F88FC697A42F7D0E52EA2EC.4AC8D17A3BED6B6E68AB92CFE3F598341A9D839A&ratebypass=yes" +] + +exports['vimeo 2'] = { + "publisher": "Vimeo", + "video": [ + "https://gcs-vimeo.akamaized.net/exp=1535802783~acl=%2A%2F823603785.mp4%2A~hmac=455bf895fafc1d034c3e793e91f96d8c87db9ce5ca7e54f9698a508f31b51c47/vimeo-prod-skyfire-std-us/01/2635/7/188175573/823603785.mp4", + "https://gcs-vimeo.akamaized.net/exp=1535802783~acl=%2A%2F823603780.mp4%2A~hmac=74b624180b954cf19ac3399164c6f9102b234663e6f6a41f1c8715c04348d0be/vimeo-prod-skyfire-std-us/01/2635/7/188175573/823603780.mp4", + "https://gcs-vimeo.akamaized.net/exp=1535802783~acl=%2A%2F823603783.mp4%2A~hmac=98f11f868f2e7ac772f72e38f71b29c4b32340f3b9f2833899538f040959c7aa/vimeo-prod-skyfire-std-us/01/2635/7/188175573/823603783.mp4", + "https://gcs-vimeo.akamaized.net/exp=1535802783~acl=%2A%2F823603782.mp4%2A~hmac=59349fbcb43c5c6f3e90f9ffbdec8ebc5013ebbb4e883e4702d66fd77dac11ea/vimeo-prod-skyfire-std-us/01/2635/7/188175573/823603782.mp4" + ], + "audio": null, + "author": "pleid", + "title": "Converse - Past meets Present - Montage", + "date": "2016-10-20T13:06:52.000Z", + "image": "https://i.vimeocdn.com/video/598160082_1280.jpg", + "description": "Converse has spent a good part of this year updating some of their classics. Our past is constantly catching up to us, but we rarely get to see the relationship between past and present. The sneaker company gave us access to some of the original footwear to create a series of dynamic, thrilling and unexpected motion pieces wherein we watch the old versions turn into the updated models.\nCREDITS\nClient: Converse USA\nAgency: Anomaly NYC\nArt Direction: Serial Cut\nAnimation: Pleid\nSound Design: Heardcity", + "lang": "en", + "logo": "https://i.vimeocdn.com/favicon/main-touch_180", + "url": "https://vimeo.com/188175573" +} + +exports['twitter 2'] = { + "publisher": "Twitter", + "video": [ + "https://video.twimg.com/amplify_video/943561675927519232/vid/240x240/mijiQdCq-p9FaO8H.mp4", + "https://video.twimg.com/amplify_video/943561675927519232/vid/480x480/qURzB_XtWBE-dvRa.mp4", + "https://video.twimg.com/amplify_video/943561675927519232/vid/720x720/h1uN7biCI-Fbzm9D.mp4" + ], + "audio": null, "author": "The Verge", - "publisher": "TwitterCard", + "title": "The Verge - Is it bad to blow into game cartridges?", "date": "2018-02-11T12:00:00.000Z", - "description": "“Is it bad to blow into game cartridges? https://t.co/Y3yAimrUnP”", "image": "https://pbs.twimg.com/media/DRg1OMRVwAEuwTK.jpg", + "description": "The Verge on Twitter: “Is it bad to blow into game cartridges? https://t.co/Y3yAimrUnP”", "lang": "es", "logo": "https://abs.twimg.com/icons/apple-touch-icon-192x192.png", "url": "https://twitter.com/verge/status/957383241714970624" } exports['facebook 1'] = { - "author": "AFC Ajax", "publisher": "Facebook", + "video": [ + "https://video-cdg2-1.xx.fbcdn.net/v/t42.1790-2/29931969_132780680897191_2050692595729825792_n.mp4?_nc_cat=0&efg=eyJ2ZW5jb2RlX3RhZyI6ImRhc2hfdjNfNDI2X2NyZl8yM19tYWluXzMuMF9mcmFnXzJfYXVkaW8ifQ==&oh=aac6c17157da877a8162db4b6f8a6eb0&oe=5B8AA468", + "https://video-cdg2-1.xx.fbcdn.net/v/t42.1790-2/29977283_989743217849704_1908790191984738304_n.mp4?_nc_cat=0&efg=eyJ2ZW5jb2RlX3RhZyI6ImRhc2hfdjNfMjU2X2NyZl8yM19tYWluXzMuMF9mcmFnXzJfdmlkZW8ifQ==&oh=bb4b0f8bbf893c8be55113af6de832c3&oe=5B8AA307", + "https://video-cdg2-1.xx.fbcdn.net/v/t42.1790-2/29980789_2051987191737646_5699976528746512384_n.mp4?_nc_cat=0&efg=eyJ2ZW5jb2RlX3RhZyI6ImRhc2hfdjNfMzgwX2NyZl8yOF9iYXNlbGluZV8zLjBfYTIwdm5jZDJfb3RwMl9mcmFnXzJfdmlkZW8ifQ==&oh=2b86aa92071f92bd9770c86f9d651bf0&oe=5B8AA4E7", + "https://video-cdg2-1.xx.fbcdn.net/v/t42.1790-2/30015739_434285990360734_6829180725728641024_n.mp4?_nc_cat=0&efg=eyJ2ZW5jb2RlX3RhZyI6ImRhc2hfdjRfaHEzX2ZyYWdfMl92aWRlbyJ9&oh=caa0d53f322ab7bad1952553b0d9b4d6&oe=5B8AA683", + "https://video-cdg2-1.xx.fbcdn.net/v/t42.1790-2/29924134_184843515654220_5433876658782208_n.mp4?_nc_cat=0&efg=eyJ2ZW5jb2RlX3RhZyI6ImRhc2hfdjNfNjQwX2NyZl8yM19tYWluXzMuMF9mcmFnXzJfdmlkZW8ifQ==&oh=d630f36ab892b4032f76a829142fa11c&oe=5B8AA526", + "https://video-cdg2-1.xx.fbcdn.net/v/t42.1790-2/10000000_205740473532514_2439904288734969856_n.mp4?_nc_cat=0&efg=eyJ2ZW5jb2RlX3RhZyI6ImRhc2hfdjNfODU0X2NyZl8yM19tYWluXzMuMF9mcmFnXzJfdmlkZW8ifQ==&oh=2531d28c98cd63fd46b3de39a8b16ff9&oe=5B8A9E9C", + "https://video-cdg2-1.xx.fbcdn.net/v/t42.1790-2/10000000_566307827066937_5067446288358834176_n.mp4?_nc_cat=0&efg=eyJ2ZW5jb2RlX3RhZyI6ImRhc2hfdjRfaHEyX2ZyYWdfMl92aWRlbyJ9&oh=5ac0d1ef4b37a002d16ac02c55f432fe&oe=5B8AA4FA", + "https://video-cdg2-1.xx.fbcdn.net/v/t42.1790-2/10000000_608377619513235_2375849764957716480_n.mp4?_nc_cat=0&efg=eyJ2ZW5jb2RlX3RhZyI6ImRhc2hfdjRfaHE1X2ZyYWdfMl92aWRlbyJ9&oh=bb1b0b07e5ef3e266d73e2f16c3c9521&oe=5B8AA2AE" + ], + "audio": null, + "author": "AFC Ajax", + "title": "̶A̶n̶d̶e̶r̶l̶e̶c̶h̶t̶ ✔️ ̶J̶u̶v̶e̶n̶t̶u̶s̶ ✔️ W I N N E R S 👉 #Aj...", "date": "2018-04-02T18:57:00.000Z", + "image": "https://scontent-cdg2-1.xx.fbcdn.net/v/t15.0-10/p720x720/27603335_1686838558030152_6228738417937612800_n.jpg?_nc_cat=0&oh=48f6c7c84736eac9eeb2b9c086e5ec50&oe=5C2D7661", "description": null, - "image": null, "lang": "es", "logo": "https://static.xx.fbcdn.net/rsrc.php/yp/r/1Dxu6XIjaTc.ico", "url": "https://www.facebook.com/afcajax/videos/1686831701364171" } -exports['youtube 1'] = { +exports['youtube 2'] = { + "publisher": "ES", + "video": [ + "https://r3---sn-aigzrn7d.googlevideo.com/videoplayback?c=WEB&mm=31%2C26&mn=sn-aigzrn7d%2Csn-4g5ednss&ms=au%2Conr&mt=1535798742&mv=m&ei=dW6KW9qtF8ec1wb-hJCABw&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ip=95.131.170.236&clen=503873&id=o-AD6ZD2aepAs4ud0h7euE2Y_dE0uZ1_Y0GVZQIMMFwXwu&keepalive=yes&gir=yes&requiressl=yes&source=youtube&initcwndbps=240000&pl=24&dur=141.440&itag=160&lmt=1507952889801167&ipbits=0&mime=video%2Fmp4&key=yt6&expire=1535820501&fvip=3&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&signature=DCC5BC32D5619CF7167A9D0AD5E5CC7E630D82CF.B85EBA0064EE53C520BB88100C58DDC992BFE405&ratebypass=yes", + "https://r3---sn-aigzrn7d.googlevideo.com/videoplayback?c=WEB&mm=31%2C26&mn=sn-aigzrn7d%2Csn-4g5ednss&ms=au%2Conr&mt=1535798742&mv=m&ei=dW6KW9qtF8ec1wb-hJCABw&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ip=95.131.170.236&clen=916551&id=o-AD6ZD2aepAs4ud0h7euE2Y_dE0uZ1_Y0GVZQIMMFwXwu&keepalive=yes&gir=yes&requiressl=yes&source=youtube&initcwndbps=240000&pl=24&dur=141.440&itag=133&lmt=1507952889799661&ipbits=0&mime=video%2Fmp4&key=yt6&expire=1535820501&fvip=3&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&signature=1D53A389B5F340AB723944F8D3F2BAF727AC5D95.20BBE3CAB8BA637A533E6F067DA85F0C1B637BE4&ratebypass=yes", + "https://r3---sn-aigzrn7d.googlevideo.com/videoplayback?c=WEB&mm=31%2C26&mn=sn-aigzrn7d%2Csn-4g5ednss&ms=au%2Conr&mt=1535798742&mv=m&ei=dW6KW9qtF8ec1wb-hJCABw&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ip=95.131.170.236&clen=2207390&id=o-AD6ZD2aepAs4ud0h7euE2Y_dE0uZ1_Y0GVZQIMMFwXwu&keepalive=yes&gir=yes&requiressl=yes&source=youtube&initcwndbps=240000&pl=24&dur=141.440&itag=134&lmt=1507952889802693&ipbits=0&mime=video%2Fmp4&key=yt6&expire=1535820501&fvip=3&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&signature=5555B2065D925F10563FC81E4C5B3125A4F1FF3E.A4EAB84294474AE066F5815F446BB3E5A3849607&ratebypass=yes", + "https://r3---sn-aigzrn7d.googlevideo.com/videoplayback?c=WEB&mm=31%2C26&mn=sn-aigzrn7d%2Csn-4g5ednss&ms=au%2Conr&mt=1535798742&mv=m&ei=dW6KW9qtF8ec1wb-hJCABw&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ip=95.131.170.236&clen=4380005&id=o-AD6ZD2aepAs4ud0h7euE2Y_dE0uZ1_Y0GVZQIMMFwXwu&keepalive=yes&gir=yes&requiressl=yes&source=youtube&initcwndbps=240000&pl=24&dur=141.440&itag=135&lmt=1507952889999957&ipbits=0&mime=video%2Fmp4&key=yt6&expire=1535820501&fvip=3&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&signature=DE41C7D2B2E71B9D57CFC036F7940C19D9B4448B.ADECA95DD10A829BE51B126CDF0D70CE40B3B360&ratebypass=yes", + "https://r3---sn-aigzrn7d.googlevideo.com/videoplayback?c=WEB&mm=31%2C26&mn=sn-aigzrn7d%2Csn-4g5ednss&ms=au%2Conr&mt=1535798742&mv=m&ei=dW6KW9qtF8ec1wb-hJCABw&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ip=95.131.170.236&clen=8054202&id=o-AD6ZD2aepAs4ud0h7euE2Y_dE0uZ1_Y0GVZQIMMFwXwu&keepalive=yes&gir=yes&requiressl=yes&source=youtube&initcwndbps=240000&pl=24&dur=141.440&itag=136&lmt=1507952890403137&ipbits=0&mime=video%2Fmp4&key=yt6&expire=1535820501&fvip=3&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&signature=20213749D025C5000B9B4F05AD1407873E874A3D.4E1F67BF49DA23F24E57216BF9A35546A50A6D24&ratebypass=yes", + "https://r3---sn-aigzrn7d.googlevideo.com/videoplayback?c=WEB&mm=31%2C26&mn=sn-aigzrn7d%2Csn-4g5ednss&ms=au%2Conr&mt=1535798742&mv=m&ei=dW6KW9qtF8ec1wb-hJCABw&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ip=95.131.170.236&clen=14903843&id=o-AD6ZD2aepAs4ud0h7euE2Y_dE0uZ1_Y0GVZQIMMFwXwu&keepalive=yes&gir=yes&requiressl=yes&source=youtube&initcwndbps=240000&pl=24&dur=141.440&itag=137&lmt=1507952977382584&ipbits=0&mime=video%2Fmp4&key=yt6&expire=1535820501&fvip=3&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&signature=05E8187F34D11C1BDC529E1E4D198B08CA274C1B.56A61CAE660F0B1D7EE3493E7FFD4BC1019A117B&ratebypass=yes" + ], + "audio": [ + "https://r3---sn-aigzrn7d.googlevideo.com/videoplayback?c=WEB&mm=31%2C26&mn=sn-aigzrn7d%2Csn-4g5ednss&ms=au%2Conr&mt=1535798742&mv=m&ei=dW6KW9qtF8ec1wb-hJCABw&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ip=95.131.170.236&clen=1187373&id=o-AD6ZD2aepAs4ud0h7euE2Y_dE0uZ1_Y0GVZQIMMFwXwu&gir=yes&requiressl=yes&source=youtube&initcwndbps=240000&pl=24&dur=141.548&itag=17&lmt=1503811657072536&ipbits=0&mime=video%2F3gpp&key=yt6&expire=1535820501&fvip=3&signature=4AF939F4FCCDDDBC3479E11BBE050FB8CE2602E7.36D1EF162722A97D7E0D1E2209E3A784B419D2A4&ratebypass=yes", + "https://r3---sn-aigzrn7d.googlevideo.com/videoplayback?c=WEB&mm=31%2C26&mn=sn-aigzrn7d%2Csn-4g5ednss&ms=au%2Conr&mt=1535798742&mv=m&ei=dW6KW9qtF8ec1wb-hJCABw&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&ip=95.131.170.236&clen=6322851&id=o-AD6ZD2aepAs4ud0h7euE2Y_dE0uZ1_Y0GVZQIMMFwXwu&gir=yes&requiressl=yes&ratebypass=yes&source=youtube&initcwndbps=240000&pl=24&dur=141.502&itag=18&lmt=1503811655561063&ipbits=0&mime=video%2Fmp4&key=yt6&expire=1535820501&fvip=3&signature=1FF6968AA3B913F6908D66AF0C9CD766CE049CF5.7A97845B24B922691386EA748CD92B7C84A62257", + "https://r3---sn-aigzrn7d.googlevideo.com/videoplayback?c=WEB&mm=31%2C26&mn=sn-aigzrn7d%2Csn-4g5ednss&ms=au%2Conr&mt=1535798742&mv=m&ei=dW6KW9qtF8ec1wb-hJCABw&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&ip=95.131.170.236&clen=7126000&id=o-AD6ZD2aepAs4ud0h7euE2Y_dE0uZ1_Y0GVZQIMMFwXwu&gir=yes&requiressl=yes&ratebypass=yes&source=youtube&initcwndbps=240000&pl=24&dur=0.000&itag=43&lmt=1503812251410108&ipbits=0&mime=video%2Fwebm&key=yt6&expire=1535820501&fvip=3&signature=C574199D94DEB600ED2B0B96D11D1CBECEDA365E.D88E911D701BD67F6742B136A4D7020B369528AA", + "https://r3---sn-aigzrn7d.googlevideo.com/videoplayback?c=WEB&mm=31%2C26&ipbits=0&ms=au%2Conr&ratebypass=yes&source=youtube&mv=m&initcwndbps=240000&pl=24&mn=sn-aigzrn7d%2Csn-4g5ednss&dur=141.502&ei=dW6KW9qtF8ec1wb-hJCABw&itag=22&sparams=dur%2Cei%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&lmt=1507953047549769&ip=95.131.170.236&mime=video%2Fmp4&key=yt6&expire=1535820501&id=o-AD6ZD2aepAs4ud0h7euE2Y_dE0uZ1_Y0GVZQIMMFwXwu&requiressl=yes&fvip=3&mt=1535798742&signature=1000EAC9C47F59E0E655EE9145732C466F47256B.81478A0BDF1948A0A05679894DF33C8729D8364E" + ], "author": "ONE Media", - "publisher": "Youtube", + "title": "Star Wars 8 THE LAST JEDI Official TRAILER (2017) Daisy Ridley, Disney Movie HD", "date": "2017-04-14T12:00:00.000Z", - "description": "Star Wars 8 El Ultimo JEDI Trailer Espanol (Subtitulado) - 2017\n© 2017 - Disney", - "image": "https://i.ytimg.com/vi/hwMkbaS_M_c/mqdefault.jpg", + "image": "https://i.ytimg.com/vi/hwMkbaS_M_c/maxresdefault.jpg", + "description": "Star Wars 8 THE LAST JEDI Official TRAILER (2017) Daisy Ridley, Disney Movie HD\n© 2017 - Disney", "lang": null, "logo": "https://www.youtube.com/yts/img/favicon_144-vfliLAfaB.png", "url": "https://www.youtube.com/watch?v=hwMkbaS_M_c" } -exports['vimeo 1'] = { - "author": "pleid", - "publisher": "Vimeo", - "date": "2016-10-20T13:06:52.000Z", - "description": "Converse has spent a good part of this year updating some of their classics. Our past is constantly catching up to us, but we rarely get to see the relationship…", - "image": "https://i.vimeocdn.com/filter/overlay?src0=https%3A%2F%2Fi.vimeocdn.com%2Fvideo%2F598160082_1280x720.jpg&src1=https%3A%2F%2Ff.vimeocdn.com%2Fimages_v6%2Fshare%2Fplay_icon_overlay.png", - "lang": "en", - "logo": "https://i.vimeocdn.com/favicon/main-touch_180", - "url": "https://vimeo.com/188175573" -} - -exports['filter by mp4 1'] = [ - "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=160&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184853954309&fvip=4&source=youtube&clen=1232192&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=B71A314816CBE56289EC15C8A1C6BF7EE74C169D.164175CF39A4BDFD2DA2DE6147BD09545AAA01DE&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=133&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184853956556&fvip=4&source=youtube&clen=2122314&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=E3307C18153DDB5E7AACAEAB9942F8B002012050.173DE890AE005232725CB8ADA7A50B7CF25692F0&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=134&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854055225&fvip=4&source=youtube&clen=5015666&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=3DEFAB448DC70677960CAF56B11FB769592FC440.5CD5B13E517668562CCE3F430CF16734E3DD63CA&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=135&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854155850&fvip=4&source=youtube&clen=9629112&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=B4B7A543DEAEEF890B8E6D2DCD83F8F07633DA9F.503058111885647E7E14FA29CA781D0AC0EACA6D&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=136&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854254121&fvip=4&source=youtube&clen=18536895&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=15992BA735625E478060A1DFD9C01D00C0702BE4.C887CC9578BFCBE0ABF4FFCA47139C40B07436E3&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=298&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184834230729&fvip=4&source=youtube&clen=30831030&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=3ED7C0216A6CBFD79647E8BD4E9463A955EB2E8D.4B933FF684BCA75D1E10C2771A101A646875F4BF&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=18&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534185137343773&fvip=4&source=youtube&clen=14141551&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&ratebypass=yes&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=250354E58D1DBCA03CBD4DC9E3FB244C5E1152A0.952F5690B6C36A18ADEA9DDA89ED78148CAFA2A4&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&dur=283.910&ei=KhRyW4nkJIvkV7yXvugN", - "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=22&mm=31%2C29&c=WEB&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mime=video%2Fmp4&ip=95.17.206.54&pl=20&signature=26B8FCB51DC7903132039080C3888D83EC65E0F1.07B5FDE4D99FF3C3354D4D164FBC3CA838F6DEFF&expire=1534224522&initcwndbps=928750&mv=m&mt=1534202758&ms=au%2Crdu&ipbits=0&lmt=1534185082305521&requiressl=yes&sparams=dur%2Cei%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&source=youtube&ratebypass=yes&dur=283.910&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&key=yt6&ei=KhRyW4nkJIvkV7yXvugN&fvip=4" -] - diff --git a/packages/metascraper-media-provider/package.json b/packages/metascraper-media-provider/package.json index dc3d42555..e91f3af0c 100644 --- a/packages/metascraper-media-provider/package.json +++ b/packages/metascraper-media-provider/package.json @@ -18,7 +18,6 @@ "dependencies": { "@metascraper/helpers": "^4.0.1", "got": "~9.1.0", - "json-future": "~2.1.2", "lodash": "~4.17.10", "youtube-dl": "~1.12.2" }, @@ -26,7 +25,6 @@ "mocha": "latest", "nyc": "latest", "puppeteer": "latest", - "should": "latest", "snap-shot": "latest", "standard": "latest" }, diff --git a/packages/metascraper-media-provider/src/get-video-info/index.js b/packages/metascraper-media-provider/src/get-video-info/index.js index ef7612028..c9b10d123 100644 --- a/packages/metascraper-media-provider/src/get-video-info/index.js +++ b/packages/metascraper-media-provider/src/get-video-info/index.js @@ -1,40 +1,46 @@ 'use strict' const { isTwitterUrl, getTwitterVideoInfo } = require('./twitter-video-info') -const getVideoInfo = require('./video-info') +const createGetVideoInfo = require('./video-info') + +const { protocol } = require('@metascraper/helpers') const { chain } = require('lodash') -const getInfo = async url => { - if (!isTwitterUrl(url)) return getVideoInfo(url) +// Local cache for successive calls +let cachedVideoInfoUrl +let cachedVideoInfo - const [videoInfo, twitterVideos] = await Promise.all([ - getVideoInfo(url), - getTwitterVideoInfo(url) - ]) +module.exports = opts => { + const getVideoInfo = createGetVideoInfo(opts) - const formats = chain(videoInfo.formats) - .reduce((acc, format, index) => { - const { url } = twitterVideos[index] - const item = { ...format, url } - return [...acc, item] - }, []) - .value() + const getInfo = async url => { + if (!isTwitterUrl(url)) return getVideoInfo(url) - return { ...videoInfo, formats } -} + const [videoInfo, twitterVideos] = await Promise.all([ + getVideoInfo(url), + getTwitterVideoInfo(url) + ]) -// Local cache for successive calls -let cachedVideoInfoUrl -let cachedVideoInfo + const formats = chain(videoInfo.formats) + .reduce((acc, format, index) => { + const { url } = twitterVideos[index] + const item = { ...format, url, protocol: protocol(url) } + return [...acc, item] + }, []) + .value() + + return { ...videoInfo, formats } + } -module.exports = async url => { - if (url === cachedVideoInfoUrl) return cachedVideoInfo - cachedVideoInfoUrl = url + return async url => { + if (url === cachedVideoInfoUrl) return cachedVideoInfo + cachedVideoInfoUrl = url - try { - cachedVideoInfo = await getInfo(url) - } catch (err) { - cachedVideoInfo = {} + try { + cachedVideoInfo = await getInfo(url) + } catch (err) { + cachedVideoInfo = {} + } + return cachedVideoInfo } - return cachedVideoInfo } diff --git a/packages/metascraper-media-provider/src/get-video-info/video-info.js b/packages/metascraper-media-provider/src/get-video-info/video-info.js index c91e07c95..bd08ca492 100644 --- a/packages/metascraper-media-provider/src/get-video-info/video-info.js +++ b/packages/metascraper-media-provider/src/get-video-info/video-info.js @@ -3,4 +3,9 @@ const youtubedl = require('youtube-dl') const { promisify } = require('util') -module.exports = promisify(youtubedl.getInfo) +const getInfo = promisify(youtubedl.getInfo) + +module.exports = ({ cacheDir } = {}) => { + const opts = cacheDir ? [`--cache-dir=${cacheDir}`] : [] + return url => getInfo(url, opts) +} diff --git a/packages/metascraper-media-provider/src/index.js b/packages/metascraper-media-provider/src/index.js index 730a80c42..4f8386c32 100644 --- a/packages/metascraper-media-provider/src/index.js +++ b/packages/metascraper-media-provider/src/index.js @@ -1,101 +1,126 @@ 'use strict' const { - overEvery, - isEmpty, - eq, - has, - get, chain, + eq, find, + has, + isEmpty, + isNil, + isString, negate, - isString + overEvery } = require('lodash') -const { isUrl, titleize } = require('@metascraper/helpers') -const path = require('path') -const getVideoInfo = require('./get-video-info') +const { + protocol: protocolFn, + extension: extensionFn, + author: authorFn, + description: descriptionFn, + title: titleFn, + url: urlFn +} = require('@metascraper/helpers') + +const createGetVideoInfo = require('./get-video-info') + +const isMIME = extension => ({ ext, url }) => + eq(ext, extension) || eq(extensionFn(url), extension) -const isMIME = ext => item => - eq(get(item, 'ext'), ext) || - path.extname(get(item, 'url')).startsWith(`.${ext}`) +const isProtocol = value => ({ protocol, url }) => + protocol ? eq(protocol, value) : protocolFn(url, value) +/* Most used formats + Seet at https://developer.mozilla.org/en-US/docs/Web/HTML/Supported_media_formats#Browser_compatibility +*/ const isMp4 = isMIME('mp4') const isMp3 = isMIME('mp3') const isM4a = isMIME('m4a') const isAac = isMIME('aac') +const isWav = isMIME('wav') + +const isHttp = isProtocol('http') +const isHttps = isProtocol('https') -const isHttp = item => eq(get(item, 'protocol'), 'http') -const isHttps = item => eq(get(item, 'protocol'), 'https') const hasAudio = item => has(item, 'abr') const hasVideo = item => has(item, 'tbr') +const hastNotVideo = negate(hasVideo) -const getFormatUrls = orderBy => (videos, filters = []) => { - const urls = chain(videos) +const getFormatUrls = ({ orderBy, filterBy }) => (formats, filters) => { + const urls = chain(formats) .filter(overEvery(filters)) .orderBy(orderBy, 'asc') .map('url') - .filter(isUrl) + .filter(url => !eq(extensionFn(url), 'm3u8')) .value() return isEmpty(urls) ? false : urls } -const getVideoUrls = getFormatUrls('tbr') -const getAudioUrls = getFormatUrls('abr') +const getVideoUrls = getFormatUrls({ orderBy: 'tbr' }) +const getAudioUrls = getFormatUrls({ orderBy: 'abr' }) const getVideo = ({ formats }) => - getVideoUrls(formats, [hasAudio, isMp4, isHttps]) || - getVideoUrls(formats, [hasAudio, isMp4, isHttp]) || - getVideoUrls(formats, [isMp4, isHttps]) || - getVideoUrls(formats, [isMp4, isHttp]) || - getVideoUrls(formats, [isMp4]) + getVideoUrls(formats, [hasVideo, hasAudio, isMp4, isHttps]) || + getVideoUrls(formats, [hasVideo, hasAudio, isMp4, isHttp]) || + getVideoUrls(formats, [hasVideo, isMp4, isHttps]) || + getVideoUrls(formats, [hasVideo, isMp4, isHttp]) || + getVideoUrls(formats, [hasVideo, hasAudio, isHttps]) || + getVideoUrls(formats, [hasVideo, hasAudio, isHttp]) || + getVideoUrls(formats, [hasVideo, isHttps]) || + getVideoUrls(formats, [hasVideo, isHttp]) const getAudio = ({ formats }) => - getAudioUrls(formats, [negate(hasVideo), isMp3, isHttps]) || - getAudioUrls(formats, [negate(hasVideo), isAac, isHttps]) || - getAudioUrls(formats, [negate(hasVideo), isM4a, isHttps]) || - getAudioUrls(formats, [negate(hasVideo), isMp3, isHttp]) || - getAudioUrls(formats, [negate(hasVideo), isAac, isHttp]) || - getAudioUrls(formats, [negate(hasVideo), isM4a, isHttp]) || - getAudioUrls(formats, [negate(hasVideo), isMp3]) || - getAudioUrls(formats, [negate(hasVideo), isAac]) || - getAudioUrls(formats, [negate(hasVideo), isM4a]) - -const getAuthor = ({ uploader, creator, uploader_id: uploaderId }) => { - const author = find( - [creator, uploader, uploaderId], - str => isString(str) && !isUrl(str, { relative: false }) - ) - return author && titleize(author, { removeBy: true }) -} + getAudioUrls(formats, [hastNotVideo, hasAudio, isMp3, isHttps]) || + getAudioUrls(formats, [hastNotVideo, hasAudio, isAac, isHttps]) || + getAudioUrls(formats, [hastNotVideo, hasAudio, isM4a, isHttps]) || + getAudioUrls(formats, [hastNotVideo, hasAudio, isWav, isHttps]) || + getAudioUrls(formats, [hastNotVideo, hasAudio, isMp3, isHttp]) || + getAudioUrls(formats, [hastNotVideo, hasAudio, isAac, isHttp]) || + getAudioUrls(formats, [hastNotVideo, hasAudio, isM4a, isHttp]) || + getAudioUrls(formats, [hastNotVideo, hasAudio, isWav, isHttp]) || + getAudioUrls(formats, [hastNotVideo, hasAudio, isHttps]) || + getAudioUrls(formats, [hastNotVideo, hasAudio, isHttp]) + +const getAuthor = ({ uploader, creator, uploader_id: uploaderId }) => + find([creator, uploader, uploaderId], str => authorFn(str)) const getPublisher = ({ extractor_key: extractorKey }) => isString(extractorKey) && extractorKey -const getTitle = ({ title: mainTitle, alt_title: secondaryTitle }) => { - const title = find([mainTitle, secondaryTitle], isString) - return title && titleize(title) -} +const getTitle = ({ title: mainTitle, alt_title: secondaryTitle }) => + find([mainTitle, secondaryTitle], titleFn) const getDate = ({ timestamp }) => - timestamp && new Date(timestamp * 1000).toISOString() + !isNil(timestamp) && new Date(timestamp * 1000).toISOString() + +const getImage = (url, { thumbnail }) => urlFn(thumbnail, { url }) + +const getDescription = (url, { description }) => descriptionFn(description) + +module.exports = opts => { + const getVideoInfo = createGetVideoInfo(opts) -module.exports = () => { return { - video: async ({ url }) => getVideo(await getVideoInfo(url)), + video: async ({ url }) => { + const payload = await getVideoInfo(url) + // console.log(JSON.stringify(payload)) + return getVideo(payload) + }, audio: async ({ url }) => getAudio(await getVideoInfo(url)), author: async ({ url }) => getAuthor(await getVideoInfo(url)), publisher: async ({ url }) => getPublisher(await getVideoInfo(url)), title: async ({ url }) => getTitle(await getVideoInfo(url)), - date: async ({ url }) => getDate(await getVideoInfo(url)) + date: async ({ url }) => getDate(await getVideoInfo(url)), + image: async ({ url }) => getImage(url, await getVideoInfo(url)), + description: async ({ url }) => getDescription(url, await getVideoInfo(url)) } } -module.exports.getFormatUrls = getFormatUrls -module.exports.getVideo = getVideo module.exports.getAudio = getAudio +module.exports.getImage = getImage module.exports.getAuthor = getAuthor +module.exports.getDate = getDate +module.exports.getFormatUrls = getFormatUrls module.exports.getPublisher = getPublisher module.exports.getTitle = getTitle -module.exports.getDate = getDate +module.exports.getVideo = getVideo diff --git a/packages/metascraper-media-provider/test/fixtures/twitter.json b/packages/metascraper-media-provider/test/fixtures/twitter.json new file mode 100644 index 000000000..444ebe545 --- /dev/null +++ b/packages/metascraper-media-provider/test/fixtures/twitter.json @@ -0,0 +1,115 @@ +{ + "display_id": "957383241714970624", + "extractor": "twitter:card", + "protocol": "https", + "description": "The Verge on Twitter: \"Is it bad to blow into game cartridges? https://t.co/Y3yAimrUnP\"", + "vcodec": "avc1.640020", + "format": "hls-1280 - 720x720", + "requested_subtitles": null, + "tbr": 1280, + "height": 720, + "_filename": "The Verge - Is it bad to blow into game cartridges-957383241714970624.mp4", + "preference": null, + "uploader": "The Verge", + "duration": "2:26.562999999999988", + "manifest_url": "https://video.twimg.com/amplify_video/943561675927519232/pl/YNw1OIz1A5FFywhq.m3u8", + "format_id": "hls-1280", + "uploader_id": "verge", + "playlist": null, + "thumbnails": [ + { + "url": "https://pbs.twimg.com/media/DRg1OMRVwAEuwTK.jpg", + "id": "0" + } + ], + "fps": null, + "title": "The Verge - Is it bad to blow into game cartridges?", + "url": "https://video.twimg.com/amplify_video/943561675927519232/pl/720x720/p0lEHBKAhtm_3T9E.m3u8", + "extractor_key": "TwitterCard", + "id": "957383241714970624", + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "playlist_index": null, + "width": 720, + "ext": "mp4", + "webpage_url": "https://twitter.com/verge/status/957383241714970624", + "formats": [ + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "https", + "format": "hls-320 - 240x240", + "url": "https://video.twimg.com/amplify_video/943561675927519232/vid/240x240/mijiQdCq-p9FaO8H.mp4", + "vcodec": "avc1.4d0015", + "tbr": 320, + "height": 240, + "width": 240, + "ext": "mp4", + "preference": null, + "fps": null, + "manifest_url": "https://video.twimg.com/amplify_video/943561675927519232/pl/YNw1OIz1A5FFywhq.m3u8", + "format_id": "hls-320", + "acodec": "mp4a.40.2" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "https", + "format": "hls-832 - 480x480", + "url": "https://video.twimg.com/amplify_video/943561675927519232/vid/480x480/qURzB_XtWBE-dvRa.mp4", + "vcodec": "avc1.4d001f", + "tbr": 832, + "height": 480, + "width": 480, + "ext": "mp4", + "preference": null, + "fps": null, + "manifest_url": "https://video.twimg.com/amplify_video/943561675927519232/pl/YNw1OIz1A5FFywhq.m3u8", + "format_id": "hls-832", + "acodec": "mp4a.40.2" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "https", + "format": "hls-1280 - 720x720", + "url": "https://video.twimg.com/amplify_video/943561675927519232/vid/720x720/h1uN7biCI-Fbzm9D.mp4", + "vcodec": "avc1.640020", + "tbr": 1280, + "height": 720, + "width": 720, + "ext": "mp4", + "preference": null, + "fps": null, + "manifest_url": "https://video.twimg.com/amplify_video/943561675927519232/pl/YNw1OIz1A5FFywhq.m3u8", + "format_id": "hls-1280", + "acodec": "mp4a.40.2" + } + ], + "fulltitle": "The Verge - Is it bad to blow into game cartridges?", + "thumbnail": "https://pbs.twimg.com/media/DRg1OMRVwAEuwTK.jpg", + "webpage_url_basename": "957383241714970624", + "acodec": "mp4a.40.2", + "_duration_raw": 146.563, + "_duration_hms": "00:02:26.563" +} diff --git a/packages/metascraper-media-provider/test/fixtures/vimeo.json b/packages/metascraper-media-provider/test/fixtures/vimeo.json new file mode 100644 index 000000000..5cad8e992 --- /dev/null +++ b/packages/metascraper-media-provider/test/fixtures/vimeo.json @@ -0,0 +1,1166 @@ +{ + "display_id": "188175573", + "extractor": "vimeo", + "protocol": "https", + "description": "Converse has spent a good part of this year updating some of their classics. Our past is constantly catching up to us, but we rarely get to see the relationship between past and present. The sneaker company gave us access to some of the original footwear to create a series of dynamic, thrilling and unexpected motion pieces wherein we watch the old versions turn into the updated models.\nCREDITS\nClient: Converse USA\nAgency: Anomaly NYC\nArt Direction: Serial Cut\nAnimation: Pleid\nSound Design: Heardcity", + "uploader_id": "pleid", + "upload_date": "20161020", + "requested_subtitles": null, + "format": "Original - 1920x1080", + "formats": [ + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-akfire_interconnect_quic-video-823603785 - 640x360 (DASH video)", + "tbr": 512, + "height": 360, + "preference": -40, + "manifest_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-akfire_interconnect_quic-video-823603785", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.64001E", + "width": 640, + "ext": "mp4", + "fragment_base_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603785/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603785/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603785/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603785/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603785/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603785/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-akfire_interconnect_quic_sep-video-823603785 - 640x360 (DASH video)", + "tbr": 512, + "height": 360, + "preference": -40, + "manifest_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/sep/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-akfire_interconnect_quic_sep-video-823603785", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.64001E", + "width": 640, + "ext": "mp4", + "fragment_base_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603785/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603785/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603785/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603785/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603785/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603785/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-fastly_skyfire-video-823603785 - 640x360 (DASH video)", + "tbr": 512, + "height": 360, + "preference": -40, + "manifest_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-fastly_skyfire-video-823603785", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.64001E", + "width": 640, + "ext": "mp4", + "fragment_base_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603785/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603785/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603785/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603785/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603785/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603785/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-fastly_skyfire_sep-video-823603785 - 640x360 (DASH video)", + "tbr": 512, + "height": 360, + "preference": -40, + "manifest_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/sep/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-fastly_skyfire_sep-video-823603785", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.64001E", + "width": 640, + "ext": "mp4", + "fragment_base_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603785/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603785/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603785/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603785/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603785/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603785/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-akfire_interconnect_quic-video-823603780 - 960x540 (DASH video)", + "tbr": 1435, + "height": 540, + "preference": -40, + "manifest_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-akfire_interconnect_quic-video-823603780", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.64001F", + "width": 960, + "ext": "mp4", + "fragment_base_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603780/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603780/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603780/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603780/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603780/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603780/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-akfire_interconnect_quic_sep-video-823603780 - 960x540 (DASH video)", + "tbr": 1435, + "height": 540, + "preference": -40, + "manifest_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/sep/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-akfire_interconnect_quic_sep-video-823603780", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.64001F", + "width": 960, + "ext": "mp4", + "fragment_base_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603780/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603780/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603780/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603780/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603780/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603780/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-fastly_skyfire-video-823603780 - 960x540 (DASH video)", + "tbr": 1435, + "height": 540, + "preference": -40, + "manifest_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-fastly_skyfire-video-823603780", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.64001F", + "width": 960, + "ext": "mp4", + "fragment_base_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603780/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603780/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603780/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603780/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603780/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603780/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-fastly_skyfire_sep-video-823603780 - 960x540 (DASH video)", + "tbr": 1435, + "height": 540, + "preference": -40, + "manifest_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/sep/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-fastly_skyfire_sep-video-823603780", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.64001F", + "width": 960, + "ext": "mp4", + "fragment_base_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603780/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603780/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603780/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603780/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603780/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603780/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-akfire_interconnect_quic-video-823603783 - 1280x720 (DASH video)", + "tbr": 2600, + "height": 720, + "preference": -40, + "manifest_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-akfire_interconnect_quic-video-823603783", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.640020", + "width": 1280, + "ext": "mp4", + "fragment_base_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603783/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603783/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603783/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603783/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603783/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603783/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-akfire_interconnect_quic_sep-video-823603783 - 1280x720 (DASH video)", + "tbr": 2600, + "height": 720, + "preference": -40, + "manifest_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/sep/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-akfire_interconnect_quic_sep-video-823603783", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.640020", + "width": 1280, + "ext": "mp4", + "fragment_base_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603783/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603783/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603783/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603783/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603783/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603783/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-fastly_skyfire-video-823603783 - 1280x720 (DASH video)", + "tbr": 2600, + "height": 720, + "preference": -40, + "manifest_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-fastly_skyfire-video-823603783", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.640020", + "width": 1280, + "ext": "mp4", + "fragment_base_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603783/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603783/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603783/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603783/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603783/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603783/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-fastly_skyfire_sep-video-823603783 - 1280x720 (DASH video)", + "tbr": 2600, + "height": 720, + "preference": -40, + "manifest_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/sep/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-fastly_skyfire_sep-video-823603783", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.640020", + "width": 1280, + "ext": "mp4", + "fragment_base_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603783/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603783/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603783/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603783/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603783/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603783/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-akfire_interconnect_quic-video-823603782 - 1920x1080 (DASH video)", + "tbr": 4925, + "height": 1080, + "preference": -40, + "manifest_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-akfire_interconnect_quic-video-823603782", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.640028", + "width": 1920, + "ext": "mp4", + "fragment_base_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603782/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603782/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603782/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603782/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603782/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603782/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-akfire_interconnect_quic_sep-video-823603782 - 1920x1080 (DASH video)", + "tbr": 4925, + "height": 1080, + "preference": -40, + "manifest_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/sep/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-akfire_interconnect_quic_sep-video-823603782", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.640028", + "width": 1920, + "ext": "mp4", + "fragment_base_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603782/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603782/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603782/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603782/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603782/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603782/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-fastly_skyfire-video-823603782 - 1920x1080 (DASH video)", + "tbr": 4925, + "height": 1080, + "preference": -40, + "manifest_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-fastly_skyfire-video-823603782", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.640028", + "width": 1920, + "ext": "mp4", + "fragment_base_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603782/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603782/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603782/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603782/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603782/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603782/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "asr": null, + "format_note": "DASH video", + "protocol": "http_dash_segments", + "format": "dash-fastly_skyfire_sep-video-823603782 - 1920x1080 (DASH video)", + "tbr": 4925, + "height": 1080, + "preference": -40, + "manifest_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/sep/video/823603785,823603783,823603782,823603780/master.mpd?base64_init=1", + "format_id": "dash-fastly_skyfire_sep-video-823603782", + "container": "mp4_dash", + "language": null, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "vcodec": "avc1.640028", + "width": 1920, + "ext": "mp4", + "fragment_base_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/sep/video/823603785,823603783,823603782,823603780/../", + "filesize": null, + "fps": 30, + "fragments": [ + { + "path": "823603782/chop/segment-0.mp4" + }, + { + "duration": 6, + "path": "823603782/chop/segment-1.m4s" + }, + { + "duration": 6, + "path": "823603782/chop/segment-2.m4s" + }, + { + "duration": 6, + "path": "823603782/chop/segment-3.m4s" + }, + { + "duration": 6, + "path": "823603782/chop/segment-4.m4s" + }, + { + "duration": 4.5, + "path": "823603782/chop/segment-5.m4s" + } + ], + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "https", + "format": "http-360p - 640x360", + "url": "https://gcs-vimeo.akamaized.net/exp=1535800514~acl=%2A%2F823603785.mp4%2A~hmac=c1d7138053fb25092b67c26ddcbd2fc8087da208a30f47a74025e62f844ed1ba/vimeo-prod-skyfire-std-us/01/2635/7/188175573/823603785.mp4", + "tbr": null, + "height": 360, + "width": 640, + "ext": "mp4", + "fps": 30, + "format_id": "http-360p" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "m3u8_native", + "format": "hls-akfire_interconnect_quic-395 - 640x360", + "url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785/playlist.m3u8", + "vcodec": "avc1.64001E", + "tbr": 395, + "height": 360, + "width": 640, + "ext": "mp4", + "preference": null, + "fps": 30, + "manifest_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/master.m3u8", + "format_id": "hls-akfire_interconnect_quic-395", + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "m3u8_native", + "format": "hls-fastly_skyfire-395 - 640x360", + "url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785/playlist.m3u8", + "vcodec": "avc1.64001E", + "tbr": 395, + "height": 360, + "width": 640, + "ext": "mp4", + "preference": null, + "fps": 30, + "manifest_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/master.m3u8", + "format_id": "hls-fastly_skyfire-395", + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "https", + "format": "http-540p - 960x540", + "url": "https://gcs-vimeo.akamaized.net/exp=1535800514~acl=%2A%2F823603780.mp4%2A~hmac=f4c67d41844557065d91bed1f1e24ac1061252f8a37ddad76af75b84f8798429/vimeo-prod-skyfire-std-us/01/2635/7/188175573/823603780.mp4", + "tbr": null, + "height": 540, + "width": 960, + "ext": "mp4", + "fps": 30, + "format_id": "http-540p" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "m3u8_native", + "format": "hls-akfire_interconnect_quic-1081 - 960x540", + "url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603780/playlist.m3u8", + "vcodec": "avc1.64001F", + "tbr": 1081, + "height": 540, + "width": 960, + "ext": "mp4", + "preference": null, + "fps": 30, + "manifest_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/master.m3u8", + "format_id": "hls-akfire_interconnect_quic-1081", + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "m3u8_native", + "format": "hls-fastly_skyfire-1081 - 960x540", + "url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603780/playlist.m3u8", + "vcodec": "avc1.64001F", + "tbr": 1081, + "height": 540, + "width": 960, + "ext": "mp4", + "preference": null, + "fps": 30, + "manifest_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/master.m3u8", + "format_id": "hls-fastly_skyfire-1081", + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "https", + "format": "http-720p - 1280x720", + "url": "https://gcs-vimeo.akamaized.net/exp=1535800514~acl=%2A%2F823603783.mp4%2A~hmac=69a7fc2bb5ccbb888ad958f68b8f6e7b480cd635249a8f6e06fa7bd5dc600a3f/vimeo-prod-skyfire-std-us/01/2635/7/188175573/823603783.mp4", + "tbr": null, + "height": 720, + "width": 1280, + "ext": "mp4", + "fps": 30, + "format_id": "http-720p" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "m3u8_native", + "format": "hls-akfire_interconnect_quic-1935 - 1280x720", + "url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603783/playlist.m3u8", + "vcodec": "avc1.640020", + "tbr": 1935, + "height": 720, + "width": 1280, + "ext": "mp4", + "preference": null, + "fps": 30, + "manifest_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/master.m3u8", + "format_id": "hls-akfire_interconnect_quic-1935", + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "m3u8_native", + "format": "hls-fastly_skyfire-1935 - 1280x720", + "url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603783/playlist.m3u8", + "vcodec": "avc1.640020", + "tbr": 1935, + "height": 720, + "width": 1280, + "ext": "mp4", + "preference": null, + "fps": 30, + "manifest_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/master.m3u8", + "format_id": "hls-fastly_skyfire-1935", + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "https", + "format": "http-1080p - 1920x1080", + "url": "https://gcs-vimeo.akamaized.net/exp=1535800514~acl=%2A%2F823603782.mp4%2A~hmac=1ff98e8349d222bbb11b2e7e3105729a578f83c632a34e252db61034b2b05d6b/vimeo-prod-skyfire-std-us/01/2635/7/188175573/823603782.mp4", + "tbr": null, + "height": 1080, + "width": 1920, + "ext": "mp4", + "fps": 30, + "format_id": "http-1080p" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "m3u8_native", + "format": "hls-akfire_interconnect_quic-3839 - 1920x1080", + "url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603782/playlist.m3u8", + "vcodec": "avc1.640028", + "tbr": 3839, + "height": 1080, + "width": 1920, + "ext": "mp4", + "preference": null, + "fps": 30, + "manifest_url": "https://174skyfiregce-vimeo.akamaized.net/exp=1535800514~acl=%2F188175573%2F%2A~hmac=ca857b9ab870f783ac48072974d3a30a68955540ecd5d6190ab9d0263cddbdec/188175573/video/823603785,823603783,823603782,823603780/master.m3u8", + "format_id": "hls-akfire_interconnect_quic-3839", + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "m3u8_native", + "format": "hls-fastly_skyfire-3839 - 1920x1080", + "url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603782/playlist.m3u8", + "vcodec": "avc1.640028", + "tbr": 3839, + "height": 1080, + "width": 1920, + "ext": "mp4", + "preference": null, + "fps": 30, + "manifest_url": "https://skyfire.vimeocdn.com/1535800514-0x9d70ea322a34a8c90f5d82e5bdc2243c65f153d8/188175573/video/823603785,823603783,823603782,823603780/master.m3u8", + "format_id": "hls-fastly_skyfire-3839", + "acodec": "none" + }, + { + "protocol": "https", + "format": "Original - 1920x1080", + "url": "https://player.vimeo.com/play/823603079?s=188175573_1535796915_d75723e51221e86af027c51e428dc264&loc=external&context=Vimeo%5CController%5CClipController.main&download=1", + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "height": 1080, + "width": 1920, + "ext": "mp4", + "filesize": 31977000, + "format_id": "Original", + "preference": 1 + } + ], + "height": 1080, + "_filename": "Converse - Past meets Present - Montage-188175573.mp4", + "like_count": 350, + "preference": 1, + "uploader": "pleid", + "duration": "29", + "format_id": "Original", + "playlist_index": null, + "subtitles": {}, + "view_count": 8923, + "playlist": null, + "thumbnails": [ + { + "url": "https://i.vimeocdn.com/video/598160082_1280.jpg", + "id": "0" + } + ], + "title": "Converse - Past meets Present - Montage", + "url": "https://player.vimeo.com/play/823603079?s=188175573_1535796915_d75723e51221e86af027c51e428dc264&loc=external&context=Vimeo%5CController%5CClipController.main&download=1", + "extractor_key": "Vimeo", + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "ext": "mp4", + "id": "188175573", + "width": 1920, + "comment_count": 3, + "uploader_url": "https://vimeo.com/pleid", + "filesize": 31977000, + "webpage_url": "https://vimeo.com/188175573", + "timestamp": 1476968812, + "fulltitle": "Converse - Past meets Present - Montage", + "thumbnail": "https://i.vimeocdn.com/video/598160082_1280.jpg", + "webpage_url_basename": "188175573", + "_duration_raw": 29, + "_duration_hms": "00:00:29" +} diff --git a/packages/metascraper-media-provider/test/fixtures/youtube-dl.json b/packages/metascraper-media-provider/test/fixtures/youtube-dl.json deleted file mode 100644 index 118d0afd4..000000000 --- a/packages/metascraper-media-provider/test/fixtures/youtube-dl.json +++ /dev/null @@ -1,706 +0,0 @@ -{ - "upload_date": "20180813", - "creator": null, - "series": null, - "chapters": null, - "height": 720, - "like_count": 1225, - "duration": 284, - "id": "2hZDd4Yfhyw", - "requested_formats": [ - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "tbr": 2696.517, - "protocol": "https", - "format": "298 - 1280x720 (720p60)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=298&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184834230729&fvip=4&source=youtube&clen=30831030&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=3ED7C0216A6CBFD79647E8BD4E9463A955EB2E8D.4B933FF684BCA75D1E10C2771A101A646875F4BF&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "avc1.4d4020", - "format_note": "720p60", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "downloader_options": { - "http_chunk_size": 10485760 - }, - "width": 1280, - "ext": "mp4", - "filesize": 30831030, - "fps": 60, - "format_id": "298", - "height": 720, - "quality": -1, - "acodec": "none" - }, - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "tbr": 127.913, - "container": "m4a_dash", - "format": "140 - audio only (DASH audio)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=140&mime=audio%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184829638266&fvip=4&source=youtube&clen=4509873&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=A906D1491BFCC44A63F6F2E65D4F54E6A212D341.97A0F255237A73DA79E95D77055B067644E714E8&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.910&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "none", - "format_note": "DASH audio", - "abr": 128, - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "downloader_options": { - "http_chunk_size": 10485760 - }, - "ext": "m4a", - "filesize": 4509873, - "protocol": "https", - "format_id": "140", - "quality": -1, - "acodec": "mp4a.40.2" - } - ], - "view_count": 94812, - "playlist": null, - "title": "Ice Cube gives his take on whether LeBron James can achieve greatness in LA | First Take | ESPN", - "format": "298 - 1280x720 (720p60)+140 - audio only (DASH audio)", - "ext": "mp4", - "playlist_index": null, - "dislike_count": 66, - "average_rating": 4.79550743103, - "abr": 128, - "uploader_url": "http://www.youtube.com/user/ESPN", - "subtitles": {}, - "fps": 60, - "stretched_ratio": null, - "season_number": null, - "annotations": null, - "webpage_url_basename": "watch", - "acodec": "mp4a.40.2", - "display_id": "2hZDd4Yfhyw", - "automatic_captions": {}, - "description": "Ice Cube says that if LeBron James can win 4 ring in 4 years then he can be considered for the greatest in Los Angeles Lakers history, but he will be respected no matter what he does.\n\n✔ Subscribe to ESPN on YouTube: http://es.pn/SUBSCRIBEtoYOUTUBE\n✔ Subscribe to ESPN FC on YouTube: http://bit.ly/SUBSCRIBEtoESPNFC\n✔ Subscribe to NBA on ESPN on YouTube: http://bit.ly/SUBSCRIBEtoNBAonESPN\n✔ Watch ESPN on YouTube TV: http://es.pn/YouTubeTV\n\nESPN on Social Media:\n► Follow on Twitter: http://www.twitter.com/espn\n► Like on Facebook: http://www.facebook.com/espn\n► Follow on Instagram: http://www.instagram.com/espn\n\nVisit ESPN on YouTube to get up-to-the-minute sports news coverage, scores, highlights and commentary for NFL, NHL, MLB, NBA, College Football, NCAA Basketball, soccer and more.\n\nMore on ESPN.com: http://www.espn.com", - "tags": [ - "ice cube first take lebron james", - "ice cube first take espn", - "ice cube", - "ice cube first take", - "ice cube lebron james espn", - "ice cube on first take today", - "ice cube on first take", - "ice cube lebron james", - "first take ice cube", - "ice cube lebron james los angeles", - "ice cube on lebron james", - "ice cube lebron james lakers", - "ice cube on lebron to lakers", - "first take", - "stephen a smith", - "max kellerman", - "first take espn", - "espn first take", - "first take stephen a smith", - "espn" - ], - "track": null, - "requested_subtitles": null, - "start_time": null, - "uploader": "ESPN", - "format_id": "298+140", - "episode_number": null, - "uploader_id": "ESPN", - "categories": [ - "Sports" - ], - "thumbnails": [ - { - "url": "https://i.ytimg.com/vi/2hZDd4Yfhyw/hqdefault.jpg", - "id": "0" - } - ], - "license": null, - "alt_title": null, - "extractor_key": "Youtube", - "vcodec": "avc1.4d4020", - "artist": null, - "thumbnail": "https://i.ytimg.com/vi/2hZDd4Yfhyw/hqdefault.jpg", - "vbr": null, - "is_live": null, - "extractor": "youtube", - "end_time": null, - "webpage_url": "https://www.youtube.com/watch?v=2hZDd4Yfhyw", - "formats": [ - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "format_note": "DASH audio", - "protocol": "https", - "format": "249 - audio only (DASH audio)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=249&mime=audio%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185555220562&fvip=4&source=youtube&clen=1767940&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=491AFF87944CC9FD25B5D21C589F90377C3FCB3E.E05E599007F8D098CC04FAA293BAE8755110A4EB&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.861&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "none", - "tbr": 52.905, - "abr": 50, - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "downloader_options": { - "http_chunk_size": 10485760 - }, - "ext": "webm", - "filesize": 1767940, - "format_id": "249", - "quality": -1, - "acodec": "opus" - }, - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "format_note": "DASH audio", - "protocol": "https", - "format": "250 - audio only (DASH audio)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=250&mime=audio%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185554470802&fvip=4&source=youtube&clen=2075656&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=26CBC1F3175DCD572EB2DE5551DF47DE6F38E953.2CD5A527DABFE81155C7F04C5FDE2F78C82C79C7&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.861&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "none", - "tbr": 67.232, - "abr": 70, - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "downloader_options": { - "http_chunk_size": 10485760 - }, - "ext": "webm", - "filesize": 2075656, - "format_id": "250", - "quality": -1, - "acodec": "opus" - }, - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "format_note": "DASH audio", - "protocol": "https", - "format": "171 - audio only (DASH audio)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=171&mime=audio%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185555965278&fvip=4&source=youtube&clen=3312137&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=210B8EFBF6E1B837217E38F9A99C7377B489F24D.02B40582854DD3E3D2BE16CCB330B6B78F52F536&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.847&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "none", - "tbr": 103.254, - "abr": 128, - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "downloader_options": { - "http_chunk_size": 10485760 - }, - "ext": "webm", - "filesize": 3312137, - "format_id": "171", - "quality": -1, - "acodec": "vorbis" - }, - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "format_note": "DASH audio", - "protocol": "https", - "format": "251 - audio only (DASH audio)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=251&mime=audio%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185555925305&fvip=4&source=youtube&clen=3710023&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=BBA4D9E803869BDFA4CA85AC3975A4E74F654DFC.C8B8D633CDAB9FE61DF583E8DBB9A2FF981C855B&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.861&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "none", - "tbr": 121.803, - "abr": 160, - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "downloader_options": { - "http_chunk_size": 10485760 - }, - "ext": "webm", - "filesize": 3710023, - "format_id": "251", - "quality": -1, - "acodec": "opus" - }, - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "tbr": 127.913, - "container": "m4a_dash", - "format": "140 - audio only (DASH audio)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=140&mime=audio%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184829638266&fvip=4&source=youtube&clen=4509873&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=A906D1491BFCC44A63F6F2E65D4F54E6A212D341.97A0F255237A73DA79E95D77055B067644E714E8&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.910&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "none", - "format_note": "DASH audio", - "abr": 128, - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "downloader_options": { - "http_chunk_size": 10485760 - }, - "ext": "m4a", - "filesize": 4509873, - "protocol": "https", - "format_id": "140", - "quality": -1, - "acodec": "mp4a.40.2" - }, - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "tbr": 96.238, - "container": "webm", - "format": "278 - 256x144 (144p)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=278&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220177397&fvip=4&source=youtube&clen=3047257&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=31AEE690FC015CBF5606A0D9102FFC96B9BCA24B.2FF11959EEB4DE6777B07B938CE8057B30DD9648&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "vp9", - "format_note": "144p", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "downloader_options": { - "http_chunk_size": 10485760 - }, - "width": 256, - "ext": "webm", - "filesize": 3047257, - "fps": 30, - "protocol": "https", - "format_id": "278", - "height": 144, - "quality": -1, - "acodec": "none" - }, - { - "format_note": "144p", - "protocol": "https", - "format": "160 - 256x144 (144p)", - "tbr": 104.641, - "height": 144, - "downloader_options": { - "http_chunk_size": 10485760 - }, - "format_id": "160", - "quality": -1, - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=160&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184853954309&fvip=4&source=youtube&clen=1232192&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=B71A314816CBE56289EC15C8A1C6BF7EE74C169D.164175CF39A4BDFD2DA2DE6147BD09545AAA01DE&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "avc1.4d400c", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 256, - "ext": "mp4", - "filesize": 1232192, - "fps": 30, - "acodec": "none" - }, - { - "format_note": "240p", - "protocol": "https", - "format": "133 - 426x240 (240p)", - "tbr": 191.392, - "height": 240, - "downloader_options": { - "http_chunk_size": 10485760 - }, - "format_id": "133", - "quality": -1, - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=133&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184853956556&fvip=4&source=youtube&clen=2122314&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=E3307C18153DDB5E7AACAEAB9942F8B002012050.173DE890AE005232725CB8ADA7A50B7CF25692F0&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "avc1.4d4015", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 426, - "ext": "mp4", - "filesize": 2122314, - "fps": 30, - "acodec": "none" - }, - { - "format_note": "240p", - "protocol": "https", - "format": "242 - 426x240 (240p)", - "tbr": 222.403, - "height": 240, - "downloader_options": { - "http_chunk_size": 10485760 - }, - "format_id": "242", - "quality": -1, - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=242&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220160992&fvip=4&source=youtube&clen=3850378&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=ADE29866D0E224F4FA327931066956085EDB8A01.473453C889918A9BB6E6B336A627D4D7B3F33824&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "vp9", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 426, - "ext": "webm", - "filesize": 3850378, - "fps": 30, - "acodec": "none" - }, - { - "format_note": "360p", - "protocol": "https", - "format": "243 - 640x360 (360p)", - "tbr": 401.458, - "height": 360, - "downloader_options": { - "http_chunk_size": 10485760 - }, - "format_id": "243", - "quality": -1, - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=243&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220165648&fvip=4&source=youtube&clen=7618198&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=C5BE6688672F7461261A4254C66466434E6B9DD5.383565CD710DFB4CAAA0A895D3C4C58BD1685374&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "vp9", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 640, - "ext": "webm", - "filesize": 7618198, - "fps": 30, - "acodec": "none" - }, - { - "format_note": "360p", - "protocol": "https", - "format": "134 - 640x360 (360p)", - "tbr": 471.132, - "height": 360, - "downloader_options": { - "http_chunk_size": 10485760 - }, - "format_id": "134", - "quality": -1, - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=134&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854055225&fvip=4&source=youtube&clen=5015666&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=3DEFAB448DC70677960CAF56B11FB769592FC440.5CD5B13E517668562CCE3F430CF16734E3DD63CA&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "avc1.4d401e", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 640, - "ext": "mp4", - "filesize": 5015666, - "fps": 30, - "acodec": "none" - }, - { - "format_note": "480p", - "protocol": "https", - "format": "244 - 854x480 (480p)", - "tbr": 757.845, - "height": 480, - "downloader_options": { - "http_chunk_size": 10485760 - }, - "format_id": "244", - "quality": -1, - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=244&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220165309&fvip=4&source=youtube&clen=13504592&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=8BE1DAB5B67AEAEDDEC26141F4EB2323434677F9.40D8005C9A361EA5C6FA16BBE4EE5C03D692FD8E&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "vp9", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 854, - "ext": "webm", - "filesize": 13504592, - "fps": 30, - "acodec": "none" - }, - { - "format_note": "480p", - "protocol": "https", - "format": "135 - 854x480 (480p)", - "tbr": 900.881, - "height": 480, - "downloader_options": { - "http_chunk_size": 10485760 - }, - "format_id": "135", - "quality": -1, - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=135&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854155850&fvip=4&source=youtube&clen=9629112&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=B4B7A543DEAEEF890B8E6D2DCD83F8F07633DA9F.503058111885647E7E14FA29CA781D0AC0EACA6D&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "avc1.4d401f", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 854, - "ext": "mp4", - "filesize": 9629112, - "fps": 30, - "acodec": "none" - }, - { - "format_note": "720p", - "protocol": "https", - "format": "247 - 1280x720 (720p)", - "tbr": 1463.626, - "height": 720, - "downloader_options": { - "http_chunk_size": 10485760 - }, - "format_id": "247", - "quality": -1, - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=247&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185220174836&fvip=4&source=youtube&clen=29547867&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=4EB17F5834B600075C7EB219E5876609A1FFC932.75C777BD0FC651CF34137FF53964B03149CA5C49&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "vp9", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 1280, - "ext": "webm", - "filesize": 29547867, - "fps": 30, - "acodec": "none" - }, - { - "format_note": "720p", - "protocol": "https", - "format": "136 - 1280x720 (720p)", - "tbr": 1592.796, - "height": 720, - "downloader_options": { - "http_chunk_size": 10485760 - }, - "format_id": "136", - "quality": -1, - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=136&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184854254121&fvip=4&source=youtube&clen=18536895&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=15992BA735625E478060A1DFD9C01D00C0702BE4.C887CC9578BFCBE0ABF4FFCA47139C40B07436E3&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "avc1.4d401f", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 1280, - "ext": "mp4", - "filesize": 18536895, - "fps": 30, - "acodec": "none" - }, - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "tbr": 2586.548, - "protocol": "https", - "format": "302 - 1280x720 (720p60)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=302&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185013746111&fvip=4&source=youtube&clen=41986806&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=46D6B16D847DEF9A654E8E15E805A488D348A2AF.3D6F1BA63D1AE2CA7736A59F18CCCF80C9003854&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.849&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "vp9", - "format_note": "720p60", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "downloader_options": { - "http_chunk_size": 10485760 - }, - "width": 1280, - "ext": "webm", - "filesize": 41986806, - "fps": 60, - "format_id": "302", - "height": 720, - "quality": -1, - "acodec": "none" - }, - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "tbr": 2696.517, - "protocol": "https", - "format": "298 - 1280x720 (720p60)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=298&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278%2C298%2C302&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534184834230729&fvip=4&source=youtube&clen=30831030&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&keepalive=yes&signature=3ED7C0216A6CBFD79647E8BD4E9463A955EB2E8D.4B933FF684BCA75D1E10C2771A101A646875F4BF&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.850&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "avc1.4d4020", - "format_note": "720p60", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "downloader_options": { - "http_chunk_size": 10485760 - }, - "width": 1280, - "ext": "mp4", - "filesize": 30831030, - "fps": 60, - "format_id": "298", - "height": 720, - "quality": -1, - "acodec": "none" - }, - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "protocol": "https", - "format": "17 - 176x144 (small)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=17&mime=video%2F3gpp&pl=20&expire=1534224522&mv=m&lmt=1534184587965484&fvip=4&source=youtube&clen=2893543&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=8035AFAEEB9BE2736E041BB5B907F28F881CB218.CAA02DA4FA239160EE8B3F091884A7EFC53B93EC&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.933&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "mp4v.20.3", - "format_note": "small", - "ext": "3gp", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 176, - "abr": 24, - "filesize": 2893543, - "format_id": "17", - "height": 144, - "resolution": "176x144", - "acodec": "mp4a.40.2" - }, - { - "protocol": "https", - "format": "36 - 320x180 (small)", - "format_note": "small", - "height": 180, - "format_id": "36", - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=36&mime=video%2F3gpp&pl=20&expire=1534224522&mv=m&lmt=1534184588565234&fvip=4&source=youtube&clen=7869391&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=12F4A55FD6D6D2484F0FD0BF775EEA23D3E97639.D1B15C0BBCDAD61CBDD72D5695B2FE3B8C4668D8&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&dur=283.933&ei=KhRyW4nkJIvkV7yXvugN&ratebypass=yes", - "vcodec": "mp4v.20.3", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 320, - "ext": "3gp", - "filesize": 7869391, - "resolution": "320x180", - "acodec": "mp4a.40.2" - }, - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "protocol": "https", - "format": "18 - 640x360 (medium)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=18&mime=video%2Fmp4&pl=20&expire=1534224522&mv=m&lmt=1534185137343773&fvip=4&source=youtube&clen=14141551&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&ratebypass=yes&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=250354E58D1DBCA03CBD4DC9E3FB244C5E1152A0.952F5690B6C36A18ADEA9DDA89ED78148CAFA2A4&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&dur=283.910&ei=KhRyW4nkJIvkV7yXvugN", - "quality": 1, - "vcodec": "avc1.42001E", - "format_note": "medium", - "ext": "mp4", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 640, - "abr": 96, - "filesize": 14141551, - "format_id": "18", - "height": 360, - "resolution": "640x360", - "acodec": "mp4a.40.2" - }, - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "protocol": "https", - "format": "43 - 640x360 (medium)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=43&mime=video%2Fwebm&pl=20&expire=1534224522&mv=m&lmt=1534185610458623&fvip=4&source=youtube&clen=22258178&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&ratebypass=yes&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mm=31%2C29&c=WEB&key=yt6&ip=95.17.206.54&signature=DA20EA5896E4F5E5B2F2B026BE63DF56AC4D5CF1.77668799BD2EB81CEBD00EA8F6191F3419947357&initcwndbps=928750&gir=yes&mt=1534202758&ms=au%2Crdu&ipbits=0&requiressl=yes&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&dur=0.000&ei=KhRyW4nkJIvkV7yXvugN", - "quality": 1, - "vcodec": "vp8.0", - "format_note": "medium", - "ext": "webm", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 640, - "abr": 128, - "filesize": 22258178, - "format_id": "43", - "height": 360, - "resolution": "640x360", - "acodec": "vorbis" - }, - { - "http_headers": { - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0 (Chrome)" - }, - "protocol": "https", - "format": "22 - 1280x720 (hd720)", - "url": "https://r6---sn-w511uxa-8aje.googlevideo.com/videoplayback?itag=22&mm=31%2C29&c=WEB&mn=sn-w511uxa-8aje%2Csn-h5q7knez&mime=video%2Fmp4&ip=95.17.206.54&pl=20&signature=26B8FCB51DC7903132039080C3888D83EC65E0F1.07B5FDE4D99FF3C3354D4D164FBC3CA838F6DEFF&expire=1534224522&initcwndbps=928750&mv=m&mt=1534202758&ms=au%2Crdu&ipbits=0&lmt=1534185082305521&requiressl=yes&sparams=dur%2Cei%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&source=youtube&ratebypass=yes&dur=283.910&id=o-AH-F7AllkKVHynK8qqnw4B_x0q76t92S_pPhPul-9lU7&key=yt6&ei=KhRyW4nkJIvkV7yXvugN&fvip=4", - "quality": 2, - "vcodec": "avc1.64001F", - "format_note": "hd720", - "ext": "mp4", - "player_url": "/yts/jsbin/player-vflM-t6FF/en_US/base.js", - "width": 1280, - "abr": 192, - "format_id": "22", - "height": 720, - "resolution": "1280x720", - "acodec": "mp4a.40.2" - } - ], - "resolution": null, - "width": 1280, - "age_limit": 0 -} diff --git a/packages/metascraper-media-provider/test/fixtures/youtube.json b/packages/metascraper-media-provider/test/fixtures/youtube.json new file mode 100644 index 000000000..c5b3bfdff --- /dev/null +++ b/packages/metascraper-media-provider/test/fixtures/youtube.json @@ -0,0 +1,654 @@ +{ + "upload_date": "20170414", + "protocol": "https", + "creator": null, + "series": null, + "format_note": "hd720", + "chapters": null, + "height": 720, + "like_count": 7501, + "duration": "2:22", + "fulltitle": "Star Wars 8 THE LAST JEDI Official TRAILER (2017) Daisy Ridley, Disney Movie HD", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "quality": 2, + "id": "hwMkbaS_M_c", + "view_count": 1075290, + "playlist": null, + "title": "Star Wars 8 THE LAST JEDI Official TRAILER (2017) Daisy Ridley, Disney Movie HD", + "_filename": "Star Wars 8 THE LAST JEDI Official TRAILER (2017) Daisy Ridley, Disney Movie HD-hwMkbaS_M_c.mp4", + "format": "22 - 1280x720 (hd720)", + "ext": "mp4", + "playlist_index": null, + "dislike_count": 525, + "average_rating": 4.73835039139, + "abr": 192, + "uploader_url": "http://www.youtube.com/user/movietrailers", + "subtitles": {}, + "season_number": null, + "annotations": null, + "webpage_url_basename": "watch", + "acodec": "mp4a.40.2", + "display_id": "hwMkbaS_M_c", + "automatic_captions": {}, + "description": "Star Wars 8 THE LAST JEDI Official TRAILER (2017) Daisy Ridley, Disney Movie HD\n© 2017 - Disney", + "tags": [ + "Official", + "Trailer", + "Movie", + "Clip", + "TV Spot", + "International", + "Film", + "Star Wars 8", + "THE LAST JEDI", + "Star Wars THE LAST JEDI" + ], + "track": null, + "requested_subtitles": null, + "start_time": null, + "uploader": "ONE Media", + "format_id": "22", + "episode_number": null, + "uploader_id": "movietrailers", + "categories": [ + "Film & Animation" + ], + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/hwMkbaS_M_c/maxresdefault.jpg", + "id": "0" + } + ], + "license": null, + "alt_title": null, + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?fvip=3&mime=video%2Fmp4&ipbits=0&lmt=1507953047549769&ratebypass=yes&key=yt6&requiressl=yes&c=WEB&expire=1535818875&ei=G2iKW_jiIMTjV6OjsqgF&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=22&source=youtube&dur=141.502&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=dur%2Cei%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&mt=1535797130&signature=1CE1646A209F0D00E8DD16784BAFCD9E0C9CF06C.0EF24C1CBF4A06AFEB7F404C3E28CE35B354F384", + "extractor_key": "Youtube", + "vcodec": "avc1.64001F", + "artist": null, + "thumbnail": "https://i.ytimg.com/vi/hwMkbaS_M_c/maxresdefault.jpg", + "is_live": null, + "extractor": "youtube", + "end_time": null, + "webpage_url": "https://www.youtube.com/watch?v=hwMkbaS_M_c", + "formats": [ + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "format_note": "DASH audio", + "protocol": "https", + "format": "249 - audio only (DASH audio)", + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=audio%2Fwebm&gir=yes&ipbits=0&lmt=1503812236172843&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=249&mt=1535797130&dur=141.441&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&clen=767256&signature=37EFCDA1816850D5475935E19742607B798823AC.D68C8173223386C5CDC7270DFAFE8A89427C0A4E&ratebypass=yes", + "vcodec": "none", + "tbr": 56.647, + "abr": 50, + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "ext": "webm", + "filesize": 767256, + "format_id": "249", + "quality": -1, + "acodec": "opus" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "format_note": "DASH audio", + "protocol": "https", + "format": "250 - audio only (DASH audio)", + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=audio%2Fwebm&gir=yes&ipbits=0&lmt=1503812231569091&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=250&mt=1535797130&dur=141.441&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&clen=1012417&signature=6EBF894E981A5348F6F460E4E4159B3C8AAB251F.BF73ECF1C1816183AE22FFF93AA158FB8AE8AFEE&ratebypass=yes", + "vcodec": "none", + "tbr": 73.815, + "abr": 70, + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "ext": "webm", + "filesize": 1012417, + "format_id": "250", + "quality": -1, + "acodec": "opus" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "tbr": 128.211, + "container": "m4a_dash", + "format": "140 - audio only (DASH audio)", + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=audio%2Fmp4&gir=yes&ipbits=0&lmt=1507952811949144&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=140&mt=1535797130&dur=141.502&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&clen=2248120&signature=E3942E3A3682B75BD7ADA0D3B23D839993029231.CA74D985A1158F2EAD22633C998E28A30F5F28A2&ratebypass=yes", + "vcodec": "none", + "format_note": "DASH audio", + "abr": 128, + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "ext": "m4a", + "filesize": 2248120, + "protocol": "https", + "format_id": "140", + "quality": -1, + "acodec": "mp4a.40.2" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "format_note": "DASH audio", + "protocol": "https", + "format": "171 - audio only (DASH audio)", + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=audio%2Fwebm&gir=yes&ipbits=0&lmt=1503812242728751&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=171&mt=1535797130&dur=141.437&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&clen=1855487&signature=5FB40BEB02901583FCF198CBA5173E5BE8406DA3.89DC7C6DE72A0913447FAD3AB2F99F2A0C30BCC7&ratebypass=yes", + "vcodec": "none", + "tbr": 136.226, + "abr": 128, + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "ext": "webm", + "filesize": 1855487, + "format_id": "171", + "quality": -1, + "acodec": "vorbis" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "format_note": "DASH audio", + "protocol": "https", + "format": "251 - audio only (DASH audio)", + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=audio%2Fwebm&gir=yes&ipbits=0&lmt=1503812234640643&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=251&mt=1535797130&dur=141.441&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&clen=1973587&signature=A4A70B7B131A1F83908D54FA680B1F0311F90726.A540C3075E6F9C7FBAE1D7258A8497335AB4C3FA&ratebypass=yes", + "vcodec": "none", + "tbr": 139.028, + "abr": 160, + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "ext": "webm", + "filesize": 1973587, + "format_id": "251", + "quality": -1, + "acodec": "opus" + }, + { + "format_note": "144p", + "protocol": "https", + "format": "160 - 256x144 (144p)", + "tbr": 103.817, + "height": 144, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "160", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fmp4&gir=yes&ipbits=0&lmt=1507952889801167&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=160&mt=1535797130&dur=141.440&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=503873&signature=5CCB9E814DC47000AE95012B9BAC5E638CF7B335.AEBC98B5759C5A7B2BFCD057539543D7FF3F0682&ratebypass=yes", + "vcodec": "avc1.4d400c", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 256, + "ext": "mp4", + "filesize": 503873, + "fps": 25, + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "tbr": 113.715, + "container": "webm", + "format": "278 - 256x144 (144p)", + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fwebm&gir=yes&ipbits=0&lmt=1503812681501403&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=278&mt=1535797130&dur=141.400&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=1069403&signature=7D7393720C189DC68DF0A30D394962A443810F61.41AAC16433B25DED2AEA16961ACDA99217ED3FB3&ratebypass=yes", + "vcodec": "vp9", + "format_note": "144p", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "downloader_options": { + "http_chunk_size": 10485760 + }, + "width": 256, + "ext": "webm", + "filesize": 1069403, + "fps": 25, + "protocol": "https", + "format_id": "278", + "height": 144, + "quality": -1, + "acodec": "none" + }, + { + "format_note": "240p", + "protocol": "https", + "format": "133 - 426x240 (240p)", + "tbr": 195.762, + "height": 240, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "133", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fmp4&gir=yes&ipbits=0&lmt=1507952889799661&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=133&mt=1535797130&dur=141.440&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=916551&signature=CB679D12966286AEDBC316AF52CB1502D31252C6.28D58615691F9E540559B860B5719E7A55C90BC0&ratebypass=yes", + "vcodec": "avc1.4d4015", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 426, + "ext": "mp4", + "filesize": 916551, + "fps": 25, + "acodec": "none" + }, + { + "format_note": "240p", + "protocol": "https", + "format": "242 - 426x240 (240p)", + "tbr": 208.406, + "height": 240, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "242", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fwebm&gir=yes&ipbits=0&lmt=1503812681641246&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=242&mt=1535797130&dur=141.400&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=1328857&signature=6599D459FC9B49A50485711E1C335653532B35D5.A2CD851AC2535957714881682AB07D1DC317F490&ratebypass=yes", + "vcodec": "vp9", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 426, + "ext": "webm", + "filesize": 1328857, + "fps": 25, + "acodec": "none" + }, + { + "format_note": "360p", + "protocol": "https", + "format": "243 - 640x360 (360p)", + "tbr": 387.135, + "height": 360, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "243", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fwebm&gir=yes&ipbits=0&lmt=1503812681686904&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=243&mt=1535797130&dur=141.400&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=2492253&signature=1FF8110CB9D75C4A8FB282C6806A5A71F6A9BF86.63C626C127A699E1854F0571712491E0D3DDBF84&ratebypass=yes", + "vcodec": "vp9", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 640, + "ext": "webm", + "filesize": 2492253, + "fps": 25, + "acodec": "none" + }, + { + "format_note": "360p", + "protocol": "https", + "format": "134 - 640x360 (360p)", + "tbr": 463.236, + "height": 360, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "134", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fmp4&gir=yes&ipbits=0&lmt=1507952889802693&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=134&mt=1535797130&dur=141.440&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=2207390&signature=90CC91716540A24CFC117B94831E34CBF2E11519.1253E700A2076681949A8077EF0A0CC13EE0A70A&ratebypass=yes", + "vcodec": "avc1.4d401e", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 640, + "ext": "mp4", + "filesize": 2207390, + "fps": 25, + "acodec": "none" + }, + { + "format_note": "480p", + "protocol": "https", + "format": "244 - 854x480 (480p)", + "tbr": 663.803, + "height": 480, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "244", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fwebm&gir=yes&ipbits=0&lmt=1503812682913964&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=244&mt=1535797130&dur=141.400&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=4032317&signature=539A0FB04E19D118BA1B236C5F5C1FDE7077810B.B6F322E499368273C442D243E9DB905F52B69F23&ratebypass=yes", + "vcodec": "vp9", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 854, + "ext": "webm", + "filesize": 4032317, + "fps": 25, + "acodec": "none" + }, + { + "format_note": "480p", + "protocol": "https", + "format": "135 - 854x480 (480p)", + "tbr": 893.337, + "height": 480, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "135", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fmp4&gir=yes&ipbits=0&lmt=1507952889999957&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=135&mt=1535797130&dur=141.440&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=4380005&signature=20C47063DBB07F696EA3ADDB14B9127CB5C5CB3A.B722ED0B37D06C014D5747F1481559B0D0B36D66&ratebypass=yes", + "vcodec": "avc1.4d401e", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 854, + "ext": "mp4", + "filesize": 4380005, + "fps": 25, + "acodec": "none" + }, + { + "format_note": "720p", + "protocol": "https", + "format": "247 - 1280x720 (720p)", + "tbr": 1353.112, + "height": 720, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "247", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fwebm&gir=yes&ipbits=0&lmt=1503812680723992&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=247&mt=1535797130&dur=141.400&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=8034497&signature=451EBFD73FA768FAF6B3F7F80C7D2D76DC69BB7D.242C2C43191B2310203D3DE6F8BF495EAC8C1645&ratebypass=yes", + "vcodec": "vp9", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 1280, + "ext": "webm", + "filesize": 8034497, + "fps": 25, + "acodec": "none" + }, + { + "format_note": "720p", + "protocol": "https", + "format": "136 - 1280x720 (720p)", + "tbr": 1610.767, + "height": 720, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "136", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fmp4&gir=yes&ipbits=0&lmt=1507952890403137&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=136&mt=1535797130&dur=141.440&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=8054202&signature=2B28B5EC1C1EE32C0BE79537EE448B1BF0441A57.09D9F53E9F626EB8DB06508DCA3AA28B4C89D183&ratebypass=yes", + "vcodec": "avc1.4d401f", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 1280, + "ext": "mp4", + "filesize": 8054202, + "fps": 25, + "acodec": "none" + }, + { + "format_note": "1080p", + "protocol": "https", + "format": "248 - 1920x1080 (1080p)", + "tbr": 2347.36, + "height": 1080, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "248", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fwebm&gir=yes&ipbits=0&lmt=1503812340015025&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=248&mt=1535797130&dur=141.400&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=15062619&signature=D8366E08756AE8B241DD5264ECE8928F03ACD759.61D7374624D57DA33C551EA94285CB566FDE8C89&ratebypass=yes", + "vcodec": "vp9", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 1920, + "ext": "webm", + "filesize": 15062619, + "fps": 25, + "acodec": "none" + }, + { + "format_note": "1080p", + "protocol": "https", + "format": "137 - 1920x1080 (1080p)", + "tbr": 2757.086, + "height": 1080, + "downloader_options": { + "http_chunk_size": 10485760 + }, + "format_id": "137", + "quality": -1, + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=video%2Fmp4&gir=yes&ipbits=0&lmt=1507952977382584&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=137&mt=1535797130&dur=141.440&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&clen=14903843&signature=16960EBA740BA2C11F88FC697A42F7D0E52EA2EC.4AC8D17A3BED6B6E68AB92CFE3F598341A9D839A&ratebypass=yes", + "vcodec": "avc1.640028", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 1920, + "ext": "mp4", + "filesize": 14903843, + "fps": 25, + "acodec": "none" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "https", + "format": "17 - 176x144 (small)", + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?fvip=3&mime=video%2F3gpp&gir=yes&ipbits=0&lmt=1503811657072536&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=17&mt=1535797130&dur=141.548&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&clen=1187373&signature=4325E9F4DAC37FB0D61B0015DC0636E753C2C5E1.2A9BC9E6B8C94607A012EFCD3A2A134E807F25C3&ratebypass=yes", + "vcodec": "mp4v.20.3", + "format_note": "small", + "ext": "3gp", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 176, + "abr": 24, + "filesize": 1187373, + "format_id": "17", + "height": 144, + "resolution": "176x144", + "acodec": "mp4a.40.2" + }, + { + "protocol": "https", + "format": "36 - 320x180 (small)", + "format_note": "small", + "height": 180, + "format_id": "36", + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?fvip=3&mime=video%2F3gpp&gir=yes&ipbits=0&lmt=1503811652398125&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=36&mt=1535797130&dur=141.548&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&clen=3506033&signature=19E76DBDC1137B269267DAD172E63FB137D5EEC7.3578D812A3DFB2651B4E51522B948F1976299462&ratebypass=yes", + "vcodec": "mp4v.20.3", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 320, + "ext": "3gp", + "filesize": 3506033, + "resolution": "320x180", + "acodec": "mp4a.40.2" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "https", + "format": "18 - 640x360 (medium)", + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?fvip=3&mime=video%2Fmp4&gir=yes&ipbits=0&lmt=1503811655561063&ratebypass=yes&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=18&mt=1535797130&dur=141.502&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&clen=6322851&signature=098FAC77C4F9A641DF056D470EBE5A4D100FB915.47064BC9130CF024C716B65E414864B88DED5FC0", + "quality": 1, + "vcodec": "avc1.42001E", + "format_note": "medium", + "ext": "mp4", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 640, + "abr": 96, + "filesize": 6322851, + "format_id": "18", + "height": 360, + "resolution": "640x360", + "acodec": "mp4a.40.2" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "https", + "format": "43 - 640x360 (medium)", + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?fvip=3&mime=video%2Fwebm&gir=yes&ipbits=0&lmt=1503812251410108&ratebypass=yes&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=43&mt=1535797130&dur=0.000&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&clen=7126000&signature=2457DCD9E41AC8777B9BA0815AFB014BC644CD20.72712447936DB69925B02DEACF6576B9B568F4D4", + "quality": 1, + "vcodec": "vp8.0", + "format_note": "medium", + "ext": "webm", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 640, + "abr": 128, + "filesize": 7126000, + "format_id": "43", + "height": 360, + "resolution": "640x360", + "acodec": "vorbis" + }, + { + "http_headers": { + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language": "en-us,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0" + }, + "protocol": "https", + "format": "22 - 1280x720 (hd720)", + "url": "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?fvip=3&mime=video%2Fmp4&ipbits=0&lmt=1507953047549769&ratebypass=yes&key=yt6&requiressl=yes&c=WEB&expire=1535818875&ei=G2iKW_jiIMTjV6OjsqgF&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=22&source=youtube&dur=141.502&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=dur%2Cei%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cexpire&mt=1535797130&signature=1CE1646A209F0D00E8DD16784BAFCD9E0C9CF06C.0EF24C1CBF4A06AFEB7F404C3E28CE35B354F384", + "quality": 2, + "vcodec": "avc1.64001F", + "format_note": "hd720", + "ext": "mp4", + "player_url": "/yts/jsbin/player-vfliK45Zi/en_US/base.js", + "width": 1280, + "abr": 192, + "format_id": "22", + "height": 720, + "resolution": "1280x720", + "acodec": "mp4a.40.2" + } + ], + "resolution": "1280x720 (hd720)", + "width": 1280, + "age_limit": 0, + "_duration_raw": 142, + "_duration_hms": "00:02:22" +} diff --git a/packages/metascraper-media-provider/test/index.js b/packages/metascraper-media-provider/test/index.js index 91e15a474..a05f141e4 100644 --- a/packages/metascraper-media-provider/test/index.js +++ b/packages/metascraper-media-provider/test/index.js @@ -1,79 +1,66 @@ 'use strict' -const { isUrl } = require('@metascraper/helpers') -const { isString } = require('lodash') const snapshot = require('snap-shot') const { promisify } = require('util') const { resolve } = require('path') -const { omit } = require('lodash') -const should = require('should') const fs = require('fs') const metascraper = require('metascraper')([ - require('metascraper-media-provider')(), + require('metascraper-publisher')(), + require('..')(), require('metascraper-author')(), require('metascraper-date')(), require('metascraper-description')(), require('metascraper-image')(), require('metascraper-lang')(), require('metascraper-logo')(), - require('metascraper-publisher')(), require('metascraper-title')(), require('metascraper-url')() ]) const readFile = promisify(fs.readFile) -const { getVideo } = require('metascraper-media-provider') - -const output = require('./fixtures/youtube-dl.json') +const { getVideo } = require('..') describe('metascraper-media-provider', () => { - it('.getVideo', () => { - snapshot(getVideo(output.formats)) + describe('.getVideo', () => { + it('twitter', () => { + snapshot(getVideo(require('./fixtures/twitter.json'))) + }) + it('vimeo', () => { + snapshot(getVideo(require('./fixtures/vimeo.json'))) + }) + it('youtube', () => { + snapshot(getVideo(require('./fixtures/youtube.json'))) + }) }) describe('provider', () => { it('vimeo', async () => { const html = await readFile(resolve(__dirname, 'fixtures/vimeo.html')) const url = 'https://vimeo.com/188175573' const metadata = await metascraper({ html, url }) - should(isUrl(metadata.video)).be.true() - should(isString(metadata.title)).be.true() - const meta = omit(metadata, ['video', 'title']) - snapshot(meta) + snapshot(metadata) }) it('twitter', async () => { const html = await readFile(resolve(__dirname, 'fixtures/twitter.html')) const url = 'https://twitter.com/verge/status/957383241714970624' - const metadata = await metascraper({ html, url }) - should(isUrl(metadata.video)).be.true() - should(isString(metadata.title)).be.true() - const meta = omit(metadata, ['video', 'title']) - snapshot(meta) + snapshot(metadata) }) it('facebook', async () => { const html = await readFile(resolve(__dirname, 'fixtures/facebook.html')) const url = 'https://www.facebook.com/afcajax/videos/1686831701364171' - const metadata = await metascraper({ html, url }) - should(isUrl(metadata.video)).be.true() - should(isString(metadata.title)).be.true() - const meta = omit(metadata, ['video', 'title']) - snapshot(meta) + snapshot(metadata) }) it('youtube', async () => { const html = await readFile(resolve(__dirname, 'fixtures/youtube.html')) const url = 'https://www.youtube.com/watch?v=hwMkbaS_M_c' - const metadata = await metascraper({ html, url }) - should(isUrl(metadata.video)).be.true() - should(isString(metadata.title)).be.true() - const meta = omit(metadata, ['video', 'title']) - snapshot(meta) + snapshot(metadata) }) }) }) diff --git a/packages/metascraper-publisher/index.js b/packages/metascraper-publisher/index.js index c1989fcb1..5df295de9 100644 --- a/packages/metascraper-publisher/index.js +++ b/packages/metascraper-publisher/index.js @@ -1,13 +1,10 @@ 'use strict' -const { isString } = require('lodash') -const condenseWhitespace = require('condense-whitespace') +const { publisher } = require('@metascraper/helpers') const REGEX_RSS = /^(.*?)\s[-|]\satom$/i const REGEX_TITLE = /^.*?[-|]\s+(.*)$/ -const validator = value => isString(value) && condenseWhitespace(value) - /** * Wrap a rule with validation and formatting logic. * @@ -17,7 +14,7 @@ const validator = value => isString(value) && condenseWhitespace(value) const wrap = rule => ({ htmlDom }) => { const value = rule(htmlDom) - return validator(value) + return publisher(value) } const getFromTitle = (text, regex) => { @@ -53,5 +50,3 @@ module.exports = () => ({ wrap($ => getFromTitle($('link[type*="xml"]').attr('title'), REGEX_RSS)) ] }) - -module.exports.validator = validator diff --git a/packages/metascraper-publisher/package.json b/packages/metascraper-publisher/package.json index e189793ba..1bf7a57c5 100644 --- a/packages/metascraper-publisher/package.json +++ b/packages/metascraper-publisher/package.json @@ -16,8 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "condense-whitespace": "~1.0.0", - "lodash": "~4.17.10" + "@metascraper/helpers": "^4.0.1" }, "devDependencies": { "mocha": "latest", diff --git a/packages/metascraper-soundcloud/index.js b/packages/metascraper-soundcloud/index.js index adb893b8e..1bb87fc6d 100644 --- a/packages/metascraper-soundcloud/index.js +++ b/packages/metascraper-soundcloud/index.js @@ -1,19 +1,19 @@ 'use strict' -const {getValue, titleize} = require('@metascraper/helpers') +const { $filter, title } = require('@metascraper/helpers') module.exports = () => ({ author: [ - ({htmlDom: $, meta, url: baseUrl}) => - titleize(getValue($, $('.soundTitle__username'))) + ({ htmlDom: $, meta, url: baseUrl }) => + title($filter($, $('.soundTitle__username'))) ], description: [ - ({htmlDom: $, meta, url: baseUrl}) => - titleize( + ({ htmlDom: $, meta, url: baseUrl }) => + title( $('.soundTitle__description') .first() .text(), - {capitalize: false} + { capitalize: false } ) ] }) diff --git a/packages/metascraper-title/index.js b/packages/metascraper-title/index.js index 347f9fed4..6cf49fcdc 100644 --- a/packages/metascraper-title/index.js +++ b/packages/metascraper-title/index.js @@ -1,13 +1,10 @@ 'use strict' -const { getValue, titleize } = require('@metascraper/helpers') -const { isString } = require('lodash') - -const validator = value => isString(value) && titleize(value) +const { $filter, title } = require('@metascraper/helpers') const wrap = rule => ({ htmlDom }) => { const value = rule(htmlDom) - return validator(value) + return title(value) } module.exports = () => ({ @@ -18,8 +15,6 @@ module.exports = () => ({ wrap($ => $('.entry-title').text()), wrap($ => $('h1[class*="title"] a').text()), wrap($ => $('h1[class*="title"]').text()), - wrap($ => getValue($, $('title'))) + wrap($ => $filter($, $('title'))) ] }) - -module.exports.validator = wrap diff --git a/packages/metascraper-url/index.js b/packages/metascraper-url/index.js index 89ed09a46..cfb6ae025 100644 --- a/packages/metascraper-url/index.js +++ b/packages/metascraper-url/index.js @@ -1,8 +1,6 @@ 'use strict' -const { getUrl, isUrl } = require('@metascraper/helpers') - -const validator = (value, url) => isUrl(value) && getUrl(url, value) +const { url: urlFn } = require('@metascraper/helpers') /** * Wrap a rule with validation and formatting logic. @@ -13,7 +11,7 @@ const validator = (value, url) => isUrl(value) && getUrl(url, value) const wrap = rule => ({ htmlDom, url }) => { const value = rule(htmlDom) - return validator(value, url) + return urlFn(value, { url }) } /** @@ -29,5 +27,3 @@ module.exports = () => ({ ({ url }) => url ] }) - -module.exports.validator = validator diff --git a/packages/metascraper-video/index.js b/packages/metascraper-video/index.js index 478a0e0bf..78b45c3fe 100644 --- a/packages/metascraper-video/index.js +++ b/packages/metascraper-video/index.js @@ -1,15 +1,6 @@ 'use strict' -const { getUrl, isUrl } = require('@metascraper/helpers') -const videoExtensions = require('video-extensions') -const { URL } = require('url') -const path = require('path') - -const isVideoUrl = url => { - const { pathname } = new URL(url) - const ext = path.extname(pathname).substring(1) - return videoExtensions.includes(ext) -} +const { url: urlFn, isVideoUrl } = require('@metascraper/helpers') /** * Wrap a rule with validation and formatting logic. @@ -23,24 +14,19 @@ const createWrapper = fn => rule => ({ htmlDom, url }) => { return fn(value, url) } -const wrap = createWrapper((value, url) => isUrl(value) && getUrl(url, value)) +const wrap = createWrapper((value, url) => urlFn(value, { url })) -const validator = (value, url) => { - if (!isUrl(value)) return false - const urlValue = getUrl(url, value) +const wrapVideo = createWrapper((value, url) => { + const urlValue = urlFn(value, { url }) return isVideoUrl(urlValue) && urlValue -} - -const wrapVideo = createWrapper(validator) +}) /** * Rules. */ module.exports = () => ({ - image: [ - wrap($ => $('video').attr('poster')) - ], + image: [wrap($ => $('video').attr('poster'))], video: [ wrapVideo($ => $('meta[property="og:video:secure_url"]').attr('content')), wrapVideo($ => $('meta[property="og:video:url"]').attr('content')), @@ -50,5 +36,3 @@ module.exports = () => ({ wrapVideo($ => $('source').attr('src')) ] }) - -module.exports.validator = validator diff --git a/packages/metascraper-video/package.json b/packages/metascraper-video/package.json index 091bd7caf..32d2dd700 100644 --- a/packages/metascraper-video/package.json +++ b/packages/metascraper-video/package.json @@ -16,8 +16,7 @@ "url": "https://github.com/microlinkhq/metascraper/issues" }, "dependencies": { - "@metascraper/helpers": "^4.0.1", - "video-extensions": "~1.1.0" + "@metascraper/helpers": "^4.0.1" }, "devDependencies": { "lodash": "latest", diff --git a/packages/metascraper-video/test/index.js b/packages/metascraper-video/test/index.js index 437dacf43..7a25274b2 100644 --- a/packages/metascraper-video/test/index.js +++ b/packages/metascraper-video/test/index.js @@ -7,7 +7,7 @@ const { omit } = require('lodash') const fs = require('fs') const metascraper = require('metascraper')([ - require('metascraper-video')(), + require('..')(), require('metascraper-author')(), require('metascraper-date')(), require('metascraper-description')(), diff --git a/packages/metascraper-youtube/index.js b/packages/metascraper-youtube/index.js index e2fded236..e2a310fbb 100644 --- a/packages/metascraper-youtube/index.js +++ b/packages/metascraper-youtube/index.js @@ -1,10 +1,8 @@ 'use strict' -const getVideoId = require('get-video-id') - -const { getValue, isUrl, titleize } = require('@metascraper/helpers') +const { $filter, author } = require('@metascraper/helpers') const isReachable = require('is-reachable') -const { isString } = require('lodash') +const getVideoId = require('get-video-id') const pLocate = require('p-locate') const THUMBAILS_RESOLUTIONS = [ @@ -16,29 +14,27 @@ const THUMBAILS_RESOLUTIONS = [ ] const getThumbnailUrl = id => { - const urls = THUMBAILS_RESOLUTIONS.map(res => `https://img.youtube.com/vi/${id}/${res}`) + const urls = THUMBAILS_RESOLUTIONS.map( + res => `https://img.youtube.com/vi/${id}/${res}` + ) return pLocate(urls, isReachable) } const wrap = rule => ({ htmlDom }) => { const value = rule(htmlDom) - return isString(value) && !isUrl(value, {relative: false}) && titleize( - value, {removeBy: true} - ) + return author(value) } module.exports = () => ({ author: [ wrap($ => $('#owner-name').text()), wrap($ => $('#channel-title').text()), - wrap($ => getValue($, $('[class*="user-info"]'))) - ], - publisher: [ - ({url}) => getVideoId(url).service === 'youtube' && 'YouTube' + wrap($ => $filter($, $('[class*="user-info"]'))) ], + publisher: [({ url }) => getVideoId(url).service === 'youtube' && 'YouTube'], image: [ ({ htmlDom, url }) => { - const {id, service} = getVideoId(url) + const { id, service } = getVideoId(url) return service === 'youtube' && id && getThumbnailUrl(id) } ] diff --git a/packages/metascraper-youtube/package.json b/packages/metascraper-youtube/package.json index a42cb1dce..e5098a0e6 100644 --- a/packages/metascraper-youtube/package.json +++ b/packages/metascraper-youtube/package.json @@ -24,7 +24,6 @@ "@metascraper/helpers": "^4.0.1", "get-video-id": "~3.1.0", "is-reachable": "~2.4.0", - "lodash": "~4.17.10", "p-locate": "~3.0.0" }, "devDependencies": { From 19405c288d25fcb0c4e7ada1ef898b8c6455f6a9 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sat, 1 Sep 2018 14:11:27 +0200 Subject: [PATCH 035/442] Video is a collection --- packages/metascraper-helpers/index.js | 44 ++++++------- packages/metascraper-logo/index.js | 4 +- .../test/fixtures/{ => video}/twitter.json | 0 .../test/fixtures/{ => video}/vimeo.json | 0 .../test/fixtures/{ => video}/youtube.json | 0 .../metascraper-media-provider/test/index.js | 6 +- .../__snapshots__/index.js.snap-shot | 66 +++++++++++-------- packages/metascraper-video/index.js | 4 +- packages/metascraper-video/test/index.js | 37 ++++++++--- .../__snapshots__/index.js.snap-shot | 3 +- 10 files changed, 95 insertions(+), 69 deletions(-) rename packages/metascraper-media-provider/test/fixtures/{ => video}/twitter.json (100%) rename packages/metascraper-media-provider/test/fixtures/{ => video}/vimeo.json (100%) rename packages/metascraper-media-provider/test/fixtures/{ => video}/youtube.json (100%) diff --git a/packages/metascraper-helpers/index.js b/packages/metascraper-helpers/index.js index 39fb96f56..f0a7341a2 100644 --- a/packages/metascraper-helpers/index.js +++ b/packages/metascraper-helpers/index.js @@ -1,8 +1,16 @@ 'use strict' -const { toLower, replace, includes, isString, trim, flow, isEmpty } = require('lodash') +const { + toLower, + replace, + includes, + isString, + trim, + flow, + isEmpty +} = require('lodash') const condenseWhitespace = require('condense-whitespace') -const videoExtensions = require('video-extensions') +const videoExtensions = require('video-extensions').concat(['gif']) const audioExtensions = require('audio-extensions') const isRelativeUrl = require('is-relative-url') const fileExtension = require('file-extension') @@ -21,17 +29,15 @@ const REGEX_LOCATION = /^[A-Z\s]+\s+[-—–]\s+/ const removeLocation = value => replace(value, REGEX_LOCATION, '') -const urlTest = (url, { relative = true }) => relative - ? isRelativeUrl(url) || urlRegex().test(url) - : urlRegex().test(url) +const urlTest = (url, { relative = true }) => + relative ? isRelativeUrl(url) || urlRegex().test(url) : urlRegex().test(url) const isUrl = (url, opts = {}) => !isEmpty(url) && urlTest(url, opts) -const absoluteUrl = (baseUrl, relativePath = '') => ( +const absoluteUrl = (baseUrl, relativePath = '') => resolveUrl(baseUrl, relativePath) -) -const sanetizeUrl = (url, opts) => ( +const sanetizeUrl = (url, opts) => _normalizeUrl(url, { normalizeHttp: false, stripWWW: false, @@ -39,16 +45,11 @@ const sanetizeUrl = (url, opts) => ( removeTrailingSlash: false, ...opts }) -) -const normalizeUrl = (baseUrl, relativePath, opts) => ( +const normalizeUrl = (baseUrl, relativePath, opts) => sanetizeUrl(absoluteUrl(baseUrl, relativePath), opts) -) -const removeByPrefix = flow([ - value => value.replace(REGEX_BY, ''), - trim -]) +const removeByPrefix = flow([value => value.replace(REGEX_BY, ''), trim]) const createTitle = flow([condenseWhitespace, smartquotes]) @@ -61,14 +62,13 @@ const titleize = (src, { capitalize = false, removeBy = false } = {}) => { const defaultFn = el => el.text().trim() -const $filter = ($, collection, fn = defaultFn) => { - const el = collection.filter((i, el) => fn($(el))).first() +const $filter = ($, domNodes, fn = defaultFn) => { + const el = domNodes.filter((i, el) => fn($(el))).first() return fn(el) } -const isAuthor = (str, opts = { relative: false }) => ( +const isAuthor = (str, opts = { relative: false }) => isString(str) && !isUrl(str, opts) -) const getAuthor = (str, opts = { removeBy: true }) => titleize(str, opts) @@ -77,9 +77,8 @@ const protocol = url => { return protocol.replace(':', '') } -const createUrlExtensionValidator = collection => url => ( +const createUrlExtensionValidator = collection => url => isUrl(url) && includes(collection, extension(url)) -) const isVideoUrl = createUrlExtensionValidator(videoExtensions) @@ -89,7 +88,8 @@ const extension = url => fileExtension(url).split('?')[0] const description = value => isString(value) && getDescription(value) -const getDescription = value => titleize(removeLocation(value), { capitalize: false }) +const getDescription = value => + titleize(removeLocation(value), { capitalize: false }) const publisher = value => isString(value) && condenseWhitespace(value) diff --git a/packages/metascraper-logo/index.js b/packages/metascraper-logo/index.js index 74b9462ed..11c23eb5f 100644 --- a/packages/metascraper-logo/index.js +++ b/packages/metascraper-logo/index.js @@ -16,8 +16,8 @@ const getDomNodeSizes = (domNodes, attr) => const getSizes = ($, collection) => chain(collection) .reduce((acc, { tag, attr }) => { - const domNode = $(tag).get() - const selectors = getDomNodeSizes(domNode, attr) + const domNodes = $(tag).get() + const selectors = getDomNodeSizes(domNodes, attr) return concat(acc, selectors) }, []) .sortBy(({ size }) => -size) diff --git a/packages/metascraper-media-provider/test/fixtures/twitter.json b/packages/metascraper-media-provider/test/fixtures/video/twitter.json similarity index 100% rename from packages/metascraper-media-provider/test/fixtures/twitter.json rename to packages/metascraper-media-provider/test/fixtures/video/twitter.json diff --git a/packages/metascraper-media-provider/test/fixtures/vimeo.json b/packages/metascraper-media-provider/test/fixtures/video/vimeo.json similarity index 100% rename from packages/metascraper-media-provider/test/fixtures/vimeo.json rename to packages/metascraper-media-provider/test/fixtures/video/vimeo.json diff --git a/packages/metascraper-media-provider/test/fixtures/youtube.json b/packages/metascraper-media-provider/test/fixtures/video/youtube.json similarity index 100% rename from packages/metascraper-media-provider/test/fixtures/youtube.json rename to packages/metascraper-media-provider/test/fixtures/video/youtube.json diff --git a/packages/metascraper-media-provider/test/index.js b/packages/metascraper-media-provider/test/index.js index a05f141e4..f83b143a8 100644 --- a/packages/metascraper-media-provider/test/index.js +++ b/packages/metascraper-media-provider/test/index.js @@ -25,13 +25,13 @@ const { getVideo } = require('..') describe('metascraper-media-provider', () => { describe('.getVideo', () => { it('twitter', () => { - snapshot(getVideo(require('./fixtures/twitter.json'))) + snapshot(getVideo(require('./fixtures/video/twitter.json'))) }) it('vimeo', () => { - snapshot(getVideo(require('./fixtures/vimeo.json'))) + snapshot(getVideo(require('./fixtures/video/vimeo.json'))) }) it('youtube', () => { - snapshot(getVideo(require('./fixtures/youtube.json'))) + snapshot(getVideo(require('./fixtures/video/youtube.json'))) }) }) describe('provider', () => { diff --git a/packages/metascraper-video/__snapshots__/index.js.snap-shot b/packages/metascraper-video/__snapshots__/index.js.snap-shot index f87781c6d..b3fda8065 100644 --- a/packages/metascraper-video/__snapshots__/index.js.snap-shot +++ b/packages/metascraper-video/__snapshots__/index.js.snap-shot @@ -1,19 +1,8 @@ -exports['video src 1'] = { - "image": "https://cdn.vox-cdn.com/thumbor/AtQQMyWrexi6-Xyk73jv6nqTO7s=/0x5:1247x658/fit-in/1200x630/cdn.vox-cdn.com/uploads/chorus_asset/file/10079811/Screen_Shot_2018_01_22_at_3.27.50_PM.png", - "video": null, - "author": "Rachel Becker", - "date": "2018-01-22T23:38:17.000Z", - "description": "The zombies are only released on the weekends, the developers promise", - "lang": null, - "logo": "https://cdn.vox-cdn.com/uploads/chorus_asset/file/7395351/android-chrome-192x192.0.png", - "publisher": "The Verge", - "title": "You can visit the Pentagon’s secret nuclear bunker inside Minecraft", - "url": "https://www.theverge.com/2018/1/22/16921092/pentagon-secret-nuclear-bunker-reconstruction-minecraft-cns-miis-model" -} - -exports['source src 1'] = { +exports[' 1'] = { "image": "https://img-9gag-fun.9cache.com/photo/aGjVLDK_460s.jpg", - "video": "https://img-9gag-fun.9cache.com/photo/aGjVLDK_460sv.mp4", + "video": [ + "https://img-9gag-fun.9cache.com/photo/aGjVLDK_460sv.mp4" + ], "author": null, "date": null, "description": "Watch the video and the fun convo of the 9GAG community", @@ -37,22 +26,26 @@ exports['og:video 1'] = { "url": "https://twitter.com/_developit/status/955905369242513414" } -exports['src:poster 1'] = { - "image": "https://thumbs.gfycat.com/TimelyHealthyArmadillo-mobile.jpg", - "video": "https://thumbs.gfycat.com/TimelyHealthyArmadillo-mobile.mp4", - "author": "Gfycat", - "date": null, - "description": "Watch Backflip GIF on Gfycat. Discover more PUBG GIFs on Gfycat", - "lang": "en", - "logo": "https://gfycat.com/static/apple-touch-icon/apple-touch-icon-180x180.png", - "publisher": "Gfycat", - "title": "Backflip - Create, Discover and Share Awesome GIFs on Gfycat", - "url": "https://thumbs.gfycat.com/TimelyHealthyArmadillo-size_restricted.gif" +exports['single src 1'] = { + "image": "https://cdn.vox-cdn.com/thumbor/AtQQMyWrexi6-Xyk73jv6nqTO7s=/0x5:1247x658/fit-in/1200x630/cdn.vox-cdn.com/uploads/chorus_asset/file/10079811/Screen_Shot_2018_01_22_at_3.27.50_PM.png", + "video": [ + "https://cdn.vox-cdn.com/thumbor/4l0C-7uGFtTfc6lWibo1ITiE2YU=/0x0:1280x720/320x213/filters:focal(538x258:742x462):gifv():no_upscale()/cdn.vox-cdn.com/uploads/chorus_image/image/58416873/2018_01_22_14_19_55.0.gif" + ], + "author": "Rachel Becker", + "date": "2018-01-22T23:38:17.000Z", + "description": "The zombies are only released on the weekends, the developers promise", + "lang": null, + "logo": "https://cdn.vox-cdn.com/uploads/chorus_asset/file/7395351/android-chrome-192x192.0.png", + "publisher": "The Verge", + "title": "You can visit the Pentagon’s secret nuclear bunker inside Minecraft", + "url": "https://www.theverge.com/2018/1/22/16921092/pentagon-secret-nuclear-bunker-reconstruction-minecraft-cns-miis-model" } exports['clips.twitch.tv 1'] = { "image": "https://clips-media-assets.twitch.tv/27434665136-offset-2366-preview.jpg", - "video": "https://clips-media-assets.twitch.tv/AT-27434665136-offset-2366-1280x720.mp4", + "video": [ + "https://clips-media-assets.twitch.tv/AT-27434665136-offset-2366-1280x720.mp4" + ], "author": null, "date": null, "description": "Shroud with the casual coffee sip to kill combo - Clipped by jpan11", @@ -65,7 +58,9 @@ exports['clips.twitch.tv 1'] = { exports['play.tv 1'] = { "image": "https://d1playscdntv-a.akamaihd.net/video/Ha35bprkDYG/processed/720.jpg", - "video": "https://d1playscdntv-a.akamaihd.net/video/Ha35bprkDYG/processed/480.mp4", + "video": [ + "https://d1playscdntv-a.akamaihd.net/video/Ha35bprkDYG/processed/480.mp4" + ], "author": "chineseouchie", "description": "Publicado por chineseouchie", "lang": "en", @@ -75,3 +70,18 @@ exports['play.tv 1'] = { "url": "https://plays.tv/video/5a6f64b1bef69a7fa9/holy-shit" } +exports['src:poster 1'] = { + "image": "https://thumbs.gfycat.com/TimelyHealthyArmadillo-mobile.jpg", + "video": [ + "https://thumbs.gfycat.com/TimelyHealthyArmadillo-mobile.mp4" + ], + "author": "Gfycat", + "date": null, + "description": "Watch Backflip GIF on Gfycat. Discover more PUBG GIFs on Gfycat", + "lang": "en", + "logo": "https://gfycat.com/static/apple-touch-icon/apple-touch-icon-180x180.png", + "publisher": "Gfycat", + "title": "Backflip - Create, Discover and Share Awesome GIFs on Gfycat", + "url": "https://thumbs.gfycat.com/TimelyHealthyArmadillo-size_restricted.gif" +} + diff --git a/packages/metascraper-video/index.js b/packages/metascraper-video/index.js index 78b45c3fe..a81f3a6b2 100644 --- a/packages/metascraper-video/index.js +++ b/packages/metascraper-video/index.js @@ -18,7 +18,7 @@ const wrap = createWrapper((value, url) => urlFn(value, { url })) const wrapVideo = createWrapper((value, url) => { const urlValue = urlFn(value, { url }) - return isVideoUrl(urlValue) && urlValue + return isVideoUrl(urlValue) && [urlValue] }) /** @@ -33,6 +33,6 @@ module.exports = () => ({ wrapVideo($ => $('meta[property="og:video"]').attr('content')), wrapVideo($ => $('meta[property="twitter:player:stream"]').attr('content')), wrapVideo($ => $('video').attr('src')), - wrapVideo($ => $('source').attr('src')) + wrapVideo($ => $('video > source').attr('src')) ] }) diff --git a/packages/metascraper-video/test/index.js b/packages/metascraper-video/test/index.js index 7a25274b2..9757c3057 100644 --- a/packages/metascraper-video/test/index.js +++ b/packages/metascraper-video/test/index.js @@ -23,16 +23,34 @@ const readFile = promisify(fs.readFile) describe('metascraper-video', () => { describe('video', () => { - it('video src', async () => { - const html = await readFile(resolve(__dirname, 'fixtures/video-src.html')) - const url = - 'https://www.theverge.com/2018/1/22/16921092/pentagon-secret-nuclear-bunker-reconstruction-minecraft-cns-miis-model' - - const metadata = await metascraper({ html, url }) - snapshot(metadata) + describe('