Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[YouTube] Workaround again 403 HTTP issues on the ANDROID InnerTube client and fix stream tests #1084

Merged
merged 3 commits into from
Jul 31, 2023
Merged

[YouTube] Workaround again 403 HTTP issues on the ANDROID InnerTube client and fix stream tests #1084

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
164c8e3
[YouTube] Workaround again 403 HTTP issues on the Android client by u…
AudricV Jul 22, 2023
7528eb2
[YouTube] Fix stream tests failures
AudricV Jul 23, 2023
522c781
[YouTube] Update stream tests mocks
AudricV Jul 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions ...java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -981,11 +981,9 @@ private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentC
.value(CPN, androidCpn)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
// Workaround getting streaming URLs which can return 403 HTTP response
// codes by using stories parameter for Android client requests
// This behavior only happen in certain countries such as UK as of
// 10.29.2022
.value("params", "8AEB")
// Workaround getting streaming URLs which return 403 HTTP response code by
// using some parameters for Android client requests
.value("params", "CgIQBg")
.done())
.getBytes(StandardCharsets.UTF_8);

Expand Down
43 changes: 21 additions & 22 deletions ...g/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ public void testTags() {}
}

public static class PublicBroadcasterTest extends DefaultStreamExtractorTest {
private static final String ID = "q6fgbYWsMgw";
private static final String ID = "cJ9to6EmElQ";
private static final int TIMESTAMP = 0;
private static final String URL = BASE_URL + ID;
private static StreamExtractor extractor;
Expand All @@ -396,41 +396,39 @@ public static void setUp() throws Exception {
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Was verbirgt sich am tiefsten Punkt des Ozeans?"; }
@Override public String expectedName() { return "Merci pour les 3 millions d'abonnés \uD83C\uDF89| ARTE"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return BASE_URL + ID; }
@Override public String expectedOriginalUrlContains() { return URL; }

@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "Dinge Erklärt – Kurzgesagt"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCwRH985XgMYXQ6NxXDo8npw"; }
@Override public long expectedUploaderSubscriberCountAtLeast() { return 1_500_000; }
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("Lasst uns abtauchen!", "Angebot von funk", "Dinge"); }
@Override public long expectedLength() { return 631; }
@Override public String expectedUploaderName() { return "ARTE"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCwI-JbGNsojunnHbFAc0M4Q"; }
@Override public long expectedUploaderSubscriberCountAtLeast() { return 3_000_000; }
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("sommets", "fans", "cadeau"); }
@Override public long expectedLength() { return 45; }
@Override public long expectedTimestamp() { return TIMESTAMP; }
@Override public long expectedViewCountAtLeast() { return 1_600_000; }
@Nullable @Override public String expectedUploadDate() { return "2019-06-12 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2019-06-12"; }
@Override public long expectedLikeCountAtLeast() { return 70000; }
@Override public long expectedViewCountAtLeast() { return 20_000; }
@Nullable @Override public String expectedUploadDate() { return "2023-07-07 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2023-07-07"; }
@Override public long expectedLikeCountAtLeast() { return 1000; }
@Override public long expectedDislikeCountAtLeast() { return -1; }
@Override public List<MetaInfo> expectedMetaInfo() throws MalformedURLException {
return Collections.singletonList(new MetaInfo(
"",
new Description("Funk is a German public broadcast service.", Description.PLAIN_TEXT),
Collections.singletonList(new URL("https://de.wikipedia.org/wiki/Funk_(Medienangebot)?wprov=yicw1")),
Collections.singletonList("Wikipedia (German)")
new Description("Arte is a French/German public broadcast service.",
Description.PLAIN_TEXT),
List.of(new URL(
"https://en.wikipedia.org/wiki/Arte?wprov=yicw1")),
List.of("Wikipedia")
));
}
@Override public boolean expectedUploaderVerified() { return true; }
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
@Override public String expectedCategory() { return "Education"; }
@Override public String expectedCategory() { return "News & Politics"; }
@Override public List<String> expectedTags() {
return Arrays.asList("Abgrund", "Algen", "Bakterien", "Challengertief", "Dumbooktopus",
"Dunkel", "Dunkelheit", "Fische", "Flohkrebs", "Hadal-Zone", "Kontinentalschelf",
"Licht", "Mariannengraben", "Meer", "Meeresbewohner", "Meeresschnee", "Mesopelagial",
"Ozean", "Photosynthese", "Plankton", "Plastik", "Polypen", "Pottwale",
"Staatsquelle", "Tauchen", "Tauchgang", "Tentakel", "Tiefe", "Tiefsee", "Tintenfische",
"Titanic", "Vampirtintenfisch", "Verschmutzung", "Viperfisch", "Wale");
return Arrays.asList("arte", "arte 3 millions", "arte remerciement",
"documentaire arte", "arte documentaire", "fan d'arte", "arte youtube");
}
// @formatter:on
}
Expand Down Expand Up @@ -559,7 +557,8 @@ void testCheckAudioStreams() throws Exception {
}

assertTrue(audioStreams.stream()
.anyMatch(audioStream -> "English".equals(audioStream.getAudioTrackName())));
.anyMatch(audioStream ->
"English original".equals(audioStream.getAudioTrackName())));

final Locale hindiLocale = LocaleCompat.forLanguageTag("hi");
assertTrue(audioStreams.stream()
Expand Down
2 changes: 1 addition & 1 deletion ...chabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public void testUploaderName() throws Exception {

@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "lofi hip hop radio - beats to relax/study to"; }
@Override public String expectedName() { return "lofi hip hop radio \uD83D\uDCDA - beats to relax/study to"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return YoutubeStreamExtractorDefaultTest.BASE_URL + ID; }
@Override public String expectedOriginalUrlContains() { return URL; }
Expand Down
23 changes: 15 additions & 8 deletions ...i/newpipe/extractor/services/youtube/extractor/stream/ageRestricted/generated_mock_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,15 @@
"responseMessage": "",
"responseHeaders": {
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
],
"cache-control": [
"private, max-age\u003d0"
],
"content-security-policy-report-only": [
"base-uri \u0027self\u0027;default-src \u0027self\u0027 https: blob:;font-src https: data:;img-src https: data: android-webview-video-poster:;media-src blob: https:;object-src \u0027none\u0027;script-src \u0027nonce-YdAQShUg0To_xhoSabl-vQ\u0027 \u0027unsafe-inline\u0027 \u0027strict-dynamic\u0027 https: http: \u0027unsafe-eval\u0027;style-src https: \u0027unsafe-inline\u0027;report-uri /cspreport",
"require-trusted-types-for \u0027script\u0027;report-uri /cspreport"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
Expand All @@ -32,16 +36,19 @@
"cross-origin"
],
"date": [
"Tue, 22 Nov 2022 10:41:17 GMT"
"Sat, 22 Jul 2023 18:24:10 GMT"
],
"expires": [
"Tue, 22 Nov 2022 10:41:17 GMT"
"Sat, 22 Jul 2023 18:24:10 GMT"
],
"origin-trial": [
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-form-factor\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
],
"report-to": [
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
Expand All @@ -50,9 +57,9 @@
"ESF"
],
"set-cookie": [
"YSC\u003dARiIxvc7aiA; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d2gvEtKcFruk; Domain\u003d.youtube.com; Expires\u003dSun, 21-May-2023 10:41:17 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+076; expires\u003dThu, 21-Nov-2024 10:41:17 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
"YSC\u003dRBncugXb0cY; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003dF1G_Uux5N7o; Domain\u003d.youtube.com; Expires\u003dThu, 18-Jan-2024 18:24:10 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+446; expires\u003dMon, 21-Jul-2025 18:24:10 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
Expand All @@ -67,7 +74,7 @@
"0"
]
},
"responseBody": "var scriptUrl \u003d \u0027https:\\/\\/www.youtube.com\\/s\\/player\\/041a7965\\/www-widgetapi.vflset\\/www-widgetapi.js\u0027;window[\u0027yt_embedsEnableIframeSrcWithIntent\u0027] \u003d true ;try{var ttPolicy\u003dwindow.trustedTypes.createPolicy(\"youtube-widget-api\",{createScriptURL:function(x){return x}});scriptUrl\u003dttPolicy.createScriptURL(scriptUrl)}catch(e){}var YT;if(!window[\"YT\"])YT\u003d{loading:0,loaded:0};var YTConfig;if(!window[\"YTConfig\"])YTConfig\u003d{\"host\":\"https://www.youtube.com\"};\nif(!YT.loading){YT.loading\u003d1;(function(){var l\u003d[];YT.ready\u003dfunction(f){if(YT.loaded)f();else l.push(f)};window.onYTReady\u003dfunction(){YT.loaded\u003d1;for(var i\u003d0;i\u003cl.length;i++)try{l[i]()}catch(e$0){}};YT.setConfig\u003dfunction(c){for(var k in c)if(c.hasOwnProperty(k))YTConfig[k]\u003dc[k]};var a\u003ddocument.createElement(\"script\");a.type\u003d\"text/javascript\";a.id\u003d\"www-widgetapi-script\";a.src\u003dscriptUrl;a.async\u003dtrue;var c\u003ddocument.currentScript;if(c){var n\u003dc.nonce||c.getAttribute(\"nonce\");if(n)a.setAttribute(\"nonce\",n)}var b\u003d\ndocument.getElementsByTagName(\"script\")[0];b.parentNode.insertBefore(a,b)})()};\n",
"responseBody": "var scriptUrl \u003d \u0027https:\\/\\/www.youtube.com\\/s\\/player\\/8e83803a\\/www-widgetapi.vflset\\/www-widgetapi.js\u0027;try{var ttPolicy\u003dwindow.trustedTypes.createPolicy(\"youtube-widget-api\",{createScriptURL:function(x){return x}});scriptUrl\u003dttPolicy.createScriptURL(scriptUrl)}catch(e){}var YT;if(!window[\"YT\"])YT\u003d{loading:0,loaded:0};var YTConfig;if(!window[\"YTConfig\"])YTConfig\u003d{\"host\":\"https://www.youtube.com\"};\nif(!YT.loading){YT.loading\u003d1;(function(){var l\u003d[];YT.ready\u003dfunction(f){if(YT.loaded)f();else l.push(f)};window.onYTReady\u003dfunction(){YT.loaded\u003d1;var i\u003d0;for(;i\u003cl.length;i++)try{l[i]()}catch(e$1271022551$0){}};YT.setConfig\u003dfunction(c){var k;for(k in c)if(c.hasOwnProperty(k))YTConfig[k]\u003dc[k]};var a\u003ddocument.createElement(\"script\");a.type\u003d\"text/javascript\";a.id\u003d\"www-widgetapi-script\";a.src\u003dscriptUrl;a.async\u003dtrue;var c\u003ddocument.currentScript;if(c){var n\u003dc.nonce||c.getAttribute(\"nonce\");if(n)a.setAttribute(\"nonce\",\nn)}var b\u003ddocument.getElementsByTagName(\"script\")[0];b.parentNode.insertBefore(a,b)})()};\n",
"latestUrl": "https://www.youtube.com/iframe_api"
}
}
19 changes: 8 additions & 11 deletions ...i/newpipe/extractor/services/youtube/extractor/stream/ageRestricted/generated_mock_1.json
View file
Open in desktop

Large diffs are not rendered by default.

19 changes: 11 additions & 8 deletions ...i/newpipe/extractor/services/youtube/extractor/stream/ageRestricted/generated_mock_2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,28 +29,31 @@
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
],
"cache-control": [
"private, max-age\u003d0"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"cross-origin-opener-policy": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Tue, 22 Nov 2022 10:41:18 GMT"
"Sat, 22 Jul 2023 18:24:11 GMT"
],
"expires": [
"Tue, 22 Nov 2022 10:41:18 GMT"
"Sat, 22 Jul 2023 18:24:11 GMT"
],
"origin-trial": [
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-form-factor\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
],
"report-to": [
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
Expand All @@ -59,9 +62,9 @@
"ESF"
],
"set-cookie": [
"YSC\u003dqOvGL5EH0kk; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 26-Feb-2020 10:41:18 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+190; expires\u003dThu, 21-Nov-2024 10:41:18 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
"YSC\u003dKzlu5knnLDg; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSun, 25-Oct-2020 18:24:11 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+519; expires\u003dMon, 21-Jul-2025 18:24:11 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
Expand Down
17 changes: 10 additions & 7 deletions ...i/newpipe/extractor/services/youtube/extractor/stream/ageRestricted/generated_mock_3.json
View file
Open in desktop

Large diffs are not rendered by default.

Loading
Loading