Skip to content

Commit

Permalink
[YouTube] Workaround HTTP 403s on streaming URLs of WEB client (after…
Browse files Browse the repository at this point in the history 
… some time or instantly for some JavaScript players), update clients info (#51)

* [YouTube] Workaround HTTP 403s on streaming URLs of WEB client (after some time or instantly for some JavaScript players), update clients info

Co-Authored-By: Audric V. <74829229+audricv@users.noreply.github.com>
Co-Authored-By: InfinityLoop1308 <96324692+infinityloop1308@users.noreply.github.com>

* Update YoutubeStreamExtractor.java

* Update YoutubeParsingHelper.java

* Update YoutubeParsingHelper.java

---------

Co-authored-by: Audric V. <74829229+audricv@users.noreply.github.com>
Co-authored-by: InfinityLoop1308 <96324692+infinityloop1308@users.noreply.github.com>
  • Loading branch information
@ShareASmile @AudricV @InfinityLoop1308
3 people authored Jul 28, 2024
1 parent 8c8ddd6 commit 55d9207
Show file tree
Hide file tree
Showing 4 changed files with 204 additions and 125 deletions.
56 changes: 36 additions & 20 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ private YoutubeParsingHelper() {
* The client version for InnerTube requests with the {@code WEB} client, used as the last
* fallback if the extraction of the real one failed.
*/
private static final String HARDCODED_CLIENT_VERSION = "2.20220809.02.00";
private static final String HARDCODED_CLIENT_VERSION = "2.20240718.01.00";

/**
* The InnerTube API key which should be used by YouTube's desktop website, used as a fallback
Expand All @@ -169,7 +169,7 @@ private YoutubeParsingHelper() {
* such as <a href="https://www.apkmirror.com/apk/google-inc/youtube/">APKMirror</a>.
* </p>
*/
private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "17.31.35";
private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "19.28.35";

/**
* The InnerTube API key used by the {@code ANDROID} client. Found with the help of
Expand All @@ -187,7 +187,7 @@ private YoutubeParsingHelper() {
* Store page of the YouTube app</a>, in the {@code What’s New} section.
* </p>
*/
private static final String IOS_YOUTUBE_CLIENT_VERSION = "17.31.4";
private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.28.1";

/**
* The InnerTube API key used by the {@code iOS} client. Found with the help of
Expand Down Expand Up @@ -235,7 +235,7 @@ private YoutubeParsingHelper() {
* information.
* </p>
*/
private static final String IOS_DEVICE_MODEL = "iPhone14,5";
private static final String IOS_DEVICE_MODEL = "iPhone16,2";

private static Random numberGenerator = new SecureRandom();

Expand Down Expand Up @@ -618,7 +618,6 @@ private static void extractClientVersionAndKeyFromHtmlSearchResultsPage()
final Stream<JsonObject> serviceTrackingParamsStream = serviceTrackingParams.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast);

clientVersion = getClientVersionFromServiceTrackingParam(
serviceTrackingParamsStream, "CSI", "cver");

Expand Down Expand Up @@ -1362,32 +1361,49 @@ public static JsonBuilder<JsonObject> prepareTvHtml5EmbedJsonBuilder(
// @formatter:on
}

public static JsonObject getWebPlayerResponse(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId) throws IOException, ExtractionException {
final byte[] body = JsonWriter.string(
prepareDesktopJsonBuilder(localization, contentCountry)
.value(VIDEO_ID, videoId)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(UTF_8);
final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER
+ "&$fields=microformat,playabilityStatus,storyboards,videoDetails";

return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().post(
url, getYouTubeHeaders(), body, localization)));
}

@Nonnull
public static byte[] createDesktopPlayerBody(
public static byte[] createTvHtml5EmbedPlayerBody(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId,
@Nonnull final String sts,
final boolean isTvHtml5DesktopJsonBuilder,
@Nonnull final String contentPlaybackNonce) throws IOException, ExtractionException {
// @formatter:off
return JsonWriter.string((isTvHtml5DesktopJsonBuilder
? prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId)
: prepareDesktopJsonBuilder(localization, contentCountry))
.object("playbackContext")
.object("contentPlaybackContext")
return JsonWriter.string(
prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId)
.object("playbackContext")
.object("contentPlaybackContext")
// Signature timestamp from the JavaScript base player is needed to get
// working obfuscated URLs
.value("signatureTimestamp", sts)
.value("referer", "https://www.youtube.com/watch?v=" + videoId)
.end()
.end()
.value(CPN, contentPlaybackNonce)
.value(VIDEO_ID, videoId)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(UTF_8);
.end()
.end()
.value(CPN, contentPlaybackNonce)
.value(VIDEO_ID, videoId)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(UTF_8);
// @formatter:on
}

Expand Down
64 changes: 55 additions & 9 deletions ...c/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@

import javax.annotation.Nonnull;

import static org.schabi.newpipe.extractor.utils.Parser.matchMultiplePatterns;

/**
* YouTube's streaming URLs of HTML5 clients are protected with a cipher, which modifies their
* {@code n} query parameter.
Expand Down Expand Up @@ -42,13 +44,54 @@ public final class YoutubeThrottlingDecrypter {
private static final String FUNCTION_NAME_REGEX = SINGLE_CHAR_VARIABLE_REGEX + "+";

private static final String ARRAY_ACCESS_REGEX = "\\[(\\d+)]";
private static final Pattern DECRYPT_FUNCTION_NAME_PATTERN = Pattern.compile(
// CHECKSTYLE:OFF
"\\(" + SINGLE_CHAR_VARIABLE_REGEX + "=String\\.fromCharCode\\(110\\),"
private static final Pattern[] DEOBFUSCATION_FUNCTION_NAME_REGEXES = {

/*
* The first regex matches the following text, where we want rDa and the array index
* accessed:
*
* a.D&&(b="nn"[+a.D],c=a.get(b))&&(c=rDa[0](c),a.set(b,c),rDa.length||rma("")
*/
Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "+=\"nn\"\\[\\+"
+ SINGLE_CHAR_VARIABLE_REGEX + "+\\." + SINGLE_CHAR_VARIABLE_REGEX + "+],"
+ SINGLE_CHAR_VARIABLE_REGEX + "+=" + SINGLE_CHAR_VARIABLE_REGEX
+ "+\\.get\\(" + SINGLE_CHAR_VARIABLE_REGEX + "+\\)\\)&&\\("
+ SINGLE_CHAR_VARIABLE_REGEX + "+=(" + SINGLE_CHAR_VARIABLE_REGEX
+ "+)\\[(\\d+)]"),

/*
* The second regex matches the following text, where we want rma:
*
* a.D&&(b="nn"[+a.D],c=a.get(b))&&(c=rDa[0](c),a.set(b,c),rDa.length||rma("")
*/
Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "+=\"nn\"\\[\\+"
+ SINGLE_CHAR_VARIABLE_REGEX + "+\\." + SINGLE_CHAR_VARIABLE_REGEX + "+],"
+ SINGLE_CHAR_VARIABLE_REGEX + "+=" + SINGLE_CHAR_VARIABLE_REGEX + "+\\.get\\("
+ SINGLE_CHAR_VARIABLE_REGEX + "+\\)\\).+\\|\\|(" + SINGLE_CHAR_VARIABLE_REGEX
+ "+)\\(\"\"\\)"),

/*
* The third regex matches the following text, where we want BDa and the array index
* accessed:
*
* (b=String.fromCharCode(110),c=a.get(b))&&(c=BDa[0](c)
*/
Pattern.compile("\\(" + SINGLE_CHAR_VARIABLE_REGEX + "=String\\.fromCharCode\\(110\\),"
+ SINGLE_CHAR_VARIABLE_REGEX + "=" + SINGLE_CHAR_VARIABLE_REGEX + "\\.get\\("
+ SINGLE_CHAR_VARIABLE_REGEX + "\\)\\)" + "&&\\(" + SINGLE_CHAR_VARIABLE_REGEX
+ "=(" + FUNCTION_NAME_REGEX + ")" + "(?:" + ARRAY_ACCESS_REGEX + ")?\\("
+ SINGLE_CHAR_VARIABLE_REGEX + "\\)");
+ SINGLE_CHAR_VARIABLE_REGEX + "\\)"),

/*
* The fourth regex matches the following text, where we want Yva and the array index
* accessed:
*
* .get("n"))&&(b=Yva[0](b)
*/
Pattern.compile("\\.get\\(\"n\"\\)\\)&&\\(" + SINGLE_CHAR_VARIABLE_REGEX
+ "=(" + FUNCTION_NAME_REGEX + ")(?:" + ARRAY_ACCESS_REGEX + ")?\\("
+ SINGLE_CHAR_VARIABLE_REGEX + "\\)")
};
// CHECKSTYLE:ON

// Escape the curly end brace to allow compatibility with Android's regex engine
Expand Down Expand Up @@ -115,11 +158,14 @@ public static String apply(@Nonnull final String streamingUrl,
}

private static String parseDecodeFunctionName(final String playerJsCode)
throws Parser.RegexException {
final Matcher matcher = DECRYPT_FUNCTION_NAME_PATTERN.matcher(playerJsCode);
if (!matcher.find()) {
throw new Parser.RegexException("Failed to find pattern \""
+ DECRYPT_FUNCTION_NAME_PATTERN + "\"");
throws ParsingException {
final Matcher matcher;
try {
matcher = matchMultiplePatterns(DEOBFUSCATION_FUNCTION_NAME_REGEXES,
playerJsCode);
} catch (final Parser.RegexException e) {
throw new ParsingException("Could not find deobfuscation function with any of the "
+ "known patterns in the base JavaScript player code", e);
}

final String functionName = matcher.group(1);
Expand Down
Loading

0 comments on commit 55d9207

Please sign in to comment.