Skip to content

Commit

Permalink
[fix/ISSUE-83] Add potoken support
Browse files Browse the repository at this point in the history
  • Loading branch information
@azihassan
azihassan committed Nov 5, 2024
1 parent 6ec1c6d commit 7cc2176
Show file tree
Hide file tree
Showing 6 changed files with 16,314 additions and 63 deletions.
14 changes: 10 additions & 4 deletions source/app.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ void main(string[] args)
bool dethrottle = true;
bool chunked;
bool displayVersion;
string cookieFile;
string poToken;

version(linux)
{
Expand All @@ -61,7 +63,9 @@ void main(string[] args)
"no-cache", "Skip caching of HTML and base.js", &noCache,
"d|dethrottle", "Attempt to dethrottle download speed by solving the N challenge (defaults to true)", &dethrottle,
"no-dethrottle", "Skip N-challenge dethrottling attempt", () { dethrottle = false; },
"version", "Displays youtube-d version", &displayVersion
"version", "Displays youtube-d version", &displayVersion,
"cookiefile", "Cookie file, required for certain formats", &cookieFile,
"potoken", "Proof of origin token, required for certain formats", &poToken
);
if(displayVersion)
{
Expand Down Expand Up @@ -97,7 +101,9 @@ void main(string[] args)
noProgress,
retry > 0 ? true : noCache, //force cache refresh on failure,
dethrottle,
chunked
chunked,
cookieFile,
poToken
);
break;
}
Expand All @@ -117,10 +123,10 @@ void main(string[] args)
}
}

void handleURL(string url, int itag, StdoutLogger logger, bool displayFormats, bool outputURL, bool parallel, bool noProgress, bool noCache, bool dethrottle, bool chunked)
void handleURL(string url, int itag, StdoutLogger logger, bool displayFormats, bool outputURL, bool parallel, bool noProgress, bool noCache, bool dethrottle, bool chunked, string cookieFile, string poToken)
{
logger.display(formatTitle("Handling " ~ url));
YoutubeVideoURLExtractor parser = Cache(logger, noCache ? Yes.forceRefresh : No.forceRefresh).makeParser(url, itag);
YoutubeVideoURLExtractor parser = Cache(logger, cookieFile, poToken, noCache ? Yes.forceRefresh : No.forceRefresh).makeParser(url, itag);
logger.displayVerbose("Downloaded video HTML");
logger.displayVerbose("Attempt to dethrottle : " ~ (dethrottle ? "Yes" : "No"));

Expand Down
109 changes: 92 additions & 17 deletions source/cache.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,35 +5,78 @@ import std.conv : to;
import std.datetime : SysTime, Clock, days;
import std.file : exists, getcwd, readText, remove, tempDir, write;
import std.net.curl : Curl, CurlOption;
import etc.c.curl : curl_slist, curl_slist_append, curl_slist_free_all;
import std.path : buildPath;
import std.typecons : Flag, Yes, No;
import std.string : indexOf;
import std.string : indexOf, format;
import std.regex : ctRegex, matchFirst;
import std.algorithm : map;
import std.algorithm : canFind, map;

import helpers : StdoutLogger, parseID, parseQueryString, parseBaseJSKey, formatTitle, formatSuccess;
import parsers : parseBaseJSURL, YoutubeVideoURLExtractor, SimpleYoutubeVideoURLExtractor, AdvancedYoutubeVideoURLExtractor;
import parsers : parseBaseJSURL, YoutubeVideoURLExtractor, SimpleYoutubeVideoURLExtractor, AdvancedYoutubeVideoURLExtractor, PlayerYoutubeVideoURLExtractor;

enum PLAYER_REQUEST_FORMAT = `{
"videoId": "cvDVjwMXiCs",
"context": {
"client": {
"userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15,gzip(gfe)",
"clientName": "WEB_EMBEDDED_PLAYER",
"clientVersion": "1.20241029.01.00",
"originalUrl": "https://www.youtube.com/embed/cvDVjwMXiCs",
"platform": "DESKTOP",
"clientScreen": "EMBED"
}
},
"serviceIntegrityDimensions": {
"poToken": "%s"
}
}`;

struct Cache
{
private StdoutLogger logger;
private string delegate(string url) downloadAsString;
private Flag!"forceRefresh" forceRefresh;
string cacheDirectory;
string poToken;

this(StdoutLogger logger, Flag!"forceRefresh" forceRefresh = No.forceRefresh)
this(StdoutLogger logger, string cookieFile, string poToken, Flag!"forceRefresh" forceRefresh = No.forceRefresh)
{
this.logger = logger;
this.forceRefresh = forceRefresh;
this.poToken = poToken;
cacheDirectory = tempDir();

downloadAsString = (string url) {
string result;
Curl curl;
/*curl_slist* headers;
scope(exit)
{
if(headers != null)
{
curl_slist_free_all(headers);
}
}*/
curl.initialize();
curl.set(CurlOption.url, url);
curl.set(CurlOption.encoding, "deflate, gzip");
curl.set(CurlOption.followlocation, true);
curl.set(CurlOption.useragent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15,gzip(gfe)");
curl.set(CurlOption.verbose, logger.verbose);
if(cookieFile != "" && cookieFile.readText().canFind("VISITOR_INFO1_LIVE"))
{
logger.display("Attaching cookie file " ~ cookieFile);
curl.set(CurlOption.cookiefile, cookieFile);
}
if(url.canFind("/v1/player") && poToken != "")
{
logger.display("Attaching proof-of-origin token " ~ poToken);
curl.set(CurlOption.postfields, poToken.format!PLAYER_REQUEST_FORMAT());

//headers = curl_slist_append(null, "Content-Type: application/json");
//curl.set(CurlOption.httpheader, headers);
}

curl.onReceive = (ubyte[] chunk) {
result ~= chunk.map!(to!(const(char))).to!string;
Expand All @@ -51,30 +94,43 @@ struct Cache
};
}

this(StdoutLogger logger, string delegate(string url) downloadAsString, Flag!"forceRefresh" forceRefresh = No.forceRefresh)
this(StdoutLogger logger, string delegate(string url) downloadAsString, string cookieFile = "", string poToken = "", Flag!"forceRefresh" forceRefresh = No.forceRefresh)
{
this(logger);
this(logger, "", "");
this.downloadAsString = downloadAsString;
this.forceRefresh = forceRefresh;
}

YoutubeVideoURLExtractor makeParser(string url, int itag)
{
string htmlCachePath = getHTMLCachePath(url) ~ ".html";
updateHTMLCache(url, htmlCachePath, itag);
string html = htmlCachePath.readText();
string html;
string player;

if(poToken != "")
{
string playerURL = "https://www.youtube.com/youtubei/v1/player?prettyPrint=false";
string playerCachePath = getHTMLCachePath(url) ~ ".json";
updatePlayerCache(playerURL, playerCachePath, itag);
player = playerCachePath.readText();
}
else
{
string htmlCachePath = getHTMLCachePath(url) ~ ".html";
updateHTMLCache(url, htmlCachePath, itag);
html = htmlCachePath.readText();
}

string baseJSURL = html.parseBaseJSURL();
string baseJSCachePath = getBaseJSCachePath(baseJSURL) ~ ".js";
updateBaseJSCache(baseJSURL, baseJSCachePath, itag);
string baseJS = baseJSCachePath.readText();

return makeParser(html, baseJS, logger);
return makeParser(html, baseJS, player, logger);
}

private void updateHTMLCache(string url, string htmlCachePath, int itag)
{
bool shouldRedownload = forceRefresh || !htmlCachePath.exists() || isStale(htmlCachePath.readText(), itag);
bool shouldRedownload = forceRefresh || !htmlCachePath.exists() || isStale(htmlCachePath.readText(), "", itag);
if(shouldRedownload)
{
logger.display("Cache miss, downloading HTML...");
Expand All @@ -87,6 +143,21 @@ struct Cache
}
}

private void updatePlayerCache(string url, string playerCachePath, int itag)
{
bool shouldRedownload = forceRefresh || !playerCachePath.exists() || isStale("", playerCachePath.readText(), itag);
if(shouldRedownload)
{
logger.display("Cache miss, downloading HTML...");
string player = this.downloadAsString(url);
playerCachePath.write(player);
}
else
{
logger.display("Cache hit (" ~ playerCachePath ~ "), skipping player JSON download...");
}
}

private void updateBaseJSCache(string url, string baseJSCachePath, int itag)
{
bool shouldRedownload = forceRefresh || !baseJSCachePath.exists();
Expand All @@ -102,9 +173,9 @@ struct Cache
}
}

private bool isStale(string html, int itag)
private bool isStale(string html, string player, int itag)
{
YoutubeVideoURLExtractor shallowParser = makeParser(html, "", logger);
YoutubeVideoURLExtractor shallowParser = makeParser(html, "", player, logger);
ulong expire = shallowParser.findExpirationTimestamp(itag);
return SysTime.fromUnixTime(expire) < Clock.currTime();
}
Expand All @@ -131,14 +202,18 @@ struct Cache
return buildPath(cacheDirectory, cacheKey);
}

private YoutubeVideoURLExtractor makeParser(string html, string baseJS, StdoutLogger logger)
private YoutubeVideoURLExtractor makeParser(string html, string baseJS, string player, StdoutLogger logger)
{
if(player != "")
{
return new PlayerYoutubeVideoURLExtractor(html, baseJS, player, poToken, logger);
}
immutable urlRegex = ctRegex!`"itag":\d+,"url":"(.*?)"`;
if(!html.matchFirst(urlRegex).empty)
{
return new SimpleYoutubeVideoURLExtractor(html, baseJS, logger);
return new SimpleYoutubeVideoURLExtractor(html, baseJS, poToken, logger);
}
return new AdvancedYoutubeVideoURLExtractor(html, baseJS, logger);
return new AdvancedYoutubeVideoURLExtractor(html, baseJS, poToken, logger);
}
}

Expand Down Expand Up @@ -246,7 +321,7 @@ unittest
}
return "tests/zoz.html".readText();
};
auto cache = Cache(new StdoutLogger(), downloadAsString, Yes.forceRefresh);
auto cache = Cache(new StdoutLogger(), downloadAsString, "", "", Yes.forceRefresh);
cache.cacheDirectory = buildPath(getcwd(), "tests");

auto parser = cache.makeParser("https://youtu.be/zoz", 18);
Expand Down
Loading

0 comments on commit 7cc2176

Please sign in to comment.