Skip to content

Commit

Permalink
[fix/ISSUE-83] Add potoken support
Browse files Browse the repository at this point in the history
  • Loading branch information
@azihassan
azihassan committed Nov 7, 2024
1 parent 6ec1c6d commit de66660
Show file tree
Hide file tree
Showing 8 changed files with 16,408 additions and 76 deletions.
14 changes: 10 additions & 4 deletions source/app.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ void main(string[] args)
bool dethrottle = true;
bool chunked;
bool displayVersion;
string cookieFile;
string poToken;

version(linux)
{
Expand All @@ -61,7 +63,9 @@ void main(string[] args)
"no-cache", "Skip caching of HTML and base.js", &noCache,
"d|dethrottle", "Attempt to dethrottle download speed by solving the N challenge (defaults to true)", &dethrottle,
"no-dethrottle", "Skip N-challenge dethrottling attempt", () { dethrottle = false; },
"version", "Displays youtube-d version", &displayVersion
"version", "Displays youtube-d version", &displayVersion,
"cookiefile", "Cookie file, required for certain formats", &cookieFile,
"potoken", "Proof of origin token, required for certain formats", &poToken
);
if(displayVersion)
{
Expand Down Expand Up @@ -97,7 +101,9 @@ void main(string[] args)
noProgress,
retry > 0 ? true : noCache, //force cache refresh on failure,
dethrottle,
chunked
chunked,
cookieFile,
poToken
);
break;
}
Expand All @@ -117,10 +123,10 @@ void main(string[] args)
}
}

void handleURL(string url, int itag, StdoutLogger logger, bool displayFormats, bool outputURL, bool parallel, bool noProgress, bool noCache, bool dethrottle, bool chunked)
void handleURL(string url, int itag, StdoutLogger logger, bool displayFormats, bool outputURL, bool parallel, bool noProgress, bool noCache, bool dethrottle, bool chunked, string cookieFile, string poToken)
{
logger.display(formatTitle("Handling " ~ url));
YoutubeVideoURLExtractor parser = Cache(logger, noCache ? Yes.forceRefresh : No.forceRefresh).makeParser(url, itag);
YoutubeVideoURLExtractor parser = Cache(logger, cookieFile, poToken, noCache ? Yes.forceRefresh : No.forceRefresh).makeParser(url, itag);
logger.displayVerbose("Downloaded video HTML");
logger.displayVerbose("Attempt to dethrottle : " ~ (dethrottle ? "Yes" : "No"));

Expand Down
197 changes: 167 additions & 30 deletions source/cache.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,78 +3,142 @@ import std.array : replace;
import std.base64 : Base64URL;
import std.conv : to;
import std.datetime : SysTime, Clock, days;
import std.file : exists, getcwd, readText, remove, tempDir, write;
import std.net.curl : Curl, CurlOption;
import std.file : exists, getcwd, readText, remove, tempDir, write, copy;
import std.net.curl : HTTP;
import std.path : buildPath;
import std.typecons : Flag, Yes, No;
import std.string : indexOf;
import std.string : indexOf, format, toLower;
import std.regex : ctRegex, matchFirst;
import std.algorithm : map;
import std.algorithm : canFind, map;
import std.zlib : UnCompress;

import helpers : StdoutLogger, parseID, parseQueryString, parseBaseJSKey, formatTitle, formatSuccess;
import parsers : parseBaseJSURL, YoutubeVideoURLExtractor, SimpleYoutubeVideoURLExtractor, AdvancedYoutubeVideoURLExtractor;
import parsers : parseBaseJSURL, YoutubeVideoURLExtractor, SimpleYoutubeVideoURLExtractor, AdvancedYoutubeVideoURLExtractor, PlayerYoutubeVideoURLExtractor;

string formatPlayerRequest(string videoId, string poToken)
{
return format!`{
"videoId": "%s",
"context": {
"client": {
"userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15,gzip(gfe)",
"clientName": "WEB_EMBEDDED_PLAYER",
"clientVersion": "1.20241029.01.00",
"originalUrl": "https://www.youtube.com/embed/%s",
"platform": "DESKTOP",
"clientScreen": "EMBED"
}
},
"serviceIntegrityDimensions": {
"poToken": "%s"
}
}`(videoId, videoId, poToken);

}

struct Cache
{
private StdoutLogger logger;
private string delegate(string url) downloadAsString;
private Flag!"forceRefresh" forceRefresh;
string cacheDirectory;
string poToken;

this(StdoutLogger logger, Flag!"forceRefresh" forceRefresh = No.forceRefresh)
this(StdoutLogger logger, string cookieFile, string poToken, Flag!"forceRefresh" forceRefresh = No.forceRefresh)
{
this.logger = logger;
this.forceRefresh = forceRefresh;
this.poToken = poToken;
cacheDirectory = tempDir();

downloadAsString = (string url) {
string result;
Curl curl;
curl.initialize();
curl.set(CurlOption.url, url);
curl.set(CurlOption.encoding, "deflate, gzip");
curl.set(CurlOption.followlocation, true);

curl.onReceive = (ubyte[] chunk) {
result ~= chunk.map!(to!(const(char))).to!string;
return chunk.length;
string responseEncoding;
auto curl = HTTP(url);
auto gzip = new UnCompress();

//curl.set(CurlOption.encoding, "deflate, gzip");
curl.addRequestHeader("Accept-Encoding", "deflate, gzip");
curl.setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15,gzip(gfe)");
curl.verbose(logger.verbose);
if(cookieFile != "" && cookieFile.readText().canFind("VISITOR_INFO1_LIVE"))
{
logger.display("Attaching cookie file " ~ cookieFile);
curl.setCookieJar(cookieFile);
}
if(url.canFind("/v1/player") && poToken != "")
{
logger.display("Attaching proof-of-origin token " ~ poToken);
curl.setPostData(formatPlayerRequest("cvDVjwMXiCs", poToken), "application/json");
//curl.set(CurlOption.postfields, formatPlayerRequest("cvDVjwMXiCs", poToken));
}

curl.onReceiveHeader = (in char[] key, in char[] value) {
if(key.idup.toLower() == "content-length")
{
logger.displayVerbose("Length of " ~ url ~ " : " ~ value);
}
if(key.idup.toLower() == "content-encoding")
{
responseEncoding = value.idup.toLower();
}
};

curl.onReceiveHeader = (in char[] header) {
if(header.indexOf("Content-Length", No.caseSensitive) == 0)
curl.onReceive = (ubyte[] chunk) {
if(responseEncoding == "gzip" || responseEncoding == "deflate")
{
logger.displayVerbose("Length of " ~ url ~ " : " ~ header["Content-Length: ".length .. $]);
ubyte[] uncompressed = cast(ubyte[]) gzip.uncompress(chunk);
result ~= uncompressed.map!(to!(const(char))).to!string;
}
else
{
result ~= chunk.map!(to!(const(char))).to!string;
}
return chunk.length;
};
curl.perform();
return result;
};
}

this(StdoutLogger logger, string delegate(string url) downloadAsString, Flag!"forceRefresh" forceRefresh = No.forceRefresh)
this(StdoutLogger logger, string delegate(string url) downloadAsString, string cookieFile = "", string poToken = "", Flag!"forceRefresh" forceRefresh = No.forceRefresh)
{
this(logger);
this(logger, cookieFile, poToken);
this.downloadAsString = downloadAsString;
this.forceRefresh = forceRefresh;
}

YoutubeVideoURLExtractor makeParser(string url, int itag)
{
string html;
string player;

string htmlCachePath = getHTMLCachePath(url) ~ ".html";
updateHTMLCache(url, htmlCachePath, itag);
string html = htmlCachePath.readText();
string playerCachePath = getHTMLCachePath(url) ~ ".json";
if(poToken != "")
{
string playerURL = "https://www.youtube.com/youtubei/v1/player?prettyPrint=false";
updatePlayerCache(url, playerURL, playerCachePath, htmlCachePath, itag);
html = htmlCachePath.readText();
player = playerCachePath.readText();
}
else
{
updateHTMLCache(url, htmlCachePath, itag);
html = htmlCachePath.readText();
}

string baseJSURL = html.parseBaseJSURL();
string baseJSCachePath = getBaseJSCachePath(baseJSURL) ~ ".js";
updateBaseJSCache(baseJSURL, baseJSCachePath, itag);
string baseJS = baseJSCachePath.readText();

return makeParser(html, baseJS, logger);
return makeParser(html, baseJS, player, logger);
}

private void updateHTMLCache(string url, string htmlCachePath, int itag)
{
bool shouldRedownload = forceRefresh || !htmlCachePath.exists() || isStale(htmlCachePath.readText(), itag);
bool shouldRedownload = forceRefresh || !htmlCachePath.exists() || isStale(htmlCachePath.readText(), "", itag);
if(shouldRedownload)
{
logger.display("Cache miss, downloading HTML...");
Expand All @@ -87,6 +151,23 @@ struct Cache
}
}

private void updatePlayerCache(string url, string playerURL, string playerCachePath, string htmlCachePath, int itag)
{
bool shouldRedownload = forceRefresh || !playerCachePath.exists() || isStale("", playerCachePath.readText(), itag);
if(shouldRedownload)
{
logger.display("Cache miss, downloading HTML and player JSON...");
string player = this.downloadAsString(playerURL);
playerCachePath.write(player);
string html = this.downloadAsString(url);
htmlCachePath.write(html);
}
else
{
logger.display("Cache hit (" ~ playerCachePath ~ "), skipping player JSON download...");
}
}

private void updateBaseJSCache(string url, string baseJSCachePath, int itag)
{
bool shouldRedownload = forceRefresh || !baseJSCachePath.exists();
Expand All @@ -102,9 +183,9 @@ struct Cache
}
}

private bool isStale(string html, int itag)
private bool isStale(string html, string player, int itag)
{
YoutubeVideoURLExtractor shallowParser = makeParser(html, "", logger);
YoutubeVideoURLExtractor shallowParser = makeParser(html, "", player, logger);
ulong expire = shallowParser.findExpirationTimestamp(itag);
return SysTime.fromUnixTime(expire) < Clock.currTime();
}
Expand All @@ -131,14 +212,18 @@ struct Cache
return buildPath(cacheDirectory, cacheKey);
}

private YoutubeVideoURLExtractor makeParser(string html, string baseJS, StdoutLogger logger)
private YoutubeVideoURLExtractor makeParser(string html, string baseJS, string player, StdoutLogger logger)
{
if(player != "")
{
return new PlayerYoutubeVideoURLExtractor(html, baseJS, player, poToken, logger);
}
immutable urlRegex = ctRegex!`"itag":\d+,"url":"(.*?)"`;
if(!html.matchFirst(urlRegex).empty)
{
return new SimpleYoutubeVideoURLExtractor(html, baseJS, logger);
return new SimpleYoutubeVideoURLExtractor(html, baseJS, poToken, logger);
}
return new AdvancedYoutubeVideoURLExtractor(html, baseJS, logger);
return new AdvancedYoutubeVideoURLExtractor(html, baseJS, poToken, logger);
}
}

Expand Down Expand Up @@ -246,7 +331,7 @@ unittest
}
return "tests/zoz.html".readText();
};
auto cache = Cache(new StdoutLogger(), downloadAsString, Yes.forceRefresh);
auto cache = Cache(new StdoutLogger(), downloadAsString, "", "", Yes.forceRefresh);
cache.cacheDirectory = buildPath(getcwd(), "tests");

auto parser = cache.makeParser("https://youtu.be/zoz", 18);
Expand Down Expand Up @@ -304,3 +389,55 @@ unittest
auto parser = cache.makeParser("https://youtu.be/zoz", 18);
assert(baseJSDownloadAttempted);
}

unittest
{
writeln("Given PlayerYoutubeVideoURLExtractor, when cache is fresh, should not download HTML and player".formatTitle());
scope(success) writeln("OK\n".formatSuccess());

SysTime tomorrow = Clock.currTime() + 1.days;
"tests/cvDVjwMXiCs.html".copy("tests/embed-fresh.html");
"tests/cvDVjwMXiCs.json".copy("tests/embed-fresh.json");
"tests/embed-fresh.json".write("tests/embed-fresh.json".readText().dup.replace("expire=1730607289", "expire=" ~ tomorrow.toUnixTime().to!string));
scope(exit)
{
remove("tests/embed-fresh.html");
remove("tests/embed-fresh.json");
}

//bool downloadAttempted;
auto downloadAsString = delegate string(string url) {
assert(false, "downloadAsString should not be called when cache is fresh");
};
auto cache = Cache(new StdoutLogger(), downloadAsString, "tests/cookies.txt", "PO_TOKEN_MOCK");
cache.cacheDirectory = buildPath(getcwd(), "tests");

auto parser = cache.makeParser("https://youtu.be/embed-fresh", 18);
//assert(!downloadAttempted);
}

unittest
{
writeln("Given PlayerYoutubeVideoURLExtractor, when cache is stale, should download HTML and player".formatTitle());
scope(success) writeln("OK\n".formatSuccess());

bool downloadAttempted;
auto downloadAsString = delegate string(string url) {
if(url == "https://youtu.be/cvDVjwMXiCs")
{
downloadAttempted = true;
return "tests/cvDVjwMXiCs.html".readText();
}
if(url == "https://www.youtube.com/youtubei/v1/player?prettyPrint=false")
{
downloadAttempted = true;
return "tests/cvDVjwMXiCs.json".readText();
}
assert(false, "downloadAsString called with unknown URL: " ~ url);
};
auto cache = Cache(new StdoutLogger(), downloadAsString, "tests/cookies.txt", "PO_TOKEN_MOCK");
cache.cacheDirectory = buildPath(getcwd(), "tests");

auto parser = cache.makeParser("https://youtu.be/cvDVjwMXiCs", 18);
assert(downloadAttempted);
}
13 changes: 13 additions & 0 deletions source/helpers.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import std.format : formattedRead;

import parsers : YoutubeFormat, AudioVisual;

//todo remove debug logs
ulong getContentLength(string url, YoutubeFormat youtubeFormat)
{
writeln("url = ", url);
Expand Down Expand Up @@ -262,6 +263,17 @@ string parseID(string url)
url.formattedRead!"https://youtube.com/shorts/%s"(id);
}
}
if(url.indexOf("/embed/") != -1)
{
if(url.startsWith("https://www"))
{
url.formattedRead!"https://www.youtube.com/embed/%s"(id);
}
else
{
url.formattedRead!"https://youtube.com/embed/%s"(id);
}
}
return id;
}

Expand All @@ -273,6 +285,7 @@ unittest
assert("https://youtu.be/-H-Fno9xbE4".parseID() == "-H-Fno9xbE4");
assert("https://www.youtube.com/shorts/_tT2ldpZHek".parseID() == "_tT2ldpZHek");
assert("https://youtube.com/shorts/_tT2ldpZHek".parseID() == "_tT2ldpZHek");
assert("https://www.youtube.com/embed/cvDVjwMXiCs".parseID() == "cvDVjwMXiCs");
assert("qlsdkqsldkj".parseID() == "");
}

Expand Down
Loading

0 comments on commit de66660

Please sign in to comment.