[feature/ISSUE-28] Cache base.js based on base.js URL and not video ID

azihassan · Dec 27, 2023 · f0c7fbf · f0c7fbf
1 parent 8f9ce43
commit f0c7fbf
Show file tree

Hide file tree

Showing 2 changed files with 64 additions and 13 deletions.
diff --git a/source/cache.d b/source/cache.d
@@ -9,7 +9,7 @@ import std.path : buildPath;
 import std.typecons : Flag, Yes, No;
 import std.string : indexOf;
 
-import helpers : StdoutLogger, parseID, parseQueryString;
+import helpers : StdoutLogger, parseID, parseQueryString, parseBaseJSKey;
 import parsers : parseBaseJSURL, YoutubeVideoURLExtractor, SimpleYoutubeVideoURLExtractor, AdvancedYoutubeVideoURLExtractor;
 
 struct Cache
@@ -35,46 +35,62 @@ struct Cache
 
     YoutubeVideoURLExtractor makeParser(string url, int itag)
     {
-        string htmlCachePath = getCachePath(url) ~ ".html";
-        string baseJSCachePath = getCachePath(url) ~ ".js";
-        updateCache(url, htmlCachePath, baseJSCachePath, itag);
-
+        string htmlCachePath = getHTMLCachePath(url) ~ ".html";
+        updateHTMLCache(url, htmlCachePath, itag);
         string html = htmlCachePath.readText();
+
+        string baseJSURL = html.parseBaseJSURL();
+        string baseJSCachePath = getBaseJSCachePath(baseJSURL) ~ ".js";
+        updateBaseJSCache(baseJSURL, baseJSCachePath, itag);
         string baseJS = baseJSCachePath.readText();
-        if(html.indexOf("signatureCipher:") == -1)
+
+        if(html.indexOf("signatureCipher") == -1)
         {
             return new SimpleYoutubeVideoURLExtractor(html, baseJS, logger);
         }
         return new AdvancedYoutubeVideoURLExtractor(html, baseJS, logger);
     }
 
-    private void updateCache(string url, string htmlCachePath, string baseJSCachePath, int itag)
+    private void updateHTMLCache(string url, string htmlCachePath, int itag)
     {
         bool shouldRedownload = forceRefresh || !htmlCachePath.exists() || isStale(htmlCachePath.readText(), itag);
         if(shouldRedownload)
         {
             logger.display("Cache miss, downloading HTML...");
             string html = this.downloadAsString(url);
             htmlCachePath.write(html);
-            string baseJS = this.downloadAsString(html.parseBaseJSURL());
-            baseJSCachePath.write(baseJS);
         }
         else
         {
             logger.display("Cache hit, skipping HTML download...");
         }
     }
 
+    private void updateBaseJSCache(string url, string baseJSCachePath, int itag)
+    {
+        bool shouldRedownload = !baseJSCachePath.exists();
+        if(shouldRedownload)
+        {
+            logger.display("base.js cache miss, downloading from " ~ url);
+            string baseJS = this.downloadAsString(url);
+            baseJSCachePath.write(baseJS);
+        }
+        else
+        {
+            logger.display("base.js cache hit, skipping download...");
+        }
+    }
+
     private bool isStale(string html, int itag)
     {
-        YoutubeVideoURLExtractor shallowParser = html.indexOf("signatureCipher:") == -1
+        YoutubeVideoURLExtractor shallowParser = html.indexOf("signatureCipher") == -1
             ? new SimpleYoutubeVideoURLExtractor(html, "", logger)
             : new AdvancedYoutubeVideoURLExtractor(html, "", logger);
         ulong expire = shallowParser.findExpirationTimestamp(itag);
         return SysTime.fromUnixTime(expire) < Clock.currTime();
     }
 
-    private string getCachePath(string url)
+    private string getHTMLCachePath(string url)
     {
         string cacheKey = url.parseID();
         if(cacheKey == "")
@@ -84,11 +100,24 @@ struct Cache
 
         return buildPath(cacheDirectory, cacheKey);
     }
+
+    private string getBaseJSCachePath(string url)
+    {
+        ///s/player/0c96dfd3/player_ias.vflset/ar_EG/base.js
+        string cacheKey = url.parseBaseJSKey();
+        if(cacheKey == "")
+        {
+            cacheKey = Base64URL.encode(cast(ubyte[]) url);
+        }
+
+        return buildPath(cacheDirectory, cacheKey);
+    }
 }
 
 unittest
 {
     writeln("Given SimpleYoutubeVideoURLExtractor, when cache is stale, should redownload HTML");
+    //"0c96dfd3.js".write("base.js");
     bool downloadAttempted;
     auto downloadAsString = delegate string(string url) {
         downloadAttempted = true;
@@ -114,7 +143,7 @@ unittest
     cache.cacheDirectory = getcwd();
 
     "zoz-fresh.html".write("zoz.html".readText().dup.replace("expire=1638935038", "expire=" ~ tomorrow.toUnixTime().to!string));
-    "zoz-fresh.js".write("base.min.js".readText());
+    //"zoz-fresh.js".write("base.min.js".readText());
 
     auto parser = cache.makeParser("https://youtu.be/zoz-fresh", 18);
     assert(!downloadAttempted);
@@ -148,7 +177,7 @@ unittest
     cache.cacheDirectory = getcwd();
 
     //mock previously cached and fresh files
-    "dQw4w9WgXcQ-fresh.js".write("base.min.js".readText());
+    //"7862ca1f.js".write("base.min.js".readText());
     "dQw4w9WgXcQ-fresh.html".write(
             "dQw4w9WgXcQ.html".readText().dup.replace("expire%3D1677997809", "expire%3D" ~ tomorrow.toUnixTime().to!string)
     );

diff --git a/source/helpers.d b/source/helpers.d
@@ -220,3 +220,25 @@ unittest
     assert("https://www.youtube.com/shorts/_tT2ldpZHek".parseID() == "_tT2ldpZHek");
     assert("qlsdkqsldkj".parseID() == "");
 }
+
+string parseBaseJSKey(string url)
+{
+    writeln("base.js url = ", url);
+    string id;
+    if(url.startsWith("https://"))
+    {
+        url = url["https://".length .. $];
+    }
+    if(url.startsWith("www.youtube.com"))
+    {
+        url = url["www.youtube.com".length .. $];
+    }
+    return url.split("/")[3];
+}
+
+unittest
+{
+    assert("/s/player/0c96dfd3/player_ias.vflset/ar_EG/base.js".parseBaseJSKey() == "0c96dfd3");
+    assert("https://www.youtube.com/s/player/0c96dfd3/player_ias.vflset/ar_EG/base.js".parseBaseJSKey() == "0c96dfd3");
+    assert("www.youtube.com/s/player/0c96dfd3/player_ias.vflset/ar_EG/base.js".parseBaseJSKey() == "0c96dfd3");
+}