Fix crawler (rate limit changes)

osmarks · Jul 15, 2024 · fc6d0c9 · fc6d0c9
1 parent 43ff921
commit fc6d0c9
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/meme-rater/crawler.py b/meme-rater/crawler.py
@@ -9,6 +9,10 @@
 
 async def fetch_list_seg(sess, list_url, query):
     async with sess.get(list_url + ".json", params=query) as res:
+        if rate_limit := res.headers.get("x-ratelimit-remaining"):
+            rl = float(rate_limit)
+            if rl <= 5.0:
+                await asyncio.sleep(float(res.headers["x-ratelimit-reset"]))
         return await res.json()
 
 async def fetch_past(sess, list_url, n):