Merge pull request #7 from AnswerDotAI/base64

Exclude images from context?
AnswerDotAI · Sep 12, 2024 · 4aef130 · 4aef130
2 parents bdd7b4e + 97556f3
commit 4aef130
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/llms_txt/core.py b/llms_txt/core.py
@@ -70,7 +70,8 @@ def _doc(kw):
     "Create a `Doc` FT object with the text retrieved from `url` as the child, and `kw` as attrs."
     url = kw.pop('url')
     re_comment = re.compile('^<!--.*-->$', flags=re.MULTILINE)
-    txt = [o for o in httpx.get(url).text.splitlines() if not re_comment.search(o)]
+    re_base64_img = re.compile(r'<img[^>]*src="data:image/[^"]*"[^>]*>')
+    txt = [o for o in httpx.get(url).text.splitlines() if not re_comment.search(o) and not re_base64_img.search(o)]
     return Doc('\n'.join(txt), **kw)
 
 # %% ../nbs/01_core.ipynb

diff --git a/nbs/01_core.ipynb b/nbs/01_core.ipynb
@@ -684,7 +684,8 @@
     "    \"Create a `Doc` FT object with the text retrieved from `url` as the child, and `kw` as attrs.\"\n",
     "    url = kw.pop('url')\n",
     "    re_comment = re.compile('^<!--.*-->$', flags=re.MULTILINE)\n",
-    "    txt = [o for o in httpx.get(url).text.splitlines() if not re_comment.search(o)]\n",
+    "    re_base64_img = re.compile(r'<img[^>]*src=\"data:image/[^\"]*\"[^>]*>')\n",
+    "    txt = [o for o in httpx.get(url).text.splitlines() if not re_comment.search(o) and not re_base64_img.search(o)]\n",
     "    return Doc('\\n'.join(txt), **kw)"
    ]
   },