feat(gutenberg): implement new reader

llama-index-integrations/readers/llama-index-readers-gutenberg/.gitignore

@@ -0,0 +1,153 @@
+llama_index/_static
+.DS_Store
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+bin/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+etc/
+include/
+lib/
+lib64/
+parts/
+sdist/
+share/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+.ruff_cache
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+notebooks/
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+pyvenv.cfg
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Jetbrains
+.idea
+modules/
+*.swp
+
+# VsCode
+.vscode
+
+# pipenv
+Pipfile
+Pipfile.lock
+
+# pyright
+pyrightconfig.json

llama-index-integrations/readers/llama-index-readers-gutenberg/BUILD

@@ -0,0 +1,7 @@
+poetry_requirements(
+    name="poetry",
+)
+
+python_requirements(
+    name="reqs",
+)

llama-index-integrations/readers/llama-index-readers-gutenberg/CHANGELOG.md

@@ -0,0 +1,5 @@
+# CHANGELOG
+
+## [0.1.2] - 2024-02-13
+
+- Add maintainers and keywords from library.json (llamahub)

llama-index-integrations/readers/llama-index-readers-gutenberg/Makefile

@@ -0,0 +1,17 @@
+GIT_ROOT ?= $(shell git rev-parse --show-toplevel)
+
+help:	## Show all Makefile targets.
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'
+
+format:	## Run code autoformatters (black).
+	pre-commit install
+	git ls-files | xargs pre-commit run black --files
+
+lint:	## Run linters: pre-commit (black, ruff, codespell) and mypy
+	pre-commit install && git ls-files | xargs pre-commit run --show-diff-on-failure --files
+
+test:	## Run tests via pytest.
+	pytest tests
+
+watch-docs:	## Build and watch documentation.
+	sphinx-autobuild docs/ docs/_build/html --open-browser --watch $(GIT_ROOT)/llama_index/

llama-index-integrations/readers/llama-index-readers-gutenberg/README.md

@@ -0,0 +1,21 @@
+# Project Gutenberg loader
+
+```bash
+pip install llama-index-readers-gutenberg
+```
+
+This loader loads books from their names from the Project Gutenberg library.
+
+## Usage
+
+Here's an example usage of the Project Gutenberg loader:
+
+```python
+from llama_index.readers.gutenberg import ProjectGutenbergReader
+
+reader = ProjectGutenbergReader()
+crime_and_punishment = reader.load_data("Crime and Punishment")
+
+# Not existing book raises exception
+reader.load_data("Not existing book")
+```

llama-index-integrations/readers/llama-index-readers-gutenberg/llama_index/readers/gutenberg/BUILD

@@ -0,0 +1 @@
+python_sources()

llama-index-integrations/readers/llama-index-readers-gutenberg/llama_index/readers/gutenberg/__init__.py

@@ -0,0 +1,3 @@
+from llama_index.readers.gutenberg.base import ProjectGutenbergReader
+
+__all__ = ["ProjectGutenbergReader"]

llama-index-integrations/readers/llama-index-readers-gutenberg/llama_index/readers/gutenberg/base.py

@@ -0,0 +1,48 @@
+import re
+import requests
+from llama_index.core.readers.base import BaseReader
+from llama_index.core.schema import Document
+
+
+class BookNotFoundException(Exception):
+    pass
+
+
+class ProjectGutenbergReader(BaseReader):
+    def load_data(self, book_title):
+        document = maybe_get_gutenberg_book(book_title)
+        return [document]
+
+
+def maybe_get_gutenberg_book(title):
+    url = f"http://gutendex.com/books/?search={title}"
+    response = requests.get(url)
+
+    try:
+        response.raise_for_status()
+    except requests.exceptions.HTTPError:
+        raise BookNotFoundException(f"Search request failed for title: {title}")
+
+    books = response.json().get("results", [])
+    book_id = None
+    for book in books:
+        if title.lower() in book["title"].lower():
+            book_id = book["id"]
+            break
+
+    if book_id is None:
+        raise BookNotFoundException(f"Book not found with title: {title}")
+
+    book_url = f"https://www.gutenberg.org/files/{book_id}/{book_id}-0.txt"
+    response = requests.get(book_url)
+
+    try:
+        response.raise_for_status()
+    except requests.exceptions.HTTPError:
+        raise BookNotFoundException(f"Book not found with title: {title}")
+
+    text = response.text
+    # Get rid of binary characters
+    text = re.sub(r"[^\x00-\x7F]+", "", text)
+
+    return Document(text=text)

llama-index-integrations/readers/llama-index-readers-gutenberg/pyproject.toml

@@ -0,0 +1,49 @@
+[build-system]
+build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core"]
+
+[tool.codespell]
+check-filenames = true
+check-hidden = true
+skip = "*.csv,*.html,*.json,*.jsonl,*.pdf,*.txt,*.ipynb"
+
+[tool.llamahub]
+contains_example = true
+import_path = "llama_index.readers.gutenberg"
+
+[tool.llamahub.class_authors]
+ProjectGutenbergReader = "Oguz Vuruskaner"
+
+[tool.mypy]
+disallow_untyped_defs = true
+exclude = ["_static", "build", "examples", "notebooks", "venv"]
+ignore_missing_imports = true
+python_version = "3.8"
+
+[tool.poetry]
+authors = ["ovuruska <oguzvuruskaner@gmail.com>"]
+description = "llama-index readers gutenberg integration ebook"
+exclude = ["**/BUILD"]
+license = "MIT"
+maintainers = ["ovuruska"]
+name = "llama-index-readers-gutenberg"
+readme = "README.md"
+version = "0.0.1"
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4.0"
+llama-index-core = "^0.10.1"
+
+[tool.poetry.group.dev.dependencies]
+types-requests = "2.28.11.8"
+
+[tool.poetry.group.dev.dependencies.black]
+extras = ["jupyter"]
+version = "<=23.9.1,>=23.7.0"
+
+[tool.poetry.group.dev.dependencies.codespell]
+extras = ["toml"]
+version = ">=v2.2.6"
+
+[[tool.poetry.packages]]
+include = "llama_index/"

llama-index-integrations/readers/llama-index-readers-gutenberg/tests/BUILD

@@ -0,0 +1 @@
+python_tests()

llama-index-integrations/readers/llama-index-readers-gutenberg/tests/test_readers_project_gutenberg_reader.py

@@ -0,0 +1,7 @@
+from llama_index.core.readers.base import BaseReader
+from llama_index.readers.gutenberg import ProjectGutenbergReader
+
+
+def test_class():
+    names_of_base_classes = [b.__name__ for b in ProjectGutenbergReader.__mro__]
+    assert BaseReader.__name__ in names_of_base_classes