tests(html): add tests and fix bugs

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

Author: ceberam

docling/backend/abstract_backend.py

@@ -12,16 +12,20 @@
 
 
 class BaseBackendOptions(BaseModel):
-    """Common options for all declarative document backends.
+    """Common options for all declarative document backends."""
 
-    This is placeholder to define all common options among declarative backends.
-    """
+    enable_remote_fetch: bool = Field(
+        False, description="Enable remote resource fetching."
+    )
+    enable_local_fetch: bool = Field(
+        False, description="Enable local resource fetching."
+    )
 
 
 class DeclarativeBackendOptions(BaseBackendOptions):
     """Default backend options for a declarative document backend."""
 
-    kind: Literal["declarative"] = "declarative"
+    kind: Literal["declarative"] = Field("declarative", exclude=True, repr=False)
 
 
 class HTMLBackendOptions(BaseBackendOptions):
@@ -30,7 +34,7 @@ class HTMLBackendOptions(BaseBackendOptions):
     This class can be extended to include options specific to HTML processing.
     """
 
-    kind: Literal["html"] = "html"
+    kind: Literal["html"] = Field("html", exclude=True, repr=False)
     image_fetch: bool = Field(
         False,
         description=(
@@ -53,9 +57,6 @@ class HTMLBackendOptions(BaseBackendOptions):
 
 
 class AbstractDocumentBackend(ABC):
-    enable_remote_fetch: bool = False
-    enable_local_fetch: bool = False
-
     @abstractmethod
     def __init__(self, in_doc: "InputDocument", path_or_stream: Union[BytesIO, Path]):
         self.file = in_doc.file

docling/backend/html_backend.py

@@ -1,6 +1,8 @@
 import base64
 import logging
+import os
 import re
+import warnings
 from contextlib import contextmanager
 from copy import deepcopy
 from io import BytesIO
@@ -313,15 +315,15 @@ def _resolve_relative_path(self, loc: str) -> str:
         abs_loc = loc
 
         if self.base_path:
-            if not loc.startswith(("http://", "https://", "data:", "file://", "/")):
+            if loc.startswith("//"):
+                # Protocol-relative URL - default to https
+                abs_loc = "https:" + loc
+            elif not loc.startswith(("http://", "https://", "data:", "file://")):
                 if HTMLDocumentBackend._is_remote_url(self.base_path):  # remote fetch
                     abs_loc = urljoin(self.base_path, loc)
                 elif self.base_path:  # local fetch
                     # For local files, resolve relative to the HTML file location
                     abs_loc = str(Path(self.base_path).parent / loc)
-            elif loc.startswith("//"):
-                # Protocol-relative URL - default to https
-                abs_loc = "https:" + loc
 
         _log.debug(f"Resolved location {loc} to {abs_loc}")
         return abs_loc
@@ -1198,13 +1200,18 @@ def get_img_hyperlink(img_tag):
     def _create_image_ref(self, src_url: str) -> Optional[ImageRef]:
         try:
             img_data = self._load_image_data(src_url)
-
             if img_data:
                 img = Image.open(BytesIO(img_data))
                 return ImageRef.from_pil(img, dpi=int(img.info.get("dpi", (72,))[0]))
-
-        except (ValidationError, UnidentifiedImageError, Exception) as e:
-            _log.warning(f"Could not process image (src={src_url}): {e}")
+        except (
+            requests.HTTPError,
+            ValidationError,
+            UnidentifiedImageError,
+            OperationNotAllowed,
+            TypeError,
+            ValueError,
+        ) as e:
+            warnings.warn(f"Could not process an image from {src_url}: {e}")
 
         return None
 
@@ -1213,33 +1220,33 @@ def _load_image_data(self, src_loc: str) -> Optional[bytes]:
             _log.debug(f"Skipping SVG file: {src_loc}")
             return None
 
-        try:
-            if HTMLDocumentBackend._is_remote_url(src_loc):
-                if not self.enable_remote_fetch:
-                    raise OperationNotAllowed(
-                        "Fetching remote resources is only allowed when set explicitly. "
-                        "options.enable_remote_fetch=True."
-                    )
-                response = requests.get(src_loc, stream=True)
-                response.raise_for_status()
-                return response.content
-            elif src_loc.startswith("data:"):
-                data = re.sub(r"^data:image/.+;base64,", "", src_loc)
-                return base64.b64decode(data)
-
-            if src_loc.startswith("file://"):
-                src_loc = src_loc[7:]
-
-            if not self.enable_local_fetch:
+        if HTMLDocumentBackend._is_remote_url(src_loc):
+            if not self.options.enable_remote_fetch:
                 raise OperationNotAllowed(
-                    "Fetching local resources is only allowed when set explicitly. "
-                    "options.enable_local_fetch=True."
+                    "Fetching remote resources is only allowed when set explicitly. "
+                    "Set options.enable_remote_fetch=True."
                 )
+            response = requests.get(src_loc, stream=True)
+            response.raise_for_status()
+            return response.content
+        elif src_loc.startswith("data:"):
+            data = re.sub(r"^data:image/.+;base64,", "", src_loc)
+            return base64.b64decode(data)
+
+        if src_loc.startswith("file://"):
+            src_loc = src_loc[7:]
+
+        if not self.options.enable_local_fetch:
+            raise OperationNotAllowed(
+                "Fetching local resources is only allowed when set explicitly. "
+                "Set options.enable_local_fetch=True."
+            )
+        # add check that file exists and can read
+        if os.path.isfile(src_loc) and os.access(src_loc, os.R_OK):
             with open(src_loc, "rb") as f:
                 return f.read()
-        except Exception as e:
-            _log.warning(f"Could not load image data: {e}")
-            return None
+        else:
+            raise ValueError("File does not exist or it is not readable.")
 
     @staticmethod
     def get_text(item: PageElement) -> str:

docling/datamodel/document.py

@@ -143,10 +143,18 @@ def __init__(
             format=InputFormat.PDF,
             backend_options=backend_options,
         )  # initialize with dummy values
-
         self.limits = limits or DocumentLimits()
         self.format = format
 
+        # check for backend incompatibilities
+        if issubclass(backend, DeclarativeDocumentBackend) and backend_options:
+            if not issubclass(
+                type(backend_options), type(backend.get_default_options())
+            ):
+                raise ValueError(
+                    "Incompatible types between backend and backend_options arguments."
+                )
+
         try:
             if isinstance(path_or_stream, Path):
                 self.file = path_or_stream
@@ -289,8 +297,7 @@ def docs(
             else:
                 options = format_options[format]
                 backend = options.backend
-                # patch against circular import
-                if "backend_options" in options.__class__.__dict__:
+                if "backend_options" in options.model_fields_set:
                     backend_options = cast("FormatOption", options).backend_options
 
             path_or_stream: Union[BytesIO, Path]