ENH: Process XRefStm

PyPDF2/_reader.py

@@ -327,7 +327,9 @@ def metadata(self) -> Optional[DocumentInformation]:
         obj = self.trailer[TK.INFO]
         retval = DocumentInformation()
         if isinstance(obj, type(None)):
-            raise PdfReadError("trailer not found or does not point to document information directory")
+            raise PdfReadError(
+                "trailer not found or does not point to document information directory"
+            )
         retval.update(obj)  # type: ignore
         return retval
 
@@ -1131,7 +1133,31 @@ def get_object(self, indirect_reference: IndirectObject) -> Optional[PdfObject]:
                 return NullObject()
             start = self.xref[indirect_reference.generation][indirect_reference.idnum]
             self.stream.seek(start, 0)
-            idnum, generation = self.read_object_header(self.stream)
+            try:
+                idnum, generation = self.read_object_header(self.stream)
+            except Exception:
+                if hasattr(self.stream, "getbuffer"):
+                    buf = bytes(self.stream.getbuffer())  # type: ignore
+                else:
+                    p = self.stream.tell()
+                    buf = self.stream.read(-1)
+                    self.stream.seek(p, 0)
+                m = re.search(
+                    rf"\s{indirect_reference.idnum}\s+{indirect_reference.generation}\s+obj".encode(),
+                    buf,
+                )
+                if m is not None:
+                    logger_warning(
+                        f"Object ID {indirect_reference.idnum},{indirect_reference.generation} ref repaired",
+                        __name__,
+                    )
+                    self.xref[indirect_reference.generation][
+                        indirect_reference.idnum
+                    ] = (m.start(0) + 1)
+                    self.stream.seek(m.start(0) + 1)
+                    idnum, generation = self.read_object_header(self.stream)
+                else:
+                    idnum = -1  # exception will be raised below
             if idnum != indirect_reference.idnum and self.xref_index:
                 # Xref table probably had bad indexes due to not being zero-indexed
                 if self.strict:
@@ -1163,13 +1189,48 @@ def get_object(self, indirect_reference: IndirectObject) -> Optional[PdfObject]:
                     retval, indirect_reference.idnum, indirect_reference.generation
                 )
         else:
-            logger_warning(
-                f"Object {indirect_reference.idnum} {indirect_reference.generation} "
-                "not defined.",
-                __name__,
+            if hasattr(self.stream, "getbuffer"):
+                buf = bytes(self.stream.getbuffer())  # type: ignore
+            else:
+                p = self.stream.tell()
+                buf = self.stream.read(-1)
+                self.stream.seek(p, 0)
+            m = re.search(
+                rf"\s{indirect_reference.idnum}\s+{indirect_reference.generation}\s+obj".encode(),
+                buf,
             )
-            if self.strict:
-                raise PdfReadError("Could not find object.")
+            if m is not None:
+                logger_warning(
+                    f"Object {indirect_reference.idnum} {indirect_reference.generation} found",
+                    __name__,
+                )
+                if indirect_reference.generation not in self.xref:
+                    self.xref[indirect_reference.generation] = {}
+                self.xref[indirect_reference.generation][indirect_reference.idnum] = (
+                    m.start(0) + 1
+                )
+                self.stream.seek(m.end(0) + 1)
+                skip_over_whitespace(self.stream)
+                self.stream.seek(-1, 1)
+                retval = read_object(self.stream, self)  # type: ignore
+
+                # override encryption is used for the /Encrypt dictionary
+                if not self._override_encryption and self._encryption is not None:
+                    # if we don't have the encryption key:
+                    if not self._encryption.is_decrypted():
+                        raise FileNotDecryptedError("File has not been decrypted")
+                    # otherwise, decrypt here...
+                    retval = cast(PdfObject, retval)
+                    retval = self._encryption.decrypt_object(
+                        retval, indirect_reference.idnum, indirect_reference.generation
+                    )
+            else:
+                logger_warning(
+                    f"Object {indirect_reference.idnum} {indirect_reference.generation} not defined.",
+                    __name__,
+                )
+                if self.strict:
+                    raise PdfReadError("Could not find object.")
         self.cache_indirect_object(
             indirect_reference.generation, indirect_reference.idnum, retval
         )
@@ -1429,7 +1490,14 @@ def _read_standard_xref_table(self, stream: StreamType) -> None:
                     pass
                 else:
                     self.xref[generation][num] = offset
-                    self.xref_free_entry[generation][num] = entry_type_b == b"f"
+                    try:
+                        self.xref_free_entry[generation][num] = entry_type_b == b"f"
+                    except Exception:
+                        pass
+                    try:
+                        self.xref_free_entry[65535][num] = entry_type_b == b"f"
+                    except Exception:
+                        pass
                 cnt += 1
                 num += 1
             read_non_whitespace(stream)
@@ -1452,6 +1520,8 @@ def _read_xref_tables_and_trailers(
             # load the xref table
             stream.seek(startxref, 0)
             x = stream.read(1)
+            if x in b"\r\n":
+                x = stream.read(1)
             if x == b"x":
                 startxref = self._read_xref(stream)
             elif xref_issue_nr:
@@ -1461,12 +1531,26 @@ def _read_xref_tables_and_trailers(
                 except Exception:
                     xref_issue_nr = 0
             elif x.isdigit():
-                xrefstream = self._read_pdf15_xref_stream(stream)
-
+                try:
+                    xrefstream = self._read_pdf15_xref_stream(stream)
+                except Exception as e:
+                    if TK.ROOT in self.trailer:
+                        logger_warning(
+                            f"Previous trailer can not be read {e.args}",
+                            __name__,
+                        )
+                        break
+                    else:
+                        raise PdfReadError(f"trailer can not be read {e.args}")
                 trailer_keys = TK.ROOT, TK.ENCRYPT, TK.INFO, TK.ID
                 for key in trailer_keys:
                     if key in xrefstream and key not in self.trailer:
                         self.trailer[NameObject(key)] = xrefstream.raw_get(key)
+                if "/XRefStm" in xrefstream:
+                    p = stream.tell()
+                    stream.seek(cast(int, xrefstream["/XRefStm"]) + 1, 0)
+                    self._read_pdf15_xref_stream(stream)
+                    stream.seek(p, 0)
                 if "/Prev" in xrefstream:
                     startxref = cast(int, xrefstream["/Prev"])
                 else:
@@ -1482,6 +1566,17 @@ def _read_xref(self, stream: StreamType) -> Optional[int]:
         for key, value in new_trailer.items():
             if key not in self.trailer:
                 self.trailer[key] = value
+        if "/XRefStm" in new_trailer:
+            p = stream.tell()
+            stream.seek(cast(int, new_trailer["/XRefStm"]) + 1, 0)
+            try:
+                self._read_pdf15_xref_stream(stream)
+            except Exception:
+                logger_warning(
+                    f"XRef object at {new_trailer['/XRefStm']} can not be read, some object may be missing",
+                    __name__,
+                )
+            stream.seek(p, 0)
         if "/Prev" in new_trailer:
             startxref = new_trailer["/Prev"]
             return startxref

PyPDF2/generic/_utils.py

@@ -79,15 +79,15 @@ def read_string_from_stream(
             try:
                 tok = escape_dict[tok]
             except KeyError:
-                if tok.isdigit():
+                if tok >= b"0" and tok <= b"7":
                     # "The number ddd may consist of one, two, or three
                     # octal digits; high-order overflow shall be ignored.
                     # Three octal digits shall be used, with leading zeros
                     # as needed, if the next character of the string is also
                     # a digit." (PDF reference 7.3.4.2, p 16)
                     for _ in range(2):
                         ntok = stream.read(1)
-                        if ntok.isdigit():
+                        if ntok >= b"0" and ntok <= b"7":
                             tok += ntok
                         else:
                             stream.seek(-1, 1)  # ntok has to be analysed

tests/test_merger.py

@@ -345,7 +345,7 @@ def test_sweep_indirect_list_newobj_is_None(caplog):
     merger.append(reader)
     merger.write("tmp-merger-do-not-commit.pdf")
     merger.close()
-    assert "Object 21 0 not defined." in caplog.text
+    # used to be: assert "Object 21 0 not defined." in caplog.text
 
     reader2 = PdfReader("tmp-merger-do-not-commit.pdf")
     reader2.pages

tests/test_reader.py

@@ -101,13 +101,15 @@ def test_read_metadata(pdf_path, expected):
 
 
 @pytest.mark.parametrize(
-    "pdf_path",
-    [EXTERNAL_ROOT / "017-unreadable-meta-data/unreadablemetadata.pdf"]
+    "pdf_path", [EXTERNAL_ROOT / "017-unreadable-meta-data/unreadablemetadata.pdf"]
 )
 def test_broken_meta_data(pdf_path):
-    with open(pdf_path, 'rb') as f:
+    with open(pdf_path, "rb") as f:
         reader = PdfReader(f)
-        with pytest.raises(PdfReadError, match=r"trailer not found or does not point to document information directory"):
+        with pytest.raises(
+            PdfReadError,
+            match=r"trailer not found or does not point to document information directory",
+        ):
             reader.metadata
 
 
@@ -1113,13 +1115,23 @@ def test_corrupted_xref_table():
     name = "BreezeMan1.pdf"
     reader = PdfReader(BytesIO(get_pdf_from_url(url, name=name)))
     reader.pages[0].extract_text()
-
     url = "https://github.com/py-pdf/PyPDF2/files/9444748/BreezeManual.failed.pdf"
     name = "BreezeMan2.pdf"
     reader = PdfReader(BytesIO(get_pdf_from_url(url, name=name)))
-    try:
-        reader.pages[0].extract_text()
-    except Exception:
-        pass  # Exception normal
-    else:
-        raise Exception("page 0 should not be corrupted")
+    reader.pages[0].extract_text()
+
+def test_reader(caplog):
+    # iss #1273
+    url = "https://github.com/py-pdf/PyPDF2/files/9464742/shiv_resume.pdf"
+    name = "shiv_resume.pdf"
+    reader = PdfReader(BytesIO(get_pdf_from_url(url, name=name)))
+    assert "Previous trailer can not be read" in caplog.text
+    caplog.clear()
+    # first call requires some reparations...
+    reader.pages[0].extract_text()
+    assert "repaired" in caplog.text
+    assert "found" in caplog.text
+    caplog.clear()
+    # ...and now no more required
+    reader.pages[0].extract_text()
+    assert caplog.text == ""

tests/test_utils.py

@@ -5,6 +5,7 @@
 import pytest
 
 import PyPDF2._utils
+from PyPDF2 import PdfReader
 from PyPDF2._utils import (
     _get_max_pdf_version_header,
     deprecate_bookmark,
@@ -19,6 +20,8 @@
 )
 from PyPDF2.errors import PdfReadError, PdfStreamError
 
+from . import get_pdf_from_url
+
 TESTS_ROOT = Path(__file__).parent.resolve()
 PROJECT_ROOT = TESTS_ROOT.parent
 RESOURCE_ROOT = PROJECT_ROOT / "resources"
@@ -243,3 +246,13 @@ def foo(old_param=1, baz=2):
         "old_param is deprecated. Use new_param instead."
     )
     assert exc.value.args[0] == expected_msg
+
+
+def test_escapedcode_followed_by_int():
+    # iss #1294
+    url = "https://github.com/timedegree/playground_files/raw/main/%E8%AE%BA%E6%96%87/AN%20EXACT%20ANALYTICAL%20SOLUTION%20OF%20KEPLER'S%20EQUATION.pdf"
+    name = "keppler.pdf"
+
+    reader = PdfReader(io.BytesIO(get_pdf_from_url(url, name=name)))
+    for page in reader.pages:
+        page.extract_text()

tests/test_xmp.py

@@ -172,8 +172,8 @@ def test_dc_subject():
 def test_issue585():
     url = "https://github.com/mstamy2/PyPDF2/files/5536984/test.pdf"
     name = "mstamy2-5536984.pdf"
-    reader = PdfReader(BytesIO(get_pdf_from_url(url, name=name)))
     with pytest.raises(PdfReadError) as exc:
+        reader = PdfReader(BytesIO(get_pdf_from_url(url, name=name)))
         reader.xmp_metadata
     assert exc.value.args[0].startswith("XML in XmpInformation was invalid")