upload v1.18.4

docs/annot.rst

@@ -315,8 +315,7 @@ There is a parent-child relationship between an annotation and its page. If the
       :arg sequence,float text_color: change the text color. 'FreeText' annotations only.
       :arg sequence,float border_color: change the border color. 'FreeText' annotations only.
       :arg sequence,float fill_color: the fill color.
-
-          * 'FreeText' annotations: If you set (or leave) this to *None*, then **no rectangle at all** will be drawn around the text, and the border color will be ignored. This will leave anything "under" the text visible.
+
           * 'Line', 'Polyline', 'Polygon' annotations: use it to give applicable line end symbols a fill color other than that of the annotation *(changed in v1.16.16)*.
 
       :arg bool cross_out: *(new in v1.17.2)* add two diagonal lines to the annotation rectangle. 'Redact' annotations only. If not desired, *False* must be specified even if the annotation was created with *False*.

docs/changes.rst

@@ -5,6 +5,8 @@ Changes in Version 1.18.4
 ---------------------------
 This version adds several features to support PDF Optional Content. Among other things, this includes OCMDs (Optional Content Membership Dictionaries) with the full scope of *"visibility expressions"* (PDF key ``/VE``), text insertions (including the :ref:`TextWriter` class) and drawings.
 
+* **Fixed** issue `#727 <https://github.com/pymupdf/PyMuPDF/issues/727>`_. Freetext annotations now support an uncolored rectangle when ``fill_color=None``.
+* **Fixed** issue `#726 <https://github.com/pymupdf/PyMuPDF/issues/726>`_. UTF-8 encoding errors are now handled for HTML / XML :meth:`Page.getText` output.
 * **Fixed** issue `#724 <https://github.com/pymupdf/PyMuPDF/issues/724>`_. Empty values are no longer stored in the PDF /Info metadata dictionary.
 * **Added** new methods :meth:`Document.set_oc` and :meth:`Document.get_oc` to set or get optional content references for **existing** image and form XObjects. These methods are similar to the same-named methods of :ref:`Annot`.
 * **Added** :meth:`Document.set_ocmd`, :meth:`Document.get_ocmd` for handling OCMDs.
@@ -25,7 +27,7 @@ As a major new feature, this version introduces support for PDF's **Optional Con
 
 * **Fixed** issue `#714 <https://github.com/pymupdf/PyMuPDF/issues/714>`_.
 * **Fixed** issue `#711 <https://github.com/pymupdf/PyMuPDF/issues/711>`_.
-* **Fixed** issue `#707 <https://github.com/pymupdf/PyMuPDF/issues/707>`_: if a PDF user password is supplied but no owner password is supplied nor present, then the user password is also used as the owner password.
+* **Fixed** issue `#707 <https://github.com/pymupdf/PyMuPDF/issues/707>`_: if a PDF user password, but no owner password is supplied nor present, then the user password is also used as the owner password.
 * **Fixed** ``expand`` and ``deflate`` parameters of methods :meth:`Document.save` and :meth:`Document.write`. Individual image and font compression should now finally work. Addresses issue `#713 <https://github.com/pymupdf/PyMuPDF/issues/713>`_.
 * **Added** a support of PDF optional content. This includes several new :ref:`Document` methods for inquiring and setting optional content status and adding optional content configurations and groups. In addition, images, form XObjects and annotations now can be bound to optional content specifications. **Resolved** issue `#709 <https://github.com/pymupdf/PyMuPDF/issues/709>`_.
 

fitz/fitz.i

@@ -7609,7 +7609,6 @@ struct Annot
             }
             return Py_BuildValue("i", oc);
         }
-        %pythoncode %{optional_content = property(getOC, doc="optional content xref")%}
 
 
         //----------------------------------------------------------------
@@ -7751,7 +7750,7 @@ struct Annot
         FITZEXCEPTION(set_oc, !result)
         PARENTCHECK(set_oc, """Set annotation optional content xref.""")
         PyObject *
-        set_optional_content(int oc=0)
+        set_oc(int oc=0)
         {
             fz_try(gctx) {
                 pdf_annot *annot = (pdf_annot *) $self;
@@ -8125,8 +8124,13 @@ struct Annot
             JM_color_FromSequence(fill_color, &nfcol, fcol);
             fz_try(gctx) {
                 pdf_dirty_annot(gctx, annot); // enforce MuPDF /AP formatting
-                if (type == PDF_ANNOT_FREE_TEXT && EXISTS(fill_color))
-                    pdf_set_annot_color(gctx, annot, nfcol, fcol);
+                if (type == PDF_ANNOT_FREE_TEXT) {
+                    if (EXISTS(fill_color)) {
+                        pdf_set_annot_color(gctx, annot, nfcol, fcol);
+                    } else{
+                        pdf_dict_del(gctx, annot->obj, PDF_NAME(IC));
+                    }
+                }
 
                 int insert_rot = (rotate >= 0) ? 1 : 0;
                 switch (type) {
@@ -8369,15 +8373,17 @@ struct Annot
                 TOOLS._update_da(self, da_str)
                 
                 for i, item in enumerate(ap_tab):
-                    if (item.endswith(b" w")
-                        and bwidth > 0
-                        and border_color is not None
-                       ):  # update border color
+                    if (
+                        item.endswith(b" w") and bwidth > 0 and border_color is not None
+                    ):  # update border color
                         ap_tab[i + 1] = color_string(border_color, "s")
                         continue
                     if item == b"BT":  # update text color
                         ap_tab[i + 1] = color_string(tcol, "f")
                         continue
+                    if fill is None:
+                        if item.endswith((b" re")) and ap_tab[i + 1] == b"f":
+                            ap_tab[i + 1] = b"n"
 
                 if dashes is not None:  # handle dashes
                     ap_tab.insert(0, dashes)
@@ -9744,21 +9750,18 @@ struct TextPage {
             fz_stext_page *this_tpage = (fz_stext_page *) $self;
             fz_try(gctx) {
                 res = fz_new_buffer(gctx, 1024);
+                out = fz_new_output_with_buffer(gctx, res);
                 switch(format) {
                     case(1):
-                        out = fz_new_output_with_buffer(gctx, res);
                         fz_print_stext_page_as_html(gctx, out, this_tpage, 0);
                         break;
                     case(3):
-                        out = fz_new_output_with_buffer(gctx, res);
                         fz_print_stext_page_as_xml(gctx, out, this_tpage, 0);
                         break;
                     case(4):
-                        out = fz_new_output_with_buffer(gctx, res);
                         fz_print_stext_page_as_xhtml(gctx, out, this_tpage, 0);
                         break;
                     default:
-                        out = fz_new_output_with_buffer(gctx, res);
                         JM_print_stext_page_as_text(gctx, out, this_tpage);
                         break;
                 }

fitz/helper-other.i

@@ -41,8 +41,8 @@ PyObject *JM_EscapeStrFromBuffer(fz_context *ctx, fz_buffer *buff)
 PyObject *JM_UnicodeFromBuffer(fz_context *ctx, fz_buffer *buff)
 {
     unsigned char *s = NULL;
-    size_t len = fz_buffer_storage(ctx, buff, &s);
-    PyObject *val = PyUnicode_FromStringAndSize((const char *) s, (Py_ssize_t) len);
+    Py_ssize_t len = (Py_ssize_t) fz_buffer_storage(ctx, buff, &s);
+    PyObject *val = PyUnicode_DecodeUTF8((const char *) s, len, "replace");
     if (!val) {
         val = EMPTY_STRING;
         PyErr_Clear();

fitz/helper-stext.i

@@ -14,7 +14,7 @@ fz_stext_page *JM_new_stext_page_from_page(fz_context *ctx, fz_page *page, fz_re
     fz_try(ctx) {
         tp = fz_new_stext_page(ctx, rect);
         dev = fz_new_stext_device(ctx, tp, &options);
-        fz_run_page_contents(ctx, page, dev, fz_identity, NULL);
+        fz_run_page(ctx, page, dev, fz_identity, NULL);
         fz_close_device(ctx, dev);
     }
     fz_always(ctx) {