Strange behavior for page.deleteAnnot on 1.16.9 compare to 1.13.20

[PCarle]

with the code bellow i hope at end only one polyannot in pixmap!

from __future__ import print_function
import sys
print("Python", sys.version, "on", sys.platform, "\n")
import fitz
print(fitz.__doc__, "\n")

import os
print (os.getcwd())
from matplotlib.pyplot import imshow
import numpy as np
from PIL import Image


def print_descr(rect, annot):
    """Print a short description to the right of an annot rect."""
    annot.parent.insertText(rect.br + (10, 0),
                    "'%s' annotation" % annot.type[1], color = red)
    
red    = (1, 0, 0)
blue   = (0, 0, 1)
gold   = (1, 1, 0)
colors = {"stroke": blue, "fill": gold}
colors2 = {"fill": blue, "stroke": gold}
border = {"width": 1.3}
displ = fitz.Rect(0, 50, 0, 50)
r = fitz.Rect(50, 100, 220, 135)
t1 = u"têxt üsès Lätiñ charß,\nEUR: €, mu: µ, super scripts: ²³!"

doc = fitz.open()
page = doc.newPage()

annot=None
for i in range (7):
    if annot is not None :
        print ("<----delete annot",i-1)
        page.deleteAnnot(annot)
    annot = page.addPolygonAnnot([r.bl, r.br, r.tr,r.tl])
    annot.setOpacity(0.5)
    annot.setBorder(border)
    annot.setColors(stroke=blue,fill=blue)
    annot.update()
    print_descr(annot.rect, annot)
    print("----> add annot",i)
    r+=displ
    


pix=page.getPixmap(matrix=fitz.Matrix(fitz.Identity).preScale(3,3),alpha=True,colorspace="RGBA")

from PIL import Image, ImageTk
mode = "RGBA"
img = Image.frombytes(mode, [pix.width, pix.height], pix.samples)

from matplotlib import pyplot as plt
import matplotlib as mpl
mpl.rc("savefig", dpi=300)
plt.figure(figsize = (20,20))
fig, ax = plt.subplots(figsize=(18, 20))
ax.imshow(np.asarray(img))

with pymupdf version 1.13.20 :

  # -*- coding: utf-8 -*-
from __future__ import print_function
import sys
print("Python", sys.version, "on", sys.platform, "\n")
import fitz
print(fitz.__doc__, "\n")

import os
print (os.getcwd())
from matplotlib.pyplot import imshow
import numpy as np
from PIL import Image


def print_descr(rect, annot):
    """Print a short description to the right of an annot rect."""
    annot.parent.insertText(rect.br + (10, 0),
                    "'%s' annotation" % annot.type[1], color = red)
    
red    = (1, 0, 0)
blue   = (0, 0, 1)
gold   = (1, 1, 0)
colors = {"stroke": blue, "fill": gold}
colors2 = {"fill": blue, "stroke": gold}
border = {"width": 1.3}
displ = fitz.Rect(0, 50, 0, 50)
r = fitz.Rect(50, 100, 220, 135)
t1 = u"têxt üsès Lätiñ charß,\nEUR: €, mu: µ, super scripts: ²³!"

doc = fitz.open()
page = doc.newPage()

annot=None
for i in range (7):
    if annot is not None :
        print ("<----delete annot",i-1)
        page.deleteAnnot(annot)
    annot = page.addPolygonAnnot([r.bl, r.br, r.tr,r.tl])
    annot.setOpacity(0.5)
    annot.setBorder(border)
    annot.setColors(colors2)
    print_descr(annot.rect, annot)
    print("----> add annot",i)
    r+=displ


pix=page.getPixmap(matrix=fitz.Matrix(fitz.Identity).preScale(3,3),alpha=True,colorspace="RGBA")

from PIL import Image, ImageTk
mode = "RGBA"
img = Image.frombytes(mode, [pix.width, pix.height], pix.samples)

from matplotlib import pyplot as plt
import matplotlib as mpl
mpl.rc("savefig", dpi=300)
plt.figure(figsize = (20,20))
fig, ax = plt.subplots(figsize=(18, 20))
ax.imshow(np.asarray(img))

result is as expected :

[JorjMcKie]

The code works correctly in both cases: there is only one (the last) annottation left!
You can verify this when you save the PDF and then look at it.
I suspect there is some different behavior in the MuPDF base library.
If you do this:

from __future__ import print_function

import os
import sys

import fitz
from PIL import Image

print(fitz.__doc__)

red = (1, 0, 0)
blue = (0, 0, 1)
displ = (0, 50, 0, 50)
r = fitz.Rect(50, 100, 220, 135)


def print_descr(rect, annot):
    """Print a short description to the right of an annot rect."""
    annot.parent.insertText(
        rect.br + (10, 0), "'%s' annotation" % annot.type[1], color=red
    )


doc = fitz.open()
page = doc.newPage()

annot = None
for i in range(7):
    if annot is not None:
        print("<----delete annot", i - 1)
        page.deleteAnnot(annot)
    annot = page.addRectAnnot(r)
    annot.setOpacity(0.5)
    annot.setBorder(width=1.3)
    annot.setColors(stroke=blue, fill=blue)
    annot.update()
    print_descr(annot.rect, annot)
    print("----> add annot", i)
    r += displ

page = None  # delete page (also works with 'del page')
page = doc[-1]  # reload the page!

pix = page.getPixmap(alpha=0)
img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
img.show()
doc.save("test.pdf")

IAW, if you reload the page before taking the pixmap, you will see a correct image.

At the moment I'm not sure how to handle this. It is a relatively unusual situation, and it is at the same time easily resolved ...

An option would be to find a way to do this "behind the curtain", when an annot.update() occurs. Seems quite radical to me, though.

[PCarle]

delete page each time i delete one annotation it's difficult for me in my code. my code is UI track mouse selection in a qtwidget (pyside) with creating/deleting multiple "polygon annotation";
so when i change selection i delete all polyannotation saved as pair (page,annotation) in a list.
i try a two pass deletion, first time i delete all polyannotations, second all pages, and reload current page...
sound good for you ?

[PCarle]

I try this:

# -*- coding: utf-8 -*-
from __future__ import print_function
import sys
print("Python", sys.version, "on", sys.platform, "\n")
import fitz
print(fitz.__doc__, "\n")

import os
print (os.getcwd())
from matplotlib.pyplot import imshow
import numpy as np
from PIL import Image


def print_descr(rect, annot):
    """Print a short description to the right of an annot rect."""
    annot.parent.insertText(rect.br + (10, 0),
                    "'%s' annotation" % annot.type[1], color = red)
    
red    = (1, 0, 0)
blue   = (0, 0, 1)
gold   = (1, 1, 0)
colors = {"stroke": blue, "fill": gold}
colors2 = {"fill": blue, "stroke": gold}
border = {"width": 1.3}
displ = fitz.Rect(0, 50, 0, 50)
r = fitz.Rect(50, 100, 220, 135)
t1 = u"têxt üsès Lätiñ charß,\nEUR: €, mu: µ, super scripts: ²³!"

doc = fitz.open()
page = doc.newPage()

annots=[]
for i in range (7):
    annot = page.addPolygonAnnot([r.bl, r.br, r.tr,r.tl])
    annot.setOpacity(0.5)
    annot.setBorder(border)
    annot.setColors(stroke=blue,fill=blue)
    annot.update()
    print_descr(annot.rect, annot)
    #doc.layout()
    print("----> add annot",i,annot.xref)
    r+=displ
    annots.append((page,annot))

for (i,(page,a)) in enumerate(annots):
    print ("<---- delete",i,a.xref)
    page.deleteAnnot(a)
    
pages_set=set([x[0]for x in annots])
for p in pages_set:
    print ("del page",p.number)
    del page
        
page=doc[-1]

pix=page.getPixmap(matrix=fitz.Matrix(fitz.Identity).preScale(3,3),alpha=True,colorspace="RGBA")

from PIL import Image, ImageTk
mode = "RGBA"
img = Image.frombytes(mode, [pix.width, pix.height], pix.samples)

from matplotlib import pyplot as plt
import matplotlib as mpl
mpl.rc("savefig", dpi=300)
plt.figure(figsize = (20,20))
fig, ax = plt.subplots(figsize=(18, 20))
ax.imshow(np.asarray(img))

and the result is :

[PCarle]

may be with adding a page.update() method ?

[JorjMcKie]

may be with adding a page.update() method ?

I am not sure yet. Of course such a method "page.update()" would have to be backed in the underlying C code. This is what I was referring to previously when I wrote

An option would be to find a way to do this "behind the curtain", when an annot.update() occurs. Seems quite radical to me, though.

The basic requirement is to enforce a recreation of the page object. This will cause all related information to be dropped, including any existing annotation objects for that page. Re-assigning the page variable to the required PDF page will then force MuPDF to re-read the relevant information.
This will happen with del page, page = None or assigning the page to another (e.g. Page) object.
There must be some new caching mechanism somewhere, which leads to this behavior ...

[JorjMcKie]

I just successfully tested this one:

from __future__ import print_function

import os
import sys

import fitz
from PIL import Image

print(fitz.__doc__)

red = (1, 0, 0)
blue = (0, 0, 1)
displ = (0, 50, 0, 50)
r = fitz.Rect(50, 100, 220, 135)


def print_descr(rect, annot):
    """Print a short description to the right of an annot rect."""
    annot.parent.insertText(
        rect.br + (10, 0), "'%s' annotation" % annot.type[1], color=red
    )


def reload(page):
    """Force reloading a page.
    Will drop all page resources including dependent objects like annotations.
    Hereafter, the page is reloaded from the document.
    """
    pno = page.number  # save page number
    doc = page.parent  # save owing PDF
    page._erase()  # free all page resources
    return doc[pno]  # reload


doc = fitz.open()
page = doc.newPage()

annot = None
for i in range(7):
    if annot is not None:
        print("<----delete annot", i - 1)
        page.deleteAnnot(annot)
    annot = page.addRectAnnot(r)
    annot.setOpacity(0.5)
    annot.setBorder({"width": 1.3})
    annot.setColors({"stroke": blue, "fill": blue})
    annot.update()
    print_descr(annot.rect, annot)
    print("----> add annot", i)
    r += displ

page = reload(page)

pix = page.getPixmap(alpha=0)
img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
img.show()

[PCarle]

It's not cool i can't get page and work on it (add delete annotations) each time in need to do something on page i need to load it modify it and unload it !
if i need some references on annotations i must use xref and get annotation in page by xref each time and change it!
xref may be stable or changes on save document ?

[JorjMcKie]

I am still investigating if there may be another solution.
The xref of a PDF object may change if the document is saved with garbage collection. So this is not a good thing to use for identification across saves or document closes. However, I haven't used yet a standard annotation field called /NM which is supposed to be an annotation's unique identifier within a page's annotation.
What I will definitely do is introducing this field and provide a mechanism to load an annotation by providing this identification.

[PCarle]

Ok I'am waiting for migrate and redefine selections codes in my application

[JorjMcKie]

This is what I am currently testing:

1. Every annotation added with PyMuPDF will receive a unique id. This is a unicode string contained in the annot.info dictionary and accessible via ``annot.info["id"].
2. This id can never be changed. It follows the pattern fitzannot-<n> (or fitzwidget-<n> for fields) with a running integer n.
3. From the page object, an annotation can be loaded via its id: annot = page.retrieve_annot(annot_id). Not every annotation has an identifying /NM. If not found, an exception is raised.
4. The list of all annotation (and widget / field) names can be retrieved by page.annot_names.

With these changes, the above script can take the following form:

import os
import sys

import fitz
from PIL import Image

print(fitz.__doc__)

red = (1, 0, 0)
blue = (0, 0, 1)
displ = (0, 50, 0, 50)
r = fitz.Rect(50, 100, 220, 135)


def print_descr(rect, annot):
    """Print a short description to the right of an annot rect."""
    annot.parent.insertText(
        rect.br + (10, 0), "'%s' annotation" % annot.type[1], color=red
    )


def reload(page):
    """Reload a page."""
    pno = page.number
    doc = page.parent
    page._erase()
    return doc[pno]


doc = fitz.open()
page = doc.newPage()

annot = None
for i in range(7):
    if annot is not None:
        print("deleting annot", annot.info["id"])
        page.deleteAnnot(annot)
    annot = page.addRectAnnot(r)
    annot.setOpacity(0.5)
    annot.setBorder(width=1.3)
    annot.setColors(stroke=blue, fill=blue)
    annot.update()
    print_descr(annot.rect, annot)
    annot_id = annot.info["id"]
    print("added annot", annot_id)  # save the annot id before refreshing the page!
    page = reload(page)  # refresh the page
    annot = page.retrieve_annot(annot_id)  # load previously added annot again
    r += displ

pix = page.getPixmap(alpha=0)
img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
img.show()
doc.save("test.pdf", garbage=3)

[JorjMcKie]

Fixed with v1.16.10 which is available on PyPI now.
The following skript should run okay with it:

import fitz
from PIL import Image

print(fitz.__doc__)

red = (1, 0, 0)
blue = (0, 0, 1)
displ = (0, 50, 0, 50)
r = fitz.Rect(50, 100, 220, 135)


def print_descr(rect, annot):
    """Print a short description to the right of an annot rect."""
    annot.parent.insertText(
        rect.br + (10, 0), "'%s' annotation" % annot.type[1], color=red
    )


doc = fitz.open()
page = doc.newPage()

annot = None
for i in range(7):
    if annot is not None:
        print(i, "deleting annot", annot.info["id"])
        page.deleteAnnot(annot)
    annot = page.addRectAnnot(r)
    annot.setOpacity(0.5)
    annot.setBorder(width=1.3)
    annot.setColors(stroke=blue, fill=blue)
    annot.update()
    print_descr(annot.rect, annot)
    annot_id = annot.info["id"]
    print(i, "added annot", annot_id)
    r += displ


page = doc.reload_page(page)  # refresh the page before taking pictures
print(annot.info["id"], "=", annot)  # just to prove: last annot is still there

pix = page.getPixmap(alpha=0)
img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
img.show()
doc.save("test.pdf", garbage=3)

[JorjMcKie]

Just some final comments before closing this:

1. The new Document method reload_page() is functionally equivalent to your requested method Page.update().
2. In contrast to my previous concern, this new method does not invalidate existing annotations / links / widgets objects of the page. The "refreshed" page object inherits all of those depending objects (as indicated in the previous post).
3. Given all this, the new feature for identifying an annotation via Annot.info["id"] is strictly speaking not necessary for the use cases you are dealing with. But it is of course nice to have it now ...

As a general comment:
If you are doing these type of things (lots of annot inserts and deletes with intermittent image creations), you should use garbage=3 if you ever save your PDF. Otherwise all the short-lived intermediate objects will also appear in the PDF file - eating up disk space unnecessarily.