Slicing through ImageData - Interactor does change image slice

[juanprietob]

I would like to set up a simple UI with an image/volume on the left side and a 3D renderer on the right side.
I can read and visualize the first slice of the image data but when using the wheel of the mouse, the slices of the image do not change.

I have tried multiple implementations (see commented code)

1. Do I need to set up a new interactor? Or why does the vtkInteractorStyleImage not work as expected?
2. Why can't I use the VtkLocalView to render the image? It only works with the VtkRemoteView (which makes the rendering window pop out)
3. I can't find another example of simple image visualization (I tried with Plotly but it is extremely slow to go through the slices)
4. Is there any other suggestion for image visualization?

from trame.app import get_server
from trame.ui.vuetify import SinglePageLayout
from trame.widgets import vtk as vtk_widgets, vuetify, trame, plotly
from trame.decorators import hot_reload

import plotly.graph_objects as go
import plotly.express as px

from vtkmodules.vtkFiltersSources import vtkConeSource
from vtkmodules.vtkRenderingCore import (
    vtkActor,
    vtkPolyDataMapper,
    vtkRenderer,
    vtkRenderWindow,
    vtkRenderWindowInteractor,
    vtkImageActor
)

from vtk import (
    vtkImageViewer,
    vtkImageViewer2,
    vtkImageData,
    vtkInteractorStyleImage,
    vtkImageResliceMapper
)

# Required for interactor initialization
from vtkmodules.vtkInteractionStyle import vtkInteractorStyleSwitch, vtkInteractorStyleImage  # noqa

# Required for rendering initialization, not necessary for
# local rendering, but doesn't hurt to include it
import vtkmodules.vtkRenderingOpenGL2  # noqa

import SimpleITK as sitk
import numpy as np
from vtk.util import numpy_support

def ReadImage(file_path):
    # Convert SimpleITK image to a NumPy array
    sitk_image = sitk.ReadImage(file_path)
    spacing = sitk_image.GetSpacing()
    sitk_array = sitk.GetArrayFromImage(sitk_image)
    if (len(sitk_array.shape) == 4):
        sitk_array = sitk_array[:,:,:,0]
        sitk_array = np.flip(sitk_array, axis=1)
    
    # sitk_array = sitk_array.transpose(2,1,0) # Transpose dimensions to match VTK's order

    # Convert NumPy array to VTK array (vtkImageData)
    vtk_data_array = numpy_support.numpy_to_vtk(num_array=sitk_array.ravel(), deep=True)

    # Create a VTK image (vtkImageData)
    vtk_image = vtkImageData()
    vtk_image.SetSpacing(sitk_image.GetSpacing())
    vtk_image.SetOrigin(sitk_image.GetOrigin())
    vtk_image.SetDimensions(sitk_image.GetSize())
    vtk_image.GetPointData().SetScalars(vtk_data_array)

    return vtk_image

image_data = ReadImage("M.dcm")
# Create the first VTK render window and renderer

# image_viewer.Render()
# image_viewer.GetRenderer().ResetCamera()
# image_viewer.Render()

renderer1 = vtkRenderer()
renderWindow1 = vtkRenderWindow()
renderWindow1.AddRenderer(renderer1)

renderWindowInteractor1 = vtkRenderWindowInteractor()
renderWindowInteractor1.SetRenderWindow(renderWindow1)

# imageViewer = vtkImageViewer()
# imageViewer.SetInputData(image_data)
# imageViewer.Render()
# imageViewer.GetRenderWindow().SetWindowName("DisplayImageExample1")
# imageViewer.SetupInteractor(renderWindowInteractor1)

# image_viewer = vtkImageViewer2()
# image_viewer.SetRenderWindow(renderWindow1)
# image_viewer.SetupInteractor(renderWindowInteractor1)
# image_viewer.SetInputData(image_data)
# image_viewer.SetSliceOrientation(vtkImageViewer2.SLICE_ORIENTATION_XY)


interactor_style = vtkInteractorStyleImage()
interactor_style.SetInteractionModeToImageSlicing()


renderWindowInteractor1.SetInteractorStyle(interactor_style)
renderWindowInteractor1.SetRenderWindow(renderWindow1)

image_actor = vtkImageActor()
image_mapper = vtkImageResliceMapper()
image_mapper.SetInputData(image_data)
image_actor.SetMapper(image_mapper)

renderer1.AddActor(image_actor)


# Create the second VTK render window and renderer
renderer2 = vtkRenderer()
renderWindow2 = vtkRenderWindow()
renderWindow2.AddRenderer(renderer2)

renderWindowInteractor2 = vtkRenderWindowInteractor()
renderWindowInteractor2.SetRenderWindow(renderWindow2)
# Create and set up the cone source for the second renderer
cone_source2 = vtkConeSource()
cone_source2.Update()

actor2 = vtkActor()
mapper2 = vtkPolyDataMapper()
mapper2.SetInputConnection(cone_source2.GetOutputPort())
actor2.SetMapper(mapper2)
renderer2.AddActor(actor2)

# Initialize the trame server with a single page layout
server = get_server(client_type = "vue2")
state, ctrl = server.state, server.controller



@hot_reload
def do_someting():
    print("do_someting::ChangeMe")

@ctrl.set("update_ui")
def update_ui():
    with SinglePageLayout(server) as layout:
    
        layout.title.set_text("My App")

        with layout.content:
             with vuetify.VContainer(
                        fluid=True,
                        classes="fill-height",
                    ):
                with vuetify.VRow(classes="fill-height"):
                    with vuetify.VCol(cols=6):
                        image_view = vtk_widgets.VtkRemoteView(renderWindow1)
                        # image_view = vtk_widgets.VtkLocalView(renderWindow1, interactor=renderWindowInteractor1)
                        # image_view.update()
                        # image_viewer.Render()
                        
                    with vuetify.VCol(cols=6):
                        vuetify.VSlider(
                            v_model=("resolution1", 8), # (var_name, initial_value)
                            min=3, max=60, step=1,                      # min/max/step
                            hide_details=True, dense=True,              # presentation params
                        )
                        vtk_widgets.VtkLocalView(renderWindow2)

update_ui()
# renderWindow2.Render()
# -----------------------------------------------------------------------------
# Automatic UI update on file change using watchdog (pip install watchdog)
# -----------------------------------------------------------------------------

try:
    import asyncio
    from pathlib import Path
    from watchdog.observers import Observer
    from watchdog.events import FileSystemEventHandler

    current_event_loop = asyncio.get_event_loop()

    def update_ui():
        with server.state:
            ctrl.update_ui()

    class UpdateUIOnChange(FileSystemEventHandler):
        def on_modified(self, event):
            current_event_loop.call_soon_threadsafe(update_ui)

    observer = Observer()
    observer.schedule(
        UpdateUIOnChange(), str(Path(__file__).parent.absolute()), recursive=False
    )
    observer.start()
except:
    print("Watchdog not installed so skipping the auto monitoring")

# Starting the server
if __name__ == "__main__":
    server.start(open_browser=False, )

[jourdain]

The current implementation of the RemoteView does not properly forward event to the interactor, especially for the wheel.
For the local view, the interactor will have to be managed/defined on the client side somehow, so not easy for your specific use case.

But you can try to leverage trame-rca to properly send all the events to the interactor. You can find an example of it here

[jourdain]

Then you need to do pip install -U vtk --extra-index-url https://wheels.vtk.org --pre maybe

[jourdain]

You should be able to use "vtk==9.3.20240302.dev0" otherwise

[juanprietob]

I was able to make the trame-rca example run. Unfortunately, I am still unable to make the slice change. I can capture the events of the mouse wheel with 2 approaches the rca-example, and also overriding the events from the vtkInteractorStyleImage (see attached code)

How can I trigger the rendering event after updating the ImageNumber?

from trame.app import asynchronous, get_server
from trame.ui.vuetify import SinglePageLayout
from trame.widgets import vtk as vtk_widgets, vuetify, trame, plotly, rca
from trame.decorators import hot_reload

import plotly.graph_objects as go
import plotly.express as px

from vtkmodules.vtkFiltersSources import vtkConeSource
from vtkmodules.vtkRenderingCore import (
    vtkActor,
    vtkPolyDataMapper,
    vtkRenderer,
    vtkRenderWindow,
    vtkRenderWindowInteractor,
    vtkImageActor,
    vtkActor2D
)

from vtk import (
    vtkImageViewer,
    vtkImageViewer2,
    vtkImageData,
    vtkInteractorStyleImage,
    vtkImageResliceMapper,
    vtkImageMapper,
    vtkCommand
)

# Required for interactor initialization
from vtkmodules.vtkInteractionStyle import vtkInteractorStyleSwitch, vtkInteractorStyleImage  # noqa
from vtkmodules.vtkWebCore import vtkRemoteInteractionAdapter, vtkWebApplication
# Required for rendering initialization, not necessary for
# local rendering, but doesn't hurt to include it
import vtkmodules.vtkRenderingOpenGL2  # noqa

import SimpleITK as sitk
import numpy as np
from vtk.util import numpy_support

import asyncio
import json
import time


# HELPER = vtkWebApplication()
# HELPER.SetImageEncoding(vtkWebApplication.ENCODING_NONE)
# HELPER.SetNumberOfEncoderThreads(4)

# class ViewAdapter:
#     def __init__(self, window, name, interactor_style):
#         self._view = window
#         self.area_name = name
#         self.animating = False
#         self._iren = window.GetInteractor()
#         self._iren.EnableRenderOff()
#         self._view.ShowWindowOff()
#         self._interactor_style = interactor_style

#     def _get_metadata(self):
#         return dict(
#             type="image/jpeg",  # mime time
#             codec="",  # video codec, not relevant here
#             w=self._view.GetSize()[0],
#             h=self._view.GetSize()[1],
#             st=int(time.time_ns() / 1000000),
#             key=("key"),  # jpegs are always keyframes
#         )

#     async def _animate(self):
#         mtime = 0
#         while self.animating:
#             data = HELPER.InteractiveRender(self._view)
#             if data is not None and mtime != data.GetMTime():
#                 mtime = data.GetMTime()
#                 self.push(memoryview(data), self._get_metadata())
#                 # await asyncio.sleep(1.0 / self.target_fps)
#             await asyncio.sleep(0)

#         HELPER.InvalidateCache(self._view)
#         content = memoryview(HELPER.StillRender(self._view))
#         self.push(content, self._get_metadata())

#     def set_streamer(self, stream_manager):
#         self.streamer = stream_manager

#     def update_size(self, origin, size):
#         width = int(size.get("w", 300))
#         height = int(size.get("h", 300))
#         self._view.SetSize(width, height)
#         content = memoryview(HELPER.StillRender(self._view))
#         self.push(content, self._get_metadata())

#     def push(self, content, meta=None):
#         if meta is not None:
#             self.last_meta = meta
#         if content is None:
#             return
#         self.streamer.push_content(self.area_name, self.last_meta, content)

#     def on_interaction(self, origin, event):        
#         event_type = event["type"]
#         if event_type == "StartInteractionEvent":
#             if not self.animating:
#                 self.animating = True
#                 asynchronous.create_task(self._animate())
#         elif event_type == "EndInteractionEvent":
#             self.animating = False
#         else:
#             event_str = json.dumps(event)
            

#             if event["type"] == "StartMouseWheel" and event["action"] == "up":
                
#                 slice_num = self._interactor_style.GetCurrentImageNumber()        
#                 self._interactor_style.SetCurrentImageNumber(slice_num + 1)

#             if event["type"] == "StartMouseWheel" and event["action"] == "down":
                
#                 slice_num = self._interactor_style.GetCurrentImageNumber()        
#                 self._interactor_style.SetCurrentImageNumber(slice_num - 1)

#             status = vtkRemoteInteractionAdapter.ProcessEvent(self._iren, event_str)    

#             # Force Render next time InteractiveRender is called
#             if status:
#                 HELPER.InvalidateCache(self._view)

class InteractorStyleImage(vtkInteractorStyleImage):
    def __init__(self, parent=None):
        super().__init__()
        self.AddObserver("MouseWheelForwardEvent", self.OnMouseWheelForward)
        self.AddObserver("MouseWheelBackwardEvent", self.OnMouseWheelBackward)
    
    def OnMouseWheelForward(self, interactor, event):        
        slice_num = interactor.GetCurrentImageNumber()        
        interactor.SetCurrentImageNumber(slice_num + 1)        
        # self.InvokeEvent(vtkCommand.InteractionEvent)
        # self.Mapper.Update()
        # self.Interactor.Render()
        # self.Render.Render()
        # print("Slice number: ", slice_num + 1)
    
    def OnMouseWheelBackward(self, interactor, event):        
        slice_num = interactor.GetCurrentImageNumber()        
        interactor.SetCurrentImageNumber(slice_num - 1)
        print("Slice number: ", slice_num - 1)
        # self.InvokeEvent(vtkCommand.InteractionEvent)
        # self.Mapper.Update()
        # self.Interactor.Render()
        # self.Render.Render()
        # print("Slice number: ", slice_num + 1)

def ReadImage(file_path):
    # Convert SimpleITK image to a NumPy array
    sitk_image = sitk.ReadImage(file_path)
    spacing = sitk_image.GetSpacing()
    sitk_array = sitk.GetArrayFromImage(sitk_image)
    if (len(sitk_array.shape) == 4):
        sitk_array = sitk_array[:,:,:,0]
        sitk_array = np.flip(sitk_array, axis=1)
    
    # sitk_array = sitk_array.transpose(2,1,0) # Transpose dimensions to match VTK's order

    # Convert NumPy array to VTK array (vtkImageData)
    # print(sitk_array.shape)
    vtk_data_array = numpy_support.numpy_to_vtk(num_array=sitk_array.ravel(), deep=True)

    # Create a VTK image (vtkImageData)
    vtk_image = vtkImageData()
    vtk_image.SetSpacing(sitk_image.GetSpacing())
    vtk_image.SetOrigin(sitk_image.GetOrigin())
    vtk_image.SetDimensions(sitk_image.GetSize())
    vtk_image.GetPointData().SetScalars(vtk_data_array)

    print(vtk_image)

    return vtk_image


image_data = ReadImage("M.dcm")
# # Create the first VTK render window and renderer
renderer1 = vtkRenderer()
renderWindow1 = vtkRenderWindow()
renderWindow1.SetOffScreenRendering(1)
renderWindow1.AddRenderer(renderer1)

interactor_style = InteractorStyleImage()    

renderWindowInteractor1 = vtkRenderWindowInteractor()
renderWindowInteractor1.SetRenderWindow(renderWindow1)
renderWindowInteractor1.SetInteractorStyle(interactor_style)    
interactor_style.SetInteractionModeToImageSlicing()

image_actor = vtkImageActor()
image_mapper = vtkImageResliceMapper()
image_mapper.SetInputData(image_data)
image_actor.SetMapper(image_mapper)

renderer1.AddActor(image_actor)

# interactor_style.Mapper = image_mapper

# Create the second VTK render window and renderer
renderer2 = vtkRenderer()
renderWindow2 = vtkRenderWindow()
renderWindow2.AddRenderer(renderer2)

renderWindowInteractor2 = vtkRenderWindowInteractor()
renderWindowInteractor2.SetRenderWindow(renderWindow2)
# Create and set up the cone source for the second renderer
cone_source2 = vtkConeSource()
cone_source2.Update()

actor2 = vtkActor()
mapper2 = vtkPolyDataMapper()
mapper2.SetInputConnection(cone_source2.GetOutputPort())
actor2.SetMapper(mapper2)
renderer2.AddActor(actor2)

# Initialize the trame server with a single page layout
server = get_server(client_type = "vue2")
state, ctrl = server.state, server.controller



@hot_reload
def do_someting():
    print("do_someting::ChangeMe")

@ctrl.set("update_ui")
def update_ui():
    with SinglePageLayout(server) as layout:
    
        layout.title.set_text("My App")

        with layout.content:
             with vuetify.VContainer(
                        fluid=True,
                        classes="fill-height",
                    ):
                with vuetify.VRow(classes="fill-height"):
                    with vuetify.VCol(cols=6):
                        # view = rca.RemoteControlledArea(
                        #     name="view_image",
                        #     display="image",
                        # )
                        vtk_widgets.VtkRemoteView(renderWindow1)
                        
                    with vuetify.VCol(cols=6):
                        vuetify.VSlider(
                            v_model=("resolution1", 8), # (var_name, initial_value)
                            min=3, max=60, step=1,                      # min/max/step
                            hide_details=True, dense=True,              # presentation params
                        )
                        vtk_widgets.VtkLocalView(renderWindow2)


# @ctrl.add("on_server_ready")
# def init_rca(**kwargs):

#     # image_data = ReadImage("/Users/juan/Desktop/M.dcm")
#     image_data = ReadImage("c:/Users/juan/Desktop/M.dcm")
#     # Create the first VTK render window and renderer
#     renderer1 = vtkRenderer()
#     renderWindow1 = vtkRenderWindow()
#     renderWindow1.SetOffScreenRendering(1)
#     renderWindow1.AddRenderer(renderer1)

#     interactor_style = vtkInteractorStyleImage()    

#     renderWindowInteractor1 = vtkRenderWindowInteractor()
#     renderWindowInteractor1.SetRenderWindow(renderWindow1)
#     renderWindowInteractor1.SetInteractorStyle(interactor_style)    
#     interactor_style.SetInteractionModeToImageSlicing()

#     view_handler = ViewAdapter(renderWindow1, "view_image", interactor_style)
#     server.controller.rc_area_register(view_handler)

#     image_actor = vtkImageActor()
#     image_mapper = vtkImageResliceMapper()
#     image_mapper.SetInputData(image_data)
#     image_actor.SetMapper(image_mapper)

#     renderer1.AddActor(image_actor)



update_ui()
# renderWindow2.Render()
# -----------------------------------------------------------------------------
# Automatic UI update on file change using watchdog (pip install watchdog)
# -----------------------------------------------------------------------------

try:
    import asyncio
    from pathlib import Path
    from watchdog.observers import Observer
    from watchdog.events import FileSystemEventHandler

    current_event_loop = asyncio.get_event_loop()

    def update_ui():
        with server.state:
            ctrl.update_ui()

    class UpdateUIOnChange(FileSystemEventHandler):
        def on_modified(self, event):
            current_event_loop.call_soon_threadsafe(update_ui)

    observer = Observer()
    observer.schedule(
        UpdateUIOnChange(), str(Path(__file__).parent.absolute()), recursive=False
    )
    observer.start()
except:
    print("Watchdog not installed so skipping the auto monitoring")

# Starting the server
if __name__ == "__main__":
    server.start(open_browser=False, )

[juanprietob]

Found a solution!

resliceViewer = vtkResliceImageViewer()
renderWindowInteractor = vtkRenderWindowInteractor()
resliceViewer.SetupInteractor(renderWindowInteractor)
resliceViewer.GetRenderWindow().SetOffScreenRendering(1)
resliceViewer.SetInputData(image_data)

And then in UI view:

view = vtk_widgets.VtkRemoteView(resliceViewer.GetRenderWindow())
ctrl.on_server_ready.add(view.update) 

[jourdain]

To properly update the view, you need to call view.update() but you can bind it to the controller by doing the following.

view = vtk_widgets.VtkRemoteView(resliceViewer.GetRenderWindow())
ctrl.view_update = view.update


# trigger a view update - instead of calling render_window.Render()
ctrl.view_update()